event_store 0.1.0

data/db/event_store_sample_data_generator.rb

@@ -0,0 +1,30 @@
+# require_relative '../../protocol_buffers/lib/protocol_buffers'
+require 'faceplate_api'
+require "faceplate_api/thermostats/test_support/event_mother"
+require 'securerandom'
+require 'time'
+include FaceplateApi
+event_names = [:firmware_version_updated, :fan_on_updated, :fan_mode_updated, :configuration_lock_updated, :display_lock_updated,
+ :mode_updated, :system_name_updated, :operation_status_updated, :relative_airflow_updated, :balance_point_updated, :indoor_temperature_updated,
+ :temperature_setpoint_updated, :sensor_added, :sensor_removed, :sensor_updated, :zone_added, :zone_removed, :zone_updated, :preset_added,
+ :preset_removed, :preset_updated, :preset_activated, :relative_humidity_setpoint_updated, :event_schedule_added, :event_schedule_removed, :event_schedule_updated, :event_schedule_activated]
+
+aggregate_ids = ["ASDFDS12939", "1SQFDS12B39", "103MMV", SecureRandom.uuid, SecureRandom.uuid, "10PM93BU37"]
+ITERATIONS = 5
+versions_per_device = (0..(event_names.length * ITERATIONS)).to_a
+
+mothers = {}
+aggregate_ids.each do |aggregate_id|
+  mother  = Thermostats::EventMother.new(device_id: aggregate_id)
+  mothers[mother] = versions_per_device.dup
+end
+
+File.open('./data.sql', 'w') do |f|
+  (event_names * ITERATIONS * ITERATIONS).shuffle.each do |name|
+    event_mother = mothers.keys.sample
+    event = event_mother.send(name)
+    version = mothers[event_mother].shift
+    f.puts "INSERT INTO events.device_events(aggregate_id, version, occurred_at, serialized_event, fully_qualified_name) values ('#{event_mother.device_id}', #{version}, '#{DateTime.now.iso8601}', '#{event.to_s}', '#{name}');"
+  end
+  f.puts 'commit;'
+end

data/db/migrations/001_create_event_store_events.rb

@@ -0,0 +1,60 @@
+require 'event_store'
+Sequel.migration do
+  up do
+
+    run %Q<CREATE TABLE #{EventStore.fully_qualified_table} (
+          id AUTO_INCREMENT PRIMARY KEY,
+          version BIGINT NOT NULL,
+          aggregate_id varchar(36) NOT NULL,
+          fully_qualified_name varchar(255) NOT NULL,
+          occurred_at TIMESTAMPTZ NOT NULL,
+          serialized_event VARBINARY(255) NOT NULL)
+
+          PARTITION BY EXTRACT(year FROM occurred_at AT TIME ZONE 'UTC')*100 + EXTRACT(month FROM occurred_at AT TIME ZONE 'UTC');
+
+          CREATE PROJECTION #{EventStore.fully_qualified_table}_super_projecion /*+createtype(D)*/
+          (
+           id ENCODING COMMONDELTA_COMP,
+           version ENCODING COMMONDELTA_COMP,
+           aggregate_id ENCODING RLE,
+           fully_qualified_name ENCODING AUTO,
+           occurred_at ENCODING BLOCKDICT_COMP,
+           serialized_event ENCODING AUTO
+          )
+          AS
+           SELECT id,
+                  version,
+                  aggregate_id,
+                  fully_qualified_name,
+                  occurred_at,
+                  serialized_event
+           FROM #{EventStore.fully_qualified_table}
+           ORDER BY aggregate_id,
+                    version
+          SEGMENTED BY HASH(aggregate_id) ALL NODES;
+
+          CREATE PROJECTION #{EventStore.fully_qualified_table}_runtime_history_projection /*+createtype(D)*/
+          (
+           version ENCODING DELTAVAL,
+           aggregate_id ENCODING RLE,
+           fully_qualified_name ENCODING RLE,
+           occurred_at ENCODING RLE,
+           serialized_event ENCODING AUTO
+          )
+          AS
+           SELECT version,
+                  aggregate_id,
+                  fully_qualified_name,
+                  occurred_at,
+                  serialized_event
+           FROM #{EventStore.fully_qualified_table}
+           ORDER BY aggregate_id,
+                    occurred_at,
+                    fully_qualified_name
+           SEGMENTED BY HASH(aggregate_id) ALL NODES;>
+  end
+
+  down do
+    run 'DROP SCHEMA #{EventStore.schema} CASCADE;'
+  end
+end

data/db/pg_migrations/001_create_event_store_events.rb

@@ -0,0 +1,17 @@
+require 'event_store'
+Sequel.migration do
+  change do
+    create_table((EventStore.schema + "__" + EventStore.table_name).to_sym) do
+      primary_key :id
+      Bignum      :version
+      index       :version
+      String      :aggregate_id
+      index       :aggregate_id
+      String      :fully_qualified_name
+      index       :fully_qualified_name
+      DateTime    :occurred_at
+      index       :occurred_at
+      bytea       :serialized_event
+    end
+  end
+end

data/event_store.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'event_store/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "event_store"
+  spec.version       = EventStore::VERSION
+  spec.authors       = ["Paul Saieg, John Colvin", "Stuart Nelson"]
+  spec.description   = ["A Ruby implementation of an EventSource (A+ES) tuned for Vertica or Postgres"]
+  spec.email         = ["classicist@gmail.com"]
+  spec.summary       = %q{Ruby implementation of an EventSource (A+ES) for the Nexia Ecosystem}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", ">= 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 2.14"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "simplecov-rcov"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "pry-debugger"
+  spec.add_development_dependency "mock_redis"
+
+  spec.add_dependency "sequel", "~> 3.42"
+  spec.add_dependency 'sequel-vertica', '~> 0.1.0'
+  spec.add_dependency 'pg', '~> 0.17.1'
+  spec.add_dependency 'redis', "~> 3.0.7"
+end

data/lib/event_store.rb

@@ -0,0 +1,113 @@
+require 'sequel'
+require 'vertica'
+require 'sequel-vertica'
+require 'redis'
+require 'event_store/version'
+require 'event_store/time_hacker'
+require 'event_store/event_appender'
+require 'event_store/aggregate'
+require 'event_store/client'
+require 'event_store/errors'
+Sequel.extension :migration
+
+module EventStore
+  Event = Struct.new(:aggregate_id, :occurred_at, :fully_qualified_name, :serialized_event, :version)
+  SerializedEvent = Struct.new(:fully_qualified_name, :serialized_event, :version, :occurred_at)
+  SNAPSHOT_DELIMITER = "__NexEvStDelim__"
+
+  def self.db_config(env, adapter)
+    raw_db_config[env.to_s][adapter.to_s]
+  end
+
+  def self.raw_db_config
+    if @raw_db_config.nil?
+      file_path = File.expand_path(__FILE__ + '/../../db/database.yml')
+      @config_file = File.open(file_path,'r')
+      @raw_db_config = YAML.load(@config_file)
+      @config_file.close
+    end
+    @raw_db_config
+  end
+
+  def self.db
+    @db
+  end
+
+  def self.redis
+    @redis
+  end
+
+  def self.connect(*args)
+    @db ||= Sequel.connect(*args)
+  end
+
+  def self.redis_connect(config_hash)
+    @redis ||= Redis.new(config_hash)
+  end
+
+  def self.local_redis_connect
+    @redis_connection ||= redis_connect raw_db_config['redis']
+  end
+
+  def self.schema
+    @schema ||= raw_db_config[@environment][@database]['schema']
+  end
+
+  def self.table_name
+    @table_name ||= raw_db_config['table_name']
+  end
+
+  def self.fully_qualified_table
+    @fully_qualified_table ||= Sequel.lit "#{schema}.#{table_name}"
+  end
+
+  def self.clear!
+    EventStore.db.from(fully_qualified_table).delete
+    EventStore.redis.flushdb
+  end
+
+  def self.postgres(db_env = :test)
+    @database = 'postgres'
+    @environment = db_env.to_s
+    local_redis_connect
+    create_db( @database, @environment)
+  end
+
+  def self.vertica(db_env = :test)
+    @database = 'vertica'
+    @environment = db_env.to_s
+    local_redis_connect
+    create_db(@database, @environment)
+  end
+
+  def self.production(database_config, redis_config)
+    self.redis_connect redis_config
+    self.connect database_config
+  end
+
+  def self.create_db(type, db_env, db_config = nil)
+    @db_type = type
+    db_config ||= self.db_config(db_env, type)
+    if type == 'vertica'
+      #To find the ip address of vertica on your local box (running in a vm)
+      #1. open Settings -> Network and select Wi-Fi
+      #2. open a terminal in the VM
+      #3. do /sbin/ifconfig (ifconfig is not in $PATH)
+      #4. the inet address for en0 is what you want
+      #Hint: if it just hangs, you have have the wrong IP
+      db_config['host'] = vertica_host
+      @migrations_dir = 'db/migrations'
+    else
+      @migrations_dir = 'db/pg_migrations'
+    end
+
+    EventStore.connect db_config
+    schema_exits = @db.table_exists?("#{schema}__schema_info".to_sym)
+    @db.run "CREATE SCHEMA #{EventStore.schema};" unless schema_exits
+    Sequel::Migrator.run(@db, @migrations_dir, :table=> "#{schema}__schema_info".to_sym)
+  end
+
+  def self.vertica_host
+    File.read File.expand_path("../../db/vertica_host_address.txt", __FILE__)
+  end
+end

data/lib/event_store/aggregate.rb

@@ -0,0 +1,72 @@
+module EventStore
+  class Aggregate
+
+    attr_reader :id, :type, :snapshot_table, :snapshot_version_table, :event_table
+
+    def initialize(id, type = EventStore.table_name)
+      @id = id
+      @type = type
+      @schema = EventStore.schema
+      @event_table = EventStore.fully_qualified_table
+      @snapshot_table = "#{@type}_snapshots_for_#{@id}"
+      @snapshot_version_table = "#{@type}_snapshot_versions_for_#{@id}"
+    end
+
+    def events
+      @events_query ||= EventStore.db.from(@event_table).where(:aggregate_id => @id.to_s).order(:version)
+    end
+
+    def snapshot
+      events_hash = auto_rebuild_snapshot(read_raw_snapshot)
+      snap = []
+      events_hash.each_pair do |key, value|
+        raw_event            = value.split(EventStore::SNAPSHOT_DELIMITER)
+        fully_qualified_name = key
+        version              = raw_event.first.to_i
+        serialized_event     = raw_event[1]
+        occurred_at          = Time.parse(raw_event.last)
+        snap << SerializedEvent.new(fully_qualified_name, serialized_event, version, occurred_at)
+      end
+      snap.sort {|a,b| a.version <=> b.version}
+    end
+
+    def rebuild_snapshot!
+      delete_snapshot!
+      corrected_events = events.all.map{|e| e[:occurred_at] = TimeHacker.translate_occurred_at_from_local_to_gmt(e[:occurred_at]); e}
+      EventAppender.new(self).store_snapshot(corrected_events)
+    end
+
+    def events_from(version_number, max = nil)
+      events.limit(max).where{ version >= version_number.to_i }.all
+    end
+
+    def last_event
+      snapshot.last
+    end
+
+    def version
+      (EventStore.redis.hget(@snapshot_version_table, :current_version) || -1).to_i
+    end
+
+    def delete_snapshot!
+      EventStore.redis.del [@snapshot_table, @snapshot_version_table]
+    end
+
+    def delete_events!
+      events.delete
+    end
+
+  private
+    def auto_rebuild_snapshot(events_hash)
+      return events_hash unless events_hash.empty?
+      event = events.select(:version).limit(1).all
+      return events_hash if event.nil?
+      rebuild_snapshot!
+      events_hash = read_raw_snapshot
+    end
+
+    def read_raw_snapshot
+      EventStore.redis.hgetall(@snapshot_table)
+    end
+  end
+end

data/lib/event_store/client.rb

@@ -0,0 +1,87 @@
+module EventStore
+  class Client
+
+    def initialize( aggregate_id, aggregate_type = EventStore.table_name)
+      @aggregate = Aggregate.new(aggregate_id, aggregate_type)
+    end
+
+    def id
+      @aggregate.id
+    end
+
+    def type
+      @aggregate.type
+    end
+
+    def event_table
+      @aggregate.event_table
+    end
+
+    def append event_data
+      event_appender.append(event_data)
+      yield(event_data) if block_given?
+      nil
+    end
+
+    def snapshot
+      raw_snapshot
+    end
+
+    def event_stream
+      translate_events raw_event_stream
+    end
+
+    def event_stream_from version_number, max=nil
+      translate_events @aggregate.events_from(version_number, max)
+    end
+
+    def peek
+      translate_event @aggregate.last_event
+    end
+
+    def raw_snapshot
+      @aggregate.snapshot
+    end
+
+    def raw_event_stream
+      @aggregate.events.all
+    end
+
+    def raw_event_stream_from version_number, max=nil
+      @aggregate.events_from(version_number, max)
+    end
+
+    def version
+      @aggregate.version
+    end
+
+    def count
+      event_stream.length
+    end
+
+    def destroy!
+      @aggregate.delete_events!
+      @aggregate.delete_snapshot!
+    end
+
+    def rebuild_snapshot!
+      @aggregate.delete_snapshot!
+      @aggregate.rebuild_snapshot!
+    end
+
+    private
+
+    def event_appender
+      EventAppender.new(@aggregate)
+    end
+
+    def translate_events(event_hashs)
+      event_hashs.map { |eh| translate_event(eh) }
+    end
+
+    def translate_event(event_hash)
+      occurred_at =  TimeHacker.translate_occurred_at_from_local_to_gmt(event_hash[:occurred_at])
+      SerializedEvent.new event_hash[:fully_qualified_name], event_hash[:serialized_event], event_hash[:version], occurred_at
+    end
+  end
+end

data/lib/event_store/errors.rb

@@ -0,0 +1,4 @@
+module EventStore
+  class AttributeMissingError < StandardError; end
+  class ConcurrencyError < StandardError; end
+end

data/lib/event_store/event_appender.rb

@@ -0,0 +1,83 @@
+module EventStore
+  class EventAppender
+
+    def initialize aggregate
+      @aggregate = aggregate
+    end
+
+    def append raw_events
+      EventStore.db.transaction do
+        set_current_version
+
+        prepared_events = raw_events.map do |raw_event|
+          event = prepare_event(raw_event)
+          validate! event
+          raise concurrency_error(event) if has_concurrency_issue?(event)
+          event
+        end
+        # All concurrency issues need to be checked before persisting any of the events
+        # Otherwise, the newly appended events may raise erroneous concurrency errors
+        result = @aggregate.events.multi_insert(prepared_events)
+        store_snapshot(prepared_events) unless result.nil?
+        result
+      end
+    end
+
+    def store_snapshot(prepared_events)
+      r = EventStore.redis
+      current_version_numbers = r.hgetall(@aggregate.snapshot_version_table)
+      current_version_numbers.default = -1
+      valid_snapshot_events = []
+      valid_snapshot_versions = []
+      prepared_events.each do |event|
+        if event[:version].to_i > current_version_numbers[event[:fully_qualified_name]].to_i
+          valid_snapshot_events   << event[:fully_qualified_name]
+          valid_snapshot_events   << (event[:version].to_s + EventStore::SNAPSHOT_DELIMITER + event[:serialized_event] + EventStore::SNAPSHOT_DELIMITER + event[:occurred_at].to_s)
+          valid_snapshot_versions << event[:fully_qualified_name]
+          valid_snapshot_versions << event[:version]
+        end
+      end
+      unless valid_snapshot_versions.empty?
+        last_version            = valid_snapshot_versions.last
+        valid_snapshot_versions << :current_version
+        valid_snapshot_versions << last_version.to_i
+        r.multi do
+          r.hmset(@aggregate.snapshot_version_table, valid_snapshot_versions)
+          r.hmset(@aggregate.snapshot_table, valid_snapshot_events)
+        end
+      end
+    end
+
+  private
+    def has_concurrency_issue? event
+      event[:version] <= current_version
+    end
+
+    def prepare_event raw_event
+      { :version              => raw_event.version.to_i,
+        :aggregate_id         => raw_event.aggregate_id,
+        :occurred_at          => Time.parse(raw_event.occurred_at.to_s).utc, #to_s truncates microseconds, which brake Time equality
+        :serialized_event     => raw_event.serialized_event,
+        :fully_qualified_name => raw_event.fully_qualified_name }
+    end
+
+    def concurrency_error event
+      ConcurrencyError.new("The version of the event being added (version #{event[:version]}) is <= the current version (version #{current_version})")
+    end
+
+    private
+    def current_version
+      @current_version ||= @aggregate.version
+    end
+    alias :set_current_version :current_version
+
+    def validate! event_hash
+      [:aggregate_id, :fully_qualified_name, :occurred_at, :serialized_event, :version].each do |attribute_name|
+        if event_hash[attribute_name].to_s.strip.empty?
+          raise AttributeMissingError, "value required for #{attribute_name}"
+        end
+      end
+    end
+
+  end
+end