nexia_event_store 0.2.10

data/db/event_store_sample_data_generator.rb

@@ -0,0 +1,30 @@
+# require_relative '../../protocol_buffers/lib/protocol_buffers'
+require 'faceplate_api'
+require "faceplate_api/thermostats/test_support"
+require 'securerandom'
+require 'time'
+include FaceplateApi
+event_names = [:firmware_version_updated, :fan_on_updated, :fan_mode_updated, :configuration_lock_updated, :display_lock_updated,
+ :mode_updated, :system_name_updated, :operation_status_updated, :relative_airflow_updated, :balance_point_updated, :indoor_temperature_updated,
+ :temperature_setpoint_updated, :sensor_added, :sensor_removed, :sensor_updated, :zone_added, :zone_removed, :zone_updated, :preset_added,
+ :preset_removed, :preset_updated, :preset_activated, :relative_humidity_setpoint_updated, :event_schedule_added, :event_schedule_removed, :event_schedule_updated, :event_schedule_activated]
+
+aggregate_ids = ["ASDFDS12939", "1SQFDS12B39", "103MMV", SecureRandom.uuid, SecureRandom.uuid, "10PM93BU37"]
+ITERATIONS = 5
+versions_per_device = (0..(event_names.length * ITERATIONS)).to_a
+
+mothers = {}
+aggregate_ids.each do |aggregate_id|
+  mother  = FaceplateApi::EventFixture.new(header: {device_id: aggregate_id}).event_mother
+  mothers[mother] = versions_per_device.dup
+end
+
+File.open('./data.sql', 'w') do |f|
+  (event_names * ITERATIONS * ITERATIONS).shuffle.each do |name|
+    event_mother = mothers.keys.sample
+    event = event_mother.send(name)
+    version = mothers[event_mother].shift
+    f.puts "INSERT INTO events.device_events(aggregate_id, version, occurred_at, serialized_event, fully_qualified_name) values ('#{event_mother.device_id}', #{version}, '#{DateTime.now.iso8601}', '#{event.to_s}', '#{name}');"
+  end
+  f.puts 'commit;'
+end

data/db/migrations/001_create_event_store_events.rb

@@ -0,0 +1,62 @@
+require 'event_store'
+Sequel.migration do
+  up do
+
+    run %Q<CREATE TABLE #{EventStore.fully_qualified_table} (
+          id AUTO_INCREMENT PRIMARY KEY,
+          version BIGINT NOT NULL,
+          aggregate_id varchar(36) NOT NULL,
+          fully_qualified_name varchar(512) NOT NULL,
+          occurred_at TIMESTAMPTZ NOT NULL,
+          serialized_event VARBINARY(32768) NOT NULL)
+
+          PARTITION BY EXTRACT(year FROM occurred_at AT TIME ZONE 'UTC')*100 + EXTRACT(month FROM occurred_at AT TIME ZONE 'UTC');
+
+          CREATE PROJECTION #{EventStore.fully_qualified_table}_super_projecion /*+createtype(D)*/
+          (
+           id ENCODING COMMONDELTA_COMP,
+           version ENCODING COMMONDELTA_COMP,
+           aggregate_id ENCODING RLE,
+           fully_qualified_name ENCODING AUTO,
+           occurred_at ENCODING BLOCKDICT_COMP,
+           serialized_event ENCODING AUTO
+          )
+          AS
+           SELECT id,
+                  version,
+                  aggregate_id,
+                  fully_qualified_name,
+                  occurred_at,
+                  serialized_event
+           FROM #{EventStore.fully_qualified_table}
+           ORDER BY aggregate_id,
+                    version
+          SEGMENTED BY HASH(aggregate_id) ALL NODES
+          KSAFE 1;
+
+          CREATE PROJECTION #{EventStore.fully_qualified_table}_runtime_history_projection /*+createtype(D)*/
+          (
+           version ENCODING DELTAVAL,
+           aggregate_id ENCODING RLE,
+           fully_qualified_name ENCODING RLE,
+           occurred_at ENCODING RLE,
+           serialized_event ENCODING AUTO
+          )
+          AS
+           SELECT version,
+                  aggregate_id,
+                  fully_qualified_name,
+                  occurred_at,
+                  serialized_event
+           FROM #{EventStore.fully_qualified_table}
+           ORDER BY aggregate_id,
+                    occurred_at,
+                    fully_qualified_name
+           SEGMENTED BY HASH(aggregate_id) ALL NODES
+           KSAFE 1;>
+  end
+
+  down do
+    run 'DROP SCHEMA #{EventStore.schema} CASCADE;'
+  end
+end

data/db/pg_migrations/001_create_event_store_events.rb

@@ -0,0 +1,17 @@
+require 'event_store'
+Sequel.migration do
+  change do
+    create_table((EventStore.schema + "__" + EventStore.table_name).to_sym) do
+      primary_key :id
+      Bignum      :version
+      index       :version
+      String      :aggregate_id
+      index       :aggregate_id
+      String      :fully_qualified_name
+      index       :fully_qualified_name
+      DateTime    :occurred_at
+      index       :occurred_at
+      bytea       :serialized_event
+    end
+  end
+end

data/db/setup_db_user.sql

@@ -0,0 +1,4 @@
+/* intentional: this will not drop and recreate the database if it already exists */
+/* Mainly for CI. If you install with brew, you likely have authentication configured as "TRUST" for all */
+CREATE USER nexia WITH UNENCRYPTED PASSWORD 'Password1';
+GRANT ALL ON DATABASE history_store TO nexia;

data/event_store.gemspec

@@ -0,0 +1,36 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'event_store/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'nexia_event_store'
+  spec.version       = EventStore::VERSION
+  spec.authors       = ['Paul Saieg, John Colvin', 'Stuart Nelson']
+  spec.description   = ['A Ruby implementation of an EventSource (A+ES) tuned for Vertica or Postgres']
+  spec.email         = ['classicist@gmail.com, jgeiger@gmail.com']
+  spec.summary       = %q{Ruby implementation of an EventSource (A+ES) for the Nexia Ecosystem}
+  spec.homepage      = 'https://github.com/nexiahome/event_store'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 0'
+  spec.add_development_dependency 'rspec', '~> 3.1'
+  spec.add_development_dependency 'simplecov', '~> 0.9'
+  spec.add_development_dependency 'simplecov-rcov', '~> 0.2'
+  spec.add_development_dependency 'guard-rspec', '~> 4.3'
+  spec.add_development_dependency 'pry-byebug', '~> 2.0'
+  spec.add_development_dependency 'mock_redis', '~> 0.13'
+
+  spec.add_dependency 'sequel', '~> 4.14'
+  spec.add_dependency 'nexia-sequel-vertica', '~> 0.2'
+  spec.add_dependency 'pg', '~> 0.17'
+  spec.add_dependency 'redis', '~> 3.1'
+  spec.add_dependency 'hiredis', '~> 0.5'
+  spec.add_development_dependency 'rspec_junit_formatter', '~> 0.2'
+end

data/lib/event_store/aggregate.rb

@@ -0,0 +1,93 @@
+module EventStore
+  class Aggregate
+
+    attr_reader :id, :type, :snapshot_table, :snapshot_version_table, :event_table
+
+    def self.count
+      EventStore.db.from( EventStore.fully_qualified_table).distinct(:aggregate_id).count
+    end
+
+    def self.ids(offset, limit)
+      EventStore.db.from( EventStore.fully_qualified_table).distinct(:aggregate_id).select(:aggregate_id).order(:aggregate_id).limit(limit, offset).all.map{|item| item[:aggregate_id]}
+    end
+
+    def initialize(id, type = EventStore.table_name)
+      @id = id
+      @type = type
+      @schema = EventStore.schema
+      @event_table = EventStore.fully_qualified_table
+      @snapshot_table = "#{@type}_snapshots_for_#{@id}"
+      @snapshot_version_table = "#{@type}_snapshot_versions_for_#{@id}"
+    end
+
+    def events
+      @events_query ||= EventStore.db.from(@event_table).where(:aggregate_id => @id.to_s).order(:version)
+    end
+
+    def snapshot
+      events_hash = auto_rebuild_snapshot(read_raw_snapshot)
+      snap = []
+      events_hash.each_pair do |key, value|
+        raw_event            = value.split(EventStore::SNAPSHOT_DELIMITER)
+        fully_qualified_name = key
+        version              = raw_event.first.to_i
+        serialized_event     = EventStore.unescape_bytea(raw_event[1])
+        occurred_at          = Time.parse(raw_event.last)
+        snap << SerializedEvent.new(fully_qualified_name, serialized_event, version, occurred_at)
+      end
+      snap.sort {|a,b| a.version <=> b.version}
+    end
+
+    def rebuild_snapshot!
+      delete_snapshot!
+      corrected_events = events.all.map{|e| e[:occurred_at] = TimeHacker.translate_occurred_at_from_local_to_gmt(e[:occurred_at]); e}
+      EventAppender.new(self).store_snapshot(corrected_events)
+    end
+
+    def events_from(version_number, max = nil)
+      events.limit(max).where{ version >= version_number.to_i }.all.map do |event|
+        event[:serialized_event] = EventStore.unescape_bytea(event[:serialized_event])
+        event
+      end
+    end
+
+    def event_stream_between(start_time, end_time, fully_qualified_names = [])
+      query = events.where(occurred_at: start_time..end_time)
+      query = query.where(fully_qualified_name: fully_qualified_names) if fully_qualified_names && fully_qualified_names.any?
+      query.all.map {|e| e[:serialized_event] = EventStore.unescape_bytea(e[:serialized_event]); e}
+    end
+
+    def event_stream
+      events.all.map {|e| e[:serialized_event] = EventStore.unescape_bytea(e[:serialized_event]); e}
+    end
+
+    def last_event
+      snapshot.last
+    end
+
+    def version
+      (EventStore.redis.hget(@snapshot_version_table, :current_version) || -1).to_i
+    end
+
+    def delete_snapshot!
+      EventStore.redis.del [@snapshot_table, @snapshot_version_table]
+    end
+
+    def delete_events!
+      events.delete
+    end
+
+  private
+    def auto_rebuild_snapshot(events_hash)
+      return events_hash unless events_hash.empty?
+      event = events.select(:version).limit(1).all
+      return events_hash if event.nil?
+      rebuild_snapshot!
+      events_hash = read_raw_snapshot
+    end
+
+    def read_raw_snapshot
+      EventStore.redis.hgetall(@snapshot_table)
+    end
+  end
+end

data/lib/event_store/client.rb

@@ -0,0 +1,99 @@
+module EventStore
+  class Client
+
+    def self.count
+      Aggregate.count
+    end
+
+    def self.ids(offset, limit)
+      Aggregate.ids(offset, limit)
+    end
+
+    def initialize( aggregate_id, aggregate_type = EventStore.table_name)
+      @aggregate = Aggregate.new(aggregate_id, aggregate_type)
+    end
+
+    def id
+      @aggregate.id
+    end
+
+    def type
+      @aggregate.type
+    end
+
+    def event_table
+      @aggregate.event_table
+    end
+
+    def append event_data
+      event_appender.append(event_data)
+      yield(event_data) if block_given?
+      nil
+    end
+
+    def snapshot
+      raw_snapshot
+    end
+
+    def event_stream
+      translate_events raw_event_stream
+    end
+
+    def event_stream_from version_number, max=nil
+      translate_events @aggregate.events_from(version_number, max)
+    end
+
+    def event_stream_between(start_time, end_time, fully_qualified_names = [])
+      translate_events @aggregate.event_stream_between(start_time, end_time, fully_qualified_names)
+    end
+
+    def peek
+      translate_event @aggregate.last_event
+    end
+
+    def raw_snapshot
+      @aggregate.snapshot
+    end
+
+    def raw_event_stream
+      @aggregate.event_stream
+    end
+
+    def raw_event_stream_from version_number, max=nil
+      @aggregate.events_from(version_number, max)
+    end
+
+    def version
+      @aggregate.version
+    end
+
+    def count
+      event_stream.length
+    end
+
+    def destroy!
+      @aggregate.delete_events!
+      @aggregate.delete_snapshot!
+    end
+
+    def rebuild_snapshot!
+      @aggregate.delete_snapshot!
+      @aggregate.rebuild_snapshot!
+    end
+
+    private
+
+    def event_appender
+      EventAppender.new(@aggregate)
+    end
+
+    def translate_events(event_hashs)
+      event_hashs.map { |eh| translate_event(eh) }
+    end
+
+    def translate_event(event_hash)
+      occurred_at =  TimeHacker.translate_occurred_at_from_local_to_gmt(event_hash[:occurred_at])
+      SerializedEvent.new event_hash[:fully_qualified_name], event_hash[:serialized_event], event_hash[:version], occurred_at
+    end
+  end
+end

data/lib/event_store/errors.rb

@@ -0,0 +1,4 @@
+module EventStore
+  class AttributeMissingError < StandardError; end
+  class ConcurrencyError < StandardError; end
+end

data/lib/event_store/event_appender.rb

@@ -0,0 +1,84 @@
+module EventStore
+  class EventAppender
+
+    def initialize aggregate
+      @aggregate = aggregate
+    end
+
+    def append raw_events
+      EventStore.db.transaction do
+        set_current_version
+
+        prepared_events = raw_events.map do |raw_event|
+          event = prepare_event(raw_event)
+          validate! event
+          raise concurrency_error(event) if has_concurrency_issue?(event)
+          event
+        end
+        # All concurrency issues need to be checked before persisting any of the events
+        # Otherwise, the newly appended events may raise erroneous concurrency errors
+        result = @aggregate.events.multi_insert(prepared_events)
+        store_snapshot(prepared_events) unless result.nil?
+        result
+      end
+    end
+
+    def store_snapshot(prepared_events)
+      r = EventStore.redis
+      current_version_numbers = r.hgetall(@aggregate.snapshot_version_table)
+      current_version_numbers.default = -1
+      valid_snapshot_events = []
+      valid_snapshot_versions = []
+      prepared_events.each do |event|
+        if event[:version].to_i > current_version_numbers[event[:fully_qualified_name]].to_i
+          valid_snapshot_events   << event[:fully_qualified_name]
+          valid_snapshot_events   << (event[:version].to_s + EventStore::SNAPSHOT_DELIMITER + event[:serialized_event] + EventStore::SNAPSHOT_DELIMITER + event[:occurred_at].to_s)
+          valid_snapshot_versions << event[:fully_qualified_name]
+          valid_snapshot_versions << event[:version]
+        end
+      end
+      unless valid_snapshot_versions.empty?
+        last_version            = valid_snapshot_versions.last
+        valid_snapshot_versions << :current_version
+        valid_snapshot_versions << last_version.to_i
+        r.multi do
+          r.hmset(@aggregate.snapshot_version_table, valid_snapshot_versions)
+          r.hmset(@aggregate.snapshot_table, valid_snapshot_events)
+        end
+      end
+    end
+
+  private
+    def has_concurrency_issue? event
+      event[:version] <= current_version
+    end
+
+    def prepare_event raw_event
+      raise ArgumentError.new("Cannot Append a Nil Event") unless raw_event
+      { :version              => raw_event.version.to_i,
+        :aggregate_id         => raw_event.aggregate_id,
+        :occurred_at          => Time.parse(raw_event.occurred_at.to_s).utc, #to_s truncates microseconds, which brake Time equality
+        :serialized_event     => EventStore.escape_bytea(raw_event.serialized_event),
+        :fully_qualified_name => raw_event.fully_qualified_name }
+    end
+
+    def concurrency_error event
+      ConcurrencyError.new("The version of the event being added (version #{event[:version]}) is <= the current version (version #{current_version})")
+    end
+
+    private
+    def current_version
+      @current_version ||= @aggregate.version
+    end
+    alias :set_current_version :current_version
+
+    def validate! event_hash
+      [:aggregate_id, :fully_qualified_name, :occurred_at, :serialized_event, :version].each do |attribute_name|
+        if event_hash[attribute_name].to_s.strip.empty?
+          raise AttributeMissingError, "value required for #{attribute_name}"
+        end
+      end
+    end
+
+  end
+end

data/lib/event_store/time_hacker.rb

@@ -0,0 +1,16 @@
+module EventStore
+  class TimeHacker
+    class << self
+      #Hack around various DB adapters that hydrate dates from the db into the local ruby timezone
+      def translate_occurred_at_from_local_to_gmt(occurred_at)
+        if occurred_at.class == Time
+          #expecting "2001-02-03 01:26:40 -0700"
+          Time.parse(occurred_at.to_s.gsub(/\s[+-]\d+$/, ' UTC'))
+        elsif occurred_at.class == DateTime
+          #expecting "2001-02-03T01:26:40+00:00"
+          Time.parse(occurred_at.iso8601.gsub('T', ' ').gsub(/[+-]\d{2}\:\d{2}/, ' UTC'))
+        end
+      end
+    end
+  end
+end