sandthorn_driver_sequel_2 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8ad8b0f35cb67c5dfa7f950eed9c8d3f2a592a4c
+  data.tar.gz: c3834db70856e0baaf045907d01c457f42eeae5d
+SHA512:
+  metadata.gz: 8ce499a3eb979ba4c263662afc0475588e74c85f05ca8481d60e7a718ff9ec865838b76c0fcdf78236f0bd75cef593d2fa4af018a69096b5449568b5de4c8d77
+  data.tar.gz: 96b6174602519e18fb8bad0d80393fe40cb40973180c93494d06dadbe85d39e1bc1484f848dd84cc682d6f2b1e2f3ebaa16812b593146f37f3344afe36321993

data/.autotest

@@ -0,0 +1,3 @@
+Autotest.add_hook :initialize do |at|
+    %w{.git spec/db coverage}.each {|exception| at.add_exception(exception)}
+end

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.sqlite3
+.idea
+spec/db_file

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format d

data/.ruby-gemset

@@ -0,0 +1 @@
+sandthorn_driver_sequel

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.1

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.0
+  - 2.0.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sandthorn_driver_sequel.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,7 @@
+interactor :simple
+guard :rspec, cmd: 'bundle exec rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/sandthorn_driver_sequel/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Morgan Hallgren
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+[![Code Climate](https://codeclimate.com/github/Sandthorn/sandthorn_driver_sequel.png)](https://codeclimate.com/github/Sandthorn/sandthorn_driver_sequel)
+
+# Sandthorn Sequel-driver 2
+
+A SQL database driver for [Sandthorn](https://github.com/Sandthorn/sandthorn), made with [Sequel](http://sequel.jeremyevans.net/).
+
+This sequel driver is a rewrite from [sandthorn_driver_sequel](https://github.com/Sandthorn/sandthorn_driver_sequel) and its purpous is to speed up batch imports. 
+
+It has removed the aggregate table and store all aggregate and event data in the events table. Its now possible to output the event data to file and import it to the database after a batch import with the COPY command. This speeds up the import of larger dataset by a magnitude of alot. A note is that the aggregate_version checks has been removed and the user has to make sure that no other write of events are written during batch import.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sandthorn_driver_sequel_2'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sandthorn_driver_sequel_2
+
+## Usage
+
+Its possible to setup the EventStore with the events_file_path option, this will make the EventStore output all its events to this file path.
+
+    SandthornDriverSequel2.driver_from_url(url: event_store_url, events_file_path: "../events.csv")
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec
+
+task :benchmark do
+	sh "ulimit -n 8192 && rspec --tag benchmark"
+end

data/lib/sandthorn_driver_sequel_2/access/event_access.rb

@@ -0,0 +1,94 @@
+module SandthornDriverSequel2
+  class EventAccess < Access::Base
+    # = EventAccess
+    # Reads and writes events.
+
+    def store_events(events)
+      events = Utilities.array_wrap(events)
+      timestamp = Time.now.utc
+      events.each do |event|
+        store_event(timestamp, event)
+      end
+      #puts aggregate.inspect
+#      aggregate.save
+    end
+
+    def find_events_by_aggregate_id(aggregate_id)
+      #aggregate_version = Sequel.qualify(storage.events_table_name, :aggregate_version)
+      #aggregate_aggregate_id = Sequel.qualify(storage.aggregates_table_name, :aggregate_id)
+      #events_aggregate_id = Sequel.qualify(storage.events_table_name, :aggregate_id)
+      wrap(storage.events.where(:aggregate_id => aggregate_id)
+        .select(
+          :sequence_number,
+          :aggregate_id,
+          :aggregate_version,
+          :aggregate_type,
+          :event_name,
+          :event_data,
+          :timestamp)
+        .order(:sequence_number)
+        .all)
+    end
+
+    # Returns events that occurred after the given snapshot
+    def after_snapshot(snapshot)
+      _aggregate_version = snapshot.aggregate_version
+      aggregate_id = snapshot.aggregate_id
+      wrap(storage.events
+        .where(aggregate_id: aggregate_id)
+        .where { aggregate_version > _aggregate_version }.all)
+    end
+
+    def get_events(*args)
+      query_builder = EventQuery.new(storage)
+      query_builder.build(*args)
+      wrap(query_builder.events)
+    end
+
+    # Returns aggregate ids.
+    # @param aggregate_type, optional,
+    def aggregate_ids(aggregate_type: nil)
+      events = storage.events
+      if aggregate_type
+        events = events.where(aggregate_type: aggregate_type.to_s)
+      end
+      events.select_map(:aggregate_id).uniq
+    end
+
+    private
+
+    def wrap(arg)
+      events = Utilities.array_wrap(arg)
+      events.map { |e| EventWrapper.new(e.values) }
+    end
+
+    def build_event_data(timestamp, event)
+      {
+          aggregate_id: event.aggregate_id,
+          aggregate_version: event.aggregate_version,
+          aggregate_type: event.aggregate_type.to_s,
+          event_name: event.event_name,
+          event_data: event.event_data,
+          timestamp: timestamp
+      }
+    end
+
+    def check_versions!(aggregate, event)
+      version = aggregate.aggregate_version
+      if version != event[:aggregate_version]
+        raise Errors::ConcurrencyError.new(event, aggregate)
+      end
+    rescue TypeError
+      raise Errors::EventFormatError, "Event has wrong format: #{event.inspect}"
+    end
+
+    def store_event(timestamp, event)
+      event = EventWrapper.new(event)
+      #aggregate.aggregate_version += 1
+      #check_versions!(aggregate, event)
+      data = build_event_data(timestamp, event)
+      storage.events.insert(data)
+    end
+
+  end
+end

data/lib/sandthorn_driver_sequel_2/access/snapshot_access.rb

@@ -0,0 +1,88 @@
+module SandthornDriverSequel2
+  class SnapshotAccess < Access::Base
+
+    def find_by_aggregate_id(aggregate_id)
+      #aggregate = aggregates.find_by_aggregate_id(aggregate_id)
+      storage.snapshots.first(aggregate_id: aggregate_id)
+    end
+
+    def find(snapshot_id)
+      storage.snapshots[snapshot_id]
+    end
+
+    def record_snapshot(aggregate_id, snapshot_data)
+      #aggregate = aggregates.find_by_aggregate_id!(aggregate_id)
+      previous_snapshot = find_by_aggregate_id(aggregate_id)
+      #if perform_snapshot?(aggregate, previous_snapshot)
+      perform_snapshot(aggregate_id, previous_snapshot, snapshot_data)
+      #end
+    end
+
+    def obsolete(aggregate_types: [], max_event_distance: 100)
+      aggregate_types.map!(&:to_s)
+      snapshot_version = Sequel.qualify(storage.snapshots_table_name, :aggregate_version)
+      aggregate_version = Sequel.qualify(storage.aggregates_table_name, :aggregate_version)
+      aggregate_id = Sequel.qualify(storage.aggregates_table_name, :aggregate_id)
+      query = storage.aggregates.left_outer_join(storage.snapshots, aggregate_id: :aggregate_id)
+      query = query.select { (aggregate_version - snapshot_version).as(distance) }
+      query = query.select_append(aggregate_id, :aggregate_type)
+      query = query.where { (aggregate_version - coalesce(snapshot_version, 0)) > max_event_distance }
+      if aggregate_types.any?
+        query = query.where(aggregate_type: aggregate_types)
+      end
+      query.all
+    end
+
+    private
+
+    def aggregates
+      @aggregates ||= AggregateAccess.new(storage)
+    end
+
+    def perform_snapshot?(aggregate, snapshot)
+      return true if snapshot.nil?
+      snapshot = SnapshotWrapper.new(snapshot)
+      aggregate.aggregate_version > snapshot.aggregate_version
+    end
+
+    def perform_snapshot(aggregate_id, previous_snapshot, snapshot_data)
+      #check_snapshot_version!(aggregate, snapshot_data)
+      if valid_snapshot?(previous_snapshot)
+        update_snapshot(previous_snapshot, snapshot_data)
+      else
+        insert_snapshot(aggregate_id, snapshot_data)
+      end
+    end
+
+    def insert_snapshot(aggregate_id, snapshot_data)
+      data = build_snapshot(snapshot_data)
+      data[:aggregate_id] = aggregate_id
+      storage.snapshots.insert(data)
+    end
+
+    def build_snapshot(snapshot_data)
+      snapshot_data = SnapshotWrapper.new(snapshot_data)
+      {
+          snapshot_data:      snapshot_data.data,
+          aggregate_version:  snapshot_data.aggregate_version
+      }
+    end
+
+    def valid_snapshot?(snapshot)
+      snapshot && snapshot[:snapshot_data]
+    end
+
+    def update_snapshot(snapshot, snapshot_data)
+      data = build_snapshot(snapshot_data)
+      storage.snapshots.where(aggregate_id: snapshot.id).update(data)
+    end
+
+    def check_snapshot_version!(aggregate, snapshot_data)
+      snapshot = SnapshotWrapper.new(snapshot_data)
+      if aggregate.aggregate_version < snapshot.aggregate_version
+        raise Errors::WrongSnapshotVersionError.new(aggregate, snapshot.aggregate_version)
+      end
+    end
+
+  end
+end

data/lib/sandthorn_driver_sequel_2/access.rb

@@ -0,0 +1,19 @@
+module SandthornDriverSequel2
+  module Access
+    class Base
+      # = Access::Base
+      # Inheriting classes use +storage+ to provide access to a
+      # particular database model/table.
+      def initialize(storage)
+        @storage = storage
+      end
+
+      private
+
+      attr_reader :storage
+    end
+  end
+end
+
+require "sandthorn_driver_sequel_2/access/event_access"
+require "sandthorn_driver_sequel_2/access/snapshot_access"

data/lib/sandthorn_driver_sequel_2/errors.rb

@@ -0,0 +1,49 @@
+module SandthornDriverSequel2::Errors
+  Error = Class.new(StandardError)
+  InternalError = Class.new(Error)
+  NoAggregateError = Class.new(Error)
+  EventFormatError = Class.new(Error)
+
+  class ConcurrencyError < Error
+    attr_reader :event, :aggregate
+    def initialize(event, aggregate)
+      @event = event
+      @aggregate = aggregate
+      super(create_message)
+    end
+
+    def create_message
+      "#{aggregate.aggregate_type} with id #{aggregate.aggregate_id}: " +
+      "expected event with version #{aggregate.aggregate_version}, but got #{event.aggregate_version}"
+    end
+  end
+
+  class WrongAggregateVersionError < Error;
+    def initialize(aggregate, version)
+      @aggregate = aggregate
+      @version = version
+      super(create_message)
+    end
+
+    def create_message
+      "#{@aggregate[:aggregate_type]} with id #{@aggregate[:aggregate_id]}" +
+      " should be at version #{@version}" +
+      " but was #{@aggregate[:aggregate_version]} in the event store."
+    end
+  end
+
+  class WrongSnapshotVersionError < Error
+    attr_reader :aggregate, :version
+    def initialize(aggregate, version)
+      @aggregate = aggregate
+      @version = version
+      super(create_message)
+    end
+
+    def create_message
+      "#{aggregate[:aggregate_type]} with id #{aggregate[:aggregate_id]}: tried to save snapshot with version "+
+      "#{version}, but current version is at #{aggregate[:aggregate_version]}"
+    end
+  end
+
+end    

data/lib/sandthorn_driver_sequel_2/event_query.rb

@@ -0,0 +1,68 @@
+module SandthornDriverSequel2
+  class EventQuery
+    def initialize(storage)
+      @storage = storage
+    end
+
+    def build(
+      aggregate_types: [],
+      take: 0,
+      after_sequence_number: 0)
+
+      aggregate_types.map!(&:to_s)
+
+      query = storage.events
+      query = add_aggregate_types(query, aggregate_types)
+      query = add_sequence_number(query, after_sequence_number)
+      query = add_select(query)
+      query = add_limit(query, take)
+      @query = query.order(:sequence_number)
+    end
+
+    def events
+      @query.all
+    end
+
+    private
+
+    attr_reader :storage
+
+    def add_limit(query, take)
+      if take > 0
+        query.limit(take)
+      else
+        query
+      end
+    end
+
+    def add_select(query)
+      query.select(*select_columns)
+    end
+
+    def select_columns
+      aggregate_version = Sequel.qualify(storage.events_table_name, :aggregate_version)
+      aggregate_id = Sequel.qualify(storage.events_table_name, :aggregate_id)
+      [
+        :aggregate_type,
+        aggregate_version,
+        aggregate_id,
+        :sequence_number,
+        :event_name,
+        :event_data,
+        :timestamp
+      ]
+    end
+
+    def add_sequence_number(query, after_sequence_number)
+      query.where { sequence_number > after_sequence_number }
+    end
+
+    def add_aggregate_types(query, aggregate_types)
+      if aggregate_types.any?
+        query = query.where( :aggregate_type => aggregate_types )
+      end
+      query
+    end
+
+  end
+end

data/lib/sandthorn_driver_sequel_2/event_store.rb

@@ -0,0 +1,138 @@
+module SandthornDriverSequel2
+  class EventStore
+    include EventStoreContext
+
+    attr_reader :driver, :context, :url
+
+    def initialize url: nil, context: nil, events_file_path: nil
+      @driver = SequelDriver.new url: url
+      @context = context
+      @url = url
+
+      driver.execute do |db|
+        @storage = Storage.new(db, context, events_file_path)
+      end
+    end
+
+    def save_events events, aggregate_id, class_name
+      driver.execute_in_transaction do |db|
+        event_access = get_event_access(db)
+        events = events.map { |event| event[:aggregate_type] = class_name; event[:aggregate_id] = aggregate_id; event;}
+        event_access.store_events(events)
+      end
+    end
+
+    def get_aggregate_events(aggregate_id)
+      driver.execute do |db|
+        events = get_event_access(db)
+        events.find_events_by_aggregate_id(aggregate_id)
+      end
+    end
+
+    def save_snapshot aggregate_snapshot, aggregate_id
+      driver.execute_in_transaction do |db|
+        snapshot_access = get_snapshot_access(db)
+        snapshot_access.record_snapshot(aggregate_id, aggregate_snapshot)
+      end
+    end
+
+    # If the aggregate has a snapshot, return events starting from the snapshots.
+    # Otherwise, return all events.
+    # TODO: needs a better name
+    def get_aggregate_events_from_snapshot(aggregate_id)
+      driver.execute do |db|
+        #snapshots = get_snapshot_access(db)
+        event_access = get_event_access(db)
+        snapshot = false#snapshots.find_by_aggregate_id(aggregate_id)
+        if snapshot
+          events = event_access.after_snapshot(snapshot)
+          snapshot_event = build_snapshot_event(snapshot)
+          events.unshift(snapshot_event)
+        else
+          event_access.find_events_by_aggregate_id(aggregate_id)
+        end
+      end
+    end
+
+    def build_snapshot_event(snapshot)
+      {
+          aggregate_version: snapshot[:aggregate_version],
+          event_data: snapshot[:snapshot_data],
+          event_name: "aggregate_set_from_snapshot"
+      }
+    end
+
+    def get_aggregate aggregate_id, *class_name
+      warn(":get_aggregate is deprecated. Use :get_aggregate_events_from_snapshot")
+      get_aggregate_events_from_snapshot(aggregate_id)
+    end
+
+    def get_aggregate_ids(aggregate_type: nil)
+      driver.execute do |db|
+        access = get_event_access(db)
+        access.aggregate_ids(aggregate_type: aggregate_type)
+      end
+    end
+
+    def get_aggregate_list_by_typename(type)
+      warn(":get_aggregate_list_by_typenames is deprecated. Use :get_aggregate_ids")
+      get_aggregate_ids(type: type)
+    end
+
+    # def get_all_types
+    #   driver.execute do |db|
+    #     access = get_aggregate_access(db)
+    #     access.aggregate_types
+    #   end
+    # end
+
+    def get_snapshot aggregate_id
+      driver.execute do |db|
+        snapshots = get_snapshot_access(db)
+        snapshot = snapshots.find_by_aggregate_id(aggregate_id)
+        transform_snapshot(snapshot)
+      end
+    end
+
+    def get_events(*args)
+      driver.execute do |db|
+        event_access = get_event_access(db)
+        event_access.get_events(*args)
+      end
+    end
+
+    def get_new_events_after_event_id_matching_classname event_id, class_name, take: 0
+      get_events(after_sequence_number: event_id, aggregate_types: Utilities.array_wrap(class_name), take: take)
+    end
+
+    def obsolete_snapshots(*args)
+      driver.execute do |db|
+        snapshots = get_snapshot_access(db)
+        snapshots.obsolete(*args)
+      end
+    end
+
+    private
+
+    def transform_snapshot(snapshot)
+      {
+          aggregate_version: snapshot.aggregate_version,
+          event_data: snapshot.snapshot_data
+      }
+    end
+
+    def get_event_access(db)
+      EventAccess.new(storage(db))
+    end
+
+    def get_snapshot_access(db)
+      SnapshotAccess.new(storage(db))
+    end
+
+    def storage(db)
+      @storage
+      #Storage.new(db, context, file_output: @file_output)
+    end
+
+  end
+end

data/lib/sandthorn_driver_sequel_2/event_store_context.rb

@@ -0,0 +1,15 @@
+module SandthornDriverSequel2
+  module EventStoreContext
+    attr_reader :context
+    def events_table_name
+      with_context_if_exists :events
+    end
+    def snapshots_table_name
+      with_context_if_exists :snapshots
+    end
+    def with_context_if_exists name
+      name = "#{context}_#{name}".to_sym if context
+      name
+    end
+  end
+end

data/lib/sandthorn_driver_sequel_2/file_output_wrappers/events.rb

@@ -0,0 +1,37 @@
+require 'forwardable'
+
+module SandthornDriverSequel2
+  module FileOutputWrapper
+    class Events
+      extend Forwardable
+      def initialize event_file, sequence_number
+        @event_file = event_file
+        @sequence_number = sequence_number
+      end
+
+      def events sequel
+        @sequel = sequel
+        self
+      end
+
+      def insert *args
+        args.each do |event|
+          @sequence_number += 1
+          event_data = String.new("#{event[:event_data]}")
+          event_data = " #{event_data}" if event_data =~ /^[\n\r]/
+          @event_file.puts "#{@sequence_number};#{event[:aggregate_id]};#{event[:aggregate_version]};#{event[:aggregate_type]};#{event[:event_name]};#{event_data};#{event[:timestamp]}"
+        end
+      end
+
+      def_delegators :@sequel, :first, :where, :join, :select, :all
+
+      def save *args
+        @event_file.write args
+      end
+
+      def flush
+        @event_file.flush
+      end
+    end
+  end
+end