sourced 0.0.1

data/lib/sourced/backends/sequel_backend.rb

@@ -0,0 +1,387 @@
+# frozen_string_literal: true
+
+require 'sequel'
+require 'json'
+require 'sourced/message'
+
+Sequel.extension :fiber_concurrency
+Sequel.extension :pg_json if defined?(PG)
+
+module Sourced
+  module Backends
+    class SequelBackend
+      def initialize(db, logger: Sourced.config.logger, prefix: 'sourced')
+        @db = connect(db)
+        @logger = logger
+        @prefix = prefix
+        @commands_table = table_name(:commands)
+        @streams_table = table_name(:streams)
+        @offsets_table = table_name(:offsets)
+        @events_table = table_name(:events)
+        logger.info("Connected to #{@db}")
+      end
+
+      def installed?
+        db.table_exists?(events_table) && db.table_exists?(streams_table) && db.table_exists?(offsets_table) && db.table_exists?(commands_table)
+      end
+
+      def schedule_commands(commands)
+        return false if commands.empty?
+
+        rows = commands.map { |c| serialize_command(c) }
+
+        db.transaction do
+          db[commands_table].multi_insert(rows)
+        end
+        true
+      end
+
+      # TODO: if the application raises an exception
+      # the command row is not deleted, so that it can be retried.
+      # However, if a command fails _permanently_ there's no point in keeping it in the queue,
+      # this ties with unresolved error handling in event handling, too.
+      def next_command(&reserve)
+        if block_given?
+          db.transaction do
+            row = db.fetch(sql_for_next_command, Time.now.utc).first
+            return unless row
+
+            cmd = deserialize_event(row)
+            yield cmd
+            db[commands_table].where(id: cmd.id).delete
+            cmd
+            # TODO: on failure, do we want to mark command as failed
+            # and put it in a dead-letter queue?
+          end
+        else
+          row = db[commands_table].order(:created_at).first
+          row ? deserialize_event(row) : nil
+        end
+      end
+
+      Stats = Data.define(:stream_count, :max_global_seq, :groups)
+
+      def stats
+        stream_count = db[streams_table].count
+        max_global_seq = db[events_table].max(:global_seq)
+        groups = db.fetch(sql_for_consumer_stats).all
+        Stats.new(stream_count, max_global_seq, groups)
+      end
+
+      def transaction(&)
+        db.transaction(&)
+      end
+
+      def append_to_stream(stream_id, events)
+        return if events.empty?
+
+        if events.map(&:stream_id).uniq.size > 1
+          raise ArgumentError, 'Events must all belong to the same stream'
+        end
+
+        db.transaction do
+          seq = events.last.seq
+          id = db[streams_table].insert_conflict(target: :stream_id, update: { seq:, updated_at: Time.now }).insert(stream_id:, seq:)
+          rows = events.map { |e| serialize_event(e, id) }
+          db[events_table].multi_insert(rows)
+        end
+        true
+      rescue Sequel::UniqueConstraintViolation => e
+        raise Sourced::ConcurrentAppendError, e.message
+      end
+
+      # @param reactor [Sourced::ReactorInterface]
+      def reserve_next_for_reactor(reactor, &)
+        group_id = reactor.consumer_info.group_id
+        handled_events = reactor.handled_events.map(&:type)
+
+        db.transaction do
+          start_from = reactor.consumer_info.start_from.call
+          row = if start_from.is_a?(Time)
+            db.fetch(sql_for_reserve_next_with_events(handled_events, true), group_id, group_id, start_from).first
+          else
+            db.fetch(sql_for_reserve_next_with_events(handled_events), group_id, group_id).first
+          end
+          return unless row
+
+          event = deserialize_event(row)
+
+          if block_given?
+            yield(event)
+            # ACK
+            ack_event(group_id, row[:stream_id_fk], row[:global_seq])
+          end
+
+          event
+        end
+      end
+
+      def ack_on(group_id, event_id, &)
+        db.transaction do
+          row = db.fetch(sql_for_ack_on, group_id, event_id).first
+          raise Sourced::ConcurrentAckError, "Stream for event #{event_id} is being concurrently processed by #{group_id}" unless row
+
+          yield if block_given?
+
+          ack_event(group_id, row[:stream_id_fk], row[:global_seq])
+        end
+      end
+
+      private def ack_event(group_id, stream_id, global_seq)
+        db[offsets_table]
+          .insert_conflict(
+            target: [:group_id, :stream_id],
+            update: { global_seq: Sequel[:excluded][:global_seq] }
+          )
+          .insert(stream_id:, group_id:, global_seq:)
+      end
+
+      private def base_events_query
+        db[events_table]
+          .select(
+            Sequel[events_table][:id],
+            Sequel[streams_table][:stream_id],
+            Sequel[events_table][:seq],
+            Sequel[events_table][:global_seq],
+            Sequel[events_table][:type],
+            Sequel[events_table][:created_at],
+            Sequel[events_table][:causation_id],
+            Sequel[events_table][:correlation_id],
+            Sequel[events_table][:metadata],
+            Sequel[events_table][:payload],
+          )
+          .join(streams_table, id: :stream_id)
+      end
+
+      def read_correlation_batch(event_id)
+        correlation_subquery = db[events_table]
+          .select(:correlation_id)
+          .where(id: event_id)
+
+        query = base_events_query
+          .where(Sequel[events_table][:correlation_id] => correlation_subquery)
+          .order(Sequel[events_table][:global_seq])
+
+        query.map do |row|
+          deserialize_event(row)
+        end
+      end
+
+      def read_event_stream(stream_id, after: nil, upto: nil)
+        _events_table = events_table # need local variable for Sequel block
+
+        query = base_events_query.where(Sequel[streams_table][:stream_id] => stream_id)
+
+        query = query.where { Sequel[_events_table][:seq] > after } if after
+        query = query.where { Sequel[_events_table][:seq] <= upto } if upto
+        query.order(Sequel[_events_table][:global_seq]).map do |row|
+          deserialize_event(row)
+        end
+      end
+
+      # For tests only
+      def clear!
+        raise 'Not in test environment' unless ENV['ENVIRONMENT'] == 'test'
+        # Truncate and restart global_seq increment first
+        db[events_table].truncate(cascade: true, only: true, restart: true)
+        db[events_table].delete
+        db[commands_table].delete
+        db[offsets_table].delete
+        db[streams_table].delete
+      end
+
+      def install
+        if @db.class.name == 'Sequel::SQLite::Database'
+          raise 'no SQLite support yet'
+        end
+
+        _streams_table = streams_table
+
+        db.create_table?(streams_table) do
+          primary_key :id
+          String :stream_id, null: false, unique: true
+          Time :updated_at, null: false, default: Sequel.function(:now)
+          Bignum :seq, null: false
+        end
+
+        logger.info("Created table #{streams_table}")
+
+        db.create_table?(offsets_table) do
+          primary_key :id
+          foreign_key :stream_id, _streams_table
+          String :group_id, null: false, index: true
+          Bignum :global_seq, null: false
+          Time :created_at, null: false, default: Sequel.function(:now)
+          index %i[group_id stream_id], unique: true
+        end
+
+        logger.info("Created table #{offsets_table}")
+
+        db.create_table?(events_table) do
+          primary_key :global_seq, type: :Bignum
+          column :id, :uuid, unique: true
+          foreign_key :stream_id, _streams_table
+          Bignum :seq, null: false
+          String :type, null: false
+          Time :created_at, null: false
+          column :causation_id, :uuid, index: true
+          column :correlation_id, :uuid
+          column :metadata, :jsonb
+          column :payload, :jsonb
+          index %i[stream_id seq], unique: true
+        end
+
+        logger.info("Created table #{events_table}")
+
+        db.create_table?(commands_table) do
+          column :id, :uuid, unique: true
+          String :stream_id, null: false, index: true
+          String :type, null: false
+          Time :created_at, null: false, index: true
+          column :causation_id, :uuid
+          column :correlation_id, :uuid
+          column :metadata, :jsonb
+          column :payload, :jsonb
+        end
+
+        logger.info("Created table #{commands_table}")
+
+        self
+      end
+
+      private
+
+      attr_reader :db, :logger, :prefix, :events_table, :streams_table, :offsets_table, :commands_table
+
+      def sql_for_next_command
+        <<~SQL
+          WITH next_command AS (
+              SELECT 
+                  id,
+                  stream_id,
+                  type,
+                  causation_id,
+                  correlation_id,
+                  metadata,
+                  payload,
+                  created_at,
+                  pg_try_advisory_xact_lock(hashtext(stream_id::text)) AS lock_obtained
+              FROM #{commands_table}
+              WHERE created_at <= ? 
+              ORDER BY created_at
+          )
+          SELECT *
+          FROM next_command
+          WHERE lock_obtained = true
+          LIMIT 1;
+        SQL
+      end
+
+      def sql_for_reserve_next_with_events(handled_events, with_time_window = false)
+        event_types = handled_events.map { |e| "'#{e}'" }
+        event_types_sql = event_types.any? ? " AND e.type IN(#{event_types.join(',')})" : ''
+        time_window_sql = with_time_window ? ' AND e.created_at > ?' : ''
+
+        <<~SQL
+          WITH candidate_events AS (
+              SELECT 
+                  e.global_seq,
+                  e.id,
+                  e.stream_id AS stream_id_fk,
+                  s.stream_id,
+                  e.seq,
+                  e.type,
+                  e.causation_id,
+                  e.correlation_id,
+                  e.metadata,
+                  e.payload,
+                  e.created_at,
+                  pg_try_advisory_xact_lock(hashtext(?::text), hashtext(s.id::text)) as lock_obtained
+              FROM #{events_table} e
+              JOIN #{streams_table} s ON e.stream_id = s.id
+              LEFT JOIN #{offsets_table} o ON o.stream_id = e.stream_id 
+                  AND o.group_id = ?
+              WHERE e.global_seq > COALESCE(o.global_seq, 0)#{event_types_sql}#{time_window_sql}
+              ORDER BY e.global_seq
+          )
+          SELECT *
+          FROM candidate_events
+          WHERE lock_obtained = true
+          LIMIT 1;
+        SQL
+      end
+
+      def sql_for_ack_on
+        <<~SQL
+          WITH candidate_rows AS (
+            SELECT 
+                e.global_seq,
+                e.stream_id AS stream_id_fk, 
+                pg_try_advisory_xact_lock(hashtext(?::text), hashtext(s.id::text)) as lock_obtained
+            FROM #{events_table} e
+            JOIN #{streams_table} s ON e.stream_id = s.id
+            WHERE e.id = ?
+          )
+          SELECT *
+          FROM candidate_rows
+          WHERE lock_obtained = true
+          LIMIT 1;
+        SQL
+      end
+
+      def sql_for_consumer_stats
+        @sql_for_consumer_stats ||= <<~SQL
+          SELECT 
+              group_id,
+              min(global_seq) as oldest_processed,
+              max(global_seq) as newest_processed,
+              count(*) as stream_count
+          FROM #{offsets_table}
+          GROUP BY group_id;
+        SQL
+      end
+
+      def table_name(name)
+        [prefix, name].join('_').to_sym
+      end
+
+      def parse_json(json)
+        return json unless json.is_a?(String)
+
+        JSON.parse(json, symbolize_names: true)
+      end
+
+      def serialize_command(cmd)
+        row = cmd.to_h.except(:seq)
+        row[:metadata] = JSON.dump(row[:metadata]) if row[:metadata]
+        row[:payload] = JSON.dump(row[:payload]) if row[:payload]
+        row
+      end
+
+      def serialize_event(event, stream_id)
+        row = event.to_h
+        row[:stream_id] = stream_id
+        row[:metadata] = JSON.dump(row[:metadata]) if row[:metadata]
+        row[:payload] = JSON.dump(row[:payload]) if row[:payload]
+        row
+      end
+
+      def deserialize_event(row)
+        row[:payload] = parse_json(row[:payload]) if row[:payload]
+        row[:metadata] = parse_json(row[:metadata]) if row[:metadata]
+        Message.from(row)
+      end
+
+      def connect(db)
+        case db
+        when Sequel::Database
+          db
+        when String, Hash
+          Sequel.connect(db)
+        else
+          raise ArgumentError, "Invalid database connection: #{db.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/sourced/backends/test_backend.rb

@@ -0,0 +1,273 @@
+# frozen_string_literal: true
+
+require 'thread'
+
+module Sourced
+  module Backends
+    class TestBackend
+      class Group
+        attr_reader :group_id
+
+        Offset = Struct.new(:stream_id, :index, :locked)
+
+        def initialize(group_id, backend)
+          @group_id = group_id
+          @backend = backend
+          @offsets = {}
+          reindex
+        end
+
+        def to_h
+          active_offsets = @offsets.values.select { |o| o.index >= 0 }
+          oldest_processed = (active_offsets.min_by(&:index)&.index || -1) + 1
+          newest_processed = (active_offsets.max_by(&:index)&.index || -1) + 1
+          stream_count = active_offsets.size
+
+          { group_id:, oldest_processed:, newest_processed:, stream_count: }
+        end
+
+        def reindex
+          backend.events.each.with_index do |e, idx|
+            @offsets[e.stream_id] ||= Offset.new(e.stream_id, -1, false)
+          end
+        end
+
+        def ack_on(event_id, &)
+          global_seq = backend.events.find_index { |e| e.id == event_id }
+          return unless global_seq
+
+          evt = backend.events[global_seq]
+          offset = @offsets[evt.stream_id]
+          if offset.locked
+            raise Sourced::ConcurrentAckError, "Stream for event #{event_id} is being concurrently processed by #{group_id}" unless row
+          else
+            offset.locked = true
+            yield
+            offset.index = global_seq
+            offset.locked = false
+          end
+        end
+
+        NOOP_FILTER = ->(_) { true } 
+
+        def reserve_next(handled_events, time_window, &)
+          time_filter = time_window.is_a?(Time) ? ->(e) { e.created_at > time_window } : NOOP_FILTER
+          evt = nil
+          offset = nil
+          index = -1
+
+          backend.events.each.with_index do |e, idx|
+            offset = @offsets[e.stream_id]
+            if offset.locked # stream locked by another consumer in the group
+              next
+            elsif idx > offset.index && handled_events.include?(e.class) && time_filter.call(e) # new event for the stream
+              evt = e
+              offset.locked = true
+              index = idx
+              break
+            else # event already consumed
+            end
+          end
+
+          if evt
+            if block_given?
+              yield(evt)
+              offset.index = index
+            end
+            offset.locked = false
+          end
+          evt
+        end
+
+        private
+
+        attr_reader :backend
+      end
+
+      def initialize
+        clear!
+        @mutex = Mutex.new
+        @in_tx = false
+        @tx_id = nil
+      end
+
+      def events = @state.events
+
+      def inspect
+        %(<#{self.class} events:#{events.size} streams:#{@state.events_by_stream_id.size}>)
+      end
+
+      class State
+        attr_reader :events, :commands, :groups, :events_by_correlation_id, :events_by_stream_id, :stream_id_seq_index
+
+        def initialize(
+          events: [], 
+          commands: [],
+          groups: Hash.new { |h, k| h[k] = Group.new(k, self) }, 
+          events_by_correlation_id: Hash.new { |h, k| h[k] = [] }, 
+          events_by_stream_id: Hash.new { |h, k| h[k] = [] },
+          stream_id_seq_index: {}
+        )
+
+          @events = events
+          @groups = groups
+          @events_by_correlation_id = events_by_correlation_id
+          @commands = commands
+          @command_locks = {}
+          @events_by_stream_id = events_by_stream_id
+          @stream_id_seq_index = stream_id_seq_index
+        end
+
+        def schedule_commands(commands)
+          @commands = (@commands + commands).sort_by(&:created_at)
+        end
+
+        def next_command(&reserve)
+          now = Time.now.utc
+
+          if block_given?
+            return nil if @commands.empty?
+            idx = @commands.index do |c|
+              !@command_locks[c.stream_id] && c.created_at <= now
+            end
+
+            return nil unless idx
+            cmd = @commands[idx]
+            @command_locks[cmd.stream_id] = true
+            begin
+              yield cmd
+              @commands.delete_at(idx)
+            ensure
+              @command_locks.delete(cmd.stream_id)
+            end
+            cmd
+          else
+            @commands.first
+          end
+        end
+
+        def copy
+          self.class.new(
+            events: events.dup,
+            commands: commands.dup,
+            groups: deep_dup(groups),
+            events_by_correlation_id: deep_dup(events_by_correlation_id),
+            events_by_stream_id: deep_dup(events_by_stream_id),
+            stream_id_seq_index: deep_dup(stream_id_seq_index)
+          )
+        end
+
+        private def deep_dup(hash)
+          hash.each.with_object(hash.dup.clear) do |(k, v), new_hash|
+            new_hash[k] = v.dup
+          end
+        end
+      end
+
+      def clear!
+        @state = State.new
+      end
+
+      def installed? = true
+
+      def reserve_next_for_reactor(reactor, &)
+        group_id = reactor.consumer_info.group_id
+        start_from = reactor.consumer_info.start_from.call
+        transaction do
+          group = @state.groups[group_id]
+          group.reserve_next(reactor.handled_events, start_from, &)
+        end
+      end
+
+      def ack_on(group_id, event_id, &)
+        transaction do
+          group = @state.groups[group_id]
+          group.ack_on(event_id, &)
+        end
+      end
+
+      def schedule_commands(commands)
+        transaction do
+          @state.schedule_commands(commands)
+        end
+      end
+
+      def next_command(&)
+        transaction do
+          @state.next_command(&)
+        end
+      end
+
+      Stats = Data.define(:stream_count, :max_global_seq, :groups)
+
+      def stats
+        stream_count = @state.events_by_stream_id.size
+        max_global_seq = events.size
+        groups = @state.groups.values.map(&:to_h).filter { |g| g[:stream_count] > 0 }
+        Stats.new(stream_count, max_global_seq, groups)
+      end
+
+      def transaction(&)
+        if @in_tx
+          yield
+        else
+          @mutex.synchronize do
+            @in_tx = true
+            @state_snapshot = @state.copy
+            result = yield
+            @in_tx = false
+            @state_snapshot = nil
+            result
+          end
+        end
+      rescue StandardError => e
+        @in_tx = false
+        @state = @state_snapshot if @state_snapshot
+        raise
+      end
+
+      def append_to_stream(stream_id, events)
+        transaction do
+          check_unique_seq!(events)
+
+          events.each do |event|
+            @state.events_by_correlation_id[event.correlation_id] << event
+            @state.events_by_stream_id[stream_id] << event
+            @state.events << event
+            @state.stream_id_seq_index[seq_key(stream_id, event)] = true
+          end
+        end
+        @state.groups.each_value(&:reindex)
+        true
+      end
+
+      def read_correlation_batch(event_id)
+        event = @state.events.find { |e| e.id == event_id }
+        return [] unless event
+        @state.events_by_correlation_id[event.correlation_id]
+      end
+
+      def read_event_stream(stream_id, after: nil, upto: nil)
+        events = @state.events_by_stream_id[stream_id]
+        events = events.select { |e| e.seq > after } if after
+        events = events.select { |e| e.seq <= upto } if upto
+        events
+      end
+
+      private
+
+      def check_unique_seq!(events)
+        duplicate = events.find do |event|
+          @state.stream_id_seq_index[seq_key(event.stream_id, event)]
+        end
+        if duplicate
+          raise Sourced::ConcurrentAppendError, "Duplicate stream_id/seq: #{duplicate.stream_id}/#{duplicate.seq}"
+        end
+      end
+
+      def seq_key(stream_id, event)
+        [stream_id, event.seq]
+      end
+    end
+  end
+end

data/lib/sourced/command_context.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'sourced/types'
+
+module Sourced
+  # A command factory to instantiate commands from Hash attributes
+  # including extra metadata.
+  # @example
+  #
+  #  ctx = Sourced::CommandContext.new(
+  #    stream_id: params[:stream_id],
+  #    metadata: {
+  #      user_id: session[:user_id]
+  #    }
+  #  )
+  #
+  # # params[:command] should be a Hash with { type: String, payload: Hash | nil }
+  #
+  #  cmd = ctx.build(params[:command])
+  #  cmd.stream_id # String
+  #  cmd.metadata[:user_id] # == session[:user_id]
+  #
+  class CommandContext
+    # @option stream_id [String]
+    # @option metadata [Hash] metadata to add to commands built by this context
+    # @option scope [Sourced::Message] Message class to use as command registry
+    def initialize(stream_id: nil, metadata: Plumb::BLANK_HASH, scope: Sourced::Command)
+      @defaults = {
+        stream_id:,
+        metadata:
+      }.freeze
+      @scope = scope
+    end
+
+    # @param attrs [Hash] attributes to lookup and buils a scope from.
+    # @return [Sourced::Message]
+    def build(attrs)
+      attrs = defaults.merge(Types::SymbolizedHash.parse(attrs))
+      scope.from(attrs)
+    end
+
+    private
+
+    attr_reader :defaults, :scope
+  end
+end