sourced 0.0.1

data/lib/sourced/projector.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Sourced
+  # Projectors react to events
+  # and update views of current state somewhere (a DB, files, etc)
+  class Projector
+    include Evolve
+    include Sync
+    extend Consumer
+
+    class << self
+      def handled_events = handled_events_for_evolve
+    end
+
+    attr_reader :id, :seq, :state
+
+    def initialize(id, backend: Sourced.config.backend, logger: Sourced.config.logger)
+      @id = id
+      @seq = 0
+      @backend = backend
+      @logger = logger
+      @state = init_state(id)
+    end
+
+    def inspect
+      %(<#{self.class} id:#{id} seq:#{seq}>)
+    end
+
+    def handle_events(events)
+      evolve(state, events)
+      save
+      [] # no commands
+    end
+
+    private
+
+    attr_reader :backend, :logger
+
+    def init_state(_id)
+      nil
+    end
+
+    def save
+      backend.transaction do
+        run_sync_blocks(state, nil, [])
+      end
+    end
+
+    # A StateStored projector fetches initial state from
+    # storage somewhere (DB, files, API)
+    # And then after reacting to events and updating state,
+    # it can save it back to the same or different storage.
+    # @example
+    #
+    #  class CartListings < Sourced::Projector::StateStored
+    #    # Fetch listing record from DB, or new one.
+    #    def init_state(id)
+    #      CartListing.find_or_initialize(id)
+    #    end
+    #
+    #    # Evolve listing record from events
+    #    evolve Carts::ItemAdded do |listing, event|
+    #      listing.total += event.payload.price
+    #    end
+    #
+    #    # Sync listing record back to DB
+    #    sync do |listing, _, _|
+    #      listing.save!
+    #    end
+    #  end
+    class StateStored < self
+      class << self
+        def handle_events(events)
+          instance = new(events.first.stream_id)
+          instance.handle_events(events)
+        end
+      end
+    end
+
+    # An EventSourced projector fetches initial state from
+    # past events in the event store.
+    # And then after reacting to events and updating state,
+    # it can save it to a DB table, a file, etc.
+    # @example
+    #
+    #  class CartListings < Sourced::Projector::EventSourced
+    #    # Initial in-memory state
+    #    def init_state(id)
+    #      { id:, total: 0 }
+    #    end
+    #
+    #    # Evolve listing record from events
+    #    evolve Carts::ItemAdded do |listing, event|
+    #      listing[:total] += event.payload.price
+    #    end
+    #
+    #    # Sync listing record to a file
+    #    sync do |listing, _, _|
+    #      File.write("/listings/#{listing[:id]}.json", JSON.dump(listing)) 
+    #    end
+    #  end
+    class EventSourced < self
+      class << self
+        def handle_events(events)
+          # The current state already includes
+          # the new events, so we need to load upto events.first.seq
+          instance = load(events.first.stream_id, upto: events.first.seq - 1)
+          instance.handle_events(events)
+        end
+
+        # Load from event history
+        #
+        # @param stream_id [String] the stream id
+        # @return [Sourced::Projector::EventSourced]
+        def load(stream_id, upto: nil)
+          new(stream_id).load(upto:)
+        end
+      end
+
+      # TODO: this is also in Decider. DRY up?
+      def load(after: nil, upto: nil)
+        events = backend.read_event_stream(id, after:, upto:)
+        if events.any?
+          @seq = events.last.seq 
+          evolve(state, events)
+        end
+        self
+      end
+    end
+  end
+end

data/lib/sourced/rails/install_generator.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record'
+
+module Sourced
+  module Rails
+    class InstallGenerator < ::Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path('templates', __dir__)
+
+      class_option :prefix, type: :string, default: 'sourced'
+
+      def copy_initializer_file
+        create_file 'config/initializers/sourced.rb' do
+          <<~CONTENT
+            # frozen_string_literal: true
+
+            require 'sourced'
+            require 'sourced/backends/active_record_backend'
+
+            # This table prefix is used to generate the initial database migrations.
+            # If you change the table prefix here,
+            # make sure to migrate your database to the new table names.
+            Sourced::Backends::ActiveRecordBackend.table_prefix = '#{table_prefix}'
+
+            # Configure Sors to use the ActiveRecord backend
+            Sourced.configure do |config|
+              config.backend = Sourced::Backends::ActiveRecordBackend.new
+              config.logger = Rails.logger
+            end
+          CONTENT
+        end
+      end
+
+      def copy_bin_file
+        copy_file 'bin_sourced', 'bin/sourced'
+        chmod 'bin/sourced', 0o755
+      end
+
+      def create_migration_file
+        migration_template 'create_sourced_tables.rb.erb', File.join(db_migrate_path, 'create_sourced_tables.rb')
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::STRING.to_f}]"
+      end
+
+      def table_prefix
+        options['prefix']
+      end
+    end
+  end
+end

data/lib/sourced/rails/railtie.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Sourced
+  module Rails
+    class Railtie < ::Rails::Railtie
+      # TODO: review this.
+      # Workers use Async, so this is needed
+      # but not sure this can be safely used with non Async servers like Puma.
+      # config.active_support.isolation_level = :fiber
+
+      generators do
+        require 'sourced/rails/install_generator'
+      end
+    end
+  end
+end

data/lib/sourced/rails/templates/bin_sors

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/environment"
+require "sourced"
+
+ActiveRecord::Base.logger = nil
+
+Sourced::Supervisor.start

data/lib/sourced/rails/templates/create_sors_tables.rb.erb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+class CreateSorsTables < ActiveRecord::Migration<%= migration_version %>
+  def change
+    # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support
+    # enable_extension 'pgcrypto'
+
+    if connection.class.name == 'ActiveRecord::ConnectionAdapters::SQLite3Adapter'
+      create_table :<%= table_prefix %>_events, id: false do |t|
+        t.string :id, null: false, index: { unique: true }
+        t.bigint :global_seq, primary_key: true
+        t.bigint :seq
+        t.string :stream_id, null: false, index: true
+        t.string :type, null: false
+        t.datetime :created_at
+        t.string :producer
+        t.string :causation_id, index: true
+        t.string :correlation_id
+        t.text :payload
+      end
+    else
+      create_table :<%= table_prefix %>_events, id: :uuid do |t|
+        t.bigserial :global_seq, index: true
+        t.bigint :seq
+        t.string :stream_id, null: false, index: true
+        t.string :type, null: false
+        t.datetime :created_at
+        t.string :producer
+        t.uuid :causation_id, index: true
+        t.uuid :correlation_id
+        t.jsonb :payload
+      end
+    end
+
+    add_index :<%= table_prefix %>_events, %i[stream_id seq], unique: true
+
+    create_table :<%= table_prefix %>_streams do |t|
+      t.text :stream_id, null: false, index: { unique: true }
+      t.boolean :locked, default: false, null: false
+    end
+
+    create_table :<%= table_prefix %>_commands do |t|
+      t.string :stream_id, null: false
+      if t.class.name == 'ActiveRecord::ConnectionAdapters::SQLite3::TableDefinition'
+        t.text :data, null: false
+        t.datetime :scheduled_at, null: false, default: -> { 'CURRENT_TIMESTAMP' }
+      else
+        t.jsonb :data, null: false
+        t.datetime :scheduled_at, null: false, default: -> { 'NOW()' }
+      end
+    end
+
+    add_foreign_key :<%= table_prefix %>_commands, :<%= table_prefix %>_streams, column: :stream_id, primary_key: :stream_id
+  end
+end

data/lib/sourced/react.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Sourced
+  module React
+    PREFIX = 'reaction'
+
+    def self.included(base)
+      super
+      base.extend ClassMethods
+    end
+
+    def react(events)
+      events.flat_map { |event| __handle_reaction(event) }
+    end
+
+    private
+
+    def __handle_reaction(event)
+      method_name = Sourced.message_method_name(React::PREFIX, event.class.to_s)
+      return [] unless respond_to?(method_name)
+
+      cmds = send(method_name, event)
+      [cmds].flatten.compact.map do |cmd|
+        cmd.with_metadata(producer: self.class.consumer_info.group_id)
+      end
+    end
+
+    module ClassMethods
+      def inherited(subclass)
+        super
+        handled_events_for_react.each do |evt_type|
+          subclass.handled_events_for_react << evt_type
+        end
+      end
+
+      # Override this with extend Sourced::Consumer
+      def consumer_info
+        Sourced::Consumer::ConsumerInfo.new(group_id: name)
+      end
+
+      # These two are the Reactor interface
+      # expected by Worker
+      def handle_events(_events)
+        raise NoMethodError, "implement .handle_events(Array<Event>) in #{self}"
+      end
+
+      def handled_events_for_react
+        @handled_events_for_react ||= []
+      end
+
+      def react(event_type, &block)
+        handled_events_for_react << event_type unless event_type.is_a?(Symbol)
+        define_method(Sourced.message_method_name(React::PREFIX, event_type.to_s), &block) if block_given?
+      end
+    end
+  end
+end

data/lib/sourced/router.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module Sourced
+  class Router
+    include Singleton
+
+    PID = Process.pid
+
+    class << self
+      public :new
+
+      def register(...)
+        instance.register(...)
+      end
+
+      def handle_command(command)
+        instance.handle_command(command)
+      end
+
+      def dispatch_next_command
+        instance.dispatch_next_command
+      end
+
+      def handle_events(events)
+        instance.handle_events(events)
+      end
+
+      def async_reactors
+        instance.async_reactors
+      end
+
+      def handle_and_ack_events_for_reactor(reactor, events)
+        instance.handle_and_ack_events_for_reactor(reactor, events)
+      end
+
+      def handle_next_event_for_reactor(reactor, process_name = nil)
+        instance.handle_next_event_for_reactor(reactor, process_name)
+      end
+    end
+
+    attr_reader :sync_reactors, :async_reactors, :backend, :logger
+
+    def initialize(backend: Sourced.config.backend, logger: Sourced.config.logger)
+      @backend = backend
+      @logger = logger
+      @decider_lookup = {}
+      @sync_reactors = Set.new
+      @async_reactors = Set.new
+    end
+
+    def register(thing)
+      if DeciderInterface === thing
+        thing.handled_commands.each do |cmd_type|
+          @decider_lookup[cmd_type] = thing
+        end
+      end
+
+      return unless ReactorInterface === thing
+
+      if thing.consumer_info.async
+        @async_reactors << thing
+      else
+        @sync_reactors << thing
+      end
+    end
+
+    def handle_command(command)
+      decider = @decider_lookup.fetch(command.class)
+      decider.handle_command(command)
+    end
+
+    def handle_events(events)
+      event_classes = events.map(&:class)
+      reactors = sync_reactors.filter do |r|
+        r.handled_events.intersect?(event_classes)
+      end
+      # TODO
+      # Reactors can return commands to run next
+      # I need to think about how to best to handle this safely
+      # Also this could potential lead to infinite recursion!
+      reactors.each do |r|
+        handle_and_ack_events_for_reactor(r, events)
+      end
+    end
+
+    def handle_next_event_for_reactor(reactor, process_name = nil)
+      backend.reserve_next_for_reactor(reactor) do |event|
+        # We're dealing with one event at a time now
+        # So reactors should return a single command, or nothing
+        log_event('handling event', reactor, event, process_name)
+        commands = reactor.handle_events([event])
+        if commands.any?
+          # This will run a new decider
+          # which may be expensive, timeout, or raise an exception
+          # TODO: handle decider errors
+          backend.schedule_commands(commands)
+        end
+
+        event
+      end
+    end
+
+    # When in sync mode, we want both events
+    # and any resulting commands to be processed syncronously
+    # and in the same transaction as events are appended to store.
+    # We could handle commands in threads or fibers,
+    # if they belong to different streams than the events,
+    # but we need to make sure to raise exceptions in the main thread.
+    # so that the transaction is rolled back.
+    def handle_and_ack_events_for_reactor(reactor, events)
+      backend.ack_on(reactor.consumer_info.group_id, events.last.id) do
+        commands = reactor.handle_events(events)
+        if commands && commands.any?
+          # TODO: Commands may or may not belong to he same stream as events
+          # if they belong to the same stream,
+          # hey need to be dispached in order to preserve per stream order
+          # If they belong to different streams, they can be dispatched in parallel
+          # or put in a command bus.
+          # TODO2: we also need to handle exceptions here
+          # TODO3: this is not tested
+          commands.each do |cmd|
+            log_event(' -> produced command', reactor, cmd)
+            handle_command(cmd)
+          end
+        end
+      end
+    end
+
+    def dispatch_next_command
+      backend.next_command do |cmd|
+        #  TODO: error handling
+        handle_command(cmd)
+      end
+    end
+
+    private
+
+    def log_event(label, reactor, event, process_name = PID)
+      logger.info "[#{process_name}]: #{reactor.consumer_info.group_id} #{label} #{event_info(event)}"
+    end
+
+    def event_info(event)
+      %([#{event.type}] stream_id:#{event.stream_id} seq:#{event.seq})
+    end
+  end
+end

data/lib/sourced/supervisor.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'async'
+require 'console'
+require 'sourced/worker'
+
+module Sourced
+  class Supervisor
+    def self.start(...)
+      new(...).start
+    end
+
+    def initialize(logger: Sourced.config.logger, count: 2)
+      @logger = logger
+      @count = count
+      @workers = []
+    end
+
+    def start
+      logger.info("Starting sync supervisor with #{@count} workers")
+      set_signal_handlers
+      @workers = @count.times.map do |i|
+        Worker.new(logger:, name: "worker-#{i}")
+      end
+      Sync do |task|
+        @workers.each do |wrk|
+          task.async do
+            wrk.poll
+          end
+        end
+      end
+    end
+
+    def stop
+      logger.info("Stopping #{@workers.size} workers")
+      @workers.each(&:stop)
+      logger.info('All workers stopped')
+    end
+
+    def set_signal_handlers
+      Signal.trap('INT') { stop }
+      Signal.trap('TERM') { stop }
+    end
+
+    private
+
+    attr_reader :logger
+  end
+end

data/lib/sourced/sync.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Sourced
+  module Sync
+    def self.included(base)
+      super
+      base.extend ClassMethods
+    end
+
+    def run_sync_blocks(state, command, events)
+      self.class.sync_blocks.each do |blk|
+        case blk
+        when Proc
+          if blk.arity == 2 # (command, events)
+            instance_exec(command, events, &blk)
+          else # (state, command, events)
+            instance_exec(state, command, events, &blk)
+          end
+        else
+          blk.call(state, command, events)
+        end
+      end
+    end
+
+    CallableInterface = Sourced::Types::Interface[:call]
+
+    class SyncReactor < SimpleDelegator
+      def handle_events(events)
+        Router.handle_and_ack_events_for_reactor(__getobj__, events)
+      end
+
+      def call(_state, _command, events)
+        handle_events(events)
+      end
+    end
+
+    module ClassMethods
+      def inherited(subclass)
+        super
+        sync_blocks.each do |blk|
+          subclass.sync_blocks << blk
+        end
+      end
+
+      def sync_blocks
+        @sync_blocks ||= []
+      end
+
+      def sync(callable = nil, &block)
+        callable ||= block
+        callable = case callable
+                   when Proc
+                     unless (2..3).include?(callable.arity)
+                       raise ArgumentError,
+                             'sync block must accept 2 or 3 arguments'
+                     end
+
+                     callable
+                   when ReactorInterface
+                     # Wrap reactors here
+                     # TODO:
+                     # If the sync reactor runs successfully
+                     # A). we want to ACK processed events for it in the offsets table
+                     # so that if the reactor is moved to async execution
+                     # it doesn't reprocess the same events again
+                     # B). The reactors .handle_events may return commands
+                     # Do we want to dispatch those commands inline?
+                     # Or is this another reason to have a separate async command bus
+                     SyncReactor.new(callable)
+                   when CallableInterface
+                     callable
+                   else
+                     raise ArgumentError, 'sync block must be a Proc, Sourced::ReactorInterface or #call interface'
+                   end
+
+        sync_blocks << callable
+      end
+    end
+  end
+end

data/lib/sourced/types.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'plumb'
+require 'time'
+require 'securerandom'
+
+module Sourced
+  module Types
+    include Plumb::Types
+
+    # A UUID string, or generate a new one
+    AutoUUID = UUID::V4.default { SecureRandom.uuid }
+
+    # Deeply symbolize keys of a hash
+    # Usage:
+    #   SymbolizedHash.parse({ 'a' => { 'b' => 'c' } }) # => { a: { b: 'c' } }
+    SymbolizedHash = Hash[
+      # String keys are converted to symbols
+      (Symbol | String.transform(::Symbol, &:to_sym)),
+      # Hash values are recursively symbolized
+      Any.defer { SymbolizedHash } | Any
+    ]
+  end
+end

data/lib/sourced/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Sourced
+  VERSION = '0.0.1'
+end