concurrent_pipeline 0.1.0

data/lib/concurrent_pipeline/processors/actor_processor.rb

@@ -0,0 +1,363 @@
+# frozen_string_literal: true
+
+require "concurrent/edge/erlang_actor"
+
+require_relative "../changeset"
+require_relative "../read_only_store"
+
+module ConcurrentPipeline
+  module Processors
+    class ActorProcessor
+      Message = Struct.new(:type, :payload) do
+        def to_s(...)
+          inspect(...)
+        end
+        def inspect(...)
+          "<Message #{type} (#{payload.class}) >"
+        end
+      end
+
+      Ms = Module.new do
+        def self.g(...)
+          Message.new(...)
+        end
+
+        def msg(...)
+          Message.new(...)
+        end
+      end
+
+      Msg = Message.method(:new) #->(*args, **opts) { Message.new(*args, **opts) }
+
+      def self.call(...)
+        new(...).call
+      end
+
+      attr_reader :store, :pipelineables, :registry, :stream
+      def initialize(store:, pipelineables:, registry:, stream:)
+        @store = store
+        @pipelineables = pipelineables
+        @registry = registry
+        @stream = stream
+      end
+
+      module PipeActor
+        module InstanceMethods
+          attr_accessor :ctx
+          def respond(msg)
+            if self.class.on_blocks.key?(msg.type)
+              instance_exec(msg, &self.class.on_blocks[msg.type])
+            else
+              instance_exec(msg, &self.class.default_block)
+            end
+          rescue => e
+            Log.warn("error: #{e.class}:#{e.message}\n---\n#{e.backtrace.join("\n")}")
+            terminate :error
+          end
+
+          def reply(...)
+            ctx.reply(...)
+          end
+
+          def terminate(...)
+            ctx.terminate(...)
+          end
+        end
+
+        def self.extended(base)
+          base.include(InstanceMethods)
+        end
+
+        def spawn(...)
+          instance = self.new(...)
+          Concurrent::ErlangActor.spawn(type: :on_pool) do
+            receive(keep: true) do |msg|
+              instance.ctx = self
+              instance.respond(msg)
+            end
+          end
+        end
+
+        def on_blocks
+          @on_blocks ||= {}
+        end
+
+        def default_block
+          @default_block
+        end
+
+        private
+
+        def on(type, &block)
+          on_blocks[type] = block
+        end
+
+        def default(&block)
+          @default_block = block
+        end
+      end
+
+      class ActorPool
+        attr_reader :pool
+        def initialize(concurrency = 10000)
+          @pool = Pool.spawn(concurrency)
+        end
+
+        def process(bodies)
+          # This can be blocking because it is called by perform
+          # which is already being called by an actor, so blocking is ok.
+          # However, we still want the pool to be limited in size across
+          # all actors.
+          bodies.map { |body|
+            pool.ask(
+              Processors::ActorProcessor::Message.new(
+                :queue,
+                body
+              )
+            )
+          }.map { _1.terminated.value! }
+        end
+
+        class Pool
+          extend Processors::ActorProcessor::PipeActor
+          attr_reader :concurrency, :queue, :processing_count
+          def initialize(concurrency= 10000)
+            @concurrency = concurrency
+            @queue = []
+            @processing_count = 0
+          end
+
+          on :queue do |msg|
+            pid = spawn_queued_actor(msg.payload)
+            queue << pid
+            try_process
+            reply(pid)
+          end
+
+          on :finished do |msg|
+            @processing_count -= 1
+            try_process()
+          end
+
+          private
+
+          def try_process
+            return if queue.empty?
+            return if processing_count >= concurrency
+
+            @processing_count += 1
+            pid = queue.shift
+            pid.tell(ctx.pid)
+          end
+
+          def spawn_queued_actor(body)
+            Concurrent::ErlangActor.spawn(type: :on_pool) do
+              receive do |sender|
+                # begin
+                body.call
+
+                sender.tell(Processors::ActorProcessor::Message.new(:finished, nil))
+              end
+            end
+          end
+        end
+      end
+
+      class Changeset
+        extend PipeActor
+
+        attr_reader :dispatch, :pipelines, :store
+        def initialize(dispatch:, store:)
+          @dispatch = dispatch
+          @store = store
+          @pipelines = []
+        end
+
+        on :changeset do |msg|
+          pipelines << msg.payload
+        end
+
+        on :flush_queue do |msg|
+          next unless pipelines.any?
+
+          diffed = store.apply(pipelines.map(&:changeset))
+          if diffed
+            dispatch.tell(Msg.(:pipelines_updated, pipelines.map(&:id)))
+          else
+            dispatch.tell(Msg.(:pipelines_processed, pipelines.map(&:id)))
+          end
+
+          @pipelines = []
+        end
+      end
+
+      class Scheduler
+        extend PipeActor
+
+        attr_reader :dispatch, :status, :store, :pipelineables, :stream
+        def initialize(dispatch:, store:, pipelineables:, stream:)
+          @dispatch = dispatch
+          @store = store
+          @pipelineables = pipelineables
+          @stream = stream
+          @status = {}
+          @unlimited_pool = ActorPool.new
+          @pools = {}
+        end
+
+        def pool_for(pipelineable)
+          @pools[pipelineable] ||= (
+            if pipelineable.concurrency
+              ActorPool.new(pipelineable.concurrency)
+            else
+              @unlimited_pool
+            end
+          )
+        end
+
+        default do |msg|
+          # we update pipeline_ids on both messages.
+          pipeline_ids = msg.payload || []
+          pipeline_ids.each do |pipeline_id|
+            status[pipeline_id] = :processed
+          end
+
+          case msg.type
+          when :requeue
+            reader = store.reader
+
+            pipelineables
+              .map { _1.build_pipelines(store: reader, stream: stream, pool: pool_for(_1)) }
+              .flatten
+              .each do |c|
+                if status[c.id] != :queued && c.should_perform?
+                  Log.debug("enqueuing: #{c.id}")
+                  status[c.id] = :queued
+                  dispatch.tell(Msg.(:enqueue, c))
+                end
+              end
+          end
+
+          if status.values.all? { _1 == :processed }
+            dispatch.tell(Msg.(:all_pipelines_processed))
+          end
+          reply :ok
+        end
+      end
+
+      class Ticker
+        extend PipeActor
+
+        attr_reader :dispatch
+        def initialize(dispatch)
+          @dispatch = dispatch
+        end
+
+        on :start do |msg|
+          loop do
+            sleep 0.1
+            dispatch.tell(Msg.(:tick))
+          end
+        end
+      end
+
+      class Dispatch
+        extend PipeActor
+
+        attr_reader :work, :changeset, :scheduler
+
+        on :init do |msg|
+          @work = msg.payload[:work]
+          @changeset = msg.payload[:changeset]
+          @scheduler = msg.payload[:scheduler]
+          reply :ok
+        end
+
+        on :tick do |msg|
+          changeset.tell(Msg.(:flush_queue))
+        end
+
+        on :enqueue do |msg|
+          work.tell(Msg.(:process, msg.payload))
+        end
+
+        on :error do |msg|
+          warn "error: #{msg.payload.class}:#{msg.payload.message}\n---\n#{msg.payload.backtrace.join("\n")}"
+          terminate :error
+        end
+
+        on :all_pipelines_processed do |msg|
+          terminate :ok
+        end
+
+        on :pipelines_updated do |msg|
+          scheduler.tell(Msg.(:requeue, msg.payload))
+        end
+
+        on :pipelines_processed do |msg|
+          scheduler.tell(Msg.(:pipelines_processed, msg.payload))
+        end
+
+        on :changeset do |msg|
+          changeset.tell(Msg.(:changeset, msg.payload))
+        end
+
+        default do |msg|
+          Log.debug("unknown message: #{msg.inspect}")
+        end
+      end
+
+      class Work
+        extend PipeActor
+
+        attr_reader :dispatch
+        def initialize(dispatch)
+          @dispatch = dispatch
+        end
+
+        on :process do |msg|
+          a = Concurrent::ErlangActor.spawn(type: :on_pool) do
+            receive do |pipeline, dispatch|
+              pipeline = msg.payload
+              Log.debug("starting perform: #{pipeline.class}: #{pipeline.id}")
+              pipeline.perform
+              Log.debug("finished perform: #{pipeline.class}: #{pipeline.id}")
+              dispatch.tell(Msg.(:changeset, pipeline))
+            rescue => e
+              dispatch.tell(Msg.(:error, e))
+            end
+          end
+
+          a.tell([msg.payload, dispatch])
+        end
+      end
+
+      def call
+        dispatch = Dispatch.spawn
+        ticker = Ticker.spawn(dispatch)
+        work = Work.spawn(dispatch)
+        changeset = Changeset.spawn(dispatch: dispatch, store: store)
+        scheduler = Scheduler.spawn(
+          dispatch: dispatch,
+          store: store,
+          pipelineables: pipelineables,
+          stream: stream
+        )
+
+        dispatch.tell(Msg.(
+          :init,
+          work: work,
+          changeset: changeset,
+          scheduler: scheduler
+        ))
+
+        Log.debug("triggering initial queue")
+
+        ticker.tell(Msg.(:start))
+        scheduler.tell(Msg.(:requeue))
+
+        dispatch.terminated.result
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/producer.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
+require_relative "processors/actor_processor"
+require_relative "registry"
+require_relative "store"
+require_relative "stores/yaml"
+
+module ConcurrentPipeline
+  class Producer
+    class Stream
+      attr_reader :receivers
+      def initialize
+        @receivers = {
+          default: -> (type, *) { Log.warn("No stream handler for type: #{type.inspect}") },
+        }
+      end
+
+      def on(type, &block)
+        receivers[type] = block
+      end
+
+      def push(type, payload)
+        receivers[type]
+          .tap { Log.warn("No stream handler for type: #{type.inspect}") if _1.nil? }
+          &.call(payload)
+      end
+    end
+
+    def initialize(data: nil, store: nil, stream: nil, dir: nil, &initialization_block)
+      raise ArgumentError.new("provide data or store but not both") if data && store
+      raise ArgumentError.new("must provide initial data, a store, or a block") unless data || store || initialization_block
+      @dir = dir
+      @data = data
+      @store = store&.reader? ? store.store : store
+      @initialization_block = initialization_block
+      @stream = stream
+    end
+
+    def call(&block)
+      changeset = self.store.changeset
+      @initialization_block&.call(changeset)
+      store.apply(changeset)
+
+      Processors::ActorProcessor.call(
+        store: store,
+        pipelineables: pipelineables,
+        registry: registry,
+        stream: stream
+      )
+
+      store.reader.all(:PipelineStep).all?(&:success?)
+    end
+
+    def data
+      store.reader.to_h
+    end
+
+    def store
+      @store ||= self.class.store.build_writer(data: @data || {}, dir: dir, registry: registry)
+    end
+
+    def stream
+      @stream || self.class.stream
+    end
+
+    def versions
+      self.class.store.versions(dir: dir, registry: registry)
+    end
+
+    def history
+      self.class.store.history(dir: dir, registry: registry)
+    end
+
+    def dir
+      @dir ||= Dir.mktmpdir
+    end
+
+    private
+
+    def registry
+      self.class.registry
+    end
+
+    def pipelineables
+      self.class.pipelineables
+    end
+
+
+    class << self
+      def store(klass = nil)
+        @store = klass || Stores::Yaml
+        @store
+      end
+
+      def pipelineables
+        @pipelineables ||= []
+      end
+
+      def registry
+        @registry ||= (
+          Registry
+            .new
+            .tap { _1.register(:PipelineStep, Pipeline::PipelineStep) }
+        )
+      end
+
+      def model(klass_or_symbol, as: nil, &block)
+        if klass_or_symbol.is_a?(Class)
+          raise ArgumentError.new("Cannot provide both a class and a block") if block
+          as ||= klass_or_symbol.name.split("::").last.to_sym
+          registry.register(as, klass_or_symbol)
+        elsif klass_or_symbol.is_a?(Symbol)
+          registry.register(klass_or_symbol, Class.new do
+            extend Model
+            instance_eval(&block)
+          end)
+        else
+          raise ArgumentError.new("Must provide either a class or a symbol")
+        end
+      end
+
+      module CustomPipelines
+      end
+
+      def stream(&block)
+        return @stream unless block
+        @stream = Stream.new.tap { _1.instance_exec(&block) }
+      end
+
+      def pipeline(klass_or_symbol = nil, **opts, &block)
+        pipelineable = (
+          if klass_or_symbol.is_a?(Class)
+            raise ArgumentError.new("Cannot provide both a class and a block") if block
+            klass_or_symbol
+          elsif klass_or_symbol.is_a?(Symbol) || klass_or_symbol.nil?
+            klass_or_symbol ||= "Pipeline#{pipelineables.count}"
+            pipeline_class = Class.new(Pipeline, &block)
+            class_name = klass_or_symbol.to_s.split("_").collect(&:capitalize).join
+            CustomPipelines.const_set(class_name, pipeline_class)
+            pipeline_class
+          else
+            raise ArgumentError.new("Must provide either a class or a symbol")
+          end
+        )
+
+        opts.each do |meth, args|
+          pipelineable.public_send(meth, *Array(args))
+        end
+
+        pipelineables << pipelineable
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/read_only_store.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ConcurrentPipeline
+  class ReadyOnlyStore
+    attr_reader :store
+    def initialize(store)
+      @store = store
+    end
+
+    def find(...)
+      store.find(...)
+    end
+
+    def all(...)
+      store.all(...)
+    end
+
+    def everything(...)
+      store.everything(...)
+    end
+  end
+end

data/lib/concurrent_pipeline/registry.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module ConcurrentPipeline
+  class Registry
+    def build(type, attributes)
+      lookup.fetch(type).new(attributes)
+    end
+
+    def register(type, klass)
+      if lookup.key?(type)
+        raise <<~TXT
+          Duplicate type: #{type} for class #{klass.name}. Use the `as:` \
+          option to avoid this collision, eg `model(MyModel, as: :MyModel).
+        TXT
+      end
+
+      lookup[type] = klass
+    end
+
+    def type_for(type)
+      return type if lookup.key?(type)
+
+      reverse_lookup = lookup.invert
+
+      return reverse_lookup[type] if reverse_lookup.key?(type)
+
+      raise ArgumentError, "Unknown type: #{type}"
+    end
+
+    private
+
+    def lookup
+      @lookup ||= {}
+    end
+  end
+end

data/lib/concurrent_pipeline/shell.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module ConcurrentPipeline
+  class Shell
+    Error = Class.new(StandardError)
+    class << self
+      Result = Struct.new(:command, :success, :stdout, :stderr, keyword_init: true) do
+        def success?
+          success
+        end
+      end
+
+      def run!(...)
+        # only returns stdout just because.
+        run(...)
+          .tap { raise "command failed: \n#{_1.inspect}" unless _1.success? }
+          .stdout
+      end
+
+      def run(command)
+        Open3.popen3(command) do |_in, stdout, stderr, wait_thr|
+          process_stdout = []
+          stdout_thr = Thread.new do
+            while line = stdout.gets&.chomp
+              yield(:stdout, line) if block_given?
+              process_stdout << line
+            end
+          end
+
+          process_stderr = []
+          stderr_thr = Thread.new do
+            while line = stderr.gets&.chomp
+              yield(:stderr, line) if block_given?
+              process_stderr << line
+            end
+          end
+
+          [
+            stderr_thr,
+            stdout_thr,
+          ].each(&:join)
+
+          Result.new(
+            command: command,
+            success: wait_thr.value.success?,
+            stdout: process_stdout.join("\n"),
+            stderr: process_stderr.join("\n"),
+          )
+        end
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/store.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ConcurrentPipeline
+  class Store
+    attr_reader :db, :changeset
+    def initialize(db:, changeset:)
+      @changeset = changeset
+      @db = db
+    end
+
+    def find(...)
+      db.find(...)
+    end
+
+    def all(...)
+      db.find(...)
+    end
+
+    def create(...)
+      changeset.create(...)
+    end
+
+    def update(...)
+      changeset.update(...)
+    end
+  end
+end

data/lib/concurrent_pipeline/stores/versioned.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "yaml"
+
+module ConcurrentPipeline
+  module Stores
+    class Versioned
+      Version = Struct.new(:data, :registry, keyword_init: true)
+        include Yaml::QueryMethods
+      end
+
+      attr_reader :data, :registry
+      def initialize(data:, registry:)
+        @data = data
+        @registry = registry
+      end
+
+      def versions
+
+      end
+    end
+  end
+end