concurrent_pipeline 0.1.0

data/lib/concurrent_pipeline/changeset.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module ConcurrentPipeline
+  class Changeset
+    Result = Struct.new(:diff) do
+      alias diff? diff
+    end
+
+    InitialDelta = Struct.new(:data, :dup, keyword_init: true) do
+      def apply(store)
+        # We fully dup the data to avoid mutating the input
+        dup_data = YAML.unsafe_load(data.to_yaml)
+        store.set(dup_data)
+        Result.new(true)
+      end
+
+      def self.from_json(json)
+        new(data: json.fetch(:delta), dup: true)
+      end
+
+      def as_json(...)
+        {
+          action: :initial,
+          delta: data
+        }
+      end
+    end
+
+    CreateDelta = Struct.new(:type, :attributes, keyword_init: true) do
+      def apply(store)
+        store.create(type: type, attributes: attributes)
+        Result.new(true)
+      end
+
+      def self.from_json(json)
+        new(
+          type: json.fetch(:type),
+          attributes: json.fetch(:attributes)
+        )
+      end
+
+      def as_json(...)
+        {
+          action: :create,
+          type: type,
+          attributes: attributes
+        }
+      end
+    end
+
+    UpdateDelta = Struct.new(:id, :type, :delta, keyword_init: true) do
+      def apply(store)
+        current_model = store.find(type, id)
+
+        # Todo: detect if changed underfoot
+
+        Result.new(
+          store.update(
+            id: id,
+            type: type,
+            attributes: current_model.attributes.merge(delta)
+          )
+        )
+      end
+
+      def self.from_json(json)
+        new(
+          id: json.fetch(:id),
+          type: json.fetch(:type),
+          delta: json.fetch(:delta),
+        )
+      end
+
+      def as_json
+        {
+          action: :update,
+          id: id,
+          type: type,
+          delta: delta
+        }
+      end
+    end
+
+    def self.from_json(registry:, json:)
+      type_map = {
+        initial: InitialDelta,
+        create: CreateDelta,
+        update: UpdateDelta,
+      }
+
+      new(
+        registry: registry
+      ).tap do |changeset|
+        json.fetch(:changes).each do |change|
+          type_map
+            .fetch(change.fetch(:action))
+            .from_json(change)
+            .then { changeset.deltas << _1 }
+        end
+      end
+    end
+
+    attr_reader :deltas, :registry
+    def initialize(registry:)
+      @registry = registry
+      @deltas = []
+    end
+
+    def deltas?
+      !@deltas.empty?
+    end
+
+    def create(type, attributes)
+      with_id = { id: SecureRandom.uuid }.merge(attributes)
+      @deltas << CreateDelta.new(type: type, attributes: with_id)
+    end
+
+    def update(model, delta)
+      type = registry.type_for(model.class)
+      @deltas << UpdateDelta.new(id: model.id, type: type, delta: delta)
+    end
+
+    def apply(...)
+      deltas.map { _1.apply(...) }
+    end
+
+    def as_json(...)
+      {
+        changes: deltas.map { _1.as_json(...) }
+      }
+    end
+  end
+end

data/lib/concurrent_pipeline/model.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module ConcurrentPipeline
+  module Model
+    module InstanceMethods
+      attr_reader :attributes
+
+      def initialize(attributes)
+        @attributes = attributes
+      end
+    end
+
+    def self.extended(base)
+      base.include(InstanceMethods)
+    end
+
+    def inherited(base)
+      base.instance_variable_set(:@attributes, attributes.dup)
+    end
+
+    def attributes
+      @attributes ||= {}
+    end
+
+    def attribute(name, **opts)
+      attributes[name] = opts
+
+      define_method(name) { attributes[name] }
+    end
+  end
+end

data/lib/concurrent_pipeline/pipeline.rb

@@ -0,0 +1,214 @@
+require "time"
+
+module ConcurrentPipeline
+  class Pipeline
+
+    # {
+    #   type: PipelineStep,
+    #   pipeline_id: [MyPipeline, :vhost, 1],
+    #   name: {string},
+    #   result: nil | :success | :failure,
+    #   completed_at: nil | {timestamp},
+    #   sequence: 3
+    # }
+
+    class PipelineStep
+      extend Model
+
+      attribute :id
+      attribute :pipeline_id
+      attribute :name
+      attribute :result
+      attribute :completed_at
+      attribute :sequence
+      attribute :error_message
+
+      def success?
+        result == :success
+      end
+    end
+
+    class Wrapper
+      attr_reader :pipeline, :pool
+      def initialize(pipeline:, pool:)
+        @pipeline = pipeline
+        @pool = pool
+      end
+
+      def id
+        pipeline_id = (
+          if pipeline.class.target_type
+            pipeline.target.id
+          end
+        )
+
+        [pipeline.class.name, pipeline_id].compact.join("__")
+      end
+
+      def perform
+        if pipeline_steps.empty?
+          create_pipeline_steps
+        else
+          pipeline_steps
+            .reject(&:completed_at)
+            .group_by(&:sequence)
+            .values
+            .first
+            .map { |step|
+              wrapper = self
+              -> () do
+                begin
+                  wrapper.pipeline.public_send(step.name)
+                  wrapper.changeset.update(
+                    step,
+                    completed_at: Time.now.iso8601,
+                    result: :success
+                  )
+                rescue => e
+                  wrapper.changeset.update(
+                    step,
+                    completed_at: Time.now.iso8601,
+                    result: :failure,
+                    error: {class: e.class, message: e.message, backtrace: e.backtrace}
+                  )
+                end
+              end
+            }
+          .then { pool.process(_1) }
+        end
+      end
+
+      def should_perform?
+        ready? && !done?
+      end
+
+      def create_pipeline_steps
+        sequence = (
+          if pipeline.respond_to?(:steps)
+            pipeline.steps
+          else
+            [:perform]
+          end
+        )
+
+        sequence.each_with_index do |sub_seq, i|
+          Array(sub_seq).each do |step_name|
+            changeset.create(
+              PipelineStep,
+              pipeline_id: id,
+              name: step_name,
+              sequence: i
+            )
+          end
+        end
+      end
+
+      def pipeline_steps
+        @pipeline_steps ||= (
+          store
+            .all(PipelineStep)
+            .select { _1.pipeline_id == id }
+            .sort_by(&:sequence)
+        )
+      end
+
+      def ready?
+        if pipeline.respond_to?(:ready?)
+          pipeline.ready?
+        else
+          true
+        end
+      end
+
+      def done?
+        if pipeline.respond_to?(:done?)
+          pipeline.done?
+        else
+          !pipeline_steps.empty? && pipeline_steps.all?(&:completed_at)
+        end
+      end
+
+      def store
+        pipeline.store
+      end
+
+      def changeset
+        pipeline.changeset
+      end
+
+      def stream(type, payload)
+        pipeline.stream.push(type, payload)
+      end
+    end
+
+    class << self
+      attr_reader(:target_type)
+
+      def build_pipelines(store:, stream:, pool:)
+        if target_type
+          store.all(target_type).map { |record|
+            Wrapper.new(
+              pipeline: new(
+                target: record,
+                store: store,
+                changeset: store.changeset,
+                stream: stream
+              ),
+              pool: pool
+            )
+          }
+        else
+          Wrapper.new(
+            pipeline: new(
+              target: nil,
+              store: store,
+              changeset: store.changeset,
+              stream: stream
+            ),
+            pool: pool
+          )
+        end
+      end
+
+      def each(type, as: nil)
+        @target_type = type
+        define_method(as) { target } if as
+        define_method(:record) { target }
+      end
+
+      def ready(...)
+        define_method(:ready?, ...)
+      end
+
+      def done(...)
+        define_method(:done?, ...)
+      end
+
+      def perform(...)
+        steps(:perform)
+        define_method(:perform, ...)
+      end
+
+      def steps(*sequence)
+        define_method(:steps) { sequence }
+      end
+
+      def concurrency(size = nil)
+        @concurrency = size if size
+        @concurrency
+      end
+    end
+
+    attr_reader :target, :store, :changeset
+    def initialize(target:, store:, changeset:, stream:)
+      @target = target
+      @store = store
+      @changeset = changeset
+      @stream = stream
+    end
+
+    def stream(type, payload)
+      @stream.push(type, payload)
+    end
+  end
+end