concurrent_pipeline 0.1.0 → 1.0.0

data/Rakefile

@@ -6,11 +6,13 @@ require "rake/testtask"
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
   t.libs << "lib"
-  t.test_files = FileList["test/**/test_*.rb"]
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+  t.warning = false
 end
 
 require "rubocop/rake_task"
 
 RuboCop::RakeTask.new
 
-task default: %i[test rubocop]
+task default: %i[test]

data/concurrent_pipeline.gemspec

@@ -30,5 +30,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency("concurrent-ruby-edge")
+  spec.add_dependency("zeitwerk")
+  spec.add_dependency("yaml")
+  spec.add_dependency("async")
 end

data/lib/concurrent_pipeline/pipeline.rb

@@ -1,214 +1,27 @@
-require "time"
+require "securerandom"
 
 module ConcurrentPipeline
   class Pipeline
+    def self.define(&block)
+      schema = Pipelines::Schema.new
+      schema.instance_exec(&block)
 
-    # {
-    #   type: PipelineStep,
-    #   pipeline_id: [MyPipeline, :vhost, 1],
-    #   name: {string},
-    #   result: nil | :success | :failure,
-    #   completed_at: nil | {timestamp},
-    #   sequence: 3
-    # }
-
-    class PipelineStep
-      extend Model
-
-      attribute :id
-      attribute :pipeline_id
-      attribute :name
-      attribute :result
-      attribute :completed_at
-      attribute :sequence
-      attribute :error_message
-
-      def success?
-        result == :success
-      end
-    end
-
-    class Wrapper
-      attr_reader :pipeline, :pool
-      def initialize(pipeline:, pool:)
-        @pipeline = pipeline
-        @pool = pool
-      end
-
-      def id
-        pipeline_id = (
-          if pipeline.class.target_type
-            pipeline.target.id
-          end
-        )
-
-        [pipeline.class.name, pipeline_id].compact.join("__")
-      end
-
-      def perform
-        if pipeline_steps.empty?
-          create_pipeline_steps
-        else
-          pipeline_steps
-            .reject(&:completed_at)
-            .group_by(&:sequence)
-            .values
-            .first
-            .map { |step|
-              wrapper = self
-              -> () do
-                begin
-                  wrapper.pipeline.public_send(step.name)
-                  wrapper.changeset.update(
-                    step,
-                    completed_at: Time.now.iso8601,
-                    result: :success
-                  )
-                rescue => e
-                  wrapper.changeset.update(
-                    step,
-                    completed_at: Time.now.iso8601,
-                    result: :failure,
-                    error: {class: e.class, message: e.message, backtrace: e.backtrace}
-                  )
-                end
-              end
-            }
-          .then { pool.process(_1) }
-        end
-      end
-
-      def should_perform?
-        ready? && !done?
-      end
-
-      def create_pipeline_steps
-        sequence = (
-          if pipeline.respond_to?(:steps)
-            pipeline.steps
-          else
-            [:perform]
-          end
-        )
-
-        sequence.each_with_index do |sub_seq, i|
-          Array(sub_seq).each do |step_name|
-            changeset.create(
-              PipelineStep,
-              pipeline_id: id,
-              name: step_name,
-              sequence: i
-            )
-          end
-        end
-      end
-
-      def pipeline_steps
-        @pipeline_steps ||= (
-          store
-            .all(PipelineStep)
-            .select { _1.pipeline_id == id }
-            .sort_by(&:sequence)
-        )
-      end
-
-      def ready?
-        if pipeline.respond_to?(:ready?)
-          pipeline.ready?
-        else
-          true
-        end
-      end
-
-      def done?
-        if pipeline.respond_to?(:done?)
-          pipeline.done?
-        else
-          !pipeline_steps.empty? && pipeline_steps.all?(&:completed_at)
-        end
-      end
-
-      def store
-        pipeline.store
-      end
-
-      def changeset
-        pipeline.changeset
-      end
-
-      def stream(type, payload)
-        pipeline.stream.push(type, payload)
-      end
+      new(schema)
     end
 
-    class << self
-      attr_reader(:target_type)
-
-      def build_pipelines(store:, stream:, pool:)
-        if target_type
-          store.all(target_type).map { |record|
-            Wrapper.new(
-              pipeline: new(
-                target: record,
-                store: store,
-                changeset: store.changeset,
-                stream: stream
-              ),
-              pool: pool
-            )
-          }
-        else
-          Wrapper.new(
-            pipeline: new(
-              target: nil,
-              store: store,
-              changeset: store.changeset,
-              stream: stream
-            ),
-            pool: pool
-          )
-        end
-      end
-
-      def each(type, as: nil)
-        @target_type = type
-        define_method(as) { target } if as
-        define_method(:record) { target }
-      end
-
-      def ready(...)
-        define_method(:ready?, ...)
-      end
-
-      def done(...)
-        define_method(:done?, ...)
-      end
-
-      def perform(...)
-        steps(:perform)
-        define_method(:perform, ...)
-      end
-
-      def steps(*sequence)
-        define_method(:steps) { sequence }
-      end
-
-      def concurrency(size = nil)
-        @concurrency = size if size
-        @concurrency
-      end
+    attr_reader :schema, :processor
+    def initialize(schema)
+      @schema = schema
+      @processor = nil
     end
 
-    attr_reader :target, :store, :changeset
-    def initialize(target:, store:, changeset:, stream:)
-      @target = target
-      @store = store
-      @changeset = changeset
-      @stream = stream
+    def process(store)
+      @processor = schema.build_processor(store)
+      @processor.call
     end
 
-    def stream(type, payload)
-      @stream.push(type, payload)
+    def errors
+      @processor&.errors || []
     end
   end
 end

data/lib/concurrent_pipeline/pipelines/processors/asynchronous.rb

@@ -0,0 +1,92 @@
+require "async"
+require "async/semaphore"
+
+module ConcurrentPipeline
+  module Pipelines
+    module Processors
+      class Asynchronous
+        def self.call(...)
+          new(...).call
+        end
+
+        attr_reader(:store, :producers, :locker, :concurrency, :enqueue_seconds, :errors)
+        def initialize(store:, producers:, concurrency: 5, enqueue_seconds: 0.1)
+          @store = store
+          @producers = producers
+          @concurrency = concurrency
+          @enqueue_seconds = enqueue_seconds
+          @locker = Locker.new
+          @errors = []
+        end
+
+        def call
+          Async { |task|
+            semaphore = Async::Semaphore.new(concurrency)
+            active_tasks = []
+            result = true
+
+            loop do
+              # Set result to false if any task has failed
+              if errors.any?
+                result = false
+                break
+              end
+
+              # Clean up finished tasks
+              active_tasks.reject!(&:finished?)
+
+              # Try to enqueue more work (only if no failure)
+              enqueued_any = enqueue_all(semaphore, active_tasks, task)
+
+              # Stop when nothing is being processed AND nothing new was enqueued
+              break if active_tasks.empty? && !enqueued_any
+
+              # Yield to allow other tasks to progress
+              sleep(enqueue_seconds)
+            end
+
+            result  # Return false if there was a failure, true otherwise
+          }.wait  # Wait for the async block to complete and return its value
+        end
+
+        def enqueue_all(semaphore, active_tasks, parent_task)
+          enqueued_any = false
+
+          producers.each do |producer|
+            producer.records(store).each do |record|
+              next if locker.locked?(producer:, record:)
+
+              enqueued_any = true
+              locker.lock(producer:, record:)
+
+              # Spawn async task
+              new_task = parent_task.async do
+                begin
+                  # Exit early if another task has already failed
+                  next if errors.any?
+
+                  semaphore.acquire do
+                    begin
+                      store.transaction do
+                        producer.call(record)
+                      end
+                    rescue => e
+                      # Append error to array to prevent async gem from logging it
+                      errors << e
+                    end
+                  end
+                ensure
+                  locker.unlock(producer:, record:)
+                end
+              end
+
+              active_tasks << new_task
+            end
+          end
+
+          enqueued_any
+        end
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/pipelines/processors/locker.rb

@@ -0,0 +1,28 @@
+require "securerandom"
+require "async"
+require "async/semaphore"
+
+module ConcurrentPipeline
+  module Pipelines
+    module Processors
+      class Locker
+        attr_reader(:locks)
+        def initialize
+          @locks = {}
+        end
+
+        def locked?(producer:, record:)
+          locks.key?([producer, record.class.record_name, record.id])
+        end
+
+        def lock(producer:, record:)
+          locks[[producer, record.class.record_name, record.id]] = true
+        end
+
+        def unlock(producer:, record:)
+          locks.delete([producer, record.class.record_name, record.id])
+        end
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/pipelines/processors/synchronous.rb

@@ -0,0 +1,50 @@
+module ConcurrentPipeline
+  module Pipelines
+    module Processors
+      class Synchronous
+        def self.call(...)
+          new(...).call
+        end
+
+        attr_reader(:store, :producers, :locker, :errors)
+        def initialize(store:, producers:)
+          @store = store
+          @producers = producers
+          @locker = Locker.new
+          @errors = []
+        end
+
+        def call
+          while(enqueue_all) do end
+          errors.empty?
+        end
+
+        def enqueue_all
+          enqueued_any = false
+
+          producers.each do |producer|
+            producer.records(store).each do |record|
+              next if locker.locked?(producer:, record:)
+
+              enqueued_any = true
+              locker.lock(producer:, record:)
+
+              begin
+                store.transaction do
+                  producer.call(record)
+                end
+              rescue => e
+                errors << e
+                return false
+              ensure
+                locker.unlock(producer:, record:)
+              end
+            end
+          end
+
+          enqueued_any
+        end
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/pipelines/schema.rb

@@ -0,0 +1,56 @@
+require "securerandom"
+
+module ConcurrentPipeline
+  module Pipelines
+    class Schema
+      PROCESSORS = {
+        sync: Processors::Synchronous,
+        async: Processors::Asynchronous,
+      }
+
+      Producer = Struct.new(:query, :block) do
+        def call(*a, **p)
+          instance_exec(*a, **p, &block)
+        end
+
+        def shell
+          Shell
+        end
+
+        def records(store)
+          if query.is_a?(Proc)
+            query.call
+          else
+            # Query is a hash with record_name and filters
+            store.where(query[:record_name], **query[:filters])
+          end
+        end
+      end
+
+      def producers
+        @producers ||= []
+      end
+
+      def processor(type, **attrs)
+        @processor = {type:, attrs:}
+      end
+
+      def build_processor(store)
+        PROCESSORS
+          .fetch(@processor.fetch(:type))
+          .new(store:, producers:, **@processor.fetch(:attrs))
+      end
+
+      def process(query_or_record_name, **filters, &block)
+        if query_or_record_name.is_a?(Proc)
+          # Lambda-based query (current behavior)
+          producers << Producer.new(query: query_or_record_name, block:)
+        else
+          # Record name with filters
+          query = { record_name: query_or_record_name, filters: }
+          producers << Producer.new(query:, block:)
+        end
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/store.rb

@@ -1,27 +1,102 @@
-# frozen_string_literal: true
+require "securerandom"
 
 module ConcurrentPipeline
   class Store
-    attr_reader :db, :changeset
-    def initialize(db:, changeset:)
-      @changeset = changeset
-      @db = db
+    def self.define(&block)
+      schema = Stores::Schema.new
+      schema.instance_exec(&block)
+
+      klass = Class.new(Store) do
+        define_method(:schema) { schema }
+      end
+
+
+      klass.new(schema.storage)
+    end
+
+    attr_reader :storage
+    def initialize(storage)
+      @storage = storage
+    end
+
+    def transaction(&block)
+      ensure_writable
+
+      if storage.in_transaction?
+        raise "Nested transactions are not supported"
+      end
+
+      storage.transaction(&block)
+
+      nil
+    end
+
+    def create(record_name, **attrs)
+      ensure_writable
+
+      storage.create(
+        name: record_name,
+        attrs: { id: SecureRandom.uuid }.merge(attrs)
+      )
+
+      nil
+    end
+
+    def update(record, **attrs)
+      ensure_writable
+
+      # Create a temporary record to apply and validate setter methods
+      temp_record = record.class.new(record.attributes)
+
+      # Apply attributes using setter methods (will raise NoMethodError if attribute doesn't exist)
+      attrs.each do |key, value|
+        temp_record.public_send("#{key}=", value)
+      end
+
+      storage.update(
+        name: record.class.record_name,
+        id: record.id,
+        attrs: temp_record.attributes
+      )
+
+      nil
+    end
+
+    def all(record_name)
+      storage
+        .all(name: record_name)
+        .map { schema.build(record_name, attrs: _1) }
     end
 
-    def find(...)
-      db.find(...)
+    def where(record_name, **filters)
+      records = all(record_name)
+
+      return records if filters.empty?
+
+      records.select do |record|
+        filters.all? do |key, value|
+          attr_value = record.public_send(key)
+          if value.respond_to?(:call)
+            value.call(attr_value)
+          else
+            attr_value == value
+          end
+        end
+      end
     end
 
-    def all(...)
-      db.find(...)
+    def versions
+      storage.versions.map { self.class.new(_1) }
     end
 
-    def create(...)
-      changeset.create(...)
+    def restore
+      self.class.new(storage.restore)
     end
 
-    def update(...)
-      changeset.update(...)
+    def ensure_writable
+      unless storage.writeable?
+        raise "Unwritable storage: Must 'restore' it before you can write to it"
+      end
     end
   end
 end

data/lib/concurrent_pipeline/stores/schema/record.rb

@@ -0,0 +1,47 @@
+module ConcurrentPipeline
+  module Stores
+    class Schema
+      class Record
+        class << self
+          def attribute(name, **options)
+            attributes << name
+            attribute_defaults[name] = options[:default] if options.key?(:default)
+
+            define_method(name) do
+              attributes[name]
+            end
+
+            define_method("#{name}=") do |value|
+              @attributes[name] = value
+            end
+          end
+
+          def attributes
+            @attributes ||= []
+          end
+
+          def attribute_defaults
+            @attribute_defaults ||= {}
+          end
+
+          def inherited(mod)
+            mod.attribute(:id)
+          end
+        end
+
+        attr_reader :attributes
+        def initialize(attributes = {})
+          # Apply defaults for missing attributes
+          defaults = self.class.attribute_defaults
+          @attributes = self.class.attributes.each_with_object({}) do |attr_name, hash|
+            if attributes.key?(attr_name)
+              hash[attr_name] = attributes[attr_name]
+            elsif defaults.key?(attr_name)
+              hash[attr_name] = defaults[attr_name]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/stores/schema.rb

@@ -0,0 +1,35 @@
+module ConcurrentPipeline
+  module Stores
+    class Schema
+      STORAGE = {
+        yaml: Storage::Yaml
+      }
+
+      def build(name, attrs:)
+        records.fetch(name).new(attrs)
+      end
+
+      def storage(type = nil, **attrs)
+        @storage = STORAGE.fetch(type).new(**attrs) if type
+        @storage
+      end
+
+      def record(name, &)
+        records[name] = Class.new(Record) do
+          define_singleton_method(:name) { "PipelineRecord.#{name}" }
+          define_singleton_method(:record_name) { name }
+
+          class_exec(&)
+
+          define_method(:inspect) do
+            "#<#{self.class.name} #{attributes.inspect[0..100]}>"
+          end
+        end
+      end
+
+      def records
+        @records ||= {}
+      end
+    end
+  end
+end