concurrent_pipeline 1.0.0 → 2.0.0

data/concurrent_pipeline.gemspec

@@ -30,7 +30,9 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency("zeitwerk")
-  spec.add_dependency("yaml")
-  spec.add_dependency("async")
+  spec.add_dependency("zeitwerk", "~> 2.7")
+  spec.add_dependency("yaml", "~> 0.4")
+  spec.add_dependency("async", "~> 2.35")
+  spec.add_dependency("activerecord", "~> 8.0")
+  spec.add_dependency("sqlite3", "~> 2.4")
 end

data/lib/concurrent_pipeline/errors.rb

@@ -0,0 +1,6 @@
+module ConcurrentPipeline
+  module Errors
+    Base = Class.new(StandardError)
+    AssertionFailure = Class.new(Base)
+  end
+end

data/lib/concurrent_pipeline/pipeline.rb

@@ -6,22 +6,18 @@ module ConcurrentPipeline
       schema = Pipelines::Schema.new
       schema.instance_exec(&block)
 
-      new(schema)
+      Class.new(Pipeline) do
+        define_singleton_method(:process) { |store| new(schema).process(store) }
+      end
     end
 
-    attr_reader :schema, :processor
+    attr_reader :schema
     def initialize(schema)
       @schema = schema
-      @processor = nil
     end
 
     def process(store)
-      @processor = schema.build_processor(store)
-      @processor.call
-    end
-
-    def errors
-      @processor&.errors || []
+      schema.build_processor(store).call
     end
   end
 end

data/lib/concurrent_pipeline/pipelines/processors/asynchronous.rb

@@ -9,56 +9,110 @@ module ConcurrentPipeline
           new(...).call
         end
 
-        attr_reader(:store, :producers, :locker, :concurrency, :enqueue_seconds, :errors)
-        def initialize(store:, producers:, concurrency: 5, enqueue_seconds: 0.1)
+        attr_reader(
+          :store,
+          :producers,
+          :locker,
+          :concurrency,
+          :enqueue_seconds,
+          :errors,
+          :before_process_hooks,
+          :timers
+        )
+        def initialize(
+          store:,
+          producers:,
+          concurrency: 5,
+          enqueue_seconds: 0.1,
+          before_process_hooks: [],
+          timers: []
+        )
           @store = store
           @producers = producers
           @concurrency = concurrency
           @enqueue_seconds = enqueue_seconds
           @locker = Locker.new
           @errors = []
+          @before_process_hooks = before_process_hooks
+          @timers = timers
+          @completed = 0
+          @start_time = nil
         end
 
         def call
+          @start_time = Time.now
           Async { |task|
             semaphore = Async::Semaphore.new(concurrency)
             active_tasks = []
-            result = true
 
-            loop do
-              # Set result to false if any task has failed
-              if errors.any?
-                result = false
-                break
+            # Start timer tasks
+            timer_tasks = timers.map { |timer|
+              task.async do
+                loop do
+                  sleep(timer.interval)
+                  begin
+                    stats = build_stats
+                    timer.block.call(stats)
+                  rescue => e
+                    # Silently ignore timer errors to not break the pipeline
+                  end
+                end
               end
+            }
+
+            begin
+              loop do
+                if errors.any?
+                  break
+                end
 
-              # Clean up finished tasks
-              active_tasks.reject!(&:finished?)
+                # Clean up finished tasks
+                active_tasks.reject!(&:finished?)
 
-              # Try to enqueue more work (only if no failure)
-              enqueued_any = enqueue_all(semaphore, active_tasks, task)
+                # Try to enqueue more work (only if no failure)
+                enqueued_any = enqueue_all(semaphore, active_tasks, task)
 
-              # Stop when nothing is being processed AND nothing new was enqueued
-              break if active_tasks.empty? && !enqueued_any
+                # Stop when nothing is being processed AND nothing new was enqueued
+                break if active_tasks.empty? && !enqueued_any
 
-              # Yield to allow other tasks to progress
-              sleep(enqueue_seconds)
+                # Yield to allow other tasks to progress
+                sleep(enqueue_seconds)
+              end
+            ensure
+              # Stop all timer tasks
+              timer_tasks.each(&:stop)
             end
 
-            result  # Return false if there was a failure, true otherwise
+            Result.new(errors:)
           }.wait  # Wait for the async block to complete and return its value
         end
 
+        def queue_size
+          locker.locks.size
+        end
+
+        private
+
+        def build_stats
+          Pipelines::Schema::Stats.new(
+            queue_size: queue_size,
+            completed: @completed,
+            time: Time.now - @start_time
+          )
+        end
+
         def enqueue_all(semaphore, active_tasks, parent_task)
           enqueued_any = false
 
           producers.each do |producer|
-            producer.records(store).each do |record|
+            records = producer.records(store)
+            records.each_with_index do |record, index|
               next if locker.locked?(producer:, record:)
 
               enqueued_any = true
               locker.lock(producer:, record:)
 
+
               # Spawn async task
               new_task = parent_task.async do
                 begin
@@ -67,9 +121,18 @@ module ConcurrentPipeline
 
                   semaphore.acquire do
                     begin
+
+                      before_process_hooks.each do |hook|
+                        Pipelines::Schema::Step.new(
+                          value: record,
+                          label: producer.label
+                        ).then { hook.call(_1) }
+                      end
+
                       store.transaction do
                         producer.call(record)
                       end
+                      @completed += 1
                     rescue => e
                       # Append error to array to prevent async gem from logging it
                       errors << e

data/lib/concurrent_pipeline/pipelines/processors/locker.rb

@@ -12,15 +12,15 @@ module ConcurrentPipeline
         end
 
         def locked?(producer:, record:)
-          locks.key?([producer, record.class.record_name, record.id])
+          locks.key?([producer, record.class.name, record.id])
         end
 
         def lock(producer:, record:)
-          locks[[producer, record.class.record_name, record.id]] = true
+          locks[[producer, record.class.name, record.id]] = true
         end
 
         def unlock(producer:, record:)
-          locks.delete([producer, record.class.record_name, record.id])
+          locks.delete([producer, record.class.name, record.id])
         end
       end
     end

data/lib/concurrent_pipeline/pipelines/processors/result.rb

@@ -0,0 +1,11 @@
+module ConcurrentPipeline
+  module Pipelines
+    module Processors
+      Result = Data.define(:errors) do
+        def success?
+          errors.empty?
+        end
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/pipelines/processors/synchronous.rb

@@ -1,3 +1,5 @@
+require "async"
+
 module ConcurrentPipeline
   module Pipelines
     module Processors
@@ -6,33 +8,100 @@ module ConcurrentPipeline
           new(...).call
         end
 
-        attr_reader(:store, :producers, :locker, :errors)
-        def initialize(store:, producers:)
+        attr_reader(
+          :store,
+          :producers,
+          :locker,
+          :errors,
+          :before_process_hooks,
+          :timers
+        )
+        def initialize(
+          store:,
+          producers:,
+          before_process_hooks: [],
+          timers: []
+        )
           @store = store
           @producers = producers
           @locker = Locker.new
           @errors = []
+          @before_process_hooks = before_process_hooks
+          @timers = timers
+          @completed = 0
+          @start_time = nil
         end
 
         def call
-          while(enqueue_all) do end
-          errors.empty?
+          @start_time = Time.now
+          Async { |task|
+            # Start timer tasks
+            timer_tasks = timers.map { |timer|
+              task.async do
+                loop do
+                  sleep(timer.interval)
+                  begin
+                    stats = build_stats
+                    timer.block.call(stats)
+                  rescue => e
+                    # Silently ignore timer errors to not break the pipeline
+                  end
+                end
+              end
+            }
+
+            begin
+              while(enqueue_all) do end
+              Result.new(errors:)
+            ensure
+              timer_tasks.each(&:stop)
+            end
+          }.wait
+        end
+
+        def queue_size
+          # Cannot use locker.locks because don't enqueue them all, just
+          # process records one-by-one.
+          producers.sum do |producer|
+            producer.records(store).count do |record|
+              !locker.locked?(producer:, record:)
+            end
+          end
+        end
+
+        private
+
+        def build_stats
+          Pipelines::Schema::Stats.new(
+            queue_size: queue_size,
+            completed: @completed,
+            time: Time.now - @start_time
+          )
         end
 
         def enqueue_all
           enqueued_any = false
 
           producers.each do |producer|
-            producer.records(store).each do |record|
+            records = producer.records(store)
+            records.each_with_index do |record, index|
               next if locker.locked?(producer:, record:)
 
               enqueued_any = true
               locker.lock(producer:, record:)
 
               begin
+                before_process_hooks.each do |hook|
+                  Pipelines::Schema::Step.new(
+                    value: record,
+                    label: producer.label
+                  ).then { hook.call(_1) }
+                end
+
                 store.transaction do
                   producer.call(record)
                 end
+                @completed += 1
               rescue => e
                 errors << e
                 return false

data/lib/concurrent_pipeline/pipelines/schema.rb

@@ -8,11 +8,19 @@ module ConcurrentPipeline
         async: Processors::Asynchronous,
       }
 
-      Producer = Struct.new(:query, :block) do
+      Step = Struct.new(:value, :queue_size, :label, keyword_init: true)
+      Timer = Data.define(:interval, :block)
+      Stats = Struct.new(:queue_size, :completed, :time, keyword_init: true)
+
+      Producer = Struct.new(:query, :block, :label, keyword_init: true) do
         def call(*a, **p)
           instance_exec(*a, **p, &block)
         end
 
+        def assert(val)
+          raise Errors::AssertionFailure.new("Post condition failed") unless val
+        end
+
         def shell
           Shell
         end
@@ -20,9 +28,10 @@ module ConcurrentPipeline
         def records(store)
           if query.is_a?(Proc)
             query.call
+          elsif query.is_a?(ActiveRecord::Relation)
+            query.reload.to_a
           else
-            # Query is a hash with record_name and filters
-            store.where(query[:record_name], **query[:filters])
+            raise "Invalid processor query type: #{query.inspect}"
           end
         end
       end
@@ -31,6 +40,18 @@ module ConcurrentPipeline
         @producers ||= []
       end
 
+      def arounds
+        @arounds ||= []
+      end
+
+      def timers
+        @timers ||= []
+      end
+
+      def before_process_hooks
+        @before_process_hooks ||= []
+      end
+
       def processor(type, **attrs)
         @processor = {type:, attrs:}
       end
@@ -38,18 +59,23 @@ module ConcurrentPipeline
       def build_processor(store)
         PROCESSORS
           .fetch(@processor.fetch(:type))
-          .new(store:, producers:, **@processor.fetch(:attrs))
+          .new(store:, producers:, before_process_hooks:, timers:, **@processor.fetch(:attrs))
       end
 
-      def process(query_or_record_name, **filters, &block)
-        if query_or_record_name.is_a?(Proc)
-          # Lambda-based query (current behavior)
-          producers << Producer.new(query: query_or_record_name, block:)
-        else
-          # Record name with filters
-          query = { record_name: query_or_record_name, filters: }
-          producers << Producer.new(query:, block:)
-        end
+      def process(query, label: nil, &block)
+        producers << Producer.new(query:, block:, label:)
+      end
+
+      def around(&block)
+        arounds << block
+      end
+
+      def before_process(&block)
+        before_process_hooks << block
+      end
+
+      def timer(interval, &block)
+        timers << Timer.new(interval:, block:)
       end
     end
   end

data/lib/concurrent_pipeline/store.rb

@@ -1,4 +1,4 @@
-require "securerandom"
+require "active_record"
 
 module ConcurrentPipeline
   class Store
@@ -10,93 +10,196 @@ module ConcurrentPipeline
         define_method(:schema) { schema }
       end
 
+      schema.records.each do |name, spec|
+        define_method(name) { class_for(spec) }
+      end
 
-      klass.new(schema.storage)
+      klass.new(schema:, version: :root)
     end
 
-    attr_reader :storage
-    def initialize(storage)
-      @storage = storage
+    attr_reader :schema, :version
+    def initialize(schema:, version:)
+      @schema = schema
+      @version = version
+      @klasses = {}
+
+      # eagerly construct classes
+      schema.records.each_value { class_for(_1) }
     end
 
-    def transaction(&block)
-      ensure_writable
+    def transaction(&)
+      base_class.transaction(&)
+    end
+
+    def versions
+      return [self] unless root?
+
+      version_files = Dir.glob(File.join(schema.dir, "versions", "*.sqlite3")).sort
+      version_files.map do |file|
+        version_num = File.basename(file, ".sqlite3").to_i
+        Store.new(schema: schema, version: version_num)
+      end
+    end
 
-      if storage.in_transaction?
-        raise "Nested transactions are not supported"
+    def restore
+      raise "Can only restore from a version snapshot, not from root" if root?
+
+      # Copy the version database to the main database
+      main_db = File.join(schema.dir, "db.sqlite3")
+      FileUtils.cp(db_path, main_db)
+
+      # Renumber versions: keep all versions up to and including this one,
+      # then create a new version snapshot for the restore
+      version_files = Dir.glob(File.join(schema.dir, "versions", "*.sqlite3")).sort
+      version_files.each do |file|
+        file_version = File.basename(file, ".sqlite3").to_i
+        if file_version > version
+          FileUtils.rm(file)
+        end
       end
 
-      storage.transaction(&block)
+      # Create a new version snapshot of the restored state
+      new_version_num = version + 1
+      version_file = File.join(schema.dir, "versions", "#{new_version_num}.sqlite3")
+      FileUtils.cp(main_db, version_file)
 
-      nil
+      # Return a new root store for the restored state
+      Store.new(schema: schema, version: :root)
     end
 
-    def create(record_name, **attrs)
-      ensure_writable
+    private
 
-      storage.create(
-        name: record_name,
-        attrs: { id: SecureRandom.uuid }.merge(attrs)
-      )
+    def schema_root_mod_name
+      @schema_root_mod_name ||+ "SchemaRoot_#{schema.object_id}_#{object_id}"
+    end
 
-      nil
+    def schema_root_mod
+      @schema_root_mod ||= (
+        if Store.const_defined?(schema_root_mod_name)
+          Store.const_get(schema_root_mod_name)
+        else
+          Module.new.tap { Store.const_set(schema_root_mod_name, _1) }
+        end
+      )
     end
 
-    def update(record, **attrs)
-      ensure_writable
+    def base_class
+      @base_class ||= (
+        me = self
+        klass = Class.new(ActiveRecord::Base) do
+          define_singleton_method(:transaction_mutex) do
+            @transaction_mutex ||= Mutex.new
+          end
 
-      # Create a temporary record to apply and validate setter methods
-      temp_record = record.class.new(record.attributes)
+          define_singleton_method(:class_name) do |name|
+            "#{me.send(:schema_root_mod)}::Record_#{name}"
+          end
 
-      # Apply attributes using setter methods (will raise NoMethodError if attribute doesn't exist)
-      attrs.each do |key, value|
-        temp_record.public_send("#{key}=", value)
-      end
+          define_singleton_method(:transaction) do |**options, &block|
+            # If already in a transaction, just call super without creating a backup
+            return super(**options, &block) if connection.transaction_open?
 
-      storage.update(
-        name: record.class.record_name,
-        id: record.id,
-        attrs: temp_record.attributes
-      )
+            transaction_mutex.synchronize do
+              result = super(**options, &block)
 
-      nil
-    end
+              timestamp = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)
+              backup_path = File.join(me.schema.dir, "versions/#{timestamp}.sqlite3")
+              FileUtils.mkdir_p(File.dirname(backup_path))
+              FileUtils.cp(me.send(:db_path), backup_path)
 
-    def all(record_name)
-      storage
-        .all(name: record_name)
-        .map { schema.build(record_name, attrs: _1) }
-    end
+              result
+            end
+          end
+        end
+
+        # ActiveRecord doesn't let this be anonymous?
+        schema_root_mod.const_set(
+          "Base",
+          klass
+        )
+
+        klass.abstract_class = true
+        klass.establish_connection(
+          adapter: 'sqlite3',
+          database: db_path
+        )
+
+        # Configure SQLite to use DELETE journal mode instead of WAL
+        # This avoids creating -wal and -shm files
+        klass.connection.execute("PRAGMA journal_mode=DELETE")
+        klass.connection.execute("PRAGMA locking_mode=NORMAL")
+
+        # Only run migrations for the root store, not for version snapshots
+        if root? && schema.migrations.any?
+          # Ensure schema_migrations table exists
+          unless klass.connection.table_exists?(:schema_migrations)
+            klass.connection.create_table(:schema_migrations, id: false) do |t|
+              t.string :version, null: false
+            end
+            klass.connection.add_index(:schema_migrations, :version, unique: true)
+          end
 
-    def where(record_name, **filters)
-      records = all(record_name)
+          # Run any migrations that haven't been run yet
+          schema.migrations.each do |migration|
+            version = migration.version.to_s
 
-      return records if filters.empty?
+            # Check if migration has already been run using quote method for SQL safety
+            existing = klass.connection.select_value(
+              "SELECT 1 FROM schema_migrations WHERE version = #{klass.connection.quote(version)} LIMIT 1"
+            )
+            next if existing
 
-      records.select do |record|
-        filters.all? do |key, value|
-          attr_value = record.public_send(key)
-          if value.respond_to?(:call)
-            value.call(attr_value)
-          else
-            attr_value == value
+            # Run the migration
+            klass.connection.instance_exec(&migration.block)
+
+            # Record that this migration has been run
+            klass.connection.execute(
+              "INSERT INTO schema_migrations (version) VALUES (#{klass.connection.quote(version)})"
+            )
           end
         end
-      end
+
+        klass
+      )
     end
 
-    def versions
-      storage.versions.map { self.class.new(_1) }
+    def class_for(spec)
+      @klasses[spec.name] ||= (
+        store = self
+        Class.new(base_class) do
+          self.table_name = spec.table
+
+          # Execute the block if present, but define a schema method to ignore schema calls
+          if spec.block
+            # Define a no-op schema method to prevent errors when the block tries to call it
+            define_singleton_method(:schema) { |*args, &blk| }
+            class_exec(&spec.block)
+            # Remove the schema method after execution
+            singleton_class.remove_method(:schema) rescue nil
+          end
+
+          # Make all records readonly if this is a version snapshot
+          if not store.send(:root?)
+            define_method(:readonly?) { true }
+          end
+        end
+      ).tap {
+        schema_root_mod.const_set("Record_#{spec.name}", _1)
+      }
     end
 
-    def restore
-      self.class.new(storage.restore)
+    def db_path
+      @db_path ||= (
+        if root?
+          File.join(schema.dir, "db.sqlite3")
+        else
+          File.join(schema.dir, "versions", "#{version}.sqlite3")
+        end
+      )
     end
 
-    def ensure_writable
-      unless storage.writeable?
-        raise "Unwritable storage: Must 'restore' it before you can write to it"
-      end
+    def root?
+      version == :root
     end
   end
 end