concurrent_pipeline 1.0.0 → 2.0.0

data/lib/concurrent_pipeline/stores/schema.rb

@@ -1,34 +1,72 @@
 module ConcurrentPipeline
   module Stores
     class Schema
-      STORAGE = {
-        yaml: Storage::Yaml
-      }
+      Record = Data.define(:name, :table, :block)
+      Migration = Data.define(:version, :block)
 
-      def build(name, attrs:)
-        records.fetch(name).new(attrs)
+      class RecordContext
+        attr_reader :schema_instance, :table_name
+
+        def initialize(schema_instance)
+          @schema_instance = schema_instance
+          @table_name = nil
+        end
+
+        def schema(table_name, &block)
+          # Store the table name for later use
+          @table_name = table_name
+
+          # Prepend schema migrations to front of the line
+          # Wrap the block in a create_table call
+          migration_block = Proc.new do
+            create_table(table_name, &block)
+          end
+
+          schema_instance.prepend_migration(table_name, &migration_block)
+        end
+
+        def method_missing(...)
+        end
+
+        def respond_to_missing?(...)
+          true
+        end
+      end
+
+      attr_reader :migrations, :records
+      def initialize
+        @migrations = []
+        @records = {}
+        @migration_counter = 1
       end
 
-      def storage(type = nil, **attrs)
-        @storage = STORAGE.fetch(type).new(**attrs) if type
-        @storage
+      def dir(path = nil)
+        @dir = path if path
+        @dir
       end
 
-      def record(name, &)
-        records[name] = Class.new(Record) do
-          define_singleton_method(:name) { "PipelineRecord.#{name}" }
-          define_singleton_method(:record_name) { name }
+      def migrate(version = @migration_counter += 1, &block)
+        migrations << Migration.new(version: version, block: block)
+      end
 
-          class_exec(&)
+      def prepend_migration(version, &block)
+        migrations.unshift(Migration.new(version: version, block: block))
+      end
 
-          define_method(:inspect) do
-            "#<#{self.class.name} #{attributes.inspect[0..100]}>"
-          end
+      def record(name, table: nil, &block)
+        # If block is given, execute it in RecordContext to extract schema calls
+        extracted_table = table
+        if block
+          context = RecordContext.new(self)
+          context.instance_exec(&block)
+          extracted_table ||= context.table_name
         end
-      end
 
-      def records
-        @records ||= {}
+        records[name] = Record.new(
+          name: name,
+          table: extracted_table,
+          block: block
+        )
       end
     end
   end

data/lib/concurrent_pipeline/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ConcurrentPipeline
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end

data/lib/concurrent_pipeline.rb

@@ -8,7 +8,7 @@ module ConcurrentPipeline
   class Error < StandardError; end
 
   class << self
-    def store(&)
+    def store(type = :yaml, &)
       Store.define(&)
     end
 

metadata

@@ -1,57 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: concurrent_pipeline
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Pete Kinnecom
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-12-13 00:00:00.000000000 Z
+date: 2026-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: yaml
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: async
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.35'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.35'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
 description:
 email:
 - git@k7u7.com
@@ -66,17 +94,16 @@ files:
 - concurrency.md
 - concurrent_pipeline.gemspec
 - lib/concurrent_pipeline.rb
+- lib/concurrent_pipeline/errors.rb
 - lib/concurrent_pipeline/pipeline.rb
 - lib/concurrent_pipeline/pipelines/processors/asynchronous.rb
 - lib/concurrent_pipeline/pipelines/processors/locker.rb
+- lib/concurrent_pipeline/pipelines/processors/result.rb
 - lib/concurrent_pipeline/pipelines/processors/synchronous.rb
 - lib/concurrent_pipeline/pipelines/schema.rb
 - lib/concurrent_pipeline/shell.rb
 - lib/concurrent_pipeline/store.rb
 - lib/concurrent_pipeline/stores/schema.rb
-- lib/concurrent_pipeline/stores/schema/record.rb
-- lib/concurrent_pipeline/stores/storage/yaml.rb
-- lib/concurrent_pipeline/stores/storage/yaml/fs.rb
 - lib/concurrent_pipeline/version.rb
 homepage: https://github.com/petekinnecom/concurrent_pipeline
 licenses:

data/lib/concurrent_pipeline/stores/schema/record.rb

@@ -1,47 +0,0 @@
-module ConcurrentPipeline
-  module Stores
-    class Schema
-      class Record
-        class << self
-          def attribute(name, **options)
-            attributes << name
-            attribute_defaults[name] = options[:default] if options.key?(:default)
-
-            define_method(name) do
-              attributes[name]
-            end
-
-            define_method("#{name}=") do |value|
-              @attributes[name] = value
-            end
-          end
-
-          def attributes
-            @attributes ||= []
-          end
-
-          def attribute_defaults
-            @attribute_defaults ||= {}
-          end
-
-          def inherited(mod)
-            mod.attribute(:id)
-          end
-        end
-
-        attr_reader :attributes
-        def initialize(attributes = {})
-          # Apply defaults for missing attributes
-          defaults = self.class.attribute_defaults
-          @attributes = self.class.attributes.each_with_object({}) do |attr_name, hash|
-            if attributes.key?(attr_name)
-              hash[attr_name] = attributes[attr_name]
-            elsif defaults.key?(attr_name)
-              hash[attr_name] = defaults[attr_name]
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/concurrent_pipeline/stores/storage/yaml/fs.rb

@@ -1,140 +0,0 @@
-require "yaml"
-require "fileutils"
-
-module ConcurrentPipeline
-  module Stores
-    module Storage
-      class Yaml
-        class Fs
-          @@mutex = Mutex.new
-
-          attr_reader :dir
-
-          def initialize(dir:)
-            @dir = dir
-            FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
-            FileUtils.mkdir_p(versions_dir) unless Dir.exist?(versions_dir)
-          end
-
-          def read_version(version_number)
-            @@mutex.synchronize do
-              if version_number == 0
-                {}
-              else
-                current_ver = unsafe_current_version_number
-
-                # If requesting the current/latest version, read from latest.yml
-                if version_number == current_ver
-                  if File.exist?(latest_file_path)
-                    data = File.read(latest_file_path).then { YAML.load(_1, aliases: true) || {} }
-                    # Normalize keys: convert record names and ID keys to strings for consistency
-                    data.transform_keys(&:to_s).transform_values do |records|
-                      records.transform_keys(&:to_s)
-                    end
-                  else
-                    {}
-                  end
-                else
-                  # Reading a historical version from versions/ directory
-                  file_path = version_file_path(version_number)
-                  if File.exist?(file_path)
-                    data = File.read(file_path).then { YAML.load(_1, aliases: true) || {} }
-                    # Normalize keys: convert record names and ID keys to strings for consistency
-                    data.transform_keys(&:to_s).transform_values do |records|
-                      records.transform_keys(&:to_s)
-                    end
-                  else
-                    {}
-                  end
-                end
-              end
-            end
-          end
-
-          def write_version(version_number, data)
-            @@mutex.synchronize do
-              # Copy current latest.yml to versions directory if it exists
-              if File.exist?(latest_file_path)
-                current_version = unsafe_current_version_number
-                if current_version > 0
-                  version_path = version_file_path(current_version)
-                  FileUtils.cp(latest_file_path, version_path)
-                end
-              end
-
-              # Write new data to latest.yml
-              File.write(latest_file_path, YAML.dump(data))
-            end
-          end
-
-          def current_version_number
-            @@mutex.synchronize do
-              unsafe_current_version_number
-            end
-          end
-
-          def version_files
-            @@mutex.synchronize do
-              unsafe_version_files
-            end
-          end
-
-          def delete_version(version_number)
-            @@mutex.synchronize do
-              current_ver = unsafe_current_version_number
-
-              # If deleting the latest version
-              if version_number == current_ver
-                File.delete(latest_file_path) if File.exist?(latest_file_path)
-              else
-                # Deleting an archived version
-                file_path = version_file_path(version_number)
-                File.delete(file_path) if File.exist?(file_path)
-              end
-            end
-          end
-
-          def restore_version(version_number)
-            @@mutex.synchronize do
-              version_path = version_file_path(version_number)
-
-              if File.exist?(version_path)
-                # Copy the version file to latest.yml
-                FileUtils.cp(version_path, latest_file_path)
-              else
-                raise "Version #{version_number} does not exist"
-              end
-            end
-          end
-
-          private
-
-          def unsafe_current_version_number
-            if File.exist?(latest_file_path)
-              # Count existing version files + 1 for the latest
-              unsafe_version_files.length + 1
-            else
-              0
-            end
-          end
-
-          def unsafe_version_files
-            Dir.glob(File.join(versions_dir, "*.yml")).sort
-          end
-
-          def version_file_path(version_num)
-            File.join(versions_dir, "%04d.yml" % version_num)
-          end
-
-          def latest_file_path
-            File.join(dir, "data.yml")
-          end
-
-          def versions_dir
-            File.join(dir, "versions")
-          end
-        end
-      end
-    end
-  end
-end

data/lib/concurrent_pipeline/stores/storage/yaml.rb

@@ -1,196 +0,0 @@
-module ConcurrentPipeline
-  module Stores
-    module Storage
-      class Yaml
-        attr_reader :fs, :version_number
-
-        def initialize(dir:, version_number: nil)
-          @fs = Fs.new(dir: dir)
-          @version_number = version_number
-        end
-
-        def in_transaction?
-          !transaction_operations.nil?
-        end
-
-        def transaction(&block)
-          begin_transaction
-          begin
-            yield
-            commit_transaction
-          rescue => e
-            rollback_transaction
-            raise e
-          end
-        end
-
-        def create(name:, attrs:)
-          in_txn = in_transaction?
-
-          raise "Cannot write to non-current version" unless writeable?
-
-          id = attrs[:id] || attrs["id"]
-          raise "Record must have an id" unless id
-
-          # Always buffer the operation
-          buffer_operation(
-            type: :create,
-            name: name.to_s,
-            id: id.to_s,
-            attrs: attrs.transform_keys(&:to_s)
-          )
-
-          # Flush immediately if not in a transaction
-          flush_buffer unless in_txn
-        end
-
-        def update(name:, id:, attrs:)
-          in_txn = in_transaction?
-
-          raise "Cannot write to non-current version" unless writeable?
-
-          # Always buffer the operation
-          buffer_operation(
-            type: :update,
-            name: name.to_s,
-            id: id.to_s,
-            attrs: attrs.transform_keys(&:to_s)
-          )
-
-          # Flush immediately if not in a transaction
-          flush_buffer unless in_txn
-        end
-
-        def all(name:)
-          data = load_data
-          records = data[name.to_s] || {}
-          records.values.map { |attrs| attrs.transform_keys(&:to_sym) }
-        end
-
-        def versions
-          current_ver = current_version_number
-          (1..current_ver).map { |idx| self.class.new(dir: fs.dir, version_number: idx) }
-        end
-
-        def restore
-          current_version = version_number || current_version_number
-
-          # Delete all versions after this one
-          fs.version_files.each_with_index do |file, idx|
-            version_num = idx + 1
-            if version_num > current_version
-              fs.delete_version(version_num)
-            end
-          end
-
-          # If restoring to a historical version (not current), move it to latest.yml
-          if version_number && version_number < current_version_number
-            fs.restore_version(version_number)
-          end
-
-          # Return a new writeable storage at this version
-          self.class.new(dir: fs.dir)
-        end
-
-        def writeable?
-          version_number.nil? || version_number == current_version_number
-        end
-
-        private
-
-        def begin_transaction
-          raise "Transaction already in progress" if transaction_operations
-
-          self.transaction_operations = []
-        end
-
-        def commit_transaction
-          raise "No transaction in progress" unless transaction_operations
-
-          flush_buffer
-        end
-
-        def rollback_transaction
-          self.transaction_operations = nil
-        end
-
-        TRANSACTION_KEY = :yaml_storage_transaction_operations
-
-        def transaction_operations
-          Fiber[TRANSACTION_KEY]
-        end
-
-        def transaction_operations=(value)
-          Fiber[TRANSACTION_KEY] = value
-        end
-
-        def buffer_operation(op)
-          # If in a transaction, append to the transaction buffer
-          if transaction_operations
-            transaction_operations << op
-          else
-            # If not in a transaction, initialize a temporary buffer
-            self.transaction_operations = [op]
-          end
-        end
-
-        def flush_buffer
-          return unless transaction_operations
-
-          # Load current data and apply all buffered operations
-          data = load_current_data
-          transaction_operations.each do |op|
-            apply_operation(data, op)
-          end
-
-          write_new_version(data)
-          self.transaction_operations = nil
-        end
-
-        def apply_operation(data, op)
-          case op[:type]
-          when :create
-            data[op[:name]] ||= {}
-            data[op[:name]][op[:id]] = op[:attrs]
-          when :update
-            records = data[op[:name]] || {}
-            if records[op[:id]]
-              records[op[:id]].merge!(op[:attrs])
-            else
-              raise "Record not found: #{op[:name]} with id #{op[:id].inspect}"
-            end
-          end
-        end
-
-        def load_current_data
-          load_data
-        end
-
-        def load_data
-          if version_number
-            # Reading a historical version from the versions/ directory
-            target_version = version_number
-            fs.read_version(target_version)
-          else
-            # Reading the current latest.yml
-            current_ver = current_version_number
-            if current_ver == 0
-              {}
-            else
-              fs.read_version(current_ver)
-            end
-          end
-        end
-
-        def write_new_version(data)
-          next_version = current_version_number + 1
-          fs.write_version(next_version, data)
-        end
-
-        def current_version_number
-          fs.current_version_number
-        end
-      end
-    end
-  end
-end