concurrent_pipeline 0.1.0 → 1.0.0

data/lib/concurrent_pipeline/stores/storage/yaml/fs.rb

@@ -0,0 +1,140 @@
+require "yaml"
+require "fileutils"
+
+module ConcurrentPipeline
+  module Stores
+    module Storage
+      class Yaml
+        class Fs
+          @@mutex = Mutex.new
+
+          attr_reader :dir
+
+          def initialize(dir:)
+            @dir = dir
+            FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+            FileUtils.mkdir_p(versions_dir) unless Dir.exist?(versions_dir)
+          end
+
+          def read_version(version_number)
+            @@mutex.synchronize do
+              if version_number == 0
+                {}
+              else
+                current_ver = unsafe_current_version_number
+
+                # If requesting the current/latest version, read from latest.yml
+                if version_number == current_ver
+                  if File.exist?(latest_file_path)
+                    data = File.read(latest_file_path).then { YAML.load(_1, aliases: true) || {} }
+                    # Normalize keys: convert record names and ID keys to strings for consistency
+                    data.transform_keys(&:to_s).transform_values do |records|
+                      records.transform_keys(&:to_s)
+                    end
+                  else
+                    {}
+                  end
+                else
+                  # Reading a historical version from versions/ directory
+                  file_path = version_file_path(version_number)
+                  if File.exist?(file_path)
+                    data = File.read(file_path).then { YAML.load(_1, aliases: true) || {} }
+                    # Normalize keys: convert record names and ID keys to strings for consistency
+                    data.transform_keys(&:to_s).transform_values do |records|
+                      records.transform_keys(&:to_s)
+                    end
+                  else
+                    {}
+                  end
+                end
+              end
+            end
+          end
+
+          def write_version(version_number, data)
+            @@mutex.synchronize do
+              # Copy current latest.yml to versions directory if it exists
+              if File.exist?(latest_file_path)
+                current_version = unsafe_current_version_number
+                if current_version > 0
+                  version_path = version_file_path(current_version)
+                  FileUtils.cp(latest_file_path, version_path)
+                end
+              end
+
+              # Write new data to latest.yml
+              File.write(latest_file_path, YAML.dump(data))
+            end
+          end
+
+          def current_version_number
+            @@mutex.synchronize do
+              unsafe_current_version_number
+            end
+          end
+
+          def version_files
+            @@mutex.synchronize do
+              unsafe_version_files
+            end
+          end
+
+          def delete_version(version_number)
+            @@mutex.synchronize do
+              current_ver = unsafe_current_version_number
+
+              # If deleting the latest version
+              if version_number == current_ver
+                File.delete(latest_file_path) if File.exist?(latest_file_path)
+              else
+                # Deleting an archived version
+                file_path = version_file_path(version_number)
+                File.delete(file_path) if File.exist?(file_path)
+              end
+            end
+          end
+
+          def restore_version(version_number)
+            @@mutex.synchronize do
+              version_path = version_file_path(version_number)
+
+              if File.exist?(version_path)
+                # Copy the version file to latest.yml
+                FileUtils.cp(version_path, latest_file_path)
+              else
+                raise "Version #{version_number} does not exist"
+              end
+            end
+          end
+
+          private
+
+          def unsafe_current_version_number
+            if File.exist?(latest_file_path)
+              # Count existing version files + 1 for the latest
+              unsafe_version_files.length + 1
+            else
+              0
+            end
+          end
+
+          def unsafe_version_files
+            Dir.glob(File.join(versions_dir, "*.yml")).sort
+          end
+
+          def version_file_path(version_num)
+            File.join(versions_dir, "%04d.yml" % version_num)
+          end
+
+          def latest_file_path
+            File.join(dir, "data.yml")
+          end
+
+          def versions_dir
+            File.join(dir, "versions")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/stores/storage/yaml.rb

@@ -0,0 +1,196 @@
+module ConcurrentPipeline
+  module Stores
+    module Storage
+      class Yaml
+        attr_reader :fs, :version_number
+
+        def initialize(dir:, version_number: nil)
+          @fs = Fs.new(dir: dir)
+          @version_number = version_number
+        end
+
+        def in_transaction?
+          !transaction_operations.nil?
+        end
+
+        def transaction(&block)
+          begin_transaction
+          begin
+            yield
+            commit_transaction
+          rescue => e
+            rollback_transaction
+            raise e
+          end
+        end
+
+        def create(name:, attrs:)
+          in_txn = in_transaction?
+
+          raise "Cannot write to non-current version" unless writeable?
+
+          id = attrs[:id] || attrs["id"]
+          raise "Record must have an id" unless id
+
+          # Always buffer the operation
+          buffer_operation(
+            type: :create,
+            name: name.to_s,
+            id: id.to_s,
+            attrs: attrs.transform_keys(&:to_s)
+          )
+
+          # Flush immediately if not in a transaction
+          flush_buffer unless in_txn
+        end
+
+        def update(name:, id:, attrs:)
+          in_txn = in_transaction?
+
+          raise "Cannot write to non-current version" unless writeable?
+
+          # Always buffer the operation
+          buffer_operation(
+            type: :update,
+            name: name.to_s,
+            id: id.to_s,
+            attrs: attrs.transform_keys(&:to_s)
+          )
+
+          # Flush immediately if not in a transaction
+          flush_buffer unless in_txn
+        end
+
+        def all(name:)
+          data = load_data
+          records = data[name.to_s] || {}
+          records.values.map { |attrs| attrs.transform_keys(&:to_sym) }
+        end
+
+        def versions
+          current_ver = current_version_number
+          (1..current_ver).map { |idx| self.class.new(dir: fs.dir, version_number: idx) }
+        end
+
+        def restore
+          current_version = version_number || current_version_number
+
+          # Delete all versions after this one
+          fs.version_files.each_with_index do |file, idx|
+            version_num = idx + 1
+            if version_num > current_version
+              fs.delete_version(version_num)
+            end
+          end
+
+          # If restoring to a historical version (not current), move it to latest.yml
+          if version_number && version_number < current_version_number
+            fs.restore_version(version_number)
+          end
+
+          # Return a new writeable storage at this version
+          self.class.new(dir: fs.dir)
+        end
+
+        def writeable?
+          version_number.nil? || version_number == current_version_number
+        end
+
+        private
+
+        def begin_transaction
+          raise "Transaction already in progress" if transaction_operations
+
+          self.transaction_operations = []
+        end
+
+        def commit_transaction
+          raise "No transaction in progress" unless transaction_operations
+
+          flush_buffer
+        end
+
+        def rollback_transaction
+          self.transaction_operations = nil
+        end
+
+        TRANSACTION_KEY = :yaml_storage_transaction_operations
+
+        def transaction_operations
+          Fiber[TRANSACTION_KEY]
+        end
+
+        def transaction_operations=(value)
+          Fiber[TRANSACTION_KEY] = value
+        end
+
+        def buffer_operation(op)
+          # If in a transaction, append to the transaction buffer
+          if transaction_operations
+            transaction_operations << op
+          else
+            # If not in a transaction, initialize a temporary buffer
+            self.transaction_operations = [op]
+          end
+        end
+
+        def flush_buffer
+          return unless transaction_operations
+
+          # Load current data and apply all buffered operations
+          data = load_current_data
+          transaction_operations.each do |op|
+            apply_operation(data, op)
+          end
+
+          write_new_version(data)
+          self.transaction_operations = nil
+        end
+
+        def apply_operation(data, op)
+          case op[:type]
+          when :create
+            data[op[:name]] ||= {}
+            data[op[:name]][op[:id]] = op[:attrs]
+          when :update
+            records = data[op[:name]] || {}
+            if records[op[:id]]
+              records[op[:id]].merge!(op[:attrs])
+            else
+              raise "Record not found: #{op[:name]} with id #{op[:id].inspect}"
+            end
+          end
+        end
+
+        def load_current_data
+          load_data
+        end
+
+        def load_data
+          if version_number
+            # Reading a historical version from the versions/ directory
+            target_version = version_number
+            fs.read_version(target_version)
+          else
+            # Reading the current latest.yml
+            current_ver = current_version_number
+            if current_ver == 0
+              {}
+            else
+              fs.read_version(current_ver)
+            end
+          end
+        end
+
+        def write_new_version(data)
+          next_version = current_version_number + 1
+          fs.write_version(next_version, data)
+        end
+
+        def current_version_number
+          fs.current_version_number
+        end
+      end
+    end
+  end
+end

data/lib/concurrent_pipeline/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ConcurrentPipeline
-  VERSION = "0.1.0"
+  VERSION = "1.0.0"
 end

data/lib/concurrent_pipeline.rb

@@ -1,15 +1,19 @@
 # frozen_string_literal: true
 
-require_relative "concurrent_pipeline/version"
-require_relative "concurrent_pipeline/model"
-require_relative "concurrent_pipeline/pipeline"
-require_relative "concurrent_pipeline/producer"
-require_relative "concurrent_pipeline/shell"
-
-require "logger"
+require "zeitwerk"
+loader = Zeitwerk::Loader.for_gem
+loader.setup
 
 module ConcurrentPipeline
   class Error < StandardError; end
-  # Your code goes here...
-  Log = Logger.new($stdout).tap { _1.level = Logger::WARN }
+
+  class << self
+    def store(&)
+      Store.define(&)
+    end
+
+    def pipeline(&)
+      Pipeline.define(&)
+    end
+  end
 end

metadata

@@ -1,17 +1,45 @@
 --- !ruby/object:Gem::Specification
 name: concurrent_pipeline
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Pete Kinnecom
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-11 00:00:00.000000000 Z
+date: 2025-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby-edge
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: async
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -31,26 +59,24 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".rubocop.yml"
+- ".claude/settings.local.json"
 - ".ruby-version"
 - README.md
 - Rakefile
 - concurrency.md
 - concurrent_pipeline.gemspec
 - lib/concurrent_pipeline.rb
-- lib/concurrent_pipeline/changeset.rb
-- lib/concurrent_pipeline/model.rb
 - lib/concurrent_pipeline/pipeline.rb
-- lib/concurrent_pipeline/processors/actor_processor.rb
-- lib/concurrent_pipeline/producer.rb
-- lib/concurrent_pipeline/read_only_store.rb
-- lib/concurrent_pipeline/registry.rb
+- lib/concurrent_pipeline/pipelines/processors/asynchronous.rb
+- lib/concurrent_pipeline/pipelines/processors/locker.rb
+- lib/concurrent_pipeline/pipelines/processors/synchronous.rb
+- lib/concurrent_pipeline/pipelines/schema.rb
 - lib/concurrent_pipeline/shell.rb
 - lib/concurrent_pipeline/store.rb
-- lib/concurrent_pipeline/stores/versioned.rb
-- lib/concurrent_pipeline/stores/yaml.rb
-- lib/concurrent_pipeline/stores/yaml/db.rb
-- lib/concurrent_pipeline/stores/yaml/history.rb
+- lib/concurrent_pipeline/stores/schema.rb
+- lib/concurrent_pipeline/stores/schema/record.rb
+- lib/concurrent_pipeline/stores/storage/yaml.rb
+- lib/concurrent_pipeline/stores/storage/yaml/fs.rb
 - lib/concurrent_pipeline/version.rb
 homepage: https://github.com/petekinnecom/concurrent_pipeline
 licenses:

data/.rubocop.yml

@@ -1,14 +0,0 @@
-AllCops:
-  TargetRubyVersion: 3.2
-  DisabledByDefault: true
-
-Style/StringLiterals:
-  Enabled: true
-  EnforcedStyle: double_quotes
-
-Style/StringLiteralsInInterpolation:
-  Enabled: true
-  EnforcedStyle: double_quotes
-
-Layout/LineLength:
-  Max: 120

data/lib/concurrent_pipeline/changeset.rb

@@ -1,133 +0,0 @@
-# frozen_string_literal: true
-
-module ConcurrentPipeline
-  class Changeset
-    Result = Struct.new(:diff) do
-      alias diff? diff
-    end
-
-    InitialDelta = Struct.new(:data, :dup, keyword_init: true) do
-      def apply(store)
-        # We fully dup the data to avoid mutating the input
-        dup_data = YAML.unsafe_load(data.to_yaml)
-        store.set(dup_data)
-        Result.new(true)
-      end
-
-      def self.from_json(json)
-        new(data: json.fetch(:delta), dup: true)
-      end
-
-      def as_json(...)
-        {
-          action: :initial,
-          delta: data
-        }
-      end
-    end
-
-    CreateDelta = Struct.new(:type, :attributes, keyword_init: true) do
-      def apply(store)
-        store.create(type: type, attributes: attributes)
-        Result.new(true)
-      end
-
-      def self.from_json(json)
-        new(
-          type: json.fetch(:type),
-          attributes: json.fetch(:attributes)
-        )
-      end
-
-      def as_json(...)
-        {
-          action: :create,
-          type: type,
-          attributes: attributes
-        }
-      end
-    end
-
-    UpdateDelta = Struct.new(:id, :type, :delta, keyword_init: true) do
-      def apply(store)
-        current_model = store.find(type, id)
-
-        # Todo: detect if changed underfoot
-
-        Result.new(
-          store.update(
-            id: id,
-            type: type,
-            attributes: current_model.attributes.merge(delta)
-          )
-        )
-      end
-
-      def self.from_json(json)
-        new(
-          id: json.fetch(:id),
-          type: json.fetch(:type),
-          delta: json.fetch(:delta),
-        )
-      end
-
-      def as_json
-        {
-          action: :update,
-          id: id,
-          type: type,
-          delta: delta
-        }
-      end
-    end
-
-    def self.from_json(registry:, json:)
-      type_map = {
-        initial: InitialDelta,
-        create: CreateDelta,
-        update: UpdateDelta,
-      }
-
-      new(
-        registry: registry
-      ).tap do |changeset|
-        json.fetch(:changes).each do |change|
-          type_map
-            .fetch(change.fetch(:action))
-            .from_json(change)
-            .then { changeset.deltas << _1 }
-        end
-      end
-    end
-
-    attr_reader :deltas, :registry
-    def initialize(registry:)
-      @registry = registry
-      @deltas = []
-    end
-
-    def deltas?
-      !@deltas.empty?
-    end
-
-    def create(type, attributes)
-      with_id = { id: SecureRandom.uuid }.merge(attributes)
-      @deltas << CreateDelta.new(type: type, attributes: with_id)
-    end
-
-    def update(model, delta)
-      type = registry.type_for(model.class)
-      @deltas << UpdateDelta.new(id: model.id, type: type, delta: delta)
-    end
-
-    def apply(...)
-      deltas.map { _1.apply(...) }
-    end
-
-    def as_json(...)
-      {
-        changes: deltas.map { _1.as_json(...) }
-      }
-    end
-  end
-end

data/lib/concurrent_pipeline/model.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module ConcurrentPipeline
-  module Model
-    module InstanceMethods
-      attr_reader :attributes
-
-      def initialize(attributes)
-        @attributes = attributes
-      end
-    end
-
-    def self.extended(base)
-      base.include(InstanceMethods)
-    end
-
-    def inherited(base)
-      base.instance_variable_set(:@attributes, attributes.dup)
-    end
-
-    def attributes
-      @attributes ||= {}
-    end
-
-    def attribute(name, **opts)
-      attributes[name] = opts
-
-      define_method(name) { attributes[name] }
-    end
-  end
-end