async_pipeline 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c923079b7f9cb16b354ff63bdde17ec7102978a533a8bf7e21ef7cef618fee39
+  data.tar.gz: 54dee3f2a5690fd5a34e3603067b2f5ce3261044a07948a35871014eaa702e3c
+SHA512:
+  metadata.gz: e4447679eea0e2298c72a946531ec422cd3caefe7d36acbcda3c61b2cddcc5ee3df9c3502586a556d0e2671f095fb6971908bcfca70277ae6a160c2115dd4f73
+  data.tar.gz: 791d1a309697cd7c940566493ff5c335900d746d310da969d8addf25b7d98e1e18442bfaef56b0a6a5239a9c6f603b1fcee1cb1b99f0279fb5c28d959b79d040

data/.rubocop.yml

@@ -0,0 +1,14 @@
+AllCops:
+  TargetRubyVersion: 3.2
+  DisabledByDefault: true
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120

data/.ruby-version

@@ -0,0 +1 @@
+3.2.3

data/README.md

@@ -0,0 +1,11 @@
+# AsyncPipeline
+
+An experimental thing to do stuff for you as you want it to do.
+
+## Installation
+
+In order to install this gem, one must first invent the universe.
+
+## Usage
+
+Probably don't. Cause I'm still figuring it out.

data/Rakefile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

data/async_pipeline.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "lib/async_pipeline/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "async_pipeline"
+  spec.version = AsyncPipeline::VERSION
+  spec.authors = ["Pete Kinnecom"]
+  spec.email = ["git@k7u7.com"]
+
+  spec.summary = "Define a pipeline of async tasks and their starting conditions"
+  spec.homepage = "https://github.com/petekinnecom/async_pipeline"
+  spec.license = "WTFPL"
+  spec.required_ruby_version = ">= 3.0.0"
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = spec.homepage
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+end

data/lib/async_pipeline/changeset.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module AsyncPipeline
+  class Changeset
+    CreateDelta = Struct.new(:type, :attributes, keyword_init: true) do
+      def apply(store)
+        store.create(type: type, attributes: attributes)
+      end
+    end
+
+    UpdateDelta = Struct.new(:model, :delta, keyword_init: true) do
+      def apply(store)
+        current_model = store.find(id: model.id, type: model.class)
+
+        # Todo: detect if changed underfoot
+
+        store.update(
+          id: model.id,
+          type: model.class,
+          attributes: current_model.attributes.merge(delta)
+        )
+      end
+    end
+
+    attr_reader :deltas
+    def initialize
+      @deltas = []
+    end
+
+    def deltas?
+      !@deltas.empty?
+    end
+
+    def create(type, attributes)
+      @deltas << CreateDelta.new(type: type, attributes: attributes)
+    end
+
+    def update(model, delta)
+      @deltas << UpdateDelta.new(model: model, delta: delta)
+    end
+
+    def apply(...)
+      deltas.each { _1.apply(...) }
+    end
+  end
+end

data/lib/async_pipeline/chunk.rb

@@ -0,0 +1,49 @@
+module AsyncPipeline
+  class Chunk
+    def self.build_chunks(store)
+      target = self.target
+      type = target.values.first
+      records = store.all(type: type)
+
+      if target.key?(:collection)
+        new(
+          target: records,
+          store: store,
+          changeset: Changeset.new
+        )
+      else
+        records.map { |record|
+          new(
+            target: record,
+            store: store,
+            changeset: Changeset.new
+          )
+        }
+      end
+    end
+
+    attr_accessor :target, :store, :changeset
+
+    def initialize(target:, store:, changeset:)
+      @target = target
+      @store = store
+      @changeset = changeset
+    end
+
+    def id
+      extra = (
+        if self.class.target.key?(:item)
+          { id: target.id }
+        else
+          {}
+        end
+      )
+
+      self.class.target.merge(extra)
+    end
+
+    def should_perform?
+      !done? && ready?
+    end
+  end
+end

data/lib/async_pipeline/model.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module AsyncPipeline
+  module Model
+    module InstanceMethods
+      attr_reader :attributes
+
+      def initialize(attributes)
+        @attributes = attributes
+      end
+    end
+
+    def self.extended(base)
+      base.include(InstanceMethods)
+    end
+
+    def inherited(base)
+      base.instance_variable_set(:@attributes, attributes.dup)
+    end
+
+    def attributes
+      @attributes ||= {}
+    end
+
+    def attribute(name, **opts)
+      attributes[name] = opts
+
+      define_method(name, &Ractor.make_shareable(Proc.new { attributes.fetch(name) }))
+    end
+  end
+end

data/lib/async_pipeline/pipeline.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative "processor"
+require_relative "registry"
+require_relative "store"
+require_relative "stores/yaml"
+
+module AsyncPipeline
+  module Pipeline
+    module InstanceMethods
+      def initialize(data)
+        @data = data
+      end
+
+      def start
+        Processor.call(
+          store: store,
+          chunkables: chunkables,
+          registry: registry
+        )
+      end
+
+      def data
+        store.to_h
+      end
+
+      private
+
+      def store
+        @store ||= self.class.store.build(data: @data, dir: dir, registry: registry)
+      end
+
+      def dir
+        @dir ||= Dir.mktmpdir
+      end
+
+      def registry
+        self.class.registry
+      end
+
+      def chunkables
+        self.class.chunkables
+      end
+    end
+
+    def self.extended(base)
+      base.include(InstanceMethods)
+    end
+
+    def inherited(base)
+      base.instance_variable_set(:@models, models.dup)
+      base.instance_variable_set(:@chunkables, chunkables.dup)
+    end
+
+    def store(klass = nil)
+      @store = klass if klass
+      @store
+    end
+
+    def chunkables
+      @chunkables ||= []
+    end
+
+    def registry
+      @registry ||= Registry.new
+    end
+
+    def model(klass, as_type:)
+      registry.register(as_type, klass)
+    end
+
+    def chunk(klass)
+      chunkables << klass
+    end
+  end
+end

data/lib/async_pipeline/processor.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require_relative "changeset"
+require_relative "read_only_store"
+
+module AsyncPipeline
+  class Processor
+    Message = Struct.new(:type, :payload) do
+      def to_s(...)
+        inspect(...)
+      end
+      def inspect(...)
+        "<Message #{type} (#{payload.class}) >"
+      end
+    end
+
+    Ms = Module.new do
+      def self.g(...)
+        Message.new(...)
+      end
+
+      def msg(...)
+        Message.new(...)
+      end
+    end
+
+    def self.Msg(type, payload = nil)
+      Message.new(type, payload)
+    end
+
+    def self.call(...)
+      new(...).call
+    end
+
+    attr_reader :store, :chunkables, :registry
+    def initialize(store:, chunkables:, registry:)
+      @store = store
+      @chunkables = chunkables
+      @registry = registry
+    end
+
+    def call
+      Log.info("scheduler_channel: #{scheduler_channel.object_id}")
+      scheduler_channel.send(Ms.g(:requeue))
+
+      workers
+      loop do
+        r, msg = Ractor.select(scheduler_channel, changeset_channel, work_channel, ticker)
+
+        case r
+        when ticker
+          case msg.type
+          when :tick
+            changeset_channel.send(Ms.g(:flush_queue))
+          end
+        when scheduler_channel
+          puts "scheduler_channel: #{msg}"
+          case msg.type
+          when :enqueue
+            work_channel.send(Ms.g(:process, msg.payload))
+          when :all_chunks_processed
+            scheduler_channel.close_outgoing
+            work_channel.close_outgoing
+            changeset_channel.close_outgoing
+          else
+            # scheduler_channel.close_outgoing
+            # raise "Unknown message: #{msg}"
+          end
+        when changeset_channel
+          case msg.type
+          when :chunks_updated
+            scheduler_channel.send(Ms.g(:requeue, msg.payload))
+          when :chunks_processed
+            scheduler_channel.send(Ms.g(:chunks_processed, msg.payload))
+          else
+            # raise "Unknown message: #{msg}"
+          end
+        when work_channel
+          case msg.type
+          when :changeset
+            changeset_channel.send(Ms.g(:changeset, msg.payload))
+          else
+            # raise "Unknown message: #{msg}"
+          end
+        end
+      end
+
+      store.to_h
+    end
+
+    def changeset_channel
+      @changeset_channel ||= Ractor.new(store) do |store, channel|
+        chunks = []
+        loop do
+          msg = receive
+          case msg.type
+          when :changeset
+            chunks << msg.payload
+          when :flush_queue
+            next unless chunks.any?
+
+            was_updated = store.apply(chunks.map(&:changeset))
+            if was_updated
+              Ractor.yield(Ms.g(:chunks_updated, chunks.map(&:id)))
+            else
+              Ractor.yield(Ms.g(:chunks_processed, chunks.map(&:id)))
+            end
+
+            chunks = []
+          end
+        end
+      end
+    end
+
+    def ticker
+      @ticker ||= Ractor.new do
+        loop do
+          sleep 0.5
+          Ractor.yield(Ms.g(:tick))
+        end
+      end
+    end
+
+    def scheduler_channel
+      @scheduler_channel ||= Ractor.new(store, chunkables, changeset_channel) do |store, chunkables, changeset_channel|
+        status = {}
+        loop do
+          msg = receive
+
+          # we update chunk_ids on both messages.
+          chunk_ids = msg.payload || []
+          chunk_ids.each do |chunk_id|
+            status[chunk_id] = :processed
+          end
+
+          case msg.type
+          when :requeue
+            reader = store.reader
+
+            Log.info "model_one: #{store.reader.all(type: :model_one).count}"
+            chunkables
+              .map { _1.build_chunks(reader) }
+              .flatten
+              .each do |c|
+                if status[c.id] != :queued && c.should_perform?
+                  puts "enqueuing: #{c.id}"
+                  status[c.id] = :queued
+                  Ractor.yield(Ms.g(:enqueue, c))
+                end
+              end
+          end
+
+          if status.values.all? { _1 == :processed }
+            Ractor.yield(Ms.g(:all_chunks_processed))
+          end
+        end
+      end
+    end
+
+    def work_channel
+      @work_channel ||= Ractor.new(work_pool) do |work_pool|
+        loop do
+          msg = receive
+          case msg.type
+          when :process
+            chunk = msg.payload
+            work_pool.send(Ms.g(:chunk, chunk))
+          when :changeset
+            Ractor.yield(Ms.g(:changeset, msg.payload))
+          end
+        end
+      end
+    end
+
+    def work_pool
+      @work_pool ||= Ractor.new do
+        loop do
+          Ractor.yield Ractor.receive
+        end
+      end
+    end
+
+    def workers
+      @workers ||= 4.times.map { |i|
+        Ractor.new(work_pool, work_channel, name: "worker-#{i}") do |work_pool, work_channel|
+          loop do
+            r, msg = Ractor.select(work_pool)
+
+            case msg.type
+            when :chunk
+              chunk = msg.payload
+              puts "starting perform: #{chunk.id}"
+              chunk.perform
+              puts "finished perform: #{chunk.id}"
+              work_channel.send(Ms.g(:changeset, chunk))
+            end
+          end
+        end
+      }
+    end
+  end
+end

data/lib/async_pipeline/read_only_store.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module AsyncPipeline
+  class ReadyOnlyStore
+    attr_reader :store
+    def initialize(store)
+      @store = store
+    end
+
+    def find(...)
+      store.find(...)
+    end
+
+    def all(...)
+      store.all(...)
+    end
+
+    def everything(...)
+      store.everything(...)
+    end
+  end
+end

data/lib/async_pipeline/registry.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module AsyncPipeline
+  class Registry
+    def build(type, attributes)
+      lookup.fetch(type).new(attributes)
+    end
+
+    def register(type, klass)
+      lookup[type] = klass
+    end
+
+    def type_for(type)
+      return type if lookup.key?(type)
+
+      reverse_lookup = lookup.invert
+
+      return reverse_lookup[type] if reverse_lookup.key?(type)
+
+      raise ArgumentError, "Unknown type: #{type}"
+    end
+
+    private
+
+    def lookup
+      @lookup ||= {}
+    end
+  end
+end

data/lib/async_pipeline/shell.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module AsyncPipeline
+  class Shell
+    class << self
+      Result = Struct.new(:success, :stdout, :stderr, keyword_init: true) do
+        def success?
+          success
+        end
+      end
+
+      def run!(...)
+        run(...).tap { raise "command failed: #{command}" unless _1.success? }
+      end
+
+      def run(command)
+        Open3.popen3(command) do |_in, stdout, stderr, wait_thr|
+          process_stdout = []
+          stdout_thr = Thread.new do
+            while line = stdout.gets&.chomp
+              yield(:stdout, line) if block_given?
+              process_stdout << line
+            end
+          end
+
+          process_stderr = []
+          stderr_thr = Thread.new do
+            while line = stderr.gets&.chomp
+              yield(:stderr, line) if block_given?
+              process_stderr << line
+            end
+          end
+
+          [
+            stderr_thr,
+            stdout_thr,
+          ].each(&:join)
+
+          Result.new(
+            success: wait_thr.value.success?,
+            stdout: process_stdout.join("\n"),
+            stderr: process_stderr.join("\n"),
+          )
+        end
+      end
+    end
+  end
+end

data/lib/async_pipeline/store.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module AsyncPipeline
+  class Store
+    attr_reader :db, :changeset
+    def initialize(db:, changeset:)
+      @changeset = changeset
+      @db = db
+    end
+
+    def find(...)
+      db.find(...)
+    end
+
+    def all(...)
+      db.find(...)
+    end
+
+    def create(...)
+      changeset.create(...)
+    end
+
+    def update(...)
+      changeset.update(...)
+    end
+  end
+end

data/lib/async_pipeline/stores/yaml.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "yaml"
+
+module AsyncPipeline
+  module Stores
+    class Yaml
+      class Reader
+        attr_reader :registry, :data
+        def initialize(registry:, data:)
+          @registry = registry
+          @data = data
+        end
+
+        def find(id:, type:)
+          type = registry.type_for(type)
+          attrs = (data[type] || []).find { |attrs| attrs[:id] == id }
+          registry.build(type, attrs)
+        end
+
+        def everything
+          data.each_with_object({}) do |(type, subdata), all_of_it|
+            all_of_it[type] = subdata.map { |attrs| registry.build(type, attrs) }
+          end
+        end
+
+        def all(type:)
+          type = registry.type_for(type)
+
+          data
+            .fetch(type, [])
+            .map { registry.build(type, _1) }
+        end
+      end
+
+      class Writer
+        attr_reader :path, :registry, :yaml_before, :data
+        def initialize(path:, registry:)
+          @path = path
+          @registry = registry
+          @yaml_before = File.read(path)
+          @data = YAML.unsafe_load(@yaml_before)
+        end
+
+        def apply(changesets)
+          changesets.each { _1.apply(self) }
+          yaml_after = data.to_yaml
+
+          if yaml_before != yaml_after
+            File.write(path, yaml_after)
+            true
+          else
+            false
+          end
+        end
+
+        def find(id:, type:)
+          type = registry.type_for(type)
+          attrs = (data[type] || []).find { |attrs| attrs[:id] == id }
+          registry.build(type, attrs)
+        end
+
+        def create(type:, attributes:)
+          type = registry.type_for(type)
+
+          data[type] ||= {}
+          data[type] << { id: SecureRandom.uuid }.merge(attributes)
+        end
+
+        def update(id:, type:, attributes:)
+          type = registry.type_for(type)
+          attrs = (data[type] || []).find { |attrs| attrs[:id] == id }
+          raise "Not found" unless attrs
+          attrs.merge!(attributes)
+        end
+      end
+
+      def self.build(data:, dir:, registry:)
+        path = File.join(dir, "data.yml")
+
+        File.write(path, data.to_yaml)
+
+        new(
+          path: path,
+          registry: registry
+        )
+      end
+
+      attr_reader :path, :registry
+      def initialize(path:, registry:)
+        @path = path
+        @registry = registry
+      end
+
+      def reader
+        Reader.new(data: read, registry: registry)
+      end
+
+      def apply(changesets)
+        Writer.new(path: path, registry: registry).apply(changesets)
+      end
+
+      def to_h
+        read
+      end
+
+      private
+
+      def read
+        YAML.unsafe_load_file(path)
+      end
+    end
+  end
+end

data/lib/async_pipeline/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module AsyncPipeline
+  VERSION = "0.1.0"
+end

data/lib/async_pipeline.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative "async_pipeline/version"
+require_relative "async_pipeline/model"
+require_relative "async_pipeline/chunk"
+require_relative "async_pipeline/pipeline"
+
+require "logger"
+
+module AsyncPipeline
+  class Error < StandardError; end
+  # Your code goes here...
+
+  class RactorLogger < Ractor
+    def self.new
+      super do
+        # STDOUT cannot be referenced but $stdout can
+        logger = ::Logger.new($stdout)
+        logger.formatter = proc do |severity, datetime, progname, msg|
+          "#{datetime.to_i} [#{severity}] #{msg}\n"
+       end
+
+        # Run the requested operations on our logger instance
+        while data = receive
+          logger.public_send(data[0], *data[1])
+        end
+      end
+    end
+
+    def log(level, msg)
+      ractor_name = Ractor.current.name || Ractor.current.inspect
+
+      send([level, "#{ractor_name}: #{msg}"])
+    end
+
+    def info(msg)
+      log(:info, msg)
+    end
+
+    def error(msg)
+      log(:error, msg)
+    end
+
+    def warn(msg)
+      log(:warn, msg)
+    end
+
+    def debug(msg)
+      log(:debug, msg)
+    end
+  end
+
+  Log = RactorLogger.new
+
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: async_pipeline
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Pete Kinnecom
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-05-02 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+- git@k7u7.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- ".ruby-version"
+- README.md
+- Rakefile
+- async_pipeline.gemspec
+- lib/async_pipeline.rb
+- lib/async_pipeline/changeset.rb
+- lib/async_pipeline/chunk.rb
+- lib/async_pipeline/model.rb
+- lib/async_pipeline/pipeline.rb
+- lib/async_pipeline/processor.rb
+- lib/async_pipeline/read_only_store.rb
+- lib/async_pipeline/registry.rb
+- lib/async_pipeline/shell.rb
+- lib/async_pipeline/store.rb
+- lib/async_pipeline/stores/yaml.rb
+- lib/async_pipeline/version.rb
+homepage: https://github.com/petekinnecom/async_pipeline
+licenses:
+- WTFPL
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/petekinnecom/async_pipeline
+  source_code_uri: https://github.com/petekinnecom/async_pipeline
+  changelog_uri: https://github.com/petekinnecom/async_pipeline
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.19
+signing_key:
+specification_version: 4
+summary: Define a pipeline of async tasks and their starting conditions
+test_files: []