shifty 0.5.0 → 0.6.0

data/lib/shifty/policy.rb

@@ -0,0 +1,124 @@
+module Shifty
+  # Shared by Worker and Gang: declares a pipeline-level policy default on
+  # every node reachable upstream through the supply chain. A worker's own
+  # policy declaration (its contract) always wins over the pipeline default.
+  module PolicyDeclarable
+    def with_policy(policy_name)
+      policy_name = Policy.validate!(policy_name)
+      each_upstream_node { |node| node.pipeline_policy = policy_name }
+      self
+    end
+
+    # Locks the assembled topology: "the pipeline you composed is the
+    # pipeline that runs" becomes a guarantee. Rewiring (supply=, Gang
+    # append/roster mutation) raises FrozenError afterward. Worker
+    # closure/context state stays mutable — only the topology freezes.
+    #
+    # Call this on the pipeline's TAIL: like with_policy, it walks the
+    # supply chain upstream, so freezing a mid-chain node leaves
+    # everything downstream of it mutable. And use this, not the bare
+    # Object#freeze — freeze! first materializes each node's lazy task
+    # and Fiber; a bare freeze skips that and the first shift would
+    # raise FrozenError from deep inside the worker.
+    def freeze!
+      each_upstream_node { |node| node.freeze_topology_node! }
+      self
+    end
+
+    private
+
+    def each_upstream_node
+      node = self
+      seen = {}.compare_by_identity
+      while node.respond_to?(:pipeline_policy=) && !seen[node]
+        seen[node] = true
+        yield node
+        node = node.supply
+      end
+    end
+  end
+
+  # Handoff policies govern how a value crosses a worker boundary.
+  # Each policy responds to #call(value, worker:) and returns the value
+  # the worker's task will receive.
+  module Policy
+    # Deeply freezes the value in place (zero copies) so any mutation,
+    # anywhere in the pipeline, raises at the worker that attempted it.
+    # IO-like values are rejected proactively: Ractor.make_shareable would
+    # otherwise freeze a live handle in place — a process-wide side effect
+    # on shared resources like loggers or $stdout.
+    Frozen = lambda do |value, worker:|
+      if value.is_a?(IO)
+        raise UnshareableValue.new(worker: worker, policy: :frozen, value: value)
+      end
+      begin
+        Ractor.make_shareable(value)
+      rescue Ractor::Error => e
+        raise UnshareableValue.new(worker: worker, policy: :frozen, value: value, cause: e)
+      end
+    end
+
+    # Hands the task a private, mutable deep copy. Marshal is the mechanism
+    # because Ractor.make_shareable(copy: true) returns a *frozen* copy,
+    # which cannot satisfy the :isolated contract of a mutable scratch value.
+    Isolated = lambda do |value, worker:|
+      Marshal.load(Marshal.dump(value))
+    rescue TypeError => e
+      raise UnshareableValue.new(worker: worker, policy: :isolated, value: value, cause: e)
+    end
+
+    # The escape hatch: the raw reference passes through untouched.
+    Shared = ->(value, worker:) { value }
+
+    TABLE = {
+      frozen: Frozen,
+      isolated: Isolated,
+      shared: Shared
+    }.freeze
+
+    ALIASES = {hardened: :isolated}.freeze
+
+    class << self
+      def resolve(name)
+        TABLE.fetch(canonical(name)) do
+          raise ArgumentError, "unknown policy #{name.inspect}"
+        end
+      end
+
+      def canonical(name)
+        if ALIASES.key?(name)
+          replacement = ALIASES[name]
+          warn "[shifty] policy :#{name} is deprecated and will be " \
+               "removed in 1.0.0; use :#{replacement} instead."
+          replacement
+        else
+          name
+        end
+      end
+
+      # Canonicalizes and validates a policy name at declaration time, so
+      # a typo fails where it was written rather than at first shift.
+      def validate!(name)
+        canonical(name).tap do |canonical_name|
+          unless TABLE.key?(canonical_name)
+            raise ArgumentError, "unknown policy #{name.inspect}"
+          end
+        end
+      end
+    end
+
+    # Wraps a worker's supply so that values the task pulls directly
+    # (e.g. filter/batch/trailing workers calling supply.shift mid-task)
+    # cross the boundary under the same policy as the primary intake.
+    class Supply
+      def initialize(supply, worker)
+        @supply = supply
+        @worker = worker
+      end
+
+      def shift
+        @worker.intake(@supply.shift)
+      end
+    end
+  end
+end

data/lib/shifty/rspec.rb

@@ -0,0 +1,37 @@
+require "shifty/testing"
+
+# Opt-in RSpec sugar for testing Shifty workers under handoff policies.
+# `require "shifty/rspec"` from your spec_helper; assumes RSpec is loaded.
+
+RSpec::Matchers.define :mutate_input do |input|
+  match do |worker|
+    Shifty::Testing.mutates_input?(worker, input)
+  end
+
+  failure_message do |worker|
+    "expected the worker's task to mutate its input #{input.inspect}, but it did not"
+  end
+
+  failure_message_when_negated do |worker|
+    "expected the worker's task not to mutate its input, but it mutated " \
+      "#{input.inspect}. It is only correct under policy :isolated (mutation " \
+      "stays local) or :shared (mutation is intentional); it will raise under :frozen."
+  end
+end
+
+# Runs the worker through the framework against deeply frozen input —
+# the strictest policy — proving the task is non-destructive and therefore
+# correct under every policy (§9.4, "test at the ceiling").
+#
+# Expects `worker` and `safe_input` to be defined with `let`.
+RSpec.shared_examples "a policy-safe worker" do
+  it "processes a deeply frozen input without raising" do
+    expect {
+      Shifty::Testing.run(worker, inputs: [safe_input], policy: :frozen)
+    }.not_to raise_error
+  end
+
+  it "does not mutate its input" do
+    expect(worker).not_to mutate_input(safe_input)
+  end
+end

data/lib/shifty/testing.rb

@@ -0,0 +1,82 @@
+require "shifty"
+
+module Shifty
+  # Test harness: runs a worker through the framework so unit tests
+  # exercise the same handoff policy the production pipeline will.
+  # Deliberately not loaded by `require "shifty"` — opt in with
+  # `require "shifty/testing"`.
+  module Testing
+    class << self
+      # Feeds the inputs to the worker via a source and collects its
+      # outputs until the end-of-stream sentinel (nil). The worker's
+      # declared/effective policy governs each handoff, exactly as in
+      # production; pass policy: to override it for policy-matrix tests.
+      def run(worker, inputs:, policy: nil, max_shifts: 10_000)
+        harness(worker, policy) do |subject|
+          subject.supply = source_for(inputs)
+          outputs = []
+          shifts = 0
+          until (value = subject.shift).nil?
+            outputs << value
+            if (shifts += 1) > max_shifts
+              raise Error, "Shifty::Testing.run exceeded #{max_shifts} shifts " \
+                "without seeing the nil end-of-stream sentinel. The worker's " \
+                "task probably converts nil into a non-nil value; let nil " \
+                "pass through (e.g. `value && ...`), or raise max_shifts:."
+            end
+          end
+          outputs
+        end
+      end
+
+      # The mutation detector (§6.4): hands the task a private mutable
+      # deep copy and reports whether the task changed it — surfacing
+      # mutation even when the current policy permits or hides it.
+      def mutates_input?(worker, input)
+        copy = begin
+          Marshal.load(Marshal.dump(input))
+        rescue TypeError => e
+          raise Error, "Shifty::Testing.mutates_input? needs a deep-copyable " \
+            "(Marshal-dumpable) input, but got an instance of #{input.class} " \
+            "(#{e.message})."
+        end
+        baseline = Marshal.dump(copy)
+        harness(worker, :shared) do |subject|
+          subject.supply = source_for([copy])
+          subject.shift
+        end
+        Marshal.dump(copy) != baseline
+      end
+
+      private
+
+      # Temporarily rewires the caller's actual worker (a dup would defeat
+      # task closures that reference their own worker, e.g. side_worker's
+      # policy check) and restores its policy, supply, and Fiber afterward,
+      # so the harness never leaves a mark on the worker under test.
+      def harness(worker, policy)
+        saved = {
+          policy: worker.instance_variable_get(:@policy),
+          supply: worker.instance_variable_get(:@supply),
+          fiber: worker.instance_variable_get(:@my_little_machine)
+        }
+        worker.instance_variable_set(:@policy, Policy.validate!(policy)) if policy
+        worker.instance_variable_set(:@my_little_machine, nil)
+        worker.instance_variable_set(:@supply, nil)
+        yield worker
+      ensure
+        worker.instance_variable_set(:@policy, saved[:policy])
+        worker.instance_variable_set(:@supply, saved[:supply])
+        worker.instance_variable_set(:@my_little_machine, saved[:fiber])
+      end
+
+      def source_for(inputs)
+        series = inputs.dup
+        Worker.new(tags: [:source]) do
+          series.each { |input| Fiber.yield input }
+          loop { Fiber.yield nil }
+        end
+      end
+    end
+  end
+end

data/lib/shifty/version.rb

@@ -1,3 +1,3 @@
 module Shifty
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/lib/shifty/worker.rb

@@ -2,22 +2,60 @@ require "ostruct"
 require "shifty/taggable"
 
 module Shifty
+  # Shifty::Worker uses Ruby Fibers for cooperative multitasking.
+  # This results in a single-threaded execution model where workers
+  # explicitly yield control to one another. This model is chosen for its
+  # simplicity and suitability for creating chainable data processing
+  # pipelines, where each worker performs a specific task and passes
+  # its output to the next worker in the chain.
   class Worker
-    attr_reader :supply, :tags
+    attr_reader :supply, :tags, :name
+    attr_accessor :pipeline_policy
 
     include Shifty::Taggable
+    include Shifty::PolicyDeclarable
 
     def initialize(p = {}, &block)
       @supply       = p[:supply]
       @task         = block || p[:task]
       @context      = p[:context] || OpenStruct.new
+      @policy       = Policy.validate!(p[:policy]) if p[:policy]
+      @name         = p[:name]
       self.criteria = p[:criteria]
       self.tags     = p[:tags]
     end
 
+    def effective_policy
+      @policy || pipeline_policy || Shifty.config.default_policy
+    end
+
+    # Applies this worker's effective policy to a value crossing its
+    # boundary. Public because Policy::Supply routes a task's own
+    # supply.shift calls back through it.
+    def intake(value)
+      Policy.resolve(effective_policy).call(value, worker: self)
+    end
+
     def shift
       ensure_ready_to_work!
       workflow.resume
+    rescue PolicyError
+      # The raising Fiber is terminated and can never be resumed; discard
+      # it so a caller that rescues the violation can keep shifting.
+      # Closure and context state survive — only the loop restarts.
+      # (A #freeze!-d worker can't rebuild its Fiber, so a violation
+      # there ends the pipeline — the topology guarantee wins.)
+      @my_little_machine = nil unless frozen?
+      raise
+    end
+
+    # Materializes lazy state (default task, the Fiber) so that freezing
+    # this instance cannot surprise it later with a FrozenError on ivar
+    # assignment mid-shift, then freezes it.
+    def freeze_topology_node!
+      ensure_ready_to_work!
+      workflow
+      freeze
     end
 
     def ready_to_work?
@@ -50,11 +88,18 @@ module Shifty
     end
 
     def workflow
+      # This is the core of the worker's execution, managed by a Fiber.
+      # The Fiber allows the worker to pause its execution (yield) and
+      # be resumed later, enabling cooperative multitasking.
+      #
+      # Handoff policy is applied at intake — the moment this worker pulls
+      # a value across its boundary — which is the single seam every value
+      # crosses regardless of how many times an upstream task yielded.
       @my_little_machine ||= Fiber.new {
         loop do
-          value = supply&.shift
+          value = intake(supply&.shift)
           if criteria_passes?
-            Fiber.yield @task.call(value, supply, @context)
+            Fiber.yield perform_task(value)
           else
             Fiber.yield value
           end
@@ -62,6 +107,22 @@ module Shifty
       }
     end
 
+    def perform_task(value)
+      @task.call(value, policy_supply, @context)
+    rescue FrozenError => e
+      raise PolicyViolation.new(
+        worker: self,
+        policy: effective_policy,
+        receiver: e.receiver,
+        value: value,
+        cause: e
+      )
+    end
+
+    def policy_supply
+      supply && Policy::Supply.new(supply, self)
+    end
+
     def default_task
       proc { |value| value }
     end
@@ -77,7 +138,49 @@ module Shifty
     def task_method_accepts_a_value?
       method(:task).arity > 0
     end
-  end
 
-  class WorkerError < StandardError; end
+    # ## Concurrency and Thread Safety
+    #
+    # ### Concurrency Model
+    # Shifty utilizes Ruby Fibers to achieve cooperative multitasking. This means
+    # that workers voluntarily yield control, allowing other workers to execute.
+    # The entire processing pipeline runs within a single thread, which removes
+    # an entire class of *preemptive* concurrency hazards (races on shared
+    # objects, the need for mutexes around Shifty's own state).
+    #
+    # Note that single-threading does NOT make the data flowing between workers
+    # safe on its own: because each value passes through every worker before the
+    # next value begins, a worker that mutates a handed-off value could silently
+    # corrupt what downstream workers observe. That hazard is orthogonal to
+    # threading and is governed by handoff immutability policies: values are
+    # deeply frozen at intake by default (:frozen), with :isolated and :shared
+    # opt-outs per worker/pipeline/global (see lib/shifty/policy.rb and the
+    # project wiki's Handoff-Policies page).
+    #
+    # ### Alternatives (Threads/Ractors)
+    # Threads and Ractors are not used for parallelism *yet*. Shifty's current
+    # goal is an easy-to-use framework for sequential data pipelines, and a
+    # single-threaded Fiber model serves that directly without the overhead of
+    # mutexes (Threads) or the sharing restrictions of Ractors. This is a
+    # scoping decision, not a rejection: the default :frozen policy's deeply
+    # frozen, shareable handoff values are deliberately Ractor-compatible and
+    # lay the groundwork for a future Ractor-backed worker type. (Fibers cannot
+    # cross Ractor boundaries, so such a worker would be a distinct type, not a
+    # retrofit of this one.)
+    #
+    # ### Thread Safety
+    # `Shifty::Worker` instances, and by extension `Shifty::Gang` or
+    # `Shifty::Roster` instances that manage these workers, are **not**
+    # inherently thread-safe if they are shared and modified across multiple
+    # user-created native threads. If users choose to integrate Shifty components
+    # into a multi-threaded application (e.g., processing multiple independent
+    # Shifty pipelines in parallel using separate Threads), they are responsible
+    # for implementing appropriate synchronization mechanisms (like mutexes)
+    # to protect shared Shifty objects from concurrent access and modification.
+    # (Handoff immutability policies govern only the *values* passed between
+    # workers, not the worker/pipeline objects themselves or their closure
+    # state — those remain the user's responsibility across threads.)
+    # For typical use cases where a Shifty pipeline is built and run, no
+    # external threading is usually involved by the user.
+  end
 end

data/lib/shifty.rb

@@ -1,4 +1,7 @@
 require_relative "shifty/version"
+require_relative "shifty/errors"
+require_relative "shifty/configuration"
+require_relative "shifty/policy"
 require_relative "shifty/worker"
 require_relative "shifty/gang"
 require_relative "shifty/roster"

data/shifty.gemspec

@@ -20,10 +20,15 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = ">= 3.2"
+
+  spec.add_dependency "ostruct"
+
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", "~> 3.9"
   spec.add_development_dependency "rspec-given", "~> 3.8"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "standard", ">= 1.28.2"
-  spec.add_development_dependency "codeclimate-test-reporter"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "benchmark-ips"
 end

metadata

@@ -1,15 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: shifty
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Joel Helbling
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-19 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -81,7 +94,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.28.2
 - !ruby/object:Gem::Dependency
-  name: codeclimate-test-reporter
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -106,20 +133,43 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".standard.yml"
+- CHANGELOG.md
 - Gemfile
 - Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - _config.yml
+- benchmark/RESULTS.md
+- benchmark/handoff_policies.rb
 - bin/console
 - bin/setup
+- docs/planning/artifacts/.discovery-notes.md
+- docs/planning/artifacts/.round1-specialist-outputs.md
+- docs/planning/artifacts/implementation-decision-log.md
+- docs/planning/artifacts/implementation-iteration-history.md
+- docs/planning/feature-implementation-plan.md
+- docs/planning/handoff-immutability-policies.md
 - docs/shifty/worker.md
+- docs/use_cases.md
+- docs/wiki/Coding-Idioms-Under-Frozen.md
+- docs/wiki/Handoff-Policies.md
+- docs/wiki/Home.md
+- docs/wiki/Migration-Guide-0.6.md
+- docs/wiki/Performance.md
+- docs/wiki/Testing-Workers.md
+- docs/wiki/Worker-Types.md
+- docs/wiki/_Sidebar.md
 - lib/shifty.rb
+- lib/shifty/configuration.rb
 - lib/shifty/dsl.rb
+- lib/shifty/errors.rb
 - lib/shifty/gang.rb
+- lib/shifty/policy.rb
 - lib/shifty/roster.rb
+- lib/shifty/rspec.rb
 - lib/shifty/taggable.rb
+- lib/shifty/testing.rb
 - lib/shifty/version.rb
 - lib/shifty/worker.rb
 - pkg/.gitkeep
@@ -128,7 +178,6 @@ homepage: https://github.com/joelhelbling/shifty
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -136,15 +185,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
-signing_key:
+rubygems_version: 4.0.15
 specification_version: 4
 summary: A functional framework aimed at extremely low coupling
 test_files: []