little_ghost 0.2.0 → 0.3.0

data/lib/little_ghost/assembly_execution.rb

@@ -0,0 +1,452 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module LittleGhost
+  class Assembly
+    MAX_STEP_OUTPUT_BYTES = 64 * 1024 # :nodoc:
+    MAX_STEP_EVENTS = 10_000 # :nodoc:
+    MAX_STEP_EVENT_BYTES = 10 * 1024 * 1024 # :nodoc:
+
+    # One attempt to execute an assembly step.
+    Attempt = Data.define(:number, :status, :started_at, :finished_at, :usage, :error) do
+      def initialize(number:, status:, started_at:, finished_at:, usage: Usage.new, error: nil)
+        super(number:, status: status.to_sym, started_at:, finished_at:, usage:, error: error&.to_s&.freeze)
+      end
+    end
+
+    # One logical child execution in a composite assembly.
+    Step = Data.define(
+      :id, :parent_id, :predecessor_ids, :branch_id, :participant,
+      :assembly_id, :assembly_kind, :status, :attempts, :usage, :output, :output_truncated
+    ) do
+      def initialize(id:, participant:, assembly_id:, assembly_kind:, status:, attempts:, usage:,
+        parent_id: nil, predecessor_ids: [], branch_id: nil, output: nil, output_truncated: false)
+        super(
+          id: id.to_s.freeze,
+          parent_id: parent_id&.to_s&.freeze,
+          predecessor_ids: Array(predecessor_ids).map { |value| value.to_s.freeze }.freeze,
+          branch_id: branch_id&.to_s&.freeze,
+          participant: participant.to_s.freeze,
+          assembly_id: assembly_id.to_s.freeze,
+          assembly_kind: assembly_kind.to_sym,
+          status: status.to_sym,
+          attempts: Array(attempts).freeze,
+          usage:,
+          output:,
+          output_truncated: output_truncated == true
+        )
+      end
+    end
+
+    # Immutable queries over the steps returned by one assembly invocation.
+    class Trajectory
+      include Enumerable
+
+      # Immutable steps in execution order.
+      attr_reader :steps
+
+      # Builds query indexes for +steps+.
+      def initialize(steps)
+        @steps = Array(steps).freeze
+        @by_id = @steps.to_h { |step| [step.id, step] }.freeze
+        freeze
+      end
+
+      # Iterates through steps in execution order.
+      def each(&block) = steps.each(&block)
+      # Finds one step by its stable ID.
+      def step(id) = @by_id[id.to_s]
+      # Returns steps whose parent is +id+.
+      def children(id) = steps.select { |item| item.parent_id == id.to_s }.freeze
+      # Returns attempts belonging to one step.
+      def attempts_for(id) = step(id)&.attempts || [].freeze
+
+      # Returns declared predecessor-to-step ID pairs.
+      def transitions
+        steps.flat_map do |item|
+          item.predecessor_ids.map { |predecessor| [predecessor, item.id].freeze }
+        end.freeze
+      end
+
+      # Indicates whether attempts from two steps overlapped in time.
+      def concurrent?(first_id, second_id)
+        first = step(first_id)
+        second = step(second_id)
+        return false unless first && second
+
+        first.attempts.any? do |left|
+          second.attempts.any? do |right|
+            left.started_at < right.finished_at && right.started_at < left.finished_at
+          end
+        end
+      end
+    end
+
+    StepExecution = Data.define(:result, :step, :events, :transition) # :nodoc:
+
+    private
+
+    def execute_assembly_step(
+      reference:,
+      participant:,
+      input:,
+      history:,
+      context:,
+      cancellation_token:,
+      deadline:,
+      settings:,
+      template_locals:,
+      template_paths:,
+      parent_operation_id:,
+      policies: {},
+      parent_id: nil,
+      predecessor_ids: [],
+      branch_id: nil,
+      checkpoint: nil,
+      build_options: {},
+      step_id: SecureRandom.uuid
+    )
+      policy = normalize_step_policy(policies)
+      attempts = []
+      usage = Usage.new
+
+      (policy.fetch(:retries) + 1).times do |index|
+        attempt_number = index + 1
+        started_at = Time.now
+        child = if build_options.any? && runtime.respond_to?(:build_agent)
+          runtime.build_agent(reference, run:, **build_options)
+        else
+          runtime.build_assembly(reference, run:, **build_options)
+        end
+        child_deadline = step_deadline(deadline, policy[:timeout])
+        attempt_events = []
+        attempt_event_bytes = 0
+        result = nil
+        attempt_usage = Usage.new
+        begin
+          with_active_assembly(child) do
+            child.stream(
+              input,
+              history:,
+              context: isolated_assembly_state(context),
+              cancellation_token:,
+              deadline: child_deadline,
+              settings:,
+              template_locals: template_locals.merge(runtime.template_locals(run:, agent: child)),
+              template_paths:,
+              parent_operation_id:,
+              checkpoint:
+            ).each do |event|
+              result = event.data[:result] if event.type == :invocation_stop
+              attempt_usage = event.data[:usage] || attempt_usage if event.type == :invocation_error
+              attempt_event_bytes = buffer_assembly_event!(attempt_events, event, bytes: attempt_event_bytes)
+            end
+          end
+          raise ProtocolError, "assembly step #{participant.inspect} did not return a result" unless result
+
+          attempt_usage = result.usage
+          usage += attempt_usage
+          attempts << Attempt.new(
+            number: attempt_number,
+            status: :completed,
+            started_at:,
+            finished_at: Time.now,
+            usage: attempt_usage
+          )
+          output, truncated = projected_step_output(result.output)
+          nested_steps = reparent_steps(result.steps, step_id)
+          step = Step.new(
+            id: step_id,
+            parent_id:,
+            predecessor_ids:,
+            branch_id:,
+            participant:,
+            assembly_id: child.class.respond_to?(:assembly_id) ? child.class.assembly_id : participant,
+            assembly_kind: child.class.respond_to?(:assembly_kind) ? child.class.assembly_kind : :assembly,
+            status: :completed,
+            attempts:,
+            usage:,
+            output:,
+            output_truncated: truncated
+          )
+          combined = copy_run_result(result, steps: [step, *nested_steps])
+          events = attempt_events.map do |event|
+            (event.type == :invocation_stop) ? StreamEvent.build(event.type, **event.data.merge(result: combined)) : event
+          end
+          transition = child.assembly_transition if child.respond_to?(:assembly_transition)
+          return StepExecution.new(result: combined, step:, events: events.freeze, transition:)
+        rescue => error
+          if own_step_timeout?(error, deadline, child_deadline, policy[:timeout])
+            error = AssemblyStepTimeoutError.new("assembly step #{participant.inspect} exceeded its timeout")
+          end
+          attempt_usage = error_usage(attempt_events, attempt_usage)
+          usage += attempt_usage
+          attempts << Attempt.new(
+            number: attempt_number,
+            status: :failed,
+            started_at:,
+            finished_at: Time.now,
+            usage: attempt_usage,
+            error: error.class.name
+          )
+          retrying = retry_step?(error, policy, index)
+          annotate_step_error(error, step_id:, attempts:, usage:, child:, participant:)
+          if block_given?
+            yield StreamEvent.build(
+              :assembly_step_error,
+              step_id:,
+              participant: participant.to_s,
+              attempt: attempt_number,
+              error_type: error.class.name,
+              usage:,
+              terminal: !retrying
+            )
+          end
+          raise error unless retrying
+
+          if block_given?
+            yield StreamEvent.build(
+              :assembly_step_retry,
+              step_id:,
+              participant: participant.to_s,
+              attempt: attempt_number,
+              error_type: error.class.name,
+              delay: policy.fetch(:retry_delay)
+            )
+          end
+          wait_for_retry(policy.fetch(:retry_delay), cancellation_token, deadline)
+        ensure
+          begin
+            child.close
+          rescue => cleanup_error
+            cleanup_usage = usage
+            annotate_step_error(
+              cleanup_error,
+              step_id:,
+              attempts:,
+              usage: cleanup_usage,
+              child:,
+              participant:
+            )
+            if block_given?
+              yield StreamEvent.build(
+                :assembly_step_error,
+                step_id:,
+                participant: participant.to_s,
+                attempt: attempt_number,
+                error_type: cleanup_error.class.name,
+                usage: cleanup_usage,
+                terminal: true
+              )
+            end
+            raise
+          end
+        end
+      end
+    end
+
+    def normalize_step_policy(values)
+      self.class.validate_step_policy!(values)
+    end
+
+    class << self
+      def validate_step_policy!(values) # :nodoc:
+        values = values.compact
+        retries = Integer(values.fetch(:retries, 0))
+        raise ArgumentError, "retries must be at least 0" if retries.negative?
+
+        retry_on = Array(values[:retry_on])
+        if retries.positive? && retry_on.empty?
+          raise ArgumentError, "retry_on is required when retries is greater than 0"
+        end
+        unless retry_on.all? { |error| error.is_a?(Class) && error <= Exception }
+          raise ArgumentError, "retry_on must contain exception classes"
+        end
+        timeout = Float(values[:timeout]) if values[:timeout]
+        retry_delay = Float(values.fetch(:retry_delay, 0))
+        raise ArgumentError, "timeout must be positive" if timeout && (!timeout.positive? || !timeout.finite?)
+        raise ArgumentError, "retry_delay must be non-negative" if retry_delay.negative? || !retry_delay.finite?
+
+        {retries:, retry_on: retry_on.freeze, timeout:, retry_delay:}.freeze
+      end
+    end
+
+    def retry_step?(error, policy, index)
+      return false if index >= policy.fetch(:retries)
+      return false if error.is_a?(CancelledError) || error.is_a?(CleanupError)
+
+      policy.fetch(:retry_on).any? { |type| error.is_a?(type) }
+    end
+
+    def step_deadline(parent_deadline, timeout)
+      local = Time.now + timeout if timeout
+      [parent_deadline, local].compact.min
+    end
+
+    def own_step_timeout?(error, parent_deadline, child_deadline, timeout)
+      error.is_a?(DeadlineExceededError) && timeout && child_deadline && child_deadline != parent_deadline
+    end
+
+    def wait_for_retry(delay, cancellation_token, deadline)
+      return if delay.zero?
+
+      stop_at = Time.now + delay
+      loop do
+        cancellation_token.raise_if_cancelled!
+        raise DeadlineExceededError, "The run deadline was reached" if deadline && Time.now >= deadline
+
+        remaining = stop_at - Time.now
+        break unless remaining.positive?
+
+        cancellation_token.wait([remaining, 0.05].min)
+      end
+    end
+
+    def error_usage(events, fallback)
+      events.reverse_each do |event|
+        return event.data[:usage] if event.type == :invocation_error && event.data[:usage]
+      end
+      fallback
+    end
+
+    def annotate_step_error(error, step_id:, attempts:, usage:, child:, participant:)
+      error.instance_variable_set(:@little_ghost_step_id, step_id)
+      error.instance_variable_set(:@little_ghost_step_attempts, attempts.dup.freeze)
+      error.instance_variable_set(:@little_ghost_step_usage, usage)
+      error.instance_variable_set(
+        :@little_ghost_step_assembly_id,
+        child.class.respond_to?(:assembly_id) ? child.class.assembly_id : participant.to_s
+      )
+      error.instance_variable_set(
+        :@little_ghost_step_assembly_kind,
+        child.class.respond_to?(:assembly_kind) ? child.class.assembly_kind : :assembly
+      )
+    end
+
+    def projected_step_output(output)
+      text = output.is_a?(String) ? output : JSON.generate(output)
+      return [deep_frozen_assembly_value(output), false] if text.bytesize <= MAX_STEP_OUTPUT_BYTES
+
+      [nil, true]
+    rescue JSON::GeneratorError, TypeError
+      [nil, true]
+    end
+
+    def reparent_steps(steps, parent_id)
+      Array(steps).map do |step|
+        next step if step.parent_id
+
+        Step.new(**step.to_h.merge(parent_id:))
+      end
+    end
+
+    def copy_run_result(result, usage: result.usage, steps: result.steps)
+      RunResult.new(
+        message: result.message,
+        stop_reason: result.stop_reason,
+        usage:,
+        messages: result.messages,
+        state: result.state,
+        structured_result: result.structured_result,
+        steps:
+      )
+    end
+
+    def isolated_assembly_state(value)
+      case value
+      when Hash
+        value.to_h { |key, item| [isolated_assembly_state(key), isolated_assembly_state(item)] }
+      when Array
+        value.map { |item| isolated_assembly_state(item) }
+      when String
+        value.dup
+      when NilClass, TrueClass, FalseClass, Numeric, Symbol
+        value
+      else
+        raise ArgumentError, "assembly context must contain only JSON-like state"
+      end
+    end
+
+    def deep_frozen_assembly_value(value)
+      copied = isolated_assembly_state(value)
+      case copied
+      when Hash
+        copied.each do |key, item|
+          deep_freeze_assembly_value(key)
+          deep_freeze_assembly_value(item)
+        end
+      when Array
+        copied.each { |item| deep_freeze_assembly_value(item) }
+      end
+      copied.freeze
+    end
+
+    def deep_freeze_assembly_value(value)
+      case value
+      when Hash
+        value.each do |key, item|
+          deep_freeze_assembly_value(key)
+          deep_freeze_assembly_value(item)
+        end
+      when Array
+        value.each { |item| deep_freeze_assembly_value(item) }
+      end
+      value.freeze
+    end
+
+    def buffer_assembly_event!(buffer, event, bytes:)
+      raise AssemblyLimitError, "assembly step emitted too many events" if buffer.length >= MAX_STEP_EVENTS
+
+      bytes += buffered_assembly_size(event)
+      raise AssemblyLimitError, "assembly step emitted too much event data" if bytes > MAX_STEP_EVENT_BYTES
+
+      buffer << event
+      bytes
+    end
+
+    def buffered_assembly_size(value, ancestors = {}, depth = 0)
+      return MAX_STEP_EVENT_BYTES + 1 if depth > 32
+
+      identity = value.object_id
+      return 0 if ancestors.key?(identity)
+
+      case value
+      when String
+        value.bytesize
+      when Hash
+        ancestors[identity] = true
+        value.sum do |key, item|
+          buffered_assembly_size(key, ancestors, depth + 1) + buffered_assembly_size(item, ancestors, depth + 1)
+        end
+      when Array
+        ancestors[identity] = true
+        value.sum { |item| buffered_assembly_size(item, ancestors, depth + 1) }
+      when Data
+        ancestors[identity] = true
+        value.members.sum { |member| buffered_assembly_size(value.public_send(member), ancestors, depth + 1) }
+      else
+        64
+      end
+    ensure
+      ancestors.delete(identity) if identity
+    end
+
+    def enqueue_assembly_event(queue, value, cancellation_token)
+      loop do
+        cancellation_token.raise_if_cancelled!
+        queue.push(value, true)
+        return
+      rescue ThreadError
+        cancellation_token.wait(0.01)
+      end
+    end
+
+    def enqueue_assembly_terminal(queue, value)
+      queue.push(value, true)
+    rescue ThreadError
+      queue.pop(true)
+      retry
+    end
+  end
+end

data/lib/little_ghost/errors.rb

@@ -13,6 +13,14 @@ module LittleGhost
   class CredentialError < ConfigurationError; end
   # Raised when an invocation payload or operation is invalid.
   class InvocationError < Error; end
+  # Base class for failures coordinating one or more assemblies.
+  class AssemblyError < Error; end
+  # Raised when an assembly exceeds its configured execution bound.
+  class AssemblyLimitError < AssemblyError; end
+  # Raised when an assembly cannot choose one valid next participant.
+  class AssemblyRoutingError < AssemblyError; end
+  # Raised when one assembly step exceeds its local timeout.
+  class AssemblyStepTimeoutError < AssemblyError; end
   # Raised when an invocation contains an unsupported input form.
   class UnsupportedInputError < InvocationError; end
   # Base class for provider response and protocol failures.

data/lib/little_ghost/execution.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module LittleGhost
+  # Runs one dormant Run in a supervised worker while the caller remains free to
+  # serve health checks, deliver interruptions, or coordinate process shutdown.
+  #
+  #   execution = agent.start_execution(message: "Investigate transfer 481") do |event|
+  #     event_buffer << event
+  #   end
+  #
+  #   execution.interrupt_response(message: "Include the latest ledger entry")
+  #   execution.wait(deadline: Time.now + 30)
+  #   execution.run.completed? # => true
+  #
+  # An execution owns its worker. The worker receives a snapshot of the caller's
+  # request-scoped ExecutionState. The Run continues to own its workspace,
+  # sandbox, session, entrypoint, and registered resources. +close+ requests
+  # cooperative cancellation and waits for both the worker and in-flight
+  # interruption calls.
+  class Execution
+    # The supervised Run and an exception raised outside the Run's ordinary
+    # terminal outcome, such as event delivery or cleanup failure.
+    attr_reader :run
+
+    class << self
+      # Starts +run+ immediately and returns its supervising Execution.
+      #
+      # The optional block receives each StreamEvent on the worker thread. It
+      # must be safe to call from that thread and should not retain sensitive
+      # event content longer than the application requires.
+      def start(run, &event_consumer)
+        new(run, event_consumer:).send(:start)
+      end
+    end
+
+    def initialize(run, event_consumer: nil) # :nodoc:
+      raise ArgumentError, "run must be a LittleGhost::Run" unless run.is_a?(Run)
+      unless event_consumer.nil? || event_consumer.respond_to?(:call)
+        raise ArgumentError, "event consumer must be callable"
+      end
+
+      @run = run
+      @event_consumer = event_consumer
+      @state = :pending
+      @error = nil
+      @mutex = Mutex.new
+      @condition = ConditionVariable.new
+      @active_interruptions = 0
+      @closing = false
+      @execution_state = ExecutionState.capture
+    end
+
+    # Returns +:pending+, +:running+, or +:finished+.
+    def state
+      @mutex.synchronize { @state }
+    end
+
+    # Returns an event-delivery or cleanup exception raised by the worker.
+    def error
+      @mutex.synchronize { @error }
+    end
+
+    # Indicates that the worker or an interruption call is still active.
+    def active?
+      @mutex.synchronize { @state != :finished || @active_interruptions.positive? }
+    end
+
+    # Indicates that the worker and all interruption calls have finished.
+    def finished?
+      !active?
+    end
+
+    # Prepares and delivers one interruption to the active run.
+    #
+    # +payload+ may be a message or a Hash containing +message+ and the options
+    # accepted by Run#interrupt_response. Runtime hooks receive the Hash before
+    # delivery, allowing them to materialize trusted application attachments.
+    # Calls may overlap, but +close+ prevents new calls and waits for calls that
+    # have already begun.
+    def interrupt_response(payload = nil, **options)
+      if payload.nil? && options.key?(:message)
+        payload = options.delete(:message)
+      end
+      interruption_started = false
+      begin_interruption!
+      interruption_started = true
+      run.interrupt_response_with do
+        prepared = run.prepare_interruption(interruption_payload(payload, options))
+        interruption_arguments(prepared, options)
+      end
+    ensure
+      finish_interruption! if interruption_started
+    end
+
+    # Requests cooperative cancellation and returns +self+.
+    def cancel
+      run.cancellation_token.cancel
+      self
+    end
+
+    # Waits for the worker and in-flight interruptions, then returns the Run.
+    #
+    # +deadline+ is an absolute Time. Reaching it raises DeadlineExceededError
+    # without cancelling the run. An event-delivery or cleanup failure raised by
+    # the worker is re-raised after all supervised work finishes.
+    def wait(deadline: nil)
+      worker = wait_until_finished(deadline:)
+      worker.join
+      caught = error
+      raise caught if caught
+
+      run
+    end
+
+    # Prevents new interruptions, requests cancellation, and waits for shutdown.
+    # The operation is idempotent. +deadline+ has the same meaning as in #wait.
+    def close(deadline: nil)
+      @mutex.synchronize { @closing = true }
+      cancel
+      wait(deadline:)
+    end
+
+    protected
+
+    def start # :nodoc:
+      @mutex.synchronize do
+        raise Error, "execution has already started" unless @state == :pending
+
+        @state = :running
+        @worker = Thread.new { execute }
+      end
+      self
+    end
+
+    private
+
+    def execute
+      ExecutionState.with(@execution_state) do
+        run.each { |event| @event_consumer&.call(event) }
+      end
+    rescue => caught
+      @mutex.synchronize { @error = caught }
+    ensure
+      @mutex.synchronize do
+        @state = :finished
+        @condition.broadcast
+      end
+    end
+
+    def begin_interruption!
+      @mutex.synchronize do
+        raise AgentInterruptError, "Execution is closing" if @closing
+        raise AgentInterruptError, "Execution has already finished" if @state == :finished
+
+        @active_interruptions += 1
+      end
+    end
+
+    def finish_interruption!
+      @mutex.synchronize do
+        @active_interruptions -= 1 if @active_interruptions.positive?
+        @condition.broadcast
+      end
+    end
+
+    def interruption_payload(payload, options)
+      values = payload.is_a?(Hash) ? payload.dup : {message: payload}
+      options.each { |key, value| values[key] = value }
+      values
+    end
+
+    def interruption_arguments(prepared, fallback)
+      unless prepared.is_a?(Hash)
+        return [prepared, fallback.slice(:interruption_id, :batch_key, :metadata, :cancellation_token, :deadline)]
+      end
+
+      values = prepared.transform_keys(&:to_sym)
+      message = values.delete(:message) { raise ArgumentError, "prepared interruption must include a message" }
+      allowed = values.slice(:interruption_id, :batch_key, :metadata, :cancellation_token, :deadline)
+      [message, fallback.slice(:interruption_id, :batch_key, :metadata, :cancellation_token, :deadline).merge(allowed)]
+    end
+
+    def wait_until_finished(deadline:)
+      monotonic_deadline = if deadline
+        monotonic_time + [deadline - Time.now, 0].max
+      end
+      @mutex.synchronize do
+        until @state == :finished && @active_interruptions.zero?
+          if monotonic_deadline
+            remaining = monotonic_deadline - monotonic_time
+            raise DeadlineExceededError, "Execution did not finish before the wait deadline" unless remaining.positive?
+
+            @condition.wait(@mutex, remaining)
+          else
+            @condition.wait(@mutex)
+          end
+        end
+        @worker
+      end
+    end
+
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end