little_ghost 0.4.0 → 0.5.0

data/lib/little_ghost/artifacts/workspace_store.rb

@@ -0,0 +1,370 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fiddle"
+require "json"
+require "securerandom"
+
+module LittleGhost
+  module Artifacts
+    # Writes immutable artifacts beneath one named Workspace path. The store
+    # applies per-artifact, total-byte, and count limits before exposing a file.
+    # It does not remove files; the Workspace provider owns their lifetime.
+    class WorkspaceStore # :nodoc:
+      ContextControl = Data.define(:cancellation_tokens, :deadlines) do # :nodoc:
+        def check!
+          cancelled = cancellation_tokens.any? do |token|
+            Support::CancellationToken.instance_method(:cancelled?).bind_call(token)
+          end
+          raise CancelledError, "The run was cancelled" if cancelled
+          if deadlines.any? { |deadline| Time.now.to_f >= deadline }
+            raise DeadlineExceededError, "The run deadline was reached"
+          end
+        end
+      end
+
+      PLATFORM_OPEN_FLAGS = case RUBY_PLATFORM
+      when /darwin/
+        {close_on_exec: 0x0100_0000}
+      when /linux/
+        {close_on_exec: 0x0008_0000}
+      end&.freeze # :nodoc:
+      OPENAT = if PLATFORM_OPEN_FLAGS # :nodoc:
+        Fiddle::Function.new(
+          Fiddle::Handle::DEFAULT["openat"],
+          [Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT, Fiddle::TYPE_VARIADIC],
+          Fiddle::TYPE_INT
+        )
+      end
+      UNLINKAT = if PLATFORM_OPEN_FLAGS # :nodoc:
+        Fiddle::Function.new(
+          Fiddle::Handle::DEFAULT["unlinkat"],
+          [Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT],
+          Fiddle::TYPE_INT
+        )
+      end
+
+      # Default maximum size of one artifact: 32 MiB.
+      DEFAULT_MAX_ARTIFACT_BYTES = 32 * 1024 * 1024
+      # Default cumulative store budget: 64 MiB.
+      DEFAULT_MAX_TOTAL_BYTES = 64 * 1024 * 1024
+      # Default maximum number of unique artifacts.
+      DEFAULT_MAX_ARTIFACTS = 100
+
+      # Uses the already-open +workspace+ and a configured named +path+.
+      def initialize(
+        workspace:,
+        path: :artifacts,
+        max_artifact_bytes: DEFAULT_MAX_ARTIFACT_BYTES,
+        max_total_bytes: DEFAULT_MAX_TOTAL_BYTES,
+        max_artifacts: DEFAULT_MAX_ARTIFACTS
+      )
+        raise ToolError, "Artifact workspace path is unavailable" unless workspace
+
+        @workspace = workspace
+        @path = path.to_sym
+        @max_artifact_bytes = positive_integer(max_artifact_bytes, :max_artifact_bytes)
+        @max_total_bytes = positive_integer(max_total_bytes, :max_total_bytes)
+        @max_artifacts = positive_integer(max_artifacts, :max_artifacts)
+        @mutex = Mutex.new
+        @artifacts = {}
+        @total_bytes = 0
+        @namespace = SecureRandom.hex(8).freeze
+        @root, @root_identity = validate_root!
+      end
+
+      # Materializes +data+ once for an identical name, media type, and byte
+      # sequence. Concurrent calls share one descriptor and consume one budget.
+      def write(data:, name: nil, media_type: nil, metadata: {}, context: nil)
+        write_batch([{data:, name:, media_type:, metadata:}], context:).fetch(0)
+      end
+
+      # Atomically materializes an Array of write keyword Hashes. If any new
+      # artifact fails, every file and budget reservation created by this call
+      # is rolled back; previously cached artifacts remain unchanged.
+      def write_batch(entries, context: nil)
+        raise ArgumentError, "artifact batch must be an array" unless entries.is_a?(Array)
+
+        context&.check!
+        control = context_control(context)
+        prepared = entries.map { |entry| prepare_entry(entry) }
+        @mutex.synchronize do
+          control&.check!
+          created = []
+          begin
+            prepared.map do |entry|
+              control&.check!
+              digest = entry.fetch(:digest)
+              next @artifacts.fetch(digest) if @artifacts.key?(digest)
+
+              reserve!(entry.fetch(:data).bytesize)
+              record = entry.merge(materialized: false)
+              created << record
+              artifact = materialize(**entry, control:)
+              @artifacts[digest] = artifact
+              record[:materialized] = true
+              artifact
+            end.freeze
+          rescue => error
+            rollback_batch(created, error:)
+            raise
+          end
+        end
+      end
+
+      # Number of bytes reserved by unique artifacts in this store.
+      def total_bytes = @mutex.synchronize { @total_bytes }
+
+      # Number of unique artifacts in this store.
+      def size = @mutex.synchronize { @artifacts.length }
+
+      private
+
+      def prepare_entry(entry)
+        raise ArgumentError, "artifact batch entries must be hashes" unless entry.is_a?(Hash)
+
+        data = String(entry.fetch(:data)).b
+        if data.bytesize > @max_artifact_bytes
+          raise ToolError, "Artifact exceeds the #{@max_artifact_bytes}-byte limit"
+        end
+        media_type = entry[:media_type]
+        name = normalized_name(entry[:name], media_type:)
+        metadata = DataMap.new(entry.fetch(:metadata, {}))
+        digest = Digest::SHA256.hexdigest(
+          [name, media_type, JSON.generate(metadata), data].join("\0")
+        )
+        {
+          data:,
+          digest:,
+          file_name: "#{@namespace}-#{digest}-#{name}",
+          name:,
+          media_type:,
+          metadata:
+        }
+      end
+
+      def context_control(context)
+        values = if context.instance_of?(RunContext)
+          {
+            cancellation_tokens: [context.cancellation_token],
+            deadlines: [context.deadline]
+          }
+        elsif context&.respond_to?(:artifact_control_values)
+          context.artifact_control_values
+        else
+          return
+        end
+        tokens = Array(values.fetch(:cancellation_tokens)).select do |token|
+          token.instance_of?(Support::CancellationToken)
+        end.freeze
+        deadlines = Array(values.fetch(:deadlines)).compact.filter_map do |deadline|
+          deadline.to_f if deadline.instance_of?(Time)
+        end.freeze
+        ContextControl.new(cancellation_tokens: tokens, deadlines: deadlines)
+      rescue KeyError, NoMethodError, TypeError
+        nil
+      end
+
+      def reserve!(bytes)
+        raise ToolError, "Artifact count exceeds the #{@max_artifacts}-item limit" if @artifacts.length >= @max_artifacts
+        if @total_bytes + bytes > @max_total_bytes
+          raise ToolError, "Artifacts exceed the #{@max_total_bytes}-byte total limit"
+        end
+
+        @total_bytes += bytes
+      end
+
+      def materialize(data:, digest:, file_name:, name:, media_type:, metadata:, control:)
+        directory = open_root
+        path = File.join(@root, file_name)
+        created = false
+        descriptor = open_at(directory.fileno, file_name, write_flags, 0o600)
+        created = true
+        file = file_for_descriptor(descriptor, "w")
+        begin
+          file.binmode
+          file.write(data)
+          file.flush
+          file.chmod(0o600)
+          validate_regular_file!(file)
+        ensure
+          file.close
+        end
+        control&.check!
+        @workspace.validate!
+        Artifact.materialized(
+          reference: @workspace.reference(path),
+          name:,
+          media_type:,
+          bytes: data.bytesize,
+          metadata:
+        )
+      rescue Errno::EEXIST
+        raise ToolError, "Artifact destination already exists"
+      rescue Errno::ELOOP
+        raise ToolError, "Artifact destination is unsafe"
+      rescue => error
+        cleanup_created_file(directory, file_name, error:) if created
+        raise
+      ensure
+        directory&.close
+      end
+
+      def rollback_batch(records, error:)
+        cleanup_errors = []
+        records.reverse_each do |record|
+          unless record.fetch(:materialized)
+            @total_bytes -= record.fetch(:data).bytesize
+            next
+          end
+
+          begin
+            unlink_entry(record.fetch(:file_name))
+            @artifacts.delete(record.fetch(:digest))
+            @total_bytes -= record.fetch(:data).bytesize
+          rescue => cleanup_error
+            cleanup_errors << cleanup_error
+          end
+        end
+        return if cleanup_errors.empty?
+
+        raise CleanupError,
+          "Artifact batch rollback failed after #{error.class} (#{cleanup_errors.first.class})"
+      end
+
+      def unlink_entry(entry)
+        directory = open_root
+        unlink_at(directory.fileno, entry)
+      ensure
+        directory&.close
+      end
+
+      def validate_root!
+        unless OPENAT && UNLINKAT
+          raise ToolError, "Secure artifact storage is unavailable on this platform"
+        end
+
+        @workspace.validate!
+        root = @workspace.path(@path)
+        stat = File.lstat(root)
+        real = File.realpath(root)
+        workspace_real = File.realpath(@workspace.root)
+        location_safe = !beneath?(root, @workspace.root) || beneath?(real, workspace_real)
+        unless stat.directory? && !stat.symlink? && location_safe
+          raise ToolError, "Artifact workspace path is unsafe"
+        end
+
+        directory = File.open(root, directory_flags)
+        descriptor_stat = directory.stat
+        unless descriptor_stat.directory?
+          raise ToolError, "Artifact workspace path is unsafe"
+        end
+        @workspace.validate!
+        [root.freeze, [descriptor_stat.dev, descriptor_stat.ino].freeze].freeze
+      rescue KeyError, Errno::ENOENT, Errno::ENOTDIR
+        raise ToolError, "Artifact workspace path is unavailable"
+      rescue SystemCallError
+        raise ToolError, "Artifact workspace path is unavailable"
+      ensure
+        directory&.close
+      end
+
+      def open_root
+        @workspace.validate!
+        directory = File.open(@root, directory_flags)
+        stat = directory.stat
+        unless stat.directory? && [stat.dev, stat.ino] == @root_identity
+          raise ToolError, "Artifact workspace path changed after initialization"
+        end
+        @workspace.validate!
+        directory
+      rescue Errno::ENOENT, Errno::ELOOP, Errno::ENOTDIR
+        directory&.close
+        raise ToolError, "Artifact workspace path changed after initialization"
+      rescue
+        directory&.close
+        raise
+      end
+
+      def open_at(directory, entry, flags, permissions)
+        descriptor = OPENAT.call(directory, entry, flags, Fiddle::TYPE_UINT, permissions)
+        raise SystemCallError.new("openat", Fiddle.last_error) if descriptor.negative?
+
+        descriptor
+      end
+
+      def unlink_at(directory, entry)
+        result = UNLINKAT.call(directory, entry, 0)
+        raise SystemCallError.new("unlinkat", Fiddle.last_error) if result.negative?
+      end
+
+      def file_for_descriptor(descriptor, mode)
+        File.for_fd(descriptor, mode, autoclose: true)
+      rescue
+        IO.for_fd(descriptor).close
+        raise
+      end
+
+      def validate_regular_file!(file)
+        stat = file.stat
+        unless stat.file? && stat.nlink == 1 && (stat.mode & 0o777) == 0o600
+          raise ToolError, "Artifact destination is unsafe"
+        end
+      end
+
+      def cleanup_created_file(directory, entry, error:)
+        unlink_at(directory.fileno, entry)
+      rescue => cleanup_error
+        raise CleanupError,
+          "Artifact cleanup failed after #{error.class} (#{cleanup_error.class})"
+      end
+
+      def write_flags
+        flags = File::WRONLY | File::CREAT | File::EXCL
+        flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
+        flags |= File::NONBLOCK if defined?(File::NONBLOCK)
+        flags |= PLATFORM_OPEN_FLAGS.fetch(:close_on_exec)
+        flags
+      end
+
+      def directory_flags
+        flags = File::RDONLY
+        flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
+        flags |= File::NONBLOCK if defined?(File::NONBLOCK)
+        flags |= PLATFORM_OPEN_FLAGS.fetch(:close_on_exec)
+        flags
+      end
+
+      def normalized_name(name, media_type:)
+        value = name.to_s
+        value = default_name(media_type) if value.empty?
+        value = File.basename(value).scrub.gsub(/[^A-Za-z0-9._-]/, "_")
+        value = "artifact" if value.empty? || value == "." || value == ".."
+        value.byteslice(0, 120)
+      end
+
+      def default_name(media_type)
+        extension = {
+          "application/json" => ".json",
+          "text/plain" => ".txt",
+          "text/markdown" => ".md",
+          "image/png" => ".png",
+          "image/jpeg" => ".jpg",
+          "application/pdf" => ".pdf"
+        }[media_type.to_s]
+        "artifact#{extension}"
+      end
+
+      def beneath?(candidate, base)
+        candidate == base || candidate.start_with?(base.end_with?(File::SEPARATOR) ? base : "#{base}#{File::SEPARATOR}")
+      end
+
+      def positive_integer(value, name)
+        integer = Integer(value)
+        raise ArgumentError, "#{name} must be positive" unless integer.positive?
+
+        integer
+      end
+    end
+  end
+end

data/lib/little_ghost/artifacts.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module LittleGhost
+  # Stores files from Run input and Tool results, then presents them to agents
+  # as bounded images, documents, previews, or Workspace references.
+  module Artifacts
+  end
+end
+
+require_relative "artifacts/workspace_store"
+require_relative "artifacts/presentation_budget"

data/lib/little_ghost/assembly.rb

@@ -14,18 +14,19 @@ module LittleGhost
   #   agent_run.response
   #   graph_run.response
   #
-  # Named subclasses are the usual form. +to_builder+ creates a mutable dynamic
-  # definition seeded by the class, while +definition+ returns the immutable
-  # snapshot used for one execution. Composite results expose Assembly::Step
-  # records through RunResult#trajectory.
+  # Applications normally subclass Agent, Workflow, Swarm, or Graph rather
+  # than Assembly directly. Each standalone call returns a top-level Run.
+  # Participants called inside another Assembly return a RunResult to their
+  # parent.
   #
-  # A standalone assembly owns a top-level Run. An assembly built by a Runtime
-  # participates in the existing run and returns a RunResult. Applications
-  # normally subclass Agent, Workflow, Swarm, or Graph rather than Assembly
-  # directly. Standalone calls automatically reuse the active Configuration's
-  # shared Runtime while keeping each Run and its resources independent.
+  # == Advanced construction
   #
-  # == What each calling form returns
+  # Named subclasses are the usual form. +to_builder+ creates a mutable
+  # definition for applications that discover participants at runtime.
+  # +definition+ returns the fixed snapshot used by one execution. Composite
+  # results record their steps in RunResult#trajectory.
+  #
+  # == Return values
   #
   # [<tt>CustomerSupportAgent.ask(...)</tt>]
   #   A named class creates and returns a top-level Run.
@@ -56,6 +57,11 @@ module LittleGhost
       #
       # Enumeration yields StreamEvent objects and returns the terminal Run.
       # The same Invocation fields accepted by .ask may be supplied as +options+.
+      # Composite assemblies also emit an +:agent_stream+ event for every
+      # normalized event from every Agent in the run, including intermediate
+      # and nested participants. Set <tt>include_agent_events: false</tt> to
+      # keep only the ordinary public stream. A standalone Agent retains its
+      # ordinary stream by default and accepts +true+ to opt in.
       def stream_ask(message, **options)
         snapshot = definition
         stream = nil
@@ -144,6 +150,7 @@ module LittleGhost
     attr_reader :workspace
     # Sandbox supplied to this Assembly, when present.
     attr_reader :sandbox
+    attr_reader :agent_stream_path # :nodoc:
 
     def initialize(run: nil, runtime: nil, workspace: nil, sandbox: nil, standalone: run.nil?) # :nodoc:
       @run = run
@@ -154,28 +161,39 @@ module LittleGhost
       @assembly_mutex = Mutex.new
       @assembly_closed = false
       @active_assemblies = []
+      @agent_stream_path = [].freeze
+    end
+
+    def bind_agent_stream_path(path) # :nodoc:
+      @agent_stream_path = Array(path).dup.freeze
+      self
     end
 
     # Builds the top-level Run used by a standalone assembly.
-    def build_run(payload) # :nodoc:
+    def build_run(payload = nil, include_agent_events_by_default: false, **payload_options) # :nodoc:
+      payload = entrypoint_payload(payload, payload_options) unless payload_options.empty?
       payload = payload.dup if payload.is_a?(Hash)
       cancellation_token = if payload.is_a?(Hash)
         payload.delete(:cancellation_token) || payload.delete("cancellation_token")
       end
-      source_class = self.class.respond_to?(:assembly_source_class) ? self.class.assembly_source_class : self.class
+      source_class = run_entrypoint_class
       options = {entrypoint_class: source_class}
       options[:execution_class] = self.class unless source_class.equal?(self.class)
       options[:agent_class] = source_class if is_a?(Agent)
       options[:cancellation_token] = cancellation_token if cancellation_token
       options[:workspace] = workspace if workspace
       options[:sandbox] = sandbox if sandbox
+      options[:include_agent_events_by_default] = true if include_agent_events_by_default
       runtime.build_run(payload, **options)
     end
 
     # Starts +payload+ on a supervised worker and returns an Execution.
+    # Composite assemblies include contextual +:agent_stream+ events in the
+    # consumer by default. Set +include_agent_events+ to +false+ in +payload+
+    # to keep only the ordinary public stream.
     def start_execution(payload, &event_consumer)
       ensure_standalone!
-      Execution.start(build_run(payload), &event_consumer)
+      Execution.start(build_stream_run(payload), &event_consumer)
     end
 
     # Runs +input+ to completion.
@@ -204,13 +222,17 @@ module LittleGhost
     #
     # A standalone stream returns its terminal Run after enumeration. A
     # run-scoped stream finishes with an +invocation_stop+ event containing its
-    # RunResult.
+    # RunResult. A standalone composite Assembly receives contextual
+    # +:agent_stream+ events from every Agent in the Run by default and may set
+    # <tt>include_agent_events: false</tt> to omit them. A standalone Agent may
+    # set the option to +true+ to include its contextual wrapper.
     def stream_ask(message, **options)
       if standalone?
         options[:deadline_at] = options.delete(:deadline) if options.key?(:deadline)
+        payload = entrypoint_payload(message, options)
         stream = nil
         return Enumerator.new do |events|
-          stream ||= build_run(entrypoint_payload(message, options)).each
+          stream ||= build_stream_run(payload).each
           stream.each { |event| events << event }
         end
       end
@@ -245,7 +267,7 @@ module LittleGhost
           target = if assembly.is_a?(Agent)
             assembly
           elsif assembly.run
-            assembly.runtime.build_assembly(assembly.class, run: assembly.run)
+            assembly.send(:build_tool_assembly)
           else
             assembly.class.new(runtime: assembly.runtime)
           end
@@ -254,7 +276,7 @@ module LittleGhost
             context: context&.state || {},
             cancellation_token: context&.cancellation_token || Support::CancellationToken.new,
             deadline: context&.deadline,
-            parent_operation_id: assembly.run&.operation_id
+            parent_operation_id: context&.agent_operation_id || assembly.run&.operation_id
           }
           if target.is_a?(Agent)
             options[:interjection_metadata] = context&.interjection_metadata
@@ -338,6 +360,30 @@ module LittleGhost
 
     private
 
+    def build_stream_run(payload) # :nodoc:
+      return build_run(payload) if is_a?(Agent)
+
+      build_run(payload, include_agent_events_by_default: true)
+    end
+
+    def run_entrypoint_class # :nodoc:
+      return self.class.assembly_source_class if self.class.respond_to?(:assembly_source_class)
+
+      self.class
+    end
+
+    def build_tool_assembly
+      builder = runtime.method(:build_assembly)
+      accepts_path = builder.parameters.any? do |kind, name|
+        kind == :keyrest || (%i[key keyreq].include?(kind) && name == :agent_stream_path)
+      end
+      options = {run:}
+      options[:agent_stream_path] = agent_stream_path if accepts_path
+      child = builder.call(self.class, **options)
+      child.bind_agent_stream_path(agent_stream_path) if child.respond_to?(:bind_agent_stream_path)
+      child
+    end
+
     def ensure_standalone!
       raise Error, "Only a standalone assembly can start an execution" unless standalone?
     end