little_ghost 0.5.0 → 0.7.0

data/lib/little_ghost/subagents/manager.rb

@@ -3,6 +3,7 @@
 require "securerandom"
 require "base64"
 require "digest"
+require "monitor"
 require "time"
 require_relative "definition"
 require_relative "control_tool"
@@ -37,7 +38,9 @@ module LittleGhost
     # With a parent session, durable definitions retain only committed compact
     # transcripts and limited state snapshots. Failed or cancelled turns never
     # become committed conversation history. Call #close to cancel and join
-    # workers owned by a directly constructed manager.
+    # workers owned by a directly constructed manager. If cooperative fiber
+    # cleanup exceeds the deadline, the manager remains closed to new work and
+    # a later #close retries cleanup.
     class Manager
       # Raised when one or more managed workers cannot stop within the cleanup
       # deadline.
@@ -100,7 +103,6 @@ module LittleGhost
         :progress_message,
         :progress_sequence
       )
-
       class Completion # :nodoc:
         def initialize
           @mutex = Mutex.new
@@ -248,12 +250,17 @@ module LittleGhost
         @cancellation_token = cancellation_token.child
         @deadline = deadline
         @observer = observer
+        @observer_events = []
+        @observer_flushing = false
+        @task_runner = runtime ? runtime.task_runner : Support::TaskRunner.new
         @parent_session = parent_session
         @parent_agent_path = AgentPath.validate!(parent_agent_path)
         @parent_link = parent_session && self.class.parent_link(parent_session)
         @registry_session = parent_session && registry_session
         @capacity = Capacity.new(max_concurrent)
         @mutex = Mutex.new
+        @close_monitor = Monitor.new
+        @close_active = false
         @registry_mutex = Mutex.new
         @restore_mutex = Mutex.new
         @condition = ConditionVariable.new
@@ -262,6 +269,8 @@ module LittleGhost
         @identity_slots = 0
         @turn_count = 0
         @closed = false
+        @close_complete = false
+        @closed_agents = {}
         restore_identities
       end
 
@@ -330,14 +339,28 @@ module LittleGhost
           raise Error, "Subagent manager is closed"
         end
 
-        turn, queued_snapshot = enqueue(
-          identity,
-          task,
-          event: "spawned",
-          count_turn: false,
-          parent_operation_id:,
-          context:
-        )
+        begin
+          turn, queued_snapshot = enqueue(
+            identity,
+            task,
+            event: "spawned",
+            count_turn: false,
+            parent_operation_id:,
+            context:
+          )
+        rescue
+          synchronize_close do
+            should_close = @mutex.synchronize do
+              if @identities.delete(subagent_id)
+                @identity_slots -= 1
+                @turn_count -= 1
+              end
+              !@closed_agents.key?(agent.object_id)
+            end
+            agent.close if should_close && agent.respond_to?(:close)
+          end
+          raise
+        end
         return {status: "working", subagent: queued_snapshot} if mode == "async"
 
         turn.completion.value(cancellation_token: @cancellation_token, deadline: @deadline)
@@ -665,61 +688,99 @@ module LittleGhost
       end
 
       # Cancels queued work, cooperatively stops workers, and closes child
-      # agents. Raises CleanupError if workers do not stop within the bound.
+      # agents. Raises CleanupError if workers do not stop within the bound. A
+      # later call retries unfinished cleanup without accepting new work.
       def close
-        workers = @mutex.synchronize do
-          return if @closed
-
-          @closed = true
-          @cancellation_token.cancel
-          @identities.each_value do |identity|
-            next if %w[idle failed cancelled persisting].include?(identity.status)
-
-            turn = identity.current
-            identity.status = "cancelled"
-            turn&.completion&.resolve(cancelled_turn(identity, turn))
-            identity.progress_message = nil
-            identity.current_turn = nil
-            identity.current = nil
-            cancel_queued_turns(identity)
-            emit("cancelled", identity, turn:)
+        synchronize_close { perform_close }
+      end
+
+      private
+
+      def synchronize_close
+        @close_monitor.synchronize do
+          return if @close_active
+
+          @close_active = true
+          begin
+            yield
+          ensure
+            @close_active = false
           end
-          @condition.broadcast
-          @identities.values.filter_map(&:worker)
         end
+      end
 
-        deadline = monotonic_time + @close_timeout
-        cooperative_deadline = monotonic_time + (@close_timeout / 2.0)
+      def perform_close
+        started_at = monotonic_time
+        deadline = started_at + @close_timeout
+        cooperative_deadline = started_at + (@close_timeout / 2.0)
+        identities = @mutex.synchronize do
+          return if @close_complete
+
+          unless @closed
+            @closed = true
+            @cancellation_token.cancel
+            @identities.each_value do |identity|
+              next if %w[idle failed cancelled persisting].include?(identity.status)
+
+              turn = identity.current
+              identity.status = "cancelled"
+              turn&.completion&.resolve(cancelled_turn(identity, turn))
+              identity.progress_message = nil
+              identity.current_turn = nil
+              identity.current = nil
+              cancel_queued_turns(identity)
+              queue_observer_event("cancelled", identity, turn:)
+            end
+            @condition.broadcast
+          end
+
+          @identities.values.dup
+        end
+        flush_observer_events
+
+        worker_entries = identities.filter_map do |identity|
+          worker = identity.worker
+          [identity, worker] if worker
+        end
+        workers = worker_entries.map(&:last)
         workers.each do |worker|
           remaining = cooperative_deadline - monotonic_time
           break unless remaining.positive?
 
-          worker.join(remaining)
+          wait_for_worker(worker, remaining)
         end
-        workers.select(&:alive?).each(&:kill)
+        workers.select(&:alive?).each(&:terminate)
         workers.each do |worker|
           remaining = deadline - monotonic_time
           break unless remaining.positive?
 
-          worker.join(remaining)
+          wait_for_worker(worker, remaining)
         end
         first_error = nil
-        survivors = workers.select(&:alive?)
-        unless survivors.empty?
+        surviving_task_entries = worker_entries.select { |_identity, worker| worker.alive? }
+        surviving_identities = surviving_task_entries.map(&:first).uniq(&:object_id)
+        unless surviving_identities.empty?
           first_error ||= CleanupError.new(
-            "#{survivors.length} subagent worker(s) did not stop within #{@close_timeout} seconds"
+            "#{surviving_identities.length} subagent worker(s) did not stop within #{@close_timeout} seconds"
           )
         end
-        agents = @mutex.synchronize { @identities.values.map(&:agent).reverse.uniq(&:object_id) }
+
+        blocked_identities = surviving_task_entries.map(&:first)
+        blocked_agents = blocked_identities.filter_map(&:agent).to_h { |agent| [agent.object_id, true] }
+        agents = identities.reverse.filter_map(&:agent).uniq(&:object_id).reject do |agent|
+          blocked_agents.key?(agent.object_id) || @mutex.synchronize { @closed_agents.key?(agent.object_id) }
+        end
         agents.each do |agent|
           agent.close if agent.respond_to?(:close)
+          @mutex.synchronize { @closed_agents[agent.object_id] = true }
         rescue => error
           first_error ||= error
         end
         raise first_error if first_error
-      end
 
-      private
+        @mutex.synchronize { @close_complete = true }
+        nil
+      end
 
       def restore_identities
         return unless @registry_session
@@ -1086,6 +1147,8 @@ module LittleGhost
             return rejection if rejection
           end
 
+          worker = @task_runner.spawn { run_identity(identity) } unless identity.worker&.alive?
+
           turn = Turn.new(
             number: identity.next_turn,
             message: message,
@@ -1100,17 +1163,14 @@ module LittleGhost
           @turn_count += 1 if count_turn
           identity.queue << turn
           identity.status = "queued" unless identity.status == "running"
-          emit(event, identity, turn:)
+          identity.worker = worker if worker
+          queue_observer_event(event, identity, turn:)
           queued_snapshot = snapshot(identity)
-          unless identity.worker&.alive?
-            execution_state = ExecutionState.capture
-            identity.worker = Thread.new do
-              ExecutionState.with(execution_state) { run_identity(identity) }
-            end
-          end
           @condition.broadcast
         end
         [turn, queued_snapshot]
+      ensure
+        flush_observer_events
       end
 
       def run_identity(identity)
@@ -1145,11 +1205,12 @@ module LittleGhost
                   identity.status = "running"
                   identity.progress_message = nil
                   identity.progress_sequence += 1
-                  emit("turn_started", identity, turn:)
+                  queue_observer_event("turn_started", identity, turn:)
                   @condition.broadcast
                   true
                 end
               end
+              flush_observer_events
               next unless should_run
 
               begin
@@ -1194,9 +1255,10 @@ module LittleGhost
         end
       ensure
         @mutex.synchronize do
-          identity.worker = nil if identity.worker == Thread.current
+          identity.worker = nil if identity.worker&.current?
           @condition.broadcast
         end
+        flush_observer_events
       end
 
       def run_agent_turn(identity, turn, options)
@@ -1221,11 +1283,12 @@ module LittleGhost
           @condition.broadcast
           if tool_use
             event_name = (event.type == :tool_start) ? "tool_started" : "tool_finished"
-            emit(event_name, identity, turn:, tool_call_id: tool_use.id, tool_name: tool_use.name)
+            queue_observer_event(event_name, identity, turn:, tool_call_id: tool_use.id, tool_name: tool_use.name)
           else
-            emit("activity", identity, turn:)
+            queue_observer_event("activity", identity, turn:)
           end
         end
+        flush_observer_events
       end
 
       def progress_message(event)
@@ -1321,9 +1384,10 @@ module LittleGhost
           }
           value[:response_truncated] = true if truncated
           turn.completion.resolve(value)
-          emit("turn_finished", identity, turn:)
+          queue_observer_event("turn_finished", identity, turn:)
           @condition.broadcast
         end
+        flush_observer_events
       end
 
       def retain_agent_conversation(identity, turn, result, persisted_response, interject_exchanges)
@@ -1384,10 +1448,11 @@ module LittleGhost
               error: identity.latest_error
             )
           end
-          emit("turn_failed", identity, turn:, error_type: error.class.name)
+          queue_observer_event("turn_failed", identity, turn:, error_type: error.class.name)
           fail_queued_turns(identity)
           @condition.broadcast
         end
+        flush_observer_events
       end
 
       def cancel_unrun_turn(identity, turn)
@@ -1400,9 +1465,10 @@ module LittleGhost
           identity.current = nil
           identity.status = "cancelled"
           cancel_queued_turns(identity)
-          emit("cancelled", identity, turn:) if newly_cancelled
+          queue_observer_event("cancelled", identity, turn:) if newly_cancelled
           @condition.broadcast
         end
+        flush_observer_events
       end
 
       def cancelled_turn(identity, turn)
@@ -1418,7 +1484,7 @@ module LittleGhost
       def cancel_queued_turns(identity)
         identity.queue.each do |turn|
           turn.completion.resolve(cancelled_turn(identity, turn))
-          emit("cancelled", identity, turn:)
+          queue_observer_event("cancelled", identity, turn:)
         end
         identity.queue.clear
       end
@@ -1432,7 +1498,7 @@ module LittleGhost
             turn: turn.number,
             error: "A previous turn failed; spawn a new identity."
           )
-          emit("turn_failed", identity, turn:)
+          queue_observer_event("turn_failed", identity, turn:)
         end
         identity.queue.clear
       end
@@ -1526,7 +1592,7 @@ module LittleGhost
         %w[idle failed cancelled].include?(identity.status)
       end
 
-      def emit(event, identity, turn: nil, **attributes)
+      def queue_observer_event(event, identity, turn: nil, **attributes)
         return unless @observer
 
         value = {
@@ -1545,9 +1611,7 @@ module LittleGhost
           end
         end
         value.merge!(attributes)
-        @observer.call(value.freeze)
-      rescue
-        nil
+        @observer_events << value.freeze
       end
 
       def observe_delegated_activity(identity)
@@ -1565,21 +1629,57 @@ module LittleGhost
           @condition.broadcast
           identity.current
         end
-        emit("activity", identity, turn:) if turn
+        if turn
+          @mutex.synchronize { queue_observer_event("activity", identity, turn:) }
+          flush_observer_events
+        end
       end
 
       def emit_factory_failure(definition, subagent_id, error, parent_operation_id: nil)
         return unless @observer
 
-        @observer.call({
-          event: "factory_failed",
-          subagent_id:,
-          kind: definition.kind,
-          status: "failed",
-          error_type: error.class.name,
-          parent_operation_id:
-        }.compact.freeze)
-      rescue
+        @mutex.synchronize do
+          @observer_events << {
+            event: "factory_failed",
+            subagent_id:,
+            kind: definition.kind,
+            status: "failed",
+            error_type: error.class.name,
+            parent_operation_id:
+          }.compact.freeze
+        end
+        flush_observer_events
+      end
+
+      def flush_observer_events
+        return unless @observer
+
+        should_flush = @mutex.synchronize do
+          next false if @observer_flushing || @observer_events.empty?
+
+          @observer_flushing = true
+        end
+        return unless should_flush
+
+        loop do
+          value = @mutex.synchronize do
+            event = @observer_events.shift
+            @observer_flushing = false unless event
+            event
+          end
+          break unless value
+
+          begin
+            @observer.call(value)
+          rescue
+            nil
+          end
+        end
+      end
+
+      def wait_for_worker(worker, timeout)
+        worker.wait(deadline: Time.now + timeout)
+      rescue DeadlineExceededError
         nil
       end
 

data/lib/little_ghost/support/cancellation_token.rb

@@ -3,8 +3,8 @@
 module LittleGhost
   module Support
     # CancellationToken lets related work stop cooperatively without killing its
-    # calling thread. Child tokens make cancellation flow through a run's tree of
-    # work.
+    # calling thread or fiber. Child tokens make cancellation flow through a
+    # run's tree of work.
     #
     # Cancellation is idempotent and flows only downward. Long-running
     # extensions should call #raise_if_cancelled! at bounded intervals.

data/lib/little_ghost/support/executor.rb

@@ -6,15 +6,38 @@ module LittleGhost
     # the final results. It gives framework extensions bounded parallelism without
     # losing cancellation or request-scoped state.
     #
-    # ExecutionState is copied to workers. +on_result+ runs on the calling
-    # thread in completion order. After all workers join, the first cleanup
-    # error, or otherwise the first input-order error, is raised.
-    class Executor
-      # Sets the maximum number of worker threads.
-      def initialize(max_concurrency: 8)
+    # ExecutionState is copied to workers. +on_result+ runs on the caller in
+    # completion order. After all workers join, the first
+    # cleanup error, or otherwise the first input-order error, is raised.
+    class Executor # :nodoc:
+      # Sets the maximum number of batch workers and their execution policy.
+      def initialize(max_concurrency: 8, runner: TaskRunner.new, wait_through_interruptions: false)
         raise ArgumentError, "max_concurrency must be at least 1" if max_concurrency < 1
 
         @max_concurrency = max_concurrency
+        @runner = runner
+        @wait_through_interruptions = wait_through_interruptions
+      end
+
+      attr_reader :runner # :nodoc:
+
+      # Submits one unit of work and returns its Task.
+      def submit(&work)
+        @runner.spawn(&work)
+      end
+
+      # Runs one unit of work and returns its result.
+      def call(&work)
+        resolve(submit(&work))
+      end
+
+      # Runs one unit of work only when runner capacity is immediately
+      # available. Returns an accepted flag and the result.
+      def try_call(&work)
+        task = @runner.try_spawn(&work)
+        return [false, nil] unless task
+
+        [true, resolve(task)]
       end
 
       # Maps +values+ with at most the configured number of workers.
@@ -34,11 +57,10 @@ module LittleGhost
         error_mutex = Mutex.new
         first_worker_error = nil
         worker_count = [@max_concurrency, items.length].min
-        execution_state = ExecutionState.capture
-
-        workers = worker_count.times.map do
-          Thread.new do
-            ExecutionState.with(execution_state) do
+        workers = []
+        begin
+          worker_count.times do
+            workers << submit do
               loop do
                 index = begin
                   queue.pop(true)
@@ -57,15 +79,29 @@ module LittleGhost
                   completions << index
                 end
               end
+            ensure
+              completions << :worker_finished
             end
           end
+        rescue
+          cancellation_token.cancel
+          workers.each(&:wait)
+          raise
         end
         callback_error = nil
         begin
-          items.length.times do
-            index = completions.pop
+          completed_items = 0
+          finished_workers = 0
+          until completed_items == items.length || finished_workers == workers.length
+            completion = completions.pop
+            if completion == :worker_finished
+              finished_workers += 1
+              next
+            end
+
+            completed_items += 1
             begin
-              on_result.call(index, results[index]) if on_result && !errors[index]
+              on_result.call(completion, results[completion]) if on_result && !errors[completion]
             rescue => error
               callback_error = error
               cancellation_token.cancel
@@ -73,15 +109,48 @@ module LittleGhost
             end
           end
         ensure
-          workers.each(&:join)
+          workers.each(&:wait)
         end
 
         raise callback_error if callback_error
+        task_error = workers.filter_map(&:error).first
+        raise task_error if task_error
+
         first_error = errors.compact.find { |error| error.is_a?(CleanupError) } || first_worker_error
         raise first_error if first_error
 
         results
       end
+
+      private
+
+      def resolve(task)
+        interruption = nil
+        loop do
+          task.wait
+          break
+        rescue Exception => error # rubocop:disable Lint/RescueException
+          raise unless @wait_through_interruptions
+
+          interruption ||= error
+          break unless task.alive?
+        end
+        raise interruption if interruption
+        raise task.error if task.error
+
+        task.result
+      end
+
+      # The process-wide Executor for work moved off a scheduler thread.
+      BLOCKING = new(
+        runner: PooledThreadRunner.new,
+        wait_through_interruptions: true
+      ) # :nodoc:
+      private_constant :BLOCKING
+
+      class << self
+        def blocking = BLOCKING # :nodoc:
+      end
     end
   end
 end

data/lib/little_ghost/support/http_client.rb

@@ -70,6 +70,7 @@ module LittleGhost
             method: :post,
             headers:,
             body:,
+            cancellation_token:,
             deadline:,
             allow_insecure_http: @allow_insecure_http,
             label: "Provider"
@@ -80,7 +81,8 @@ module LittleGhost
 
       # Executes a bounded request and returns the complete response body.
       def request(uri:, method: :get, headers: {}, body: nil, allow_insecure_http: false,
-        cancellation_token: nil, deadline: nil)
+        cancellation_token: nil, deadline: nil, label: "HTTP request", max_response_bytes: @max_response_bytes)
+        response_limit = positive_integer(max_response_bytes, :max_response_bytes)
         response_body = +""
         each_chunk(
           uri:,
@@ -89,8 +91,14 @@ module LittleGhost
           body:,
           cancellation_token:,
           deadline: deadline || Time.now + @read_timeout,
-          allow_insecure_http:
-        ) { |chunk| response_body << chunk }
+          allow_insecure_http:,
+          label:
+        ) do |chunk|
+          if response_body.bytesize + chunk.bytesize > response_limit
+            raise ProtocolError, "#{label} response exceeded #{response_limit} bytes"
+          end
+          response_body << chunk
+        end
         response_body
       end
 
@@ -119,7 +127,12 @@ module LittleGhost
         http.write_timeout = remaining_timeout(deadline, @read_timeout)
         http.request(request) do |response|
           unless response.is_a?(Net::HTTPSuccess)
-            response_body = read_limited(response, @max_error_body_bytes)
+            response_body = read_limited(
+              response,
+              @max_error_body_bytes,
+              cancellation_token:,
+              deadline:
+            )
             raise Providers::HTTPError.new(
               "#{label} failed with HTTP #{response.code}",
               status: response.code.to_i,
@@ -165,9 +178,10 @@ module LittleGhost
         [remaining, maximum].min
       end
 
-      def read_limited(response, limit)
+      def read_limited(response, limit, cancellation_token:, deadline:)
         body = +""
         response.read_body do |chunk|
+          check_control!(cancellation_token, deadline)
           remaining = limit - body.bytesize
           body << chunk.byteslice(0, remaining) if remaining.positive?
           break if body.bytesize >= limit

data/lib/little_ghost/support/interruptible_stream.rb

@@ -8,7 +8,9 @@ module LittleGhost
     #
     # The producer receives an emitter callable. Ending enumeration early stops
     # and joins the producer; CleanupError is raised if it cannot be stopped
-    # within the fixed shutdown bound.
+    # within the fixed shutdown bound. The producer remains on a worker thread
+    # even when the consumer uses a Fiber scheduler, allowing cleanup to
+    # interrupt a provider SDK or socket read that remains blocked.
     #
     #   stream = LittleGhost::Support::InterruptibleStream.new(
     #     cancellation_token: token
@@ -36,7 +38,7 @@ module LittleGhost
       end
 
       # Yields produced values, raising producer, cancellation, deadline, or
-      # cleanup errors in the consuming thread.
+      # cleanup errors in the caller.
       def each
         return enum_for(__method__) unless block_given?