mistri 0.4.1 → 0.5.0

data/lib/mistri/retry_policy.rb

@@ -4,8 +4,8 @@ module Mistri
   # When a failed turn is worth retrying, and how long to wait. Transient
   # failures (rate limits, overload, server errors, timeouts, dropped or
   # truncated streams) retry with jittered exponential backoff, honoring the
-  # provider's retry-after when it sent one. Everything else — auth, invalid
-  # requests, our own bugs — fails fast.
+  # provider's retry-after when it sent one. Everything else (auth, invalid
+  # requests, our own bugs) fails fast.
   #
   # attempts counts retries, not calls: attempts 3 means up to four requests.
   class RetryPolicy

data/lib/mistri/session.rb

@@ -66,6 +66,11 @@ module Mistri
       entries.reverse_each.find { |entry| entry["type"] == "compaction" }
     end
 
+    # The inbox: entry types queued for the loop's next turn boundary, each
+    # mapped to the marker key its fold leaves on the consuming message
+    # entry.
+    INBOX = { "steer" => "steer_id", "subagent_report" => "report_id" }.freeze
+
     # Queue a message for a running exchange from any process. The loop folds
     # pending steers into the transcript at the next turn boundary, so the
     # model sees them mid-run; one that arrives as the model finishes cleanly
@@ -74,12 +79,64 @@ module Mistri
       append("steer", "id" => SecureRandom.uuid, "message" => Message.user(text).to_h)
     end
 
-    # Steers not yet folded into the transcript, oldest first. The folding
-    # message entry carries the steer id, so consumption is derived from the
-    # log alone and reads the same from every process.
+    # A sub-agent's report, queued for this session the way a steer is: it
+    # folds into the transcript at the next turn boundary as a labeled block
+    # the model can react to ("[Magpie finished] <report>"), while the typed
+    # entry keeps name, status, and the raw text for hosts to render as a
+    # report card rather than a fake user message. One report per child,
+    # ever: a duplicate delivery (a redelivered queue job, a lease race) is
+    # dropped, and the return says which happened. Reports normally arrive
+    # via SubAgent.run_dispatched; call this directly only from a custom
+    # dispatch path.
+    def deliver_report(name:, session_id:, status:, text: nil) # rubocop:disable Naming/PredicateMethod
+      already = entries.any? do |entry|
+        entry["type"] == "subagent_report" && entry["session_id"] == session_id
+      end
+      return false if already
+
+      append("subagent_report", "id" => SecureRandom.uuid, "name" => name,
+                                "session_id" => session_id, "status" => status, "report" => text,
+                                "message" => Message.user(report_label(name, status, text)).to_h)
+      true
+    end
+
+    # Everything queued for the loop's next turn boundary (steers and
+    # sub-agent reports), oldest first, in arrival order. The folding
+    # message entry carries the source entry's id under its marker key, so
+    # consumption is derived from the log alone and reads the same from
+    # every process. A host that wakes an idle session when a steer arrives
+    # should watch this instead: a report deserves the same pickup.
+    def pending_inbox
+      log = entries
+      folded = log.flat_map { |entry| entry.values_at(*INBOX.values) }.compact.to_set
+      log.select { |entry| INBOX.key?(entry["type"]) && !folded.include?(entry["id"]) }
+    end
+
+    # The steer-only slice of the inbox, oldest first.
     def pending_steers
-      folded = entries.filter_map { |entry| entry["steer_id"] }
-      entries.select { |entry| entry["type"] == "steer" && !folded.include?(entry["id"]) }
+      pending_inbox.select { |entry| entry["type"] == "steer" }
+    end
+
+    # Sub-agents this session has spawned, in spawn order: one Child window
+    # per link entry, each reading the child's own session. Derived from the
+    # log alone, like everything else here.
+    def children
+      entries.filter_map do |entry|
+        next unless entry["type"] == "subagent"
+
+        Child.new(name: entry["name"], session_id: entry["session_id"], store: @store)
+      end
+    end
+
+    # The session as a reader renders it: entries in order with inline image
+    # bytes stripped, and, with include_children, every sub-agent's own
+    # transcript spliced in after its link entry. Spliced entries carry an
+    # "origin" key shaped exactly like the live stream's event origins
+    # ("Magpie#ab12cd34", nesting joined with ">"), so a UI that rebuilds
+    # from this sees what it saw live, lanes included, running children's
+    # progress-so-far included. The raw log stays available as #entries.
+    def transcript(include_children: false)
+      splice(entries, include_children: include_children, prefix: nil, seen: Set[@id])
     end
 
     # Record a human's decision on a parked tool call. Decisions are session
@@ -137,6 +194,39 @@ module Mistri
       Message.user("#{Compaction::SUMMARY_PREFACE}\n\n#{summary}")
     end
 
+    # Each link entry opens its child's log in place, depth first, the way
+    # nested lanes opened live. The seen set makes expansion idempotent per
+    # child (a repeated or self-referencing link renders but never expands
+    # twice), so a hostile log cannot loop this.
+    def splice(log, include_children:, prefix:, seen:)
+      log.flat_map do |entry|
+        rendered = Child.strip_images(entry)
+        rendered = rendered.merge("origin" => prefix) if prefix
+        next [rendered] unless include_children && expandable?(entry, seen)
+
+        seen << entry["session_id"]
+        origin = "#{entry["name"]}##{entry["session_id"][0, 8]}"
+        origin = "#{prefix}>#{origin}" if prefix
+        child_log = self.class.new(store: @store, id: entry["session_id"]).entries
+        [rendered, *splice(child_log, include_children: true, prefix: origin, seen: seen)]
+      end
+    end
+
+    def expandable?(entry, seen)
+      entry["type"] == "subagent" && !seen.include?(entry["session_id"])
+    end
+
+    # How a report reads to the model: labeled with the worker's name and
+    # fate, so the parent knows exactly who finished and how.
+    def report_label(name, status, text)
+      case status
+      when "done" then "[#{name} finished] #{text}"
+      when "failed" then "[#{name} failed] #{text}"
+      when "stopped" then "[#{name} was stopped]"
+      else "[#{name} ended: #{status}]"
+      end
+    end
+
     def decide(call_id, approved:, note:)
       unless open_approvals.any? { |open| open[:call].id == call_id }
         raise ConfigurationError, "no open approval for #{call_id.inspect}"

data/lib/mistri/skill.rb

@@ -3,7 +3,7 @@
 module Mistri
   # One expert playbook: a name the model selects by, a description that
   # earns the selection, and the full body it reads before acting. Build
-  # them from anywhere — Skills.load reads a directory, and a host with
+  # them from anywhere: Skills.load reads a directory, and a host with
   # skills in a database constructs these directly.
   Skill = Data.define(:name, :description, :body) do
     def initialize(name:, description: "", body: "")

data/lib/mistri/skills.rb

@@ -3,7 +3,7 @@
 module Mistri
   # Loads skills and wires them into an agent: their one-line descriptions
   # ride the system prompt, and the model pulls a full body on demand with
-  # the read_skill tool — so a large library costs almost nothing until a
+  # the read_skill tool, so a large library costs almost nothing until a
   # skill is actually used.
   module Skills
     module_function

data/lib/mistri/spawner.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module Mistri
+  # Host policy for spawning workers, as one object: the tool pool children
+  # may draw from, the curated types, the model allowlist, the headcount
+  # cap, and the dispatcher that makes background mode real. #tool builds
+  # the spawn_agent tool the top-level agent holds; SubAgent.spawner and
+  # SubAgent.pack are the front doors.
+  #
+  # Every policy violation answers the model in band (unknown type, missing
+  # instructions, over capacity, workspace sharing in background mode);
+  # only host configuration mistakes raise, and they raise at construction.
+  class Spawner
+    def initialize(provider:, tools: [], types: {}, models: [], max_children: 4,
+                   dispatcher: nil, needs_approval: false, **agent_options)
+      SubAgent.forbid_gated!(tools)
+      if tools.any? { |tool| tool.name == "spawn_agent" }
+        raise ConfigurationError, "the spawn tool never goes in its own pool"
+      end
+
+      @provider = provider
+      @pool = tools
+      # Symbol keys are natural Ruby; the wire speaks strings. One
+      # normalization here and lookup, schema, and menu all agree.
+      @types = types.transform_keys(&:to_s)
+      @models = models
+      @max_children = max_children
+      @dispatcher = dispatcher
+      @needs_approval = needs_approval
+      @agent_options = agent_options
+      validate_types!
+    end
+
+    def tool
+      spawner = self
+      Tool.define("spawn_agent", description, needs_approval: @needs_approval,
+                                              schema: schema) do |args, context|
+        spawner.spawn(args, context)
+      end
+    end
+
+    def spawn(args, context)
+      crowded = over_capacity(context.session)
+      return crowded if crowded
+
+      worker = resolve_worker(args)
+      return worker if worker.is_a?(String)
+
+      if args["mode"] == "background" && @dispatcher
+        if args["workspace"] == "parent"
+          return "A worker sharing your workspace must run inline: a blocked parent " \
+                 "cannot write concurrently, a working one can. Drop workspace or " \
+                 "drop background."
+        end
+
+        return dispatch(args, worker, context)
+      end
+
+      SubAgent.run_child(label: label_for(args), provider: worker[:provider],
+                         system: worker[:system], tools: worker[:tools],
+                         task: args.fetch("task"), context: context, **@agent_options)
+    end
+
+    private
+
+    # Create the child session and its lifecycle entries, hand the
+    # dispatcher the serializable spec plus an in-process runner closing
+    # over the reconstructed pieces, and answer with a truthful receipt:
+    # what the child's status says after dispatch, not what the mode
+    # promised. The runner closes over the spawn-time emit, so in-process
+    # dispatchers keep streaming to whoever watched the spawn.
+    def dispatch(args, worker, context)
+      store = context.session ? context.session.store : Stores::Memory.new
+      child = Session.new(store: store)
+      label = label_for(args)
+      context.session&.append("subagent", "name" => label, "session_id" => child.id)
+      child.append(Child::DISPATCHED, {})
+      spec = spec_for(args, worker, child, label, context)
+      emit = context.emit
+      options = @agent_options
+      runner = lambda do
+        SubAgent.run_dispatched(spec, provider: worker[:provider], system: worker[:system],
+                                      tools: worker[:tools], store: store, emit: emit,
+                                      **options)
+      end
+      @dispatcher.call(spec, runner)
+      receipt(label, child, store)
+    end
+
+    def spec_for(args, worker, child, label, context)
+      { "name" => label, "session_id" => child.id,
+        "parent_session_id" => context.session&.id,
+        "type" => args["type"] || "general-purpose",
+        "instructions" => args["instructions"], "task" => args.fetch("task"),
+        "tool_names" => worker[:tools].map(&:name),
+        "model" => (worker[:provider].model if worker[:provider].respond_to?(:model)),
+        "workspace" => args["workspace"] || "own" }
+    end
+
+    def receipt(label, child, store)
+      status = Child.new(name: label, session_id: child.id, store: store).status
+      state = if %i[running queued].include?(status)
+                "is working in the background"
+              else
+                "already finished (#{status})"
+              end
+      ToolResult.new(
+        content: "#{label} #{state} (agent id #{child.id[0, 8]}). Keep working: its " \
+                 "report will arrive in your context when it finishes. Meanwhile " \
+                 "read_agent checks on it (wait: true blocks for the report), " \
+                 "steer_agent adjusts it, stop_agent stops it.",
+        ui: { "agent" => label, "session_id" => child.id, "mode" => "background" }
+      )
+    end
+
+    # A worker's system prompt, tools, and provider, resolved from its
+    # type; a String answers the model in band instead of raising.
+    def resolve_worker(args)
+      type = args["type"].to_s
+      return general_worker(args) if type.empty? || type == "general-purpose"
+
+      definition = @types[type]
+      unless definition
+        return "Unknown worker type #{type.inspect}; available: " \
+               "#{["general-purpose", *@types.keys].join(", ")}."
+      end
+
+      system = [definition.render, args["instructions"]]
+               .reject { |part| part.to_s.strip.empty? }.join("\n\n")
+      chosen = args["tools"].nil? || args["tools"].empty? ? definition.tool_names : args["tools"]
+      { system: system, tools: pick(chosen),
+        provider: typed_provider(args["model"], definition.model) }
+    end
+
+    def general_worker(args)
+      system = args["instructions"].to_s
+      if system.strip.empty?
+        return "A general-purpose worker needs instructions: write its system prompt, " \
+               "or pick a type."
+      end
+
+      { system: system, tools: pick(args["tools"]),
+        provider: child_provider(args["model"]) }
+    end
+
+    # An explicit model choice goes through the allowlist; otherwise a
+    # typed worker runs on its definition's model, and without one it
+    # inherits the parent's provider. Host-curated definitions are
+    # trusted the way the pool is.
+    def typed_provider(requested, definition_model)
+      return child_provider(requested) unless requested.to_s.empty?
+      return @provider if definition_model.to_s.empty?
+
+      Mistri.provider(definition_model)
+    end
+
+    def child_provider(requested)
+      return @provider if requested.nil? || requested.to_s.empty?
+      unless @models.include?(requested)
+        raise ArgumentError,
+              "model #{requested.inspect} is not allowed; available: #{@models.join(", ")}"
+      end
+
+      Mistri.provider(requested)
+    end
+
+    def over_capacity(session)
+      return nil unless session
+
+      busy = session.children.count { |child| Child::LIVE.include?(child.status) }
+      return nil if busy < @max_children
+
+      "You already have #{busy} workers running; wait for one to finish or stop one."
+    end
+
+    # Types fail at construction, never mid-spawn: every definition must
+    # render without vars (spawn types are self-contained prompts) and
+    # declare only tools the pool actually carries.
+    def validate_types!
+      return if @types.empty?
+      if @types.key?("general-purpose")
+        raise ConfigurationError, "\"general-purpose\" is the built-in type; pick another name"
+      end
+
+      pool_names = @pool.map(&:name)
+      @types.each do |name, definition|
+        begin
+          definition.render
+        rescue ConfigurationError => e
+          raise ConfigurationError,
+                "type #{name.inspect} cannot be a spawn type: #{e.message}"
+        end
+        missing = definition.tool_names - pool_names
+        unless missing.empty?
+          raise ConfigurationError,
+                "type #{name.inspect} declares tools the pool lacks: #{missing.join(", ")}"
+        end
+      end
+    end
+
+    def pick(names)
+      return @pool if names.nil? || names.empty?
+
+      by_name = @pool.to_h { |tool| [tool.name, tool] }
+      names.map do |name|
+        by_name.fetch(name) do
+          raise ArgumentError,
+                "unknown tool #{name.inspect}; available: #{by_name.keys.join(", ")}"
+        end
+      end
+    end
+
+    def schema
+      pool_names = @pool.map(&:name)
+      type_names = ["general-purpose", *@types.keys]
+      types = @types
+      models = @models
+      dispatcher = @dispatcher
+      default = @provider.model if @provider.respond_to?(:model)
+      fallback = default ? " (default: #{default})" : ""
+      lambda do
+        string :name, "A short name for this worker, shown wherever its events appear"
+        string :task, "The child's complete task", required: true
+        if types.any?
+          string :type, "The kind of worker (default: general-purpose)", enum: type_names
+          string :instructions, "The worker's system prompt (required for " \
+                                "general-purpose; appended to a typed worker's own)"
+        else
+          string :instructions, "The child's system prompt", required: true
+        end
+        if pool_names.any?
+          array :tools, "Subset of tools to grant (default: all, or the type's own list)",
+                items: { type: "string", enum: pool_names }
+        end
+        string :model, "Model for the child#{fallback}", enum: models if models.any?
+        if dispatcher
+          string :mode, "inline blocks until the report; background returns a receipt " \
+                        "now and you keep working (default: inline)",
+                 enum: %w[inline background]
+          string :workspace, "own gives the worker its own file space; parent shares " \
+                             "yours and requires inline mode (default: own)",
+                 enum: %w[own parent]
+        end
+      end
+    end
+
+    def description
+      text = SubAgent::SPAWNER_DESCRIPTION
+      unless @types.empty?
+        text += " Typed workers come ready-made (#{@types.keys.join(", ")}): their " \
+                "instructions, tools, and model are set; add instructions only to " \
+                "focus them. general-purpose workers are yours to compose."
+      end
+      if @dispatcher
+        text += " Use background mode when the work is long and you can keep helping " \
+                "meanwhile: you get a receipt now, its report arrives on its own when " \
+                "the worker finishes, and the console tools manage it in between."
+      end
+      text
+    end
+
+    def label_for(args)
+      SubAgent.sanitize_label(args["name"], fallback: "spawn")
+    end
+  end
+end

data/lib/mistri/stores/active_record.rb

@@ -20,21 +20,36 @@ module Mistri
     #   end
     #   add_index :mistri_entries, [:session_id, :position], unique: true
     #
-    # The unique index is load-bearing: one session has one writer at a time
-    # (the loop is serial), and if a host ever runs two agents on one session,
-    # a colliding append raises instead of silently corrupting entry order.
+    # The unique index is load-bearing: a session's loop is serial, but other
+    # writers append alongside it by design (a steer from a web process, a
+    # background child's report from a job), so appends are optimistic. Two
+    # writers that pick the same position collide on the index and the loser
+    # retries at the next one; entry order stays intact without a lock.
     #
     # Reads select only (position, payload) and sort in Ruby: ORDER BY over
     # rows carrying multi-megabyte payloads exhausts MySQL's sort buffer.
     class ActiveRecord
+      # Concurrent writers converge in one or two retries; a session would
+      # need this many simultaneous appenders to exhaust them.
+      APPEND_ATTEMPTS = 5
+
       def initialize(model)
         @model = model
       end
 
       def append(id, entry)
-        position = @model.where(session_id: id).maximum(:position).to_i + 1
-        @model.create!(session_id: id, position: position, payload: JSON.generate(entry))
-        nil
+        payload = JSON.generate(entry)
+        attempts = 0
+        begin
+          position = @model.where(session_id: id).maximum(:position).to_i + 1
+          @model.create!(session_id: id, position: position, payload: payload)
+          nil
+        rescue ::ActiveRecord::RecordNotUnique
+          attempts += 1
+          raise if attempts >= APPEND_ATTEMPTS
+
+          retry
+        end
       end
 
       def load(id)

data/lib/mistri/stores/jsonl.rb

@@ -13,8 +13,10 @@ module Mistri
         FileUtils.mkdir_p(dir)
       end
 
+      # One write call per line: concurrent appenders (a steer, a child's
+      # report) interleave whole lines, never fragments.
       def append(id, entry)
-        File.open(path(id), "a") { |file| file.puts(JSON.generate(entry)) }
+        File.write(path(id), "#{JSON.generate(entry)}\n", mode: "a")
         nil
       end