insika 0.1.0 → 0.2.0

data/lib/insika/tool_output_compressor.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Insika
+  # Mechanical (no-LLM) compression of tool output in the replayed transcript
+  # (A3/C3, hermes phase 1): identical tool results are stored ONCE — the first
+  # occurrence full — and every byte-identical repeat is replaced with a compact
+  # back-reference to it plus the original's first line. The cheap half of
+  # compaction: a search tool that returns the same catalog page over N turns
+  # currently pays for the full body N times, and that accumulation is what
+  # blows the budget first.
+  #
+  # Deliberately crude on purpose: SHA-256 digest, so ONLY exact matches dedupe —
+  # a differing detail is kept, never "close enough" dropped. Deterministic per
+  # input (the same transcript compresses identically every turn, prompt-cache
+  # friendly). Never touches user/assistant content and never mutates the input
+  # (symbol or string keys are preserved per message).
+  #
+  # Consumer: the history path of Context::Providers::Session, opt-in via
+  # AgentProfile#tool_output_compression. The LLM half of compaction (real
+  # summarization) remains F5 — decided with data, not by matrix.
+  class ToolOutputCompressor
+    # Below this a back-reference can never cost less than the body it
+    # replaces (an early bail; the REAL gate is the length check per repeat).
+    MIN_LENGTH = 200
+    # The mechanical "1-line summary": the original's first line, capped here
+    # (160 chars ≈ ~40 tokens — a bounded, informative stub).
+    SUMMARY_LIMIT = 160
+
+    # The boilerplate deliberately carries NO positional pointer ("see above"):
+    # the budget cut evicts the OLDEST unit first, which is the original this
+    # back-reference points at — a "↑" that survives its target leaves a dangle
+    # in the model's context (C3). The summary is the content; it stands alone.
+    BOILERPLATE = "[repeated tool output — byte-identical to an earlier result " \
+                  "in this conversation; One-line summary: "
+
+    class << self
+      # [message, ...] with repeated tool results back-referenced. A non-tool
+      # message or a tool result that is not a String passes through untouched.
+      # A repeat is replaced ONLY when the back-reference is strictly shorter
+      # than the content it would replace (in the 200–~260-char band a
+      # back-reference is LONGER than the original — replacing there would
+      # GROW the transcript, C3).
+      def compress_transcript(messages)
+        seen = {}
+        Array(messages).map do |m|
+          content = text_of(m)
+          next m unless content && content.length >= MIN_LENGTH
+
+          digest = Digest::SHA256.hexdigest(content)
+          if seen.key?(digest)
+            replacement = back_reference(seen[digest][:summary])
+            replacement.length < content.length ? replace_content(m, replacement) : m
+          else
+            seen[digest] = { summary: summarize(content) }
+            m
+          end
+        end
+      end
+
+      private
+
+      # -> String | nil: the content of a role:tool message, nil otherwise
+      # (compression is for tool RESULTS only; a person's words are never touched).
+      def text_of(m)
+        return nil unless m.is_a?(Hash)
+        return nil unless (m[:role] || m["role"]).to_s == "tool"
+
+        c = m[:content] || m["content"]
+        c.is_a?(String) ? c : nil
+      end
+
+      def back_reference(summary)
+        "#{BOILERPLATE}#{summary}]"
+      end
+
+      # A dup with the content replaced under the SAME key style as the original
+      # (symbol-keyed message -> symbol content, string-keyed -> string content).
+      def replace_content(m, text)
+        out = m.dup
+        if m.key?(:content)
+          out[:content] = text
+        else
+          out["content"] = text
+        end
+        out
+      end
+
+      # The mechanical one-line summary: the first line, whitespace-collapsed and
+      # capped. For a multi-line JSON tool body the first line is a real head; for
+      # a single-line blob it is a truncated head — never the whole body.
+      def summarize(content)
+        first = content.to_s.split(/\r?\n/).first.to_s
+        collapsed = first.gsub(/\s+/, " ").strip
+        collapsed.length <= SUMMARY_LIMIT ? collapsed : "#{collapsed[0, SUMMARY_LIMIT]}…"
+      end
+    end
+  end
+end

data/lib/insika/tool_store.rb

@@ -9,7 +9,7 @@ module Insika
   # McpStore masks the env): headers whose name is in `secret_headers` never
   # leave in plaintext to the UI — they become the sentinel `__OCULTO__`. Only `get_raw`/
   # `all_raw` (consumed by DataDefinedTool/overlay, never the screen) return the
-  # real values. Phase 5, Step A.
+  # real values.,.
   #
   # Record in the ConfigStore:
   #   { "definition" => { ...ToolDefinition#to_h... },

data/lib/insika/tool_trace_store.rb

@@ -3,7 +3,7 @@
 require "json"
 
 module Insika
-  # Per-session TOOL-CALL trace, for debugging in the Studio (FOLLOWUP §3.1). One
+  # Per-session TOOL-CALL trace, for debugging in the Studio. One
   # record per session in the raw backend (scope "tool_traces") — RUNTIME data, next
   # to sessions/tasks, NOT config (hence the raw `store:`, like SessionStore,
   # not the ConfigStore). A capped LIST of entries; ToolEnvelope writes one

data/lib/insika/tools/concurrency.rb

@@ -5,7 +5,7 @@ require "async/semaphore"
 
 module Insika
   module Tools
-    # Fan-out INSIDE a single tool (RFC-0010 §B1): when a tool must gather N
+    # Fan-out INSIDE a single tool (§): when a tool must gather N
     # independent I/O calls (stock + price + promo + delivery) into ONE result,
     # `gather` runs them CONCURRENTLY on the turn's reactor and returns their values
     # IN ORDER. Because a data-tool spends its time on the HTTP wait, those waits
@@ -13,7 +13,7 @@ module Insika
     #
     # This is the "aggregator tool" pattern: the model makes ONE tool call and the
     # concurrency is an implementation detail of that tool — no change to the agent
-    # loop, no dependency on parallel tool execution (item 30). Use it in a custom
+    # loop, no dependency on parallel tool execution. Use it in a custom
     # Ruby tool's #execute:
     #
     #   def execute(store_id:)

data/lib/insika/tools/data_defined_tool.rb

@@ -10,7 +10,6 @@ module Insika
     # ToolDefinition (the same pattern as A2ARemote). It makes an HTTP call described
     # in config — no Ruby code per tool. Since it inherits RubyLLM::Tool (pulls in the gem),
     # it is NOT required in lib/insika.rb; the overlay loads it lazily at registration
-    # (Step B). Phase 5, Step A.
     #
     # Contract preserved by duck-typing: it overrides name/description/parameters/
     # execute; RubyLLM's params_schema derives from #parameters automatically.
@@ -28,7 +27,7 @@ module Insika
         super()
       end
 
-      # Turn context (Phase 6/D2/G3): the registry tool does NOT receive TurnState,
+      # Turn context: the registry tool does NOT receive TurnState,
       # so the Executor DEPOSITS the turn ids here, per-turn
       # (chat/agent/tenant/store). They resolve {{ctx.*}} — SEPARATE from the model's
       # {{param}} — to emit X-Chat-Id/X-Store-Id/X-Agent-Id. They come from the TURN, never from
@@ -46,8 +45,8 @@ module Insika
 
       # FULL (nested) JSON Schema straight into RubyLLM's params_schema — it is what
       # the providers serialize (OpenAI/Anthropic/Gemini/Bedrock prefer
-      # params_schema; parameters is just a fallback). Provider-agnostic (Phase 7/F6) and
-      # the only form that expresses nesting (object/array/enum). Phase 7, Step A.
+      # params_schema; parameters is just a fallback). Provider-agnostic and
+      # the only form that expresses nesting (object/array/enum).,.
       def params_schema = @definition.parameters
 
       # FLAT top-level view for discovery (tool_search calls #parameters on the resolved
@@ -121,7 +120,7 @@ module Insika
       end
 
       # ctx.* -> TURN context (deposited by the Executor); the rest -> MODEL args
-      # (kwargs). The split is the D2/R2 trust boundary: the model does
+      # (kwargs). The split is the/R2 trust boundary: the model does
       # not choose which chat/store the tool accesses.
       def resolve(name, kwargs)
         prefix = Insika::ToolDefinition::CTX_PREFIX

data/lib/insika/tools/load_skill.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "ruby_llm"
+require "time"
 
 module Insika
   module Tools
@@ -22,19 +23,76 @@ module Insika
       # `param :name` (verified: the param is still present).
       def name = "load_skill"
 
-      def initialize(catalog, allowed_names)
+      # trace_recorder/state are OPTIONAL (nil = no trace, parity): this tool is
+      # deliberately NOT enveloped (ToolAssembly#wrap_tools), and the envelope is
+      # what records the tool trace — so without recording HERE, the one call an
+      # operator most needs to audit is the only one missing from the Studio's
+      # trace. Same shape as ToolSearch, which also emits its own event.
+      # `agent` selects WHICH body a name resolves to: an agent that specialized a
+      # shared skill must be served its own version, under the same bare name. nil =
+      # the shared scope only (parity).
+      def initialize(catalog, allowed_names, trace_recorder: nil, state: nil, agent: nil)
         @catalog = catalog
         @allowed = Array(allowed_names).map(&:to_s)
+        @trace_recorder = trace_recorder
+        @state = state
+        @agent = agent
         super()
       end
 
       def execute(name:)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        result = load(name)
+        trace(name, result, started)
+        result
+      end
+
+      private
+
+      def load(name)
         return { error: "skill '#{name}' not available for this agent" } unless @allowed.include?(name.to_s)
 
-        skill = @catalog.find(name)
+        skill = @catalog.find(name, agent: @agent)
         return { error: "skill '#{name}' not found" } unless skill
 
-        skill.body
+        with_companions(skill)
+      end
+
+      # A skill's declared `companions:` come back in the SAME call, so the model
+      # cannot end up holding half a recipe (the reference table without the procedure
+      # that reads it — measured on a real pack, and the searches came out malformed).
+      #
+      # A lone skill returns its bare body, byte for byte as before: only a skill that
+      # actually declares companions pays the wrapper. Restricted to `@allowed`, which
+      # is the LAZY allowed set — a companion that is eager is already in the prompt in
+      # full, so fetching it again would only buy a duplicate.
+      def with_companions(skill)
+        extras = Array(skill.companions).filter_map do |name|
+          next unless @allowed.include?(name.to_s) && name.to_s != skill.name
+
+          @catalog.find(name, agent: @agent)
+        end
+        return skill.body if extras.empty?
+
+        ([skill] + extras).uniq(&:name)
+                          .map { |s| %(<skill name="#{s.name}">\n#{s.body}\n</skill>) }.join("\n\n")
+      end
+
+      # Mirrors ToolEnvelope#trace (same entry shape, so the Studio renders it
+      # like any other call). Clipping/masking is the ToolTraceStore's job.
+      # NEVER breaks the turn — the trace is observability.
+      def trace(name, result, started)
+        return unless @trace_recorder && @state&.task&.session_id
+
+        @trace_recorder.record(
+          session_id: @state.task.session_id,
+          entry: { "turn" => @state.turn, "tool" => "load_skill", "call_id" => "",
+                   "args" => { "name" => name.to_s }, "result" => result,
+                   "ms" => ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round,
+                   "at" => Time.now.utc.iso8601 }
+        )
+      rescue StandardError
+        nil
       end
     end
   end

data/lib/insika/tools/stuck_signal.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+
+module Insika
+  module Tools
+    # The agent's deterministic "I cannot proceed" signal (WS5). A
+    # system builtin like remember: the model calls it when it determines it cannot
+    # continue (out of scope, missing data, a case a human must take over). It ends
+    # the turn with `outcome: :stuck` recorded in the contract (the executor reads
+    # `state.stuck_outcome` at stage 8/9 and tags the terminal event) and a final
+    # message: the model's lead-in when it wrote one, else this tool's `message`.
+    #
+    # The ENGINE does not decide what "stuck" means — the consumer does. This tool
+    # is the deterministic signal the Agent.Shop subscribes to (`:turn_stuck` /
+    # `outcome: "stuck"`) to run its escalation (CRM/operator, which answers with
+    # `MessageOrigin::OPERATOR`). Nothing about handoff, pause or resume lives here.
+    class StuckSignal < RubyLLM::Tool
+      description "Signal that you cannot proceed and end the turn. Use when the " \
+                  "request is out of your scope, you lack the data to help, or a human " \
+                  "must take over. Write your final sentence to the customer first."
+      param :reason, desc: "Why you cannot proceed (goes to the operator, not the customer)"
+      param :message, desc: "Optional final message if you wrote none", required: false
+
+      def name = "signal_stuck"
+
+      def initialize(state:, **)
+        @state = state
+        super()
+      end
+
+      def execute(reason:, message: nil)
+        @state.stuck_outcome = { reason: reason.to_s, message: message.to_s }
+        # A Halt ends the tool loop here, so the turn cannot continue after declaring
+        # stuck. The payload's `say` is the fallback final message when the model
+        # wrote no lead-in (the executor's halt_answer already prefers the lead-in).
+        RubyLLM::Tool::Halt.new(Insika::ToolDefinition.wrap_halt(
+                                  { "reason" => reason.to_s },
+                                  message.to_s
+                                ))
+      end
+    end
+  end
+end

data/lib/insika/tools/subagent.rb

@@ -5,18 +5,18 @@ require_relative "agent_enum"
 
 module Insika
   module Tools
-    # In-process delegation to a CHILD agent (RFC-0010, item 21) — the Flue
+    # In-process delegation to a CHILD agent — the Flue
     # `session.task()` primitive. A system tool (like remember/load_skill): wired
     # by the ChatBuilder ONLY when `profile.subagents` is present, so `require
     # "ruby_llm"` stays in this file (loaded lazily in create_chat). NOT enveloped
     # (system tool) — in the synchronous mode the child lives in the parent's
-    # envelope and re-runs on the parent's resume (RFC-0010 §6).
+    # envelope and re-runs on the parent's resume.
     #
     # It holds no delegation logic itself: `execute` reads the parent TurnState
     # (the subagents allowlist + resolved model for inheritance + depth) and hands
     # off to `Executor#run_subagent`, which spawns the isolated child turn and
     # returns its result. The child result = its text + the linked child session
-    # id (§4.3 R3).
+    # id (R3).
     class Subagent < RubyLLM::Tool
       description "Delegates a self-contained task to a specialized child agent. " \
                   "The child runs in an ISOLATED context (it does not see this " \
@@ -67,7 +67,7 @@ module Insika
         return { dispatched: true, agent: result[:agent], session_id: result[:session_id] } if result[:dispatched]
 
         # sync: link the child session id alongside the text so a multi-step parent
-        # can reference it and the transcript stays auditable (§4.3 R3 / §7).
+        # can reference it and the transcript stays auditable (R3).
         { text: result[:text], session_id: result[:session_id] }
       end
     end

data/lib/insika/tools/subagents.rb

@@ -5,7 +5,7 @@ require_relative "agent_enum"
 
 module Insika
   module Tools
-    # PARALLEL fan-out of child agents (RFC-0010 §A): delegate N self-contained
+    # PARALLEL fan-out of child agents: delegate N self-contained
     # tasks at once and get all answers back together, in ONE parent turn. Sibling
     # of `spawn_subagent` (single); this one always sync-joins — the children run
     # concurrently (their provider waits overlap on the reactor) and the combined

data/lib/insika/turn_output.rb

@@ -25,7 +25,7 @@ module Insika
   # deliberate:
   #
   # · The customer-visible stream is per MESSAGE, not per token. `ttft_ms` still
-  #   measures the provider's first token, so item 34's baselines stay comparable;
+  #   measures the provider's first token, so's baselines stay comparable;
   #   what moved is when the customer can read it. For the WhatsApp edge this
   #   changes nothing — the dispatcher accumulated the deltas into one message
   #   anyway — and the Studio keeps its live typing off `:intermediate`.
@@ -44,7 +44,7 @@ module Insika
     # decides and publishes once, at the end of the stage.
     attr_reader :candidate
 
-    # filter: Safety::OutputFilter | nil (RFC-0009 §3.2 — nil = stream untouched).
+    # filter: Safety::OutputFilter | nil (nil = stream untouched).
     # emit:   ->(type, data) — the Executor's emitter, already bound to the task.
     # public_intermediate: the agent opted this channel in (`edge_stream`), so the
     #   narration is TAGGED and `/v1/responses` gives it its own frame. Default false:

data/lib/insika/turn_state.rb

@@ -11,11 +11,11 @@ module Insika
                   :allowed_skills,
                   :chat,               # the turn's RubyLLM::Chat instance
                   :session,            # the turn's SessionStore::Session | nil (set at stage 2;
-                  #                      read by create_chat for the per-chat model pin, §10)
-                  :model_selection,    # resolved ModelSelection (v2, §10): model/provider/source/
+                  #                      read by create_chat for the per-chat model pin)
+                  :model_selection,    # resolved ModelSelection: model/provider/source/
                   #                      pinned/params/fallbacks. Set at stage 5; surfaced in usage.
                   :halt_reason,        # set by Middleware when short-circuiting (halt-as-FAILURE)
-                  :halt_response,      # set by a Middleware for the GRACEFUL halt (RFC-0009 §3.1):
+                  :halt_response,      # set by a Middleware for the GRACEFUL halt:
                   #                      the safe reply the turn completes with, WITHOUT touching the
                   #                      LLM. Distinct from halt_reason — a completion, not a failure.
                   :guardrail_block,    # audit metadata the guardrail sets alongside halt_response
@@ -25,7 +25,11 @@ module Insika
                   #                      (after_task); the Executor emits one :guardrail_flagged each.
                   :response_content,   # the turn's final assistant text, set at stage 6/on halt so the
                   #                      after_task validator can inspect it.
-                  :output_filter       # per-turn Safety::OutputFilter (nil = off); redacts the stream.
+                  :output_filter,      # per-turn Safety::OutputFilter (nil = off); redacts the stream.
+                  :stuck_outcome       # set by the signal_stuck system tool (WS5):
+                  #                      { reason:, message: } when the agent declared it cannot
+                  #                      proceed. The Executor tags the terminal event with
+                  #                      outcome: "stuck" and emits :turn_stuck. nil = normal turn.
 
     # Internal (not part of the contract): per-CALL correlation between RubyLLM's
     # tool callbacks and the tool decorators — `current_tool_call` keys the
@@ -34,7 +38,7 @@ module Insika
     #
     # They live in FIBER STORAGE, not in ivars, and that is the whole point:
     # `before_tool_call` → `tool.call` → `after_tool_result` all run in the SAME
-    # fiber, and with `ToolConcurrency` (item 30) there is one fiber PER CALL. A
+    # fiber, and with `ToolConcurrency` there is one fiber PER CALL. A
     # single slot on this shared object would let one in-flight call overwrite
     # another's — a side-effect recorded under the wrong id (so a resume skips the
     # wrong tool, or re-runs a non-idempotent one) and a mislabelled event. Both
@@ -56,7 +60,7 @@ module Insika
       Fiber[NAME_KEY] = name
     end
 
-    # Internal (§11 R1): the chat's message count RIGHT AFTER `assemble` (seeded
+    # Internal (R1): the chat's message count RIGHT AFTER `assemble` (seeded
     # history) and BEFORE `ask`. persist_turn slices `chat.messages.drop(baseline)`
     # to serialize the turn's real exchange — user + assistant(tool_calls) + tool
     # results + final assistant — into the transcript. nil = no chat recorded
@@ -70,13 +74,13 @@ module Insika
     # capability_registry or empty profile.capabilities (parity).
     attr_accessor :capability_names
 
-    # Internal (RFC-0015): the turn's resolved QueuePolicy. Read at stage 6 to decide
+    # Internal: the turn's resolved QueuePolicy. Read at stage 6 to decide
     # whether this run accepts steered messages, and how they are worded. Resolved once
     # per turn, in build_turn_state — an edit to the agent mid-run does not change the
     # rules the run started under.
     attr_accessor :queue_policy
 
-    # Internal (item 33): true when this turn re-enters the pipeline via
+    # Internal: true when this turn re-enters the pipeline via
     # resume_task/recovery. The EdgeLimiter reads it to NEVER re-count or block a
     # turn that was already admitted — a crash/pause under a saturated window must
     # not swallow a legitimate message with the rate-limit reply.
@@ -86,13 +90,13 @@ module Insika
     # (`remember` tool). Set in run_pipeline; nil = DEFAULT_TENANT in the MemoryStore.
     attr_accessor :tenant
 
-    # Internal (Phase 6/D2/G4): turn context deposited into the data-tools to
+    # Internal: turn context deposited into the data-tools to
     # resolve {{ctx.*}} (chat_id/agent_id/tenant/store_id) and emit
     # X-Chat-Id/X-Store-Id/X-Agent-Id. A Hash of symbols, set in run_pipeline.
     # Comes from the TURN, never from the model's args (R2). Distinct from `tenant` (memory).
     attr_accessor :turn_context
 
-    # Internal (Phase 6, observability): the turn's token usage (input/output/
+    # Internal (observability): the turn's token usage (input/output/
     # total/cached + model), captured from the provider's response at stage 6. Goes
     # to the terminal event (:task_completed) — feeds the usage of
     # /v1/responses and the Telemetry (OTEL). nil = turn with no model response
@@ -111,19 +115,19 @@ module Insika
     # by the coordinator for await(:approval)).
     attr_accessor :requires_approval, :approval_coordinator, :actor
 
-    # Internal (item 30 / D4): the turn's shared in-flight cap for tool calls —
+    # Internal: the turn's shared in-flight cap for tool calls —
     # ONE Async::Semaphore(tool_concurrency), installed by ToolAssembly#wrap_tools
     # and acquired by every ToolEnvelope, INCLUDING the ones tool_search promotes
     # mid-turn (they read it off the state, so the cap survives promotion).
     # nil = concurrency off: no gate, no overhead, serial execution unchanged.
     attr_accessor :tool_gate
 
-    # Item 30: parallel tool calls, resolved PER TURN and read by ChatBuilder
+    # parallel tool calls, resolved PER TURN and read by ChatBuilder
     # (whether to hand the gem `concurrency:`) and ToolAssembly (the gate's size).
     #
     # `requested_tool_concurrency` is what the operator configured;
     # `tool_concurrency` is what this turn actually gets. They differ for exactly
-    # one reason — D3: `Executor#request_approval` blocks on `actor.await(:approval)`,
+    # one reason —: `Executor#request_approval` blocks on `actor.await(approval)`,
     # and the mailbox is one queue per TASK. Two fibers waiting there share it,
     # `dequeue` wakes exactly one, the message is consumed, and the other fiber
     # hangs until `approval_timeout` (~1h). So a turn that can suspend for a human

data/lib/insika/turn_timing.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Insika
-  # Opt-in per-turn latency breakdown (item 34 / §13.1, action 1: locate the
+  # Opt-in per-turn latency breakdown (locate the
   # TTFB cost with real-turn data). OFF unless INSIKA_TURN_TIMING is set — when
   # off the Executor never allocates one and the hot path pays only `nil&.mark`.
   #
@@ -16,7 +16,7 @@ module Insika
   #
   # `ttft_ms` is the PROVIDER's first token, not the first byte the customer can
   # read: TurnOutput publishes a message once it ends, so the customer-visible
-  # answer lands inside `gen_ms`. Measuring the provider is the point — item 34's
+  # answer lands inside `gen_ms`. Measuring the provider is the point —'s
   # baselines (~720 ms, provider-bound) stay comparable across that change.
   class TurnTiming
     # EnvSchema owns "is this flag on?" (1/true/yes/on) — the same predicate that

data/lib/insika/usage_ledger.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Insika
-  # Fixed-window counters on the KV Store (item 33 / §12 G7): the durable side of
+  # Fixed-window counters on the KV Store: the durable side of
   # the edge limits. One scope, keys shaped "kind:id:window_start" — the window
   # start bucketed on the epoch keeps every process/worker on the SAME bucket
   # without coordination (the Store's transaction serializes the read-modify-write,

data/lib/insika/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Insika
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end