insika 0.0.1 → 0.2.0

data/lib/insika/tools/tool_search.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+
+module Insika
+  module Tools
+    # Level 2 of TOOLS progressive disclosure (analog of LoadSkill):
+    # searches the deferred catalog and PROMOTES the relevant ones into the live chat via
+    # chat.with_tools (verified to propagate on the next round of the same `ask`
+    # in ruby_llm 1.16). `require "ruby_llm"` stays in THIS file (it inherits from
+    # RubyLLM::Tool) — it does not enter lib/insika.rb; the Executor loads it lazily in
+    # configure_chat (like LoadSkill).
+    class ToolSearch < RubyLLM::Tool
+      description "Searches and enables additional tools by describing the need"
+      param :query, desc: "What you need to do (e.g.: 'send email', 'generate invoice')"
+
+      # RubyLLM::Tool#name derives from self.class.name — for a nested class
+      # (Insika::Tools::ToolSearch) it produces "insika--tools--tool_search", not
+      # "tool_search". Explicit override: the name the model calls must match
+      # the catalog/docs/tests.
+      def name = "tool_search"
+
+      def initialize(catalog, deferred_allowed, chat, tool_registry:, event_stream:,
+                     checkpoint_store:, state:)
+        @catalog = catalog
+        @deferred_allowed = Array(deferred_allowed).map(&:to_s)
+        @chat = chat
+        @tool_registry = tool_registry
+        @event_stream = event_stream
+        @checkpoint_store = checkpoint_store
+        @state = state
+        @promoted = [] # names already promoted IN THIS chat — idempotency
+        super()
+      end
+
+      def execute(query:)
+        matches = @catalog.search(query, within: @deferred_allowed)
+        emit_tool_search(query, matches.map(&:name))
+        if matches.empty?
+          return { matched: [], message: "no tool found for '#{query}'" }
+        end
+
+        new_matches = matches.reject { |m| @promoted.include?(m.name) }
+        promote(new_matches) unless new_matches.empty?
+
+        { matched: matches.map { |m| describe(m) } }
+      end
+
+      private
+
+      # Instantiates (via tool_registry), wraps in the SAME ToolEnvelope as the eager
+      # ones (profile timeout + state's skip_side_effects) and promotes via
+      # chat.with_tools. A NotFoundError (misaligned catalog) drops only
+      # that match — the search does not break.
+      def promote(entries)
+        timeout = @state.profile.limits[:tool_timeout] || 60
+        wrapped = entries.filter_map do |entry|
+          tool = @tool_registry.resolve(entry.name)
+          @promoted << entry.name
+          ToolEnvelope.new(tool, state: @state, checkpoint_store: @checkpoint_store,
+                                 tool_registry: @tool_registry, timeout: timeout,
+                                 skip_side_effects: Array(@state.skip_side_effects))
+        rescue Insika::NotFoundError
+          nil
+        end
+        @chat.with_tools(*wrapped) unless wrapped.empty?
+      end
+
+      # Mirrors :skill_activated, but emitted by the tool itself (it has event_stream/
+      # state in the constructor). Without a monotonic `seq` (private to the Executor) — a
+      # documented gap, not a blocker.
+      def emit_tool_search(query, matched_names)
+        @event_stream.emit(Insika::Event.new(
+                             type: :tool_search,
+                             data: { query: query, matched: matched_names },
+                             meta: { task_id: @state.task.id, session_id: @state.task.session_id }
+                           ))
+      end
+
+      def describe(entry)
+        tool = @tool_registry.resolve(entry.name)
+        {
+          name: entry.name,
+          description: entry.description,
+          parameters: tool.parameters.transform_values do |p|
+            { type: p.type, description: p.description, required: p.required }
+          end
+        }
+      rescue Insika::NotFoundError
+        { name: entry.name, description: entry.description, parameters: {} }
+      end
+    end
+  end
+end

data/lib/insika/turn_output.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module Insika
+  # WHAT REACHES THE CUSTOMER — the turn's publishing rule.
+  #
+  # A turn is not one assistant message. Between the user's message and the answer
+  # the model may narrate the loop ("deixa eu buscar isso pra você"), apologise for
+  # a tool that failed, or — when it has no tool to call — reason in prose. All of
+  # it arrives as ordinary `content` chunks, indistinguishable at the token level
+  # from the answer, so streaming every chunk straight to `:content` published all
+  # of it. Running a real store's prompt proved what that costs: 132 deltas of an
+  # English monologue ("Let me check the tools I actually have… Actually, let me
+  # reconsider.") went out as `response.output_text.delta` — the bytes a WhatsApp
+  # customer would have read.
+  #
+  # The rule, enforced here: **`:content` carries the ANSWER — the text of the
+  # assistant message that ENDS the turn.** Everything else rides `:intermediate`,
+  # which the Studio and the trace render (that is the operator's window into what
+  # the model narrated) and which `/v1/responses` deliberately does not translate,
+  # exactly like `:thinking`.
+  #
+  # Which message is the last one is only knowable when it ends — a message that
+  # carries tool calls is never the answer — so text is buffered per message and
+  # published at the boundary (RubyLLM's `after_message`). Two consequences, both
+  # deliberate:
+  #
+  # · The customer-visible stream is per MESSAGE, not per token. `ttft_ms` still
+  #   measures the provider's first token, so's baselines stay comparable;
+  #   what moved is when the customer can read it. For the WhatsApp edge this
+  #   changes nothing — the dispatcher accumulated the deltas into one message
+  #   anyway — and the Studio keeps its live typing off `:intermediate`.
+  # · A turn that dies mid-message publishes nothing. Half a sentence was never an
+  #   answer; the fragment is still on the stream as `:intermediate` for whoever is
+  #   debugging it.
+  #
+  # `halt_when` (PR #130) is the one case where narration IS the turn: the tool
+  # already answered the customer, and the model's lead-in ("vou te inscrever
+  # agora") is all the turn is worth. That text is retained and published by the
+  # Executor's halt branch — see #halt_text.
+  class TurnOutput
+    # The text of the message that ended the turn, or nil if no boundary said so.
+    # It is a CANDIDATE, not the published answer: the `:agent` after-hook runs
+    # after the message ends and may replace the whole response, so the Executor
+    # decides and publishes once, at the end of the stage.
+    attr_reader :candidate
+
+    # filter: Safety::OutputFilter | nil (nil = stream untouched).
+    # emit:   ->(type, data) — the Executor's emitter, already bound to the task.
+    # public_intermediate: the agent opted this channel in (`edge_stream`), so the
+    #   narration is TAGGED and `/v1/responses` gives it its own frame. Default false:
+    #   an internal event stays internal unless someone said otherwise.
+    def initialize(filter:, emit:, public_intermediate: false)
+      @filter = filter
+      @emit = emit
+      @public_intermediate = public_intermediate
+      @pending = +""           # text of the message currently streaming
+      @last_intermediate = +"" # text of the last message that turned out NOT to be the answer
+      @candidate = nil
+    end
+
+    # One provider chunk. Publishes the redacted slice as `:intermediate` — live,
+    # chunk by chunk, because the operator surfaces want to watch it happen — and
+    # holds it until the message boundary decides what it was.
+    def push(text)
+      slice = @filter ? @filter.push(text) : text.to_s
+      return if slice.empty?
+
+      @pending << slice
+      emit_intermediate(slice)
+    end
+
+    # A message ended (RubyLLM `after_message`). Only an assistant message decides
+    # anything — a `role: tool` result is the gem's bookkeeping. A message carrying
+    # tool calls is intermediate by definition: the model asked for something, so
+    # it was not done talking.
+    def message_ended(message)
+      return unless assistant?(message)
+
+      flush
+      text = @pending
+      @pending = +""
+      tool_calls?(message) ? @last_intermediate = text : @candidate = text
+    end
+
+    # Releases the redactor's retained tail (the sliding buffer holds back a value
+    # that might still be growing into a match) into the current message.
+    def flush
+      return unless @filter
+
+      tail = @filter.flush.to_s
+      return if tail.empty?
+
+      @pending << tail
+      emit_intermediate(tail)
+    end
+
+    # Emits the answer and returns it. Empty text emits no event — an empty turn is
+    # what the consumer suppresses.
+    def publish(text)
+      answer = text.to_s
+      @emit.call(:content, { delta: answer }) unless answer.empty?
+      answer
+    end
+
+    # The narration of the message that halted the turn. Normally the message
+    # boundary already moved it aside (`@last_intermediate`); a transport that does
+    # not report boundaries leaves it in `@pending`, and the lead-in is worth the
+    # same either way.
+    def halt_text = @last_intermediate.empty? ? @pending : @last_intermediate
+
+    private
+
+    # `public: true` is the whole difference between an event the edge drops and one
+    # it translates. The flag travels on the EVENT because `frame_for` is a pure
+    # static mapper with no agent in scope — the profile is read once, here.
+    def emit_intermediate(text)
+      data = { delta: text }
+      data[:public] = true if @public_intermediate
+      @emit.call(:intermediate, data)
+    end
+
+    def assistant?(message) = field(message, :role).to_s == "assistant"
+
+    # RubyLLM::Message answers `tool_call?`; a double may only carry the field.
+    def tool_calls?(message)
+      return !!message.tool_call? if message.respond_to?(:tool_call?)
+
+      calls = field(message, :tool_calls)
+      calls.respond_to?(:empty?) ? !calls.empty? : !calls.nil?
+    end
+
+    def field(message, name)
+      return message.public_send(name) if message.respond_to?(name)
+      return message[name] || message[name.to_s] if message.respond_to?(:[])
+
+      nil
+    end
+  end
+end

data/lib/insika/turn_state.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Insika
+  # MUTABLE on purpose (the only exception to the Data types):
+  # the Middleware MODIFIES the execution — the links write into these fields.
+  class TurnState
+    attr_reader :task, :profile, :turn # turn identity (1-based)
+    attr_accessor :message,            # input (Middleware may rewrite)
+                  :context,            # ContextPackage from the Builder
+                  :allowed_tools,      # Resolution from the Policy Engine
+                  :allowed_skills,
+                  :chat,               # the turn's RubyLLM::Chat instance
+                  :session,            # the turn's SessionStore::Session | nil (set at stage 2;
+                  #                      read by create_chat for the per-chat model pin)
+                  :model_selection,    # resolved ModelSelection: model/provider/source/
+                  #                      pinned/params/fallbacks. Set at stage 5; surfaced in usage.
+                  :halt_reason,        # set by Middleware when short-circuiting (halt-as-FAILURE)
+                  :halt_response,      # set by a Middleware for the GRACEFUL halt:
+                  #                      the safe reply the turn completes with, WITHOUT touching the
+                  #                      LLM. Distinct from halt_reason — a completion, not a failure.
+                  :guardrail_block,    # audit metadata the guardrail sets alongside halt_response
+                  #                      ({category:, source:, action:, detail:}); the Executor (single
+                  #                      emitter) turns it into :guardrail_blocked.
+                  :guardrail_flags,    # [{category:, source:, detail:}] appended by the OutputValidator
+                  #                      (after_task); the Executor emits one :guardrail_flagged each.
+                  :response_content,   # the turn's final assistant text, set at stage 6/on halt so the
+                  #                      after_task validator can inspect it.
+                  :output_filter,      # per-turn Safety::OutputFilter (nil = off); redacts the stream.
+                  :stuck_outcome       # set by the signal_stuck system tool (WS5):
+                  #                      { reason:, message: } when the agent declared it cannot
+                  #                      proceed. The Executor tags the terminal event with
+                  #                      outcome: "stuck" and emits :turn_stuck. nil = normal turn.
+
+    # Internal (not part of the contract): per-CALL correlation between RubyLLM's
+    # tool callbacks and the tool decorators — `current_tool_call` keys the
+    # side-effect checkpoint / resume skip / trace, `current_tool_name` labels the
+    # :tool_result event.
+    #
+    # They live in FIBER STORAGE, not in ivars, and that is the whole point:
+    # `before_tool_call` → `tool.call` → `after_tool_result` all run in the SAME
+    # fiber, and with `ToolConcurrency` there is one fiber PER CALL. A
+    # single slot on this shared object would let one in-flight call overwrite
+    # another's — a side-effect recorded under the wrong id (so a resume skips the
+    # wrong tool, or re-runs a non-idempotent one) and a mislabelled event. Both
+    # silent. One writer per fiber needs no lock; serial execution is unchanged,
+    # since a lone fiber writes and reads its own storage.
+    #
+    # Read/written ONLY through here so the rule has one home.
+    CALL_KEY = :insika_tool_call
+    NAME_KEY = :insika_tool_name
+
+    def current_tool_call = Fiber[CALL_KEY]
+    def current_tool_name = Fiber[NAME_KEY]
+
+    def current_tool_call=(call)
+      Fiber[CALL_KEY] = call
+    end
+
+    def current_tool_name=(name)
+      Fiber[NAME_KEY] = name
+    end
+
+    # Internal (R1): the chat's message count RIGHT AFTER `assemble` (seeded
+    # history) and BEFORE `ask`. persist_turn slices `chat.messages.drop(baseline)`
+    # to serialize the turn's real exchange — user + assistant(tool_calls) + tool
+    # results + final assistant — into the transcript. nil = no chat recorded
+    # (workflow/halt) → persist_turn falls back to the {user, assistant} pair.
+    attr_accessor :chat_baseline
+
+    # Internal: impl_name(String) -> STABLE name of the capability that
+    # resolved it, computed by resolve_capabilities BEFORE the policy_request and
+    # consulted AFTER @policy_engine.decide, at the post-Policy junction, to
+    # decide which impls enter as Capability::ResolvedTool. {} = no
+    # capability_registry or empty profile.capabilities (parity).
+    attr_accessor :capability_names
+
+    # Internal: the turn's resolved QueuePolicy. Read at stage 6 to decide
+    # whether this run accepts steered messages, and how they are worded. Resolved once
+    # per turn, in build_turn_state — an edit to the agent mid-run does not change the
+    # rules the run started under.
+    attr_accessor :queue_policy
+
+    # Internal: true when this turn re-enters the pipeline via
+    # resume_task/recovery. The EdgeLimiter reads it to NEVER re-count or block a
+    # turn that was already admitted — a crash/pause under a saturated window must
+    # not swallow a legitimate message with the rate-limit reply.
+    attr_accessor :resumed
+
+    # Internal (memory): the turn's tenant (from the Command), scope of the write path
+    # (`remember` tool). Set in run_pipeline; nil = DEFAULT_TENANT in the MemoryStore.
+    attr_accessor :tenant
+
+    # Internal: turn context deposited into the data-tools to
+    # resolve {{ctx.*}} (chat_id/agent_id/tenant/store_id) and emit
+    # X-Chat-Id/X-Store-Id/X-Agent-Id. A Hash of symbols, set in run_pipeline.
+    # Comes from the TURN, never from the model's args (R2). Distinct from `tenant` (memory).
+    attr_accessor :turn_context
+
+    # Internal (observability): the turn's token usage (input/output/
+    # total/cached + model), captured from the provider's response at stage 6. Goes
+    # to the terminal event (:task_completed) — feeds the usage of
+    # /v1/responses and the Telemetry (OTEL). nil = turn with no model response
+    # (workflow) or provider without counts.
+    attr_accessor :usage
+
+    # Internal (Tool Search): ids of side-effects already completed in the
+    # interrupted turn, propagated to the tools PROMOTED by tool_search (the same `skip`
+    # that the eager tools' wrap_tools receives). Set in run_pipeline;
+    # nil = new turn (Array(nil) => []).
+    attr_accessor :skip_side_effects
+
+    # Approval gate. `requires_approval` = names of tools that require
+    # approval (Resolution); `approval_coordinator` = object (the Executor) that
+    # creates the PendingAction/suspends/waits; `actor` = the turn's mailbox (used
+    # by the coordinator for await(:approval)).
+    attr_accessor :requires_approval, :approval_coordinator, :actor
+
+    # Internal: the turn's shared in-flight cap for tool calls —
+    # ONE Async::Semaphore(tool_concurrency), installed by ToolAssembly#wrap_tools
+    # and acquired by every ToolEnvelope, INCLUDING the ones tool_search promotes
+    # mid-turn (they read it off the state, so the cap survives promotion).
+    # nil = concurrency off: no gate, no overhead, serial execution unchanged.
+    attr_accessor :tool_gate
+
+    # parallel tool calls, resolved PER TURN and read by ChatBuilder
+    # (whether to hand the gem `concurrency:`) and ToolAssembly (the gate's size).
+    #
+    # `requested_tool_concurrency` is what the operator configured;
+    # `tool_concurrency` is what this turn actually gets. They differ for exactly
+    # one reason —: `Executor#request_approval` blocks on `actor.await(approval)`,
+    # and the mailbox is one queue per TASK. Two fibers waiting there share it,
+    # `dequeue` wakes exactly one, the message is consumed, and the other fiber
+    # hangs until `approval_timeout` (~1h). So a turn that can suspend for a human
+    # runs its tools serially. Per-TURN and not per-profile because
+    # `requires_approval` comes from the Resolution: it can be empty on a turn
+    # whose profile does list approvals.
+    def requested_tool_concurrency
+      n = ((profile.respond_to?(:limits) && profile.limits) || {})[:tool_concurrency].to_i
+      n > 1 ? n : nil
+    end
+
+    def tool_concurrency
+      return nil unless Array(requires_approval).empty?
+
+      requested_tool_concurrency
+    end
+
+    def initialize(task:, profile:, turn:, message:)
+      @task = task
+      @profile = profile
+      @turn = turn
+      @message = message
+      @capability_names = {}
+      # Fiber storage is INHERITED by fibers created later, so a turn spawned from
+      # inside a tool call (a subagent child) would start out carrying its
+      # parent's correlation. Clearing at turn start keeps a child from keying its
+      # own side-effects under the parent's tool_call id.
+      self.current_tool_call = nil
+      self.current_tool_name = nil
+    end
+  end
+end

data/lib/insika/turn_timing.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Insika
+  # Opt-in per-turn latency breakdown (locate the
+  # TTFB cost with real-turn data). OFF unless INSIKA_TURN_TIMING is set — when
+  # off the Executor never allocates one and the hot path pays only `nil&.mark`.
+  #
+  # Splits a turn into the three windows that answer "is TTFB local or provider?":
+  #   prep_ms  — prep_start -> ask: ALL local work before the provider call
+  #              (context build, policy, guardrail detectors, chat assembly).
+  #   ttft_ms  — ask -> first_token: the provider round-trip to the 1st token.
+  #   gen_ms   — first_token -> done: streaming the rest of the response.
+  #
+  # Marks are monotonic; `mark` is first-write-wins so `first_token` records the
+  # FIRST content chunk even though it is called on every chunk.
+  #
+  # `ttft_ms` is the PROVIDER's first token, not the first byte the customer can
+  # read: TurnOutput publishes a message once it ends, so the customer-visible
+  # answer lands inside `gen_ms`. Measuring the provider is the point —'s
+  # baselines (~720 ms, provider-bound) stay comparable across that change.
+  class TurnTiming
+    # EnvSchema owns "is this flag on?" (1/true/yes/on) — the same predicate that
+    # validates the :boolean keys, so a spelling `insika env` accepts is a spelling
+    # the reader honours.
+    def self.enabled?(env = ENV)
+      Insika::EnvSchema.truthy?(Insika::EnvSchema.read("INSIKA_TURN_TIMING", env))
+    end
+
+    def initialize
+      @marks = {}
+    end
+
+    def mark(name)
+      @marks[name] ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    # -> Hash of phase deltas in ms (only the windows whose endpoints both fired;
+    # a workflow turn has no ask/first_token, so those are simply absent).
+    def to_h
+      {
+        prep_ms: delta(:prep_start, :ask),
+        ttft_ms: delta(:ask, :first_token),
+        gen_ms: delta(:first_token, :done),
+        total_ms: delta(:prep_start, :done)
+      }.compact
+    end
+
+    private
+
+    def delta(from, to)
+      return nil unless @marks[from] && @marks[to]
+
+      ((@marks[to] - @marks[from]) * 1000).round(2)
+    end
+  end
+end

data/lib/insika/usage_ledger.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Insika
+  # Fixed-window counters on the KV Store: the durable side of
+  # the edge limits. One scope, keys shaped "kind:id:window_start" — the window
+  # start bucketed on the epoch keeps every process/worker on the SAME bucket
+  # without coordination (the Store's transaction serializes the read-modify-write,
+  # in-process via the fiber semaphore and cross-process via BEGIN IMMEDIATE).
+  #
+  # Growth is bounded: each `add` garbage-collects the (kind, id) pair's PREVIOUS
+  # window key, so an active entity holds at most 2 keys at any time and an idle
+  # one converges to 1.
+  class UsageLedger
+    SCOPE = "usage_counters"
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # Adds `by` to the current window's counter of (kind, id) -> the NEW total.
+    def add(kind, id, window:, by: 1, now: Time.now)
+      start = window_start(now, window)
+      key = key_for(kind, id, start)
+      @store.transaction do
+        total = @store.get(SCOPE, key).to_i + by
+        @store.set(SCOPE, key, total)
+        @store.delete(SCOPE, key_for(kind, id, start - window))
+        total
+      end
+    end
+
+    # Current window's total for (kind, id). Missing/expired -> 0.
+    def count(kind, id, window:, now: Time.now)
+      @store.get(SCOPE, key_for(kind, id, window_start(now, window))).to_i
+    end
+
+    private
+
+    def window_start(now, window)
+      (now.to_i / window) * window
+    end
+
+    def key_for(kind, id, start)
+      "#{kind}:#{id}:#{start}"
+    end
+  end
+end

data/lib/insika/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Insika
-  VERSION = "0.0.1"
+  VERSION = "0.2.0"
 end