insika 0.8.0 → 1.0.1

data/lib/insika/fence.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Insika
+  # Third-party text — a product description, a review, an FAQ body, a fact a
+  # customer dictated — reaches the model as bytes, and the model reads every
+  # one of them. Those bytes can carry zero-width characters, bidi overrides, a
+  # forged "\n\nassistant:" turn marker or a tag shaped like the engine's own
+  # transcript markup. This is the ONE sanitizer for that class of input:
+  # pure Ruby, stdlib only, no IO, every pattern bounded so a hostile megabyte
+  # finishes in linear time on the reactor.
+  #
+  # It never re-wraps. The engine already renders data inside fixed labels
+  # (<memory>, <knowledge>, a tool result) and the FenceNotice provider names
+  # them to the model; what this module does is make sure the DATA cannot
+  # reproduce those labels or a turn boundary.
+  module Fence
+    module_function
+
+    # Zero-width, bidi and format controls: the usual carriers of hidden text.
+    INVISIBLE = /[\u00AD\u061C\u180E\u200B-\u200F\u2028-\u202E\u2060-\u2064\u2066-\u2069\u206A-\u206F\uFE00-\uFE0F\uFEFF\uFFF9-\uFFFB\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]/
+
+    # C0/C1 controls except tab, newline and carriage return.
+    CONTROL = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/
+
+    # Tag names that read as transcript or tool-call markup to a model, plus the
+    # labels the engine itself renders around data. A copy of any of them inside
+    # DATA is a forgery — `<fact>` inside a fact value would end the fact early.
+    TAG_NAMES = %w[
+      system human user assistant transcript conversation
+      tool_result tool_use function_calls function_results invoke
+      memory knowledge fact note concept briefing conversation_summary request_context
+    ].freeze
+
+    # Only tag-SHAPED text: bare (`<system>`), closing (`</memory>`), self-closing,
+    # optionally namespaced (`<ns:invoke>`), with at most eight `name="value"`
+    # attributes. Bare words are not attributes, so `<system requirements>`
+    # passes. Quantifiers are bounded and non-adjacent — linear on unclosed input.
+    TAG_ATTRS = /(?:[ \t]+[\w:.-]{1,40}[ \t]*=[ \t]*(?:"[^"]{0,200}"|'[^']{0,200}'|[^\s"'>]{1,200})){0,8}/
+    TAG = /<[ \t]*\/?[ \t]*(?:[a-z][\w.-]{0,30}:)?(?:#{TAG_NAMES.join('|')})\b#{TAG_ATTRS.source}[ \t]*\/?>/i
+
+    # Provider special tokens (`<|im_start|>`, `<|endoftext|>`).
+    SPECIAL_TOKEN = /<\|[^|<>\r\n]{1,64}\|>/
+
+    # A forged turn boundary: a blank line (or the very start), a full role word,
+    # a colon. A mid-sentence "user:" and a one-letter list marker ("A:") do not
+    # match. The colon becomes " -", so the words survive as prose.
+    TURN_MARKER = /(\A[ \t]*|(?:\r\n|\r|\n)[ \t]*(?:\r\n|\r|\n)[ \t]*)(human|assistant|system|user)[ \t]*:/i
+
+    REMOVED = "[removed]"
+    TRUNCATED = " …[truncated]"
+    # One string leaf after the envelope; `Settings fencing.max_chars` overrides.
+    DEFAULT_MAX_CHARS = 12_000
+
+    # -> String. `max_chars` bounds the result INCLUDING the suffix.
+    def sanitize_text(text, max_chars: nil)
+      s = Coercion.utf8(text).unicode_normalize(:nfkc)
+      s = s.gsub(INVISIBLE, "").gsub(CONTROL, " ")
+      # To a fixpoint: a tag nested inside another (`</memory</memory>>`) must
+      # not reassemble once the inner one goes. Each round shrinks the string or
+      # stops, so the loop is bounded by the number of tags.
+      loop do
+        stripped = s.gsub(TAG, REMOVED).gsub(SPECIAL_TOKEN, REMOVED)
+        break if stripped == s
+
+        s = stripped
+      end
+      s = s.gsub(TURN_MARKER) { "#{Regexp.last_match(1)}#{Regexp.last_match(2)} -" }
+      cap(s, max_chars)
+    end
+
+    # Walks Hash/Array leaves; String leaves are sanitized, keys and every other
+    # type pass through untouched (numbers, booleans, nil are not text).
+    def sanitize_value(obj, max_chars: nil)
+      case obj
+      when String then sanitize_text(obj, max_chars: max_chars)
+      when Hash then obj.transform_values { |v| sanitize_value(v, max_chars: max_chars) }
+      when Array then obj.map { |v| sanitize_value(v, max_chars: max_chars) }
+      else obj
+      end
+    end
+
+    # The per-agent switch, read the same way everywhere (envelope, providers,
+    # doctor): an operator's `fencing true`, a form's "1" or a JSON `true`.
+    def enabled?(profile)
+      profile.respond_to?(:fencing) && Coercion.truthy?(profile.fencing)
+    end
+
+    def cap(text, max_chars)
+      return text if max_chars.nil? || text.length <= max_chars
+      return text[0, max_chars] if max_chars <= TRUNCATED.length
+
+      text[0, max_chars - TRUNCATED.length] + TRUNCATED
+    end
+    private_class_method :cap
+  end
+end

data/lib/insika/golden_store.rb

@@ -148,6 +148,9 @@ module Insika
       # when present. A case that lost its reference in a round-trip would stop being
       # compared against the incumbent and the report would look identical.
       h["reference"] = golden.reference unless golden.reference.empty?
+      # And `state`: a snapshot case that lost its snapshot would replay from an
+      # empty conversation and fail for a reason that has nothing to do with the agent.
+      h["state"] = golden.state unless golden.state.empty?
       # `tenant` follows the same omit-when-default rule: "platform" is the
       # loader's own default, so leaving it out reproduces the same case; an
       # explicit tenant is a case's isolation boundary and must never round-trip away.

data/lib/insika/harvest.rb

@@ -89,7 +89,7 @@ module Insika
       # first-10 audit reads back.
 
       # ask:   ->(prompt) { "<raw model text>" } | something answering #content
-      #        (+ #input_tokens/#output_tokens/#cached_tokens for cost).
+      #        (+ RubyLLM-compatible #tokens for cost).
       # model: the ref recorded as the candidate's `proposer` ("utility_model"
       #        default).
       attr_reader :model
@@ -110,7 +110,7 @@ module Insika
       # max_proposals: cap on surviving raw skills. Drops counted, never fixed.
       def mine(prompt:, message_counts:, max_proposals: 10)
         answer = @ask.call(prompt)
-        raw = parse(text_of(answer))
+        raw = parse(answer)
         skills = []
         dropped = DROP_KEYS.to_h { |k| [k, 0] }
         seen = {}
@@ -142,20 +142,25 @@ module Insika
       # discipline). The cached prefix is INCLUDED in the spent total (E1: the
       # run's cost is the harvest's only side spend).
       def cost_of(answer)
-        return nil unless answer.respond_to?(:input_tokens) && answer.respond_to?(:output_tokens)
+        return nil unless answer.respond_to?(:tokens)
 
-        input = answer.input_tokens.to_i
-        output = answer.output_tokens.to_i
-        cached = answer.respond_to?(:cached_tokens) ? answer.cached_tokens.to_i : 0
-        spent = input + output + cached
+        usage = answer.tokens
+        input = usage.input.to_i
+        output = usage.output.to_i
+        cached = usage.cache_read.to_i
+        spent = input + output + cached + usage.cache_write.to_i
         spent.positive? ? { "spent" => spent, "cached" => cached } : nil
       end
 
-      # Fences stripped, parsed STRICTLY (the Proposer's discipline): a model
-      # that improvises a schema fails here instead of producing half a skill.
-      def parse(raw)
-        body = raw.strip.gsub(/\A```(?:json)?\s*|\s*```\z/, "")
-        parsed = JSON.parse(body)
+      # Native responses own JSON parsing; injected text asks retain their
+      # existing fenced-JSON contract. Domain validation still runs below.
+      def parse(answer)
+        parsed = if answer.respond_to?(:parsed)
+                   answer.parsed
+                 else
+                   JSON.parse(text_of(answer).strip.gsub(/\A```(?:json)?\s*|\s*```\z/, ""))
+                 end
+        parsed = parsed["items"] if parsed.is_a?(Hash)
         raise Unusable, "the miner's answer is not an array" unless parsed.is_a?(Array)
 
         parsed
@@ -214,6 +219,10 @@ module Insika
         return nil if ref.nil?
 
         provider, model = split_ref(ref)
+        # A bare "model" ref falls back to the agent's own provider (stamped
+        # by RunHarvest) — RubyLLM refuses a nil provider with
+        # assume_model_exists, so a bare ref used to fail every run.
+        provider ||= Coercion.presence(config && config["provider"])
         factory = ask_factory || ->(m, p) { ruby_llm_ask(m, p, llm: llm) }
         Miner.new(ask: factory.call(model, provider), model: ref)
       end
@@ -232,10 +241,13 @@ module Insika
         require "ruby_llm"
         llm ||= RubyLLM
         lambda do |prompt|
+          schema = { "type" => "object", "properties" => { "items" => SKILL_SCHEMA.json_schema },
+                     "required" => ["items"], "additionalProperties" => false }
           llm.chat(model: model, provider: provider, assume_model_exists: true)
-             .with_temperature(0).ask(prompt)
+             .with_temperature(0).with_schema("schema" => schema, "strict" => false)
+             .ask("#{prompt}\n\nReturn the requested array inside a JSON object with the key \"items\".")
         end
       end
     end
   end
-end
+end

data/lib/insika/harvest_engine.rb

@@ -125,6 +125,7 @@ module Insika
     def due_sessions
       due = []
       @session_store.each_id do |id|
+        next if Insika::SessionStore.synthetic?(id) # not customers — and skips reading the record
         session = @session_store.find(id)
         next unless session
         next if @harvest_store.mined?(id)

data/lib/insika/harvest_store.rb

@@ -196,6 +196,18 @@ module Insika
 
     # gated -> awaiting_approval. The refusal parks: a conversion REFUSAL
     # keeps the candidate AT gated (the page shows the ruler's hole).
+    # A parked candidate (gated on a missing ruler: no judge, no criterion,
+    # no origin) goes back to pending so the operator can re-run the gate
+    # once the hole is fixed. The last reports stay until the new gate
+    # overwrites them. gated -> pending only.
+    def reopen(candidate_id)
+      update_candidate(candidate_id) do |record|
+        guard_candidate_state!(record, "gated")
+        record["status"] = "pending"
+        record["updated_at"] = timestamp
+      end
+    end
+
     def mark_awaiting(candidate_id)
       update_candidate(candidate_id) do |record|
         guard_candidate_state!(record, "gated")

data/lib/insika/http_client.rb

@@ -24,8 +24,21 @@ module Insika
 
     class ResponseTooLarge < Insika::Error; end
 
-    def initialize(max_bytes: MAX_BYTES)
+    DNS_TTL = 60 # seconds
+    # A connection idle longer than this is reopened instead of reused. Kept well
+    # under common server idle timeouts: Net::HTTP retries only idempotent methods,
+    # so a POST on a socket the server already closed would fail the tool call.
+    KEEP_ALIVE = 5 # seconds
+    MAX_IDLE_PER_HOST = 16
+
+    def initialize(max_bytes: MAX_BYTES, dns_ttl: DNS_TTL, keep_alive: KEEP_ALIVE)
       @max_bytes = max_bytes
+      @dns_ttl = dns_ttl
+      @keep_alive = keep_alive
+      @dns = {}
+      @idle = Hash.new { |h, k| h[k] = [] }
+      @pid = Process.pid
+      @lock = Mutex.new
     end
 
     def request(method:, url:, headers: {}, body: nil, timeout: nil)
@@ -34,9 +47,7 @@ module Insika
       headers.each { |k, v| req[k] = v }
       req.body = body if body && !body.to_s.empty?
 
-      t = timeout || DEFAULT_TIMEOUT
-      opts = { use_ssl: uri.scheme == "https", open_timeout: t, read_timeout: t }
-      Net::HTTP.start(uri.host, uri.port, opts) do |http|
+      with_connection(uri, timeout || DEFAULT_TIMEOUT) do |http|
         result = nil
         http.request(req) do |resp|
           # Accumulate in BINARY: Net::HTTP yields ASCII-8BIT chunks and a
@@ -55,9 +66,81 @@ module Insika
           # The redirect TARGET, so a moved API is reported as "moved to <url>"
           # instead of a bare 3xx with the empty body servers send with it.
           result[:location] = resp["location"].to_s if resp["location"]
+          # The server's own processing time (Rack::Runtime's X-Runtime, seconds),
+          # so a slow call can be split into server work and everything else.
+          runtime = Float(resp["x-runtime"], exception: false)
+          result[:server_ms] = (runtime * 1000).round if runtime
         end
         result
       end
     end
+
+    private
+
+    # Reuses an idle keep-alive connection to the same origin (a new TLS connection
+    # costs several round trips). One caller holds a connection at a time; one that
+    # raised is closed, never returned, since its state is unknown.
+    def with_connection(uri, timeout)
+      key = [uri.scheme, uri.host, uri.port]
+      http = checkout(key) || connect(uri, timeout)
+      http.read_timeout = timeout
+      result = yield http
+      checkin(key, http)
+      result
+    rescue StandardError
+      http&.finish if http&.started?
+      raise
+    end
+
+    def connect(uri, timeout)
+      # `ipaddr` connects to the cached address; TLS (SNI, certificate check) and the
+      # Host header still use the hostname.
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.ipaddr = address(uri.host)
+      http.use_ssl = uri.scheme == "https"
+      http.open_timeout = timeout
+      http.keep_alive_timeout = @keep_alive
+      http.start
+    end
+
+    def checkout(key)
+      @lock.synchronize do
+        reset_after_fork
+        @idle[key].pop
+      end
+    end
+
+    def checkin(key, http)
+      kept = @lock.synchronize do
+        next false if @idle[key].size >= MAX_IDLE_PER_HOST
+
+        @idle[key].push(http)
+        true
+      end
+      http.finish unless kept
+    end
+
+    # A forked child must not share its parent's sockets.
+    def reset_after_fork
+      return if @pid == Process.pid
+
+      @pid = Process.pid
+      @idle.clear
+    end
+
+    # Net::HTTP resolves with getaddrinfo, which blocks the whole reactor under
+    # Falcon: a burst of N tool calls queued N lookups and froze every turn of the
+    # process meanwhile. Resolving once per host per TTL makes that rare.
+    # ponytail: the lookup on a miss still blocks; a fiber-aware resolver removes it
+    # if one host per minute ever shows up in a profile.
+    def address(host)
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      hit = @lock.synchronize { @dns[host] }
+      return hit[0] if hit && hit[1] > now
+
+      ip = Addrinfo.getaddrinfo(host, nil, nil, :STREAM).first.ip_address
+      @lock.synchronize { @dns[host] = [ip, now + @dns_ttl] }
+      ip
+    end
   end
 end

data/lib/insika/kits.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Insika
+  # Kits: named bundles of skills, tools and tool groups that agents subscribe to
+  # (AgentProfile#kits). Stored in Settings["kits"]; applied at READ time by
+  # AgentDefaults::ProfileSource, so editing a kit reaches every subscriber on its
+  # next turn and the agent's own record never holds a kit's items.
+  #
+  # A kit only ADDS to an explicit allowlist. An agent whose allowlist is "all"
+  # (nil) already sees everything, so the kit leaves it alone.
+  module Kits
+    NAME = /\A[a-z0-9][a-z0-9_-]{0,63}\z/
+    LISTS = %w[skills tools tool_groups].freeze
+
+    module_function
+
+    # -> the normalized kit { "description", "skills", "tools", "tool_groups" }.
+    def validate!(name, kit)
+      raise Insika::ValidationError, "kit name must be a lowercase slug" unless name.to_s.match?(NAME)
+      raise Insika::ValidationError, "kit must be an object" unless kit.is_a?(Hash)
+
+      kit = kit.transform_keys(&:to_s)
+      out = { "description" => kit["description"].to_s.strip }
+      LISTS.each do |key|
+        value = kit[key]
+        raise Insika::ValidationError, "kit #{key} must be a list" unless value.nil? || value.is_a?(Array)
+
+        out[key] = Array(value).map { |v| v.to_s.strip }.reject(&:empty?).uniq
+      end
+      out
+    end
+
+    # -> the profile with its kits' items unioned in (the same object when there is
+    # nothing to add). Unknown kit names are ignored: a deleted kit just stops applying.
+    def apply(profile, kits_by_name)
+      # A malformed stored kit is skipped, never raised: one bad record must not stop
+      # every turn of every subscriber (the same rule as AgentDefaults).
+      chosen = Array(profile.kits).filter_map { |name| kits_by_name[name] }.grep(Hash)
+      return profile if chosen.empty?
+
+      pick = ->(key) { chosen.flat_map { |k| Array(k[key]) }.uniq }
+      changes = {}
+      unless profile.tools_allow.nil? && profile.tools_allow_groups.nil?
+        tools = pick.("tools")
+        groups = pick.("tool_groups")
+        changes[:tools_allow] = Array(profile.tools_allow) | tools if tools.any?
+        changes[:tools_allow_groups] = Array(profile.tools_allow_groups) | groups if groups.any?
+      end
+      skills = pick.("skills")
+      changes[:skills] = Array(profile.skills) | skills if !profile.skills.nil? && skills.any?
+      changes.empty? ? profile : profile.with(**changes)
+    end
+  end
+end

data/lib/insika/knowledge.rb

@@ -244,7 +244,7 @@ module Insika
       attr_reader :model, :types
 
       # ask:   ->(prompt) { "<raw model text>" } | something answering #content
-      #        (+ #input_tokens/#output_tokens/#cached_tokens for cost).
+      #        (+ RubyLLM-compatible #tokens for cost).
       # model: the ref recorded on the concept's provenance ("utility_model"
       #        default).
       # types: the allowed concept types (the profile's `knowledge.types`,
@@ -262,7 +262,7 @@ module Insika
       #      cost: { "spent" => N, "cached" => N } | nil }
       def extract(prompt:, max_concepts: 10)
         answer = @ask.call(prompt)
-        raw = parse(text_of(answer))
+        raw = parse(answer)
         concepts = []
         dropped = DROP_KEYS.to_h { |k| [k, 0] }
         seen = {}
@@ -291,21 +291,25 @@ module Insika
       def text_of(answer) = (answer.respond_to?(:content) ? answer.content : answer).to_s
 
       def cost_of(answer)
-        return nil unless answer.respond_to?(:input_tokens) && answer.respond_to?(:output_tokens)
+        return nil unless answer.respond_to?(:tokens)
 
-        input = answer.input_tokens.to_i
-        output = answer.output_tokens.to_i
-        cached = answer.respond_to?(:cached_tokens) ? answer.cached_tokens.to_i : 0
-        spent = input + output + cached
+        usage = answer.tokens
+        input = usage.input.to_i
+        output = usage.output.to_i
+        cached = usage.cache_read.to_i
+        spent = input + output + cached + usage.cache_write.to_i
         spent.positive? ? { "spent" => spent, "cached" => cached } : nil
       end
 
-      # Fences stripped, parsed STRICTLY: a model that improvises a schema
-      # fails here instead of producing half a concept, never persisted
-      # half-parsed.
-      def parse(raw)
-        body = raw.strip.gsub(/\A```(?:json)?\s*|\s*```\z/, "")
-        parsed = JSON.parse(body)
+      # Native responses own JSON parsing; injected text asks retain their
+      # existing fenced-JSON contract. Domain validation still runs below.
+      def parse(answer)
+        parsed = if answer.respond_to?(:parsed)
+                   answer.parsed
+                 else
+                   JSON.parse(text_of(answer).strip.gsub(/\A```(?:json)?\s*|\s*```\z/, ""))
+                 end
+        parsed = parsed["items"] if parsed.is_a?(Hash)
         raise Unusable, "the extractor's answer is not an array" unless parsed.is_a?(Array)
 
         parsed
@@ -368,12 +372,17 @@ module Insika
       # Temperature 0: extraction should be deterministic for the same turn.
       # `ruby_llm` required lazily so nothing loads a provider gem until an
       # extractor is actually configured (load_guard stays green).
-      def ruby_llm_ask(model, provider, llm: nil)
+      def ruby_llm_ask(model, provider, llm: nil, schema: nil)
         require "ruby_llm"
         llm ||= RubyLLM
         lambda do |prompt|
+          unless schema
+            prompt = "#{prompt}\n\nReturn the requested array inside a JSON object with the key \"items\"."
+          end
+          response_schema = schema || { "type" => "object", "properties" => { "items" => CONCEPT_SCHEMA.json_schema },
+                                        "required" => ["items"], "additionalProperties" => false }
           llm.chat(model: model, provider: provider, assume_model_exists: true)
-             .with_temperature(0).ask(prompt)
+             .with_temperature(0).with_schema("schema" => response_schema, "strict" => false).ask(prompt)
         end
       end
     end
@@ -434,7 +443,7 @@ module Insika
       # -> { verdict: :related, merged_body: String } | { verdict: :contradicting }
       def resolve(existing_body:, new_body:)
         answer = @ask.call(prompt_for(existing_body, new_body))
-        parsed = parse(text_of(answer))
+        parsed = parse(answer)
         raise Unusable, "not a JSON object" unless VERDICT_SCHEMA.call(parsed).success?
 
         classify(parsed)
@@ -455,9 +464,12 @@ module Insika
 
       def text_of(answer) = (answer.respond_to?(:content) ? answer.content : answer).to_s
 
-      def parse(raw)
-        body = raw.strip.gsub(/\A```(?:json)?\s*|\s*```\z/, "")
-        parsed = JSON.parse(body)
+      def parse(answer)
+        parsed = if answer.respond_to?(:parsed)
+                   answer.parsed
+                 else
+                   JSON.parse(text_of(answer).strip.gsub(/\A```(?:json)?\s*|\s*```\z/, ""))
+                 end
         raise Unusable, "the consolidator's answer is not an object" unless parsed.is_a?(Hash)
 
         parsed
@@ -492,7 +504,9 @@ module Insika
         return nil if ref.nil?
 
         provider, model = ExtractorFactory.split_ref(ref)
-        factory = ask_factory || ->(m, p) { ExtractorFactory.ruby_llm_ask(m, p, llm: llm) }
+        factory = ask_factory || ->(m, p) {
+          ExtractorFactory.ruby_llm_ask(m, p, llm: llm, schema: Consolidator::VERDICT_SCHEMA.json_schema)
+        }
         Consolidator.new(ask: factory.call(model, provider), model: ref)
       end
     end

data/lib/insika/llm_configurator.rb

@@ -48,6 +48,7 @@ module Insika
             next
           end
 
+          remember_boot(config, api)
           if set_accessor(config, "#{api}_api_key", key)
             base = fetch(rec, :base_url)
             set_accessor(config, "#{api}_api_base", base) if base && !base.to_s.empty?
@@ -61,16 +62,43 @@ module Insika
       { applied: applied, skipped: skipped }
     end
 
-    # UNDOES a provider's config in RubyLLM at runtime (delete without a restart,
-    # clears `<api>_api_key`/`<api>_api_base`. A provider that RubyLLM
+    # Re-applies the stored providers when they changed since THIS process last
+    # applied them. The Studio's `apply` reaches only the worker that served the
+    # edit; every other worker (and every worker after a restart) gets the
+    # provider here. Called per chat build, so it mutates the config only when the
+    # store changed, which is rare. Inside a turn the read comes from the
+    # ConfigStore cache. A provider this method applied and that no longer has a
+    # key (deleted, or its key cleared) is unapplied; a key it never applied, such
+    # as one from the environment, is never cleared. A store error keeps the
+    # current config: the next call retries, and the chat is not failed for it.
+    def refresh
+      records = @provider_store.all_raw
+      digest = records.hash
+      return if digest == @refreshed_digest
+
+      applied = apply(records)[:applied].map(&:to_s)
+      (Array(@refreshed_apis) - applied).each { |api| unapply(api) }
+      @refreshed_apis = applied
+      @refreshed_digest = digest
+    rescue StandardError => e
+      warn "[insika] LLM providers not refreshed, keeping the current ones: #{e.class}: #{e.message}"
+    end
+
+    # UNDOES a provider's config in RubyLLM at runtime (delete without a restart):
+    # `<api>_api_key`/`<api>_api_base` go back to what they were before this
+    # configurator first set them (nil when nothing was). A provider that RubyLLM
     # doesn't recognize (no accessor) -> unapplied: false (nothing applied, nothing to undo).
     # -> { unapplied: bool }.
     def unapply(api)
       api = api.to_s
+      # never applied here (another worker served the edit): nothing of ours to undo
+      return { unapplied: false } unless boot_values.key?(api)
+
       unapplied = false
       with_config do |config|
-        unapplied = set_accessor(config, "#{api}_api_key", nil)
-        set_accessor(config, "#{api}_api_base", nil)
+        key, base = boot_values[api]
+        unapplied = set_accessor(config, "#{api}_api_key", key)
+        set_accessor(config, "#{api}_api_base", base)
       end
       { unapplied: unapplied }
     end
@@ -86,6 +114,20 @@ module Insika
       end
     end
 
+    # The key/base a provider had before this configurator first overrode it (the
+    # environment's, or nil), so removing the authored one puts those back instead
+    # of leaving the provider without credentials.
+    def remember_boot(config, api)
+      return if boot_values.key?(api.to_s)
+
+      boot_values[api.to_s] = %w[key base].map do |part|
+        getter = "#{api}_api_#{part}"
+        config.public_send(getter) if config.respond_to?(getter)
+      end
+    end
+
+    def boot_values = (@boot_values ||= {})
+
     def set_accessor(config, name, value)
       setter = "#{name}="
       return false unless config.respond_to?(setter)

data/lib/insika/llm_provider_store.rb

@@ -38,9 +38,7 @@ module Insika
     end
 
     # -> [Hash] all with REAL api_key (for the configurator). Never goes to the screen.
-    def all_raw
-      apis.filter_map { |a| raw(a) }
-    end
+    def all_raw = @cs.all(SCOPE).compact
 
     # Upsert with secret reconciliation. `attrs` (string|symbol keys):
     #   api (required), base_url, auth_header, api_key (sentinel-aware), models[]

data/lib/insika/llm_trace_store.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Insika
+  # Content-free native diagnostics. Task deletion owns their lifetime.
+  class LLMTraceStore
+    SCOPE = "llm_traces"
+    MAX_PER_TASK = 200
+    TEXT_FIELDS = %w[type operation agent provider model status request_id exception_class at].freeze
+    NUMBER_FIELDS = %w[turn duration_ms input_tokens output_tokens cache_read_tokens cache_write_tokens thinking_tokens cost].freeze
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # Also used at the telemetry boundary; never stringify arbitrary objects.
+    def self.sanitize(entry)
+      entry.each_with_object({}) do |(key, value), result|
+        if TEXT_FIELDS.include?(key) && value.is_a?(String)
+          result[key] = Coercion.utf8(value)[0, 256].gsub(/[[:cntrl:]]/, "")
+        elsif NUMBER_FIELDS.include?(key) && (value.nil? ||
+              ((value.is_a?(Integer) || value.is_a?(Float)) && value.finite? && value >= 0 && value <= 1e18))
+          result[key] = value
+        end
+      end
+    end
+
+    def record(task_id:, entry:) = record_many(task_id: task_id, entries: [entry])
+
+    # A turn's events in one transaction: one read and one write of the trace,
+    # instead of one transaction per event.
+    def record_many(task_id:, entries:)
+      return if entries.empty?
+
+      @store.transaction do
+        next unless @store.get(TaskStore::SCOPE, "#{TaskStore::KEY_PREFIX}#{task_id}")
+
+        trace = for_task(task_id)
+        all = trace["entries"] + entries.map { |entry| self.class.sanitize(entry) }
+        @store.set(SCOPE, task_id.to_s, {
+          "entries" => all.last(MAX_PER_TASK),
+          "truncated" => trace["truncated"] || all.size > MAX_PER_TASK
+        })
+      end
+    rescue StandardError
+      nil
+    end
+
+    def for_task(task_id)
+      @store.get(SCOPE, task_id.to_s) || { "entries" => [], "truncated" => false }
+    end
+
+    def clear(task_id) = @store.delete(SCOPE, task_id.to_s)
+  end
+end