insika 0.1.0 → 0.3.0

data/lib/insika/retention.rb

@@ -0,0 +1,281 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  # WS8 (phase 2): retention — the engine forgets what is old enough to forget.
+  # Age-based purge of the CONVERSATION footprint: sessions (and their
+  # per-session tool/context traces), terminal tasks (and their checkpoints),
+  # customer/tenant memory cells and outcome records. The knob is data, not
+  # code: `settings.retention_days` (Integer; nil/0 = OFF — parity, nothing is
+  # ever swept by default).
+  #
+  # Cadence: the Tick calls `run` every pass; an internal DAILY claim (the
+  # tick's window idiom — one key, a timestamp, 24 h) makes the actual sweep
+  # once a day, so the O(n) scans never ride the 60 s loop. A sweep that
+  # crashes propagates to the tick, which logs and keeps ticking.
+  class Retention
+    SCOPE = "retention"
+    KEY = "claim"
+    BUDGET_KEY = "budget_claim"
+    # the MEMORY TTL's own daily claim. Deliberately NOT the age-based
+    # KEY — memory TTLs sweep on their own knob (`memory_ttl_days`), gated by
+    # neither retention_days nor the age-based claim (D5).
+    MEMORY_TTL_KEY = "memory_ttl_claim"
+    WINDOW = 86_400 # one sweep per day, at most
+
+    TERMINAL = %w[completed failed cancelled].freeze
+
+    def initialize(session_store:, task_store:, checkpoint_store:,
+                   memory_store:, outcome_store:, tool_trace_store: nil,
+                   context_trace_store: nil, outbox_store: nil, shadow_pair_store: nil,
+                    settings_store: nil, budget_ledger: nil, funnel_store: nil,
+                    followup_store: nil, contact_store: nil, proposal_store: nil,
+                    model_visible_trace_store: nil,
+                     store:, window: WINDOW, now: nil, harvest_store: nil)
+      @session_store = session_store
+      @task_store = task_store
+      @checkpoint_store = checkpoint_store
+      @memory_store = memory_store
+      @outcome_store = outcome_store
+      @tool_trace_store = tool_trace_store
+      @context_trace_store = context_trace_store
+      @model_visible_trace_store = model_visible_trace_store #  ; nil = parity
+      @outbox_store = outbox_store
+      @shadow_pair_store = shadow_pair_store
+      @settings_store = settings_store
+      @budget_ledger = budget_ledger # WS2 counter GC; nil = nothing to sweep
+      @funnel_store = funnel_store   #  ; nil = nothing to sweep
+      @followup_store = followup_store #  ; nil = nothing to sweep
+      @contact_store = contact_store   #  ; nil = nothing to sweep
+      @proposal_store = proposal_store #  ; nil = nothing to sweep
+      @store = store
+      @window = window
+      @now = now # injectable for specs (a deterministic "today")
+      @harvest_store = harvest_store #  ; nil = nothing to sweep
+    end
+
+    # the sweep reads the memory store's cells/records (specs seed
+    # facts through it).
+    attr_reader :memory_store
+
+    # -> { claimed: false } |
+    #    { claimed: true, sessions:, tasks:, outcomes:, memory:, deliveries: }.
+    # Either shape may carry `budget_cells:` — the budget counter GC is NOT
+    # gated by retention_days (see #sweep_budget_cells). Either shape may carry
+    # `memory_ttl:` — the   memory TTL sweep, gated by ITS OWN daily
+    # claim and knob, never by retention_days (see #sweep_memory_ttl).
+    def run
+      budget_cells = sweep_budget_cells
+      memory_ttl = sweep_memory_ttl
+      days = retention_days
+      unless days.to_i.positive? && claim_window
+        summary = { claimed: false }
+        summary[:budget_cells] = budget_cells if budget_cells
+        summary[:memory_ttl] = memory_ttl if memory_ttl
+        return summary
+      end
+
+      cutoff = now - (days.to_i * 86_400)
+      summary = { claimed: true, sessions: sweep_sessions(cutoff),
+                  tasks: sweep_tasks(cutoff), outcomes: sweep_outcomes(cutoff),
+                  memory: @memory_store.prune_older_than(cutoff),
+                  deliveries: sweep_outbox(cutoff),
+                  pairs: sweep_shadow_pairs(cutoff) }
+      summary[:funnel] = sweep_funnel(cutoff) if @funnel_store
+      # the follow-up footprint ages out with the rest — records
+      # and contact cells under the SAME retention_days gate (nil collaborator
+      # = nothing to sweep, base graph parity).
+      summary[:followups] = @followup_store.delete_older_than(cutoff) if @followup_store
+      summary[:contacts] = @contact_store.delete_older_than(cutoff) if @contact_store
+      # proposals are evidence OF a transcript — when the
+      # transcript dies, the proposal's excerpt is gone and the pending fact is
+      # stale. Pending AND terminal rows age out together; a proposal is
+      # re-derivable (D2), so pruning is never data loss. The session MARKERS
+      # are never pruned (the store's rule — the marker is the claim).
+      summary[:proposals] = @proposal_store.delete_older_than(cutoff) if @proposal_store
+      # candidates (pending AND terminal), log rows and
+      # snapshots are DERIVED data of transcripts (D11) — when the transcripts
+      # die, the candidates' excerpts are gone; they are re-derivable (D2), so
+      # pruning is never data loss. The session markers are never pruned.
+      summary[:harvest] = @harvest_store.delete_older_than(cutoff) if @harvest_store
+      summary[:budget_cells] = budget_cells if budget_cells
+      summary[:memory_ttl] = memory_ttl if memory_ttl
+      summary
+    end
+
+    private
+
+    # The BudgetLedger's expired cells (WS2). Deliberately OUTSIDE the
+    # retention_days gate, on its OWN daily claim: those rows are engine
+    # bookkeeping whose window already rolled over, not customer content, so a
+    # deployment that keeps its conversations forever (retention OFF — the
+    # default) must still not grow budget rows forever. -> count | nil (no
+    # ledger, or another worker holds today's claim).
+    def sweep_budget_cells
+      return nil unless @budget_ledger && claim(BUDGET_KEY)
+
+      @budget_ledger.prune(now: now)
+    end
+
+    def retention_days
+      return nil unless @settings_store
+
+      value = @settings_store.get["retention_days"]
+      value.to_s.empty? ? nil : Integer(value)
+    rescue ArgumentError, TypeError
+      nil # a non-numeric value reads as OFF — never a crash at sweep time
+    end
+
+    # the memory TTL sweep — TWO independent expiry clocks under
+    # ONE daily claim:
+    #   1. per-fact expires_at (prune_expired — past dates physically removed);
+    #   2. per-cell TTL by `memory_ttl_days` (age by the cell's updated_at).
+    # A fact with an explicit expires_at is EXCLUDED from the age-based pass
+    # (the explicit override owns that fact's life). Runs on its OWN claim and
+    # knob — a deployment with retention_days off still honors memory TTLs.
+    # -> Integer (removed) | nil (no knob, or another worker holds the claim).
+    def sweep_memory_ttl
+      ttl = memory_ttl_setting
+      return nil if ttl.nil?
+      return nil unless claim(MEMORY_TTL_KEY)
+
+      removed = @memory_store.prune_expired(now)
+      ttl_cutoffs(ttl).each do |scope, cutoff|
+        removed += @memory_store.prune_older_than(cutoff, scope: scope)
+      end
+      removed
+    end
+
+    # settings["memory_ttl_days"]: Integer | Hash{ "<tenant>" => days, "*" => days }.
+    # nil/empty/blank -> nil (OFF — parity). A non-numeric value -> nil (never
+    # a crash at sweep time, the retention_days rescue pattern).
+    def memory_ttl_setting
+      return nil unless @settings_store
+
+      raw = @settings_store.get["memory_ttl_days"]
+      case raw
+      when Integer then raw
+      when Hash
+        map = raw.each_with_object({}) do |(k, v), acc|
+          acc[k.to_s] = Integer(v.to_s)
+        rescue ArgumentError, TypeError
+          next
+        end
+        map.empty? ? nil : map
+      end
+    end
+
+    # -> [[scope, Time]] — one per existing cell with a resolved TTL. The
+    # setting is passed in (one settings-store read per sweep — the caller
+    # already resolved it).
+    def ttl_cutoffs(setting = memory_ttl_setting)
+      return [] unless setting
+
+      @memory_store.cells.filter_map do |cell|
+        days = cell_ttl(cell, setting)
+        next if days.nil? || days <= 0
+
+        [cell[:scope], now - days * 86_400]
+      end
+    end
+
+    # Customer cell "memory:acme:c-1" -> map["acme"]; tenant/bare cell
+    # "memory:acme" or "memory:c-123" -> map["acme"] / map["c-123"]; fallback
+    # map["*"]; an Integer setting -> every cell gets it.
+    def cell_ttl(cell, setting)
+      return setting if setting.is_a?(Integer)
+
+      key = cell[:tenant] || cell[:customer]
+      v = key && setting[key]
+      v ||= setting["*"]
+      v
+    end
+
+    # Sessions untouched past the cutoff, and their per-session traces.
+    def sweep_sessions(cutoff)
+      removed = 0
+      @session_store.each_id.each do |id|
+        session = @session_store.find(id)
+        next unless session && session.updated_at && session.updated_at < cutoff.iso8601
+
+        @tool_trace_store&.clear(id)
+        @context_trace_store&.clear(id)
+        @session_store.delete(id)
+        removed += 1
+      end
+      removed
+    end
+
+    # TERMINAL tasks untouched past the cutoff, and their checkpoints. A
+    # non-terminal task (queued/running) is never touched here — the Recovery
+    # sweep owns those lives. the model-visible traces are
+    # transcripts — they die next to their checkpoints.
+    def sweep_tasks(cutoff)
+      removed = 0
+      @task_store.each_id.each do |id|
+        task = @task_store.find(id)
+        next unless task && TERMINAL.include?(task.status.to_s)
+        next unless task.updated_at && task.updated_at < cutoff.iso8601
+
+        @checkpoint_store.purge(id)
+        @model_visible_trace_store&.purge(id)
+        @task_store.delete(id)
+        removed += 1
+      end
+      removed
+    end
+
+    def sweep_outcomes(cutoff)
+      @outcome_store ? @outcome_store.delete_older_than(cutoff) : 0
+    end
+
+    # The delivered/failed outbox records past the cutoff. Their `payload` is
+    # the answer the customer received — conversation content, so it ages out
+    # with the rest of the footprint instead of living in the store forever.
+    def sweep_outbox(cutoff)
+      @outbox_store ? @outbox_store.delete_older_than(cutoff) : 0
+    end
+
+    # Shadow pairs  past the cutoff, TERMINAL statuses only
+    # (judged/incomplete) — an open/complete pair older than the window is
+    # still someone's unjudged evidence, exactly like the outbox's rule.
+    def sweep_shadow_pairs(cutoff)
+      @shadow_pair_store ? @shadow_pair_store.delete_older_than(cutoff) : 0
+    end
+
+    # the funnel DAY CELLS die with the outcomes they fold — same
+    # retention_days gate, same daily claim. Cursors/baselines live while their
+    # agent does. nil collaborator = nothing to sweep (base graph, parity).
+    def sweep_funnel(cutoff)
+      @funnel_store.delete_older_than(cutoff)
+    end
+
+    def claim_window = claim(KEY)
+
+    # The daily claim: one key, a timestamp, a 24 h window — the tick's
+    # claim_window idiom (a key per day would be a slow leak; the window is
+    # long enough that the store never grows). One key per SWEEP (the age-based
+    # one, the budget GC): they gate on different knobs, so a single shared key
+    # would let whichever ran first starve the other for a day.
+    def claim(key)
+      now_time = now
+      @store.transaction do
+        current = @store.get(SCOPE, key)
+        last = current && begin
+          Time.iso8601(current["claimed_at"].to_s)
+        rescue ArgumentError
+          nil
+        end
+        if last.nil? || (now_time - last) >= @window
+          @store.set(SCOPE, key, { "claimed_at" => now_time.iso8601 })
+          true
+        else
+          false
+        end
+      end
+    end
+
+    def now = @now || Time.now.utc
+  end
+end

data/lib/insika/routing.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Insika
+  # WS4: intent routing as DATA. When `AgentProfile#routes` is present, the
+  # turn's message is classified into one of the configured routes with a cheap
+  # model BEFORE the agent chat is assembled. This class owns the PURE parts —
+  # normalizing the route config, generating the classifier prompt from the
+  # route descriptions, and parsing the model's answer back into a route — so
+  # they are testable without a provider. The ask itself and its usage are the
+  # Executor's (a pre-stage call counted in the turn's usage and trace).
+  #
+  # Config shape (string keys at the persistence boundary — the pack and the
+  # Studio store it like any other free-form hash):
+  #
+  #   { "shopping" => "the customer wants to browse products",
+  #     "order"    => { "description" => "asks about an existing order",
+  #                     "delegate" => "order-agent" },     # hand the turn to that agent
+  #     "human"    => { "description" => "the customer is frustrated or asks for a person",
+  #                     "stuck" => true, "message" => "..." }, # end the turn :stuck (WS5)
+  #     "default"  => "shopping",                            # deterministic fallback
+  #     "model"    => "deepseek-v4-flash" }                  # the CHEAP classifier
+  #
+  # `default`/`model`/`provider` are reserved top-level keys; everything else is
+  # a route name (single token). A route value is a description String or a Hash
+  # with `description` + optional `delegate` (an existing agent the turn is
+  # handed to) / `stuck` (the turn ends with the WS5 stuck outcome) / `message`
+  # (the consumer-facing lead-in for a stuck route).
+  class Routing
+    RESERVED = %w[model provider default].freeze
+    # A name the classifier can actually answer with: one lowercase token.
+    NAME = /\A[a-z0-9][a-z0-9_-]*\z/
+    # The turn's usage fields the classifier call can contribute.
+    TOKEN_FIELDS = %i[input_tokens output_tokens cached_tokens cache_creation_tokens total_tokens].freeze
+
+    Entry = Data.define(:name, :description, :delegate, :stuck, :message)
+
+    class << self
+      # -> { entries: [Entry], default: String, model: String, provider: String }
+      # | nil (routes absent/empty = routing off). Raises ValidationError on a
+      # route name the classifier could never answer with (spaces/symbols) or on
+      # a config with zero routes — a routing config that silently routes nothing
+      # is a config error, not a quiet off.
+      def normalize(routes)
+        return nil unless routes.is_a?(Hash) && !routes.empty?
+
+        bad = routes.keys.map(&:to_s).reject { |k| RESERVED.include?(k) || k.match?(NAME) }
+        unless bad.empty?
+          raise Insika::ValidationError,
+                "route names may not contain spaces or symbols: #{bad.join(', ')}"
+        end
+
+        entries = routes.each_with_object([]) do |(name, value), acc|
+          next if RESERVED.include?(name.to_s)
+
+          cfg = value.is_a?(Hash) ? stringify(value) : { "description" => value.to_s }
+          acc << Entry.new(name: name.to_s, description: cfg["description"].to_s,
+                           delegate: cfg["delegate"].to_s, stuck: cfg["stuck"] == true,
+                           message: cfg["message"].to_s)
+        end
+        raise Insika::ValidationError, "routes must define at least one route" if entries.empty?
+
+        default = routes["default"].to_s
+        default = entries.first.name if default.empty?
+        unless entries.any? { |e| e.name == default }
+          raise Insika::ValidationError,
+                "default route '#{default}' is not a configured route"
+        end
+        { entries: entries, default: default,
+          model: routes["model"].to_s, provider: routes["provider"].to_s }
+      end
+
+      # The classifier's instructions, auto-generated from the route descriptions
+      # (config-over-code: no per-route prompt file).
+      def classifier_prompt(meta)
+        lines = meta[:entries].map do |e|
+          "-#{e.name}: #{e.description.empty? ? e.name : e.description}"
+        end
+        <<~PROMPT
+          Classify the customer's message into exactly one of these intents.
+          Reply with ONLY the intent name — nothing else.
+
+          #{lines.join("\n")}
+        PROMPT
+      end
+
+      # -> Symbol: the route the model named; the DEFAULT when it named nothing
+      # usable (prose, punctuation, an unknown name, empty). Deterministic by
+      # construction — a chatty or confused classifier can never invent a route.
+      def parse(text, meta)
+        token = text.to_s.strip.split(/\s+/).first.to_s.downcase
+                    .delete_suffix(".").delete_suffix(",")
+        names = meta[:entries].map(&:name)
+        (names.include?(token) ? token : meta[:default]).to_sym
+      end
+    end
+
+    def self.stringify(hash)
+      hash.each_with_object({}) { |(k, v), acc| acc[k.to_s] = v }
+    end
+  end
+end

data/lib/insika/safety/config.rb

@@ -2,7 +2,7 @@
 
 module Insika
   module Safety
-    # Per-agent guardrail configuration (RFC-0009 §3.3 / D6), read from
+    # Per-agent guardrail configuration, read from
     # `profile.guardrails`. OPT-IN like `capabilities`: an agent that says nothing
     # gets the CONSERVATIVE default — deterministic detectors ON, LLM moderator OFF.
     #
@@ -16,9 +16,20 @@ module Insika
     #     moderator:  "provider/model"|nil,  # LLM moderator model; nil = deterministic only
     #     strictness: "low"|"medium"|"high", # which input categories fire (default medium)
     #     responses:  { <category> => "<safe reply>", ... }  # per-agent override, see below
+    #     corpora:    { "languages" => ["en"], "extra" => { "abuse" => ["/\\bdupa\\b/i"] } }
     #   }
     #
-    # `responses` is the CONFIGURATION-OVER-CONVENTION knob (RFC-0009 §7). The engine
+    # `corpora`  is the removability knob for the shipped
+    # pt-BR corpus: `languages` filters the shipped families (nil = all,
+    # [] = none, ["en"] = the EN-only corpus — dropping the pt-BR input
+    # heuristics AND the CPF/CNPJ output redaction, a documented consequence),
+    # `extra` adds source-string patterns per family. Absent = the full
+    # shipped default, byte-for-byte today's behavior (parity). Validation
+    # fails loud (Insika::ValidationError naming the value): the doctor's
+    # guardrail-corpora check compiles every declaration, so a typo'd language
+    # surfaces at boot — `insika doctor` exits non-zero — never mid-turn.
+    #
+    # `responses` is the CONFIGURATION-OVER-CONVENTION knob. The engine
     # ships neutral built-in refusals (Safety::SafeResponses::DEFAULTS), but this is
     # OSS across arbitrary businesses/languages, so we never hard-bake tone: an agent
     # overrides the safe reply per category (`injection`/`sexual`/`abuse`/`escalate`/
@@ -37,14 +48,18 @@ module Insika
 
       DEFAULT_STRICTNESS = :medium
 
-      attr_reader :input, :output, :moderator, :strictness, :responses
+      attr_reader :input, :output, :moderator, :strictness, :responses, :corpora
 
-      def initialize(input:, output:, moderator:, strictness:, responses: {})
+      def initialize(input:, output:, moderator:, strictness:, responses: {}, corpora: nil)
         @input = input
         @output = output
         @moderator = moderator
         @strictness = strictness
         @responses = responses # { "category" => "safe reply" }, agent override map
+        @corpora = corpora     # { "languages" => [...]?, "extra" => {...}? } | nil (nil = the shipped default)
+        # Built ONCE at construction: the compiled corpus the whole turn reads.
+        @corpus = Corpus.compile(languages: corpora && corpora["languages"],
+                                 extra: (corpora && corpora["extra"]) || {})
       end
 
       # Builds a Config from a profile. A nil/empty `guardrails` -> the
@@ -58,10 +73,22 @@ module Insika
           output: bool(h.fetch(:output, true)),
           moderator: presence(h[:moderator]),
           strictness: normalize_strictness(h[:strictness]),
-          responses: normalize_responses(h[:responses])
+          responses: normalize_responses(h[:responses]),
+          corpora: normalize_corpora(h[:corpora])
         )
       end
 
+      # The compiled corpus for this agent  — never nil: absent
+      # corpora -> the full shipped default.
+      def corpus = @corpus
+
+      # The resolved corpus languages (for the doctor's enumeration, C3):
+      # nil corpora (or nil languages) = ALL shipped languages.
+      def corpus_languages
+        langs = @corpora && @corpora["languages"]
+        langs.nil? ? Corpus::DEFAULTS.keys : langs
+      end
+
       # Input categories the deterministic scan should run, per strictness.
       def input_categories = STRICTNESS_CATEGORIES.fetch(@strictness, STRICTNESS_CATEGORIES[DEFAULT_STRICTNESS])
 
@@ -103,7 +130,20 @@ module Insika
         end
       end
 
-      private_class_method :symbolize, :bool, :presence, :normalize_strictness, :normalize_responses
+      # `corpora` -> { "languages" => ...?, "extra" => ...? } with STRING keys |
+      # nil when absent/empty. Validation of the values happens in
+      # Corpus.compile (from_hash fails loud with the value named).
+      def self.normalize_corpora(v)
+        return nil unless v.is_a?(Hash)
+
+        h = v.each_with_object({}) { |(k, val), acc| acc[k.to_s] = val }
+        return nil if h["languages"].nil? && h["extra"].nil?
+
+        { "languages" => h["languages"], "extra" => h["extra"] }
+      end
+
+      private_class_method :symbolize, :bool, :presence, :normalize_strictness, :normalize_responses,
+                           :normalize_corpora
     end
   end
 end