insika 0.2.0 → 0.3.0

data/lib/insika/session_store.rb

@@ -23,7 +23,14 @@ module Insika
     KEY_PREFIX = "session:"
 
     Session = Data.define(:id, :messages, :vars, :memory_refs,
-                          :created_at, :updated_at)
+                          :created_at, :updated_at, :briefing, :evidence) do
+      # Trailing members with defaults: an old record without the "briefing" /
+      # "evidence" keys reads as empty/nil without a migration.
+      def initialize(id:, messages:, vars:, memory_refs:, created_at:, updated_at:,
+                     briefing: nil, evidence: nil)
+        super
+      end
+    end
 
     # store: any Insika::Store (Memory, SQLite, ...) — injected by the
     # composition root (config/wiring.rb). SessionStore does not know the
@@ -44,6 +51,8 @@ module Insika
         "messages" => [],
         "vars" => deep_stringify(vars),
         "memory_refs" => [],
+        "briefing" => { "fields" => {}, "next_step" => nil },
+        "evidence" => { "ids" => [], "ungrounded" => 0 },
         "created_at" => now,
         "updated_at" => now
       }
@@ -89,6 +98,58 @@ module Insika
       to_session(record)
     end
 
+    # appends this turn's evidence (ids + ungrounded delta) to the
+    # session record. RMW like append_messages — the SessionActor serializes
+    # same-session turns; the copy is in the method comment.
+    def append_evidence(id, ids:, ungrounded:)
+      record = fetch!(id)
+      ev = record["evidence"] ||= { "ids" => [], "ungrounded" => 0 }
+      fresh = (ev["ids"] + Array(ids).map(&:to_s).reject(&:empty?)).uniq.last(EvidenceLedger::MAX_IDS)
+      ev["ids"] = fresh
+      ev["ungrounded"] = ev["ungrounded"].to_i + ungrounded.to_i
+      record["updated_at"] = timestamp
+      @store.set(SCOPE, key_for(id), record)
+      to_session(record)
+    end
+
+    # -> Session. Upsert ONE briefing field. The pack owns the schema; the
+    # engine validates nothing about field NAMES here (the tools do, at the
+    # write edge). value is a String (anything else -> to_s); a BLANK value
+    # (after strip) REMOVES the key — absence means "not yet asked" (
+    # D4). NotFoundError if the session does not exist.
+    #
+    # CONCURRENCY NOTE: an unlocked RMW (read -> mutate -> set), like
+    # append_messages — but the SessionActor argument does NOT apply here. A
+    # briefing writer is a system tool, never enveloped, and with
+    # tool_concurrency > 1 the gem runs each call in its OWN fiber — outside the
+    # actor's per-session turn serialization. The RMW is still safe, for a
+    # different reason: nothing in the read/mutate/set path suspends. Store
+    # get/set are synchronous (the SQLite write semaphore is a non-yielding fast
+    # path when free), and a fiber only switches at a scheduler suspension point
+    # — so no other writer can interleave mid-RMW (measured: N concurrent
+    # writers lose nothing). It holds ONLY while that path never suspends; an
+    # async store (a real yield in get/set) would need a lock or CAS.
+    def update_briefing(id, field:, value:)
+      record = fetch!(id)
+      briefing = record["briefing"] ||= { "fields" => {}, "next_step" => nil }
+      value = Coercion.presence(Coercion.utf8(value.to_s))
+      value ? briefing["fields"][field.to_s] = value : briefing["fields"].delete(field.to_s)
+      record["updated_at"] = timestamp
+      @store.set(SCOPE, key_for(id), record)
+      to_session(record)
+    end
+
+    # -> Session. Upsert the agreed next step; a blank text clears to nil
+    # NotFoundError if absent.
+    def set_next_step(id, text:)
+      record = fetch!(id)
+      briefing = record["briefing"] ||= { "fields" => {}, "next_step" => nil }
+      briefing["next_step"] = Coercion.presence(Coercion.utf8(text.to_s))
+      record["updated_at"] = timestamp
+      @store.set(SCOPE, key_for(id), record)
+      to_session(record)
+    end
+
     # -> bool (delegates to the backend: false for a nonexistent id)
     def delete(id)
       @store.delete(SCOPE, key_for(id))
@@ -126,7 +187,9 @@ module Insika
         vars: record["vars"],
         memory_refs: record["memory_refs"],
         created_at: record["created_at"],
-        updated_at: record["updated_at"]
+        updated_at: record["updated_at"],
+        briefing: record["briefing"] || { "fields" => {}, "next_step" => nil },
+        evidence: record["evidence"]
       )
     end
 

data/lib/insika/settings_store.rb

@@ -31,6 +31,16 @@ module Insika
       "turn_timeout" => 120,
       "tool_timeout" => 30,
       "compaction" => { "enabled" => false, "keep_last" => 20 },
+      # Data lifecycle (WS8, phase 2): the RETENTION window in days. The
+      # tick's Retention sweep purges sessions (+traces), terminal tasks
+      # (+checkpoints), memory cells and outcomes older than this. nil/0 =
+      # OFF (parity — nothing is ever swept by default).
+      "retention_days" => nil,
+      # memory TTL. Integer = default for every cell; Hash =
+      # per-tenant { "<tenant>" => days, "*" => days } (ops-authored). nil = OFF.
+      # Swept by the Retention tick on its OWN daily claim, NOT gated by
+      # retention_days (D5). Additive key — reads overlay DEFAULTS.
+      "memory_ttl_days" => nil,
       # LLM config v2. Platform-wide model layer, resolved by the
       # ModelResolver under an agent that pins no model of its own:
       #   default_model/default_provider -> the platform default (Chat > Agent > HERE)

data/lib/insika/shadow_pair_store.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require "digest"
+require "time"
+
+module Insika
+  # C2 — one durable record per mirrored exchange (shadow mode),
+  # written by TWO INDEPENDENT HALVES: ours at the turn's terminal, the
+  # incumbent's at the mirror. Both land on the same key — a digest of
+  # (channel, external_id, event_id), deterministic and order-free — so the two
+  # writers converge without an index and without ordering assumptions.
+  #
+  # It stores and it counts; it does not judge, does not fold a verdict, does
+  # not know what the criterion says.
+  #
+  # Status transitions (never backwards):
+  #
+  #                 ┌─ record_incumbent ─┐
+  #   (nothing) ────┤                    ├─▶ open ─▶ complete ─▶ judged
+  #                 └─ record_ours ─────┘       └─▶ silent   (never judged)
+  #                                             └─▶ incomplete (expire)
+  class ShadowPairStore
+    SCOPE = "shadow_pairs"
+    KEY_PREFIX = "pair:"
+
+    STATUSES = %i[open complete silent judged incomplete].freeze
+
+    Pair = Data.define(
+      :id, :channel, :agent, :session_id, :task_id, :event_id,
+      :inbound, :incumbent_reply, :insika_reply,
+      :status, :verdict, :criterion_sha, :created_at, :updated_at
+    ) do
+      def complete?      = %i[complete silent].include?(status)
+      def judged?        = status == :judged
+      def outcome        = verdict && verdict["outcome"]
+      def human_assisted? = verdict && verdict["vs"] == "human-assisted"
+    end
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # The correlation key BOTH writers compute independently. SHA-256 hex of
+    # "<channel>\0<external_id>\0<event_id>", truncated to 32 — deterministic,
+    # order-free, and it keeps a phone number out of the store's key space.
+    def self.key_for(channel:, external_id:, event_id:)
+      Digest::SHA256.hexdigest("#{channel}\0#{external_id}\0#{event_id}")[0, 32]
+    end
+
+    # Our half. Upsert: creates the record or fills our fields on the incumbent's.
+    # `reply` may be "" — a turn that published nothing (halt_when, an out-of-band
+    # tool) is recorded as :silent rather than left invisible. -> Pair
+    def record_ours(id:, channel:, agent:, session_id:, task_id:, event_id:,
+                    inbound:, reply:, criterion_sha:)
+      upsert(id) do |record, created|
+        record["channel"] = channel.to_s
+        record["event_id"] = event_id.to_s
+        record["agent"] = agent
+        record["session_id"] = session_id&.to_s
+        record["task_id"] = task_id&.to_s
+        record["inbound"] = inbound.to_s
+        record["insika_reply"] = reply.to_s
+        record["criterion_sha"] = criterion_sha
+        created
+      end
+    end
+
+    # The incumbent's half (the mirror contract). Same upsert shape; the fields
+    # this half owns are the reply and, on first write, the timestamp the mirror
+    # reports. Never overwrites our half's fields. First-write-wins is enforced
+    # HERE, inside the transaction: the customer received ONE reply, and two
+    # concurrent mirror retries must not let the second rewrite the evidence.
+    # -> Pair
+    def record_incumbent(id:, channel:, event_id:, external_id:, reply:, at: nil)
+      upsert(id, at: at) do |record, created|
+        record["channel"] = channel.to_s
+        record["event_id"] = event_id.to_s
+        record["incumbent_reply"] = reply.to_s if record["incumbent_reply"].nil?
+        created
+      end
+    end
+
+    # -> Pair | nil
+    def find(id)
+      record = @store.get(SCOPE, key_for(id))
+      record && to_pair(record)
+    end
+
+    # Lazy scan over a SNAPSHOT of the keys (deleting under a live enumeration
+    # would skip records — the same rule OutboxStore applies).
+    def each(&block)
+      return enum_for(:each) unless block_given?
+
+      @store.list(SCOPE, KEY_PREFIX).each do |key|
+        record = @store.get(SCOPE, key)
+        yield to_pair(record) if record
+      end
+    end
+
+    # -> [Pair] created at or after `time`.
+    def since(time)
+      cutoff = time.utc.iso8601
+      each.select { |p| p.created_at.to_s >= cutoff }
+    end
+
+    # -> [Pair] status :complete, oldest first — the judging queue. `silent`
+    # pairs are NEVER here:   finding that pairwise is
+    # systematically unfair to a tool that delivers out of band is not
+    # something to average away.
+    def unjudged(limit: nil, agent: nil)
+      pairs = each.select { |p| p.status == :complete }
+                   .sort_by { |p| p.created_at.to_s }
+      pairs = pairs.select { |p| p.agent.to_s == agent.to_s } if agent
+      limit ? pairs.first(limit.to_i) : pairs
+    end
+
+    # -> { open:, complete:, silent:, judged:, incomplete: }
+    def counts(since: nil)
+      pairs = since ? self.since(since) : each.to_a
+      STATUSES.to_h { |s| [s, pairs.count { |p| p.status == s }] }
+    end
+
+    # The panel's Verdict as data. status -> :judged. -> Pair
+    def record_verdict(id, verdict:)
+      @store.transaction do
+        key = key_for(id)
+        record = @store.get(SCOPE, key)
+        raise Insika::NotFoundError, "shadow pair not found: #{id}" unless record
+
+        record["verdict"] = Coercion.deep_stringify(verdict)
+        record["status"] = "judged"
+        record["updated_at"] = timestamp
+        @store.set(SCOPE, key, record)
+        to_pair(record)
+      end
+    end
+
+    # An `open` pair older than the cutoff will never complete. -> count moved.
+    # `complete`/`silent`/`judged` are never touched. Update-style only: a pair
+    # deleted between the scan and the write (retention, LGPD purge) is left
+    # deleted — an upsert here would resurrect it as a ghost :incomplete record
+    # carrying none of its fields.
+    def expire(older_than:)
+      cutoff = older_than.utc.iso8601
+      moved = 0
+      each.select { |p| p.status == :open && p.created_at.to_s < cutoff }.each do |pair|
+        @store.transaction do
+          key = key_for(pair.id)
+          record = @store.get(SCOPE, key)
+          next unless record && record["status"] == "open"
+
+          record["status"] = "incomplete"
+          record["updated_at"] = timestamp
+          @store.set(SCOPE, key, record)
+          moved += 1
+        end
+      end
+      moved
+    end
+
+    # -> Integer. Keys only, no record materialization — a count of pairs must
+    # not pay for customer text (doctor's shadow-off check, the Studio).
+    def size = @store.list(SCOPE, KEY_PREFIX).length
+
+    # LGPD / retention (C9): drops every pair of these sessions, whatever its
+    # status — the pair holds the customer's own words. -> count removed.
+    def purge_sessions(session_ids)
+      wanted = Array(session_ids).map(&:to_s)
+      return 0 if wanted.empty?
+
+      doomed = each.select { |p| wanted.include?(p.session_id.to_s) }
+      doomed.each { |p| @store.delete(SCOPE, key_for(p.id)) }
+      doomed.size
+    end
+
+    # Retention: pairs created before the cutoff, TERMINAL statuses only
+    # (`judged`/`incomplete`) — an `open`/`complete` record older than the
+    # window is still someone's unjudged evidence. -> count removed.
+    def delete_older_than(time)
+      cutoff = time.utc.iso8601
+      doomed = each.select do |p|
+        %i[judged incomplete].include?(p.status) && p.created_at.to_s < cutoff
+      end
+      doomed.each { |p| @store.delete(SCOPE, key_for(p.id)) }
+      doomed.size
+    end
+
+    private
+
+    # The shared upsert: read -> merge only the fields THIS half owns (never
+    # overwrite the other's with nil) -> recompute status -> write. Runs inside
+    # Store#transaction so two halves landing in the same instant serialize on
+    # the backend's lock — the same claim mechanic OutboxStore#claim uses.
+    def upsert(id, at: nil, &fill)
+      @store.transaction do
+        key = key_for(id)
+        record = @store.get(SCOPE, key)
+        created = record.nil?
+        unless record
+          record = {
+            "id" => id.to_s, "channel" => nil, "agent" => nil, "session_id" => nil,
+            "task_id" => nil, "event_id" => nil, "inbound" => nil,
+            "incumbent_reply" => nil, "insika_reply" => nil, "status" => "open",
+            "verdict" => nil, "criterion_sha" => nil,
+            "created_at" => arrival_time(at), "updated_at" => timestamp
+          }
+        end
+        fill.call(record, created)
+        record["status"] = status_for(record)
+        record["updated_at"] = timestamp
+        @store.set(SCOPE, key, record)
+        to_pair(record)
+      end
+    end
+
+    # Status recomputation, in one place (never backwards — a judged pair stays
+    # judged, an expired one stays incomplete):
+    def status_for(record)
+      return record["status"] if %w[judged incomplete].include?(record["status"])
+      return "open" if record["insika_reply"].nil? || record["incumbent_reply"].nil?
+      return "silent" if record["insika_reply"].to_s.strip.empty?
+
+      "complete"
+    end
+
+    def key_for(id) = "#{KEY_PREFIX}#{id}"
+
+    # The mirror's reported time on first write; nil = now. A String rides
+    # through as-is (it is the wire format); a Time is normalized to ISO8601.
+    # A String is ALSO normalized to UTC ISO8601: the mirrors report local
+    # offsets (+09:00, -03:00) and every comparison against created_at
+    # (since/expire/retention/unjudged ordering) is lexicographic — two offsets
+    # would make those comparisons lie. Unparseable input keeps the old
+    # ride-through behaviour rather than refusing the pair.
+    def arrival_time(at)
+      return timestamp if at.nil?
+      return at.utc.iso8601 unless at.is_a?(String)
+
+      Time.iso8601(at).utc.iso8601
+    rescue ArgumentError
+      at
+    end
+
+    def to_pair(record)
+      Pair.new(
+        id: record["id"], channel: record["channel"], agent: record["agent"],
+        session_id: record["session_id"], task_id: record["task_id"],
+        event_id: record["event_id"], inbound: record["inbound"],
+        incumbent_reply: record["incumbent_reply"], insika_reply: record["insika_reply"],
+        status: record["status"].to_sym, verdict: record["verdict"],
+        criterion_sha: record["criterion_sha"],
+        created_at: record["created_at"], updated_at: record["updated_at"]
+      )
+    end
+
+    def timestamp = Time.now.utc.iso8601
+  end
+end

data/lib/insika/skill_catalog.rb

@@ -34,6 +34,10 @@ module Insika
       @skills, @agent_skills = load_all
     end
 
+    # The SkillStore the catalog overlays — the composition root hands it to
+    # the harvest (the dedup reads the AUTHORED skills the catalog serves).
+    attr_reader :store
+
     # `agent` (an agent id) resolves the AGENT SCOPE first, then the shared one — the
     # same precedence chain the catalog already runs for store-over-disk and
     # workspace-over-managed-over-bundled, with one more dimension.

data/lib/insika/soak/envelope.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "digest"
+require "yaml"
+
+module Insika
+  module Soak
+    # The frozen pre-declared envelope for a soak run. Parses the
+    # fenced `yaml` block inside a markdown file, validates it completely or not
+    # at all, and exposes the SHA-256 of the WHOLE file — every hourly snapshot
+    # is stamped with it, so editing the envelope mid-run turns the run
+    # `invalid` instead of producing a verdict nobody can defend.
+    #
+    # No defaults, no tolerance: an envelope with a hole is not a
+    # pre-declaration. Contrast with EnvSchema (tolerant, warn-and-boot) — that
+    # resilience protects a live service; this strictness protects a claim.
+    class Envelope
+      # Required keys. A missing one is a ConfigError, not a default.
+      REQUIRED = %i[
+        version target duration_hours warmup_hours
+        arrival turns_per_hour session_turns concurrency_cap web_concurrency
+        rss_growth_ratio prep_p95_drift_ratio
+        restarts_max error_rate_ceiling no_usage_rate_ceiling
+        coverage_min_ratio gap_seconds_max hourly_turn_floor
+      ].freeze
+
+      # Written after E1, before E2 (techspec D3). Absent -> `calibrated?` is
+      # false and the 72h run refuses to start; the 4h dry run does not require them.
+      CALIBRATED = %i[rss_ceiling_mb prep_p95_ceiling_ms total_p95_ceiling_ms].freeze
+
+      # Keys validated as ratios: must be strictly greater than 1 (the gate is
+      # "no more than 1.15x", so a ratio of 1 or below is a nonsense gate).
+      RATIO_KEYS = %i[rss_growth_ratio prep_p95_drift_ratio].freeze
+
+      # Keys validated as positive integers.
+      POSITIVE_COUNT_KEYS = %i[
+        duration_hours warmup_hours turns_per_hour session_turns
+        concurrency_cap web_concurrency gap_seconds_max hourly_turn_floor
+      ].freeze
+
+      # Keys validated as rates in [0, 1].
+      RATE_KEYS = %i[error_rate_ceiling no_usage_rate_ceiling coverage_min_ratio].freeze
+
+      # Raises Insika::ConfigError: file missing, no fenced yaml block,
+      # unparseable, a REQUIRED key absent, or a value out of range.
+      def self.load(path)
+        bytes = File.binread(path)
+        parse(bytes, sha: "sha256:#{Digest::SHA256.hexdigest(bytes)}")
+      rescue Errno::ENOENT
+        raise ConfigError, "soak envelope not found: #{path}"
+      end
+
+      # Pure parse (specs, fixtures). `sha` is stamped verbatim.
+      def self.parse(source, sha:)
+        new(values: validate(YAML.safe_load(fenced_yaml(source)) || {}), sha: sha)
+      rescue Psych::SyntaxError => e
+        raise ConfigError, "soak envelope yaml is unparseable: #{e.message.lines.first.to_s.strip}"
+      end
+
+      def self.fenced_yaml(source)
+        match = source.to_s.match(/```yaml\s*\n(.*?)\n```/m)
+        raise ConfigError, "soak envelope has no fenced yaml block" unless match
+
+        # Markdown fence content may be indented; YAML cares about the column.
+        lines = match[1].lines
+        indent = lines.reject { |l| l.strip.empty? }.map { |l| l[/\A */].length }.min
+        lines.map { |l| l[indent..] || "\n" }.join
+      end
+      private_class_method :fenced_yaml
+
+      def self.validate(raw)
+        values = raw.each_with_object({}) do |(k, v), acc|
+          acc[k.to_sym] = v if k.to_s =~ /\A[a-z_][a-z0-9_]*\z/
+        end
+
+        REQUIRED.each do |key|
+          raise ConfigError, "soak envelope is missing required key: #{key}" if values[key].nil?
+        end
+
+        RATIO_KEYS.each do |key|
+          v = values[key]
+          unless v.is_a?(Numeric) && v > 1
+            raise ConfigError, "soak envelope key #{key} must be > 1 (got #{v.inspect})"
+          end
+        end
+
+        POSITIVE_COUNT_KEYS.each do |key|
+          v = values[key]
+          unless v.is_a?(Integer) && v.positive?
+            raise ConfigError, "soak envelope key #{key} must be a positive integer (got #{v.inspect})"
+          end
+        end
+
+        unless values[:restarts_max].is_a?(Integer) && values[:restarts_max] >= 0
+          raise ConfigError, "soak envelope key restarts_max must be >= 0 (got #{values[:restarts_max].inspect})"
+        end
+
+        RATE_KEYS.each do |key|
+          v = values[key]
+          unless v.is_a?(Numeric) && v >= 0 && v <= 1
+            raise ConfigError, "soak envelope key #{key} must be a rate in 0..1 (got #{v.inspect})"
+          end
+        end
+
+        unless values[:warmup_hours] < values[:duration_hours]
+          raise ConfigError, "soak envelope key warmup_hours must be < duration_hours " \
+                             "(got #{values[:warmup_hours]} >= #{values[:duration_hours]})"
+        end
+
+        CALIBRATED.each do |key|
+          v = values[key]
+          next if v.nil?
+
+          unless v.is_a?(Numeric) && v.positive?
+            raise ConfigError, "soak envelope key #{key} must be a positive number when set (got #{v.inspect})"
+          end
+        end
+
+        values.freeze
+      end
+      private_class_method :validate
+
+      attr_reader :sha, :values
+
+      def initialize(values:, sha:)
+        @values = values
+        @sha = sha
+      end
+
+      def [](key) = values[key]
+
+      def calibrated? = CALIBRATED.all? { |key| !values[key].nil? }
+
+      # E1 shape (<= 8h): the dry run skips the calibrated ceilings.
+      def dry_run? = values[:duration_hours] <= 8
+
+      def to_h = values.merge(sha: sha)
+    end
+  end
+end