insika 0.2.0 → 0.7.0

data/lib/insika/artifact_store.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "time"
+
+module Insika
+  # The report destination: one record per run, no versioning, the listing
+  # IS the history. A store, not a CMS.
+  #
+  # The tenant is a BINDING of the row — inherited from the agent that saved
+  # the artifact (the tool reads it from the turn context, never from the
+  # model), so a purge is a tenant-prefix scan and store A's report can never
+  # appear in, or be linked from, store B.
+  #
+  # Record key: "<tenant>:<agent>:<id>" — per-tenant / per-agent scans are
+  # prefixes; the id is the LAST segment, so a find is a suffix match (the
+  # followup_store.rb idiom — there is no stable prefix for an id alone).
+  # Blank tenant -> the literal "platform" (the outcome_store.rb rule, so the
+  # purge prefix scans line up).
+  class ArtifactStore
+    SCOPE = "artifacts"
+
+    # The mime allowlist — a page, not an attachment: no binaries, no uploads.
+    MIMES = %w[text/html text/markdown image/svg+xml].freeze
+
+    # The size cap on `content` (INSIKA_ARTIFACT_MAX_BYTES; the audio-message
+    # precedent is 1 MB — an artifact is a page, not an attachment).
+    DEFAULT_MAX_BYTES = 1_000_000
+    TITLE_MAX = 200
+
+    Record = Data.define(:id, :tenant, :agent, :task_id, :title, :mime,
+                         :content, :created_at)
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # -> Record. Validates the mime allowlist, a non-empty title (<= 200
+    # chars) and content within `max_bytes` — ValidationError otherwise (the
+    # tool returns it to the model as `{ error: }`).
+    def create(tenant:, agent:, task_id:, title:, mime:, content:,
+               id: SecureRandom.uuid, now: Time.now.utc, max_bytes: DEFAULT_MAX_BYTES)
+      mime = "text/html" if mime.to_s.empty?
+      raise Insika::ValidationError, "mime must be one of #{MIMES.join(', ')}, got #{mime.inspect}" unless MIMES.include?(mime.to_s)
+
+      title = title.to_s
+      raise Insika::ValidationError, "title is required (1..#{TITLE_MAX} chars)" if title.strip.empty? || title.length > TITLE_MAX
+
+      content = content.to_s
+      raise Insika::ValidationError, "content is required" if content.empty?
+      raise Insika::ValidationError, "content exceeds #{max_bytes} bytes (#{content.bytesize})" if content.bytesize > max_bytes
+
+      record = { "id" => id.to_s, "tenant" => tenant_id(tenant), "agent" => agent.to_s,
+                 "task_id" => task_id.to_s, "title" => title, "mime" => mime.to_s,
+                 "content" => content, "created_at" => now.iso8601 }
+      @store.set(SCOPE, key(record), record)
+      to_record(record)
+    end
+
+    # -> Record | nil. Suffix match over the scope's keys (no stable prefix
+    # for an id alone — the followup_store.rb idiom).
+    def find(id)
+      key = @store.list(SCOPE).find { |k| k.end_with?(":#{id}") }
+      key && to_record(@store.get(SCOPE, key))
+    end
+
+    # -> [Record] — one agent's artifacts, newest first (the listing IS the
+    # history; the Studio tab's read).
+    def for_agent(tenant:, agent:)
+      prefix = "#{tenant_id(tenant)}:#{agent}:"
+      @store.list(SCOPE).filter_map do |k|
+        next unless k.start_with?(prefix)
+
+        to_record(@store.get(SCOPE, k))
+      end.sort_by { |r| [r.created_at.to_s, r.id] }.reverse
+    end
+
+    # -> [Record] — EVERY agent's artifacts for the tenant, newest first (same
+    # prefix scan as #purge, but reads instead of deletes). Needs the CALLER
+    # to know the right tenant string — see #all below for the Studio, which
+    # doesn't.
+    def for_tenant(tenant:)
+      prefix = "#{tenant_id(tenant)}:"
+      @store.list(SCOPE).filter_map do |k|
+        next unless k.start_with?(prefix)
+
+        to_record(@store.get(SCOPE, k))
+      end.sort_by { |r| [r.created_at.to_s, r.id] }.reverse
+    end
+
+    # -> [Record] — every artifact in the store, optionally narrowed to one
+    # agent, with NO tenant guess. For the Studio only: it is already
+    # operator-only (sees every tenant's agents), and unlike #for_agent/
+    # #for_tenant (hot, tenant-scoped runtime paths that a real multi-tenant
+    # caller uses because it KNOWS its own tenant) the Studio often does not
+    # — `save_artifact` binds tenant from the turn (`agent` when the Playground
+    # dispatched it, `"platform"` for a plain single-tenant API turn), so a
+    # page guessing one fixed string is wrong for the other half of the time.
+    # Filtering by the record's own `agent` field sidesteps the guess entirely.
+    def all(agent: nil)
+      @store.list(SCOPE).filter_map do |k|
+        record = to_record(@store.get(SCOPE, k))
+        next if record.nil?
+        next if agent && record.agent != agent.to_s
+
+        record
+      end.sort_by { |r| [r.created_at.to_s, r.id] }.reverse
+    end
+
+    # -> bool (did it exist?).
+    def delete(id)
+      key = @store.list(SCOPE).find { |k| k.end_with?(":#{id}") }
+      key ? @store.delete(SCOPE, key) : false
+    end
+
+    # -> count removed. The tenant-erasure reach — one tenant's artifacts die
+    # with it, never a neighbour's.
+    def purge(tenant:)
+      prefix = "#{tenant_id(tenant)}:"
+      keys = @store.list(SCOPE).select { |k| k.start_with?(prefix) }
+      keys.each { |k| @store.delete(SCOPE, k) }
+      keys.size
+    end
+
+    # -> count removed. The retention knob's reach (`artifact_ttl_days`) —
+    # the guarantee that PII inside a report expires even though no reader
+    # can see inside the opaque HTML.
+    def delete_older_than(time)
+      cutoff = time.utc.iso8601
+      removed = 0
+      @store.list(SCOPE).each do |k|
+        record = @store.get(SCOPE, k)
+        next unless record && record["created_at"].to_s < cutoff
+
+        @store.delete(SCOPE, k)
+        removed += 1
+      end
+      removed
+    end
+
+    private
+
+    def key(record)
+      "#{record['tenant']}:#{record['agent']}:#{record['id']}"
+    end
+
+    def tenant_id(tenant)
+      t = tenant.to_s
+      t.empty? ? "platform" : t
+    end
+
+    def to_record(rec)
+      return nil if rec.nil?
+
+      Record.new(id: rec["id"], tenant: rec["tenant"], agent: rec["agent"],
+                 task_id: rec["task_id"], title: rec["title"], mime: rec["mime"],
+                 content: rec["content"], created_at: rec["created_at"])
+    end
+  end
+end

data/lib/insika/balloon_splitter.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Insika
+  #   — pure: confirmed answer text -> the balloons a progressive
+  # channel should POST. Paragraphs are the unit (`\n\n` is the seam E1 cares
+  # about); the sentence split fires only as a SOFT cap for one paragraph that
+  # grew past `SENTENCE_AFTER`. It never splits inside a fenced code block.
+  #
+  # No events, no outbox, no `:intermediate` — if it is handed loop narration
+  # that is a caller bug; the splitter has no way to tell.
+  module BalloonSplitter
+    # Soft cap after which a single paragraph is split on sentences. WhatsApp's
+    # hard cap is ~4096; this is a readability cap, not a platform one.
+    SENTENCE_AFTER = 600
+
+    module_function
+
+    # -> [String], at least one when `text` is non-blank, else [].
+    def split(text)
+      raw = Insika::Coercion.presence(text)
+      return [] if raw.nil?
+
+      paragraphs = raw.split(PARAGRAPH_BREAK)
+      balloons = group_fenced(paragraphs)
+      balloons.flat_map { |balloon| split_long(balloon) }
+    end
+
+    # A fenced code block (` ``` ` … ` ``` `) spans paragraphs and stays ONE
+    # balloon — newlines inside it are code, not a seam. A fence that never
+    # closes still ends as one balloon (garbage in, one balloon out).
+    def group_fenced(paragraphs)
+      balloons = []
+      fence = nil
+      paragraphs.each do |para|
+        piece = para.strip
+        next if piece.empty?
+
+        if fence
+          fence << piece
+          # a line starting with ``` closes the block, wherever it lands — the
+          # close fence of ordinary markdown shares a paragraph with its code.
+          if closes_fence?(piece)
+            balloons << fence.join("\n\n")
+            fence = nil
+          end
+        elsif piece.lines.first.to_s.start_with?(FENCE)
+          fence = [piece]
+          # one paragraph may open AND close the block (```ruby\nx = 1\n```) —
+          # the closer is a later line of the same piece.
+          if closes_fence?(piece, opener: true)
+            balloons << fence.join("\n\n")
+            fence = nil
+          end
+        else
+          balloons << piece
+        end
+      end
+      balloons << fence.join("\n\n") if fence
+      balloons
+    end
+
+    # Does this paragraph close an open fence? Any line starting with the fence
+    # marker counts. `opener: true` skips the FIRST line — the line that opened
+    # the block, which cannot be its own closer.
+    def closes_fence?(piece, opener: false)
+      lines = piece.lines
+      lines[(opener ? 1 : 0)..].any? { |line| line.start_with?(FENCE) }
+    end
+
+    # The soft cap: a paragraph longer than SENTENCE_AFTER splits after sentence
+    # closers followed by whitespace — which is why `3.9s` and `www.` survive
+    # (their period is not followed by whitespace) and a decimal does too. A
+    # leftover without a closer stays one balloon.
+    #
+    # Sentences are then RE-GROUPED into ~SENTENCE_AFTER blocks (
+    # the split is a cap, not a mandate). Without the regroup, a 680-char
+    # paragraph of short sentences would atomize into one balloon per sentence —
+    # a paragraph that was ONE bubble becomes 40 WhatsApp messages for no
+    # latency win.
+    def split_long(balloon)
+      return [balloon] if balloon.length <= SENTENCE_AFTER
+
+      parts = balloon.split(SENTENCE_BOUNDARY).map(&:strip).reject(&:empty?)
+      return [balloon] if parts.empty?
+
+      parts.each_with_object([+""]) do |sentence, balloons|
+        if balloons.last.empty?
+          balloons.last << sentence
+        elsif balloons.last.length + sentence.length + 1 <= SENTENCE_AFTER
+          balloons.last << " #{sentence}"
+        else
+          balloons << +sentence
+        end
+      end
+    end
+
+    PARAGRAPH_BREAK = /\n\s*\n+/
+    FENCE = "```"
+    SENTENCE_BOUNDARY = /(?<=[.!?…])\s+(?=\S)/
+    private_constant :PARAGRAPH_BREAK, :FENCE, :SENTENCE_BOUNDARY
+  end
+end

data/lib/insika/budget_ledger.rb

@@ -18,9 +18,11 @@ module Insika
   # BEGIN IMMEDIATE: no lost update. No enforcement here — the middleware is
   # WS2; this file is only correct accounting.
   #
-  # Growth is bounded like UsageLedger: each `add` deletes the (id)'s previous
-  # day AND previous month cell, so an active scope holds at most 4 keys and an
-  # idle one converges to 2.
+  # Growth is bounded on the hot path AND swept: each `add` deletes the (id)'s
+  # previous day and previous month cell (an ACTIVE scope holds at most 4 keys),
+  # and `prune` — the daily sweep on the tick — drops everything else: the cells
+  # of an id that went idle for more than one window and the alert flags, which
+  # the hot path never collects.
   class BudgetLedger
     SCOPE = "budget_counters"
     ALERT_SCOPE = "budget_alerts"
@@ -87,6 +89,25 @@ module Insika
       !@store.get(ALERT_SCOPE, alert_key(cell_id(tenant, agent), window, now, level)).nil?
     end
 
+    # The GC of both scopes: drops every cell whose window is not the CURRENT
+    # one. `add`'s two deletes only reach the IMMEDIATELY previous day/month, so
+    # a scope that goes idle for two days leaves its counter behind forever, and
+    # the alert flags were never collected at all — unbounded row growth the WS8
+    # retention sweep does not reach (that one is age-based over CONTENT; these
+    # are counters with no timestamp). Every key of both scopes ENDS in its
+    # bucket, so one rule sweeps both. -> count of cells removed.
+    def prune(now: Time.now)
+      day = daily_bucket(now)
+      month = month_bucket(now)
+      @store.transaction do
+        [SCOPE, ALERT_SCOPE].sum do |scope|
+          stale = @store.list(scope).select { |k| past?(k.rpartition(":").last.to_i, day, month) }
+          stale.each { |k| @store.delete(scope, k) }
+          stale.size
+        end
+      end
+    end
+
     private
 
     # No tenant (single_tenant default) is a LITERAL "platform" cell, never a
@@ -95,6 +116,16 @@ module Insika
       [tenant || "platform", agent].join(":")
     end
 
+    # Is that bucket a window STRICTLY BEHIND the live one? The two kinds of
+    # bucket cannot collide — an epoch-day is a multiple of 86_400 (~1.7e9), a
+    # calendar month is year*12+month (~24e3) — so the magnitude tells them
+    # apart. STRICTLY behind, never "not the current one": a host whose clock
+    # runs minutes ahead writes tomorrow's cell around midnight, and a sweeper
+    # that deleted it would hand that tenant a fresh day of budget.
+    def past?(bucket, day, month)
+      bucket >= DAY ? bucket < day : bucket.positive? && bucket < month
+    end
+
     def bump(id, bucket, by)
       total = @store.get(SCOPE, key(id, bucket)).to_i + by
       @store.set(SCOPE, key(id, bucket), total)

data/lib/insika/cache_series_store.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Insika
+  #   — per-AGENT cache-hit series (scope "cache_series"), for the
+  # Studio agent-detail plot. Sessions do not stamp their agent, so the
+  # per-session context trace cannot answer "cache-hit over time for THIS
+  # agent"; this capped list can. Entries are counts and a category name only —
+  # PII-free by construction. No retention hook: the cap bounds growth.
+  class CacheSeriesStore
+    SCOPE = "cache_series"
+    MAX_PER_AGENT = 200 # oldest dropped; one entry per turn, so 200 is a
+    # rolling window, not a leak path
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # Appends a sanitized entry for the agent; caps. Rescues everything — the
+    # series never breaks the turn.
+    def record(agent:, entry:)
+      return if agent.to_s.empty?
+
+      list = (@store.get(SCOPE, agent.to_s) || []) + [sanitize(entry)]
+      @store.set(SCOPE, agent.to_s, list.last(MAX_PER_AGENT))
+    rescue StandardError
+      nil
+    end
+
+    # -> [Hash] the agent's series, chronological. [] if none.
+    def for_agent(agent) = @store.get(SCOPE, agent.to_s) || []
+
+    private
+
+    def sanitize(entry)
+      e = entry.is_a?(Hash) ? entry : {}
+      {
+        "at" => (e[:at] || e["at"])&.to_s,
+        "turn" => int(e[:turn] || e["turn"]),
+        "hit_pct" => int_or_nil(e[:hit_pct] || e["hit_pct"]),
+        "cached_tokens" => int(e[:cached_tokens] || e["cached_tokens"]),
+        "prompt_tokens" => int(e[:prompt_tokens] || e["prompt_tokens"]),
+        "invalidation_reason" => (e[:invalidation_reason] || e["invalidation_reason"])&.to_s
+      }
+    end
+
+    def int(value) = Integer(value || 0)
+    def int_or_nil(value) = value.nil? ? nil : Integer(value)
+  end
+end

data/lib/insika/channel_delivery.rb

@@ -17,7 +17,7 @@ module Insika
   #      party with outages, so "keep trying" is a real requirement and "keep trying
   #      forever" is a real outage of ours.
   #
-  # It is NOT a job queue: no scheduler, no priorities, no fan-out. The moment it
+  # It is NOT a job queue: no priorities, no fan-out. The moment it
   # grows one, the thing to do is take a real queue, not to finish building this.
   class ChannelDelivery
     MAX_ATTEMPTS = 3
@@ -26,7 +26,8 @@ module Insika
     BACKOFF_SECONDS = [1, 5].freeze
 
     def initialize(channels:, outbox:, session_store:, event_stream: nil,
-                   max_attempts: MAX_ATTEMPTS, backoff: BACKOFF_SECONDS, sleeper: nil)
+                   max_attempts: MAX_ATTEMPTS, backoff: BACKOFF_SECONDS, sleeper: nil,
+                   shadow_pairs: nil, criterion_sha: nil)
       @channels = channels
       @outbox = outbox
       @session_store = session_store
@@ -34,29 +35,66 @@ module Insika
       @max_attempts = max_attempts
       @backoff = Array(backoff)
       @sleeper = sleeper || method(:default_sleep)
+      @shadow_pairs = shadow_pairs
+      @criterion_sha = criterion_sha
     end
 
-    # The turn committed an answer. -> the Delivery to dispatch, or nil when there
-    # is nothing to deliver, which is the common case and must stay cheap:
+    # the pair store and the frozen criterion's sha. Both default to
+    # nil (parity — a graph without them behaves exactly as today); the server
+    # root sets them at boot, after the criterion file has been loaded and
+    # refused-or-accepted (the graph itself reads no env and no file).
+    attr_writer :shadow_pairs, :criterion_sha
+
+    # Confirmed answer -> 0..N pending Deliveries, in order .
+    # A progressive channel splits on paragraphs (BalloonSplitter); everything
+    # else is the single whole-answer row.
+    # -> [] when there is nothing to send (the cheap exits):
     #   · the turn did not come in through a channel,
     #   · the channel is Shape A (answers on its own stream — no `deliver`),
     #   · the answer is empty (a turn that died mid-message published nothing, and
     #     half a sentence was never an answer),
+    #   · the channel is in SHADOW mode: the answer is recorded as a
+    #     pair and nothing is dispatched — zero outbox writes, ever (E1),
     #   · or we do not know who to send it to.
-    def record(task:, channel_id:, content:)
-      return nil if content.to_s.strip.empty?
-
+    #
+    # `attachments` (evidence cards) ride the outbox payload as an
+    # ADDITIVE key on the LAST balloon — a Shape B channel that reads `payload`
+    # ignores it (JSON contract, additive); one that renders cards consumes it.
+    def record_balloons(task:, channel_id:, content:, progressive:, attachments: nil)
       channel = @channels&.find(channel_id)
-      return nil unless channel.respond_to?(:deliver)
+      return [] unless channel.respond_to?(:deliver)
+      # Shadow records ONE pair for the whole answer — a balloon per paragraph
+      # would mint N pairs for one turn.
+      if shadow?(channel)
+        record_shadow(task, channel_id, content)
+        return []
+      end
+
+      return [] if content.to_s.strip.empty?
 
       to = recipient(channel, task.session_id)
-      return nil if to.nil? || to.empty?
+      return [] if to.nil? || to.empty?
+
+      parts = progressive ? Insika::BalloonSplitter.split(content) : [content.to_s]
+      parts = parts.reject { |p| p.to_s.strip.empty? }
+      return [] if parts.empty?
+
+      multi = parts.size > 1
+      parts.each_with_index.map do |part, i|
+        last = i == parts.size - 1
+        create_pending(task, channel_id, part, to,
+                       index: multi ? i : nil, final: multi ? last : nil,
+                       attachments: last ? attachments : nil)
+      end
+    end
+
+    def shadow?(channel) = channel.respond_to?(:shadow?) && channel.shadow?
 
-      @outbox.create(
-        channel: channel_id, to: to, task_id: task.id, session_id: task.session_id,
-        payload: { "session_id" => task.session_id.to_s, "task_id" => task.id.to_s,
-                   "content" => content.to_s }
-      )
+    # does this channel flush progressively? Duck-typed — a channel
+    # that does not answer `progressive?` is `:at_end`.
+    def progressive?(channel_id)
+      channel = @channels&.find(channel_id)
+      channel.respond_to?(:progressive?) && channel.progressive?
     end
 
     # Claim + POST + bounded retry. Safe to call twice: the second caller loses the
@@ -90,6 +128,73 @@ module Insika
 
     private
 
+    # One outbox row for a confirmed balloon. `index`/`final` ride the payload
+    # only when non-nil — a single-balloon progressive turn is indistinguishable
+    # from an `:at_end` one on the wire. `index` also lands on the RECORD, which
+    # is what the boot sweep orders by. `attachments` :
+    # validated for the outbox — malformed entries dropped, never a turn failure.
+    def create_pending(task, channel_id, content, to, index: nil, final: nil, attachments: nil)
+      payload = { "session_id" => task.session_id.to_s, "task_id" => task.id.to_s,
+                  "content" => content.to_s }
+      payload["index"] = index if index
+      payload["final"] = final unless final.nil?
+      atts = Insika::Evidence.valid_attachments(attachments)
+      payload["attachments"] = atts unless atts.empty?
+      @outbox.create(channel: channel_id, to: to, task_id: task.id, session_id: task.session_id,
+                     payload: payload, index: index.to_i)
+    end
+
+    # Our half of the shadow pair (C3). One store upsert on the turn's terminal,
+    # then nil — `Executor#finalize_channel_delivery` returns without dispatching.
+    # The ordering rules: no event_id -> :shadow_unpairable (C1 makes this
+    # unreachable through the relay; a plugin channel could still get it wrong);
+    # no pair store wired -> the same event (fail-closed, nothing delivered);
+    # no recipient -> the same event (a pair keyed on an empty external_id can
+    # never meet the mirror's half).
+    def record_shadow(task, channel_id, content)
+      command = task.respond_to?(:command) ? task.command : nil
+      payload = command.is_a?(Hash) ? (command["payload"] || command[:payload] || {}) : {}
+      agent = payload["agent"] || payload[:agent]
+      message = payload["message"] || payload[:message]
+      event_id = Insika::Coercion.presence(payload["event_id"] || payload[:event_id])
+      if event_id.nil? || @shadow_pairs.nil?
+        emit_shadow(:shadow_unpairable, channel_id, agent, nil, silent: nil)
+        return nil
+      end
+
+      channel = @channels&.find(channel_id)
+      external_id = recipient(channel, task.session_id)
+      # The same empty-recipient guard the delivery path has: a pair keyed on an
+      # empty external_id can never meet the mirror's half (its digest differs),
+      # so the pair would sit :open forever. C1 makes this unreachable through
+      # the relay; a plugin channel could still get it wrong.
+      if Insika::Coercion.presence(external_id).nil?
+        emit_shadow(:shadow_unpairable, channel_id, agent, nil, silent: nil)
+        return nil
+      end
+
+      silent = content.to_s.strip.empty?
+      id = Insika::ShadowPairStore.key_for(channel: channel_id, external_id: external_id,
+                                           event_id: event_id)
+      @shadow_pairs.record_ours(id: id, channel: channel_id, agent: agent,
+                                session_id: task.session_id, task_id: task.id,
+                                event_id: event_id, inbound: message.to_s,
+                                reply: content.to_s, criterion_sha: @criterion_sha)
+      emit_shadow(:shadow_recorded, channel_id, agent, id, silent: silent)
+      nil
+    end
+
+    # Metadata only: the stream reaches every subscriber and stays free of
+    # customer content, per the Studio's own emit_operator_action rule.
+    def emit_shadow(type, channel, agent, pair_id, silent:)
+      return unless @event_stream
+
+      data = { channel: channel.to_s, agent: agent, pair_id: pair_id }.compact
+      data[:silent] = silent unless silent.nil?
+      @event_stream.emit(Insika::Event.new(type: type, data: data,
+                                           meta: { at: Time.now.utc.iso8601 }))
+    end
+
     def attempt(delivery, channel)
       last_error = nil