insika 0.1.0 → 0.3.0

data/lib/insika/balloon_splitter.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Insika
+  #   — pure: confirmed answer text -> the balloons a progressive
+  # channel should POST. Paragraphs are the unit (`\n\n` is the seam E1 cares
+  # about); the sentence split fires only as a SOFT cap for one paragraph that
+  # grew past `SENTENCE_AFTER`. It never splits inside a fenced code block.
+  #
+  # No events, no outbox, no `:intermediate` — if it is handed loop narration
+  # that is a caller bug; the splitter has no way to tell.
+  module BalloonSplitter
+    # Soft cap after which a single paragraph is split on sentences. WhatsApp's
+    # hard cap is ~4096; this is a readability cap, not a platform one.
+    SENTENCE_AFTER = 600
+
+    module_function
+
+    # -> [String], at least one when `text` is non-blank, else [].
+    def split(text)
+      raw = Insika::Coercion.presence(text)
+      return [] if raw.nil?
+
+      paragraphs = raw.split(PARAGRAPH_BREAK)
+      balloons = group_fenced(paragraphs)
+      balloons.flat_map { |balloon| split_long(balloon) }
+    end
+
+    # A fenced code block (` ``` ` … ` ``` `) spans paragraphs and stays ONE
+    # balloon — newlines inside it are code, not a seam. A fence that never
+    # closes still ends as one balloon (garbage in, one balloon out).
+    def group_fenced(paragraphs)
+      balloons = []
+      fence = nil
+      paragraphs.each do |para|
+        piece = para.strip
+        next if piece.empty?
+
+        if fence
+          fence << piece
+          # a line starting with ``` closes the block, wherever it lands — the
+          # close fence of ordinary markdown shares a paragraph with its code.
+          if closes_fence?(piece)
+            balloons << fence.join("\n\n")
+            fence = nil
+          end
+        elsif piece.lines.first.to_s.start_with?(FENCE)
+          fence = [piece]
+          # one paragraph may open AND close the block (```ruby\nx = 1\n```) —
+          # the closer is a later line of the same piece.
+          if closes_fence?(piece, opener: true)
+            balloons << fence.join("\n\n")
+            fence = nil
+          end
+        else
+          balloons << piece
+        end
+      end
+      balloons << fence.join("\n\n") if fence
+      balloons
+    end
+
+    # Does this paragraph close an open fence? Any line starting with the fence
+    # marker counts. `opener: true` skips the FIRST line — the line that opened
+    # the block, which cannot be its own closer.
+    def closes_fence?(piece, opener: false)
+      lines = piece.lines
+      lines[(opener ? 1 : 0)..].any? { |line| line.start_with?(FENCE) }
+    end
+
+    # The soft cap: a paragraph longer than SENTENCE_AFTER splits after sentence
+    # closers followed by whitespace — which is why `3.9s` and `www.` survive
+    # (their period is not followed by whitespace) and a decimal does too. A
+    # leftover without a closer stays one balloon.
+    #
+    # Sentences are then RE-GROUPED into ~SENTENCE_AFTER blocks (
+    # the split is a cap, not a mandate). Without the regroup, a 680-char
+    # paragraph of short sentences would atomize into one balloon per sentence —
+    # a paragraph that was ONE bubble becomes 40 WhatsApp messages for no
+    # latency win.
+    def split_long(balloon)
+      return [balloon] if balloon.length <= SENTENCE_AFTER
+
+      parts = balloon.split(SENTENCE_BOUNDARY).map(&:strip).reject(&:empty?)
+      return [balloon] if parts.empty?
+
+      parts.each_with_object([+""]) do |sentence, balloons|
+        if balloons.last.empty?
+          balloons.last << sentence
+        elsif balloons.last.length + sentence.length + 1 <= SENTENCE_AFTER
+          balloons.last << " #{sentence}"
+        else
+          balloons << +sentence
+        end
+      end
+    end
+
+    PARAGRAPH_BREAK = /\n\s*\n+/
+    FENCE = "```"
+    SENTENCE_BOUNDARY = /(?<=[.!?…])\s+(?=\S)/
+    private_constant :PARAGRAPH_BREAK, :FENCE, :SENTENCE_BOUNDARY
+  end
+end

data/lib/insika/baseline_store.rb

@@ -3,8 +3,8 @@
 require "time"
 
 module Insika
-  # The ACCEPTED state of an agent's golden set (RFC-0008 §3.4, promoted to a store
-  # by RFC-0013 §3.7.3). One record per agent in the ConfigStore (scope "baselines"):
+  # The ACCEPTED state of an agent's golden set (promoted to a store
+  # by). One record per agent in the ConfigStore (scope "baselines"):
   #
   #   { "at" => iso8601, "cases" => { "<case id>" => { "pass" => bool, "score" => n } } }
   #

data/lib/insika/budget_ledger.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Insika
+  # Fixed-window spend counters for BUDGETS (WS2): the accounting layer the
+  # edge middleware rolls against. One scope, cells keyed
+  # "tenant:agent:window:calendar-bucket":
+  #
+  #   · DAILY    — the UTC calendar day (epoch/86400 IS midnight-aligned).
+  #   · MONTHLY  — (year * 12 + month) of the UTC calendar month: a budget month
+  #                is the CALENDAR month, however many days long it is (a fixed
+  #                N-day window drifts its bucket start across month lengths).
+  #                UTC like the daily, so both rollovers agree on any host.
+  #
+  # Built on the UsageLedger vocabulary (tenant/agent instead of kind/id) but
+  # on the store directly, with the increment riding `@store.transaction` — the
+  # exact read-modify-write discipline WS2's enforcement will build on. Two
+  # processes (or two SQLite handles) racing the same cell serialize on
+  # BEGIN IMMEDIATE: no lost update. No enforcement here — the middleware is
+  # WS2; this file is only correct accounting.
+  #
+  # Growth is bounded on the hot path AND swept: each `add` deletes the (id)'s
+  # previous day and previous month cell (an ACTIVE scope holds at most 4 keys),
+  # and `prune` — the daily sweep on the tick — drops everything else: the cells
+  # of an id that went idle for more than one window and the alert flags, which
+  # the hot path never collects.
+  class BudgetLedger
+    SCOPE = "budget_counters"
+    ALERT_SCOPE = "budget_alerts"
+    DAY = 86_400
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # Adds `by` across both windows; -> { daily:, monthly: } the NEW totals
+    # for (tenant, agent). Atomic per call: one transaction, both bumps.
+    def add(tenant:, agent:, by:, now: Time.now)
+      id = cell_id(tenant, agent)
+      @store.transaction do
+        daily = bump(id, daily_bucket(now), by)
+        monthly = bump(id, month_bucket(now), by)
+        @store.delete(SCOPE, key(id, daily_bucket(now - DAY)))   # previous day cell
+        @store.delete(SCOPE, key(id, month_bucket(now) - 1))     # previous calendar month cell
+        { daily: daily, monthly: monthly }
+      end
+    end
+
+    # -> { daily:, monthly: } current totals for (tenant, agent). Purely
+    # read; an expired window reads as 0 (rolls over at the boundary).
+    def current(tenant:, agent:, now: Time.now)
+      id = cell_id(tenant, agent)
+      { daily: @store.get(SCOPE, key(id, daily_bucket(now))).to_i,
+        monthly: @store.get(SCOPE, key(id, month_bucket(now))).to_i }
+    end
+
+    # Seconds until the window's bucket rolls over (the retry_after the
+    # enforcement quotes when a hard budget refuses a turn). Both windows are
+    # UTC-aligned (the daily via the epoch, the monthly via UTC components) so a
+    # non-UTC host never quotes a negative or local-midnight reset.
+    def reset_in(window, now: Time.now)
+      case window
+      when :daily then DAY - (now.to_i % DAY)
+      when :monthly then (next_utc_month_start(now) - now).to_i
+      end
+    end
+
+    # "1× per window" alert markers (the soft enforcement's event): a flag per
+    # (id, window, level, bucket) so a budget that stays over the threshold
+    # cannot spam one event per turn. `level:` separates DISTINCT triggers in
+    # the same window (WS2): the `alert_at` crossing and the real soft-cap
+    # crossing each warn once — the cap event must not be swallowed by the
+    # 80% marker having fired earlier. Marked/read in the same transaction
+    # discipline. -> bool: had the window already been marked?
+    def mark_alert(tenant:, agent:, window:, level: nil, now: Time.now)
+      id = cell_id(tenant, agent)
+      flag = alert_key(id, window, now, level)
+      @store.transaction do
+        # `next`, NOT `return`: a non-local return from inside the block skips
+        # the store's COMMIT and leaks the BEGIN IMMEDIATE open — the 2nd turn
+        # over a threshold then locks the whole backend (WS2).
+        next true unless @store.get(ALERT_SCOPE, flag).nil?
+
+        @store.set(ALERT_SCOPE, flag, 1)
+        false
+      end
+    end
+
+    def alerted?(tenant:, agent:, window:, level: nil, now: Time.now)
+      !@store.get(ALERT_SCOPE, alert_key(cell_id(tenant, agent), window, now, level)).nil?
+    end
+
+    # The GC of both scopes: drops every cell whose window is not the CURRENT
+    # one. `add`'s two deletes only reach the IMMEDIATELY previous day/month, so
+    # a scope that goes idle for two days leaves its counter behind forever, and
+    # the alert flags were never collected at all — unbounded row growth the WS8
+    # retention sweep does not reach (that one is age-based over CONTENT; these
+    # are counters with no timestamp). Every key of both scopes ENDS in its
+    # bucket, so one rule sweeps both. -> count of cells removed.
+    def prune(now: Time.now)
+      day = daily_bucket(now)
+      month = month_bucket(now)
+      @store.transaction do
+        [SCOPE, ALERT_SCOPE].sum do |scope|
+          stale = @store.list(scope).select { |k| past?(k.rpartition(":").last.to_i, day, month) }
+          stale.each { |k| @store.delete(scope, k) }
+          stale.size
+        end
+      end
+    end
+
+    private
+
+    # No tenant (single_tenant default) is a LITERAL "platform" cell, never a
+    # null-key collision with some other scope.
+    def cell_id(tenant, agent)
+      [tenant || "platform", agent].join(":")
+    end
+
+    # Is that bucket a window STRICTLY BEHIND the live one? The two kinds of
+    # bucket cannot collide — an epoch-day is a multiple of 86_400 (~1.7e9), a
+    # calendar month is year*12+month (~24e3) — so the magnitude tells them
+    # apart. STRICTLY behind, never "not the current one": a host whose clock
+    # runs minutes ahead writes tomorrow's cell around midnight, and a sweeper
+    # that deleted it would hand that tenant a fresh day of budget.
+    def past?(bucket, day, month)
+      bucket >= DAY ? bucket < day : bucket.positive? && bucket < month
+    end
+
+    def bump(id, bucket, by)
+      total = @store.get(SCOPE, key(id, bucket)).to_i + by
+      @store.set(SCOPE, key(id, bucket), total)
+      total
+    end
+
+    # the UTC calendar day's start (epoch is aligned to midnight UTC).
+    def daily_bucket(now)
+      (now.to_i / DAY) * DAY
+    end
+
+    # the UTC calendar month as one integer (2026-08 -> 24296). UTC, not local:
+    # the month boundary must agree with the daily epoch-day boundary on a
+    # non-UTC host (WS2), or the cap resets at a different moment than the day.
+    def month_bucket(now)
+      u = now.utc
+      u.year * 12 + u.month
+    end
+
+    # Midnight UTC of the 1st of the window's NEXT month — December-safe
+    # (Time.utc(y, 13, 1) raises; y+1/1 is the calendar answer).
+    def next_utc_month_start(now)
+      u = now.utc
+      u.month == 12 ? Time.utc(u.year + 1, 1, 1) : Time.utc(u.year, u.month + 1, 1)
+    end
+
+    def key(id, bucket)
+      "#{id}:#{bucket}"
+    end
+
+    # One alert flag per (id, window, level, calendar bucket): daily cells are
+    # keyed by day, monthly by (year*12+month) — a flag dies with its window.
+    def alert_key(id, window, now, level = nil)
+      bucket = window == :monthly ? month_bucket(now) : daily_bucket(now)
+      level ? "#{id}:#{window}:#{level}:#{bucket}" : "#{id}:#{window}:#{bucket}"
+    end
+  end
+end

data/lib/insika/cache_series_store.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Insika
+  #   — per-AGENT cache-hit series (scope "cache_series"), for the
+  # Studio agent-detail plot. Sessions do not stamp their agent, so the
+  # per-session context trace cannot answer "cache-hit over time for THIS
+  # agent"; this capped list can. Entries are counts and a category name only —
+  # PII-free by construction. No retention hook: the cap bounds growth.
+  class CacheSeriesStore
+    SCOPE = "cache_series"
+    MAX_PER_AGENT = 200 # oldest dropped; one entry per turn, so 200 is a
+    # rolling window, not a leak path
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # Appends a sanitized entry for the agent; caps. Rescues everything — the
+    # series never breaks the turn.
+    def record(agent:, entry:)
+      return if agent.to_s.empty?
+
+      list = (@store.get(SCOPE, agent.to_s) || []) + [sanitize(entry)]
+      @store.set(SCOPE, agent.to_s, list.last(MAX_PER_AGENT))
+    rescue StandardError
+      nil
+    end
+
+    # -> [Hash] the agent's series, chronological. [] if none.
+    def for_agent(agent) = @store.get(SCOPE, agent.to_s) || []
+
+    private
+
+    def sanitize(entry)
+      e = entry.is_a?(Hash) ? entry : {}
+      {
+        "at" => (e[:at] || e["at"])&.to_s,
+        "turn" => int(e[:turn] || e["turn"]),
+        "hit_pct" => int_or_nil(e[:hit_pct] || e["hit_pct"]),
+        "cached_tokens" => int(e[:cached_tokens] || e["cached_tokens"]),
+        "prompt_tokens" => int(e[:prompt_tokens] || e["prompt_tokens"]),
+        "invalidation_reason" => (e[:invalidation_reason] || e["invalidation_reason"])&.to_s
+      }
+    end
+
+    def int(value) = Integer(value || 0)
+    def int_or_nil(value) = value.nil? ? nil : Integer(value)
+  end
+end

data/lib/insika/channel_delivery.rb

@@ -3,7 +3,7 @@
 require "time"
 
 module Insika
-  # Hands a finished turn's answer to a Shape B channel (RFC-0011 §6.5). The turn
+  # Hands a finished turn's answer to a Shape B channel. The turn
   # ended; the recipient is not on any connection; the reply has to travel out of
   # band and survive a crash on the way. Three moves, in this order, and the order
   # is the whole design:
@@ -26,7 +26,8 @@ module Insika
     BACKOFF_SECONDS = [1, 5].freeze
 
     def initialize(channels:, outbox:, session_store:, event_stream: nil,
-                   max_attempts: MAX_ATTEMPTS, backoff: BACKOFF_SECONDS, sleeper: nil)
+                   max_attempts: MAX_ATTEMPTS, backoff: BACKOFF_SECONDS, sleeper: nil,
+                   shadow_pairs: nil, criterion_sha: nil)
       @channels = channels
       @outbox = outbox
       @session_store = session_store
@@ -34,29 +35,66 @@ module Insika
       @max_attempts = max_attempts
       @backoff = Array(backoff)
       @sleeper = sleeper || method(:default_sleep)
+      @shadow_pairs = shadow_pairs
+      @criterion_sha = criterion_sha
     end
 
-    # The turn committed an answer. -> the Delivery to dispatch, or nil when there
-    # is nothing to deliver, which is the common case and must stay cheap:
+    # the pair store and the frozen criterion's sha. Both default to
+    # nil (parity — a graph without them behaves exactly as today); the server
+    # root sets them at boot, after the criterion file has been loaded and
+    # refused-or-accepted (the graph itself reads no env and no file).
+    attr_writer :shadow_pairs, :criterion_sha
+
+    # Confirmed answer -> 0..N pending Deliveries, in order .
+    # A progressive channel splits on paragraphs (BalloonSplitter); everything
+    # else is the single whole-answer row.
+    # -> [] when there is nothing to send (the cheap exits):
     #   · the turn did not come in through a channel,
     #   · the channel is Shape A (answers on its own stream — no `deliver`),
     #   · the answer is empty (a turn that died mid-message published nothing, and
-    #     half a sentence was never an answer — P19),
+    #     half a sentence was never an answer),
+    #   · the channel is in SHADOW mode: the answer is recorded as a
+    #     pair and nothing is dispatched — zero outbox writes, ever (E1),
     #   · or we do not know who to send it to.
-    def record(task:, channel_id:, content:)
-      return nil if content.to_s.strip.empty?
-
+    #
+    # `attachments` (evidence cards) ride the outbox payload as an
+    # ADDITIVE key on the LAST balloon — a Shape B channel that reads `payload`
+    # ignores it (JSON contract, additive); one that renders cards consumes it.
+    def record_balloons(task:, channel_id:, content:, progressive:, attachments: nil)
       channel = @channels&.find(channel_id)
-      return nil unless channel.respond_to?(:deliver)
+      return [] unless channel.respond_to?(:deliver)
+      # Shadow records ONE pair for the whole answer — a balloon per paragraph
+      # would mint N pairs for one turn.
+      if shadow?(channel)
+        record_shadow(task, channel_id, content)
+        return []
+      end
+
+      return [] if content.to_s.strip.empty?
 
       to = recipient(channel, task.session_id)
-      return nil if to.nil? || to.empty?
+      return [] if to.nil? || to.empty?
+
+      parts = progressive ? Insika::BalloonSplitter.split(content) : [content.to_s]
+      parts = parts.reject { |p| p.to_s.strip.empty? }
+      return [] if parts.empty?
+
+      multi = parts.size > 1
+      parts.each_with_index.map do |part, i|
+        last = i == parts.size - 1
+        create_pending(task, channel_id, part, to,
+                       index: multi ? i : nil, final: multi ? last : nil,
+                       attachments: last ? attachments : nil)
+      end
+    end
+
+    def shadow?(channel) = channel.respond_to?(:shadow?) && channel.shadow?
 
-      @outbox.create(
-        channel: channel_id, to: to, task_id: task.id, session_id: task.session_id,
-        payload: { "session_id" => task.session_id.to_s, "task_id" => task.id.to_s,
-                   "content" => content.to_s }
-      )
+    # does this channel flush progressively? Duck-typed — a channel
+    # that does not answer `progressive?` is `:at_end`.
+    def progressive?(channel_id)
+      channel = @channels&.find(channel_id)
+      channel.respond_to?(:progressive?) && channel.progressive?
     end
 
     # Claim + POST + bounded retry. Safe to call twice: the second caller loses the
@@ -90,6 +128,73 @@ module Insika
 
     private
 
+    # One outbox row for a confirmed balloon. `index`/`final` ride the payload
+    # only when non-nil — a single-balloon progressive turn is indistinguishable
+    # from an `:at_end` one on the wire. `index` also lands on the RECORD, which
+    # is what the boot sweep orders by. `attachments` :
+    # validated for the outbox — malformed entries dropped, never a turn failure.
+    def create_pending(task, channel_id, content, to, index: nil, final: nil, attachments: nil)
+      payload = { "session_id" => task.session_id.to_s, "task_id" => task.id.to_s,
+                  "content" => content.to_s }
+      payload["index"] = index if index
+      payload["final"] = final unless final.nil?
+      atts = Insika::Evidence.valid_attachments(attachments)
+      payload["attachments"] = atts unless atts.empty?
+      @outbox.create(channel: channel_id, to: to, task_id: task.id, session_id: task.session_id,
+                     payload: payload, index: index.to_i)
+    end
+
+    # Our half of the shadow pair (C3). One store upsert on the turn's terminal,
+    # then nil — `Executor#finalize_channel_delivery` returns without dispatching.
+    # The ordering rules: no event_id -> :shadow_unpairable (C1 makes this
+    # unreachable through the relay; a plugin channel could still get it wrong);
+    # no pair store wired -> the same event (fail-closed, nothing delivered);
+    # no recipient -> the same event (a pair keyed on an empty external_id can
+    # never meet the mirror's half).
+    def record_shadow(task, channel_id, content)
+      command = task.respond_to?(:command) ? task.command : nil
+      payload = command.is_a?(Hash) ? (command["payload"] || command[:payload] || {}) : {}
+      agent = payload["agent"] || payload[:agent]
+      message = payload["message"] || payload[:message]
+      event_id = Insika::Coercion.presence(payload["event_id"] || payload[:event_id])
+      if event_id.nil? || @shadow_pairs.nil?
+        emit_shadow(:shadow_unpairable, channel_id, agent, nil, silent: nil)
+        return nil
+      end
+
+      channel = @channels&.find(channel_id)
+      external_id = recipient(channel, task.session_id)
+      # The same empty-recipient guard the delivery path has: a pair keyed on an
+      # empty external_id can never meet the mirror's half (its digest differs),
+      # so the pair would sit :open forever. C1 makes this unreachable through
+      # the relay; a plugin channel could still get it wrong.
+      if Insika::Coercion.presence(external_id).nil?
+        emit_shadow(:shadow_unpairable, channel_id, agent, nil, silent: nil)
+        return nil
+      end
+
+      silent = content.to_s.strip.empty?
+      id = Insika::ShadowPairStore.key_for(channel: channel_id, external_id: external_id,
+                                           event_id: event_id)
+      @shadow_pairs.record_ours(id: id, channel: channel_id, agent: agent,
+                                session_id: task.session_id, task_id: task.id,
+                                event_id: event_id, inbound: message.to_s,
+                                reply: content.to_s, criterion_sha: @criterion_sha)
+      emit_shadow(:shadow_recorded, channel_id, agent, id, silent: silent)
+      nil
+    end
+
+    # Metadata only: the stream reaches every subscriber and stays free of
+    # customer content, per the Studio's own emit_operator_action rule.
+    def emit_shadow(type, channel, agent, pair_id, silent:)
+      return unless @event_stream
+
+      data = { channel: channel.to_s, agent: agent, pair_id: pair_id }.compact
+      data[:silent] = silent unless silent.nil?
+      @event_stream.emit(Insika::Event.new(type: type, data: data,
+                                           meta: { at: Time.now.utc.iso8601 }))
+    end
+
     def attempt(delivery, channel)
       last_error = nil
 
@@ -111,7 +216,7 @@ module Insika
     end
 
     # The consumer's own key for this conversation. Written into the session's vars
-    # when the channel minted it (§4.3); the channel's own id parser is the fallback
+    # when the channel minted it; the channel's own id parser is the fallback
     # for a session created before those vars existed.
     def recipient(channel, session_id)
       session = session_id && @session_store.find(session_id)
@@ -130,14 +235,17 @@ module Insika
     def emit(delivery)
       return unless @event_stream
 
-      @event_stream.emit(Insika::Event.new(
-                           type: :channel_delivered,
-                           data: { channel: delivery.channel, outbox_id: delivery.id,
-                                   status: delivery.status.to_s, attempts: delivery.attempts,
-                                   error: delivery.last_error },
-                           meta: { task_id: delivery.task_id, session_id: delivery.session_id,
-                                   at: Time.now.utc.iso8601 }
-                         ))
+      data = { channel: delivery.channel, outbox_id: delivery.id,
+               status: delivery.status.to_s, attempts: delivery.attempts,
+               error: delivery.last_error }
+      meta = { task_id: delivery.task_id, session_id: delivery.session_id,
+               at: Time.now.utc.iso8601 }
+      # :delivery_failed is the ALERT face of a failed delivery (WS6) — emitted
+      # alongside :channel_delivered so the delivery audit stream is unchanged.
+      @event_stream.emit(Insika::Event.new(type: :channel_delivered, data: data, meta: meta))
+      if delivery.status == :failed
+        @event_stream.emit(Insika::Event.new(type: :delivery_failed, data: data, meta: meta))
+      end
     end
 
     # Async when there is a reactor (production: the retry must not block the

data/lib/insika/channel_registry.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Insika
-  # The channels this deployment speaks, by id (RFC-0011 §4.2). A `Registry` like
+  # The channels this deployment speaks, by id. A `Registry` like
   # tools and workflows — same plugin bookkeeping, so `deregister_plugin` rolls a
   # half-registered plugin back exactly as it does for a tool.
   #