terret-core 0.1.0 → 0.1.1

data/lib/terret/redactor.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Terret
+  # ctx[:redactor] — §13's "credentials never enter the session log", in the
+  # two layers docs/exec.md §6 describes.
+  #
+  # The FIRST is a tools/post_execute listener: the waterfall every tool
+  # result already passes through, so a tool that happened to hand back a
+  # credential is rewritten before the loop appends it — and before any other
+  # post_execute listener, or any in-memory consumer of the Result, reads it.
+  #
+  # The SECOND is a Sessions scrubber, and it is the one that makes the
+  # promise true rather than likely: a secret can reach the log through a path
+  # that never touches a tool result at all — a user's own message, an
+  # injected steer, a plugin's durable event — and register_scrubber runs over
+  # every String of every append regardless of type. Because that runs INSIDE
+  # normalize_payload, the stored bytes and every projection derived from them
+  # agree by construction, so the log invariant needs nothing special here.
+  #
+  # What this is not: comprehensive. Patterns are regexp sources on this row's
+  # config, so it catches shapes a deployment named and nothing else. Catching a
+  # secret by its exact bytes rather than by a named shape is ctx[:credentials]'s
+  # job (plan §6.9, Terret::Credentials): it registers every value it resolves
+  # as its own append-boundary scrubber, running alongside this one.
+  class Redactor < Hames::Service
+    service_key :redactor
+    inject :sessions
+    config_schema patterns:    { type: Array, default: [],
+                                 doc: "regexp/string patterns scrubbed from tool output and the log" },
+                  replacement: { type: String, default: "[REDACTED]",
+                                 doc: "text a matched secret is replaced with" }
+
+    DEFAULT_REPLACEMENT = "[REDACTED]"
+
+    def start(ctx)
+      @patterns = compile(config[:patterns])
+
+      ctx.on("tools/post_execute") { |result, next_| next_.(redact_result(result)) }
+      # Registered through the seam rather than as a second listener: this
+      # runs at the append boundary itself (Sessions#register_scrubber), which
+      # is the whole reason the backstop is trustworthy.
+      ctx[:sessions].register_scrubber(method(:redact))
+    end
+
+    # Patterns are compiled, so they are the one thing a hot swap has to
+    # re-derive; the replacement token is read per call and is already live.
+    # An uncompilable pattern raises here, and the loader rolls the row back.
+    def reconfigure(config)
+      @patterns = compile(config[:patterns])
+    end
+
+    # The redaction itself, and the method the Sessions scrubber calls. A
+    # non-String is handed straight back: this is reached from the tools
+    # pipeline too, where a result's content may be anything a handler
+    # returned.
+    def redact(text)
+      return text unless text.is_a?(String)
+
+      # gsub's BLOCK form, not its string form, which would read `\0`/`\1` in
+      # a deployment's replacement token as backreferences — a token
+      # containing `\0` would paste the matched secret back in and leave a
+      # redactor that silently un-redacts.
+      @patterns.reduce(text) { |acc, pattern| acc.gsub(pattern) { replacement } }
+    end
+
+    # Public because a reader downstream has to recognize this mechanism's own
+    # mark: the loop refuses to replay a resumed tool call whose stored
+    # arguments carry it (Loop#redaction_token).
+    def replacement = config[:replacement] || DEFAULT_REPLACEMENT
+
+    private
+
+    # Both fields, because both are model-visible: derive_messages projects a
+    # tool/result's error alongside its content, so a handler that echoes the
+    # credential it was handed into a Failure message leaks through exactly
+    # the same door. The append scrubber would catch either, but this layer
+    # exists so the Result object itself is clean the moment it leaves the
+    # pipeline.
+    def redact_result(result)
+      return result unless result.is_a?(Tools::Result)
+
+      result.with(content: redact(result.content), error: redact(result.error))
+    end
+
+    # A pattern that does not compile is a configuration bug, so it fails at
+    # boot (or at the reconfigure that introduced it) rather than raising on
+    # every append forever after. A Regexp is accepted as itself, so a plugin
+    # composing rows in Ruby need not stringify what it already has.
+    def compile(patterns)
+      Array(patterns).map do |pattern|
+        next pattern if pattern.is_a?(Regexp)
+
+        begin
+          Regexp.new(pattern.to_s)
+        rescue RegexpError => e
+          raise RegexpError, "redactor pattern #{pattern.inspect} does not compile: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/terret/sessions.rb

@@ -8,6 +8,12 @@ module Terret
   SessionEvent = Data.define(:id, :session_id, :seq, :at, :type, :payload)
 
   class LogInvariantViolation < StandardError; end
+  class NonPrimitivePayload < StandardError; end
+  # A registered scrubber broke its contract. Distinct from
+  # NonPrimitivePayload on purpose: that one means the DATA was unstorable
+  # (something a caller can fix by encoding it), this one means a PLUGIN is
+  # broken, and the two must not be rescued by the same handler.
+  class ScrubberContractViolation < StandardError; end
 
   # ctx.sessions — the append-only session log. The single source of the
   # context the model sees: derive_messages projects model history from it,
@@ -15,36 +21,114 @@ module Terret
   # exactly what replaying the log yields ("model-visible means logged").
   class Sessions < Hames::Service
     service_key :sessions
+    inject :session_store
+    config_schema({}) # the session log takes no config
 
     Session = Struct.new(:id, :events, :parent_id, keyword_init: true)
 
+    # Payload keys whose values the harness minted to point at something else:
+    # tool call ids and their approval foreign keys, the part tag decode_part
+    # dispatches on, lineage, verdicts, the live allow list.
+    # A pattern written for a credential — long hex, a UUID shape — matches
+    # these too, and rewriting one protects nothing (none of it is
+    # model-carried) while it can wreck the log for good: two tool calls that
+    # collapse to one id are a request providers reject, a mangled tag makes
+    # the session undecodable, and a rewritten pattern list silently changes
+    # what an agent may run. Add a key here only when the harness itself
+    # generates its value. A tool NAME is deliberately absent: the model
+    # chooses it, so it is content — and redacting one fails safe, because a
+    # name that stops resolving comes back as a not-found error rather than
+    # collapsing two calls onto one identifier.
+    STRUCTURAL_KEYS = %i[id call_id type verdict status agent parent_id
+                         from boundary upto_seq n patterns].freeze
+
+    # Keys whose value is a structural CONTAINER rather than a leaf: the
+    # exemption reaches through them to the identifiers inside, because an
+    # assistant message's encoded parts each carry their own tag and id.
+    STRUCTURAL_CONTAINERS = %i[parts].freeze
+
     def start(ctx)
       @ctx = ctx
-      @store = {}
-      @dir = config[:jsonl_dir] # optional persistence
+      @store = ctx[:session_store]
+      @cache = {}
+      @locks = {}
+      @locks_mutex = Mutex.new
+      @emit_mutex = Mutex.new
+      @emit_queue = []
+      @emitting = false
+      @scrubbers = []
+    end
+
+    # Rewrite every String of every durable payload on its way into the log
+    # (§13's log boundary; docs/exec.md §6). Registration is an effect, so the
+    # returned disposer unregisters and unloading the owning plugin reaps it.
+    #
+    # This is the boundary the scrubbing has to happen at rather than anywhere
+    # downstream: the stored event and every projection derived from it —
+    # derive_messages, and so both sides of the digest assert_log_invariant!
+    # compares — read the same already-scrubbed bytes, so "model-visible means
+    # logged" holds by construction. A read-time filter over the projection
+    # would leave the secret in the log itself and split the digest in two.
+    #
+    # Scrubbers fold in registration order: each is handed the previous one's
+    # output, so a later one can rewrite what an earlier one produced.
+    # What a live in-memory value would look like once stored, for a caller
+    # that has to compare one against a payload already in the log — the
+    # approvals gate matches a recorded verdict on the call's args, and
+    # scrubbing rewrites one side of that comparison. CONTENT scope, because
+    # such a value always sits nested under a content key (`args`), so its own
+    # keys get no structural exemption.
+    def stored_form(value) = normalize_payload(value, structural: false)
+
+    # Whether anything is registered. Callers that must reshape what they
+    # append to give a scrubber a fair look at it — the loop's chunk carry —
+    # ask this so they can stay exactly as they were when nobody is scrubbing.
+    def scrubbing? = !@scrubbers.empty?
+
+    # `ctx:` decides the registration's LIFETIME, not its reach: the scrubber
+    # list is the service's, so a scrubber always sees every append, but a
+    # caller passing its forked agent context ties ownership to that fork —
+    # the same bleed Registry#register closed, where a registration made by an
+    # agent outlived the agent that made it.
+    def register_scrubber(callable, ctx: @ctx)
+      ctx.effect do
+        @scrubbers << callable
+        # Identity, not ==: removing "the entry equal to this callable" would
+        # take a twin down with it if the same object were registered twice,
+        # and a scrubber that defines its own == could unregister a stranger.
+        lambda do
+          i = @scrubbers.rindex { |s| s.equal?(callable) }
+          @scrubbers.delete_at(i) if i
+        end
+      end
     end
 
     def create(id: SecureRandom.hex(6), parent_id: nil)
       s = Session.new(id:, events: [], parent_id:)
-      @store[id] = s
+      @cache[id] = s
       append(id, "session/created", { parent_id: })
       s
     end
 
-    def fetch(id) = @store.fetch(id)
+    def fetch(id) = @cache.fetch(id)
 
     def append(session_id, type, payload = {})
       decl = Hames.event(type)
       raise Hames::ContractError, "#{type} is not a durable event" unless decl.durable
 
       s = fetch(session_id)
-      ev = SessionEvent.new(
-        id: SecureRandom.hex(8), session_id:, seq: s.events.length,
-        at: Time.now.utc, type: type.to_s, payload: payload
-      )
-      s.events << ev
-      persist(ev)
-      @ctx.emit("session/event", ev)
+      normalized = normalize_payload(payload)
+      ev = lock_for(session_id).synchronize do
+        e = SessionEvent.new(
+          id: SecureRandom.hex(8), session_id:, seq: s.events.length,
+          at: Time.now.utc, type: type.to_s, payload: normalized
+        )
+        # durable first: if the store raises, nothing believes the event happened
+        @store.append(e)
+        s.events << e
+        e
+      end
+      fan_out(ev)
       ev
     end
 
@@ -52,12 +136,15 @@ module Terret
     def derive_messages(session_id, upto: nil)
       events = fetch(session_id).events
       events = events.take(upto) if upto
-      events.filter_map do |ev|
+      apply_compaction(events).filter_map do |ev|
         case ev.type
         when "user/message", "context/injected"
           LLM::Message.new(role: :user, parts: [LLM::Text.new(text: ev.payload[:text])])
+        when "session/compacted"
+          LLM::Message.new(role: :user, parts: [LLM::Text.new(text: ev.payload[:summary])])
         when "assistant/message"
-          LLM::Message.new(role: :assistant, parts: ev.payload[:parts])
+          LLM::Message.new(role: :assistant,
+                           parts: ev.payload[:parts].map { |p| LLM.decode_part(p) })
         when "tool/result"
           LLM::Message.new(role: :tool, parts: [
             LLM::ToolResult.new(id: ev.payload[:id], content: ev.payload[:content],
@@ -67,6 +154,30 @@ module Terret
       end
     end
 
+    # Lifetime spend, projected from the log: sums every step/end's usage.
+    # A step whose provider sent no usage still counts as a step; its costs
+    # count as zero rather than poisoning the sum.
+    def usage(session_id)
+      out = { prompt_tokens: 0, completion_tokens: 0, cost: 0.0, steps: 0 }
+      fetch(session_id).events.each do |ev|
+        next unless ev.type == "step/end"
+
+        out[:steps] += 1
+        u = ev.payload[:usage] or next
+        out[:prompt_tokens]     += u[:prompt_tokens]     || 0
+        out[:completion_tokens] += u[:completion_tokens] || 0
+        out[:cost]              += u[:cost]              || 0.0
+      end
+      out
+    end
+
+    # The latest session/titled's title, or nil. Metadata, not model history.
+    def title(session_id)
+      fetch(session_id).events.reverse_each
+                       .find { |e| e.type == "session/titled" }
+                       &.payload&.[](:title)
+    end
+
     # The enforcement point for "model-visible means logged": the loop calls
     # this with the message list it is about to send; a mismatch against the
     # log projection raises in dev/test.
@@ -78,29 +189,221 @@ module Terret
             "outbound request diverges from session log projection for #{session_id}"
     end
 
+    # Forks the in-memory working set; resume a store-only session first.
     def fork(source_id, boundary: nil, child_id: SecureRandom.hex(6))
       src = fetch(source_id)
       child = Session.new(id: child_id, events: [], parent_id: source_id)
-      @store[child_id] = child
+      @cache[child_id] = child
       events = boundary ? src.events.take(boundary) : src.events.dup
-      events.each { |ev| child.events << ev.with(session_id: child_id) }
+      events.each do |ev|
+        copy = ev.with(session_id: child_id)
+        @store.append(copy)
+        child.events << copy
+      end
       append(child_id, "session/forked", { from: source_id, boundary: boundary })
       child
     end
 
+    def read(session_id, from_seq: 0) = @store.read(session_id, from_seq: from_seq)
+
+    def session_ids = @store.session_ids
+
+    # Rebuild a session's working set from the durable store. Idempotent: a
+    # session already in memory is returned as-is (write-through keeps the
+    # store equal). New appends continue after the last recorded seq.
+    def resume(session_id)
+      return @cache[session_id] if @cache.key?(session_id)
+
+      events = @store.read(session_id)
+      raise KeyError, "unknown session #{session_id}" if events.empty?
+
+      @cache[session_id] = Session.new(id: session_id, events: events,
+                                       parent_id: parent_id_from(events))
+    end
+
     private
 
-    def digest(messages)
-      Digest::SHA256.hexdigest(messages.map(&:inspect).join("\x1e"))
+    # Seq assignment, the durable write, and the memory push are one critical
+    # section per session. The store write is a yield point — JSONL opens a
+    # file, and a fiber scheduler switches there — so without this two
+    # appenders (a connection frame and a turn, say) read the same
+    # events.length and both claim it. Mutex is fiber-aware under the
+    # scheduler, so this parks a fiber rather than stalling the reactor.
+    def lock_for(session_id)
+      @locks_mutex.synchronize { @locks[session_id] ||= Mutex.new }
     end
 
-    def persist(ev)
-      return unless @dir
+    # Fan-out runs OUTSIDE the append lock, and in seq order. A listener that
+    # appends while handling an event — the compactor and the titler both do,
+    # on turn/end — would otherwise deliver its nested event to every other
+    # subscriber before the event it reacted to, so a socket tail would see
+    # the log out of order. Queueing behind the delivery in flight fixes that,
+    # at a price worth naming: a listener's own append returns before that
+    # event fans out. Assumes one reactor; the flag is mutex-guarded so
+    # threaded appenders cannot lose a queued event between the two.
+    def fan_out(ev)
+      @emit_mutex.synchronize do
+        @emit_queue << ev
+        return if @emitting
 
-      File.open(File.join(@dir, "#{ev.session_id}.jsonl"), "a") do |f|
-        f.puts JSON.generate(type: ev.type, seq: ev.seq, at: ev.at.iso8601,
-                             payload: ev.payload.inspect)
+        @emitting = true
       end
+      drain_emits
+    end
+
+    def drain_emits
+      while (ev = next_emit)
+        @ctx.emit("session/event", ev)
+      end
+    rescue Exception
+      @emit_mutex.synchronize { @emitting = false }
+      raise
+    end
+
+    def next_emit
+      @emit_mutex.synchronize do
+        ev = @emit_queue.shift
+        @emitting = false unless ev
+        ev
+      end
+    end
+
+    # Compacted history is still model-visible, so it lives in the log and
+    # projects as a user message standing in for everything at or before its
+    # boundary. The latest compaction wins; superseded ones drop out.
+    def apply_compaction(events)
+      latest = nil
+      events.each { |ev| latest = ev if ev.type == "session/compacted" }
+      return events unless latest
+
+      survivors = events.select do |ev|
+        ev.seq > latest.payload[:upto_seq] && ev.type != "session/compacted"
+      end
+      [latest] + survivors
+    end
+
+    def parent_id_from(events)
+      forked = events.reverse.find { |e| e.type == "session/forked" }
+      forked ? forked.payload[:from] : events.first&.payload&.[](:parent_id)
+    end
+
+    # The primitives contract: durable payloads hold only strings, numbers,
+    # booleans, nil, arrays, and symbol-keyed hashes of the same. Symbols in
+    # value position become strings; string keys become symbols. Anything
+    # else raises — typed objects are encoded at the edges (LLM.encode_part).
+    #
+    # `scrub` and `structural` carry the redaction scope down the recursion:
+    # see STRUCTURAL_KEYS for what the exemption is and #child_scope for
+    # where it stops.
+    def normalize_payload(value, scrub: true, structural: true)
+      case value
+      when Integer, true, false, nil then value
+      when String
+        utf8 = begin
+          value.encoding == Encoding::UTF_8 ? value : value.encode(Encoding::UTF_8)
+        rescue EncodingError
+          raise NonPrimitivePayload,
+                "#{value.encoding.name} string does not convert to UTF-8; scrub it first"
+        end
+        unless utf8.valid_encoding?
+          raise NonPrimitivePayload, "invalid UTF-8 string is not storable; scrub it first"
+        end
+
+        # After validation, so a scrubber is always handed storable UTF-8 —
+        # and re-validated below, because it hands something back.
+        scrub && !@scrubbers.empty? ? scrub_string(utf8) : utf8
+      when Float
+        raise NonPrimitivePayload, "non-finite Float is not storable" unless value.finite?
+
+        value
+        # Through the String arm rather than straight to the store: a symbol
+        # in value position is content like any other, and `to_s` on an
+        # ASCII-only symbol hands back a US-ASCII string that no scrubber
+        # would have seen and no encoding check would have normalized.
+      when Symbol then normalize_payload(value.to_s, scrub:, structural:)
+      when Array then value.map { |v| normalize_payload(v, scrub:, structural:) }
+      when Hash
+        value.each_with_object({}) do |(k, v), out|
+          key = case k
+                when Symbol then k
+                when String then k.to_sym
+                else raise NonPrimitivePayload, "#{k.class} is not a storable hash key"
+                end
+          # child_scope reads the un-scrubbed key: the structural exemption is
+          # decided by the real field NAME, before that name is ever rewritten.
+          child_scrub, child_structural = child_scope(key, scrub, structural)
+          # A KEY is content too once we have descended past the structural
+          # surface. An MCP tool's structured_content is a Hash whose keys the
+          # tool authored, and a secret-shaped one reached the log verbatim
+          # because only values recursed. Field names AT a structural level are
+          # the log's own vocabulary the projection reads by name, so they are
+          # never rewritten (`scrub_key` returns them untouched); the value
+          # exemption STRUCTURAL_KEYS carries is unaffected.
+          stored = scrub_key(key, scrub, structural)
+          raise NonPrimitivePayload, "duplicate key #{stored.inspect} after coercion" if out.key?(stored)
+
+          out[stored] = normalize_payload(v, scrub: child_scrub, structural: child_structural)
+        end
+      else
+        raise NonPrimitivePayload, "#{value.class} is not storable; encode it first"
+      end
+    end
+
+    # Where a key sits decides whether its value is exempt, because the same
+    # NAME means different things at different depths: `parts[0][:name]` is
+    # the tool a model called, while `args[:name]` is a terminal name the
+    # model itself wrote and `args[:content]` is a whole file. So the
+    # exemption holds only while nothing content-bearing has been entered:
+    # descending into anything outside these two lists turns it off for good,
+    # and everything below that point is scrubbed whatever it is called.
+    def child_scope(key, scrub, structural)
+      return [false, false] unless scrub # already inside an exempt subtree
+      return [false, false] if structural && STRUCTURAL_KEYS.include?(key)
+      return [true, true]   if structural && STRUCTURAL_CONTAINERS.include?(key)
+
+      [true, false]
+    end
+
+    # A key scrubs exactly where its sibling leaves do: only in content scope
+    # (past the structural surface) and only when something is registered.
+    # Routed through the String arm rather than scrub_string directly, so the
+    # key is UTF-8-encoded and validated before a scrubber sees it — the same
+    # contract every value String already meets. With no scrubber registered
+    # the key is handed back untouched, so a session that never scrubs stores
+    # byte-identical keys to before.
+    def scrub_key(key, scrub, structural)
+      return key unless scrub && !structural && !@scrubbers.empty?
+
+      normalize_payload(key.to_s, scrub: true, structural: false).to_sym
+    end
+
+    # Fold a payload string through the registered scrubbers, checking each
+    # answer. A scrubber runs after the primitives contract has already
+    # admitted the value, so a bad answer would land in the store unexamined:
+    # a nil where text was, or bytes the store's JSON generator rejects a
+    # frame later, with nothing left to say which plugin did it. Model-reachable
+    # data must never crash the harness, but a broken PLUGIN may and should —
+    # so this raises, naming the offender (a Proc's inspect carries its
+    # source location) rather than repairing what it cannot guess at.
+    def scrub_string(text)
+      @scrubbers.reduce(text) do |acc, scrubber|
+        out = scrubber.call(acc)
+        unless out.is_a?(String)
+          raise ScrubberContractViolation,
+                "scrubber #{scrubber.inspect} returned #{out.class}, not a String"
+        end
+        unless out.encoding == Encoding::UTF_8 && out.valid_encoding?
+          raise ScrubberContractViolation,
+                "scrubber #{scrubber.inspect} returned #{out.encoding.name} that is not valid " \
+                "UTF-8; a scrubber is handed UTF-8 and must hand UTF-8 back"
+        end
+
+        out
+      end
+    end
+
+    def digest(messages)
+      Digest::SHA256.hexdigest(messages.map(&:inspect).join("\x1e"))
     end
   end
 end

data/lib/terret/store.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "time"
+
+module Terret
+  # ctx[:session_store] — the persistence seam behind ctx.sessions. Providers
+  # durably record SessionEvents (whose payloads are primitives by the append
+  # contract) and hand them back exactly. Swapping the provider is a config
+  # row edit; Sessions never knows which one it is talking to. start is
+  # idempotent on every provider so an instance can ride across reboots.
+  module Store
+    # In-memory provider: the test default. Events are immutable, so sharing
+    # objects with Sessions' working set costs nothing.
+    class Memory < Hames::Service
+      service_key :session_store
+      config_schema({}) # the test default takes no config
+
+      def start(_ctx)
+        @events ||= Hash.new { |h, k| h[k] = [] }
+      end
+
+      def append(event) = @events[event.session_id] << event
+
+      def read(session_id, from_seq: 0)
+        @events.fetch(session_id, []).select { |ev| ev.seq >= from_seq }
+      end
+
+      # Order is provider-defined; recency must be derived from event timestamps.
+      def session_ids = @events.keys
+    end
+
+    # JSONL provider: one file per session, one JSON envelope per line, made
+    # for grepping. at carries microseconds so Time round-trips exactly.
+    class JSONL < Hames::Service
+      service_key :session_store
+      config_schema dir: { type: String, required: true,
+                           doc: "directory holding one JSONL file per session" }
+
+      def start(_ctx)
+        @dir ||= config.fetch(:dir).tap { |d| FileUtils.mkdir_p(d) }
+      end
+
+      def append(event)
+        File.open(path(event.session_id), "a") do |f|
+          f.puts JSON.generate(
+            id: event.id, session_id: event.session_id, seq: event.seq,
+            at: event.at.iso8601(6), type: event.type, payload: event.payload
+          )
+        end
+      end
+
+      def read(session_id, from_seq: 0)
+        return [] unless File.exist?(path(session_id))
+
+        File.foreach(path(session_id)).filter_map do |line|
+          h = begin
+            JSON.parse(line, symbolize_names: true)
+          rescue JSON::ParserError
+            next # a torn line (crash mid-write) loses itself, not the session
+          end
+          next if h[:seq] < from_seq
+
+          SessionEvent.new(id: h[:id], session_id: h[:session_id], seq: h[:seq],
+                           at: Time.iso8601(h[:at]), type: h[:type], payload: h[:payload])
+        end
+      end
+
+      def session_ids
+        Dir[File.join(@dir, "*.jsonl")].map { |f| File.basename(f, ".jsonl") }
+      end
+
+      private
+
+      def path(session_id) = File.join(@dir, "#{session_id}.jsonl")
+    end
+  end
+end