samagotchi 0.141.0 → 0.142.0

data/lib/samagotchi/session_commands.rb

@@ -23,6 +23,7 @@ require_relative "working_set_command"
 require_relative "recap_command"
 require_relative "guardrails/parent_policy"
 require_relative "system_one/log"
+require_relative "system_one/approval_label"
 require_relative "system_one/record"
 
 module Samagotchi
@@ -480,6 +481,8 @@ module Samagotchi
         return reply("could not label System 1 auto-allow #{index} (see the guardrails log)", status: :error)
       end
 
+      SystemOne::ApprovalLabel.save(SystemOne::Runs.for_sessions_dir(@engine.session_state_dir), entry["id"], SystemOne::ApprovalLabel::WRONG)
+
       call = shorten_call(entry["call"].to_s.gsub(/\s+/, " "))
       probability = entry["p"].nil? ? "unknown" : format("%.2f", entry["p"].to_f)
       reply("labelled auto-allow #{index}: #{entry["rule"]}: #{call} (p=#{probability}); " \

data/lib/samagotchi/session_manager.rb

@@ -6,6 +6,7 @@ require "time"
 require "securerandom"
 require "rbconfig"
 require_relative "chi_home"
+require_relative "system_one/runs"
 require_relative "child_env"
 require_relative "client_id"
 require_relative "analytics_file"
@@ -1043,7 +1044,9 @@ module Samagotchi
     # @raise [OwnedByTUI] a chi REPL owns it (never stopped from here)
     # @raise [DeleteRefused] a worker owns it and +stop+ is false, or it
     #   outlived the wait
-    def self.delete_session(id_or_prefix, state_dir: nil, stop: false, wait: STOP_WAIT)
+    # @param purge_runs [Boolean] also delete System 1's saved runs of it; a
+    #   session a worker left empty has none worth the read
+    def self.delete_session(id_or_prefix, state_dir: nil, stop: false, wait: STOP_WAIT, purge_runs: true)
       sd = state_dir || Session.default_state_dir
       given = id_or_prefix.to_s
       # Only a plain id: anything else could name a path outside the state dir.
@@ -1072,9 +1075,17 @@ module Samagotchi
       PluginSessionState.remove(id)
       removed.concat(ContextSources.remove_session(id, state_dir: sd))
       Participants.end_sessions([id], by: "delete", state_dir: sd)
+      purge_session_runs(id, sd) if purge_runs
       { id: id, removed: removed, stopped: !owner.nil? }
     end
 
+    # Never a reason a delete fails: one warn, and the sweep ages them out.
+    private_class_method def self.purge_session_runs(id, state_dir)
+      SystemOne::Runs.purge_session(id, sessions_dir: state_dir)
+    rescue StandardError => e
+      Log.warn(:worker, "session_runs_purge_failed", sid: id, error: e.class.name, msg: e.message.to_s[0, 200])
+    end
+
     # @return [Boolean] whether the session had no owner within +timeout+ seconds
     def self.wait_for_owner_release(session_id, timeout:, state_dir:)
       deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout.to_f
@@ -1216,7 +1227,7 @@ module Samagotchi
     private_class_method def self.discard_left_session(session_id, state_dir:, default_model:)
       return unless discardable?(session_id, state_dir: state_dir, default_model: default_model)
 
-      delete_session(session_id, state_dir: state_dir)
+      delete_session(session_id, state_dir: state_dir, purge_runs: false)
       Log.info(:worker, "discarded_empty", sid: session_id)
     rescue DeleteRefused, OwnedByTUI, ArgumentError, SystemCallError => e
       Log.info(:worker, "kept_session", sid: session_id, error: e.class.name, msg: e.message)

data/lib/samagotchi/session_retention.rb

@@ -10,6 +10,7 @@ require_relative "model_profile"
 require_relative "session_manager"
 require_relative "plugin_session_state"
 require_relative "context_sources"
+require_relative "system_one/runs"
 
 module Samagotchi
   # Which sessions go: `chi sessions prune` / `clean` (.prune) and the lazy
@@ -109,6 +110,7 @@ module Samagotchi
         return if age < interval
       end
       result = prune(state_dir: sd)
+      sweep_runs(sd)
       FileUtils.touch(marker)
       if result[:deleted].any?
         Log.info(:worker, "retention_pruned", echo: "[retention] pruned #{result[:deleted].size} sessions (kept #{result[:kept].size})", deleted: result[:deleted].size, kept: result[:kept].size)
@@ -119,6 +121,14 @@ module Samagotchi
       nil
     end
 
+    # System 1's saved runs last as long as sessions, by day file. Never a
+    # reason the session sweep fails.
+    private_class_method def self.sweep_runs(state_dir)
+      SystemOne::Runs.sweep_at(File.dirname(state_dir))
+    rescue StandardError => e
+      Log.warn(:worker, "retention_runs_failed", echo: "[retention] saved runs: #{e.class}: #{e.message}", error: e.class.name)
+    end
+
     # The caller's value (a sessions prune flag) or session.retention_days.
     private_class_method def self.resolve_days(val)
       return val.to_i if !val.nil? && val.to_s.strip != ""

data/lib/samagotchi/side_judgments.rb

@@ -16,6 +16,7 @@ module Samagotchi
   class SideJudgments
     def initialize
       @threads = {}
+      @asides = []
       @mutex = Mutex.new
     end
 
@@ -42,9 +43,19 @@ module Samagotchi
       end
     end
 
+    # Run the block on its own thread, in no kind: telling a caller there
+    # was no ask. #join waits for it too. A raise goes to +warn+.
+    # @return [Thread]
+    def aside(warn:, &block)
+      @mutex.synchronize do
+        @asides.select!(&:alive?)
+        Thread.new { run(warn, nil, &block) }.tap { |thread| @asides << thread }
+      end
+    end
+
     # Wait for every running ask, up to +seconds+ in all.
     def join(seconds)
-      threads = @mutex.synchronize { @threads.values }
+      threads = @mutex.synchronize { @threads.values + @asides }
       deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds
       threads.each do |thread|
         next if thread == Thread.current

data/lib/samagotchi/system_one/answer.rb

@@ -6,16 +6,20 @@ module Samagotchi
   module SystemOne
     # One judgment the System One API answered: for an :noul question,
     # `noul` is the probability of "yes"; a :choice answer carries `choice`
-    # (the criterion chosen) and `confidence`; a :score answer carries
-    # `score` (the level's index) and `confidence`. `probabilities` is what
-    # the model put on each option, when it says.
-    Answer = Data.define(:id, :type, :noul, :choice, :score, :confidence, :probabilities) do
-      def initialize(id:, type:, noul: nil, choice: nil, score: nil, confidence: nil, probabilities: nil) = super
+    # (the criterion chosen) and `confidence`; a :score answer carries `score`
+    # and `level`. The server's `score` is the probability-weighted level index
+    # (Σ index × its probability), not a level number: with levels 0..n-1 it
+    # sits between 0 and n-1, and for a two-level question it is exactly the
+    # probability of level 1. `level` is the name of the level with the most
+    # probability, from the response's `legend`. `probabilities` is what the
+    # model put on each option, when it says (for a score, keyed by level index).
+    Answer = Data.define(:id, :type, :noul, :choice, :score, :level, :confidence, :probabilities) do
+      def initialize(id:, type:, noul: nil, choice: nil, score: nil, level: nil, confidence: nil, probabilities: nil) = super
 
       # The response's answers entry as the wire sends it, or nil when it is
-      # not usable: no type, a type the contract doesn't have, a noul
-      # outside 0..1, a score that isn't a whole number, … A nil means "no
-      # judgment for this question", never a guess.
+      # not usable: no type, a type the contract doesn't have, a noul outside
+      # 0..1, a score that isn't a number or past the last level, … A nil means
+      # "no judgment for this question", never a guess.
       def self.from_wire(id, raw)
         return nil unless raw.is_a?(Hash)
 
@@ -37,9 +41,13 @@ module Samagotchi
           choice = raw["choice"].to_s.strip
           choice.empty? ? nil : new(**answer, choice: choice)
         else
-          score = raw["score"]
-          score = score.to_i if score.is_a?(Float) && score.to_i == score
-          score.is_a?(Integer) && score >= 0 ? new(**answer, score: score) : nil
+          # The legend names the levels; without it there is no name for the
+          # answer, but the weighted score still stands.
+          legend = levels(raw["legend"])
+          return nil if bad_field(raw, "legend", legend)
+
+          score = level_score(raw["score"], legend, odds)
+          score.nil? ? nil : new(**answer, score: score, level: top_level(legend, odds))
         end
       end
 
@@ -73,6 +81,50 @@ module Samagotchi
           weights
         end
       end
+
+      # The legend as the server writes it: a map of level index => name, or
+      # the level names in order. nil when absent or unreadable.
+      private_class_method def self.levels(value)
+        case value
+        when Hash
+          pairs = value.map { |index, name| [index.to_s.strip, name.to_s.strip] }
+          return nil if pairs.empty? || pairs.any? { |index, name| !index.match?(/\A\d+\z/) || name.empty? }
+
+          pairs.to_h { |index, name| [index.to_i, name] }
+        when Array
+          names = value.map { |name| name.to_s.strip }
+          return nil if names.empty? || names.any?(&:empty?)
+
+          names.each_with_index.to_h { |name, index| [index, name] }
+        end
+      end
+
+      # The score as sent: the weighted level index, a whole number kept whole.
+      # Not a number, or past the last level the response names, is no answer.
+      private_class_method def self.level_score(value, legend, odds)
+        return nil unless value.is_a?(Numeric)
+
+        score = value.to_f
+        return nil if score.nan? || score.infinite? || score.negative?
+
+        count = legend&.size || (odds.is_a?(Array) ? odds.size : odds&.size)
+        # A hair past the last level is the probabilities' own rounding, not a
+        # level out of range.
+        return nil if count && score > count - 1 + 1e-9
+
+        score == score.to_i ? score.to_i : score
+      end
+
+      # The level with the most probability, named by the legend; nil when
+      # there are no probabilities, or no legend to name it with.
+      private_class_method def self.top_level(legend, odds)
+        return nil if legend.nil? || legend.empty? || odds.nil? || odds.empty?
+
+        weights = legend.keys.map { |index| odds.is_a?(Array) ? odds[index] : odds[index.to_s] }
+        return nil if weights.any?(&:nil?)
+
+        legend[legend.keys[weights.index(weights.max)]]
+      end
     end
   end
 end

data/lib/samagotchi/system_one/approval_label.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative "../question_answer"
+require_relative "approval_question"
+require_relative "runs"
+
+module Samagotchi
+  module SystemOne
+    # What an answer to an approval says about the ask, for the judgments log
+    # (its value) and the saved run (all of it). The scope they took, a
+    # denial, or none for an ask that was cancelled, which is not an answer.
+    # A parent agent's answer is not the user's: its allow labels nothing, and
+    # its deny is a weak label only where the parent could have allowed — a
+    # parent that was not allowed to permit says nothing about what the user
+    # would have done (docs/guardrails.md, "System 1").
+    module ApprovalLabel
+      Label = Data.define(:value, :by, :strength)
+
+      # `/guardrails wrong`, the web's "should have asked".
+      WRONG = Label.new(value: "should_have_asked", by: "user", strength: "strong")
+      NONE = Label.new(value: "none", by: "none", strength: nil)
+
+      module_function
+
+      # @param answer [QuestionAnswer, Object] anything else: the ask was cancelled
+      # @param scopes [Array<String>] the scopes the ask offered
+      # @param parent_may_permit [#call] whether the session's parent may
+      #   allow this ask (read only for a parent's deny)
+      # @return [Label, nil] nil: this answer is no label
+      def of(answer, scopes:, parent_may_permit:)
+        return NONE unless answer.is_a?(Samagotchi::QuestionAnswer)
+
+        index = answer.selected_indices.first
+        allowed = !index.nil? && index < scopes.size
+        if answer.parent_agent?
+          return nil if allowed || !parent_may_permit.call
+
+          return Label.new(value: "deny", by: "parent", strength: "weak")
+        end
+
+        value = if allowed
+                  scopes[index] == "once" ? "allow_once" : "allow_wide"
+                else
+                  "deny"
+                end
+        Label.new(value: value, by: "user", strength: "strong")
+      end
+
+      # On the saved run +id+ names; an id from before runs has none.
+      # @param runs [Runs, nil]
+      def save(runs, id, label)
+        runs&.label(id, label.value, by: label.by, strength: label.strength, question: ApprovalQuestion::ID)
+      end
+    end
+  end
+end

data/lib/samagotchi/system_one/approval_question.rb

@@ -15,6 +15,8 @@ module Samagotchi
     # the memory an ask names and the earlier asks it resembles.
     module ApprovalQuestion
       ID = "allow"
+      # Its saved asks' name (Runs).
+      JUDGMENT = "approvals.allow"
       # What the judgment is asked, in the model's terms. Frozen: a change
       # here is a change in behaviour, not a wording tweak.
       INSTRUCTIONS = "A coding agent wants to run this tool call, and a guardrail rule asked its user to approve it. " \

data/lib/samagotchi/system_one/client.rb

@@ -17,7 +17,7 @@ module Samagotchi
     # while the judge is asked is not a failed judgment: its
     # LLM::RequestCancelled goes on to the caller, which asks no one.
     #
-    #   client = SystemOne::Client.from_config(host_registry: registry)
+    #   client = SystemOne::Client.from_config(host_registry: registry, mode: "audit")
     #   client&.ask(state: "git push origin main", questions: {"allow" => question})
     class Client
       MODES = %w[off audit on].freeze
@@ -33,19 +33,26 @@ module Samagotchi
         @tag = tag.to_sym
       end
 
-      # nil when judgment is off (the default) or no host is named, so the
-      # caller needs no other check. The settings are read every call: a
-      # config.yml change takes effect on the next ask.
+      # nil when +mode+ is off or no host is named, so the caller needs no
+      # other check. The settings are read every call: a config.yml change
+      # takes effect on the next ask.
+      # @param mode [String, Symbol] the caller's own: there is no default,
+      #   so nothing gets "on" by leaving it out, and no use reads another's
+      #   mode key (approvals pass guardrails.system_one.mode); host, model and
+      #   timeout are system_one.*, every use's
       # @param model [String, nil] a model for this call only (chi guardrails
-      #   replay --model), else the configured one
-      def self.from_config(host_registry:, env: ENV, model: nil, mode: nil, tag: :guardrails)
-        mode = (mode.nil? ? Config.get("guardrails.system_one.mode") : mode).to_s.strip.downcase
+      #   replay --model, a bundle's judgment's own), else the configured one
+      # @param host [String, nil] a host for this call only (a bundle's
+      #   judgment's own, never core's: JudgmentSettings), else the configured one
+      def self.from_config(host_registry:, mode:, env: ENV, model: nil, host: nil, tag: :guardrails)
+        mode = mode.to_s.strip.downcase
         return nil unless MODES.include?(mode) && mode != "off"
+        return nil if moved_keys_left?
 
-        host = Config.get("guardrails.system_one.host").to_s.strip
+        host = (host.nil? ? Config.get("system_one.host") : host).to_s.strip
         return nil if host.empty?
 
-        model = (model.nil? ? Config.get("guardrails.system_one.model") : model).to_s.strip
+        model = (model.nil? ? Config.get("system_one.model") : model).to_s.strip
         timeout_ms = configured_timeout_ms
 
         new(backend: HttpBackend.new(host_registry: host_registry, host: host, model: model.empty? ? nil : model,
@@ -53,39 +60,57 @@ module Samagotchi
             mode: mode, host: host, model: model.empty? ? nil : model, tag: tag)
       end
 
-      # guardrails.system_one.timeout_ms, else the default: the whole
+      # system_one.timeout_ms, else the default: the whole
       # request's limit, and how long a turn waits for an answer (toolkit
       # suggestion's on mode).
       # @return [Integer]
       def self.configured_timeout_ms
-        timeout_ms = Config.get("guardrails.system_one.timeout_ms").to_i
+        timeout_ms = Config.get("system_one.timeout_ms").to_i
         timeout_ms.positive? ? timeout_ms : DEFAULT_TIMEOUT_MS
       end
 
+      # Whether config.yml still sets a key that moved to system_one (the
+      # warning names it). A file half moved is no judgment: with the host
+      # moved and the model not, the host's first model would answer under
+      # approvals' clean record (Record is model-blind) and could auto-allow.
+      def self.moved_keys_left?
+        section = ConfigFile.section("guardrails")
+        own = section.is_a?(Hash) ? section["system_one"] || section["system-one"] : nil
+        own.is_a?(Hash) && own.keys.map { |key| key.to_s.tr("-", "_") }.intersect?(Config::SYSTEM_ONE_MOVED_KEYS)
+      end
+      private_class_method :moved_keys_left?
+
       # @param state [String, Hash, Array] what the question is about, as the
       #   contract takes it
       # @param questions [Hash{String => Question|Hash}] the questions, by id
+      # @param observer [#call, nil] told (**facts) what the return value
+      #   hides: why it is nil (failure:, failure_text:) and what the backend
+      #   said (model:, served_model:, raw:, usage:), on this thread, before
+      #   #ask returns (Recording)
+      # @param run_id [String, nil] Recording's: a bare client saves no run,
+      #   and a caller holding a bare client asks the same way
       # @return [Hash{String => Answer}, nil] one Answer per question, or nil
       #   when there is no judgment at all
-      def ask(state:, questions:, cancel_controller: nil)
+      def ask(state:, questions:, cancel_controller: nil, observer: nil, run_id: nil)
         questions = SystemOne.normalize_questions(questions)
         return nil if questions.empty?
 
         answers = begin
-          @backend.ask(state: state, questions: questions, cancel_controller: cancel_controller)
+          @backend.ask(state: state, questions: questions, cancel_controller: cancel_controller,
+                       **(observer ? { observer: observer } : {}))
         rescue LLM::RequestCancelled
           raise
         rescue StandardError => e
-          return no_judgment("#{e.class}: #{e.message}", error: e.class.name)
+          return no_judgment("#{e.class}: #{e.message}", failure: e.class.name, observer: observer, error: e.class.name)
         end
 
         unless answers.is_a?(Hash)
-          return no_judgment("the backend returned #{answers.inspect}")
+          return no_judgment("the backend returned #{answers.inspect}", failure: "bad_response", observer: observer)
         end
 
         answers = questions.to_h { |question| [question.id, answers[question.id]] }
         missing = questions.reject { |question| answers[question.id].is_a?(Answer) }
-        return no_judgment("no answer for #{missing.map(&:id).join(", ")}") unless missing.empty?
+        return no_judgment("no answer for #{missing.map(&:id).join(", ")}", failure: "no_answer", observer: observer) unless missing.empty?
 
         answers
       end
@@ -93,7 +118,8 @@ module Samagotchi
       private
 
       # One warn, one nil: the ask goes to the user.
-      def no_judgment(reason, **fields)
+      def no_judgment(reason, failure:, observer:, **fields)
+        observer&.call(failure: failure, failure_text: reason)
         # Qualified: inside SystemOne, the bare Log is now SystemOne::Log.
         Samagotchi::Log.warn(@tag, "system_one_failed", echo: "[samagotchi:#{@tag}] system one: #{reason}", host: @host, **fields)
         nil

data/lib/samagotchi/system_one/fake_backend.rb

@@ -26,7 +26,7 @@ module Samagotchi
         @calls = 0
       end
 
-      def ask(state:, questions:, cancel_controller: nil)
+      def ask(state:, questions:, cancel_controller: nil, observer: nil)
         @calls += 1
         raise BackendError, @error if @failing
 

data/lib/samagotchi/system_one/http_backend.rb

@@ -36,13 +36,17 @@ module Samagotchi
         @env = env
       end
 
+      # @param observer [#call, nil] told (**facts) the model it resolved,
+      #   before the request, then what the response said beside the parsed
+      #   answers: served_model:, raw: (each answer as sent, score legends
+      #   and all) and usage: (Splash's input_tokens, output_tokens)
       # @return [Hash{String => Answer}] one Answer per question, nil for a
       #   question the response answered badly
       # @raise [UnsupportedHost] the host is unknown, not an openai host, or
       #   names no model
       # @raise [BackendError] the request failed, the status isn't 200, or
       #   the body isn't a usable response
-      def ask(state:, questions:, cancel_controller: nil)
+      def ask(state:, questions:, cancel_controller: nil, observer: nil)
         entry = @host_registry.find_entry(@host)
         raise UnsupportedHost, "host #{@host.inspect} is not in hosts:" unless entry
         unless entry.api&.to_sym == :openai
@@ -52,6 +56,7 @@ module Samagotchi
         model = @model || host_model(entry)
         raise UnsupportedHost, "host #{@host} names no model for System One" if model.nil? || model.empty?
 
+        observer&.call(model: model)
         url = "#{entry.openai_base_url.to_s.chomp("/")}/systemone"
         questions = SystemOne.normalize_questions(questions)
         body = { model: model, state: state, questions: questions.to_h { |question| [question.id, question.to_wire] } }
@@ -64,7 +69,9 @@ module Samagotchi
           raise BackendError, "#{@host}: #{status} #{error_detail(response.body)}"
         end
 
-        answers = answers_of(response.body)
+        data = reply_of(response.body)
+        answers = data["answers"]
+        observer&.call(served_model: data["model"], raw: answers, usage: data["usage"])
         questions.to_h { |question| [question.id, Answer.from_wire(question.id, answers[question.id])] }
       end
 
@@ -99,14 +106,15 @@ module Samagotchi
         end
       end
 
-      def answers_of(body)
+      # The response's JSON, its answers map checked.
+      def reply_of(body)
         data = JSON.parse(body.to_s)
         raise BackendError, "the response is not JSON: #{body.to_s[0, 200]}" unless data.is_a?(Hash)
 
         answers = data["answers"]
         raise BackendError, "the response has no answers map" unless answers.is_a?(Hash)
 
-        answers
+        data
       rescue JSON::ParserError => e
         raise BackendError, "the response is not JSON: #{e.message[0, 200]}"
       end

data/lib/samagotchi/system_one/judgment_settings.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "../config"
+
+module Samagotchi
+  module SystemOne
+    # A bundle's judgment's settings: its entry in config.yml's
+    # system_one.judgments, read at each ask, as Client's are. Off until the
+    # user sets a mode. +host+ and +model+ are nil for the shared
+    # system_one.host and .model.
+    #
+    # Core's judgments have none, whatever the file says (Config::
+    # RESERVED_JUDGMENTS): Record and Similar are model-blind, so a model
+    # swapped for approvals alone would inherit a clean record and could
+    # auto-allow.
+    JudgmentSettings = Data.define(:mode, :host, :model) do
+      def initialize(mode: "off", host: nil, model: nil) = super
+
+      def off? = mode == "off"
+
+      # The model for Client.from_config: its own; else, on a host of its own,
+      # that host's first ("": the shared model belongs to the shared host);
+      # else nil, the shared one.
+      def client_model = model || (host ? "" : nil)
+
+      # @param name [String] the judgment's full name
+      # @param section [Hash, nil] config.yml's system_one section
+      # @return [JudgmentSettings]
+      def self.of(name, section: ConfigFile.section("system_one"))
+        return new if Config::RESERVED_JUDGMENTS.include?(name.to_s.split(".").first)
+
+        entry = entries(section)[name.to_s]
+        return new unless entry.is_a?(Hash)
+
+        new(mode: Config.judgment_mode(entry["mode"]) || "off", host: text(entry["host"]), model: text(entry["model"]))
+      rescue StandardError
+        new
+      end
+
+      # The names system_one.judgments sets.
+      # @return [Array<String>]
+      def self.names(section: ConfigFile.section("system_one"))
+        entries(section).keys
+      rescue StandardError
+        []
+      end
+
+      def self.entries(section)
+        judgments = section["judgments"] if section.is_a?(Hash)
+        judgments.is_a?(Hash) ? judgments.transform_keys(&:to_s) : {}
+      end
+      private_class_method :entries
+
+      def self.text(value)
+        word = value.to_s.strip
+        word.empty? ? nil : word
+      end
+      private_class_method :text
+    end
+  end
+end