samagotchi 0.141.0 → 0.142.0

data/lib/samagotchi/system_one/judgments.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+require_relative "../config"
+require_relative "../log"
+require_relative "../llm/errors"
+require_relative "../side_judgments"
+require_relative "judgment_settings"
+require_relative "question"
+require_relative "recording"
+require_relative "runs"
+
+module Samagotchi
+  module SystemOne
+    # Bundles' judgments (chi.judgment, ctx.judge, ctx.judge_later): what the
+    # user set for each, then the same Client and Recording core's uses ask
+    # through. It adds no power: a bundle's run is scoped to its name, and off
+    # until the user sets a mode. Core's uses stay outside (their threads,
+    # gates and contention are their own) and can't be set here (D2,
+    # JudgmentSettings).
+    class Judgments
+      # What chi.judgment declared. +questions+ is an Array of Question, or a
+      # callable given the ctx, returning what normalize_questions takes.
+      Declaration = Data.define(:name, :questions, :positive, :negative, :shared_host) do
+        # @raise [ArgumentError] anything the declaration can't be
+        def self.build(bundle:, name:, questions:, positive: nil, negative: nil, shared_host: :skip)
+          full = "#{bundle}.#{name}"
+          # The first word is the bundle: "a.b" + "x" and "a" + "b.x" must not meet.
+          raise ArgumentError, "judgment #{full.inspect}: a bundle with a dot in its name can't declare one" if bundle.to_s.include?(".")
+
+          unless Runs.declarable?(full)
+            raise ArgumentError, "judgment #{full.inspect}: a full name is dot-separated words of a-z, 0-9, _ and -, " \
+                                 "with no first word of #{Runs::RESERVED.join(", ")} and no #{Runs::RESERVED_SUFFIX} at its end"
+          end
+          unless SHARED_HOST.include?(shared_host)
+            raise ArgumentError, "judgment #{full}: shared_host: is #{SHARED_HOST.map(&:inspect).join(" or ")}, not #{shared_host.inspect}"
+          end
+
+          new(name: full, questions: questions.respond_to?(:call) ? questions : checked(full, questions),
+              positive: Array(positive).map(&:to_s).freeze, negative: Array(negative).map(&:to_s).freeze, shared_host: shared_host)
+        end
+
+        # The questions of an ask now.
+        # @return [Array<Question>]
+        def questions_for(context)
+          questions.respond_to?(:call) ? self.class.checked(name, questions.call(context)) : questions
+        end
+
+        def self.checked(full, questions)
+          list = SystemOne.normalize_questions(questions)
+          raise ArgumentError, "judgment #{full} has no questions" if list.empty?
+
+          list.freeze
+        end
+
+        # What each of its runs says of the label values, if anything.
+        def run_fields = { positive: positive, negative: negative }.reject { |_key, values| values.empty? }
+      end
+
+      SHARED_HOST = %i[ask skip].freeze
+
+      # What an ask came to. +reason+ says why there's no answer: :off,
+      # :no_host, :no_session, :failed, :cancelled, :shared_host or :busy; nil
+      # with answers. +mode+ is the judgment's ("off", "audit", "on"): in audit
+      # the bundle looks and doesn't act. +run_id+ is nil when no run is saved.
+      Result = Data.define(:answers, :run_id, :mode, :reason) do
+        def self.none(reason, mode: "off", run_id: nil) = new(answers: {}.freeze, run_id: run_id, mode: mode, reason: reason)
+
+        def no_judgment? = answers.empty?
+        def audit? = mode == "audit"
+
+        # An :noul question's probability of yes; nil with no answer, or for
+        # another type (its answer is #answers[id]).
+        # @param id [String, Symbol, nil] nil: the only question's
+        # @return [Float, nil]
+        def p(id = nil)
+          answer = id.nil? ? (answers.values.first if answers.size == 1) : answers[id.to_s]
+          answer&.noul
+        end
+      end
+
+      # One ask, made ready on the caller's thread.
+      Ask = Data.define(:client, :questions, :state, :run_id, :mode, :shared_host)
+
+      # The seams are what the Engine knows, read at each ask. They default to
+      # nothing, for a chi with no Engine (a bundle's install check): nothing
+      # is ever answered there.
+      # @param settings [#call] (name) → JudgmentSettings
+      # @param client_for [#call] (name, settings) → a Client, nil with no host
+      # @param runs [#call] → the Runs to save in
+      # @param where [#call] → Recording::Context, nil with no session
+      # @param side [SideJudgments]
+      # @param shared_host_busy [#call] (host) → whether a turn is running on it
+      # @param participant [#call] → whether a participant's turn runs: its
+      #   text is never saved
+      def initialize(settings: JudgmentSettings.method(:of), client_for: ->(_name, _settings) {}, runs: -> { Runs.new }, where: -> {},
+                     side: SideJudgments.new, shared_host_busy: ->(_host) { false }, participant: -> { false })
+        @settings = settings
+        @client_for = client_for
+        @runs = runs
+        @where = where
+        @side = side
+        @shared_host_busy = shared_host_busy
+        @participant = participant
+        @declarations = {}
+        @mutex = Mutex.new
+      end
+
+      # @param declaration [Declaration]
+      def declare(declaration)
+        @mutex.synchronize { @declarations[declaration.name] = declaration }
+      end
+
+      def declared?(name) = @mutex.synchronize { @declarations.key?(name.to_s) }
+
+      # Ask now, on this thread.
+      # @param context [Object] what a questions builder gets (the ctx)
+      # @return [Result]
+      # @raise [ArgumentError] +name+ isn't declared, or +state+ can't be one
+      def judge(name, state:, context: nil, cancel_controller: nil)
+        ask = prepare(name, state, context)
+        return ask if ask.is_a?(Result)
+
+        shared_host?(ask) ? skipped(ask, :shared_host) : asked(ask, cancel_controller)
+      end
+
+      # Ask on +name+'s side thread, one at a time; the block gets the Result
+      # on another thread, always once.
+      # @return [String, nil] the run's id, nil when no run will be saved
+      # @raise [ArgumentError] as #judge
+      def later(name, state:, context: nil, &block)
+        ask = prepare(name, state, context)
+        return tell(name, block, ask).run_id if ask.is_a?(Result)
+        return tell(name, block, skipped(ask, :shared_host)).run_id if shared_host?(ask)
+
+        started = @side.start(name, warn: warner(name, "judge_later_failed"),
+                                    finish: ->(result) { deliver(name, block, result || Result.none(:failed, mode: ask.mode, run_id: ask.run_id)) }) do
+          asked(ask, nil)
+        end
+        tell(name, block, skipped(ask, :busy)) unless started
+        ask.run_id
+      end
+
+      # A later line on a run (Runs#label, #acted); false when it isn't saved.
+      # Nothing is saved in a participant's turn, as an ask there saves no run.
+      def label(run_id, value, by:, strength: nil, question: nil)
+        return false if @participant.call
+
+        @runs.call.label(run_id, value, by: by, strength: strength, question: question)
+      end
+
+      def acted(run_id, what)
+        return false if @participant.call
+
+        @runs.call.acted(run_id, what)
+      end
+
+      # The system_one.judgments names no plugin declared, once the plugins
+      # have loaded: a typo there is a judgment that stays off. Core's names
+      # have their own warning (Config.validate_yaml_sections).
+      # @return [Array<String>]
+      def undeclared
+        known = @mutex.synchronize { @declarations.keys }
+        (JudgmentSettings.names - known).reject { |name| Config::RESERVED_JUDGMENTS.include?(name.split(".").first) }
+      end
+
+      private
+
+      # The Ask, or the Result that says why there is none.
+      def prepare(name, state, context)
+        declaration = @mutex.synchronize { @declarations[name.to_s] }
+        raise ArgumentError, "no judgment #{name} declared (chi.judgment)" unless declaration
+        unless state.is_a?(String) || state.is_a?(Hash) || state.is_a?(Array)
+          raise ArgumentError, "judgment #{name}: state: is a String, a Hash or an Array, not #{state.class}"
+        end
+
+        settings = @settings.call(declaration.name)
+        return Result.none(:off) if settings.off?
+
+        client = client_for(declaration, settings)
+        return Result.none(:no_host, mode: settings.mode) if client.nil?
+
+        where = where_now(declaration)
+        return Result.none(:no_session, mode: settings.mode) if where.nil?
+
+        ready(declaration, settings, client, where, state, context)
+      end
+
+      def where_now(declaration)
+        @where.call
+      rescue StandardError => e
+        warner(declaration.name, "judgment_session_unread").call(e)
+        nil
+      end
+
+      def ready(declaration, settings, client, where, state, context)
+        runs = @participant.call ? Runs.new : @runs.call
+        Ask.new(client: Recording.new(client, judgment: declaration.name, runs: runs, context: where, fields: declaration.run_fields),
+                questions: declaration.questions_for(context), state: copy_of(declaration, state),
+                run_id: runs.on? ? Runs.new_id(declaration.name) : nil, mode: settings.mode, shared_host: declaration.shared_host)
+      end
+
+      # A state nested too deep to copy (a cycle) is a bad state: a
+      # SystemStackError would go past a hook's rescue and end the turn.
+      def copy_of(declaration, state)
+        Runs.scrubbed(state)
+      rescue SystemStackError
+        raise ArgumentError, "judgment #{declaration.name}: state: is nested too deep"
+      end
+
+      def client_for(declaration, settings)
+        @client_for.call(declaration.name, settings)
+      rescue StandardError => e
+        warner(declaration.name, "judgment_client_failed").call(e)
+        nil
+      end
+
+      # Whether this ask would evict a running turn's cached prompt, and its
+      # bundle said to skip then. A check that fails counts as yes: no ask.
+      def shared_host?(ask)
+        ask.shared_host == :skip && @shared_host_busy.call(ask.client.host)
+      rescue StandardError => e
+        Samagotchi::Log.warn(:plugins, "judgment_shared_host_check_failed", error: e.class.name)
+        true
+      end
+
+      def skipped(ask, reason)
+        ask.client.skip(state: ask.state, questions: ask.questions, reason: reason.to_s, run_id: ask.run_id)
+        Result.none(reason, mode: ask.mode, run_id: ask.run_id)
+      end
+
+      def asked(ask, cancel_controller)
+        answers = ask.client.ask(state: ask.state, questions: ask.questions, cancel_controller: cancel_controller, run_id: ask.run_id)
+        return Result.none(:failed, mode: ask.mode, run_id: ask.run_id) if answers.nil?
+
+        Result.new(answers: answers.freeze, run_id: ask.run_id, mode: ask.mode, reason: nil)
+      rescue LLM::RequestCancelled
+        Result.none(:cancelled, mode: ask.mode, run_id: ask.run_id)
+      end
+
+      # The block gets +result+ on another thread, as an asked one does.
+      # @return [Result] +result+
+      def tell(name, block, result)
+        @side.aside(warn: warner(name, "judge_later_failed")) { deliver(name, block, result) } if block
+        result
+      end
+
+      def deliver(name, block, result)
+        block&.call(result)
+      rescue StandardError => e
+        warner(name, "judge_later_block_failed").call(e)
+      end
+
+      def warner(name, event)
+        lambda { |error|
+          Samagotchi::Log.warn(:plugins, event, judgment: name, error: "#{error.class}: #{error.message}"[0, 300])
+        }
+      end
+    end
+  end
+end

data/lib/samagotchi/system_one/log.rb

@@ -48,10 +48,12 @@ module Samagotchi
         append(line)
       end
 
-      # @param fields [Hash] the judgment's own fields (rule, tool, p, …)
+      # @param fields [Hash] the judgment's own fields (rule, tool, p, …); its
+      #   :id if the caller has one (a saved run's), else a new one
       # @return [String, nil] the judgment's id, nil when nothing was written
       def judgment(fields)
-        id = SecureRandom.uuid
+        fields = fields.dup
+        id = fields.delete(:id) || SecureRandom.uuid
         entry("judgment", { "id" => id }.merge(fields)) ? id : nil
       end
 

data/lib/samagotchi/system_one/recording.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require_relative "../llm/errors"
+require_relative "../log"
+require_relative "question"
+require_relative "runs"
+
+module Samagotchi
+  module SystemOne
+    # A Client that saves each ask as a run (Runs): the same duck type, the
+    # same answers and the same raises. It sits where a use builds its
+    # client, so gates, threads and domain logs stay the use's own. A run
+    # that can't be saved is one warn and never changes the answer.
+    #
+    #   client = Recording.wrap(Client.from_config(...), judgment: "approvals.allow", runs: runs, context: context)
+    #   client&.ask(state:, questions:, cancel_controller:)
+    class Recording
+      # What a run says about its session, taken on the caller's thread:
+      # the ask itself may run on its own. +chain_link+ is the parent or the
+      # session this one continues (a chain is derived, SessionChain, and
+      # following it costs a listing), +session_model+ the session's model.
+      Context = Data.define(:session_id, :chain_link, :session_model, :test_run) do
+        def initialize(session_id: nil, chain_link: nil, session_model: nil, test_run: nil) = super
+
+        def to_h = super.transform_keys(&:to_s)
+
+        def self.of(session, session_model: nil)
+          new(session_id: session&.id, chain_link: session&.parent_id || session&.continues,
+              session_model: session_model, test_run: session&.test_run ? true : nil)
+        end
+      end
+
+      # The wrapped client, or nil for none: the caller's nil check stays.
+      # @param recorded_mode [String, nil] the use's mode as a run says it,
+      #   where the client's isn't it (toolkit and context build theirs "on"
+      #   in audit too); nil: the client's
+      def self.wrap(client, judgment:, runs:, context: Context.new, recorded_mode: nil)
+        client && new(client, judgment: judgment, runs: runs, context: context, recorded_mode: recorded_mode)
+      end
+
+      # @param fields [Hash] more for each of its runs (a bundle's judgment's
+      #   positive: and negative: label values)
+      def initialize(client, judgment:, runs:, context: Context.new, recorded_mode: nil, fields: {})
+        @client = client
+        @judgment = judgment
+        @runs = runs
+        @context = context
+        @recorded_mode = recorded_mode
+        @fields = fields
+      end
+
+      def host = @client.host
+      def model = @client.model
+      def mode = @client.mode
+      def backend = @client.backend
+
+      # Client#ask, recorded. A cancelled ask or one that raises records its
+      # run, then raises as before.
+      # @param run_id [String, nil] the run's id when the caller needs it at
+      #   once (it is passed in, not read back: one client serves several
+      #   threads); nil: a new one
+      def ask(state:, questions:, cancel_controller: nil, run_id: nil)
+        facts = {}
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        answers = @client.ask(state: state, questions: questions, cancel_controller: cancel_controller,
+                              observer: ->(**told) { facts.merge!(told) })
+        record(run_id, state, questions, answers, facts, started)
+        answers
+      rescue LLM::RequestCancelled
+        record(run_id, state, questions, nil, facts.merge(failure: "cancelled"), started)
+        raise
+      rescue StandardError => e
+        record(run_id, state, questions, nil, facts.merge(failure: e.class.name, failure_text: e.message), started)
+        raise
+      end
+
+      # An ask that wasn't made: its run, at once, asked: false, so a later
+      # replay can answer the state with no eviction today (docs/system-one.md).
+      # @param reason [String] why: the run's +skipped+
+      def skip(state:, questions:, reason:, run_id: nil)
+        save(questions) do
+          { id: run_id || Runs.new_id(@judgment), state: state, asked: false, skipped: reason, model: said(:model) }
+        end
+      end
+
+      private
+
+      def record(run_id, state, questions, answers, facts, started)
+        save(questions) do
+          { id: run_id || Runs.new_id(@judgment), state: state, asked: true, model: facts[:model] || said(:model),
+            served_model: facts[:served_model], answers: facts[:raw] || answers&.transform_values(&:to_h), usage: facts[:usage],
+            failure: facts[:failure], failure_text: facts[:failure_text]&.to_s&.slice(0, 500),
+            latency_ms: ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round }
+        end
+      end
+
+      # One run: the block's fields, and what every run says.
+      def save(questions)
+        return unless @runs.on?
+
+        questions = SystemOne.normalize_questions(questions)
+        return if questions.empty? # Client#ask asked nothing.
+
+        @runs.record(judgment: @judgment, questions: questions, **@context.to_h.transform_keys(&:to_sym), **@fields,
+                     origin: "live", mode: @recorded_mode || said(:mode), host: said(:host), **yield)
+      rescue StandardError, SystemStackError => e # a state nested too deep to copy
+        # Qualified: inside SystemOne, the bare Log is SystemOne::Log.
+        Samagotchi::Log.warn(:system_one, "system_one_run_failed",
+                             echo: "[samagotchi:system_one] a run wasn't saved: #{e.class}: #{e.message}",
+                             judgment: @judgment, error: e.class.name)
+      end
+
+      # What the client says about itself; a stand-in may say nothing.
+      def said(name) = @client.respond_to?(name) ? @client.public_send(name) : nil
+    end
+  end
+end

data/lib/samagotchi/system_one/runs.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "securerandom"
+require "time"
+
+require_relative "../config"
+require_relative "../log"
+require_relative "question"
+require_relative "runs_store"
+
+module Samagotchi
+  module SystemOne
+    # The saved asks: one run per System 1 ask (Recording), what was sent and
+    # answered, and later lines on it, for a later comparison or fine-tune
+    # (docs/system-one.md). A Runs without a store saves nothing. Local only.
+    #
+    # A run's id is "<judgment>/<uuid>": it carries its judgment, so a later
+    # line can find its way with the id alone. Later lines never hold up the
+    # caller: a failed one is one warn and a false.
+    class Runs
+      # What the specs read: lines in memory, by judgment.
+      class MemoryStore
+        attr_reader :lines, :versions
+
+        def initialize
+          @lines = Hash.new { |hash, judgment| hash[judgment] = [] }
+          @versions = Hash.new { |hash, judgment| hash[judgment] = [] }
+          @mutex = Mutex.new
+        end
+
+        def append(judgment, line)
+          @mutex.synchronize { @lines[judgment] << line }
+          true
+        end
+
+        def version(judgment, line)
+          @mutex.synchronize do
+            @versions[judgment] << line unless @versions[judgment].include?(line)
+          end
+          true
+        end
+      end
+
+      # Names core keeps (a first word, as a bundle's scope would be) and the
+      # suffix a later replay's runs use: a bundle's judgment takes none.
+      RESERVED = Config::RESERVED_JUDGMENTS
+      RESERVED_SUFFIX = ".replay"
+      # The total the sweep keeps (docs/system-one.md).
+      MAX_BYTES = 100 * 1024 * 1024
+
+      # Runs on the state dir +root+ (where the sessions folder sits), saving
+      # unless system_one.runs is off.
+      def self.in(root)
+        new(store: Config.get("system_one.runs").to_s == "off" ? nil : RunsStore.new(root))
+      end
+
+      # .in, from the sessions folder.
+      def self.for_sessions_dir(sessions_dir) = self.in(File.dirname(sessions_dir))
+
+      # The session sweep: runs past session.retention_days (0: forever), then
+      # the oldest over the total cap. Whatever system_one.runs says now.
+      # @return [Integer] the day files deleted
+      def self.sweep_at(root, days: Config.get("session.retention_days").to_i, max_bytes: MAX_BYTES)
+        RunsStore.new(root).sweep(days: days, max_bytes: max_bytes)
+      end
+
+      # Explicit `chi sessions delete`: that session's runs and their later
+      # lines go, whatever system_one.runs says now.
+      def self.purge_session(session_id, sessions_dir:)
+        RunsStore.new(File.dirname(sessions_dir)).purge_session(session_id)
+      end
+
+      # Whether a bundle may declare +name+ (safe as a folder, not core's).
+      def self.declarable?(name)
+        text = name.to_s
+        text.match?(RunsStore::NAME) && !RESERVED.include?(text.split(".").first) && !text.end_with?(RESERVED_SUFFIX)
+      end
+
+      # A new run's id.
+      def self.new_id(judgment) = "#{judgment}/#{SecureRandom.uuid}"
+
+      # The judgment a run id carries; nil for anything else (a judgments
+      # log's id from before runs, a name that isn't safe as a folder).
+      def self.judgment_of(run_id)
+        judgment, uuid = run_id.to_s.split("/", 2)
+        judgment if judgment.to_s.match?(RunsStore::NAME) && !uuid.to_s.empty?
+      end
+
+      # The digests a run groups by: the questions' wording and, apart, their
+      # criteria, which can vary per session (a toolkit's kits).
+      # @param questions [Array<Question>]
+      # @return [Hash{String => String}]
+      def self.digests(questions)
+        wording = questions.map { |question| [question.id, question.type.to_s, question.instructions] }
+        criteria = questions.map { |question| [question.id, question.criteria] }
+        { "instructions_digest" => digest(wording), "criteria_digest" => digest(criteria) }
+      end
+
+      def self.digest(value) = Digest::SHA256.hexdigest(JSON.generate(value))[0, 16]
+      private_class_method :digest
+
+      # A copy +value+ can be written as JSON from: bytes that aren't UTF-8
+      # (a command's output in a state) are replaced, in strings and keys.
+      def self.scrubbed(value)
+        case value
+        when String then value.dup.force_encoding(Encoding::UTF_8).scrub("�")
+        when Symbol then scrubbed(value.to_s)
+        when Hash then value.to_h { |key, item| [scrubbed(key.to_s), scrubbed(item)] }
+        when Array then value.map { |item| scrubbed(item) }
+        else value
+        end
+      end
+
+      # @param store [#append, #version, nil] nil: nothing is saved
+      def initialize(store: nil)
+        @store = store
+      end
+
+      def on? = !@store.nil?
+
+      # Save one ask, its questions once per digest pair. Raises what its
+      # store raises: Recording holds the one warn.
+      # @param id [String] the run's id
+      # @param judgment [String] its judgment's name
+      # @param questions [Array<Question>] as asked
+      # @param fields [Hash] the run's other fields; nil ones are left out
+      # @return [Boolean] whether it was saved
+      def record(id:, judgment:, questions:, **fields)
+        return false unless on?
+
+        digests = self.class.digests(questions)
+        wire = questions.to_h { |question| [question.id, question.to_wire] }
+        @store.version(judgment, digests.merge("questions" => self.class.scrubbed(wire)))
+        @store.append(judgment, { "kind" => "run", "id" => id, "judgment" => judgment, "at" => now }
+                                  .merge(digests).merge(self.class.scrubbed(fields.compact)))
+      end
+
+      # What the user (or +by+) said about a run.
+      # @param value [String] the use's own word
+      # @param by [String] who said it
+      # @param strength [String, nil] "strong" or "weak", where the use has
+      #   the difference
+      # @param question [String, nil] the question it answers
+      # @return [Boolean] whether it was saved
+      def label(run_id, value, by:, strength: nil, question: nil)
+        later(run_id, "label", value: value, by: by, strength: strength, question: question)
+      end
+
+      # What the use did with the answer: never known when the run is written.
+      # @return [Boolean] whether it was saved
+      def acted(run_id, what) = later(run_id, "acted", what: what)
+
+      # What the gates said at ask time, which can't be recomputed later.
+      # @return [Boolean] whether it was saved
+      def gates(run_id, **fields) = later(run_id, "gates", **fields)
+
+      private
+
+      def later(run_id, kind, **fields)
+        return false unless on?
+
+        judgment = self.class.judgment_of(run_id)
+        return false if judgment.nil?
+
+        @store.append(judgment, { "kind" => kind, "run_id" => run_id.to_s, "at" => now }.merge(self.class.scrubbed(fields.compact)))
+      rescue StandardError, SystemStackError => e
+        # Qualified: inside SystemOne, the bare Log is SystemOne::Log.
+        Samagotchi::Log.warn(:system_one, "system_one_run_line_failed",
+                             echo: "[samagotchi:system_one] a #{kind} line wasn't saved: #{e.class}: #{e.message}",
+                             judgment: judgment, error: e.class.name)
+        false
+      end
+
+      def now = Time.now.utc.iso8601(3)
+    end
+  end
+end