samagotchi 0.35.0 → 0.37.0

data/lib/samagotchi/broadcast/triage.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+require_relative "../idle_client"
+require_relative "../idle_target"
+require_relative "../cancellation_controller"
+require_relative "../log"
+require_relative "tags"
+
+module Samagotchi
+  module Broadcast
+    # Whether a broadcast concerns a recipient its tags didn't match: a
+    # small model reads the note and the session's ScopeCard and answers
+    # yes or no (Triage::LLM). Many recipients are judged in parallel under
+    # one deadline (Triage.judge_all); whatever isn't judged by then, or
+    # fails, is delivered unchecked (fail open: a missed recipient is worse
+    # than an extra note).
+    module Triage
+      # One recipient's verdict.
+      # @!attribute relevant [Boolean] it gets the note
+      # @!attribute p [Float, nil] the model's P(yes) from its logprobs,
+      #   1.0 or 0.0 from a plain answer, nil when no model judged it
+      # @!attribute reason [String] why, for the output and the log
+      # @!attribute by [String] who decided: "tags" (a shared tag),
+      #   "model", "fallback" (delivered unchecked: deadline, error, an
+      #   answer that is no yes or no), "scope" (the note's first line
+      #   names another project), "all" (--all) or "repl" (a chi REPL owns
+      #   it: no notes)
+      # @!attribute match [Match, nil] the shared tag (by "tags")
+      Verdict = Data.define(:relevant, :p, :reason, :by, :match) do
+        def initialize(match: nil, **) = super
+
+        def unchecked? = by == "fallback"
+      end
+
+      # The spike's prompt (plain yes/no beat JSON with a p and a reason
+      # for every model tried: writing a reason pushes towards "relevant").
+      SYSTEM = <<~TEXT
+        You route short notes that a developer broadcasts to their running coding-agent sessions.
+        You get one note and one session's card (project, branch, title, what the session was started for, its latest recap, the last request).
+        Decide whether the note may affect THIS session's current work: would the agent working on it want to know?
+        A note can start with an informal scope line (a project, a component, a team); use it as a hint.
+        Notes about things unrelated to the session's work (social, other projects, other components) are not relevant.
+        Answer with exactly one word: yes or no.
+      TEXT
+
+      DEFAULT_PARALLEL = 4
+      DEFAULT_DEADLINE = 20.0
+      DEFAULT_THRESHOLD = 0.5
+      # How long threads still asking get to stop once the deadline passed.
+      JOIN_GRACE = 1.0
+
+      module_function
+
+      # @return [Verdict] delivered unchecked, +why+ in the reason
+      def unchecked(why) = Verdict.new(relevant: true, p: nil, reason: "unchecked: #{why}", by: "fallback")
+
+      # The messages a model judges +note+ for +card+ by.
+      def messages(note, card)
+        [{ role: "system", content: SYSTEM }, { role: "user", content: "NOTE:\n#{note}\n\nSESSION CARD:\n#{card}" }]
+      end
+
+      # The one project the note's first line names, as a whole word (any
+      # case), among +projects+ (the recipients' project names); nil when it
+      # names none or several. A scope line ("shopfront/checkout", "agentd
+      # web") then keeps the note to that project's sessions; anything
+      # fuzzier is the model's job.
+      # @param projects [Array<String>]
+      # @return [String, nil]
+      def scope_project(note, projects)
+        line = note.to_s.lines.first.to_s.downcase
+        named = projects.compact.uniq.select do |project|
+          line.match?(/(?<![[:alnum:]_-])#{Regexp.escape(project.downcase)}(?![[:alnum:]_-])/)
+        end
+        named.one? ? named.first : nil
+      end
+
+      # Every card's verdict: a shared tag delivers (Tags.match); a note
+      # whose first line names one of the projects (scope_project) skips
+      # the other projects' sessions; the model judges the rest
+      # (judge_all, +judge+ its keywords).
+      # @param cards [Array<ScopeCard>]
+      # @param note_tags [Array<Tag>]
+      # @return [Hash{String => Verdict}] by card id, in +cards+' order
+      def verdicts(note, cards, note_tags:, new_backend:, **judge)
+        scope = scope_project(note, cards.map(&:project))
+        decided = cards.to_h do |card|
+          match = Tags.match(note_tags, card.tags)
+          verdict = if match
+                      Verdict.new(relevant: true, p: nil, reason: match.reason, by: "tags", match: match)
+                    elsif scope && card.project != scope
+                      Verdict.new(relevant: false, p: nil, reason: "scope line names #{scope}", by: "scope")
+                    end
+          [card.id, verdict]
+        end
+        judged = judge_all(note, cards.reject { |card| decided[card.id] }, new_backend: new_backend, **judge)
+        cards.to_h { |card| [card.id, decided[card.id] || judged.fetch(card.id)] }
+      end
+
+      # Judges +cards+ with at most +parallel+ backends at a time, one per
+      # thread (a backend keeps per-host state), all under +deadline+
+      # seconds. A card not judged by then is delivered unchecked; the
+      # requests still running are cancelled.
+      # @param new_backend [#call] (CancellationController) → a backend:
+      #   #judge(note, card) → Verdict
+      # @param cards [Array<ScopeCard>]
+      # @return [Hash{String => Verdict}] by card id
+      def judge_all(note, cards, new_backend:, parallel: DEFAULT_PARALLEL, deadline: DEFAULT_DEADLINE,
+                    clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+        return {} if cards.empty?
+
+        queue = Queue.new
+        cards.each { |card| queue << card }
+        queue.close
+        results = {}
+        lock = Mutex.new
+        changed = ConditionVariable.new
+        cancel = CancellationController.new
+        ends = clock.call + deadline.to_f
+        finished = 0
+        threads = Array.new(parallel.to_i.clamp(1, cards.size)) do
+          Thread.new do
+            card = nil
+            backend = new_backend.call(cancel)
+            while (card = queue.pop)
+              verdict = judge_one(backend, note, card)
+              lock.synchronize do
+                results[card.id] = verdict unless cancel.cancelled?
+                changed.broadcast
+              end
+              card = nil
+            end
+          rescue StandardError => e
+            # No backend (or a bug around it): this thread's card and the
+            # ones it takes from here on are delivered unchecked at once,
+            # so nobody waits for the deadline.
+            Log.exception(:broadcast, "triage_thread_failed", e)
+            failed = unchecked("triage failed (#{e.class})")
+            lock.synchronize { results[card.id] ||= failed if card }
+            while (card = queue.pop)
+              lock.synchronize { results[card.id] ||= failed }
+            end
+          ensure
+            lock.synchronize do
+              finished += 1
+              changed.broadcast
+            end
+          end
+        end
+        lock.synchronize do
+          loop do
+            break if results.size == cards.size || finished == threads.size
+
+            left = ends - clock.call
+            break if left <= 0
+
+            changed.wait(lock, left)
+          end
+          cancel.cancel!(:deadline)
+          queue.clear
+        end
+        threads.each { |thread| thread.join(JOIN_GRACE) || thread.kill }
+        cards.to_h { |card| [card.id, results[card.id] || unchecked("triage deadline")] }
+      end
+
+      def judge_one(backend, note, card)
+        backend.judge(note, card)
+      rescue Samagotchi::LLM::RequestCancelled
+        unchecked("triage deadline")
+      rescue StandardError => e
+        Log.exception(:broadcast, "triage_failed", e)
+        unchecked("triage failed (#{e.class})")
+      end
+      private_class_method :judge_one
+
+      # A triage model on an OpenAI-compatible host (an IdleTarget), asked
+      # through its own IdleClient: thinking off, a few tokens, the first
+      # token's logprobs when the host gives them. P(yes) / (P(yes) +
+      # P(no)) from those is graded against +threshold+; a plain yes or no
+      # is 1.0 or 0.0.
+      class LLM
+        MAX_TOKENS = 16
+        TOP_LOGPROBS = 5
+        LOGPROB_OPTIONS = { logprobs: true, top_logprobs: TOP_LOGPROBS }.freeze
+        # Hosts (base urls) that refused logprobs with a 400: asked without
+        # them from then on, in this process.
+        @no_logprobs = Set.new
+        @no_logprobs_lock = Mutex.new
+
+        class << self
+          def logprobs?(base_url) = @no_logprobs_lock.synchronize { !@no_logprobs.include?(base_url) }
+
+          def no_logprobs!(base_url) = @no_logprobs_lock.synchronize { @no_logprobs << base_url }
+
+          # Specs: forget the hosts that refused.
+          def reset_logprobs! = @no_logprobs_lock.synchronize { @no_logprobs.clear }
+        end
+
+        # @param target [IdleTarget]
+        # @param timeout [Numeric] each request's
+        # @param client [IdleClient, nil] specs inject one
+        def initialize(target:, timeout:, threshold: DEFAULT_THRESHOLD, cancel_controller: nil, client: nil)
+          @target = target
+          @threshold = threshold.to_f
+          @cancel = cancel_controller
+          @client = client || IdleClient.for(target, timeout: timeout, purpose: "broadcast")
+        end
+
+        # @return [Verdict]
+        # @raise [Samagotchi::LLM::RequestCancelled] the deadline passed
+        def judge(note, card)
+          answer = ask(Triage.messages(note, card))
+          word = self.class.word(answer.text)
+          return Triage.unchecked("the model answered #{answer.text.to_s.strip[0, 40].inspect}") unless word
+
+          p = self.class.logprob_p(answer.top_logprobs, word: word)
+          relevant = p ? p >= @threshold : word == "yes"
+          reason = "model: #{relevant ? "yes" : "no"}#{format(" (p %.2f)", p) if p}"
+          Verdict.new(relevant: relevant, p: p || (word == "yes" ? 1.0 : 0.0), reason: reason, by: "model")
+        rescue IdleClient::SummarizeError => e
+          Triage.unchecked("the model failed (#{e.message[0, 120]})")
+        end
+
+        # "yes" or "no": the answer's first such word (thinking left out),
+        # nil when it has neither.
+        def self.word(text)
+          text.to_s.gsub(%r{<think>.*?</think>}m, "").strip.downcase[/\A\W*(yes|no)\b/, 1]
+        end
+
+        # P(yes) / (P(yes) + P(no)) over the first token's alternatives
+        # ("Yes", " yes" count as yes), when that token is the answer +word+
+        # itself: its likeliest alternative is yes or no and agrees with
+        # +word+. nil otherwise (an answer starting "**", "<think>" or a
+        # quote: those alternatives say nothing about yes or no).
+        # @param top [Array<Samagotchi::LLM::TokenLogprob>]
+        # @return [Float, nil]
+        def self.logprob_p(top, word:)
+          first = top.max_by(&:logprob)
+          return nil unless first && first.token.strip.downcase == word
+
+          sum = ->(want) { top.select { |t| t.token.strip.downcase == want }.sum { |t| Math.exp(t.logprob) } }
+          yes = sum.call("yes")
+          no = sum.call("no")
+          (yes / (yes + no)).round(4)
+        end
+
+        private
+
+        def ask(messages)
+          base_url = @target.base_url
+          options = self.class.logprobs?(base_url) ? LOGPROB_OPTIONS : {}
+          begin
+            @client.ask(messages, max_tokens: MAX_TOKENS, cancel_controller: @cancel, kind: "broadcast", options: options)
+          rescue IdleClient::SummarizeError => e
+            raise unless !options.empty? && e.cause.is_a?(Samagotchi::LLM::BadRequest)
+
+            # The host won't give logprobs (a server that refuses "n and
+            # logprobs"): ask again without, and leave them out from now on.
+            self.class.no_logprobs!(base_url)
+            Log.info(:broadcast, "logprobs_refused", model: @target.label, detail: e.cause.message.to_s[0, 200])
+            options = {}
+            retry
+          end
+        end
+      end
+    end
+  end
+end

data/lib/samagotchi/broadcast/triage_log.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "time"
+require_relative "../paths"
+
+module Samagotchi
+  module Broadcast
+    # What each broadcast decided, one JSON line per broadcast in
+    # <state>/samagotchi/broadcast/log.jsonl: its id, time, note text, the
+    # note's tags and every recipient's verdict (no scope cards: they quote
+    # the user's prompts). `chi broadcast deliver` adds a "corrected" line
+    # for the sessions it delivered to anyway: labels for the threshold and
+    # a future classifier. Not a list sessions read: broadcasts stay
+    # fire-and-forget. Over MAX_BYTES the file becomes log.jsonl.1 (one old
+    # file kept).
+    class TriageLog
+      MAX_BYTES = 2 * 1024 * 1024
+
+      # One recipient's line in a broadcast.
+      # @!attribute result [String] "delivered", "skipped" or "failed"
+      # @!attribute p [Float, nil] the triage model's P(yes), nil when no
+      #   model judged it
+      # @!attribute by [String] Triage::Verdict#by
+      # @!attribute tags [Array<String>] the session's tags (Tag#label)
+      Entry = Data.define(:session, :result, :p, :reason, :by, :tags)
+
+      # A `chi broadcast deliver`: when, and each session's result.
+      # @!attribute sessions [Array<Delivery>]
+      Correction = Data.define(:at, :sessions)
+
+      # One session a correction delivered to (or tried).
+      Delivery = Data.define(:session, :result)
+
+      # A broadcast as read back, with the corrections made to it since.
+      # @!attribute at [Time]
+      # @!attribute recipients [Array<Entry>]
+      # @!attribute corrections [Array<Correction>]
+      Record = Data.define(:id, :at, :text, :note_tags, :triage_model, :recipients, :corrections) do
+        # @return [Entry, nil]
+        def entry(session) = recipients.find { |e| e.session == session }
+
+        # Whether +session+ got it, from the broadcast or a correction.
+        def delivered?(session)
+          entry(session)&.result == "delivered" ||
+            corrections.any? { |c| c.sessions.any? { |d| d.session == session && d.result == "delivered" } }
+        end
+
+        def to_json_hash
+          { id: id, at: at.iso8601, text: text, note_tags: note_tags, triage_model: triage_model,
+            recipients: recipients.map(&:to_h),
+            corrections: corrections.map { |c| { at: c.at.iso8601, sessions: c.sessions.map(&:to_h) } } }
+        end
+      end
+
+      # No broadcast with that id, or several start with it.
+      class NotFound < StandardError; end
+
+      # @return [String] <state>/samagotchi/broadcast/log.jsonl
+      def self.default_path(env: ENV) = File.join(Paths.state_dir(env: env), "broadcast", "log.jsonl")
+
+      attr_reader :path
+
+      def initialize(path = self.class.default_path)
+        @path = path
+      end
+
+      # @param at [Time]
+      # @param recipients [Array<Entry>]
+      def append_broadcast(id:, at:, text:, note_tags:, triage_model:, recipients:)
+        append(type: "broadcast", id: id, at: at.iso8601, text: text, note_tags: note_tags, triage_model: triage_model,
+               recipients: recipients.map(&:to_h))
+      end
+
+      # @param sessions [Array<Delivery>]
+      def append_correction(id:, at:, sessions:)
+        append(type: "corrected", id: id, at: at.iso8601, sessions: sessions.map(&:to_h))
+      end
+
+      # Every broadcast in the log (the rotated file's first), oldest
+      # first, with its corrections. A line that doesn't parse is skipped.
+      # @return [Array<Record>]
+      def records
+        found = {}
+        [rotated_path, @path].each do |file|
+          next unless File.file?(file)
+
+          File.foreach(file) do |line|
+            data = parse(line)
+            next unless data
+
+            if data["type"] == "broadcast"
+              found[data["id"]] = record(data)
+            elsif data["type"] == "corrected" && (previous = found[data["id"]])
+              found[data["id"]] = previous.with(corrections: previous.corrections + [correction(data)])
+            end
+          end
+        end
+        found.values
+      end
+
+      # The broadcast +given+ names: its id, with or without "b-", or the
+      # start of one.
+      # @return [Record]
+      # @raise [NotFound]
+      def find(given)
+        want = given.to_s.strip.delete_prefix("b-")
+        raise NotFound, "no broadcast id given" if want.empty?
+
+        matches = records.select { |r| r.id.delete_prefix("b-").start_with?(want) }
+        return matches.first if matches.one?
+        raise NotFound, "no broadcast #{given} in the log (chi broadcast log)" if matches.empty?
+
+        raise NotFound, "#{given} names #{matches.size} broadcasts (#{matches.map(&:id).join(", ")}); give more of the id"
+      end
+
+      private
+
+      def rotated_path = "#{@path}.1"
+
+      def append(data)
+        FileUtils.mkdir_p(File.dirname(@path))
+        File.rename(@path, rotated_path) if File.file?(@path) && File.size(@path) > MAX_BYTES
+        File.open(@path, File::WRONLY | File::APPEND | File::CREAT, 0o600) do |file|
+          file.flock(File::LOCK_EX)
+          file.write("#{JSON.generate(data)}\n")
+        end
+      end
+
+      def parse(line)
+        data = JSON.parse(line)
+        data.is_a?(Hash) && data["id"].is_a?(String) ? data : nil
+      rescue JSON::ParserError
+        nil
+      end
+
+      def record(data)
+        Record.new(id: data["id"], at: time(data["at"]), text: data["text"].to_s, note_tags: Array(data["note_tags"]),
+                   triage_model: data["triage_model"],
+                   recipients: Array(data["recipients"]).filter_map { |e| entry(e) }, corrections: [])
+      end
+
+      def entry(data)
+        return nil unless data.is_a?(Hash)
+
+        Entry.new(session: data["session"].to_s, result: data["result"].to_s, p: data["p"], reason: data["reason"].to_s,
+                  by: data["by"].to_s, tags: Array(data["tags"]))
+      end
+
+      def correction(data)
+        sessions = Array(data["sessions"]).filter_map do |d|
+          Delivery.new(session: d["session"].to_s, result: d["result"].to_s) if d.is_a?(Hash)
+        end
+        Correction.new(at: time(data["at"]), sessions: sessions)
+      end
+
+      def time(value)
+        Time.iso8601(value.to_s)
+      rescue ArgumentError
+        Time.at(0)
+      end
+    end
+  end
+end

data/lib/samagotchi/broadcast/triage_model.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative "../config"
+require_relative "../idle_target"
+require_relative "../model_profile"
+
+module Samagotchi
+  # Loaded on first use, as the CLI commands do: host_registry pulls in the
+  # LLM layer.
+  autoload :HostRegistry, File.expand_path("../host_registry", __dir__)
+
+  module Broadcast
+    # Which model a broadcast's triage asks: broadcast.triage_model /
+    # triage_host_ref / triage_base_url; with none of them set, the recap's
+    # (recap.*); with none of those, default.model. Settings that can't be
+    # resolved are a problem, not a fallback to the next ones: triage then
+    # delivers unchecked and says why.
+    module TriageModel
+      # @!attribute target [IdleTarget, nil] nil when +problem+ says why
+      # @!attribute setting [String] where it came from: "broadcast.triage_model",
+      #   "recap.model" or "default.model" (the prefix of the keys read)
+      Choice = Data.define(:target, :setting, :problem)
+
+      SETTINGS = [%w[broadcast.triage_model broadcast.triage_host_ref broadcast.triage_base_url],
+                  %w[recap.model recap.host_ref recap.base_url]].freeze
+
+      module_function
+
+      # @param get [#call] a Config key → its value
+      # @return [Choice]
+      def resolve(host_registry: nil, get: ->(key) { Config.get(key) })
+        registry = host_registry || HostRegistry.new
+        SETTINGS.each do |model, host_ref, base_url|
+          target = IdleTarget.resolve(model: get.call(model), host_ref: get.call(host_ref), base_url: get.call(base_url),
+                                      host_registry: registry)
+          return Choice.new(target: target, setting: model, problem: nil) if target
+        rescue IdleTarget::Unresolved => e
+          return Choice.new(target: nil, setting: model, problem: problem(e, model, host_ref))
+        end
+        model = ModelProfile.required_model_name(get.call("default.model"))
+        Choice.new(target: IdleTarget.of_model(registry, model), setting: "default.model", problem: nil)
+      rescue StandardError => e
+        Choice.new(target: nil, setting: "default.model", problem: e.message.to_s.lines.first.to_s.strip[0, 200])
+      end
+
+      def problem(error, model_key, host_ref_key)
+        case error.reason
+        when :host_ref_unknown then "#{host_ref_key} '#{error.host_ref}' is not in hosts:"
+        when :model_host_mismatch then "#{model_key} '#{error.model}' names host '#{error.named}', not #{host_ref_key} '#{error.host_ref}'"
+        else "#{model_key} is missing (a host or base_url alone names no model)"
+        end
+      end
+      private_class_method :problem
+    end
+  end
+end