samagotchi 0.2.0 → 0.4.0

data/lib/samagotchi/session_metrics.rb

@@ -14,13 +14,12 @@ module Samagotchi
   # event to its SessionObserver): the REPL, the -p/--non-interactive/--resume
   # paths and SessionManager background workers.
   #
-  # Collected dimensions:
-  #   - tokens (input/output/total), with a provenance flag (:server|:estimate)
-  #   - turn count, tool calls (total / per-tool / error count)
-  #   - iterations (tool-call rounds) per turn, aggregated
-  #   - generation latency (monotonic clock across generation_* events)
-  #   - cancellations and network retries
-  #   - session wall-clock (first activity -> last activity)
+  # Each finished turn leaves a record (status, timings, model, prompt and
+  # completion tokens, tool calls, iterations, retries); session totals are
+  # sums over the records, so a worker that stops and wakes again (a new
+  # collector) keeps counting: #session_id= loads the records already saved.
+  # Only prompt and completion counts, which every backend reports the same
+  # way, are kept.
   #
   # A snapshot is surfaced via Engine#session_state_snapshot and (optionally)
   # persisted to a sibling analytics.json next to the session file. The event
@@ -43,6 +42,21 @@ module Samagotchi
       # Buffered chars/4 estimate for the current generation; committed to the
       # turn total only if the generation ends with no server token data.
       :gen_estimate_sum,
+      # This generation's prompt (the server's count; the largest seen, as it
+      # comes cumulative per request) and whether it is still streaming.
+      :gen_prompt_max,
+      :gen_open,
+      # The turn's generations, folded as each one ends: the count, the last
+      # one's prompt and completion, the sums, and which kinds of counts
+      # (server / estimate) they came from.
+      :generations,
+      :prompt_last,
+      :prompt_sum,
+      :completion_last,
+      :completion_sum,
+      :token_sources,
+      :retries,
+      :model,
       :id,
       :started_at,
       :started_monotonic,
@@ -50,23 +64,37 @@ module Samagotchi
       keyword_init: true
     )
 
+    # How full the context was after the session's last counted turn, from
+    # its saved analytics.json: a percentage, or nil when the file, the count
+    # or the window is missing (the session lists read it per row).
+    # @param session_dir [String]
+    # @return [Float, nil]
+    def self.saved_context_pct(session_dir)
+      context = JSON.parse(File.read(File.join(session_dir, "analytics.json")))["context"]
+      return nil unless context.is_a?(Hash)
+
+      used = context["used_tokens"]
+      window = context["window_tokens"]
+      return nil unless used.is_a?(Numeric) && window.is_a?(Numeric) && window.positive?
+
+      used * 100.0 / window
+    rescue JSON::ParserError, SystemCallError, TypeError
+      nil
+    end
+
+    # "ctx 12%" for the session lists, "" when unknown.
+    # @param pct [Float, nil]
+    # @return [String]
+    def self.context_label(pct)
+      pct ? "ctx #{pct.round}%" : ""
+    end
+
     def initialize(clock: nil, wall_clock: nil)
       @mutex = Monitor.new
       @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
       @wall_clock = wall_clock || -> { Time.now }
       @session_id = nil
-      @turns = 0
-      @tokens_in = 0
-      @tokens_out = 0
-      @tokens_total = 0
-      @token_source = nil # :server | :estimate
-      @tool_calls_total = 0
-      @tool_calls_by_tool = Hash.new(0)
-      @tool_errors = 0
-      @iterations_total = 0
-      @gen_latency_ms = 0
-      @cancellations = 0
-      @retries = 0
+      @state_dir = nil
       @started_at = nil
       @last_activity_at = nil
       @turn = nil
@@ -83,14 +111,25 @@ module Samagotchi
       end
     end
 
+    # The sessions dir analytics.json is read from and saved to (nil: the
+    # XDG default). Set it before #session_id=.
+    attr_writer :state_dir
+
     # Set the session id the collector is aggregating for. Safe to call multiple
-    # times; the first non-empty id wins so later turns don't clobber it.
+    # times; the first non-empty id wins so later turns don't clobber it. The
+    # first one loads the records a earlier process saved for it, once
+    # (#snapshot, on hot paths, never reads the disk).
     # @param id [String, nil]
     # @return [void]
     def session_id=(id)
       return if id.nil? || id.to_s.empty?
 
-      @mutex.synchronize { @session_id ||= id.to_s }
+      @mutex.synchronize do
+        next if @session_id
+
+        @session_id = id.to_s
+        load_persisted
+      end
     end
 
     # Event sink. Safe to call from any thread; errors are isolated by the
@@ -124,20 +163,25 @@ module Samagotchi
           end
           if @turn
             @turn.gen_started_at = monotonic_time
-            @turn.gen_completion_max = 0
-            @turn.gen_had_server = false
-            @turn.gen_estimate_sum = 0
+            @turn.generations += 1
+            reset_generation_tokens
+            @turn.gen_open = true
           end
         end
       when :generation_completed, :generation_cancelled
         record_served_model(event)
         record_generation_completed
       when :generation_retrying
-        @mutex.synchronize { @retries += 1 }
+        # The retry streams from the start: its counts replace the ones so far.
+        @mutex.synchronize do
+          if @turn
+            @turn.retries += 1
+            reset_generation_tokens
+          end
+        end
       when :turn_completed
         end_turn(status: "completed")
       when :turn_canceled
-        @mutex.synchronize { @cancellations += 1 }
         end_turn(status: "canceled", reason: event[:cancellation_reason])
       when :turn_failed
         end_turn(status: "failed")
@@ -148,32 +192,36 @@ module Samagotchi
       nil
     end
 
+    # The session totals are sums over the turn records (the ones loaded from
+    # disk and this process's), with the running turn's calls, iterations and
+    # finished generations counted at once, so /stats agrees mid-turn.
     # @return [Hash] the current summary snapshot
     def snapshot
       @mutex.synchronize do
+        records = @turn_records
+        turn = @turn
+        in_flight = turn ? turn.tool_calls_by_id.values : []
+        tools = @tool_records + in_flight
         {
           session_id: @session_id,
-          turns: @turns,
-          tokens_in: @tokens_in,
-          tokens_out: @tokens_out,
-          tokens_total: @tokens_total,
-          token_source: @token_source,
-          context_window_tokens: @context_window_tokens,
-          context_window_source: @context_window_source,
+          turns: records.size + (turn ? 1 : 0),
+          cancellations: records.count { |record| record[:status] == "canceled" },
+          tokens: {
+            prompt_sum: sum(records, :prompt_tokens_sum) + (turn&.prompt_sum || 0),
+            completion_sum: sum(records, :completion_tokens) + (turn&.completion_sum || 0),
+            source: combined_source(records.map { |record| record[:token_source] } + (turn&.token_sources || []))
+          },
+          context: context_block(records),
           profile: @profile,
           profile_source: @profile_source,
           served_model: @served_model,
           served_model_for: @served_model_for,
-          # The running turn's calls, iterations and finished generations
-          # count at once (the per-tool counts do too), so /stats agrees
-          # mid-turn; end_turn moves them into the totals.
-          tool_calls_total: @tool_calls_total + (@turn&.tool_calls || 0),
-          tool_calls_by_tool: @tool_calls_by_tool.dup,
-          tool_errors: @tool_errors + (@turn&.tool_errors || 0),
-          iterations_total: @iterations_total + (@turn&.iteration_count || 0),
-          gen_latency_ms: @gen_latency_ms + (@turn&.gen_latency_accum || 0),
-          cancellations: @cancellations,
-          retries: @retries,
+          tool_calls_total: tools.size,
+          tool_calls_by_tool: tools.map { |tool| tool[:tool].to_s }.reject(&:empty?).tally,
+          tool_errors: @tool_records.count { |tool| tool[:status] == "error" },
+          iterations_total: sum(records, :iterations) + (turn&.iteration_count || 0),
+          gen_latency_ms: (sum(records, :gen_ms) + (turn&.gen_latency_accum || 0)).round,
+          retries: sum(records, :retries) + (turn&.retries || 0),
           started_at: @started_at,
           last_activity_at: @last_activity_at,
           session_duration_ms: elapsed_ms(@session_started_monotonic),
@@ -197,11 +245,10 @@ module Samagotchi
       # Omitting state_dir lets Session.session_dir fall back to the default
       # (XDG) location; passing an explicit nil would override it and break
       # File.join.
-      dir = state_dir ? Session.session_dir(sid, state_dir: state_dir) : Session.session_dir(sid)
-      FileUtils.mkdir_p(dir)
+      FileUtils.mkdir_p(dir = session_dir(sid, state_dir || @state_dir))
       path = File.join(dir, "analytics.json")
       temp_path = "#{path}.tmp"
-      File.write(temp_path, JSON.pretty_generate(merged_persisted_snapshot(path)) + "\n")
+      File.write(temp_path, JSON.pretty_generate(snapshot.merge(active_turn: nil, active_tools: [])) + "\n")
       File.rename(temp_path, path)
       true
     rescue StandardError
@@ -212,10 +259,12 @@ module Samagotchi
 
     def begin_turn(event)
       @mutex.synchronize do
-        @session_id ||= event[:session_id].to_s if event[:session_id]
+        unless @session_id || event[:session_id].to_s.empty?
+          @session_id = event[:session_id].to_s
+          load_persisted
+        end
         @started_at ||= now.iso8601(3)
         @session_started_monotonic ||= monotonic_time
-        @turns += 1
         @turn = TurnState.new(
           session_id: @session_id,
           iteration_count: 0,
@@ -226,6 +275,16 @@ module Samagotchi
           gen_completion_max: 0,
           gen_had_server: false,
           gen_estimate_sum: 0,
+          gen_prompt_max: 0,
+          gen_open: false,
+          generations: 0,
+          prompt_last: nil,
+          prompt_sum: 0,
+          completion_last: 0,
+          completion_sum: 0,
+          token_sources: [],
+          retries: 0,
+          model: nil,
           id: SecureRandom.uuid,
           started_at: now.iso8601(3),
           started_monotonic: monotonic_time,
@@ -239,6 +298,9 @@ module Samagotchi
     def end_turn(status: "completed", reason: nil)
       @mutex.synchronize do
         if @turn
+          # A turn that failed mid-stream never saw its generation end:
+          # what it got to still counts.
+          close_generation if @turn.gen_open
           finished_at = now
           record = {
             id: @turn.id,
@@ -248,12 +310,9 @@ module Samagotchi
             duration_ms: elapsed_ms(@turn.started_monotonic)
           }
           record[:cancellation_reason] = reason.to_s unless reason.nil? || reason.to_s.empty?
+          record.merge!(turn_token_fields(@turn))
           @turn_records << record
         end
-        @iterations_total += @turn.iteration_count if @turn
-        @gen_latency_ms += @turn.gen_latency_accum if @turn
-        @tool_calls_total += @turn.tool_calls if @turn
-        @tool_errors += @turn.tool_errors if @turn
         @turn = nil
       end
     end
@@ -266,32 +325,69 @@ module Samagotchi
       @mutex.synchronize do
         @served_model = event[:served_model]
         @served_model_for = event[:requested_model]
+        @turn.model = event[:served_model] if @turn
       end
     end
 
     def record_generation_completed
-      # Finalize this generation's completion tokens. Server timings are
-      # cumulative per generation; we sum the per-generation max into the turn
-      # total. If the generation reported no server tokens we commit the buffered
-      # chars/4 estimate instead. The two paths are mutually exclusive per
-      # generation, so a final chunk carrying timings while earlier chunks did
-      # not will not double count.
-      @mutex.synchronize do
-        return unless @turn
+      @mutex.synchronize { close_generation if @turn&.gen_open }
+    end
 
-        if @turn.gen_had_server
-          @tokens_out += @turn.gen_completion_max
-        else
-          @tokens_out += @turn.gen_estimate_sum
-        end
-        @tokens_total = @tokens_in + @tokens_out
-        started = @turn.gen_started_at
-        if started
-          elapsed_ms = (monotonic_time - started) * 1000.0
-          @turn.gen_latency_accum += elapsed_ms if elapsed_ms > 0
-          @turn.gen_started_at = nil
-        end
+    # Finalize the open generation's tokens into the turn. Server counts are
+    # cumulative per generation, so its max is its count. If the generation
+    # reported no server tokens we commit the buffered chars/4 estimate
+    # instead (there is no prompt count then). The two paths are mutually
+    # exclusive per generation, so a final chunk carrying timings while
+    # earlier chunks did not will not double count. Caller holds the mutex.
+    def close_generation
+      turn = @turn
+      turn.gen_open = false
+      if turn.gen_had_server
+        completion = turn.gen_completion_max
+        turn.prompt_last = turn.gen_prompt_max
+        turn.prompt_sum += turn.gen_prompt_max
+        turn.token_sources |= ["server"]
+      else
+        completion = turn.gen_estimate_sum
+        turn.token_sources |= ["estimate"] if completion.positive?
       end
+      turn.completion_last = completion
+      turn.completion_sum += completion
+      started = turn.gen_started_at
+      return unless started
+
+      elapsed_ms = (monotonic_time - started) * 1000.0
+      turn.gen_latency_accum += elapsed_ms if elapsed_ms > 0
+      turn.gen_started_at = nil
+    end
+
+    # Caller holds the mutex.
+    def reset_generation_tokens
+      @turn.gen_completion_max = 0
+      @turn.gen_prompt_max = 0
+      @turn.gen_had_server = false
+      @turn.gen_estimate_sum = 0
+    end
+
+    # The turn record's model and token fields. The context used at the end
+    # is the last prompt plus the last answer (an estimated last generation
+    # has no prompt count, so the turn's last known prompt stands in).
+    def turn_token_fields(turn)
+      sources = turn.token_sources
+      {
+        model: turn.model,
+        generations: turn.generations,
+        prompt_tokens: turn.prompt_last,
+        prompt_tokens_sum: turn.prompt_sum,
+        completion_tokens: turn.completion_sum,
+        context_used_tokens: turn.prompt_last && (turn.prompt_last + turn.completion_last),
+        token_source: sources.size > 1 ? "mixed" : sources.first,
+        gen_ms: turn.gen_latency_accum.round,
+        tool_calls: turn.tool_calls,
+        tool_errors: turn.tool_errors,
+        iterations: turn.iteration_count,
+        retries: turn.retries
+      }
     end
 
     def record_tool_call_start(event)
@@ -300,7 +396,6 @@ module Samagotchi
 
         @turn.tool_calls += 1
         tool = event[:tool].to_s
-        @tool_calls_by_tool[tool] += 1 unless tool.empty?
         iteration = event[:iteration].to_i
         call_index = event[:call_index].to_i
         key = tool_key(iteration, call_index)
@@ -339,10 +434,9 @@ module Samagotchi
     end
 
     # Accumulate token counts from a streamed generation_chunk payload.
-    # Server-first: prefer real timings/usage. Input tokens keep a running MAX
-    # (the prompt grows across tool-call iterations); completion tokens are
-    # tracked as a per-generation MAX and summed at generation_completed so
-    # multi-generation turns count every generation. When a generation reports
+    # Server-first: prefer real timings/usage. Prompt and completion tokens
+    # are tracked as a per-generation MAX (the server's counts are cumulative
+    # per request) and folded into the turn when the generation ends. When a generation reports
     # no server tokens at all we fall back to the chars/4 estimate summed across
     # its chunks (mutually exclusive with server counting to avoid double
     # counting a final chunk that carries timings while earlier chunks do not).
@@ -353,13 +447,11 @@ module Samagotchi
       usage = TokenUsage.from_payload(payload)
       if usage
         @mutex.synchronize do
-          @token_source = :server
-          @tokens_in = [@tokens_in, usage[:prompt_tokens].to_i].max
           if @turn
+            @turn.gen_prompt_max = [@turn.gen_prompt_max, usage[:prompt_tokens].to_i].max
             @turn.gen_completion_max = [@turn.gen_completion_max, usage[:completion_tokens].to_i].max
             @turn.gen_had_server = true
           end
-          @tokens_total = @tokens_in + @tokens_out
         end
       else
         @mutex.synchronize do
@@ -367,9 +459,7 @@ module Samagotchi
           return unless chars && chars.positive?
           return if @turn && @turn.gen_had_server
 
-          @token_source ||= :estimate
           @turn.gen_estimate_sum += TokenUsage.estimate(payload_content(payload, event)) if @turn
-          @tokens_total = @tokens_in + @tokens_out
         end
       end
     end
@@ -425,30 +515,62 @@ module Samagotchi
       end
     end
 
-    def merged_persisted_snapshot(path)
-      current = snapshot.merge(active_turn: nil, active_tools: [])
-      return current unless File.file?(path)
+    def session_dir(sid, state_dir)
+      # Omitting state_dir lets Session.session_dir fall back to the default
+      # (XDG) location; passing an explicit nil would override it.
+      state_dir ? Session.session_dir(sid, state_dir: state_dir) : Session.session_dir(sid)
+    end
+
+    # The records an earlier process saved for this session (a missing or
+    # broken file, or an older shape: nothing, or zeros where keys are
+    # missing). Caller holds the mutex.
+    def load_persisted
+      path = File.join(session_dir(@session_id, @state_dir), "analytics.json")
+      return unless File.file?(path)
 
       prior = JSON.parse(File.read(path))
-      return current unless prior.is_a?(Hash)
+      return unless prior.is_a?(Hash)
+
+      @turn_records = loaded_records(prior["turn_records"]) + @turn_records
+      @tool_records = loaded_records(prior["tool_records"]) + @tool_records
+      @started_at = earliest_timestamp(prior["started_at"], @started_at)
+      @last_activity_at ||= prior["last_activity_at"]
+      # The window last seen, until this process's first generation reports.
+      window = prior["context"].is_a?(Hash) ? prior["context"] : {}
+      if window["window_tokens"] && @context_window_tokens.nil?
+        @context_window_tokens = window["window_tokens"]
+        @context_window_source = window["window_source"]
+      end
+    rescue JSON::ParserError, SystemCallError
+      nil
+    end
 
-      current.merge(
-        started_at: earliest_timestamp(prior["started_at"], current[:started_at]),
-        last_activity_at: latest_timestamp(prior["last_activity_at"], current[:last_activity_at]),
-        turn_records: merge_records(prior["turn_records"], current[:turn_records]),
-        tool_records: merge_records(prior["tool_records"], current[:tool_records])
-      )
-    rescue JSON::ParserError
-      current
+    def loaded_records(records)
+      Array(records).filter_map { |record| record.transform_keys(&:to_sym) if record.is_a?(Hash) && record["id"] }
+    end
+
+    # The context as the newest turn with a count left it, and the window
+    # now; readers compute the percentage.
+    def context_block(records)
+      last = records.reverse_each.find { |record| record[:context_used_tokens].is_a?(Numeric) }
+      {
+        used_tokens: last&.dig(:context_used_tokens),
+        window_tokens: @context_window_tokens,
+        window_source: @context_window_source&.to_s,
+        source: last&.dig(:token_source),
+        at: last&.dig(:completed_at)
+      }
     end
 
-    def merge_records(prior, current)
-      (Array(prior) + Array(current)).each_with_object({}) do |record, by_id|
-        next unless record.is_a?(Hash)
+    def sum(records, key)
+      records.sum { |record| record[key].is_a?(Numeric) ? record[key] : 0 }
+    end
 
-        id = record["id"] || record[:id]
-        by_id[id] = record if id
-      end.values
+    # server, estimate, or mixed when both kinds of counts (or a turn that
+    # was already mixed) are in; nil before any.
+    def combined_source(sources)
+      kinds = sources.compact.map(&:to_s).flat_map { |source| source == "mixed" ? %w[server estimate] : [source] }.uniq
+      kinds.size > 1 ? "mixed" : kinds.first
     end
 
     def earliest_timestamp(*timestamps)
@@ -456,11 +578,5 @@ module Samagotchi
     rescue ArgumentError
       timestamps.compact.first
     end
-
-    def latest_timestamp(*timestamps)
-      timestamps.compact.max_by { |value| Time.iso8601(value.to_s) }
-    rescue ArgumentError
-      timestamps.compact.last
-    end
   end
 end

data/lib/samagotchi/steer.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Samagotchi
+  # A plugin's steer: text a plugin puts into the running turn (Engine#steer),
+  # the way a UI's steering does. The drain a loop gets returns items: a
+  # String (a user line, as ever) or {text:, source:} (a steer). The user
+  # lines merge into one user message as before; each steer follows it as its
+  # own user message marked kind: "steer" and source: (the ContextNote keys,
+  # which every message copy keeps). The model reads it as user text.
+  module Steer
+    KIND = "steer"
+
+    # What one drain brought: the merged user text (nil when none) and how
+    # many lines made it, and the steers in order.
+    Merge = Struct.new(:content, :count, :steers, keyword_init: true) do
+      def empty? = content.nil? && steers.empty?
+
+      # The messages it appends: the user's first, then each steer.
+      def messages
+        list = content ? [{ role: "user", content: content }] : []
+        list + steers.map { |steer| Steer.message(**steer) }
+      end
+
+      # The :pending_input_merged fields beside iteration and answer;
+      # steers: only when there are some, so plain merges stay as they were.
+      def event_fields
+        fields = { count: count, content: content }
+        fields[:steers] = steers unless steers.empty?
+        fields
+      end
+    end
+
+    module_function
+
+    def message(text:, source:)
+      { role: "user", kind: KIND, source: source.to_s, content: text.to_s }
+    end
+
+    def steer?(message)
+      (message[:kind] || message["kind"]).to_s == KIND
+    end
+
+    # A user message that is a prompt or the user's steering, not a steer:
+    # what "the last prompt" and "user turns" count.
+    def prompt?(message)
+      message.is_a?(Hash) && (message[:role] || message["role"]).to_s == "user" && !steer?(message)
+    end
+
+    # Call a loop's drain: +at_answer+ goes only to a drain that takes it
+    # (the Engine's); a caller's own drain (a queue's #drain) is called bare.
+    # A failing drain drains nothing (the loop keeps going).
+    def drain(pending_input, at_answer:)
+      takes = pending_input.respond_to?(:parameters) &&
+              pending_input.parameters.any? { |type, name| %i[key keyreq].include?(type) && name == :at_answer }
+      takes ? pending_input.call(at_answer: at_answer) : pending_input.call
+    rescue StandardError
+      nil
+    end
+
+    # @return [Merge]
+    def merge(items)
+      items = Array(items)
+      lines = items.grep_v(Hash)
+      steers = items.grep(Hash).filter_map do |item|
+        text = (item[:text] || item["text"]).to_s.strip
+        { source: (item[:source] || item["source"]).to_s, text: text } unless text.empty?
+      end
+      content = lines.map { |line| line.to_s.strip }.reject(&:empty?).join("\n\n")
+      Merge.new(content: content.empty? ? nil : content, count: content.empty? ? 0 : lines.length, steers: steers)
+    end
+  end
+end