samagotchi 0.139.0 → 0.140.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +62 -1
- data/README.md +2 -1
- data/docs/architecture.md +6 -4
- data/docs/cli.md +4 -2
- data/docs/configuration.md +36 -2
- data/docs/context.md +185 -44
- data/docs/guardrails.md +11 -0
- data/docs/internals/background-tasks.md +1 -1
- data/docs/internals/engine.md +7 -1
- data/docs/internals/prompt-caching.md +10 -1
- data/docs/internals/tool-guardrails.md +1 -1
- data/docs/plugins.md +35 -17
- data/docs/sessions.md +3 -2
- data/lib/samagotchi/bridge/turn_accumulator.rb +4 -1
- data/lib/samagotchi/bundle_command.rb +21 -21
- data/lib/samagotchi/bundles/dev/manifest.yml +2 -2
- data/lib/samagotchi/bundles/github-pr/manifest.yml +4 -4
- data/lib/samagotchi/bundles/github-pr/plugin.rb +72 -176
- data/lib/samagotchi/bundles/mcp/manifest.yml +2 -2
- data/lib/samagotchi/bundles/mcp/plugin.rb +8 -3
- data/lib/samagotchi/bundles/system/manifest.yml +2 -2
- data/lib/samagotchi/bundles/system/self_map.md +2 -2
- data/lib/samagotchi/cancellation_controller.rb +4 -4
- data/lib/samagotchi/config.rb +8 -0
- data/lib/samagotchi/context_candidate_choice.rb +154 -0
- data/lib/samagotchi/context_candidates.rb +259 -0
- data/lib/samagotchi/context_candidates_log.rb +90 -0
- data/lib/samagotchi/context_command.rb +33 -8
- data/lib/samagotchi/context_fetch.rb +2 -2
- data/lib/samagotchi/context_judge.rb +194 -0
- data/lib/samagotchi/context_report_command.rb +161 -0
- data/lib/samagotchi/context_sources.rb +67 -23
- data/lib/samagotchi/cut.rb +21 -0
- data/lib/samagotchi/cut_policy.rb +6 -5
- data/lib/samagotchi/engine.rb +189 -32
- data/lib/samagotchi/hook_routes.rb +3 -2
- data/lib/samagotchi/kernel_loop.rb +8 -6
- data/lib/samagotchi/launch_mode.rb +5 -2
- data/lib/samagotchi/llm/chat_loop.rb +3 -2
- data/lib/samagotchi/llm/openai_chat.rb +1 -1
- data/lib/samagotchi/memory_bundle/status.rb +13 -3
- data/lib/samagotchi/owner_prompt.rb +39 -0
- data/lib/samagotchi/plugin/api.rb +1 -0
- data/lib/samagotchi/plugin/attached_context.rb +91 -11
- data/lib/samagotchi/plugin/context.rb +7 -1
- data/lib/samagotchi/project_env.rb +16 -2
- data/lib/samagotchi/prompt_warmup.rb +9 -1
- data/lib/samagotchi/session_access/contract.rb +2 -0
- data/lib/samagotchi/session_access/local.rb +10 -0
- data/lib/samagotchi/session_access/remote.rb +14 -0
- data/lib/samagotchi/session_access.rb +5 -0
- data/lib/samagotchi/session_commands.rb +28 -5
- data/lib/samagotchi/side_judgments.rb +84 -0
- data/lib/samagotchi/steer_cut.rb +2 -1
- data/lib/samagotchi/terminal_ui.rb +15 -0
- data/lib/samagotchi/testing/boundary.rb +3 -1
- data/lib/samagotchi/testing/host.rb +4 -1
- data/lib/samagotchi/toolkit_report_command.rb +2 -1
- data/lib/samagotchi/toolkit_suggest.rb +3 -18
- data/lib/samagotchi/turn_note.rb +6 -6
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +10 -3
- data/lib/samagotchi/web/auto_update.rb +8 -1
- data/lib/samagotchi/web/public/app.js +11 -3
- data/lib/samagotchi/web/public/composer.css +1 -1
- data/lib/samagotchi/web/public/index.html +10 -9
- data/lib/samagotchi/web/public/notifications.js +10 -1
- data/lib/samagotchi/web/public/revise_stack.js +6 -5
- data/lib/samagotchi/web/public/update_notice.js +21 -8
- data/lib/samagotchi/web/session_access_routes.rb +1 -1
- data/lib/samagotchi/web/session_hub.rb +5 -2
- data/lib/samagotchi/web/update_check.rb +14 -8
- data/lib/samagotchi/worker.rb +5 -0
- metadata +9 -1
|
@@ -5,6 +5,7 @@ require "time"
|
|
|
5
5
|
require_relative "context_sources"
|
|
6
6
|
require_relative "context_fetch"
|
|
7
7
|
require_relative "context_providers"
|
|
8
|
+
require_relative "context_report_command"
|
|
8
9
|
require_relative "memory_paths"
|
|
9
10
|
require_relative "session"
|
|
10
11
|
require_relative "cli/command"
|
|
@@ -23,7 +24,7 @@ module Samagotchi
|
|
|
23
24
|
include RedactedInspect
|
|
24
25
|
|
|
25
26
|
USAGE = <<~TEXT
|
|
26
|
-
Usage: chi context <add|push|ls|show|refresh|rm|mute|unmute> [options] [TARGET]
|
|
27
|
+
Usage: chi context <add|push|ls|show|refresh|rm|mute|unmute|report> [options] [TARGET]
|
|
27
28
|
add NAME (--cmd CMD | --push) [--every SECONDS] [--why TEXT] [--hint TEXT] TARGET
|
|
28
29
|
add URL [--why TEXT] [--hint TEXT] TARGET a URL an installed bundle's provider knows (github-pr: a PR)
|
|
29
30
|
a source: CMD prints its text (plain, or JSON {"text", "summary", "wake", "hint"});
|
|
@@ -31,17 +32,18 @@ module Samagotchi
|
|
|
31
32
|
(seconds, at least 30; default context.every_seconds). --why: why it's attached;
|
|
32
33
|
--hint: one line, often its URL. NAME: a-z, 0-9 and -, up to 40.
|
|
33
34
|
push NAME [-m TEXT] [TARGET] new text for a --push source (stdin without -m; text or JSON)
|
|
34
|
-
ls [TARGET] [--format json] the sources, their age and state
|
|
35
|
+
ls [TARGET] [--format json] the sources, their age and state; a session's candidates
|
|
35
36
|
show NAME [--json] [TARGET] the text (--json: the source and its snapshot)
|
|
36
37
|
refresh NAME [TARGET] run NAME's command now, here (a live worker absorbs the result)
|
|
37
38
|
rm NAME TARGET detach a source
|
|
38
39
|
mute|unmute NAME ID... a project's source, ignored by one session
|
|
40
|
+
report [--days N] the candidate judge's counts (this machine's log)
|
|
39
41
|
TARGET: session ids or unique prefixes, or --project (this git repository's
|
|
40
42
|
sessions, all of them). Inside a chi session (its execute) the default is that
|
|
41
43
|
session. Find ids with: chi sessions list [--live]
|
|
42
44
|
TEXT
|
|
43
45
|
|
|
44
|
-
SUBCOMMANDS = %w[add push ls show refresh rm mute unmute].freeze
|
|
46
|
+
SUBCOMMANDS = %w[add push ls show refresh rm mute unmute report].freeze
|
|
45
47
|
|
|
46
48
|
FLAGS = {
|
|
47
49
|
"add" => CLI::Flags.new(help: HELP_WORDS) do |f|
|
|
@@ -67,7 +69,8 @@ module Samagotchi
|
|
|
67
69
|
"refresh" => CLI::Flags.new(help: HELP_WORDS) { |f| f.switch "--project" },
|
|
68
70
|
"rm" => CLI::Flags.new(help: HELP_WORDS) { |f| f.switch "--project" },
|
|
69
71
|
"mute" => CLI::Flags.new(help: HELP_WORDS),
|
|
70
|
-
"unmute" => CLI::Flags.new(help: HELP_WORDS)
|
|
72
|
+
"unmute" => CLI::Flags.new(help: HELP_WORDS),
|
|
73
|
+
"report" => ContextReportCommand::FLAGS
|
|
71
74
|
}.freeze
|
|
72
75
|
|
|
73
76
|
# Where a command acts: a session (its own sources, then its project's)
|
|
@@ -106,6 +109,10 @@ module Samagotchi
|
|
|
106
109
|
return 0
|
|
107
110
|
end
|
|
108
111
|
return usage_error("unknown subcommand #{@sub}") unless SUBCOMMANDS.include?(@sub)
|
|
112
|
+
# Its own command (its own flags and usage): this machine's log.
|
|
113
|
+
if @sub == "report"
|
|
114
|
+
return ContextReportCommand.new(@argv[1..], stdout: @stdout, stderr: @stderr, state_dir: @state_dir).run
|
|
115
|
+
end
|
|
109
116
|
|
|
110
117
|
parsed = parse_flags(FLAGS.fetch(@sub), @argv[1..])
|
|
111
118
|
return parsed if parsed.is_a?(Integer)
|
|
@@ -217,16 +224,34 @@ module Samagotchi
|
|
|
217
224
|
|
|
218
225
|
targets = targets(options, args) or return CLI::Exit::FAILED
|
|
219
226
|
rows = targets.flat_map { |target| @access.context_entries(target).map { |entry| row(target, entry) } }
|
|
227
|
+
candidates = targets.flat_map { |target| @access.context_candidates(target).map { |entry| candidate_row(target, entry) } }
|
|
220
228
|
if format == "json"
|
|
221
|
-
@stdout.puts(JSON.pretty_generate(rows))
|
|
222
|
-
elsif rows.empty?
|
|
223
|
-
@stdout.puts("no attached context")
|
|
229
|
+
@stdout.puts(JSON.pretty_generate(rows + candidates))
|
|
224
230
|
else
|
|
225
|
-
print_rows(rows, labels: targets.size > 1)
|
|
231
|
+
rows.empty? ? @stdout.puts("no attached context") : print_rows(rows, labels: targets.size > 1)
|
|
232
|
+
print_candidates(candidates, labels: targets.size > 1)
|
|
226
233
|
end
|
|
227
234
|
0
|
|
228
235
|
end
|
|
229
236
|
|
|
237
|
+
# A candidate's line of ls (its --format json object, marked candidate).
|
|
238
|
+
# @param entry [SessionAccess::ContextCandidate]
|
|
239
|
+
def candidate_row(target, entry)
|
|
240
|
+
candidate = entry.candidate
|
|
241
|
+
{ "target" => target.label, "session_id" => target.session_id, "candidate" => true, "name" => candidate.name,
|
|
242
|
+
"status" => entry.status, "title" => candidate.title, "url" => candidate.url, "cmd" => candidate.cmd,
|
|
243
|
+
"mentions" => candidate.mentions, "proposed_by" => candidate.proposed_by, "at" => candidate.at }.compact
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def print_candidates(rows, labels:)
|
|
247
|
+
return if rows.empty?
|
|
248
|
+
|
|
249
|
+
@stdout.puts("candidates (attached when the session's owner mentions one):")
|
|
250
|
+
lines = rows.map { |r| [*(labels ? [r["target"]] : []), r["name"], r["status"], r["title"].to_s] }
|
|
251
|
+
widths = lines.transpose.map { |column| column.map(&:length).max }
|
|
252
|
+
lines.each { |line| @stdout.puts(" #{line.zip(widths).map { |cell, w| cell.ljust(w) }.join(" ").rstrip}") }
|
|
253
|
+
end
|
|
254
|
+
|
|
230
255
|
def run_show(options, args)
|
|
231
256
|
name = args.shift
|
|
232
257
|
return usage_error("give the source's NAME") unless name
|
|
@@ -109,8 +109,8 @@ module Samagotchi
|
|
|
109
109
|
|
|
110
110
|
def env_for(location, name)
|
|
111
111
|
previous = location.snapshot_path(name)
|
|
112
|
-
# The session's project env (ProjectEnv) under the source's own keys.
|
|
113
|
-
ProjectEnv.
|
|
112
|
+
# The session's project env (ProjectEnv.spawn_env) under the source's own keys.
|
|
113
|
+
ProjectEnv.spawn_env.merge("SAMAGOTCHI_CONTEXT_NAME" => name,
|
|
114
114
|
"SAMAGOTCHI_CONTEXT_PREVIOUS" => File.exist?(previous) ? previous : nil)
|
|
115
115
|
end
|
|
116
116
|
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "config"
|
|
4
|
+
require_relative "context_candidates_log"
|
|
5
|
+
require_relative "log"
|
|
6
|
+
require_relative "system_one/question"
|
|
7
|
+
|
|
8
|
+
module Samagotchi
|
|
9
|
+
# System 1's judgment of a session's candidate context
|
|
10
|
+
# (docs/context.md, "The judge"): is this session's work about the
|
|
11
|
+
# candidate? One ask per candidate, its state neutral and small: the
|
|
12
|
+
# candidate's title (and describe), the owner's own prompts (the first
|
|
13
|
+
# substantive one and the latest, each bounded) and, for a delegate
|
|
14
|
+
# child, what its parent session has attached. Never the branch or the
|
|
15
|
+
# checkout (bare place facts), never the source's why, never "found for
|
|
16
|
+
# this checkout". The instructions are constant (one for a session, one
|
|
17
|
+
# for a delegate child): the candidate is in the state, so the server's
|
|
18
|
+
# prefix cache holds and nothing a plugin wrote is read as instructions.
|
|
19
|
+
#
|
|
20
|
+
# The same ask has a second question, "Has the user stated a task yet?":
|
|
21
|
+
# under TASK_MIN the candidate stays unjudged and is asked again at the
|
|
22
|
+
# next owner prompt. After a judged ask, one re-ask once
|
|
23
|
+
# context.candidates.reask_after_turns more owner prompts came, then
|
|
24
|
+
# done (#due?). What it asked and answered is a `judged` line in
|
|
25
|
+
# ContextCandidatesLog. In on mode ContextCandidateChoice acts on it
|
|
26
|
+
# (attach, card, or nothing); audit only logs.
|
|
27
|
+
module ContextJudge
|
|
28
|
+
MODES = %w[off audit on].freeze
|
|
29
|
+
ABOUT_ID = "about"
|
|
30
|
+
TASK_ID = "task"
|
|
31
|
+
# The bound on each prompt in the state.
|
|
32
|
+
PROMPT_MAX = 1500
|
|
33
|
+
# Below this the user hasn't stated a task yet: the ask is deferred.
|
|
34
|
+
TASK_MIN = 0.5
|
|
35
|
+
# A prompt of this many words or fewer is not the first substantive one
|
|
36
|
+
# while a longer one follows (#conversation).
|
|
37
|
+
SHORT_WORDS = 3
|
|
38
|
+
ABOUT = "A user works with a coding agent session. The state names a candidate source of context and the " \
|
|
39
|
+
"user's messages in the session so far (the first one with a task, and the latest). " \
|
|
40
|
+
"Is this session's work about the candidate? Work on something else is not."
|
|
41
|
+
CHILD = " This session was started by another agent session: the conversation's first message is that " \
|
|
42
|
+
"session's request, and started_from lists what that session has attached (the work it is doing)."
|
|
43
|
+
TASK = "A user works with a coding agent session; the state holds the user's messages so far. " \
|
|
44
|
+
"Has the user stated a task yet? A greeting or small talk alone is not one."
|
|
45
|
+
|
|
46
|
+
# One ask's snapshot, taken on the turn's (or the plugin's) thread.
|
|
47
|
+
# +conversation+ is the owner's prompts as the state carries them,
|
|
48
|
+
# +parent_attached+ nil unless the session is a delegate child,
|
|
49
|
+
# +owner_prompts+ how many owner prompts the session had (the re-ask
|
|
50
|
+
# counts them), +phase+ "turn" (a turn's start or end) or "late" (a
|
|
51
|
+
# proposal after the prompts).
|
|
52
|
+
Job = Data.define(:session_id, :name, :title, :describe, :conversation, :parent_attached, :owner_prompts, :phase)
|
|
53
|
+
# The thresholds a judged line's would is read with, and the mode.
|
|
54
|
+
Settings = Data.define(:mode, :threshold, :offer_threshold, :reask_after_turns) do
|
|
55
|
+
def self.from_config
|
|
56
|
+
new(mode: ContextJudge.mode, threshold: Config.get("context.candidates.threshold").to_f,
|
|
57
|
+
offer_threshold: Config.get("context.candidates.offer_threshold").to_f,
|
|
58
|
+
reask_after_turns: [Config.get("context.candidates.reask_after_turns").to_i, 1].max)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
module_function
|
|
63
|
+
|
|
64
|
+
# context.candidates.mode: off (the default: no conversation text goes
|
|
65
|
+
# to a System 1 host), audit or on. An unknown value reads as off.
|
|
66
|
+
def mode
|
|
67
|
+
value = Config.get("context.candidates.mode").to_s.strip.downcase
|
|
68
|
+
MODES.include?(value) ? value : "off"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The bounded conversation the state carries: the first substantive
|
|
72
|
+
# owner prompt and the latest (one when they are the same). +skip+
|
|
73
|
+
# owner prompts at the start are not substantive (an earlier ask said
|
|
74
|
+
# no task was stated in them), nor is a short one (SHORT_WORDS words or
|
|
75
|
+
# fewer: "hi", "ok go") while a longer one follows, as the earlier
|
|
76
|
+
# prompts may have come before any candidate was there to ask about. A
|
|
77
|
+
# delegate child (+child+) keeps its first: the parent's request.
|
|
78
|
+
# @param texts [Array<String>] the owner's prompts, oldest first
|
|
79
|
+
# @return [Array<String>]
|
|
80
|
+
def conversation(texts, skip: 0, child: false)
|
|
81
|
+
texts = Array(texts).map(&:to_s).reject { |text| text.strip.empty? }
|
|
82
|
+
return [] if texts.empty?
|
|
83
|
+
|
|
84
|
+
first = child ? 0 : first_substantive(texts, skip)
|
|
85
|
+
picked = first == texts.size - 1 ? [texts.last] : [texts[first], texts.last]
|
|
86
|
+
picked.map { |text| bound(text) }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def first_substantive(texts, skip)
|
|
90
|
+
from = skip < texts.size ? skip : 0
|
|
91
|
+
(from...texts.size).find { |index| texts[index].split.size > SHORT_WORDS } || from
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def bound(text)
|
|
95
|
+
text = text.strip
|
|
96
|
+
text.length > PROMPT_MAX ? "#{text[0, PROMPT_MAX - 1]}…" : text
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# The ask's state.
|
|
100
|
+
# @return [Hash{String => Object}]
|
|
101
|
+
def state(job)
|
|
102
|
+
candidate = job.describe.to_s.strip.empty? ? job.title : "#{job.title} — #{job.describe}"
|
|
103
|
+
state = { "candidate" => candidate, "conversation" => job.conversation }
|
|
104
|
+
state["started_from"] = { "parent_attached" => job.parent_attached } if job.parent_attached
|
|
105
|
+
state
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The two questions, constant for a session (+child+: a delegate child).
|
|
109
|
+
def questions(child:)
|
|
110
|
+
{ ABOUT_ID => SystemOne::Question.new(id: ABOUT_ID, type: "noul", instructions: child ? ABOUT + CHILD : ABOUT),
|
|
111
|
+
TASK_ID => SystemOne::Question.new(id: TASK_ID, type: "noul", instructions: TASK) }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# How many owner prompts at the start no ask found a task in: the
|
|
115
|
+
# latest deferred ask's count (any candidate of the session).
|
|
116
|
+
def skip(records)
|
|
117
|
+
records.select { |line| line["kind"] == "judged" && line["stated"] == false }
|
|
118
|
+
.map { |line| line["owner_prompts"].to_i }.max || 0
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Whether candidate +name+ is asked now, at +owner_prompts+: never twice
|
|
122
|
+
# at one count; unjudged (no ask, or only deferred ones) → yes; one
|
|
123
|
+
# judged ask → once +reask_after_turns+ more owner prompts came; two →
|
|
124
|
+
# done.
|
|
125
|
+
def due?(records, name, owner_prompts:, reask_after_turns:)
|
|
126
|
+
mine = records.select { |line| line["kind"] == "judged" && line["name"] == name }
|
|
127
|
+
return false if mine.any? { |line| line["owner_prompts"].to_i >= owner_prompts }
|
|
128
|
+
|
|
129
|
+
judged = mine.select { |line| line["stated"] == true }
|
|
130
|
+
return true if judged.empty?
|
|
131
|
+
return false if judged.size >= 2
|
|
132
|
+
|
|
133
|
+
owner_prompts >= judged.first["owner_prompts"].to_i + reask_after_turns
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# The proposed candidates due an ask (+rows+: ContextCandidates
|
|
137
|
+
# .with_status), as Jobs. +texts+ the owner's prompts, oldest first;
|
|
138
|
+
# +parent_attached+ the parent's sources' titles for a delegate child,
|
|
139
|
+
# else nil.
|
|
140
|
+
# @return [Array<Job>]
|
|
141
|
+
def jobs(rows, records, session_id:, texts:, parent_attached:, settings:, phase:)
|
|
142
|
+
owner_prompts = texts.size
|
|
143
|
+
return [] if owner_prompts.zero?
|
|
144
|
+
|
|
145
|
+
conversation = conversation(texts, skip: skip(records), child: !parent_attached.nil?)
|
|
146
|
+
rows.filter_map do |candidate, status|
|
|
147
|
+
next unless status == "proposed"
|
|
148
|
+
next unless due?(records, candidate.name, owner_prompts: owner_prompts,
|
|
149
|
+
reask_after_turns: settings.reask_after_turns)
|
|
150
|
+
|
|
151
|
+
Job.new(session_id: session_id, name: candidate.name, title: candidate.title, describe: candidate.describe,
|
|
152
|
+
conversation: conversation, parent_attached: parent_attached, owner_prompts: owner_prompts,
|
|
153
|
+
phase: phase)
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# What on mode would do with +p+: attach (≥ threshold), offer (≥
|
|
158
|
+
# offer_threshold) or none.
|
|
159
|
+
def would(probability, settings)
|
|
160
|
+
return "attach" if probability >= settings.threshold
|
|
161
|
+
return "offer" if probability >= settings.offer_threshold
|
|
162
|
+
|
|
163
|
+
"none"
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# One ask (on SideJudgments' thread) and its judged line. No answer
|
|
167
|
+
# (System 1 said nothing: its own warn) writes nothing, so the
|
|
168
|
+
# candidate is asked again at the next owner prompt.
|
|
169
|
+
# @return [Hash, nil] the line's fields
|
|
170
|
+
def ask(client:, job:, log:, settings:, cancel_controller: nil)
|
|
171
|
+
answers = client.ask(state: state(job), questions: questions(child: !job.parent_attached.nil?),
|
|
172
|
+
cancel_controller: cancel_controller)
|
|
173
|
+
about = answers&.dig(ABOUT_ID)&.noul
|
|
174
|
+
task = answers&.dig(TASK_ID)&.noul
|
|
175
|
+
return nil if about.nil? || task.nil?
|
|
176
|
+
|
|
177
|
+
stated = task >= TASK_MIN
|
|
178
|
+
fields = { "title" => job.title, "phase" => job.phase, "owner_prompts" => job.owner_prompts,
|
|
179
|
+
"p" => about.round(4), "task_p" => task.round(4), "stated" => stated,
|
|
180
|
+
"would" => stated ? would(about, settings) : nil, "mode" => settings.mode,
|
|
181
|
+
"threshold" => settings.threshold, "offer_threshold" => settings.offer_threshold,
|
|
182
|
+
"host" => client.host, "model" => client.model, "child" => job.parent_attached ? true : nil,
|
|
183
|
+
"conversation" => job.conversation.map { |text| ContextCandidatesLog.short(text) } }
|
|
184
|
+
ContextCandidatesLog.event("judged", session_id: job.session_id, name: job.name, log: log, **fields.transform_keys(&:to_sym))
|
|
185
|
+
fields
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def warn(error)
|
|
189
|
+
Log.warn(:context, "context_judge_failed",
|
|
190
|
+
echo: "[samagotchi:context] candidate judge: #{error.class}: #{error.message}",
|
|
191
|
+
error: error.class.name, msg: error.message.to_s[0, 200])
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
end
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
require_relative "cli/command"
|
|
6
|
+
require_relative "cli/exit"
|
|
7
|
+
require_relative "cli/flags"
|
|
8
|
+
require_relative "config"
|
|
9
|
+
require_relative "context_candidates_log"
|
|
10
|
+
require_relative "session"
|
|
11
|
+
|
|
12
|
+
module Samagotchi
|
|
13
|
+
# `chi context report [--days N]`: counts from context_candidates.jsonl
|
|
14
|
+
# (ContextCandidatesLog), no prompts: what was proposed and mentioned,
|
|
15
|
+
# the judge's asks, what it would attach at each threshold, and how that
|
|
16
|
+
# agrees with what the user did with the same candidates (attached,
|
|
17
|
+
# mentioned, or declined); on mode's attaches (and how many the user
|
|
18
|
+
# removed later), cards and /context attach|decline. This machine only
|
|
19
|
+
# (SessionAccess::Contract).
|
|
20
|
+
class ContextReportCommand
|
|
21
|
+
include CLI::Command
|
|
22
|
+
|
|
23
|
+
USAGE = <<~TEXT
|
|
24
|
+
Usage: chi context report [--days N]
|
|
25
|
+
Counts from the candidate judge's log (context_candidates.jsonl): candidates
|
|
26
|
+
proposed, mentioned, attached or declined by you, withdrawn; the judge's asks
|
|
27
|
+
(judged, or no task stated yet); how many candidates it would attach at each
|
|
28
|
+
threshold (each candidate's latest judged ask); and how that agrees with what
|
|
29
|
+
you did with them. On mode: the judge's attaches, how many you removed later,
|
|
30
|
+
its cards, and /context attach|decline. Default: the last 14 days.
|
|
31
|
+
TEXT
|
|
32
|
+
|
|
33
|
+
DEFAULT_DAYS = 14
|
|
34
|
+
THRESHOLDS = [0.5, 0.6, 0.7, 0.8, 0.9].freeze
|
|
35
|
+
FLAGS = CLI::Flags.new(args: false) { |flags| flags.value "--days" }.freeze
|
|
36
|
+
|
|
37
|
+
# @param argv [Array<String>] the arguments after "context report"
|
|
38
|
+
# @param state_dir [String, nil] the sessions folder (a spec seam)
|
|
39
|
+
# @param now [Time] the report's clock (a spec seam)
|
|
40
|
+
def initialize(argv, stdout: $stdout, stderr: $stderr, state_dir: nil, now: Time.now)
|
|
41
|
+
@argv = argv.dup
|
|
42
|
+
@stdout = stdout
|
|
43
|
+
@stderr = stderr
|
|
44
|
+
@state_dir = state_dir || Session.default_state_dir
|
|
45
|
+
@now = now
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def run
|
|
49
|
+
parsed = parse_flags(FLAGS, @argv)
|
|
50
|
+
return parsed if parsed.is_a?(Integer)
|
|
51
|
+
|
|
52
|
+
days = parsed.options[:days]
|
|
53
|
+
return usage_error("--days takes a positive whole number") if days && !days.to_s.match?(/\A[1-9]\d*\z/)
|
|
54
|
+
|
|
55
|
+
report(days ? Integer(days, 10) : DEFAULT_DAYS)
|
|
56
|
+
CLI::Exit::OK
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# The counts, as a Hash (the report prints these).
|
|
60
|
+
def self.counts(records, threshold:, offer_threshold:)
|
|
61
|
+
by_candidate = records.group_by { |line| [line["session_id"], line["name"]] }
|
|
62
|
+
kinds = ->(kind) { records.count { |line| line["kind"] == kind } }
|
|
63
|
+
judged = records.select { |line| line["kind"] == "judged" }
|
|
64
|
+
latest = by_candidate.transform_values do |lines|
|
|
65
|
+
lines.reverse.find { |line| line["kind"] == "judged" && line["stated"] == true }&.fetch("p", nil)
|
|
66
|
+
end.compact
|
|
67
|
+
choices = by_candidate.transform_values { |lines| user_choice(lines) }.compact
|
|
68
|
+
{
|
|
69
|
+
sessions: records.map { |line| line["session_id"] }.uniq.size,
|
|
70
|
+
proposed: kinds.call("proposed"), mentioned: kinds.call("mentioned"), attached: kinds.call("attached"),
|
|
71
|
+
declined: kinds.call("declined"), detached: kinds.call("detached"), withdrawn: kinds.call("withdrawn"),
|
|
72
|
+
asks: judged.size, stated: judged.count { |line| line["stated"] == true },
|
|
73
|
+
unstated: judged.count { |line| line["stated"] == false }, candidates_judged: latest.size,
|
|
74
|
+
would: THRESHOLDS.to_h { |t| [t, latest.values.count { |p| p.to_f >= t }] },
|
|
75
|
+
threshold: threshold, offer_threshold: offer_threshold,
|
|
76
|
+
agreement: agreement(latest, choices, threshold)
|
|
77
|
+
}.merge(on_mode(records, by_candidate))
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# On mode's own lines: the judge's attaches, those the user removed
|
|
81
|
+
# later (its false positives), the cards, and /context attach|decline.
|
|
82
|
+
def self.on_mode(records, by_candidate)
|
|
83
|
+
by_command = ->(kind) { records.count { |line| line["kind"] == kind && line["by"] == "command" } }
|
|
84
|
+
removed = by_candidate.values.count do |lines|
|
|
85
|
+
index = lines.index { |line| line["kind"] == "judge_attached" }
|
|
86
|
+
index && lines.drop(index + 1).any? { |line| %w[declined detached].include?(line["kind"]) && line["by"] != "command" }
|
|
87
|
+
end
|
|
88
|
+
{ judge_attached: records.count { |line| line["kind"] == "judge_attached" }, judge_removed: removed,
|
|
89
|
+
carded: records.count { |line| line["kind"] == "carded" }, command_attached: by_command.call("attached"),
|
|
90
|
+
command_declined: by_command.call("declined") }
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# What the user last did with a candidate: "attached" (attached by
|
|
94
|
+
# them, or mentioned) or "declined"; nil when neither.
|
|
95
|
+
def self.user_choice(lines)
|
|
96
|
+
last = lines.reverse.find { |line| %w[mentioned attached declined detached].include?(line["kind"]) }
|
|
97
|
+
return nil if last.nil? || last["kind"] == "detached"
|
|
98
|
+
|
|
99
|
+
last["kind"] == "declined" ? "declined" : "attached"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# The judged candidates the user also chose for: agree (would attach
|
|
103
|
+
# what they attached, or not what they declined), attached what they
|
|
104
|
+
# declined, missed what they attached.
|
|
105
|
+
def self.agreement(latest, choices, threshold)
|
|
106
|
+
both = latest.keys & choices.keys
|
|
107
|
+
result = { candidates: both.size, agree: 0, wrong_attach: 0, missed: 0 }
|
|
108
|
+
both.each do |key|
|
|
109
|
+
would = latest[key].to_f >= threshold
|
|
110
|
+
if would == (choices[key] == "attached") then result[:agree] += 1
|
|
111
|
+
elsif would then result[:wrong_attach] += 1
|
|
112
|
+
else result[:missed] += 1
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
result
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
private
|
|
119
|
+
|
|
120
|
+
def command_name = "chi context report"
|
|
121
|
+
|
|
122
|
+
def report(days)
|
|
123
|
+
cutoff = @now - (days * 86_400)
|
|
124
|
+
records = ContextCandidatesLog.log_for(@state_dir).records.select { |line| after?(line["at"], cutoff) }
|
|
125
|
+
counts = self.class.counts(records, threshold: Config.get("context.candidates.threshold").to_f,
|
|
126
|
+
offer_threshold: Config.get("context.candidates.offer_threshold").to_f)
|
|
127
|
+
print_counts(counts, days)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def after?(iso, cutoff)
|
|
131
|
+
Time.iso8601(iso.to_s) >= cutoff
|
|
132
|
+
rescue ArgumentError
|
|
133
|
+
false
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def print_counts(counts, days)
|
|
137
|
+
@stdout.puts("chi context report: the last #{days} day#{"s" unless days == 1}, #{counts[:sessions]} session" \
|
|
138
|
+
"#{"s" unless counts[:sessions] == 1} (mode #{Config.get("context.candidates.mode")})")
|
|
139
|
+
@stdout.puts(" candidates: proposed #{counts[:proposed]} · mentioned #{counts[:mentioned]} · " \
|
|
140
|
+
"attached by you #{counts[:attached]} · declined #{counts[:declined]} · " \
|
|
141
|
+
"detached #{counts[:detached]} · withdrawn #{counts[:withdrawn]}")
|
|
142
|
+
@stdout.puts(" on mode: judge attached #{counts[:judge_attached]} · you removed #{counts[:judge_removed]} of them " \
|
|
143
|
+
"later · carded #{counts[:carded]} · /context attach #{counts[:command_attached]} · " \
|
|
144
|
+
"/context decline #{counts[:command_declined]} (the card's buttons)")
|
|
145
|
+
@stdout.puts(" judge: #{counts[:asks]} asks · judged #{counts[:stated]} · no task yet #{counts[:unstated]} · " \
|
|
146
|
+
"#{counts[:candidates_judged]} candidates judged")
|
|
147
|
+
return if counts[:candidates_judged].zero?
|
|
148
|
+
|
|
149
|
+
@stdout.puts(" would attach (each candidate's latest judged ask):")
|
|
150
|
+
counts[:would].each do |threshold, count|
|
|
151
|
+
mark = { counts[:threshold] => " threshold", counts[:offer_threshold] => " offer_threshold" }[threshold]
|
|
152
|
+
@stdout.puts(format(" ≥ %.1f %d%s", threshold, count, mark))
|
|
153
|
+
end
|
|
154
|
+
agreement = counts[:agreement]
|
|
155
|
+
@stdout.puts(" against your own choice (#{agreement[:candidates]} judged candidates you attached, " \
|
|
156
|
+
"mentioned or declined), at #{counts[:threshold]}: agree #{agreement[:agree]} · " \
|
|
157
|
+
"would attach what you declined #{agreement[:wrong_attach]} · " \
|
|
158
|
+
"would miss what you attached #{agreement[:missed]}")
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
@@ -22,7 +22,7 @@ module Samagotchi
|
|
|
22
22
|
# sessions/<id>/subscriptions.json {name => Subscription}; the worker writes it
|
|
23
23
|
# sessions/<id>/muted/<name> a marker: this session ignores <name>
|
|
24
24
|
# sessions/<id>/declined/<name> a marker (its hint inside): the user removed <name>
|
|
25
|
-
# sessions/<id>/
|
|
25
|
+
# sessions/<id>/candidates/<name>.json a Candidate: a source a plugin proposed here (ctx.context.propose)
|
|
26
26
|
# The guardrails protect the root (ProtectedPaths, CHI_TEXT): a source's
|
|
27
27
|
# command runs later, outside the gate.
|
|
28
28
|
module ContextSources
|
|
@@ -40,7 +40,10 @@ module Samagotchi
|
|
|
40
40
|
SUBSCRIPTIONS_FILE = "subscriptions.json"
|
|
41
41
|
MUTED_DIR = "muted"
|
|
42
42
|
DECLINED_DIR = "declined"
|
|
43
|
-
|
|
43
|
+
CANDIDATES_DIR = "candidates"
|
|
44
|
+
# A candidate keeps this many mentions, each one line of up to LINE_MAX_CHARS.
|
|
45
|
+
MENTIONS_MAX = 10
|
|
46
|
+
DESCRIBE_MAX_CHARS = 1000
|
|
44
47
|
|
|
45
48
|
# A name, a command, a value the store can't take.
|
|
46
49
|
class Invalid < ArgumentError; end
|
|
@@ -48,20 +51,23 @@ module Samagotchi
|
|
|
48
51
|
# A source's definition. +cmd+ nil: pushed only (chi context push).
|
|
49
52
|
# +provider+: the bundle whose context provider made it (a URL), whose
|
|
50
53
|
# folder its cmd's {bundle_dir} names (ContextProviders.command_for).
|
|
51
|
-
|
|
52
|
-
|
|
54
|
+
# +title+: what a person reads it as (a candidate's, when one was
|
|
55
|
+
# attached; nil for most).
|
|
56
|
+
Source = Data.define(:name, :cmd, :every_seconds, :why, :hint, :scope, :added_by, :created_at, :provider, :title) do
|
|
57
|
+
def initialize(name:, cmd:, every_seconds:, why:, hint:, scope:, added_by:, created_at:, provider: nil, title: nil) = super
|
|
53
58
|
|
|
54
59
|
def push? = cmd.nil?
|
|
55
60
|
|
|
56
61
|
def to_h
|
|
57
62
|
{ "name" => name, "cmd" => cmd, "every_seconds" => every_seconds, "why" => why, "hint" => hint,
|
|
58
|
-
"scope" => scope, "added_by" => added_by, "created_at" => created_at, "provider" => provider
|
|
63
|
+
"scope" => scope, "added_by" => added_by, "created_at" => created_at, "provider" => provider,
|
|
64
|
+
"title" => title }.compact
|
|
59
65
|
end
|
|
60
66
|
|
|
61
67
|
def self.from_h(data)
|
|
62
68
|
new(name: data["name"].to_s, cmd: data["cmd"], every_seconds: data["every_seconds"]&.to_i,
|
|
63
69
|
why: data["why"], hint: data["hint"], scope: data["scope"].to_s, added_by: data["added_by"],
|
|
64
|
-
created_at: data["created_at"], provider: data["provider"])
|
|
70
|
+
created_at: data["created_at"], provider: data["provider"], title: data["title"])
|
|
65
71
|
end
|
|
66
72
|
end
|
|
67
73
|
|
|
@@ -133,10 +139,25 @@ module Samagotchi
|
|
|
133
139
|
# What a command printed or a push sent (the contract, ContextSources.parse_output).
|
|
134
140
|
Fetched = Data.define(:text, :summary, :wake, :hint)
|
|
135
141
|
|
|
136
|
-
# A
|
|
137
|
-
#
|
|
138
|
-
#
|
|
139
|
-
|
|
142
|
+
# A source a plugin proposed for a session, not attached
|
|
143
|
+
# (ctx.context.propose): its spec (+url+, through an installed bundle's
|
|
144
|
+
# provider, or +cmd+), +title+ (what a person or a judge reads),
|
|
145
|
+
# +describe+ (optional, more of the same), +mentions+ (the words in an
|
|
146
|
+
# owner prompt that name it), +proposed_by+ ("plugin:<bundle>") and
|
|
147
|
+
# +at+, when it was proposed (ISO 8601). Written once: whether it is
|
|
148
|
+
# attached or declined is the store's (a source of its name, a declined
|
|
149
|
+
# marker), never kept here.
|
|
150
|
+
Candidate = Data.define(:name, :url, :cmd, :title, :describe, :mentions, :proposed_by, :at) do
|
|
151
|
+
def to_h
|
|
152
|
+
{ "url" => url, "cmd" => cmd, "title" => title, "describe" => describe, "mentions" => mentions,
|
|
153
|
+
"proposed_by" => proposed_by, "at" => at }.compact
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def self.from_h(name, data)
|
|
157
|
+
new(name: name, url: data["url"], cmd: data["cmd"], title: data["title"].to_s, describe: data["describe"],
|
|
158
|
+
mentions: Array(data["mentions"]).map(&:to_s), proposed_by: data["proposed_by"], at: data["at"])
|
|
159
|
+
end
|
|
160
|
+
end
|
|
140
161
|
|
|
141
162
|
# A source as one session sees it: the definition, where it lives, and
|
|
142
163
|
# +shadowed+ for a project source hidden by a session one of its name.
|
|
@@ -288,22 +309,45 @@ module Samagotchi
|
|
|
288
309
|
end
|
|
289
310
|
end
|
|
290
311
|
|
|
291
|
-
def
|
|
312
|
+
def candidate_path(name) = File.join(dir, CANDIDATES_DIR, "#{ContextSources.check_name!(name)}.json")
|
|
313
|
+
|
|
314
|
+
# Session only: records +candidate+ unless one of its name is here
|
|
315
|
+
# already (the first proposal stays as it was). Neither an add nor a
|
|
316
|
+
# decline clears it; deleting the session does.
|
|
317
|
+
# @return [Candidate] the one recorded (+candidate+ or the earlier one)
|
|
318
|
+
def add_candidate(candidate)
|
|
319
|
+
existing = candidate(candidate.name)
|
|
320
|
+
return existing if existing
|
|
321
|
+
|
|
322
|
+
FileUtils.mkdir_p(File.join(dir, CANDIDATES_DIR))
|
|
323
|
+
AtomicFile.write(candidate_path(candidate.name), "#{JSON.generate(candidate.to_h)}\n")
|
|
324
|
+
candidate
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
# @return [Candidate, nil]
|
|
328
|
+
def candidate(name)
|
|
329
|
+
data = ContextSources.read_json(candidate_path(name))
|
|
330
|
+
data ? Candidate.from_h(name, data) : nil
|
|
331
|
+
end
|
|
292
332
|
|
|
293
|
-
#
|
|
294
|
-
#
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
FileUtils.
|
|
299
|
-
|
|
300
|
-
offer
|
|
333
|
+
# Removes the candidate +name+ (an attached source of its name stays).
|
|
334
|
+
# @return [Boolean] whether there was one
|
|
335
|
+
def remove_candidate(name)
|
|
336
|
+
path = candidate_path(name)
|
|
337
|
+
existed = File.exist?(path)
|
|
338
|
+
FileUtils.rm_f(path)
|
|
339
|
+
existed
|
|
301
340
|
end
|
|
302
341
|
|
|
303
|
-
# @return [
|
|
304
|
-
def
|
|
305
|
-
|
|
306
|
-
|
|
342
|
+
# @return [Array<Candidate>] by name
|
|
343
|
+
def candidates
|
|
344
|
+
folder = File.join(dir, CANDIDATES_DIR)
|
|
345
|
+
return [] unless Dir.exist?(folder)
|
|
346
|
+
|
|
347
|
+
Dir.children(folder).sort.filter_map do |file|
|
|
348
|
+
name = file.delete_suffix(".json")
|
|
349
|
+
candidate(name) if file.end_with?(".json") && name.match?(NAME_RE) && !RESERVED_NAMES.include?(name)
|
|
350
|
+
end
|
|
307
351
|
end
|
|
308
352
|
|
|
309
353
|
# Session only: the markers and the worker's subscription file.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Samagotchi
|
|
4
|
+
# Why a generation was cut: what CancellationController#cancel_generation!
|
|
5
|
+
# is given (a bundle's stop_generation, or Engine#cut_for_steer) and what
|
|
6
|
+
# the loops keep on a cut generation. +by+ is who stopped it (the hook's
|
|
7
|
+
# label, or "steer"), +reason+ what it said, +steer+ true when a queued user
|
|
8
|
+
# message cut it (no nudge, no retry attempt spent), +source+ that message,
|
|
9
|
+
# for the steer_cut row. The same value rides a whole-turn stop
|
|
10
|
+
# (stop_turn): the cancel note reads its +by+ and +reason+.
|
|
11
|
+
Cut = Data.define(:by, :reason, :steer, :source) do
|
|
12
|
+
def initialize(by: nil, reason: nil, steer: false, source: nil) = super
|
|
13
|
+
|
|
14
|
+
# A hook's cut (stop_generation, or stop_turn): the bundle +by+ and its
|
|
15
|
+
# +reason+.
|
|
16
|
+
def self.hook(by: nil, reason: nil) = new(by: by, reason: reason)
|
|
17
|
+
|
|
18
|
+
# Engine#cut_for_steer: a queued message cut the streaming generation.
|
|
19
|
+
def self.steer(source) = new(by: "steer", reason: "a new message", steer: true, source: source)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "cut"
|
|
3
4
|
require_relative "turn_note"
|
|
4
5
|
|
|
5
6
|
module Samagotchi
|
|
@@ -30,7 +31,7 @@ module Samagotchi
|
|
|
30
31
|
|
|
31
32
|
module_function
|
|
32
33
|
|
|
33
|
-
# +cut+: the
|
|
34
|
+
# +cut+: the Cut (what stop_generation passed). +emit+ takes an
|
|
34
35
|
# event Hash; +inject+ puts queued input in (true when there was any).
|
|
35
36
|
# +finish_reason+ and +thinking_chars+: the cut generation's, for the
|
|
36
37
|
# retry's row.
|
|
@@ -40,13 +41,13 @@ module Samagotchi
|
|
|
40
41
|
return STOPPED if cancel_controller.cancelled?
|
|
41
42
|
|
|
42
43
|
# The row under the cut step, above the message it was cut for.
|
|
43
|
-
emit.call({ type: :steer_cut, iteration: iteration, source: cut
|
|
44
|
-
return AGAIN if inject.call || cut
|
|
44
|
+
emit.call({ type: :steer_cut, iteration: iteration, source: cut.source.to_s }) if cut.steer
|
|
45
|
+
return AGAIN if inject.call || cut.steer
|
|
45
46
|
|
|
46
47
|
if empty_retry.left?
|
|
47
|
-
empty_retry.nudge!(conversation, TurnNote.cut_retry(cut
|
|
48
|
+
empty_retry.nudge!(conversation, TurnNote.cut_retry(cut.by, cut.reason),
|
|
48
49
|
emit: emit, iteration: iteration, finish_reason: finish_reason,
|
|
49
|
-
thinking_chars: thinking_chars, stopped_by: cut
|
|
50
|
+
thinking_chars: thinking_chars, stopped_by: cut.by)
|
|
50
51
|
return AGAIN
|
|
51
52
|
end
|
|
52
53
|
empty_retry.drop_nudge!(conversation)
|