samagotchi 0.101.0 → 0.103.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +67 -1
  3. data/bin/chi +31 -196
  4. data/docs/architecture.md +62 -6
  5. data/docs/cli.md +154 -7
  6. data/docs/configuration.md +32 -5
  7. data/docs/guardrails.md +2 -1
  8. data/docs/internals/bundles.md +1 -1
  9. data/docs/internals/engine.md +1 -1
  10. data/docs/internals/llm-context-forget.md +25 -1
  11. data/docs/internals/prompt-caching.md +8 -1
  12. data/docs/internals/remote-sessions.md +303 -0
  13. data/docs/internals/working-set.md +174 -15
  14. data/docs/plugins.md +32 -5
  15. data/docs/sessions.md +38 -14
  16. data/docs/sub-agent.md +8 -0
  17. data/docs/testing.md +7 -0
  18. data/lib/samagotchi/answer_command.rb +27 -25
  19. data/lib/samagotchi/bridge/turn_accumulator.rb +16 -0
  20. data/lib/samagotchi/bridge.rb +33 -7
  21. data/lib/samagotchi/bridge_client/event_stream.rb +81 -12
  22. data/lib/samagotchi/bridge_client/stream_request.rb +77 -0
  23. data/lib/samagotchi/bridge_client.rb +22 -38
  24. data/lib/samagotchi/bundle_command/check.rb +5 -1
  25. data/lib/samagotchi/bundle_command/try/command.rb +6 -6
  26. data/lib/samagotchi/bundle_command/try/sandbox.rb +3 -3
  27. data/lib/samagotchi/bundle_command.rb +4 -4
  28. data/lib/samagotchi/bundles/claude-peer/manifest.yml +2 -2
  29. data/lib/samagotchi/bundles/claude-peer/scripts/pair +30 -1
  30. data/lib/samagotchi/bundles/codex-peer/manifest.yml +2 -2
  31. data/lib/samagotchi/bundles/codex-peer/scripts/pair +49 -4
  32. data/lib/samagotchi/bundles/guardrails/guardrails/rules.yml +7 -4
  33. data/lib/samagotchi/bundles/guardrails/manifest.yml +1 -1
  34. data/lib/samagotchi/bundles/system/manifest.yml +1 -1
  35. data/lib/samagotchi/chi_invocation.rb +43 -0
  36. data/lib/samagotchi/cli/command.rb +3 -2
  37. data/lib/samagotchi/cli/flags.rb +5 -0
  38. data/lib/samagotchi/cli/parent_wait.rb +22 -13
  39. data/lib/samagotchi/commands/registry.rb +12 -1
  40. data/lib/samagotchi/config.rb +89 -53
  41. data/lib/samagotchi/context_command.rb +80 -73
  42. data/lib/samagotchi/engine.rb +138 -15
  43. data/lib/samagotchi/idle_recap.rb +7 -1
  44. data/lib/samagotchi/idle_scheduler.rb +16 -1
  45. data/lib/samagotchi/kernel_loop.rb +13 -1
  46. data/lib/samagotchi/launch_mode.rb +22 -0
  47. data/lib/samagotchi/llm_context_edit.rb +32 -10
  48. data/lib/samagotchi/llm_context_forget.rb +24 -8
  49. data/lib/samagotchi/llm_context_view.rb +19 -7
  50. data/lib/samagotchi/memory_bundle/installer.rb +2 -0
  51. data/lib/samagotchi/memory_bundle/manifest.rb +13 -2
  52. data/lib/samagotchi/message_parts.rb +8 -3
  53. data/lib/samagotchi/note_command.rb +8 -7
  54. data/lib/samagotchi/parent_report.rb +55 -43
  55. data/lib/samagotchi/plugin/context.rb +2 -2
  56. data/lib/samagotchi/prompt.rb +1 -1
  57. data/lib/samagotchi/recap_command.rb +68 -0
  58. data/lib/samagotchi/recap_setup.rb +27 -14
  59. data/lib/samagotchi/remote_command.rb +170 -0
  60. data/lib/samagotchi/remotes.rb +130 -0
  61. data/lib/samagotchi/run_options.rb +233 -0
  62. data/lib/samagotchi/send_command/busy.rb +58 -0
  63. data/lib/samagotchi/send_command/parent_gate.rb +23 -10
  64. data/lib/samagotchi/send_command/wait_target.rb +16 -3
  65. data/lib/samagotchi/send_command.rb +186 -118
  66. data/lib/samagotchi/session.rb +48 -16
  67. data/lib/samagotchi/session_access/contract.rb +129 -0
  68. data/lib/samagotchi/session_access/dispatch.rb +133 -0
  69. data/lib/samagotchi/session_access/http_transport.rb +109 -0
  70. data/lib/samagotchi/session_access/local.rb +349 -0
  71. data/lib/samagotchi/session_access/remote.rb +638 -0
  72. data/lib/samagotchi/session_access/web_bridge_client.rb +181 -0
  73. data/lib/samagotchi/session_access.rb +141 -0
  74. data/lib/samagotchi/session_archive_command.rb +9 -5
  75. data/lib/samagotchi/session_commands.rb +53 -6
  76. data/lib/samagotchi/session_config.rb +411 -0
  77. data/lib/samagotchi/session_delete_command.rb +8 -4
  78. data/lib/samagotchi/session_manager.rb +7 -2
  79. data/lib/samagotchi/session_setup.rb +15 -10
  80. data/lib/samagotchi/sessions_command.rb +42 -16
  81. data/lib/samagotchi/terminal_ui/attach_launcher.rb +54 -18
  82. data/lib/samagotchi/terminal_ui/attach_link.rb +104 -0
  83. data/lib/samagotchi/terminal_ui/attached_loop.rb +82 -76
  84. data/lib/samagotchi/terminal_ui/formatting.rb +10 -3
  85. data/lib/samagotchi/terminal_ui/local_link.rb +130 -0
  86. data/lib/samagotchi/terminal_ui/version_lines.rb +5 -3
  87. data/lib/samagotchi/terminal_ui.rb +8 -6
  88. data/lib/samagotchi/testing/bundle.rb +3 -2
  89. data/lib/samagotchi/title_command.rb +9 -11
  90. data/lib/samagotchi/version.rb +1 -1
  91. data/lib/samagotchi/web/app.rb +387 -79
  92. data/lib/samagotchi/web/public/app.js +2 -0
  93. data/lib/samagotchi/web/public/commands.js +8 -2
  94. data/lib/samagotchi/web/public/data.js +6 -4
  95. data/lib/samagotchi/web/public/index.html +37 -3
  96. data/lib/samagotchi/web/public/stage_frame.js +7 -2
  97. data/lib/samagotchi/web/public/stage_view.js +3 -2
  98. data/lib/samagotchi/web/public/stream_handlers.js +7 -1
  99. data/lib/samagotchi/web/public/turn_html.js +5 -0
  100. data/lib/samagotchi/web/public/working_set.js +247 -24
  101. data/lib/samagotchi/web/public/working_set_rail.js +39 -12
  102. data/lib/samagotchi/web/public/working_set_view.js +198 -25
  103. data/lib/samagotchi/web/session_access_routes.rb +343 -0
  104. data/lib/samagotchi/worker.rb +9 -6
  105. data/lib/samagotchi/worker_sidecar.rb +12 -5
  106. data/lib/samagotchi/working_set.rb +132 -20
  107. data/lib/samagotchi/working_set_command.rb +222 -0
  108. data/lib/samagotchi/working_set_pin.rb +34 -0
  109. data/lib/samagotchi/working_set_steer.rb +414 -0
  110. metadata +23 -2
  111. data/lib/samagotchi/bundle_session_settings.rb +0 -251
@@ -0,0 +1,222 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "working_set"
4
+ require_relative "working_set_steer"
5
+ require_relative "tools/memory"
6
+
7
+ module Samagotchi
8
+ # /working-set: with no arguments the session's working set (WorkingSet,
9
+ # as the web's list shows it: each item, what was done to it, its state
10
+ # and tokens, then the changes the next turn starts with); `drop KEY…`,
11
+ # `undrop KEY…`, `pin KEY…`, `unpin KEY…`, `add KEY…` and `unadd KEY…`
12
+ # steer it (WorkingSetSteer). An add asks the model, in the next turn's
13
+ # note, to read the item (a file that exists, a memory, an http(s) URL).
14
+ # KEY: a path (absolute or the session's folder's), memory:NAME, a URL;
15
+ # one with spaces quoted ("my dir/a.rb", WorkingSetSteer.words), as the
16
+ # listing shows it.
17
+ # A pin is saved on the session (Session#working_set_pins); the turn-start
18
+ # note names it only where the model has forget_outputs (the strategy's
19
+ # forget layer), or to cancel a pin or an unpin it named.
20
+ #
21
+ # An op runs between turns (mid_turn :queue: a line sent while a turn
22
+ # runs waits for its end), on a copy of the session's messages that
23
+ # replaces them, so the next turn's first request is the first with it.
24
+ class WorkingSetCommand
25
+ NAME = "/working-set"
26
+ USAGE = "usage: #{NAME} [drop KEY… | undrop KEY… | pin KEY… | unpin KEY… | add KEY… | unadd KEY…] " \
27
+ "(KEY: a path, memory:NAME or a URL; \"quote\" one with spaces)".freeze
28
+ VERBS = [WorkingSetSteer::DROP, WorkingSetSteer::UNDROP, WorkingSetSteer::PIN, WorkingSetSteer::UNPIN,
29
+ WorkingSetSteer::ADD, WorkingSetSteer::UNADD].freeze
30
+ OP_WORDS = { WorkingSetSteer::DROP => "dropped", WorkingSetSteer::UNDROP => "undropped",
31
+ WorkingSetSteer::PIN => "pinned", WorkingSetSteer::UNPIN => "unpinned",
32
+ WorkingSetSteer::ADD => "added", WorkingSetSteer::UNADD => "unadded" }.freeze
33
+ PIN_OPS = [WorkingSetSteer::PIN, WorkingSetSteer::UNPIN].freeze
34
+
35
+ # @param engine [Engine] #session, #working_set_strategy,
36
+ # #clone_messages, #replace_messages
37
+ # @param save [#call] saves the session after a change
38
+ def initialize(engine:, save:)
39
+ @engine = engine
40
+ @save = save
41
+ end
42
+
43
+ # @param args [String] what follows /working-set
44
+ # @return [Array(String, Array<Symbol>, Symbol)] the reply, what
45
+ # changed (:messages, :working_set_pins; none when nothing did), and
46
+ # :ok or :error (every key refused)
47
+ # @raise [ArgumentError] a line that isn't one (its message and USAGE)
48
+ def run(args)
49
+ verb, *words = WorkingSetSteer.words(args)
50
+ return [listing, [], :ok] if verb.nil?
51
+ raise ArgumentError, USAGE unless VERBS.include?(verb) && !words.empty?
52
+
53
+ session = @engine.session or raise ArgumentError, "no session yet"
54
+ cwd = session.working_directory
55
+ strategy = @engine.working_set_strategy
56
+ messages = @engine.clone_messages(session.messages)
57
+ pins = session.working_set_pins
58
+ lines = []
59
+ done = 0
60
+ noted = false
61
+ words.each do |word|
62
+ target = WorkingSetSteer.target(word, cwd: cwd)
63
+ result = op(verb, messages, pins, target, strategy, cwd)
64
+ [result.undone, result].compact.select { |each| noted?(each, messages, strategy) }.each do |each|
65
+ messages = WorkingSetSteer.record!(messages, each)
66
+ noted = true
67
+ end
68
+ pins = WorkingSetSteer.pinning(pins, result) if PIN_OPS.include?(result.op)
69
+ lines << done_line(result.undone) if result.undone
70
+ lines << done_line(result, strategy)
71
+ done += 1
72
+ rescue WorkingSetSteer::Refused => e
73
+ lines << "#{verb} #{WorkingSetSteer.quote(word)}: #{e.message}"
74
+ end
75
+ changed = done.positive? ? save(session, noted && messages, pins) : []
76
+ [lines.join("\n"), changed, done.positive? ? :ok : :error]
77
+ end
78
+
79
+ # The set under the strategy the next turn runs, and the pending ops.
80
+ def listing
81
+ session = @engine.session
82
+ return "no session yet" unless session
83
+
84
+ messages = Array(session.messages)
85
+ items = WorkingSet.build(messages, cwd: session.working_directory, strategy: @engine.working_set_strategy,
86
+ pins: session.working_set_pins).items
87
+ return "working set: empty (the files, memories and URLs the turns read show here)\n#{USAGE}" if items.empty?
88
+
89
+ rows = items.map { |item| [WorkingSetSteer.quote(name(item, session.working_directory)), words(item)] }
90
+ width = rows.map { |row, _| row.length }.max
91
+ lines = ["working set (#{items.size} item#{"s" unless items.size == 1}):"]
92
+ lines.concat(rows.map { |row, text| " #{row.ljust(width)} #{text}" })
93
+ pending = WorkingSetSteer.pending_ops(messages)
94
+ unless pending.empty?
95
+ lines << "at the next turn's start: #{pending.map { |op| "#{OP_WORDS[op["op"]]} #{op["label"]}" }.join(", ")}"
96
+ end
97
+ lines << USAGE
98
+ lines.join("\n")
99
+ end
100
+
101
+ private
102
+
103
+ def op(verb, messages, pins, target, strategy, cwd)
104
+ case verb
105
+ when WorkingSetSteer::DROP then WorkingSetSteer.drop(messages, target, strategy: strategy, cwd: cwd, pins: pins)
106
+ when WorkingSetSteer::UNDROP
107
+ sent = !WorkingSetSteer.pending?(messages, WorkingSetSteer::DROP, target)
108
+ WorkingSetSteer.undrop(messages, target, strategy: strategy, cwd: cwd, sent: sent)
109
+ when WorkingSetSteer::PIN then WorkingSetSteer.pin(messages, pins, target, strategy: strategy, cwd: cwd)
110
+ when WorkingSetSteer::UNPIN then WorkingSetSteer.unpin(pins, target)
111
+ when WorkingSetSteer::ADD
112
+ WorkingSetSteer.add(messages, target, strategy: strategy, cwd: cwd, found: method(:missing))
113
+ else WorkingSetSteer.unadd(messages, target)
114
+ end
115
+ end
116
+
117
+ # Why an added +target+ can't be read (a Refused reason), nil when it
118
+ # can: a file that is there, a memory the project or system scope
119
+ # holds (memory_read's look). A URL is fetched as it is.
120
+ def missing(target)
121
+ case target.kind
122
+ when WorkingSet::FILE
123
+ return :directory if File.directory?(target.key)
124
+
125
+ File.file?(target.key) ? nil : :no_file
126
+ when WorkingSet::MEMORY
127
+ return :bad_memory if target.key.empty? || Tools::MemoryRead.invalid_name?(target.key)
128
+
129
+ found = %w[project system].any? { |scope| File.file?(Tools::MemoryRead.memory_path(target.key, scope)) }
130
+ found ? nil : :no_memory
131
+ end
132
+ end
133
+
134
+ # Whether the turn-start note lists +done+: a drop or an undrop always;
135
+ # a pin or an unpin where the model has forget_outputs, or when it
136
+ # cancels one the note lists.
137
+ def noted?(done, messages, strategy)
138
+ return true unless PIN_OPS.include?(done.op)
139
+
140
+ forget?(strategy) || WorkingSetSteer.pending?(messages, WorkingSetSteer::INVERSE.fetch(done.op), done.target)
141
+ end
142
+
143
+ def forget?(strategy) = Array(strategy).include?(:forget)
144
+
145
+ # Puts the ops' messages (nil: unchanged) and pins in place and saves
146
+ # the session.
147
+ # @return [Array<Symbol>] what changed
148
+ def save(session, messages, pins)
149
+ changed = []
150
+ if messages
151
+ @engine.replace_messages(messages)
152
+ changed << :messages
153
+ end
154
+ unless pins == session.working_set_pins
155
+ session.working_set_pins = pins
156
+ changed << :working_set_pins
157
+ end
158
+ @save.call(session) unless changed.empty?
159
+ changed
160
+ end
161
+
162
+ def name(item, cwd)
163
+ case item.kind
164
+ when WorkingSet::FILE then WorkingSetSteer.target(item.key, cwd: cwd).label
165
+ when WorkingSet::MEMORY then "#{WorkingSetSteer::MEMORY_PREFIX}#{item.key}"
166
+ else item.key
167
+ end
168
+ end
169
+
170
+ def words(item)
171
+ parts = [item.ops.join(", ")]
172
+ parts << "×#{item.count}" if item.count > 1
173
+ parts << item.state unless item.state == WorkingSet::LIVE
174
+ # A drop on some of its reads (the rest forgotten or still sent): undrop takes it back.
175
+ parts << "part dropped" if item.has_drop && item.state != WorkingSet::DROPPED
176
+ parts << "pinned" if item.pinned
177
+ parts << "~#{item.tokens} tokens" if item.tokens.positive?
178
+ parts.join(" · ")
179
+ end
180
+
181
+ def done_line(done, strategy = nil)
182
+ ids = done.ids.empty? ? "" : " (#{done.ids.join(", ")})"
183
+ "#{op_line(done, "#{OP_WORDS[done.op]} #{done.target.label}#{ids}", strategy)}#{with_line(done)}"
184
+ end
185
+
186
+ # A memory_read of several names is one output: the op acted on the
187
+ # others' read too. Says which names went with it.
188
+ def with_line(done)
189
+ return "" if done.shared_with.empty?
190
+
191
+ names = done.shared_with.map { |name| "memory #{name}" }.join(", ")
192
+ "; the same memory_read holds #{names}: #{OP_WORDS[done.op]} with it (that read only)"
193
+ end
194
+
195
+ def op_line(done, what, strategy)
196
+ case done.op
197
+ when WorkingSetSteer::PIN
198
+ off = forget?(strategy) ? "" : " (the strategy has no forget layer now: the pin holds when it has)"
199
+ "#{what}: forget_outputs refuses its outputs, its later reads too#{off}"
200
+ when WorkingSetSteer::UNPIN then "#{what}: the model may forget it again"
201
+ when WorkingSetSteer::ADD
202
+ "#{what}: the next turn's note asks the model to read it first (one tool call)"
203
+ when WorkingSetSteer::UNADD then "#{what}: its add was never sent; nothing changes"
204
+ when WorkingSetSteer::DROP
205
+ frees = done.freed_chars.positive? ? "frees #{WorkingSetSteer.k(done.freed_chars)}" : "frees nothing, the stubs are as long"
206
+ "#{what}: the next turn's first request sends the stubs (#{frees}; the server reads the " \
207
+ "#{WorkingSetSteer.k(done.tail_chars)} after them again)"
208
+ else
209
+ undrop_line(what, done)
210
+ end
211
+ end
212
+
213
+ def undrop_line(what, done)
214
+ if done.free
215
+ "#{what}: its drop was never sent; nothing changes"
216
+ else
217
+ "#{what}: the next request sends it whole again (the server reads the #{WorkingSetSteer.k(done.tail_chars)} " \
218
+ "from it on again)"
219
+ end
220
+ end
221
+ end
222
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Samagotchi
4
+ # One item of the working set the user pinned (/working-set pin): its
5
+ # identity as WorkingSet names an item, +kind+ file|memory|url and +key+
6
+ # (a file's absolute path, a memory's name, a URL). The model's
7
+ # forget_outputs refuses its outputs, the ones it has and the ones it
8
+ # reads later (WorkingSetSteer). Saved as the session's
9
+ # working_set_pins ([{"kind", "key"}]); a respawn, a continue and a fork
10
+ # keep them (SessionSetup).
11
+ WorkingSetPin = Data.define(:kind, :key) do
12
+ def initialize(kind:, key:)
13
+ super(kind: kind.to_s, key: key.to_s)
14
+ end
15
+
16
+ # The session file's form.
17
+ def to_file = { "kind" => kind, "key" => key }
18
+
19
+ # +value+'s pins, in order, no repeats: pins, or a file's Hashes
20
+ # (string or symbol keys); anything else (a hand-edited entry without
21
+ # a kind or key) is left out.
22
+ # @return [Array<WorkingSetPin>] frozen
23
+ def self.list(value)
24
+ Array(value).filter_map do |entry|
25
+ next entry if entry.is_a?(WorkingSetPin)
26
+ next unless entry.is_a?(Hash)
27
+
28
+ kind = entry["kind"] || entry[:kind]
29
+ key = entry["key"] || entry[:key]
30
+ new(kind: kind, key: key) unless kind.to_s.empty? || key.to_s.empty?
31
+ end.uniq.freeze
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,414 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+ require_relative "llm_context_edit"
5
+ require_relative "llm_context_view"
6
+ require_relative "llm_context_apply"
7
+ require_relative "token_usage"
8
+ require_relative "working_set"
9
+ require_relative "working_set_pin"
10
+
11
+ module Samagotchi
12
+ # The user steering the working set (/working-set, the web's popover):
13
+ # drop an item (every content output of it still sent goes out of the
14
+ # prompt as a stub) and undrop it; pin one (the model's forget_outputs
15
+ # refuses its outputs, LLMContextForget) and unpin it. Pure, on a
16
+ # conversation the caller
17
+ # owns (Engine#clone_messages' copies: an op stores new edits Hashes,
18
+ # LLMContextEdit.store) and runs between turns, so the next turn's first
19
+ # request is the first with the change (prompt caching: one break where
20
+ # the new prompt is fresh anyway).
21
+ #
22
+ # A drop is the user's layer (LLMContextEdit kind :drop, by "user"),
23
+ # saved applied: the view sends it whatever the strategy, none too
24
+ # (LLMContextView#sends?). One edit per id: a drop on an output that
25
+ # holds an edit already (a forget sent whole under none, a staged one, a
26
+ # stale stub) keeps that edit's saved Hash in its +was+, and an undrop
27
+ # puts it back. The outputs are WorkingSet.outputs_of's, the projection's
28
+ # own pairing, so the set and the op never disagree.
29
+ #
30
+ # A pin is the session's (Session#working_set_pins, a WorkingSetPin per
31
+ # item), not an edit: it sends nothing different, so it breaks no cache
32
+ # and survives a strategy switch. Pin and drop exclude each other: a
33
+ # drop asks for an unpin first, and a pin takes a pending drop off (one
34
+ # never sent: free), else asks for an undrop first.
35
+ #
36
+ # An add is a request: the turn-start note asks the model to read the
37
+ # item (read, memory_read, web_fetch) before the task, so its output is
38
+ # a real tool run (stamps, ids, forget, the projection), on both loops.
39
+ # Nothing sent changes until then; it costs the model one step. An item
40
+ # the model has whole already is refused; a stale or forgotten one is
41
+ # its refresh. Pending until the next turn sends the note
42
+ # (WorkingSet.pending_adds); an unadd takes it back.
43
+ #
44
+ # Every op since the last turn is listed in one tail system message
45
+ # (kind "working_set", #record!), so the model learns what changed and
46
+ # doesn't read a dropped file again by reflex. An inverse op before the
47
+ # next turn cancels its op there (a drop then an undrop: nothing sent,
48
+ # nothing noted).
49
+ module WorkingSetSteer
50
+ BY = "user"
51
+ NOTE_KIND = WorkingSet::NOTE_KIND
52
+ # The tail note's ops: [{"op", "kind", "key", "label"}], until the turn
53
+ # that sends it (the chat loop's kept entries leave the key out).
54
+ OPS = WorkingSet::NOTE_OPS
55
+ NOTE_OPEN = "[WORKING SET: "
56
+ DROP = "drop"
57
+ UNDROP = "undrop"
58
+ PIN = "pin"
59
+ UNPIN = "unpin"
60
+ ADD = WorkingSet::ADD
61
+ UNADD = "unadd"
62
+ INVERSE = { DROP => UNDROP, UNDROP => DROP, PIN => UNPIN, UNPIN => PIN, ADD => UNADD, UNADD => ADD }.freeze
63
+ MEMORY_PREFIX = "memory:"
64
+ URL = %r{\A[a-z][a-z0-9+.-]*://}i
65
+
66
+ # Why an op can't be done: the one place each refusal's words live
67
+ # (/working-set and forget_outputs add context only).
68
+ class Refused < StandardError
69
+ MESSAGES = {
70
+ dropped: "dropped by the user",
71
+ pinned_by_user: "pinned by the user: it stays in your context",
72
+ unpin_first: "pinned by the user: unpin it first",
73
+ undrop_first: "dropped by the user: undrop it first",
74
+ already_pinned: "already pinned",
75
+ not_pinned: "not pinned",
76
+ not_in_set: "not in the working set",
77
+ only_edited: "nothing to drop: only edited or written (no read of it is sent)",
78
+ derived: "nothing to drop: an older session's output that chi can't name",
79
+ already_dropped: "already dropped",
80
+ not_sent: "nothing to drop: none of it is sent now (forgotten or stubbed)",
81
+ not_dropped: "not dropped",
82
+ no_file: "no such file",
83
+ directory: "a directory: add its files",
84
+ no_memory: "no such memory",
85
+ bad_memory: "not a memory name (no /, \\ or ..)",
86
+ not_http: "not an http(s) URL: web_fetch can't fetch it",
87
+ already_sent: "already in the model's context (sent whole): nothing to add",
88
+ already_added: "already added: the next turn reads it",
89
+ not_added: "not added (no add of it is pending)"
90
+ }.freeze
91
+
92
+ # @return [Symbol] a key of MESSAGES
93
+ attr_reader :reason
94
+
95
+ def initialize(reason)
96
+ @reason = reason
97
+ super(MESSAGES.fetch(reason))
98
+ end
99
+
100
+ # The words for +reason+, for a caller that only reports it.
101
+ def self.text(reason) = MESSAGES.fetch(reason)
102
+ end
103
+
104
+ # What a KEY names: an item's identity (WorkingSet's kind and key) and
105
+ # how replies name it (a file relative to the session's folder).
106
+ Target = Data.define(:kind, :key, :label)
107
+
108
+ # What an op did: the ids it changed, and the chars it frees (a drop)
109
+ # and the tail the server reads again from its earliest entry on.
110
+ # +free+: an undrop that took back a drop never sent (nothing changes),
111
+ # a pin or an unpin (nothing sent changes). +undone+: the pending drop
112
+ # a pin took off (a Done), else nil. +shared_with+: the other memories a
113
+ # memory_read of several names holds in an output the op acted on (the
114
+ # op acts on that output whole, so on them too).
115
+ Done = Data.define(:target, :op, :ids, :freed_chars, :tail_chars, :free, :undone, :shared_with) do
116
+ def initialize(undone: nil, shared_with: [], **fields) = super
117
+ end
118
+
119
+ module_function
120
+
121
+ # A word quoted ("…" or '…'): runs to its closing quote, spaces and
122
+ # all; a backslash escapes the quote or itself. Else a bare word.
123
+ WORD = /\G\s*(?:"((?:[^"\\]|\\.)*)"?|'((?:[^'\\]|\\.)*)'?|(\S+))/m
124
+ # What a quoted word unescapes (\" \' \\); any other backslash stays.
125
+ ESCAPE = /\\(["'\\])/
126
+ # A word that needs quoting on a line: whitespace, or a leading quote.
127
+ QUOTE = /\s|\A["']/
128
+
129
+ # The KEY words of a /working-set line (what follows the verb):
130
+ # whitespace-separated, a quoted word keeps its spaces ("my dir/a.rb").
131
+ # An unclosed quote runs to the end. The web's working_set.js
132
+ # (steerWords) reads lines the same way.
133
+ # @return [Array<String>]
134
+ def words(text)
135
+ text.to_s.scan(WORD).filter_map do |double, single, bare|
136
+ quoted = double || single
137
+ quoted ? quoted.gsub(ESCAPE, '\\1') : bare
138
+ end.reject(&:empty?)
139
+ end
140
+
141
+ # +word+ as a line names it: quoted when it has whitespace or starts
142
+ # with a quote (#words reads it back), else as it is.
143
+ def quote(word)
144
+ word = word.to_s
145
+ word.match?(QUOTE) ? %("#{word.gsub(/["\\]/) { |char| "\\#{char}" }}") : word
146
+ end
147
+
148
+ # +word+ (a path, absolute or the session's folder's; memory:NAME; a
149
+ # URL) as a Target.
150
+ def target(word, cwd:)
151
+ word = word.to_s.strip
152
+ if word.start_with?(MEMORY_PREFIX)
153
+ name = word.delete_prefix(MEMORY_PREFIX).strip
154
+ Target.new(kind: WorkingSet::MEMORY, key: name, label: "memory #{name}")
155
+ elsif word.match?(URL)
156
+ url = word.gsub(/\s+/, " ")
157
+ Target.new(kind: WorkingSet::URL, key: url, label: url)
158
+ else
159
+ path = File.expand_path(word, cwd.to_s.empty? ? Dir.pwd : cwd)
160
+ Target.new(kind: WorkingSet::FILE, key: path, label: relative(path, cwd))
161
+ end
162
+ end
163
+
164
+ # A path as replies show it: below +cwd+ relative, else as it is.
165
+ def relative(path, cwd)
166
+ base = "#{cwd.to_s.chomp("/")}/"
167
+ !cwd.to_s.empty? && path.start_with?(base) ? path.delete_prefix(base) : path
168
+ end
169
+
170
+ # Drops +target+'s content outputs that are still sent under
171
+ # +strategy+ (the next turn's: LLMContextView's form), saving an applied
172
+ # :drop on each.
173
+ # +pins+: the session's (a pinned item asks for an unpin first).
174
+ # @return [Done]
175
+ # @raise [Refused]
176
+ def drop(conversation, target, strategy:, cwd:, pins: [], now: Time.now.utc.iso8601(3))
177
+ raise Refused, :unpin_first if pinned?(pins, target)
178
+
179
+ outputs = WorkingSet.outputs_of(conversation, kind: target.kind, key: target.key, strategy: strategy, cwd: cwd)
180
+ raise Refused, (known?(conversation, target, strategy, cwd) ? :only_edited : :not_in_set) if outputs.empty?
181
+
182
+ named = outputs.reject(&:derived)
183
+ raise Refused, :derived if named.empty?
184
+ raise Refused, :already_dropped if named.all? { |output| drop_of(conversation, output) }
185
+
186
+ sent = named.select(&:sent)
187
+ raise Refused, :not_sent if sent.empty?
188
+
189
+ edits = sent.map do |output|
190
+ was = LLMContextEdit.on(conversation[output.index])[output.id]
191
+ [output.index, LLMContextEdit.new(id: output.id, kind: LLMContextEdit::DROP, note: "", by: BY, staged_at: now,
192
+ applied_at: now, was: was&.to_h)]
193
+ end
194
+ freed, tail = LLMContextApply.weigh(conversation, layers: layers(strategy), edits: edits)
195
+ edits.each { |index, edit| LLMContextEdit.store(conversation[index], edit) }
196
+ Done.new(target: target, op: DROP, ids: edits.map { |_, edit| edit.id }, freed_chars: freed, tail_chars: tail,
197
+ free: false, shared_with: shared_with(sent))
198
+ end
199
+
200
+ # Takes +target+'s drops off, putting back what each output held before
201
+ # it (+was+). +sent+: whether the drop reached a request (else the
202
+ # undrop is free).
203
+ # @return [Done]
204
+ # @raise [Refused]
205
+ def undrop(conversation, target, strategy:, cwd:, sent: true)
206
+ outputs = WorkingSet.outputs_of(conversation, kind: target.kind, key: target.key, strategy: strategy, cwd: cwd)
207
+ dropped = outputs.filter_map { |output| (edit = drop_of(conversation, output)) && [output, edit] }
208
+ if dropped.empty?
209
+ raise Refused, (outputs.empty? && !known?(conversation, target, strategy, cwd) ? :not_in_set : :not_dropped)
210
+ end
211
+
212
+ dropped.each do |output, edit|
213
+ entry = conversation[output.index]
214
+ LLMContextEdit.remove(entry, output.id)
215
+ before = edit.was && LLMContextEdit.from_h(output.id, edit.was)
216
+ LLMContextEdit.store(entry, before) if before
217
+ end
218
+ first = dropped.map { |output, _| output.index }.min
219
+ view = LLMContextView.new(strategy: strategy)
220
+ tail = sent ? LLMContextView.chars(view.messages(conversation)[first..]) : 0
221
+ Done.new(target: target, op: UNDROP, ids: dropped.map { |output, _| output.id }, freed_chars: 0, tail_chars: tail,
222
+ free: !sent, shared_with: shared_with(dropped.map(&:first)))
223
+ end
224
+
225
+ # Pins +target+ (an item the set holds; an edit-only one too: its later
226
+ # reads are covered). A drop of it still pending (made since the last
227
+ # turn) is taken off first, free, as Done#undone; a sent one asks for
228
+ # an undrop first. The caller adds the pin (#pinning).
229
+ # @param pins [Array<WorkingSetPin>] the session's
230
+ # @return [Done] +ids+: the outputs the pin holds now
231
+ # @raise [Refused]
232
+ def pin(conversation, pins, target, strategy:, cwd:)
233
+ outputs = WorkingSet.outputs_of(conversation, kind: target.kind, key: target.key, strategy: strategy, cwd: cwd)
234
+ raise Refused, :not_in_set if outputs.empty? && !known?(conversation, target, strategy, cwd)
235
+ raise Refused, :already_pinned if pinned?(pins, target)
236
+
237
+ undone = nil
238
+ if outputs.any? { |output| drop_of(conversation, output) }
239
+ raise Refused, :undrop_first unless pending?(conversation, DROP, target)
240
+
241
+ undone = undrop(conversation, target, strategy: strategy, cwd: cwd, sent: false)
242
+ end
243
+ named = outputs.reject(&:derived)
244
+ Done.new(target: target, op: PIN, ids: named.map(&:id), freed_chars: 0, tail_chars: 0, free: true, undone: undone,
245
+ shared_with: shared_with(named))
246
+ end
247
+
248
+ # Unpins +target+: the model may forget it again. Only the pins are
249
+ # asked (a rollback may have taken the item out of the set).
250
+ # @return [Done]
251
+ # @raise [Refused]
252
+ def unpin(pins, target)
253
+ raise Refused, :not_pinned unless pinned?(pins, target)
254
+
255
+ Done.new(target: target, op: UNPIN, ids: [], freed_chars: 0, tail_chars: 0, free: true)
256
+ end
257
+
258
+ # Adds +target+: the next turn's note asks the model to read it. +found+
259
+ # answers whether the item exists (a file, a memory; the caller's
260
+ # look). Refused for an item the model is sent whole now (live, a read
261
+ # of it sent), a dropped one (undrop it), and one added already.
262
+ # @param found [#call] target → nil when it exists, else a Refused reason
263
+ # @return [Done]
264
+ # @raise [Refused]
265
+ def add(conversation, target, strategy:, cwd:, found: ->(_) {})
266
+ raise Refused, :already_added if pending?(conversation, ADD, target)
267
+
268
+ reason = target.kind == WorkingSet::URL && !target.key.match?(%r{\Ahttps?://}i) ? :not_http : found.call(target)
269
+ raise Refused, reason if reason
270
+
271
+ outputs = WorkingSet.outputs_of(conversation, kind: target.kind, key: target.key, strategy: strategy, cwd: cwd)
272
+ raise Refused, :undrop_first if outputs.any? { |output| drop_of(conversation, output) }
273
+ raise Refused, :already_sent if outputs.any?(&:sent) && live?(conversation, target, strategy, cwd)
274
+
275
+ Done.new(target: target, op: ADD, ids: [], freed_chars: 0, tail_chars: 0, free: true)
276
+ end
277
+
278
+ # Takes back +target+'s pending add (never sent: free).
279
+ # @return [Done]
280
+ # @raise [Refused]
281
+ def unadd(conversation, target)
282
+ raise Refused, :not_added unless pending?(conversation, ADD, target)
283
+
284
+ Done.new(target: target, op: UNADD, ids: [], freed_chars: 0, tail_chars: 0, free: true)
285
+ end
286
+
287
+ # +pins+ after +done+ (a pin or an unpin). A new frozen Array.
288
+ # @return [Array<WorkingSetPin>]
289
+ def pinning(pins, done)
290
+ pin = WorkingSetPin.new(kind: done.target.kind, key: done.target.key)
291
+ rest = Array(pins).reject { |held| held == pin }
292
+ (done.op == PIN ? rest + [pin] : rest).freeze
293
+ end
294
+
295
+ # The other memory names +outputs+ hold (WorkingSet::Output#shared_with).
296
+ def shared_with(outputs) = outputs.flat_map(&:shared_with).uniq
297
+
298
+ # Whether +pins+ hold +target+.
299
+ def pinned?(pins, target)
300
+ Array(pins).any? { |pin| pin.kind == target.kind && pin.key == target.key }
301
+ end
302
+
303
+ # +output+'s drop (an LLMContextEdit), or nil.
304
+ def drop_of(conversation, output)
305
+ edit = LLMContextEdit.on(conversation[output.index])[output.id]
306
+ edit if edit&.kind == LLMContextEdit::DROP
307
+ end
308
+
309
+ # Whether the set holds +target+ at all (an edit-only file does).
310
+ def known?(conversation, target, strategy, cwd)
311
+ WorkingSet.build(conversation, cwd: cwd, strategy: strategy).items.any? do |item|
312
+ item.kind == target.kind && item.key == target.key
313
+ end
314
+ end
315
+
316
+ # Whether the set holds +target+ live (not stale, forgotten, dropped).
317
+ def live?(conversation, target, strategy, cwd)
318
+ WorkingSet.build(conversation, cwd: cwd, strategy: strategy).items.any? do |item|
319
+ item.kind == target.kind && item.key == target.key && item.state == WorkingSet::LIVE
320
+ end
321
+ end
322
+
323
+ def layers(strategy) = LLMContextView.new(strategy: strategy).layers
324
+
325
+ # ── the tail note ───────────────────────────────────────────────────
326
+
327
+ # The index of the note the next turn will send, or nil
328
+ # (WorkingSet.pending_note_index).
329
+ def pending_note_index(messages) = WorkingSet.pending_note_index(messages)
330
+
331
+ # The ops the pending note lists ([{"op", "kind", "key", "label"}]).
332
+ def pending_ops(messages) = WorkingSet.pending_ops(messages)
333
+
334
+ # Whether +target+'s +op+ is pending (made since the last turn).
335
+ def pending?(messages, op, target)
336
+ pending_ops(messages).any? { |entry| same?(entry, op, target) }
337
+ end
338
+
339
+ # +messages+ with +done+'s op in the pending note: its inverse there
340
+ # cancels both (the note goes when nothing is left), else it is added
341
+ # (the note appended when there is none). A new Array; the note is a new
342
+ # Hash.
343
+ # @return [Array<Hash>]
344
+ def record!(messages, done)
345
+ index = pending_note_index(messages)
346
+ ops = index ? pending_ops(messages) : []
347
+ inverse = ops.index { |entry| same?(entry, INVERSE.fetch(done.op), done.target) }
348
+ if inverse
349
+ ops = ops.reject.with_index { |_, i| i == inverse }
350
+ else
351
+ ops = ops.reject { |entry| same?(entry, done.op, done.target) }
352
+ ops += [{ "op" => done.op, "kind" => done.target.kind, "key" => done.target.key, "label" => done.target.label }]
353
+ end
354
+ messages = messages.dup
355
+ if ops.empty?
356
+ messages.delete_at(index) if index
357
+ elsif index
358
+ messages[index] = note(ops)
359
+ else
360
+ messages << note(ops)
361
+ end
362
+ messages
363
+ end
364
+
365
+ def same?(entry, op, target)
366
+ entry["op"] == op && entry["kind"] == target.kind && entry["key"] == target.key
367
+ end
368
+
369
+ # The note listing +ops+: a tail system message, escaped like a context
370
+ # note (Prompt::ESCAPED_KINDS: it carries paths and URLs).
371
+ def note(ops)
372
+ drops = labels(ops, DROP)
373
+ undrops = labels(ops, UNDROP)
374
+ parts = []
375
+ unless drops.empty?
376
+ parts << "the user dropped #{drops.join(", ")} from your context: you are sent a stub instead of " \
377
+ "#{drops.size == 1 ? "its outputs" : "their outputs"}; read #{drops.size == 1 ? "it" : "them"} again " \
378
+ "only if the task needs it"
379
+ end
380
+ unless undrops.empty?
381
+ parts << "the user brought back #{undrops.join(", ")}: #{undrops.size == 1 ? "its outputs are" : "their outputs are"} " \
382
+ "sent whole again"
383
+ end
384
+ pins = labels(ops, PIN)
385
+ parts << "the user pinned #{pins.join(", ")}: forget_outputs won't forget #{pins.size == 1 ? "it" : "them"}" unless pins.empty?
386
+ unpins = labels(ops, UNPIN)
387
+ parts << "the user unpinned #{unpins.join(", ")}: you may forget #{unpins.size == 1 ? "it" : "them"} again" unless unpins.empty?
388
+ adds = ops.select { |entry| entry["op"] == ADD }
389
+ unless adds.empty?
390
+ parts << "the user added #{adds.map { |entry| entry["label"] }.join(", ")} to your context: before anything " \
391
+ "else, load #{adds.size == 1 ? "it" : "them"} with #{adds.map { |entry| call_of(entry) }.join(", ")}"
392
+ end
393
+ { role: "system", kind: NOTE_KIND, content: "#{NOTE_OPEN}since your last turn #{parts.join("; ")}.]", OPS => ops }
394
+ end
395
+
396
+ def labels(ops, op) = ops.select { |entry| entry["op"] == op }.map { |entry| entry["label"] }
397
+
398
+ # The tool call that loads an added item, as the note asks for it: a
399
+ # file by its absolute path (a read resolves a relative one against
400
+ # the process's folder, which may not be the session's).
401
+ def call_of(entry)
402
+ tool = { WorkingSet::MEMORY => "memory_read", WorkingSet::URL => "web_fetch" }.fetch(entry["kind"], "read")
403
+ %(#{tool} "#{entry["key"]}")
404
+ end
405
+
406
+ # "12.4k tokens", chars/4.
407
+ def k(chars)
408
+ tokens = (chars / TokenUsage::CHARS_PER_TOKEN).ceil
409
+ tokens >= 1000 ? format("%.1fk tokens", tokens / 1000.0) : "#{tokens} tokens"
410
+ end
411
+
412
+ private_class_method :relative, :shared_with, :drop_of, :known?, :live?, :layers, :same?, :labels, :call_of
413
+ end
414
+ end