samagotchi 0.100.1 → 0.102.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +50 -1
  3. data/bin/chi +14 -10
  4. data/docs/architecture.md +10 -2
  5. data/docs/cli.md +100 -5
  6. data/docs/configuration.md +26 -4
  7. data/docs/internals/bundles.md +1 -1
  8. data/docs/internals/engine.md +1 -0
  9. data/docs/internals/llm-context-forget.md +14 -0
  10. data/docs/internals/prompt-caching.md +8 -1
  11. data/docs/internals/working-set.md +205 -0
  12. data/docs/plugins.md +6 -3
  13. data/docs/sessions.md +28 -11
  14. data/docs/testing.md +8 -1
  15. data/lib/samagotchi/bridge/turn_accumulator.rb +16 -0
  16. data/lib/samagotchi/bridge.rb +9 -3
  17. data/lib/samagotchi/bundle_command/check.rb +5 -1
  18. data/lib/samagotchi/bundle_command/try/command.rb +6 -6
  19. data/lib/samagotchi/bundle_command/try/sandbox.rb +3 -3
  20. data/lib/samagotchi/bundle_command.rb +4 -4
  21. data/lib/samagotchi/bundles/system/manifest.yml +1 -1
  22. data/lib/samagotchi/commands/registry.rb +12 -1
  23. data/lib/samagotchi/config.rb +92 -53
  24. data/lib/samagotchi/engine.rb +153 -4
  25. data/lib/samagotchi/host_keys.rb +35 -18
  26. data/lib/samagotchi/idle_scheduler.rb +16 -1
  27. data/lib/samagotchi/llm_context_edit.rb +32 -10
  28. data/lib/samagotchi/llm_context_forget.rb +12 -3
  29. data/lib/samagotchi/llm_context_view.rb +19 -7
  30. data/lib/samagotchi/memory_bundle/installer.rb +2 -0
  31. data/lib/samagotchi/memory_bundle/manifest.rb +13 -2
  32. data/lib/samagotchi/message_parts.rb +8 -3
  33. data/lib/samagotchi/plugin/context.rb +2 -2
  34. data/lib/samagotchi/prompt.rb +1 -1
  35. data/lib/samagotchi/recap_command.rb +68 -0
  36. data/lib/samagotchi/recap_setup.rb +27 -14
  37. data/lib/samagotchi/send_command.rb +14 -11
  38. data/lib/samagotchi/session.rb +19 -12
  39. data/lib/samagotchi/session_commands.rb +44 -2
  40. data/lib/samagotchi/session_config.rb +411 -0
  41. data/lib/samagotchi/session_setup.rb +9 -9
  42. data/lib/samagotchi/sessions_command.rb +6 -2
  43. data/lib/samagotchi/terminal_ui/attach_launcher.rb +5 -5
  44. data/lib/samagotchi/terminal_ui/attached_loop.rb +2 -1
  45. data/lib/samagotchi/terminal_ui/formatting.rb +10 -3
  46. data/lib/samagotchi/terminal_ui.rb +8 -6
  47. data/lib/samagotchi/testing/bundle.rb +3 -2
  48. data/lib/samagotchi/testing/fake_executable.rb +6 -16
  49. data/lib/samagotchi/testing/fake_executable_launcher +6 -0
  50. data/lib/samagotchi/tool_response.rb +10 -6
  51. data/lib/samagotchi/tool_runner.rb +25 -6
  52. data/lib/samagotchi/version.rb +1 -1
  53. data/lib/samagotchi/web/app.rb +76 -11
  54. data/lib/samagotchi/web/public/app.js +35 -3
  55. data/lib/samagotchi/web/public/chip_popover.js +56 -17
  56. data/lib/samagotchi/web/public/commands.js +8 -2
  57. data/lib/samagotchi/web/public/data.js +7 -0
  58. data/lib/samagotchi/web/public/index.html +145 -1
  59. data/lib/samagotchi/web/public/info_bar.js +16 -4
  60. data/lib/samagotchi/web/public/info_bar_model.js +7 -3
  61. data/lib/samagotchi/web/public/refs.js +10 -1
  62. data/lib/samagotchi/web/public/stage_model.js +6 -5
  63. data/lib/samagotchi/web/public/stage_view.js +34 -2
  64. data/lib/samagotchi/web/public/stream_handlers.js +10 -1
  65. data/lib/samagotchi/web/public/turn_html.js +5 -0
  66. data/lib/samagotchi/web/public/turn_view.js +16 -4
  67. data/lib/samagotchi/web/public/working_set.js +612 -0
  68. data/lib/samagotchi/web/public/working_set_rail.js +278 -0
  69. data/lib/samagotchi/web/public/working_set_view.js +541 -0
  70. data/lib/samagotchi/web/server.rb +6 -4
  71. data/lib/samagotchi/worker.rb +9 -6
  72. data/lib/samagotchi/working_set.rb +238 -0
  73. data/lib/samagotchi/working_set_command.rb +119 -0
  74. data/lib/samagotchi/working_set_steer.rb +250 -0
  75. metadata +11 -2
  76. data/lib/samagotchi/bundle_session_settings.rb +0 -251
@@ -0,0 +1,238 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "tool_response"
4
+ require_relative "tool_ids"
5
+ require_relative "token_usage"
6
+ require_relative "file_ref"
7
+ require_relative "llm_context_edit"
8
+ require_relative "llm_context_stale"
9
+ require_relative "llm_context_view"
10
+ require_relative "tools/memory"
11
+
12
+ module Samagotchi
13
+ # The working set as the model really has it (the web's rail, chip,
14
+ # popover and footprints overlay it, web.working_set): every file,
15
+ # memory and URL the conversation's tool runs touched, each still +live+,
16
+ # +stale+ (a file changed since the model last saw it) or +forgotten+
17
+ # (nothing of it is still sent under the session's current strategy),
18
+ # with what its content outputs cost in tokens as sent. Pure: built from
19
+ # a conversation, its folder, a strategy and a stat, no session, no
20
+ # Engine. See docs/internals/working-set.md.
21
+ #
22
+ # The runs are the ones that pair with their calls for sure
23
+ # (LLMContextStale.paired_runs, the call as dispatched: aliases resolve
24
+ # as the loops resolve them); a "ran as:" corrected call or an output
25
+ # holding the run separator pairs with none and is left out. A failed run
26
+ # (its output an "Error", a denied call's too) and an interrupted one
27
+ # touch nothing.
28
+ #
29
+ # Content outputs (read, memory_read, web_fetch) carry the tokens and
30
+ # decide forgotten; edit, write and memory_write add ops and stamps only
31
+ # (their content is in the call's arguments, which no strategy cuts).
32
+ # Stale: the item's latest run left a stamp (ToolRun#stamp) and the file
33
+ # now differs from it in mtime or size, or is gone; a run with no stamp
34
+ # (an old session's) never makes a file stale.
35
+ module WorkingSet
36
+ FILE = "file"
37
+ MEMORY = "memory"
38
+ URL = "url"
39
+ LIVE = "live"
40
+ STALE = "stale"
41
+ FORGOTTEN = "forgotten"
42
+ # Forgotten, every output of it by the user's drop (WorkingSetSteer).
43
+ DROPPED = "dropped"
44
+
45
+ # One thing the model touched: +kind+ file|memory|url; +key+ a file's
46
+ # absolute path (FileRef), a memory's name, a URL (whitespace
47
+ # collapsed); +state+ live|stale|forgotten|dropped; +tokens+ its content
48
+ # outputs as sent; +ops+ read|edit for a file, read|write for a memory,
49
+ # read for a URL, in first-seen order; +count+ the runs that touched
50
+ # it; +first_id+ the first one's chi id (ToolIds).
51
+ Item = Data.define(:kind, :key, :state, :tokens, :ops, :count, :first_id)
52
+
53
+ # A projection and the conversation's size (entries) it was built
54
+ # from: the web keeps the newer of an event's and a GET's by it.
55
+ Snapshot = Data.define(:items, :at) do
56
+ def empty? = items.empty?
57
+
58
+ # The JSON the event and the route carry: {items: [...], at:}.
59
+ def to_h = { items: items.map(&:to_h), at: at }
60
+ end
61
+
62
+ FILE_OPS = { "read" => "read", "edit" => "edit", "write" => "edit" }.freeze
63
+ CONTENT = %w[read memory_read web_fetch].freeze
64
+ MEMORY_READ = Tools::MemoryRead::NAME
65
+ MEMORY_WRITE = Tools::MemoryWrite::NAME
66
+ WEB_FETCH = "web_fetch"
67
+ # A stale stub that an edit (not a later read) made: the model wrote
68
+ # the file, so what it has of it is still current (llm_context.stale_edits).
69
+ BY_EDIT = /superseded by a later (?:edit|write)\z/
70
+ # A partial memory_read's last part (Tools::MemoryRead.call).
71
+ MISSING = /\AError: memory not found: (.*)\z/m
72
+
73
+ # What one paired run touched: its item's identity and op, whether its
74
+ # output is content, its stored and sent texts, its stamp.
75
+ Touch = Data.define(:kind, :key, :op, :content, :run, :sent, :share)
76
+ private_constant :Touch
77
+
78
+ module_function
79
+
80
+ # @param conversation [Array<Hash>] the stored entries (symbol keys at
81
+ # the top, as Session.load has them)
82
+ # @param cwd [String, nil] the session's folder: a relative path's base
83
+ # @param strategy [Symbol, Array<Symbol>] :none or the layers, as
84
+ # LLMContextView takes it (the session's current one)
85
+ # @param stat [#call] File.stat, or a stand-in (specs)
86
+ # @return [Snapshot]
87
+ def build(conversation, cwd:, strategy:, stat: File.method(:stat))
88
+ conversation = Array(conversation)
89
+ view = LLMContextView.new(strategy: strategy)
90
+ items = all_touches(conversation, cwd, view).group_by { |touch| [touch.kind, touch.key] }.map do |(kind, key), group|
91
+ item(kind, key, group, conversation, view, stat)
92
+ end
93
+ Snapshot.new(items: items, at: conversation.size)
94
+ end
95
+
96
+ # One content output (read, memory_read, web_fetch) of an item: its
97
+ # chi +id+ (ToolIds), the +index+ of the tool_response entry holding
98
+ # it, whether the id is +derived+ (a legacy entry's: nothing can name
99
+ # it) and whether the model is still +sent+ it under the strategy.
100
+ Output = Data.define(:id, :index, :derived, :sent)
101
+
102
+ # The content outputs of the item +kind+/+key+ (#build's identity: a
103
+ # file's absolute path, a memory's name, a URL), in conversation
104
+ # order, by the same pairing #build uses, so what acts on an item's
105
+ # outputs and the projection never disagree. Edit-only items have
106
+ # none. A memory_read of several names is one output of each.
107
+ # @return [Array<Output>]
108
+ def outputs_of(conversation, kind:, key:, strategy:, cwd:)
109
+ conversation = Array(conversation)
110
+ view = LLMContextView.new(strategy: strategy)
111
+ all_touches(conversation, cwd, view).filter_map do |touch|
112
+ next unless touch.content && touch.kind == kind && touch.key == key
113
+
114
+ ref = touch.run.ref
115
+ Output.new(id: ref.id, index: touch.run.index, derived: ref.derived?,
116
+ sent: still_sent?(touch, conversation, view))
117
+ end
118
+ end
119
+
120
+ # Every paired run's touches, with what +view+ sends of its output.
121
+ def all_touches(conversation, cwd, view)
122
+ stored = LLMContextStale.paired_runs(conversation)
123
+ # Under none with no drop the view sends the conversation as it is.
124
+ messages = view.messages(conversation)
125
+ sent = if messages.equal?(conversation)
126
+ {}
127
+ else
128
+ LLMContextStale.paired_runs(messages).to_h { |run| [run.ref.id, run.text] }
129
+ end
130
+ stored.flat_map { |run| touches(run, sent.fetch(run.ref.id, run.text), cwd) }
131
+ end
132
+
133
+ # The items +run+ touched (none when it failed or was interrupted).
134
+ def touches(run, sent, cwd)
135
+ name = run.call[:name].to_s
136
+ return [] if failed?(run.text)
137
+
138
+ base = { run: run, sent: sent, content: CONTENT.include?(name), share: 1 }
139
+ if FILE_OPS.key?(name)
140
+ path = FileRef.for(name, run.call, cwd: cwd)&.path
141
+ path ? [Touch.new(kind: FILE, key: path, op: FILE_OPS[name], **base)] : []
142
+ elsif name == MEMORY_READ
143
+ names = found_names(run)
144
+ names.map { |memory| Touch.new(kind: MEMORY, key: memory, op: "read", **base.merge(share: names.size)) }
145
+ elsif name == MEMORY_WRITE
146
+ memory = run.call[:path].to_s.strip
147
+ memory.empty? ? [] : [Touch.new(kind: MEMORY, key: memory, op: "write", **base)]
148
+ elsif name == WEB_FETCH
149
+ url = run.call[:content].to_s.gsub(/\s+/, " ").strip
150
+ url.empty? ? [] : [Touch.new(kind: URL, key: url, op: "read", **base)]
151
+ else
152
+ []
153
+ end
154
+ end
155
+
156
+ # LLMContextStale's failed rule (an "Error" after the "[name]" lead; a
157
+ # denied call's "Error:" too), or an interrupted run.
158
+ def failed?(text)
159
+ ToolResponse.interrupted_text?(text) || text.sub(ToolResponse::LEAD, "").lstrip.start_with?("Error")
160
+ end
161
+
162
+ # A memory_read's names (Tools::MemoryRead.parse_names: none for the
163
+ # index listing), less the ones its output says weren't found.
164
+ def found_names(run)
165
+ names = Tools::MemoryRead.parse_names(run.call[:content].to_s)
166
+ last = run.text.split(ToolResponse::SEPARATOR).last.to_s
167
+ missing = (match = MISSING.match(last)) ? match[1].split(",").map(&:strip) : []
168
+ names - missing
169
+ end
170
+
171
+ def item(kind, key, group, conversation, view, stat)
172
+ first = group.first
173
+ contents = group.select(&:content)
174
+ tokens = contents.sum { |touch| touch.sent.length / TokenUsage::CHARS_PER_TOKEN / touch.share }.ceil
175
+ state = if contents.any? && contents.none? { |touch| still_sent?(touch, conversation, view) }
176
+ contents.all? { |touch| dropped?(touch, conversation) } ? DROPPED : FORGOTTEN
177
+ elsif kind == FILE && changed?(key, group.last.run, conversation, stat)
178
+ STALE
179
+ else
180
+ LIVE
181
+ end
182
+ Item.new(kind: kind, key: key, state: state, tokens: tokens, ops: group.map(&:op).uniq, count: group.size,
183
+ first_id: first.run.ref.id)
184
+ end
185
+
186
+ # Whether the model is still sent +touch+'s output: whole (with the
187
+ # forget layer's "[#id] " after its lead, when it shows ids), or a
188
+ # stub that keeps some of it: a forget's kept lines, a stale stub an
189
+ # edit made (the model wrote the file).
190
+ def still_sent?(touch, conversation, view)
191
+ stored = touch.run.text
192
+ sent = touch.sent
193
+ return true if sent == stored || sent == with_id(stored, touch.run.ref.id)
194
+
195
+ edit = LLMContextEdit.on(conversation[touch.run.index])[touch.run.ref.id]
196
+ return true unless edit && view.sends?(edit)
197
+
198
+ edit.kind == :forget ? edit.keep? : edit.note.match?(BY_EDIT)
199
+ end
200
+
201
+ # Whether +touch+'s output holds the user's applied drop.
202
+ def dropped?(touch, conversation)
203
+ edit = LLMContextEdit.on(conversation[touch.run.index])[touch.run.ref.id]
204
+ !edit.nil? && edit.kind == LLMContextEdit::DROP && edit.applied?
205
+ end
206
+
207
+ def with_id(text, id)
208
+ match = text.match(ToolResponse::LEAD)
209
+ match ? "#{match[0]}[##{id}] #{match.post_match}" : nil
210
+ end
211
+
212
+ # Whether the file at +path+ differs from +run+'s stamp (the item's
213
+ # latest run): no stamp, not changed.
214
+ def changed?(path, run, conversation, stat)
215
+ stamp = stamp_of(conversation[run.index], run.ref.run)
216
+ return false unless stamp
217
+
218
+ now = stat.call(path)
219
+ mtime_ns = (now.mtime.to_i * 1_000_000_000) + now.mtime.nsec
220
+ mtime_ns != field(stamp, :mtime_ns) || now.size != field(stamp, :size)
221
+ rescue SystemCallError
222
+ # Gone (or unreadable): not what the model saw.
223
+ true
224
+ end
225
+
226
+ # +entry+'s tool_stamps at the run's place, a Hash with symbol keys
227
+ # (live) or string keys (after a reload); nil when none.
228
+ def stamp_of(entry, place)
229
+ stamps = entry[:tool_stamps]
230
+ stamp = stamps.is_a?(Array) ? stamps[place] : nil
231
+ stamp.is_a?(Hash) && field(stamp, :mtime_ns) ? stamp : nil
232
+ end
233
+
234
+ def field(hash, key) = hash.key?(key) ? hash[key] : hash[key.to_s]
235
+
236
+ private_class_method :all_touches, :touches, :failed?, :found_names, :item, :still_sent?, :dropped?, :with_id, :changed?, :stamp_of, :field
237
+ end
238
+ end
@@ -0,0 +1,119 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "working_set"
4
+ require_relative "working_set_steer"
5
+
6
+ module Samagotchi
7
+ # /working-set: with no arguments the session's working set (WorkingSet,
8
+ # as the web's list shows it: each item, what was done to it, its state
9
+ # and tokens, then the changes the next turn starts with); `drop KEY…`
10
+ # and `undrop KEY…` steer it (WorkingSetSteer). KEY: a path (absolute or
11
+ # the session's folder's), memory:NAME, a URL.
12
+ #
13
+ # An op runs between turns (mid_turn :queue: a line sent while a turn
14
+ # runs waits for its end), on a copy of the session's messages that
15
+ # replaces them, so the next turn's first request is the first with it.
16
+ class WorkingSetCommand
17
+ NAME = "/working-set"
18
+ USAGE = "usage: #{NAME} [drop KEY… | undrop KEY…] (KEY: a path, memory:NAME or a URL)".freeze
19
+ VERBS = [WorkingSetSteer::DROP, WorkingSetSteer::UNDROP].freeze
20
+ OP_WORDS = { WorkingSetSteer::DROP => "dropped", WorkingSetSteer::UNDROP => "undropped" }.freeze
21
+
22
+ # @param engine [Engine] #session, #working_set_strategy,
23
+ # #clone_messages, #replace_messages
24
+ # @param save [#call] saves the session after a change
25
+ def initialize(engine:, save:)
26
+ @engine = engine
27
+ @save = save
28
+ end
29
+
30
+ # @param args [String] what follows /working-set
31
+ # @return [Array(String, Boolean, Symbol)] the reply, whether the
32
+ # conversation changed, and :ok or :error (every key refused)
33
+ # @raise [ArgumentError] a line that isn't one (its message and USAGE)
34
+ def run(args)
35
+ verb, *words = args.to_s.split
36
+ return [listing, false, :ok] if verb.nil?
37
+ raise ArgumentError, USAGE unless VERBS.include?(verb) && !words.empty?
38
+
39
+ session = @engine.session or raise ArgumentError, "no session yet"
40
+ cwd = session.working_directory
41
+ strategy = @engine.working_set_strategy
42
+ messages = @engine.clone_messages(session.messages)
43
+ lines = []
44
+ done = 0
45
+ words.each do |word|
46
+ target = WorkingSetSteer.target(word, cwd: cwd)
47
+ result = if verb == WorkingSetSteer::DROP
48
+ WorkingSetSteer.drop(messages, target, strategy: strategy, cwd: cwd)
49
+ else
50
+ sent = !WorkingSetSteer.pending?(messages, WorkingSetSteer::DROP, target)
51
+ WorkingSetSteer.undrop(messages, target, strategy: strategy, cwd: cwd, sent: sent)
52
+ end
53
+ messages = WorkingSetSteer.record!(messages, result)
54
+ lines << done_line(result)
55
+ done += 1
56
+ rescue WorkingSetSteer::Refused => e
57
+ lines << "#{verb} #{word}: #{e.message}"
58
+ end
59
+ if done.positive?
60
+ @engine.replace_messages(messages)
61
+ @save.call(session)
62
+ end
63
+ [lines.join("\n"), done.positive?, done.positive? ? :ok : :error]
64
+ end
65
+
66
+ # The set under the strategy the next turn runs, and the pending ops.
67
+ def listing
68
+ session = @engine.session
69
+ return "no session yet" unless session
70
+
71
+ messages = Array(session.messages)
72
+ items = WorkingSet.build(messages, cwd: session.working_directory, strategy: @engine.working_set_strategy).items
73
+ return "working set: empty (the files, memories and URLs the turns read show here)\n#{USAGE}" if items.empty?
74
+
75
+ rows = items.map { |item| [name(item, session.working_directory), words(item)] }
76
+ width = rows.map { |row, _| row.length }.max
77
+ lines = ["working set (#{items.size} item#{"s" unless items.size == 1}):"]
78
+ lines.concat(rows.map { |row, text| " #{row.ljust(width)} #{text}" })
79
+ pending = WorkingSetSteer.pending_ops(messages)
80
+ unless pending.empty?
81
+ lines << "at the next turn's start: #{pending.map { |op| "#{OP_WORDS[op["op"]]} #{op["label"]}" }.join(", ")}"
82
+ end
83
+ lines << USAGE
84
+ lines.join("\n")
85
+ end
86
+
87
+ private
88
+
89
+ def name(item, cwd)
90
+ case item.kind
91
+ when WorkingSet::FILE then WorkingSetSteer.target(item.key, cwd: cwd).label
92
+ when WorkingSet::MEMORY then "#{WorkingSetSteer::MEMORY_PREFIX}#{item.key}"
93
+ else item.key
94
+ end
95
+ end
96
+
97
+ def words(item)
98
+ parts = [item.ops.join(", ")]
99
+ parts << "×#{item.count}" if item.count > 1
100
+ parts << item.state unless item.state == WorkingSet::LIVE
101
+ parts << "~#{item.tokens} tokens" if item.tokens.positive?
102
+ parts.join(" · ")
103
+ end
104
+
105
+ def done_line(done)
106
+ what = "#{OP_WORDS[done.op]} #{done.target.label} (#{done.ids.join(", ")})"
107
+ if done.op == WorkingSetSteer::DROP
108
+ frees = done.freed_chars.positive? ? "frees #{WorkingSetSteer.k(done.freed_chars)}" : "frees nothing, the stubs are as long"
109
+ "#{what}: the next turn's first request sends the stubs (#{frees}; the server reads the " \
110
+ "#{WorkingSetSteer.k(done.tail_chars)} after them again)"
111
+ elsif done.free
112
+ "#{what}: its drop was never sent; nothing changes"
113
+ else
114
+ "#{what}: the next request sends it whole again (the server reads the #{WorkingSetSteer.k(done.tail_chars)} " \
115
+ "from it on again)"
116
+ end
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,250 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+ require_relative "llm_context_edit"
5
+ require_relative "llm_context_view"
6
+ require_relative "llm_context_apply"
7
+ require_relative "token_usage"
8
+ require_relative "working_set"
9
+
10
+ module Samagotchi
11
+ # The user steering the working set (/working-set, the web's popover):
12
+ # drop an item (every content output of it still sent goes out of the
13
+ # prompt as a stub) and undrop it. Pure, on a conversation the caller
14
+ # owns (Engine#clone_messages' copies: an op stores new edits Hashes,
15
+ # LLMContextEdit.store) and runs between turns, so the next turn's first
16
+ # request is the first with the change (prompt caching: one break where
17
+ # the new prompt is fresh anyway).
18
+ #
19
+ # A drop is the user's layer (LLMContextEdit kind :drop, by "user"),
20
+ # saved applied: the view sends it whatever the strategy, none too
21
+ # (LLMContextView#sends?). One edit per id: a drop on an output that
22
+ # holds an edit already (a forget sent whole under none, a staged one, a
23
+ # stale stub) keeps that edit's saved Hash in its +was+, and an undrop
24
+ # puts it back. The outputs are WorkingSet.outputs_of's, the projection's
25
+ # own pairing, so the set and the op never disagree.
26
+ #
27
+ # Every op since the last turn is listed in one tail system message
28
+ # (kind "working_set", #record!), so the model learns what changed and
29
+ # doesn't read a dropped file again by reflex. An inverse op before the
30
+ # next turn cancels its op there (a drop then an undrop: nothing sent,
31
+ # nothing noted).
32
+ module WorkingSetSteer
33
+ BY = "user"
34
+ NOTE_KIND = "working_set"
35
+ # The tail note's ops: [{"op", "kind", "key", "label"}], until the turn
36
+ # that sends it (the chat loop's kept entries leave the key out).
37
+ OPS = :working_set_ops
38
+ NOTE_OPEN = "[WORKING SET: "
39
+ DROP = "drop"
40
+ UNDROP = "undrop"
41
+ INVERSE = { DROP => UNDROP, UNDROP => DROP }.freeze
42
+ MEMORY_PREFIX = "memory:"
43
+ URL = %r{\A[a-z][a-z0-9+.-]*://}i
44
+
45
+ # Why an op can't be done: the one place each refusal's words live
46
+ # (/working-set and forget_outputs add context only).
47
+ class Refused < StandardError
48
+ MESSAGES = {
49
+ dropped: "dropped by the user",
50
+ not_in_set: "not in the working set",
51
+ only_edited: "nothing to drop: only edited or written (no read of it is sent)",
52
+ derived: "nothing to drop: an older session's output that chi can't name",
53
+ already_dropped: "already dropped",
54
+ not_sent: "nothing to drop: none of it is sent now (forgotten or stubbed)",
55
+ not_dropped: "not dropped"
56
+ }.freeze
57
+
58
+ # @return [Symbol] a key of MESSAGES
59
+ attr_reader :reason
60
+
61
+ def initialize(reason)
62
+ @reason = reason
63
+ super(MESSAGES.fetch(reason))
64
+ end
65
+
66
+ # The words for +reason+, for a caller that only reports it.
67
+ def self.text(reason) = MESSAGES.fetch(reason)
68
+ end
69
+
70
+ # What a KEY names: an item's identity (WorkingSet's kind and key) and
71
+ # how replies name it (a file relative to the session's folder).
72
+ Target = Data.define(:kind, :key, :label)
73
+
74
+ # What an op did: the ids it changed, and the chars it frees (a drop)
75
+ # and the tail the server reads again from its earliest entry on.
76
+ # +free+: an undrop that took back a drop never sent (nothing changes).
77
+ Done = Data.define(:target, :op, :ids, :freed_chars, :tail_chars, :free)
78
+
79
+ module_function
80
+
81
+ # +word+ (a path, absolute or the session's folder's; memory:NAME; a
82
+ # URL) as a Target.
83
+ def target(word, cwd:)
84
+ word = word.to_s.strip
85
+ if word.start_with?(MEMORY_PREFIX)
86
+ name = word.delete_prefix(MEMORY_PREFIX).strip
87
+ Target.new(kind: WorkingSet::MEMORY, key: name, label: "memory #{name}")
88
+ elsif word.match?(URL)
89
+ url = word.gsub(/\s+/, " ")
90
+ Target.new(kind: WorkingSet::URL, key: url, label: url)
91
+ else
92
+ path = File.expand_path(word, cwd.to_s.empty? ? Dir.pwd : cwd)
93
+ Target.new(kind: WorkingSet::FILE, key: path, label: relative(path, cwd))
94
+ end
95
+ end
96
+
97
+ # A path as replies show it: below +cwd+ relative, else as it is.
98
+ def relative(path, cwd)
99
+ base = "#{cwd.to_s.chomp("/")}/"
100
+ !cwd.to_s.empty? && path.start_with?(base) ? path.delete_prefix(base) : path
101
+ end
102
+
103
+ # Drops +target+'s content outputs that are still sent under
104
+ # +strategy+ (the next turn's: LLMContextView's form), saving an applied
105
+ # :drop on each.
106
+ # @return [Done]
107
+ # @raise [Refused]
108
+ def drop(conversation, target, strategy:, cwd:, now: Time.now.utc.iso8601(3))
109
+ outputs = WorkingSet.outputs_of(conversation, kind: target.kind, key: target.key, strategy: strategy, cwd: cwd)
110
+ raise Refused, (known?(conversation, target, strategy, cwd) ? :only_edited : :not_in_set) if outputs.empty?
111
+
112
+ named = outputs.reject(&:derived)
113
+ raise Refused, :derived if named.empty?
114
+ raise Refused, :already_dropped if named.all? { |output| drop_of(conversation, output) }
115
+
116
+ sent = named.select(&:sent)
117
+ raise Refused, :not_sent if sent.empty?
118
+
119
+ edits = sent.map do |output|
120
+ was = LLMContextEdit.on(conversation[output.index])[output.id]
121
+ [output.index, LLMContextEdit.new(id: output.id, kind: LLMContextEdit::DROP, note: "", by: BY, staged_at: now,
122
+ applied_at: now, was: was&.to_h)]
123
+ end
124
+ freed, tail = LLMContextApply.weigh(conversation, layers: layers(strategy), edits: edits)
125
+ edits.each { |index, edit| LLMContextEdit.store(conversation[index], edit) }
126
+ Done.new(target: target, op: DROP, ids: edits.map { |_, edit| edit.id }, freed_chars: freed, tail_chars: tail,
127
+ free: false)
128
+ end
129
+
130
+ # Takes +target+'s drops off, putting back what each output held before
131
+ # it (+was+). +sent+: whether the drop reached a request (else the
132
+ # undrop is free).
133
+ # @return [Done]
134
+ # @raise [Refused]
135
+ def undrop(conversation, target, strategy:, cwd:, sent: true)
136
+ outputs = WorkingSet.outputs_of(conversation, kind: target.kind, key: target.key, strategy: strategy, cwd: cwd)
137
+ dropped = outputs.filter_map { |output| (edit = drop_of(conversation, output)) && [output, edit] }
138
+ if dropped.empty?
139
+ raise Refused, (outputs.empty? && !known?(conversation, target, strategy, cwd) ? :not_in_set : :not_dropped)
140
+ end
141
+
142
+ dropped.each do |output, edit|
143
+ entry = conversation[output.index]
144
+ LLMContextEdit.remove(entry, output.id)
145
+ before = edit.was && LLMContextEdit.from_h(output.id, edit.was)
146
+ LLMContextEdit.store(entry, before) if before
147
+ end
148
+ first = dropped.map { |output, _| output.index }.min
149
+ view = LLMContextView.new(strategy: strategy)
150
+ tail = sent ? LLMContextView.chars(view.messages(conversation)[first..]) : 0
151
+ Done.new(target: target, op: UNDROP, ids: dropped.map { |output, _| output.id }, freed_chars: 0, tail_chars: tail,
152
+ free: !sent)
153
+ end
154
+
155
+ # +output+'s drop (an LLMContextEdit), or nil.
156
+ def drop_of(conversation, output)
157
+ edit = LLMContextEdit.on(conversation[output.index])[output.id]
158
+ edit if edit&.kind == LLMContextEdit::DROP
159
+ end
160
+
161
+ # Whether the set holds +target+ at all (an edit-only file does).
162
+ def known?(conversation, target, strategy, cwd)
163
+ WorkingSet.build(conversation, cwd: cwd, strategy: strategy).items.any? do |item|
164
+ item.kind == target.kind && item.key == target.key
165
+ end
166
+ end
167
+
168
+ def layers(strategy) = LLMContextView.new(strategy: strategy).layers
169
+
170
+ # ── the tail note ───────────────────────────────────────────────────
171
+
172
+ # The index of the note the next turn will send (one after the last
173
+ # turn's entries: only system messages follow it), or nil.
174
+ def pending_note_index(messages)
175
+ (messages.size - 1).downto(0) do |index|
176
+ message = messages[index]
177
+ return nil unless message[:role].to_s == "system"
178
+ return index if message[:kind].to_s == NOTE_KIND
179
+ end
180
+ nil
181
+ end
182
+
183
+ # The ops the pending note lists ([{"op", "kind", "key", "label"}]).
184
+ def pending_ops(messages)
185
+ index = pending_note_index(messages)
186
+ index ? Array(messages[index][OPS]).select { |op| op.is_a?(Hash) } : []
187
+ end
188
+
189
+ # Whether +target+'s +op+ is pending (made since the last turn).
190
+ def pending?(messages, op, target)
191
+ pending_ops(messages).any? { |entry| same?(entry, op, target) }
192
+ end
193
+
194
+ # +messages+ with +done+'s op in the pending note: its inverse there
195
+ # cancels both (the note goes when nothing is left), else it is added
196
+ # (the note appended when there is none). A new Array; the note is a new
197
+ # Hash.
198
+ # @return [Array<Hash>]
199
+ def record!(messages, done)
200
+ index = pending_note_index(messages)
201
+ ops = index ? pending_ops(messages) : []
202
+ inverse = ops.index { |entry| same?(entry, INVERSE.fetch(done.op), done.target) }
203
+ if inverse
204
+ ops = ops.reject.with_index { |_, i| i == inverse }
205
+ else
206
+ ops = ops.reject { |entry| same?(entry, done.op, done.target) }
207
+ ops += [{ "op" => done.op, "kind" => done.target.kind, "key" => done.target.key, "label" => done.target.label }]
208
+ end
209
+ messages = messages.dup
210
+ if ops.empty?
211
+ messages.delete_at(index) if index
212
+ elsif index
213
+ messages[index] = note(ops)
214
+ else
215
+ messages << note(ops)
216
+ end
217
+ messages
218
+ end
219
+
220
+ def same?(entry, op, target)
221
+ entry["op"] == op && entry["kind"] == target.kind && entry["key"] == target.key
222
+ end
223
+
224
+ # The note listing +ops+: a tail system message, escaped like a context
225
+ # note (Prompt::ESCAPED_KINDS: it carries paths and URLs).
226
+ def note(ops)
227
+ drops = ops.select { |entry| entry["op"] == DROP }.map { |entry| entry["label"] }
228
+ undrops = ops.select { |entry| entry["op"] == UNDROP }.map { |entry| entry["label"] }
229
+ parts = []
230
+ unless drops.empty?
231
+ parts << "the user dropped #{drops.join(", ")} from your context: you are sent a stub instead of " \
232
+ "#{drops.size == 1 ? "its outputs" : "their outputs"}; read #{drops.size == 1 ? "it" : "them"} again " \
233
+ "only if the task needs it"
234
+ end
235
+ unless undrops.empty?
236
+ parts << "the user brought back #{undrops.join(", ")}: #{undrops.size == 1 ? "its outputs are" : "their outputs are"} " \
237
+ "sent whole again"
238
+ end
239
+ { role: "system", kind: NOTE_KIND, content: "#{NOTE_OPEN}since your last turn #{parts.join("; ")}.]", OPS => ops }
240
+ end
241
+
242
+ # "12.4k tokens", chars/4.
243
+ def k(chars)
244
+ tokens = (chars / TokenUsage::CHARS_PER_TOKEN).ceil
245
+ tokens >= 1000 ? format("%.1fk tokens", tokens / 1000.0) : "#{tokens} tokens"
246
+ end
247
+
248
+ private_class_method :relative, :drop_of, :known?, :layers, :same?
249
+ end
250
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: samagotchi
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.100.1
4
+ version: 0.102.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dmitry Dedov
@@ -162,6 +162,7 @@ files:
162
162
  - docs/internals/llm-context-live.md
163
163
  - docs/internals/prompt-caching.md
164
164
  - docs/internals/tool-guardrails.md
165
+ - docs/internals/working-set.md
165
166
  - docs/memory.md
166
167
  - docs/plugins.md
167
168
  - docs/releasing.md
@@ -210,7 +211,6 @@ files:
210
211
  - lib/samagotchi/bundle_command/try/sandbox.rb
211
212
  - lib/samagotchi/bundle_command/try/trace.rb
212
213
  - lib/samagotchi/bundle_needs.rb
213
- - lib/samagotchi/bundle_session_settings.rb
214
214
  - lib/samagotchi/bundles/agents-md/manifest.yml
215
215
  - lib/samagotchi/bundles/agents-md/plugin.rb
216
216
  - lib/samagotchi/bundles/btw/manifest.yml
@@ -492,6 +492,7 @@ files:
492
492
  - lib/samagotchi/queued_command.rb
493
493
  - lib/samagotchi/reasoning_history.rb
494
494
  - lib/samagotchi/reasoning_history_command.rb
495
+ - lib/samagotchi/recap_command.rb
495
496
  - lib/samagotchi/recap_setup.rb
496
497
  - lib/samagotchi/recap_store.rb
497
498
  - lib/samagotchi/redacted_inspect.rb
@@ -516,6 +517,7 @@ files:
516
517
  - lib/samagotchi/session_archive_command.rb
517
518
  - lib/samagotchi/session_chain.rb
518
519
  - lib/samagotchi/session_commands.rb
520
+ - lib/samagotchi/session_config.rb
519
521
  - lib/samagotchi/session_continue.rb
520
522
  - lib/samagotchi/session_delete_command.rb
521
523
  - lib/samagotchi/session_inbox.rb
@@ -568,6 +570,7 @@ files:
568
570
  - lib/samagotchi/testing/boundary.rb
569
571
  - lib/samagotchi/testing/bundle.rb
570
572
  - lib/samagotchi/testing/fake_executable.rb
573
+ - lib/samagotchi/testing/fake_executable_launcher
571
574
  - lib/samagotchi/testing/fake_http.rb
572
575
  - lib/samagotchi/testing/helpers.rb
573
576
  - lib/samagotchi/testing/host.rb
@@ -759,6 +762,9 @@ files:
759
762
  - lib/samagotchi/web/public/vendor/directives/repeat.js
760
763
  - lib/samagotchi/web/public/vendor/directives/unsafe-html.js
761
764
  - lib/samagotchi/web/public/vendor/lit-html.js
765
+ - lib/samagotchi/web/public/working_set.js
766
+ - lib/samagotchi/web/public/working_set_rail.js
767
+ - lib/samagotchi/web/public/working_set_view.js
762
768
  - lib/samagotchi/web/qr.rb
763
769
  - lib/samagotchi/web/server.rb
764
770
  - lib/samagotchi/web/session_hub.rb
@@ -771,6 +777,9 @@ files:
771
777
  - lib/samagotchi/worker_signals.rb
772
778
  - lib/samagotchi/worker_task_wakes.rb
773
779
  - lib/samagotchi/worker_wakes.rb
780
+ - lib/samagotchi/working_set.rb
781
+ - lib/samagotchi/working_set_command.rb
782
+ - lib/samagotchi/working_set_steer.rb
774
783
  - lib/samagotchi/yaml_lines.rb
775
784
  homepage: https://github.com/dm1try/samagotchi
776
785
  licenses: