@possumtech/rummy 0.4.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/.env.example +21 -4
  2. package/PLUGINS.md +389 -194
  3. package/README.md +25 -8
  4. package/SPEC.md +850 -373
  5. package/bin/demo.js +166 -0
  6. package/bin/rummy.js +9 -3
  7. package/biome/no-fallbacks.grit +50 -0
  8. package/lang/en.json +2 -2
  9. package/migrations/001_initial_schema.sql +88 -37
  10. package/package.json +6 -4
  11. package/service.js +50 -9
  12. package/src/agent/AgentLoop.js +460 -331
  13. package/src/agent/ContextAssembler.js +4 -2
  14. package/src/agent/Entries.js +655 -0
  15. package/src/agent/ProjectAgent.js +30 -18
  16. package/src/agent/TurnExecutor.js +232 -379
  17. package/src/agent/XmlParser.js +242 -67
  18. package/src/agent/budget.js +56 -0
  19. package/src/agent/errors.js +22 -0
  20. package/src/agent/httpStatus.js +39 -0
  21. package/src/agent/known_checks.sql +8 -4
  22. package/src/agent/known_queries.sql +9 -13
  23. package/src/agent/known_store.sql +275 -118
  24. package/src/agent/materializeContext.js +102 -0
  25. package/src/agent/runs.sql +10 -7
  26. package/src/agent/schemes.sql +14 -3
  27. package/src/agent/turns.sql +9 -9
  28. package/src/hooks/HookRegistry.js +6 -5
  29. package/src/hooks/Hooks.js +44 -3
  30. package/src/hooks/PluginContext.js +35 -21
  31. package/src/{server → hooks}/RpcRegistry.js +2 -1
  32. package/src/hooks/RummyContext.js +140 -37
  33. package/src/hooks/ToolRegistry.js +36 -35
  34. package/src/llm/LlmProvider.js +64 -90
  35. package/src/llm/errors.js +21 -0
  36. package/src/plugins/ask_user/README.md +1 -1
  37. package/src/plugins/ask_user/ask_user.js +37 -12
  38. package/src/plugins/ask_user/ask_userDoc.js +2 -23
  39. package/src/plugins/ask_user/ask_userDoc.md +10 -0
  40. package/src/plugins/budget/README.md +27 -23
  41. package/src/plugins/budget/budget.js +261 -69
  42. package/src/plugins/cp/README.md +2 -2
  43. package/src/plugins/cp/cp.js +31 -13
  44. package/src/plugins/cp/cpDoc.js +2 -23
  45. package/src/plugins/cp/cpDoc.md +7 -0
  46. package/src/plugins/engine/README.md +2 -2
  47. package/src/plugins/engine/engine.sql +4 -4
  48. package/src/plugins/engine/turn_context.sql +10 -10
  49. package/src/plugins/env/README.md +20 -5
  50. package/src/plugins/env/env.js +47 -8
  51. package/src/plugins/env/envDoc.js +2 -23
  52. package/src/plugins/env/envDoc.md +13 -0
  53. package/src/plugins/error/README.md +16 -0
  54. package/src/plugins/error/error.js +151 -0
  55. package/src/plugins/file/README.md +6 -6
  56. package/src/plugins/file/file.js +15 -7
  57. package/src/plugins/get/README.md +1 -1
  58. package/src/plugins/get/get.js +125 -49
  59. package/src/plugins/get/getDoc.js +2 -43
  60. package/src/plugins/get/getDoc.md +36 -0
  61. package/src/plugins/hedberg/README.md +1 -2
  62. package/src/plugins/hedberg/hedberg.js +8 -4
  63. package/src/plugins/hedberg/matcher.js +16 -17
  64. package/src/plugins/hedberg/normalize.js +0 -48
  65. package/src/plugins/helpers.js +43 -3
  66. package/src/plugins/index.js +146 -123
  67. package/src/plugins/instructions/README.md +35 -9
  68. package/src/plugins/instructions/instructions.js +126 -12
  69. package/src/plugins/instructions/instructions.md +25 -0
  70. package/src/plugins/instructions/instructions_104.md +7 -0
  71. package/src/plugins/instructions/instructions_105.md +46 -0
  72. package/src/plugins/instructions/instructions_106.md +0 -0
  73. package/src/plugins/instructions/instructions_107.md +0 -0
  74. package/src/plugins/instructions/instructions_108.md +8 -0
  75. package/src/plugins/instructions/protocol.js +12 -0
  76. package/src/plugins/known/README.md +2 -2
  77. package/src/plugins/known/known.js +77 -45
  78. package/src/plugins/known/knownDoc.js +2 -29
  79. package/src/plugins/known/knownDoc.md +8 -0
  80. package/src/plugins/log/README.md +48 -0
  81. package/src/plugins/log/log.js +109 -0
  82. package/src/plugins/mv/README.md +2 -2
  83. package/src/plugins/mv/mv.js +57 -24
  84. package/src/plugins/mv/mvDoc.js +2 -29
  85. package/src/plugins/mv/mvDoc.md +10 -0
  86. package/src/plugins/ollama/README.md +15 -0
  87. package/src/{llm/OllamaClient.js → plugins/ollama/ollama.js} +40 -18
  88. package/src/plugins/openai/README.md +17 -0
  89. package/src/plugins/openai/openai.js +120 -0
  90. package/src/plugins/openrouter/README.md +27 -0
  91. package/src/plugins/openrouter/openrouter.js +121 -0
  92. package/src/plugins/persona/README.md +20 -0
  93. package/src/plugins/persona/persona.js +9 -16
  94. package/src/plugins/policy/README.md +21 -0
  95. package/src/plugins/policy/policy.js +29 -14
  96. package/src/plugins/prompt/README.md +1 -1
  97. package/src/plugins/prompt/prompt.js +63 -18
  98. package/src/plugins/rm/README.md +1 -1
  99. package/src/plugins/rm/rm.js +58 -14
  100. package/src/plugins/rm/rmDoc.js +2 -24
  101. package/src/plugins/rm/rmDoc.md +13 -0
  102. package/src/plugins/rpc/README.md +2 -2
  103. package/src/plugins/rpc/rpc.js +515 -296
  104. package/src/plugins/set/README.md +1 -1
  105. package/src/plugins/set/set.js +318 -77
  106. package/src/plugins/set/setDoc.js +2 -35
  107. package/src/plugins/set/setDoc.md +22 -0
  108. package/src/plugins/sh/README.md +28 -5
  109. package/src/plugins/sh/sh.js +52 -8
  110. package/src/plugins/sh/shDoc.js +2 -23
  111. package/src/plugins/sh/shDoc.md +13 -0
  112. package/src/plugins/skill/README.md +23 -0
  113. package/src/plugins/skill/skill.js +14 -17
  114. package/src/plugins/stream/README.md +101 -0
  115. package/src/plugins/stream/stream.js +290 -0
  116. package/src/plugins/telemetry/README.md +1 -1
  117. package/src/plugins/telemetry/telemetry.js +148 -74
  118. package/src/plugins/think/README.md +1 -1
  119. package/src/plugins/think/think.js +14 -1
  120. package/src/plugins/think/thinkDoc.js +2 -17
  121. package/src/plugins/think/thinkDoc.md +7 -0
  122. package/src/plugins/unknown/README.md +3 -3
  123. package/src/plugins/unknown/unknown.js +56 -21
  124. package/src/plugins/unknown/unknownDoc.js +2 -25
  125. package/src/plugins/unknown/unknownDoc.md +11 -0
  126. package/src/plugins/update/README.md +1 -1
  127. package/src/plugins/update/update.js +67 -5
  128. package/src/plugins/update/updateDoc.js +2 -27
  129. package/src/plugins/update/updateDoc.md +8 -0
  130. package/src/plugins/xai/README.md +23 -0
  131. package/src/{llm/XaiClient.js → plugins/xai/xai.js} +58 -37
  132. package/src/server/ClientConnection.js +64 -37
  133. package/src/server/SocketServer.js +23 -10
  134. package/src/server/protocol.js +11 -0
  135. package/src/sql/functions/slugify.js +13 -1
  136. package/src/sql/v_model_context.sql +27 -31
  137. package/src/sql/v_run_log.sql +9 -14
  138. package/EXCEPTIONS.md +0 -46
  139. package/src/agent/KnownStore.js +0 -338
  140. package/src/agent/ResponseHealer.js +0 -188
  141. package/src/llm/OpenAiClient.js +0 -100
  142. package/src/llm/OpenRouterClient.js +0 -100
  143. package/src/plugins/budget/recovery.js +0 -47
  144. package/src/plugins/instructions/preamble.md +0 -37
  145. package/src/plugins/performed/README.md +0 -15
  146. package/src/plugins/performed/performed.js +0 -45
  147. package/src/plugins/previous/README.md +0 -16
  148. package/src/plugins/previous/previous.js +0 -60
  149. package/src/plugins/progress/README.md +0 -16
  150. package/src/plugins/progress/progress.js +0 -26
  151. package/src/plugins/summarize/README.md +0 -19
  152. package/src/plugins/summarize/summarize.js +0 -32
  153. package/src/plugins/summarize/summarizeDoc.js +0 -28
@@ -1,189 +1,346 @@
1
- -- PREP: upsert_known_entry
2
- INSERT INTO known_entries (
3
- run_id, loop_id, turn, path, body, status, fidelity, hash
4
- , attributes, tokens, updated_at
1
+ -- PREP: upsert_entry
2
+ -- Content-layer upsert. Returns entry id for the subsequent run_view write.
3
+ -- Null :attributes on UPDATE path means "don't touch existing attributes"
4
+ -- — so UPDATE reads :attributes directly, not excluded.attributes (which
5
+ -- would have been coerced to '{}' by the VALUES clause).
6
+ INSERT INTO entries (
7
+ scope, path, body, attributes, hash, updated_at
5
8
  )
6
9
  VALUES (
7
- :run_id, :loop_id, :turn, :path, :body, :status, :fidelity, :hash
8
- , COALESCE(:attributes, '{}')
9
- , countTokens(:body)
10
- , COALESCE(:updated_at, CURRENT_TIMESTAMP)
10
+ :scope, :path, :body, COALESCE(:attributes, '{}'), :hash
11
+ , CURRENT_TIMESTAMP
11
12
  )
12
- ON CONFLICT (run_id, path) DO UPDATE SET
13
+ ON CONFLICT (scope, path) DO UPDATE SET
13
14
  body = excluded.body
14
- , status = excluded.status
15
- , fidelity = excluded.fidelity
16
- , hash = COALESCE(excluded.hash, known_entries.hash)
17
- , attributes = COALESCE(excluded.attributes, known_entries.attributes)
18
- , loop_id = excluded.loop_id
15
+ , attributes = COALESCE(:attributes, entries.attributes)
16
+ , hash = COALESCE(:hash, entries.hash)
17
+ , updated_at = CURRENT_TIMESTAMP
18
+ RETURNING id;
19
+
20
+ -- PREP: upsert_run_view
21
+ -- View-layer upsert. Called after upsert_entry with the returned entry id.
22
+ INSERT INTO run_views (
23
+ run_id, entry_id, loop_id, turn, state, outcome, visibility, updated_at
24
+ )
25
+ VALUES (
26
+ :run_id, :entry_id, :loop_id, :turn, :state, :outcome, :visibility
27
+ , CURRENT_TIMESTAMP
28
+ )
29
+ ON CONFLICT (run_id, entry_id) DO UPDATE SET
30
+ loop_id = excluded.loop_id
19
31
  , turn = excluded.turn
20
- , tokens = countTokens(excluded.body)
21
- , write_count = known_entries.write_count + 1
22
- , updated_at = COALESCE(excluded.updated_at, CURRENT_TIMESTAMP);
23
-
24
- -- PREP: recount_tokens
25
- UPDATE known_entries
26
- SET tokens = :tokens
27
- WHERE run_id = :run_id AND path = :path;
28
-
29
- -- PREP: get_stale_tokens
30
- SELECT path, body
31
- FROM known_entries
32
- WHERE
33
- run_id = :run_id
34
- AND turn = :turn;
32
+ , state = excluded.state
33
+ , outcome = excluded.outcome
34
+ , visibility = excluded.visibility
35
+ , write_count = run_views.write_count + 1
36
+ , updated_at = CURRENT_TIMESTAMP;
37
+
38
+ -- Helper fragment: "the entry this run's view references at this path".
39
+ -- Every UPDATE/DELETE resolves its target this way so the logic is
40
+ -- correct whether the entry lives in the run's own scope or a shared one.
41
+
42
+ -- PREP: append_entry_body
43
+ -- Streaming entry body growth. Appends a chunk to the existing body.
44
+ UPDATE entries
45
+ SET
46
+ body = body || :chunk
47
+ , updated_at = CURRENT_TIMESTAMP
48
+ WHERE id = (
49
+ SELECT e.id FROM entries AS e
50
+ JOIN run_views AS rv ON rv.entry_id = e.id
51
+ WHERE rv.run_id = :run_id AND e.path = :path
52
+ LIMIT 1
53
+ );
35
54
 
36
55
  -- PREP: delete_known_entry
37
- DELETE FROM known_entries
38
- WHERE run_id = :run_id AND path = :path;
56
+ -- Removes the view only. Entry is left for future GC; may be shared.
57
+ DELETE FROM run_views
58
+ WHERE run_id = :run_id AND entry_id = (
59
+ SELECT e.id FROM entries AS e
60
+ JOIN run_views AS rv ON rv.entry_id = e.id
61
+ WHERE rv.run_id = :run_id AND e.path = :path
62
+ LIMIT 1
63
+ );
39
64
 
40
65
  -- PREP: delete_file_entries_by_pattern
41
- DELETE FROM known_entries
42
- WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
66
+ DELETE FROM run_views
67
+ WHERE run_id = :run_id AND entry_id IN (
68
+ SELECT e.id FROM entries AS e
69
+ JOIN run_views AS rv ON rv.entry_id = e.id
70
+ WHERE
71
+ rv.run_id = :run_id
72
+ AND hedmatch(:pattern, e.path)
73
+ AND e.scheme IS NULL
74
+ );
75
+
76
+ -- PREP: resolve_known_entry_view
77
+ UPDATE run_views
78
+ SET
79
+ state = :state
80
+ , outcome = :outcome
81
+ , updated_at = CURRENT_TIMESTAMP
82
+ WHERE run_id = :run_id AND entry_id = (
83
+ SELECT e.id FROM entries AS e
84
+ JOIN run_views AS rv ON rv.entry_id = e.id
85
+ WHERE rv.run_id = :run_id AND e.path = :path
86
+ LIMIT 1
87
+ );
43
88
 
44
- -- PREP: resolve_known_entry
45
- UPDATE known_entries
89
+ -- PREP: resolve_known_entry_body
90
+ UPDATE entries
46
91
  SET
47
- status = :status
48
- , body = :body
92
+ body = :body
49
93
  , updated_at = CURRENT_TIMESTAMP
50
- WHERE run_id = :run_id AND path = :path;
94
+ WHERE id = (
95
+ SELECT e.id FROM entries AS e
96
+ JOIN run_views AS rv ON rv.entry_id = e.id
97
+ WHERE rv.run_id = :run_id AND e.path = :path
98
+ LIMIT 1
99
+ );
51
100
 
52
- -- PREP: set_file_fidelity
53
- UPDATE known_entries
101
+ -- PREP: set_file_visibility
102
+ UPDATE run_views
54
103
  SET
55
- fidelity = :fidelity
104
+ visibility = :visibility
56
105
  , updated_at = CURRENT_TIMESTAMP
57
- WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
106
+ WHERE run_id = :run_id AND entry_id IN (
107
+ SELECT e.id FROM entries AS e
108
+ JOIN run_views AS rv ON rv.entry_id = e.id
109
+ WHERE
110
+ rv.run_id = :run_id
111
+ AND hedmatch(:pattern, e.path)
112
+ AND e.scheme IS NULL
113
+ );
58
114
 
59
115
  -- PREP: promote_path
60
- UPDATE known_entries
116
+ UPDATE run_views
61
117
  SET
62
- fidelity = 'full'
63
- , status = 200
118
+ visibility = 'visible'
119
+ , state = 'resolved'
120
+ , outcome = NULL
64
121
  , turn = :turn
65
122
  , updated_at = CURRENT_TIMESTAMP
66
- WHERE run_id = :run_id AND path = :path;
123
+ WHERE run_id = :run_id AND entry_id = (
124
+ SELECT e.id FROM entries AS e
125
+ JOIN run_views AS rv ON rv.entry_id = e.id
126
+ WHERE rv.run_id = :run_id AND e.path = :path
127
+ LIMIT 1
128
+ );
67
129
 
68
130
  -- PREP: demote_path
69
- UPDATE known_entries
131
+ UPDATE run_views
70
132
  SET
71
- fidelity = 'archive'
133
+ visibility = 'archived'
72
134
  , updated_at = CURRENT_TIMESTAMP
73
- WHERE run_id = :run_id AND path = :path;
135
+ WHERE run_id = :run_id AND entry_id = (
136
+ SELECT e.id FROM entries AS e
137
+ JOIN run_views AS rv ON rv.entry_id = e.id
138
+ WHERE rv.run_id = :run_id AND e.path = :path
139
+ LIMIT 1
140
+ );
74
141
 
75
- -- PREP: set_fidelity
76
- -- Tokens unchanged — always reflects full body cost.
77
- UPDATE known_entries
142
+ -- PREP: set_visibility
143
+ UPDATE run_views
78
144
  SET
79
- fidelity = :fidelity
145
+ visibility = :visibility
80
146
  , updated_at = CURRENT_TIMESTAMP
81
- WHERE run_id = :run_id AND path = :path;
147
+ WHERE run_id = :run_id AND entry_id = (
148
+ SELECT e.id FROM entries AS e
149
+ JOIN run_views AS rv ON rv.entry_id = e.id
150
+ WHERE rv.run_id = :run_id AND e.path = :path
151
+ LIMIT 1
152
+ );
82
153
 
83
154
  -- PREP: get_entry_body
84
- SELECT body
85
- FROM known_entries
86
- WHERE run_id = :run_id AND path = :path;
155
+ SELECT e.body AS body
156
+ FROM run_views AS rv
157
+ JOIN entries AS e ON e.id = rv.entry_id
158
+ WHERE rv.run_id = :run_id AND e.path = :path;
87
159
 
88
160
  -- PREP: get_entry_state
89
- SELECT status, fidelity, scheme, turn
90
- FROM known_entries
91
- WHERE run_id = :run_id AND path = :path;
161
+ SELECT rv.state, rv.outcome, rv.visibility, e.scheme, rv.turn
162
+ FROM run_views AS rv
163
+ JOIN entries AS e ON e.id = rv.entry_id
164
+ WHERE rv.run_id = :run_id AND e.path = :path;
92
165
 
93
166
  -- PREP: get_file_states_by_pattern
94
- SELECT path, status, fidelity, turn
95
- FROM known_entries
96
- WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL
97
- ORDER BY path;
167
+ SELECT e.path, rv.state, rv.outcome, rv.visibility, rv.turn
168
+ FROM run_views AS rv
169
+ JOIN entries AS e ON e.id = rv.entry_id
170
+ WHERE
171
+ rv.run_id = :run_id
172
+ AND hedmatch(:pattern, e.path)
173
+ AND e.scheme IS NULL
174
+ ORDER BY e.path;
98
175
 
99
176
  -- PREP: update_entry_attributes
100
- UPDATE known_entries
177
+ UPDATE entries
101
178
  SET
102
179
  attributes = json_patch(attributes, :attributes)
103
180
  , updated_at = CURRENT_TIMESTAMP
104
- WHERE run_id = :run_id AND path = :path;
181
+ WHERE id = (
182
+ SELECT e.id FROM entries AS e
183
+ JOIN run_views AS rv ON rv.entry_id = e.id
184
+ WHERE rv.run_id = :run_id AND e.path = :path
185
+ LIMIT 1
186
+ );
105
187
 
106
188
  -- PREP: get_entry_attributes
107
- SELECT attributes
108
- FROM known_entries
109
- WHERE run_id = :run_id AND path = :path;
189
+ SELECT e.attributes AS attributes
190
+ FROM run_views AS rv
191
+ JOIN entries AS e ON e.id = rv.entry_id
192
+ WHERE rv.run_id = :run_id AND e.path = :path;
110
193
 
111
194
  -- PREP: promote_by_pattern
112
- UPDATE known_entries
195
+ UPDATE run_views
113
196
  SET
114
- fidelity = 'full'
115
- , status = 200
197
+ visibility = 'visible'
198
+ , state = 'resolved'
199
+ , outcome = NULL
116
200
  , turn = :turn
117
201
  , updated_at = CURRENT_TIMESTAMP
118
- WHERE
119
- run_id = :run_id
120
- AND hedmatch(:path, path)
121
- AND (:body IS NULL OR hedsearch(:body, body));
202
+ WHERE run_id = :run_id AND entry_id IN (
203
+ SELECT e.id FROM entries AS e
204
+ JOIN run_views AS rv ON rv.entry_id = e.id
205
+ WHERE
206
+ rv.run_id = :run_id
207
+ AND hedmatch(:path, e.path)
208
+ AND (:body IS NULL OR hedsearch(:body, e.body))
209
+ );
122
210
 
123
211
  -- PREP: demote_by_pattern
124
- UPDATE known_entries
212
+ UPDATE run_views
125
213
  SET
126
- fidelity = 'archive'
214
+ visibility = 'archived'
127
215
  , updated_at = CURRENT_TIMESTAMP
128
- WHERE
129
- run_id = :run_id
130
- AND hedmatch(:path, path)
131
- AND (:body IS NULL OR hedsearch(:body, body));
216
+ WHERE run_id = :run_id AND entry_id IN (
217
+ SELECT e.id FROM entries AS e
218
+ JOIN run_views AS rv ON rv.entry_id = e.id
219
+ WHERE
220
+ rv.run_id = :run_id
221
+ AND hedmatch(:path, e.path)
222
+ AND (:body IS NULL OR hedsearch(:body, e.body))
223
+ );
132
224
 
133
225
  -- PREP: get_entries_by_pattern
134
- SELECT path, body, scheme, status, fidelity, tokens, attributes
135
- FROM known_entries
226
+ SELECT
227
+ e.path, e.body, e.scheme, rv.state, rv.outcome, rv.visibility
228
+ , countTokens(e.body) AS tokens, e.attributes
229
+ FROM run_views AS rv
230
+ JOIN entries AS e ON e.id = rv.entry_id
136
231
  WHERE
137
- run_id = :run_id
138
- AND hedmatch(:path, path)
139
- AND (:body IS NULL OR hedsearch(:body, body))
140
- ORDER BY path
232
+ rv.run_id = :run_id
233
+ AND hedmatch(:path, e.path)
234
+ AND (:body IS NULL OR hedsearch(:body, e.body))
235
+ ORDER BY e.path
141
236
  LIMIT
142
237
  COALESCE(:limit, -1)
143
- OFFSET
144
- COALESCE(:offset, 0);
238
+ OFFSET COALESCE(:offset, 0);
145
239
 
146
240
  -- PREP: delete_entries_by_pattern
147
- DELETE FROM known_entries
148
- WHERE
149
- run_id = :run_id
150
- AND hedmatch(:path, path)
151
- AND (:body IS NULL OR hedsearch(:body, body));
241
+ DELETE FROM run_views
242
+ WHERE run_id = :run_id AND entry_id IN (
243
+ SELECT e.id FROM entries AS e
244
+ JOIN run_views AS rv ON rv.entry_id = e.id
245
+ WHERE
246
+ rv.run_id = :run_id
247
+ AND hedmatch(:path, e.path)
248
+ AND (:body IS NULL OR hedsearch(:body, e.body))
249
+ );
152
250
 
153
251
  -- PREP: update_body_by_pattern
154
- UPDATE known_entries
252
+ UPDATE entries
155
253
  SET
156
254
  body = :new_body
157
- , tokens = countTokens(:new_body)
158
- , write_count = write_count + 1
159
255
  , updated_at = CURRENT_TIMESTAMP
160
- WHERE
161
- run_id = :run_id
162
- AND hedmatch(:path, path)
163
- AND (:body IS NULL OR hedsearch(:body, body));
164
-
165
- -- PREP: restore_summarized_prompts
166
- -- Restore prompt entries demoted to summary by a recovery phase that was
167
- -- interrupted (e.g. server crash). Safe to call unconditionally at loop
168
- -- start: if the full prompt would overflow, Prompt Demotion handles it.
169
- UPDATE known_entries
256
+ WHERE id IN (
257
+ SELECT e.id FROM entries AS e
258
+ JOIN run_views AS rv ON rv.entry_id = e.id
259
+ WHERE
260
+ rv.run_id = :run_id
261
+ AND hedmatch(:path, e.path)
262
+ AND (:body IS NULL OR hedsearch(:body, e.body))
263
+ );
264
+
265
+ -- PREP: bump_write_count_by_pattern
266
+ -- Companion to update_body_by_pattern. write_count lives on run_views.
267
+ UPDATE run_views
170
268
  SET
171
- fidelity = 'full'
269
+ write_count = write_count + 1
172
270
  , updated_at = CURRENT_TIMESTAMP
173
- WHERE run_id = :run_id AND scheme = 'prompt' AND fidelity = 'summary';
271
+ WHERE run_id = :run_id AND entry_id IN (
272
+ SELECT e.id FROM entries AS e
273
+ JOIN run_views AS rv ON rv.entry_id = e.id
274
+ WHERE
275
+ rv.run_id = :run_id
276
+ AND hedmatch(:path, e.path)
277
+ AND (:body IS NULL OR hedsearch(:body, e.body))
278
+ );
279
+
280
+ -- PREP: get_turn_demotion_targets
281
+ -- Rows that demote_turn_entries is about to flip. Return shape
282
+ -- matches the old RETURNING (path, tokens) for caller compatibility.
283
+ -- State filter: skip failed/cancelled entries (they're already not
284
+ -- contributing visible context — demoting them would be misleading).
285
+ -- Scheme filter: skip known/unknown — these are the model's deliverables,
286
+ -- not housekeeping. Auto-demoting just-created knowns punishes the
287
+ -- correct Distill+Demote pattern.
288
+ SELECT e.path, countTokens(e.body) AS tokens
289
+ FROM run_views AS rv
290
+ JOIN entries AS e ON e.id = rv.entry_id
291
+ WHERE
292
+ rv.run_id = :run_id
293
+ AND rv.turn = :turn
294
+ AND rv.visibility = 'visible'
295
+ AND rv.state NOT IN ('failed', 'cancelled')
296
+ AND e.scheme NOT IN ('known', 'unknown');
174
297
 
175
298
  -- PREP: demote_turn_entries
176
- -- Demote all full entries from a turn to summary with 413 status.
177
- -- Tokens unchanged always reports full cost regardless of fidelity.
178
- UPDATE known_entries
299
+ -- View-layer only visibility lives on run_views. State untouched.
300
+ -- Call get_turn_demotion_targets first if you need the list of what
301
+ -- was demoted (used by budget plugin for the overflow error body).
302
+ -- Scheme filter mirrors get_turn_demotion_targets — never demote the
303
+ -- model's deliverables (known/unknown) along with housekeeping.
304
+ UPDATE run_views
179
305
  SET
180
- fidelity = 'summary'
181
- , status = 413
306
+ visibility = 'summarized'
182
307
  , updated_at = CURRENT_TIMESTAMP
183
308
  WHERE
184
309
  run_id = :run_id
185
310
  AND turn = :turn
186
- AND fidelity = 'full'
187
- AND status < 400
188
- RETURNING path, tokens;
311
+ AND visibility = 'visible'
312
+ AND state NOT IN ('failed', 'cancelled')
313
+ AND NOT EXISTS (
314
+ SELECT 1
315
+ FROM entries AS e
316
+ WHERE
317
+ e.id = run_views.entry_id
318
+ AND e.scheme IN ('known', 'unknown')
319
+ );
320
+
321
+ -- PREP: get_run_visible_targets
322
+ -- All visible entries across the run, oldest promotion first. Used by
323
+ -- budget postDispatch as the fallback demotion set when this-turn
324
+ -- demotion yields nothing but the packet still overflows (promotions
325
+ -- from prior turns the model forgot to demote themselves).
326
+ SELECT e.path, countTokens(e.body) AS tokens, rv.turn
327
+ FROM run_views AS rv
328
+ JOIN entries AS e ON e.id = rv.entry_id
329
+ WHERE
330
+ rv.run_id = :run_id
331
+ AND rv.visibility = 'visible'
332
+ AND rv.state NOT IN ('failed', 'cancelled')
333
+ ORDER BY rv.turn, e.id;
189
334
 
335
+ -- PREP: demote_run_visible
336
+ -- Broad cross-turn demotion. Separate prep from demote_turn_entries
337
+ -- so the caller's intent (surgical this-turn vs fallback all-visible)
338
+ -- stays explicit.
339
+ UPDATE run_views
340
+ SET
341
+ visibility = 'summarized'
342
+ , updated_at = CURRENT_TIMESTAMP
343
+ WHERE
344
+ run_id = :run_id
345
+ AND visibility = 'visible'
346
+ AND state NOT IN ('failed', 'cancelled');
@@ -0,0 +1,102 @@
1
+ import ContextAssembler from "./ContextAssembler.js";
2
+ import { countTokens } from "./tokens.js";
3
+
4
+ /**
5
+ * Rebuild turn_context from v_model_context, then assemble messages.
6
+ * Called at turn start and again by the budget plugin when it needs a
7
+ * fresh measurement after mutating visibility.
8
+ */
9
+ export default async function materializeContext({
10
+ db,
11
+ hooks,
12
+ runId,
13
+ loopId,
14
+ turn,
15
+ systemPrompt,
16
+ mode,
17
+ toolSet,
18
+ contextSize,
19
+ demoted,
20
+ }) {
21
+ await db.clear_turn_context.run({ run_id: runId, turn });
22
+ const viewRows = await db.get_model_context.all({ run_id: runId });
23
+ // Per-entry token accounting (see SPEC @token_accounting): captured
24
+ // here while we still have the raw body, then merged onto rows after
25
+ // the read-back roundtrip through turn_context.
26
+ const tokenAccounting = new Map();
27
+ for (const row of viewRows) {
28
+ // schemeOf() yields NULL (or "") for bare file paths — translate
29
+ // to "file" so the view lookup finds the file scheme handler.
30
+ const scheme = row.scheme ? row.scheme : "file";
31
+ const attrs = row.attributes ? JSON.parse(row.attributes) : null;
32
+ // Log entries live at log://turn_N/action/slug. Dispatch projection
33
+ // to the action plugin's view (set, update, search, etc.) by
34
+ // extracting the action segment from the path.
35
+ let projectionKey = scheme;
36
+ if (scheme === "log") {
37
+ const m = row.path.match(/^log:\/\/turn_\d+\/([^/]+)\//);
38
+ if (m) projectionKey = m[1];
39
+ }
40
+ const baseEntry = {
41
+ path: row.path,
42
+ scheme,
43
+ body: row.body,
44
+ attributes: attrs,
45
+ category: row.category,
46
+ };
47
+ const visibleProjection = await hooks.tools.view(projectionKey, {
48
+ ...baseEntry,
49
+ visibility: "visible",
50
+ });
51
+ const summarizedProjection = await hooks.tools.view(projectionKey, {
52
+ ...baseEntry,
53
+ visibility: "summarized",
54
+ });
55
+ const vTokens = countTokens(visibleProjection);
56
+ const sTokens = countTokens(summarizedProjection);
57
+ tokenAccounting.set(row.path, { vTokens, sTokens });
58
+ const projectedBody =
59
+ row.visibility === "visible" ? visibleProjection : summarizedProjection;
60
+ await db.insert_turn_context.run({
61
+ run_id: runId,
62
+ loop_id: loopId,
63
+ turn,
64
+ ordinal: row.ordinal,
65
+ path: row.path,
66
+ visibility: row.visibility,
67
+ state: row.state,
68
+ outcome: row.outcome,
69
+ body: projectedBody,
70
+ attributes: row.attributes,
71
+ category: row.category,
72
+ source_turn: row.turn,
73
+ });
74
+ }
75
+ const rows = await db.get_turn_context.all({ run_id: runId, turn });
76
+ for (const row of rows) {
77
+ const t = tokenAccounting.get(row.path);
78
+ if (!t) continue;
79
+ row.vTokens = t.vTokens;
80
+ row.sTokens = t.sTokens;
81
+ row.aTokens = t.vTokens - t.sTokens;
82
+ }
83
+ const lastCtx = await db.get_last_context_tokens.get({ run_id: runId });
84
+ // First turn of a new run has no prior context.
85
+ let lastContextTokens = 0;
86
+ if (lastCtx) lastContextTokens = lastCtx.context_tokens;
87
+
88
+ const messages = await ContextAssembler.assembleFromTurnContext(
89
+ rows,
90
+ {
91
+ type: mode,
92
+ systemPrompt,
93
+ contextSize,
94
+ demoted,
95
+ toolSet,
96
+ lastContextTokens,
97
+ turn,
98
+ },
99
+ hooks,
100
+ );
101
+ return { rows, messages, lastContextTokens };
102
+ }
@@ -44,7 +44,7 @@ SELECT
44
44
  FROM known_entries AS ke
45
45
  WHERE
46
46
  ke.run_id = r.id
47
- AND ke.scheme = 'summarize'
47
+ AND ke.path LIKE 'log://turn_%/update/%'
48
48
  ORDER BY ke.id DESC
49
49
  LIMIT 1
50
50
  ) AS summary
@@ -79,14 +79,17 @@ WHERE id = :run_id
79
79
  RETURNING next_turn - 1 AS turn;
80
80
 
81
81
  -- PREP: fork_known_entries
82
- INSERT INTO known_entries (
83
- run_id, loop_id, turn, path, body, status, fidelity
84
- , hash, attributes, tokens, refs, write_count
82
+ -- Cheap fork: copy only view rows. Entries stay shared between parent
83
+ -- and child. Child's subsequent writes diverge via upsert into a new
84
+ -- run-scoped entry.
85
+ INSERT INTO run_views (
86
+ run_id, entry_id, loop_id, turn, state, outcome, visibility
87
+ , write_count, refs
85
88
  )
86
89
  SELECT
87
- :new_run_id, NULL, turn, path, body, status, fidelity
88
- , hash, attributes, tokens, refs, write_count
89
- FROM known_entries
90
+ :new_run_id, entry_id, NULL, turn, state, outcome, visibility
91
+ , write_count, refs
92
+ FROM run_views
90
93
  WHERE run_id = :parent_run_id;
91
94
 
92
95
  -- PREP: get_active_runs
@@ -1,6 +1,17 @@
1
1
  -- PREP: upsert_scheme
2
- INSERT OR REPLACE INTO schemes (name, model_visible, category)
3
- VALUES (:name, :model_visible, :category);
2
+ INSERT INTO schemes (name, model_visible, category, default_scope, writable_by)
3
+ VALUES (
4
+ :name
5
+ , :model_visible
6
+ , :category
7
+ , COALESCE(:default_scope, 'run')
8
+ , COALESCE(:writable_by, '["model","plugin"]')
9
+ )
10
+ ON CONFLICT (name) DO UPDATE SET
11
+ model_visible = excluded.model_visible
12
+ , category = excluded.category
13
+ , default_scope = excluded.default_scope
14
+ , writable_by = excluded.writable_by;
4
15
 
5
16
  -- PREP: get_all_schemes
6
- SELECT name, model_visible, category FROM schemes;
17
+ SELECT name, model_visible, category, default_scope, writable_by FROM schemes;
@@ -39,12 +39,12 @@ WHERE run_id = :run_id AND context_tokens > 0
39
39
  ORDER BY sequence DESC
40
40
  LIMIT 1;
41
41
 
42
- -- PREP: get_run_log
43
- SELECT ke.path, ke.status, ke.body, ke.attributes
44
- FROM known_entries AS ke
45
- JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
46
- WHERE
47
- ke.run_id = :run_id
48
- AND ke.scheme IS NOT NULL
49
- AND s.category NOT IN ('knowledge')
50
- ORDER BY ke.id;
42
+ -- PREP: get_turns_by_run
43
+ SELECT
44
+ id, run_id, loop_id, sequence, context_tokens, prompt_tokens,
45
+ cached_tokens, completion_tokens, reasoning_tokens, total_tokens, cost
46
+ FROM turns
47
+ WHERE run_id = :run_id
48
+ ORDER BY sequence;
49
+
50
+ -- get_run_log retired — use get_results (v_run_log) instead.
@@ -29,9 +29,8 @@ export default class HookRegistry {
29
29
  await p.callback(rummy);
30
30
  if (this.#debug) {
31
31
  const duration = (performance.now() - start).toFixed(2);
32
- console.log(
33
- `[PIPELINE] Processor ${p.callback.name || "anonymous"} took ${duration}ms`,
34
- );
32
+ const name = p.callback.name ? p.callback.name : "anonymous";
33
+ console.log(`[PIPELINE] Processor ${name} took ${duration}ms`);
35
34
  }
36
35
  }
37
36
  }
@@ -46,7 +45,8 @@ export default class HookRegistry {
46
45
  }
47
46
 
48
47
  async applyFilters(tag, value, ...args) {
49
- const hooks = this.#filters.get(tag) || [];
48
+ const hooks = this.#filters.get(tag);
49
+ if (!hooks) return value;
50
50
  let result = value;
51
51
  for (const h of hooks) {
52
52
  result = await h.callback(result, ...args);
@@ -71,7 +71,8 @@ export default class HookRegistry {
71
71
  }
72
72
 
73
73
  async emitEvent(tag, ...args) {
74
- const hooks = this.#events.get(tag) || [];
74
+ const hooks = this.#events.get(tag);
75
+ if (!hooks) return;
75
76
  for (const h of hooks) {
76
77
  await h.callback(...args);
77
78
  }