@possumtech/rummy 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/.env.example +1 -0
  2. package/FIDELITY_CONTRACT.md +172 -0
  3. package/migrations/001_initial_schema.sql +3 -3
  4. package/package.json +1 -1
  5. package/src/agent/AgentLoop.js +1 -2
  6. package/src/agent/ContextAssembler.js +2 -0
  7. package/src/agent/KnownStore.js +1 -2
  8. package/src/agent/ResponseHealer.js +54 -1
  9. package/src/agent/TurnExecutor.js +51 -6
  10. package/src/agent/XmlParser.js +150 -41
  11. package/src/agent/known_store.sql +18 -11
  12. package/src/hooks/PluginContext.js +8 -2
  13. package/src/hooks/RummyContext.js +6 -3
  14. package/src/hooks/ToolRegistry.js +23 -27
  15. package/src/plugins/ask_user/ask_user.js +2 -2
  16. package/src/plugins/ask_user/ask_userDoc.js +4 -2
  17. package/src/plugins/budget/README.md +6 -4
  18. package/src/plugins/budget/budget.js +29 -9
  19. package/src/plugins/cp/cp.js +5 -5
  20. package/src/plugins/cp/cpDoc.js +0 -8
  21. package/src/plugins/engine/engine.sql +1 -1
  22. package/src/plugins/env/env.js +4 -4
  23. package/src/plugins/env/envDoc.js +2 -2
  24. package/src/plugins/file/file.js +2 -7
  25. package/src/plugins/get/get.js +31 -10
  26. package/src/plugins/get/getDoc.js +26 -37
  27. package/src/plugins/helpers.js +2 -2
  28. package/src/plugins/instructions/instructions.js +6 -5
  29. package/src/plugins/instructions/preamble.md +41 -33
  30. package/src/plugins/known/known.js +17 -16
  31. package/src/plugins/known/knownDoc.js +1 -13
  32. package/src/plugins/mv/mv.js +6 -6
  33. package/src/plugins/mv/mvDoc.js +2 -13
  34. package/src/plugins/previous/previous.js +10 -14
  35. package/src/plugins/progress/progress.js +22 -5
  36. package/src/plugins/prompt/prompt.js +14 -11
  37. package/src/plugins/rm/rm.js +4 -4
  38. package/src/plugins/rm/rmDoc.js +4 -8
  39. package/src/plugins/rpc/rpc.js +1 -1
  40. package/src/plugins/set/set.js +10 -12
  41. package/src/plugins/set/setDoc.js +4 -4
  42. package/src/plugins/sh/sh.js +4 -4
  43. package/src/plugins/sh/shDoc.js +2 -2
  44. package/src/plugins/skill/skill.js +2 -1
  45. package/src/plugins/summarize/summarize.js +2 -2
  46. package/src/plugins/summarize/summarizeDoc.js +9 -10
  47. package/src/plugins/telemetry/telemetry.js +36 -11
  48. package/src/plugins/think/think.js +2 -1
  49. package/src/plugins/think/thinkDoc.js +3 -5
  50. package/src/plugins/unknown/unknown.js +21 -14
  51. package/src/plugins/unknown/unknownDoc.js +2 -6
  52. package/src/plugins/update/update.js +2 -2
  53. package/src/plugins/update/updateDoc.js +9 -6
  54. package/src/sql/functions/slugify.js +13 -1
  55. package/src/sql/v_model_context.sql +3 -3
@@ -154,6 +154,11 @@ export default class XmlParser {
154
154
  const commands = [];
155
155
  const warnings = [];
156
156
  const textChunks = [];
157
+
158
+ // Pre-flight: balance unclosed attribute quotes that would otherwise
159
+ // cause htmlparser2 to consume the rest of input as a single attribute
160
+ // value, silently dropping every subsequent tool call.
161
+ const balanced = XmlParser.#balanceAttrQuotes(normalized, warnings);
157
162
  let current = null;
158
163
  let ended = false;
159
164
  let capped = false;
@@ -162,35 +167,50 @@ export default class XmlParser {
162
167
  {
163
168
  onopentag(name, attrs) {
164
169
  if (capped) return;
165
- if (!ALL_TOOLS.has(name)) {
166
- if (current) {
170
+
171
+ if (current) {
172
+ // Empty-body case: current tool opened but got no text
173
+ // content before a new tag. The model likely meant current
174
+ // to self-close but typed it in paired form, or emitted a
175
+ // mismatched close tag that htmlparser2 silently dropped.
176
+ // Close current, open new.
177
+ const hasBody = current.rawBody.trim() !== "";
178
+ const hasNestedOpens = (current.nested || []).length > 0;
179
+ if (!hasBody && !hasNestedOpens && ALL_TOOLS.has(name)) {
180
+ warnings.push(
181
+ `Unclosed <${current.name}> before <${name}> — recovered`,
182
+ );
183
+ commands.push(
184
+ resolveCommand(current.name, current.attrs, current.rawBody),
185
+ );
186
+ current = null;
187
+ } else {
188
+ // Nested tag inside a body with content — treat as body
189
+ // text. Tool bodies are opaque: the model writing a plan
190
+ // with <get/> in it, SEARCH/REPLACE in <set>, or XML
191
+ // examples in <known> all need to survive intact. Track
192
+ // nested opens on a stack so matching closes pop off and
193
+ // orphan closes (typos) still trigger recovery.
167
194
  const attrStr = Object.entries(attrs)
168
- .map(([k, v]) => v === "" ? k : `${k}="${v}"`)
195
+ .map(([k, v]) => (v === "" ? k : `${k}="${v}"`))
169
196
  .join(" ");
170
197
  current.rawBody += attrStr
171
198
  ? `<${name} ${attrStr}>`
172
199
  : `<${name}>`;
200
+ current.nested ||= [];
201
+ current.nested.push(name);
202
+ return;
173
203
  }
174
- return;
175
204
  }
176
205
 
177
- // Known tool opened while another is still open — close the old one.
178
- if (current) {
179
- warnings.push(
180
- `Unclosed <${current.name}> before <${name}> — recovered`,
181
- );
182
- commands.push(
183
- resolveCommand(current.name, current.attrs, current.rawBody),
184
- );
185
- }
206
+ if (!ALL_TOOLS.has(name)) return;
186
207
 
187
208
  if (commands.length >= XmlParser.MAX_COMMANDS) {
188
209
  capped = true;
189
- current = null;
190
210
  return;
191
211
  }
192
212
 
193
- current = { name, attrs, rawBody: "" };
213
+ current = { name, attrs, rawBody: "", nested: [] };
194
214
  },
195
215
 
196
216
  ontext(text) {
@@ -204,28 +224,52 @@ export default class XmlParser {
204
224
 
205
225
  onclosetag(name, isImplied) {
206
226
  if (capped) return;
207
- if (current && name === current.name) {
208
- if (ended) {
209
- warnings.push(`Unclosed <${name}> tagcontent captured anyway`);
227
+
228
+ if (current) {
229
+ // Matching nested close pop stack, keep as text.
230
+ const nested = current.nested;
231
+ if (
232
+ nested.length > 0 &&
233
+ nested[nested.length - 1] === name
234
+ ) {
235
+ nested.pop();
236
+ current.rawBody += `</${name}>`;
237
+ return;
210
238
  }
211
- commands.push(
212
- resolveCommand(current.name, current.attrs, current.rawBody),
213
- );
214
- current = null;
215
- } else if (current && ALL_TOOLS.has(name)) {
216
- // Mismatched close tag for a known tool close current tag,
217
- // don't swallow subsequent commands as body text.
218
- warnings.push(
219
- `Mismatched </${name}> closing <${current.name}> — recovered`,
220
- );
221
- commands.push(
222
- resolveCommand(current.name, current.attrs, current.rawBody),
223
- );
224
- current = null;
225
- } else if (current) {
239
+
240
+ // Matching close for outer tool — finalize.
241
+ if (name === current.name && nested.length === 0) {
242
+ if (ended) {
243
+ warnings.push(
244
+ `Unclosed <${name}> tag — content captured anyway`,
245
+ );
246
+ }
247
+ commands.push(
248
+ resolveCommand(current.name, current.attrs, current.rawBody),
249
+ );
250
+ current = null;
251
+ return;
252
+ }
253
+
254
+ // Orphan close for a known tool (likely typo) — recover.
255
+ if (ALL_TOOLS.has(name)) {
256
+ warnings.push(
257
+ `Mismatched </${name}> closing <${current.name}> — recovered`,
258
+ );
259
+ commands.push(
260
+ resolveCommand(current.name, current.attrs, current.rawBody),
261
+ );
262
+ current = null;
263
+ return;
264
+ }
265
+
266
+ // Unknown orphan close — text.
226
267
  current.rawBody += `</${name}>`;
227
- } else if (isImplied && ALL_TOOLS.has(name)) {
228
- // Self-closing tag that htmlparser2 auto-closed
268
+ return;
269
+ }
270
+
271
+ if (isImplied && ALL_TOOLS.has(name)) {
272
+ // Self-closing tag that htmlparser2 auto-closed at top level
229
273
  }
230
274
  },
231
275
 
@@ -240,7 +284,7 @@ export default class XmlParser {
240
284
  },
241
285
  );
242
286
 
243
- parser.write(normalized);
287
+ parser.write(balanced);
244
288
  ended = true;
245
289
  parser.end();
246
290
 
@@ -263,6 +307,36 @@ export default class XmlParser {
263
307
  return { commands, warnings, unparsed };
264
308
  }
265
309
 
310
+ /**
311
+ * Repair a specific malformed-tag pattern: an attribute value opened with
312
+ * `="` that never closes before the next tag. Without repair, htmlparser2
313
+ * consumes the rest of input as one giant attribute value and silently
314
+ * drops every subsequent tool call.
315
+ *
316
+ * Pattern matched: <TAG ... ATTR="text-with-no-quote</NEXT>
317
+ * Repair: <TAG ... ATTR="text-with-no-quote"></NEXT>
318
+ *
319
+ * Conservative — only triggers when the value contains no quote, no `>`,
320
+ * and is followed by another tag opening or close. Well-formed input is
321
+ * untouched.
322
+ */
323
+ static #balanceAttrQuotes(content, warnings) {
324
+ let fixes = 0;
325
+ const repaired = content.replace(
326
+ /(<\w+\s[^<>]*?\w+=")([^"<>]*?)(<\/?\w+)/g,
327
+ (_, opening, value, nextTag) => {
328
+ fixes++;
329
+ return `${opening}${value}">${nextTag}`;
330
+ },
331
+ );
332
+ if (fixes > 0) {
333
+ warnings.push(
334
+ `Repaired ${fixes} malformed attribute(s) — close all attribute values with a quote.`,
335
+ );
336
+ }
337
+ return repaired;
338
+ }
339
+
266
340
  /**
267
341
  * Normalize native tool call formats to rummy XML.
268
342
  * Models sometimes emit their training-format tool calls instead of
@@ -276,12 +350,30 @@ export default class XmlParser {
276
350
  );
277
351
 
278
352
  // Qwen/gemma: <|tool_call>call:NAME{key:"value"}<tool_call|>
353
+ // NAME may be namespaced with any of /, :, or . separators
354
+ // (e.g. `rummy.nvim/get`, `rummy:get`) — extract the trailing word
355
+ // sequence as the tool name. Value forms observed in the wild:
356
+ // key="v" / key:"v" / key:v (unquoted) / key:<|"|>v<|"|> (gemma chat-quotes)
279
357
  result = result.replace(
280
- /<\|tool_call>call:(\w+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
281
- (_, name, params) => {
282
- if (!ALL_TOOLS.has(name)) return _;
283
- const valueMatch = params.match(/["']([^"']+)["']/);
284
- const body = valueMatch?.[1] || "";
358
+ /<\|tool_call>call:([\w.:/-]+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
359
+ (match, qualifiedName, params) => {
360
+ const name = qualifiedName.match(/\w+$/)?.[0] ?? qualifiedName;
361
+ if (!ALL_TOOLS.has(name)) {
362
+ return `<error>Unknown tool '${qualifiedName}' in <|tool_call> format. Use XML tool commands listed above.</error>`;
363
+ }
364
+ const valueMatch = params.match(
365
+ /[=:]\s*(?:<\|"\|>([^<]*?)<\|"\|>|"([^"]*)"|'([^']*)'|([^,}]+))/,
366
+ );
367
+ const body = (
368
+ valueMatch?.[1] ??
369
+ valueMatch?.[2] ??
370
+ valueMatch?.[3] ??
371
+ valueMatch?.[4] ??
372
+ ""
373
+ ).trim();
374
+ if (!body) {
375
+ return `<error>Could not extract argument from <|tool_call> ${match}. Use XML format like <${name}>value</${name}>.</error>`;
376
+ }
285
377
  return `<${name}>${body}</${name}>`;
286
378
  },
287
379
  );
@@ -319,6 +411,23 @@ export default class XmlParser {
319
411
  },
320
412
  );
321
413
 
414
+ // Catch-all: any remaining <|tool_call> tokens are malformed native
415
+ // attempts (no {} block, missing close, wrong shape entirely). Replace
416
+ // each with an <error> so the model gets feedback on its next turn and
417
+ // learns to switch to XML. Lazy-match up to the next native close, the
418
+ // next XML close tag, or end of input — preserves any trailing valid XML.
419
+ // Error body must NOT contain literal <get>/<set>/etc. — those would
420
+ // re-enter the parser as phantom tool calls. Describe the format in
421
+ // prose instead and point at the tool docs above.
422
+ result = result.replace(
423
+ /<\|tool_call>[\s\S]*?(?:<\|?tool_call\|?>|<\/\w+>|$)/g,
424
+ () =>
425
+ "<error>Native tool call format not supported. Use the XML tool commands listed above (e.g. a get tag with a path attribute, or a set tag with path and body).</error>",
426
+ );
427
+
428
+ // Strip any orphan chat-format quote tokens left after replacement.
429
+ result = result.replace(/<\|"\|>/g, '"');
430
+
322
431
  return result;
323
432
  }
324
433
  }
@@ -59,7 +59,7 @@ WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
59
59
  -- PREP: promote_path
60
60
  UPDATE known_entries
61
61
  SET
62
- fidelity = 'full'
62
+ fidelity = 'promoted'
63
63
  , status = 200
64
64
  , turn = :turn
65
65
  , updated_at = CURRENT_TIMESTAMP
@@ -68,7 +68,7 @@ WHERE run_id = :run_id AND path = :path;
68
68
  -- PREP: demote_path
69
69
  UPDATE known_entries
70
70
  SET
71
- fidelity = 'archive'
71
+ fidelity = 'archived'
72
72
  , updated_at = CURRENT_TIMESTAMP
73
73
  WHERE run_id = :run_id AND path = :path;
74
74
 
@@ -111,7 +111,7 @@ WHERE run_id = :run_id AND path = :path;
111
111
  -- PREP: promote_by_pattern
112
112
  UPDATE known_entries
113
113
  SET
114
- fidelity = 'full'
114
+ fidelity = 'promoted'
115
115
  , status = 200
116
116
  , turn = :turn
117
117
  , updated_at = CURRENT_TIMESTAMP
@@ -123,7 +123,7 @@ WHERE
123
123
  -- PREP: demote_by_pattern
124
124
  UPDATE known_entries
125
125
  SET
126
- fidelity = 'archive'
126
+ fidelity = 'archived'
127
127
  , updated_at = CURRENT_TIMESTAMP
128
128
  WHERE
129
129
  run_id = :run_id
@@ -163,27 +163,34 @@ WHERE
163
163
  AND (:body IS NULL OR hedsearch(:body, body));
164
164
 
165
165
  -- PREP: restore_summarized_prompts
166
- -- Restore prompt entries demoted to summary by a recovery phase that was
166
+ -- Restore prompt entries demoted by a recovery phase that was
167
167
  -- interrupted (e.g. server crash). Safe to call unconditionally at loop
168
168
  -- start: if the full prompt would overflow, Prompt Demotion handles it.
169
169
  UPDATE known_entries
170
170
  SET
171
- fidelity = 'full'
171
+ fidelity = 'promoted'
172
172
  , updated_at = CURRENT_TIMESTAMP
173
- WHERE run_id = :run_id AND scheme = 'prompt' AND fidelity = 'summary';
173
+ WHERE run_id = :run_id AND scheme = 'prompt' AND fidelity = 'demoted';
174
174
 
175
175
  -- PREP: demote_turn_entries
176
- -- Demote all full entries from a turn to summary with 413 status.
176
+ -- Demote all promoted entries from a turn for budget claw-back.
177
+ -- Action schemes (set/rm/mv/cp) at status 200 keep their status — those
178
+ -- represent committed side effects (files written/removed) that can't be
179
+ -- clawed back; only the body in context is demoted, not the truth of what
180
+ -- happened. Everything else flips to 413 since promotion was reversed.
177
181
  -- Tokens unchanged — always reports full cost regardless of fidelity.
178
182
  UPDATE known_entries
179
183
  SET
180
- fidelity = 'summary'
181
- , status = 413
184
+ fidelity = 'demoted'
185
+ , status = CASE
186
+ WHEN scheme IN ('set', 'rm', 'mv', 'cp') AND status = 200 THEN 200
187
+ ELSE 413
188
+ END
182
189
  , updated_at = CURRENT_TIMESTAMP
183
190
  WHERE
184
191
  run_id = :run_id
185
192
  AND turn = :turn
186
- AND fidelity = 'full'
193
+ AND fidelity = 'promoted'
187
194
  AND status < 400
188
195
  RETURNING path, tokens;
189
196
 
@@ -69,10 +69,16 @@ export default class PluginContext {
69
69
  this.#hooks.tools.ensureTool(this.#name);
70
70
  }
71
71
 
72
+ // Mark this plugin's tool as hidden from model-facing tool lists.
73
+ // Handler still dispatches if the model emits the tag.
74
+ markHidden() {
75
+ this.#hooks.tools.markHidden(this.#name);
76
+ }
77
+
72
78
  /**
73
79
  * Register a named callback for this plugin.
74
80
  * "handler" registers the tool handler.
75
- * "full"/"summary" register fidelity projections.
81
+ * "promoted"/"demoted" register fidelity projections.
76
82
  * "docs" sets tool documentation.
77
83
  * Everything else resolves to a hook event.
78
84
  */
@@ -82,7 +88,7 @@ export default class PluginContext {
82
88
  this.#hooks.tools.onHandle(this.#name, callback, priority);
83
89
  return;
84
90
  }
85
- if (event === "full" || event === "summary") {
91
+ if (event === "promoted" || event === "demoted") {
86
92
  this.#hooks.tools.onView(this.#name, callback, event);
87
93
  return;
88
94
  }
@@ -103,9 +103,12 @@ export default class RummyContext {
103
103
 
104
104
  async set({ path, body, status = 200, fidelity, attributes } = {}) {
105
105
  if (!path) {
106
- const slugify = (await import("../sql/functions/slugify.js")).default;
107
- const base = slugify(body || "");
108
- path = `known://${base || Date.now()}`;
106
+ path = await this.entries.slugPath(
107
+ this.runId,
108
+ "known",
109
+ body || "",
110
+ attributes?.summary,
111
+ );
109
112
  }
110
113
  await this.entries.upsert(
111
114
  this.runId,
@@ -32,12 +32,21 @@ export default class ToolRegistry {
32
32
  #tools = new Map();
33
33
  #handlers = new Map();
34
34
  #views = new Map();
35
+ #hidden = new Set();
35
36
 
36
37
  ensureTool(scheme) {
37
38
  if (this.#tools.has(scheme)) return;
38
39
  this.#tools.set(scheme, Object.freeze({}));
39
40
  }
40
41
 
42
+ // Mark a tool as hidden — handler still dispatches if the model emits the
43
+ // tag, but the tool is excluded from all model-facing tool lists. Used for
44
+ // legacy/internal schemes (e.g. <known>, <unknown>) we want to retire
45
+ // without deleting.
46
+ markHidden(scheme) {
47
+ this.#hidden.add(scheme);
48
+ }
49
+
41
50
  get(name) {
42
51
  return this.#tools.get(name);
43
52
  }
@@ -53,7 +62,7 @@ export default class ToolRegistry {
53
62
  list.sort((a, b) => a.priority - b.priority);
54
63
  }
55
64
 
56
- onView(scheme, fn, fidelity = "full") {
65
+ onView(scheme, fn, fidelity = "promoted") {
57
66
  if (!this.#views.has(scheme)) this.#views.set(scheme, new Map());
58
67
  this.#views.get(scheme).set(fidelity, fn);
59
68
  }
@@ -67,32 +76,11 @@ export default class ToolRegistry {
67
76
  );
68
77
  }
69
78
 
70
- const attrs =
71
- typeof entry.attributes === "string"
72
- ? JSON.parse(entry.attributes)
73
- : entry.attributes;
74
- const summary = typeof attrs?.summary === "string" ? attrs.summary : null;
75
-
76
- const fidelity = entry.fidelity || "full";
79
+ const fidelity = entry.fidelity || "promoted";
77
80
  const fn = fidelityMap.get(fidelity);
78
- if (!fn) {
79
- // No view for this fidelity — fall back on model-authored summary
80
- return summary || "";
81
- }
82
-
83
- const body = await fn(entry);
84
-
85
- // Prepend summary keywords above plugin output at summary fidelity
86
- if (fidelity === "summary" && summary && body) {
87
- return `${summary}\n${body}`;
88
- }
89
-
90
- // Fall back to summary attribute when plugin returns empty
91
- if (fidelity === "summary" && summary && !body) {
92
- return summary;
93
- }
81
+ if (!fn) return "";
94
82
 
95
- return body;
83
+ return fn(entry);
96
84
  }
97
85
 
98
86
  hasView(scheme) {
@@ -113,15 +101,23 @@ export default class ToolRegistry {
113
101
  return sortByPriority([...this.#tools.keys()]);
114
102
  }
115
103
 
104
+ // Names advertised to the model — registered tools minus hidden ones.
105
+ // Use this anywhere a tool list is shown to the model.
106
+ get advertisedNames() {
107
+ return sortByPriority(
108
+ [...this.#tools.keys()].filter((n) => !this.#hidden.has(n)),
109
+ );
110
+ }
111
+
116
112
  /**
117
113
  * Compute the active tool set for a loop.
118
- * All exclusions — mode, flags — handled here. One mechanism.
114
+ * All exclusions — mode, flags, hidden — handled here. One mechanism.
119
115
  */
120
116
  resolveForLoop(
121
117
  mode,
122
118
  { noInteraction = false, noWeb = false, noProposals = false } = {},
123
119
  ) {
124
- const excluded = new Set();
120
+ const excluded = new Set(this.#hidden);
125
121
  if (mode === "ask") excluded.add("sh");
126
122
  if (noInteraction) excluded.add("ask_user");
127
123
  if (noWeb) excluded.add("search");
@@ -7,8 +7,8 @@ export default class AskUser {
7
7
  this.#core = core;
8
8
  core.registerScheme();
9
9
  core.on("handler", this.handler.bind(this));
10
- core.on("full", this.full.bind(this));
11
- core.on("summary", this.summary.bind(this));
10
+ core.on("promoted", this.full.bind(this));
11
+ core.on("demoted", this.summary.bind(this));
12
12
  core.filter("instructions.toolDocs", async (docsMap) => {
13
13
  docsMap.ask_user = docs;
14
14
  return docsMap;
@@ -2,13 +2,15 @@
2
2
  // Text goes to the model. Rationale stays in source.
3
3
  // Changing ANY line requires reading ALL rationales first.
4
4
  const LINES = [
5
- ['## <ask_user question="[Question?]">[option1; option2; ...]</ask_user>'],
5
+ [
6
+ '## <ask_user question="[Question?]">[option1; option2; ...]</ask_user> - Ask the user a question',
7
+ ],
6
8
  [
7
9
  "* YOU SHOULD use for decisions, preferences, or approvals the user must make",
8
10
  "Positive framing. Shows what ask_user IS for.",
9
11
  ],
10
12
  [
11
- "* YOU SHOULD use <get> to find information before asking the user",
13
+ "* YOU SHOULD use <get></get> to find information before asking the user",
12
14
  "Gentle redirect. Encourages self-sufficiency.",
13
15
  ],
14
16
  [
@@ -15,9 +15,11 @@ tools run uninterrupted. Enforcement happens at boundaries.
15
15
  the incoming prompt). Model runs in the headroom.
16
16
 
17
17
  2. **Post-dispatch Turn Demotion**: after all tools dispatch, check
18
- context. If over ceiling → demote ALL entries from this turn to
19
- summary (every scheme except `budget`). Write `budget://` entry
20
- listing what was demoted. Model sees it next turn and adapts.
18
+ context. If over ceiling → demote ALL entries from this turn
19
+ (every scheme except `budget`/`system`/`prompt`/`instructions`,
20
+ and 4xx error states stay promoted). Write `budget://` entry with
21
+ directive to demote irrelevant entries and promote fewer next time.
22
+ Model sees it next turn and adapts.
21
23
 
22
24
  3. **LLM rejection** (`isContextExceeded`): turn-1 token estimate
23
25
  drift causes LLM to reject. Same demotion pattern.
@@ -36,4 +38,4 @@ tools run uninterrupted. Enforcement happens at boundaries.
36
38
 
37
39
  - **Hook**: `hooks.budget.enforce` — pre-LLM ceiling check.
38
40
  - **Scheme**: `budget://` — logging category, model-visible. `onView`
39
- renders body at all fidelity levels (summary shows full content).
41
+ renders body at all fidelity levels (demoted shows full content).
@@ -82,22 +82,42 @@ export default class Budget {
82
82
  turn,
83
83
  });
84
84
 
85
- // Also summarize the prompt
85
+ // Also demote the prompt
86
86
  const promptRow = rows.find((r) => r.scheme === "prompt");
87
87
  if (promptRow) {
88
- await store.setFidelity(runId, promptRow.path, "summary");
88
+ await store.setFidelity(runId, promptRow.path, "demoted");
89
89
  }
90
90
 
91
- // Write budget entry
91
+ // Rewrite get-result bodies — the get handler claimed "promoted" success
92
+ // before this panic ran. Without rewriting, the model reads conflicting
93
+ // signals next turn (status=413 but body says "promoted").
94
+ for (const entry of demotedEntries) {
95
+ if (!entry.path.startsWith("get://")) continue;
96
+ await db.resolve_known_entry.run({
97
+ run_id: runId,
98
+ path: entry.path,
99
+ body: `Demoted by budget. See budget://${loopId}/${turn}.`,
100
+ status: 413,
101
+ });
102
+ }
103
+
104
+ // Write budget entry — terse, actionable. Path list dropped since
105
+ // demoted entries already render at fidelity="demoted" in <knowns>/<files>.
106
+ // "tokens remaining" dropped too — the number was over-optimistic (it
107
+ // treated re-demoted files as freeing their full-body tokens when their
108
+ // demoted-view renderings return to baseline). Model reads the truthful
109
+ // remaining in next turn's progress line.
110
+ //
111
+ // The 50% rule is the key directive: it forces the model to sum
112
+ // promotion costs (which is the behavior we want), and the threshold
113
+ // gives a concrete ceiling for the next try. Twofer — abiding by the
114
+ // rule requires budget awareness as a side effect.
92
115
  const ceiling = Math.floor(contextSize * CEILING_RATIO);
93
116
  const totalDemoted = demotedEntries.reduce((s, r) => s + r.tokens, 0);
94
- const pathList = demotedEntries
95
- .map((r) => `${r.path} (${r.tokens} tokens)`)
96
- .join("\n");
97
117
  const body = [
98
- `Error 413: Context overflowed by ${postBudget.overflow} tokens.`,
99
- `${demotedEntries.length} entries (${totalDemoted} tokens total) demoted. Budget: ${ceiling} tokens.`,
100
- pathList,
118
+ `413 Token Budget Error: overflowed by ${postBudget.overflow} tokens. Token Budget: ${ceiling}.`,
119
+ `Your ${demotedEntries.length} promotions from last turn (${totalDemoted} tokens total) were demoted to fit.`,
120
+ `Required: sum the tokens="N" of your promotions and new entries before emitting. A single turn must add no more than 50% of remaining Token Budget.`,
101
121
  ].join("\n");
102
122
 
103
123
  await store.upsert(runId, turn, `budget://${loopId}/${turn}`, body, 413, {
@@ -8,8 +8,8 @@ export default class Cp {
8
8
  this.#core = core;
9
9
  core.registerScheme();
10
10
  core.on("handler", this.handler.bind(this));
11
- core.on("full", this.full.bind(this));
12
- core.on("summary", this.summary.bind(this));
11
+ core.on("promoted", this.full.bind(this));
12
+ core.on("demoted", this.summary.bind(this));
13
13
  core.filter("instructions.toolDocs", async (docsMap) => {
14
14
  docsMap.cp = docs;
15
15
  return docsMap;
@@ -19,7 +19,7 @@ export default class Cp {
19
19
  async handler(entry, rummy) {
20
20
  const { entries: store, sequence: turn, runId, loopId } = rummy;
21
21
  const { path, to } = entry.attributes;
22
- const VALID = { stored: 1, summary: 1, index: 1, full: 1, archive: 1 };
22
+ const VALID = { promoted: 1, demoted: 1, archived: 1 };
23
23
  const fidelity = VALID[entry.attributes.fidelity]
24
24
  ? entry.attributes.fidelity
25
25
  : undefined;
@@ -53,7 +53,7 @@ export default class Cp {
53
53
  return `# cp ${entry.attributes.from || ""} ${entry.attributes.to || ""}`;
54
54
  }
55
55
 
56
- summary(entry) {
57
- return this.full(entry);
56
+ summary() {
57
+ return "";
58
58
  }
59
59
  }
@@ -11,14 +11,6 @@ const LINES = [
11
11
  'Example: <cp path="known://plan_*">known://archive_</cp>',
12
12
  "Glob batch copy across known entries.",
13
13
  ],
14
- [
15
- "* Source path accepts patterns: `src/*.js`, `known://draft_*`",
16
- "Pattern support consistent with get/rm.",
17
- ],
18
- [
19
- "* Use `preview` to check matches before pattern-based bulk copy",
20
- "Safety pattern consistent with rm.",
21
- ],
22
14
  ];
23
15
 
24
16
  export default LINES.map(([text]) => text).join("\n");
@@ -6,7 +6,7 @@ FROM known_entries AS ke
6
6
  JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
7
7
  WHERE
8
8
  ke.run_id = :run_id
9
- AND ke.fidelity IN ('full', 'summary')
9
+ AND ke.fidelity IN ('promoted', 'demoted')
10
10
  AND s.model_visible = 1
11
11
  ORDER BY ke.turn, ke.refs, ke.tokens DESC;
12
12
 
@@ -7,8 +7,8 @@ export default class Env {
7
7
  this.#core = core;
8
8
  core.registerScheme();
9
9
  core.on("handler", this.handler.bind(this));
10
- core.on("full", this.full.bind(this));
11
- core.on("summary", this.summary.bind(this));
10
+ core.on("promoted", this.full.bind(this));
11
+ core.on("demoted", this.summary.bind(this));
12
12
  core.filter("instructions.toolDocs", async (docsMap) => {
13
13
  docsMap.env = docs;
14
14
  return docsMap;
@@ -27,7 +27,7 @@ export default class Env {
27
27
  return `# env ${entry.attributes.command || ""}\n${entry.body}`;
28
28
  }
29
29
 
30
- summary(entry) {
31
- return entry.attributes.command || "";
30
+ summary() {
31
+ return "";
32
32
  }
33
33
  }
@@ -12,11 +12,11 @@ const LINES = [
12
12
  "Git history. Shows env for read-only investigation.",
13
13
  ],
14
14
  [
15
- '* YOU MUST NOT use <env/> to read or list files — use <get path="*" preview/> instead',
15
+ '* YOU MUST NOT use <env></env> to read or list files — use <get path="*"/> instead',
16
16
  "Prevents cat/ls through shell. Forces file access through get.",
17
17
  ],
18
18
  [
19
- "* YOU MUST NOT use <env/> for commands with side effects",
19
+ "* YOU MUST NOT use <env></env> for commands with side effects",
20
20
  "Separates exploration from action. env = observe only.",
21
21
  ],
22
22
  ];