@possumtech/rummy 2.1.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/.env.example +40 -15
  2. package/.xai.key +1 -0
  3. package/PLUGINS.md +169 -53
  4. package/README.md +38 -32
  5. package/SPEC.md +366 -179
  6. package/bin/digest.js +1097 -0
  7. package/biome/no-fallbacks.grit +2 -2
  8. package/gemini.key +1 -0
  9. package/lang/en.json +10 -1
  10. package/migrations/001_initial_schema.sql +9 -2
  11. package/package.json +19 -8
  12. package/service.js +1 -0
  13. package/src/agent/AgentLoop.js +76 -26
  14. package/src/agent/ContextAssembler.js +2 -0
  15. package/src/agent/Entries.js +238 -60
  16. package/src/agent/ProjectAgent.js +44 -0
  17. package/src/agent/TurnExecutor.js +99 -30
  18. package/src/agent/XmlParser.js +206 -111
  19. package/src/agent/errors.js +35 -0
  20. package/src/agent/known_queries.sql +1 -1
  21. package/src/agent/known_store.sql +3 -42
  22. package/src/agent/materializeContext.js +30 -1
  23. package/src/agent/runs.sql +8 -18
  24. package/src/agent/tokens.js +0 -1
  25. package/src/agent/turns.sql +1 -0
  26. package/src/hooks/Hooks.js +26 -0
  27. package/src/hooks/RummyContext.js +12 -1
  28. package/src/lib/hedberg/README.md +60 -0
  29. package/src/lib/hedberg/hedberg.js +60 -0
  30. package/src/lib/hedberg/marker.js +158 -0
  31. package/src/{plugins → lib}/hedberg/matcher.js +1 -2
  32. package/src/llm/LlmProvider.js +41 -3
  33. package/src/llm/openaiStream.js +17 -0
  34. package/src/plugins/ask_user/ask_user.js +12 -2
  35. package/src/plugins/ask_user/ask_userDoc.md +1 -5
  36. package/src/plugins/budget/README.md +29 -24
  37. package/src/plugins/budget/budget.js +166 -110
  38. package/src/plugins/cli/README.md +3 -4
  39. package/src/plugins/cli/cli.js +31 -5
  40. package/src/plugins/cloudflare/cloudflare.js +136 -0
  41. package/src/plugins/cp/cp.js +41 -4
  42. package/src/plugins/cp/cpDoc.md +5 -6
  43. package/src/plugins/engine/engine.sql +1 -1
  44. package/src/plugins/env/README.md +5 -4
  45. package/src/plugins/env/env.js +7 -4
  46. package/src/plugins/env/envDoc.md +7 -8
  47. package/src/plugins/error/error.js +56 -15
  48. package/src/plugins/file/README.md +12 -3
  49. package/src/plugins/file/file.js +2 -2
  50. package/src/plugins/get/get.js +59 -36
  51. package/src/plugins/get/getDoc.md +10 -34
  52. package/src/plugins/google/google.js +115 -0
  53. package/src/plugins/hedberg/hedberg.js +13 -56
  54. package/src/plugins/helpers.js +66 -12
  55. package/src/plugins/index.js +1 -2
  56. package/src/plugins/instructions/README.md +44 -47
  57. package/src/plugins/instructions/instructions-system.md +44 -0
  58. package/src/plugins/instructions/instructions-user.md +53 -0
  59. package/src/plugins/instructions/instructions.js +58 -189
  60. package/src/plugins/known/README.md +6 -7
  61. package/src/plugins/known/known.js +24 -30
  62. package/src/plugins/log/log.js +41 -32
  63. package/src/plugins/mv/mv.js +40 -1
  64. package/src/plugins/mv/mvDoc.md +1 -8
  65. package/src/plugins/ollama/ollama.js +4 -3
  66. package/src/plugins/openai/openai.js +4 -3
  67. package/src/plugins/openrouter/openrouter.js +14 -4
  68. package/src/plugins/persona/README.md +11 -13
  69. package/src/plugins/persona/default.md +29 -0
  70. package/src/plugins/persona/persona.js +10 -66
  71. package/src/plugins/policy/policy.js +23 -22
  72. package/src/plugins/prompt/README.md +37 -27
  73. package/src/plugins/prompt/prompt.js +13 -19
  74. package/src/plugins/rm/rm.js +18 -0
  75. package/src/plugins/rm/rmDoc.md +5 -6
  76. package/src/plugins/rpc/rpc.js +3 -3
  77. package/src/plugins/set/set.js +205 -323
  78. package/src/plugins/set/setDoc.md +47 -17
  79. package/src/plugins/sh/README.md +6 -5
  80. package/src/plugins/sh/sh.js +8 -5
  81. package/src/plugins/sh/shDoc.md +7 -8
  82. package/src/plugins/skill/README.md +37 -14
  83. package/src/plugins/skill/skill.js +200 -101
  84. package/src/plugins/skill/skillDoc.js +3 -0
  85. package/src/plugins/skill/skillDoc.md +9 -0
  86. package/src/plugins/stream/README.md +7 -6
  87. package/src/plugins/stream/finalize.js +100 -0
  88. package/src/plugins/stream/stream.js +13 -45
  89. package/src/plugins/telemetry/telemetry.js +27 -4
  90. package/src/plugins/think/think.js +2 -3
  91. package/src/plugins/think/thinkDoc.md +2 -4
  92. package/src/plugins/unknown/README.md +1 -1
  93. package/src/plugins/unknown/unknown.js +17 -19
  94. package/src/plugins/update/update.js +4 -51
  95. package/src/plugins/update/updateDoc.md +21 -6
  96. package/src/plugins/xai/xai.js +68 -102
  97. package/src/plugins/yolo/yolo.js +102 -75
  98. package/src/sql/functions/hedmatch.js +1 -1
  99. package/src/sql/functions/hedreplace.js +1 -1
  100. package/src/sql/functions/hedsearch.js +1 -1
  101. package/src/sql/functions/slugify.js +16 -2
  102. package/BENCH_ENVIRONMENT.md +0 -230
  103. package/CLIENT_INTERFACE.md +0 -396
  104. package/last_run.txt +0 -5617
  105. package/scriptify/ask_run.js +0 -77
  106. package/scriptify/cache_probe.js +0 -66
  107. package/scriptify/cache_probe_grok.js +0 -74
  108. package/src/agent/budget.js +0 -33
  109. package/src/agent/config.js +0 -38
  110. package/src/plugins/hedberg/README.md +0 -71
  111. package/src/plugins/hedberg/docs.md +0 -0
  112. package/src/plugins/hedberg/edits.js +0 -55
  113. package/src/plugins/hedberg/normalize.js +0 -17
  114. package/src/plugins/hedberg/sed.js +0 -49
  115. package/src/plugins/instructions/instructions.md +0 -34
  116. package/src/plugins/instructions/instructions_104.md +0 -8
  117. package/src/plugins/instructions/instructions_105.md +0 -39
  118. package/src/plugins/instructions/instructions_106.md +0 -22
  119. package/src/plugins/instructions/instructions_107.md +0 -17
  120. package/src/plugins/instructions/instructions_108.md +0 -0
  121. package/src/plugins/known/knownDoc.js +0 -3
  122. package/src/plugins/known/knownDoc.md +0 -8
  123. package/src/plugins/unknown/unknownDoc.js +0 -3
  124. package/src/plugins/unknown/unknownDoc.md +0 -11
  125. package/turns/cli_1777462658211/turn_001.txt +0 -772
  126. package/turns/cli_1777462658211/turn_002.txt +0 -606
  127. package/turns/cli_1777462658211/turn_003.txt +0 -667
  128. package/turns/cli_1777462658211/turn_004.txt +0 -297
  129. package/turns/cli_1777462658211/turn_005.txt +0 -301
  130. package/turns/cli_1777462658211/turn_006.txt +0 -262
  131. package/turns/cli_1777465095132/turn_001.txt +0 -715
  132. package/turns/cli_1777465095132/turn_002.txt +0 -236
  133. package/turns/cli_1777465095132/turn_003.txt +0 -287
  134. package/turns/cli_1777465095132/turn_004.txt +0 -694
  135. package/turns/cli_1777465095132/turn_005.txt +0 -422
  136. package/turns/cli_1777465095132/turn_006.txt +0 -365
  137. package/turns/cli_1777465095132/turn_007.txt +0 -885
  138. package/turns/cli_1777465095132/turn_008.txt +0 -1277
  139. package/turns/cli_1777465095132/turn_009.txt +0 -736
  140. /package/src/{plugins → lib}/hedberg/patterns.js +0 -0
@@ -76,6 +76,7 @@ export default class TurnExecutor {
76
76
  contextSize,
77
77
  systemPrompt: null,
78
78
  loopPrompt,
79
+ signal,
79
80
  },
80
81
  );
81
82
  await this.#hooks.turn.started.emit({
@@ -88,14 +89,18 @@ export default class TurnExecutor {
88
89
 
89
90
  await this.#hooks.processTurn(rummy);
90
91
 
91
- const systemPrompt =
92
- await this.#hooks.instructions.resolveSystemPrompt(rummy);
92
+ // Run persona feeds the assembly.system chain (persona plugin's
93
+ // participant at priority 150). Loaded once per turn; the system
94
+ // prompt is built directly by the chain — no resolveSystemPrompt
95
+ // indirection.
96
+ const runRow = await this.#db.get_run_by_id.get({ id: currentRunId });
93
97
 
94
98
  const budgetCtx = {
95
99
  runId: currentRunId,
96
100
  loopId: currentLoopId,
97
101
  turn,
98
- systemPrompt,
102
+ systemPrompt: "",
103
+ persona: runRow.persona,
99
104
  mode,
100
105
  toolSet,
101
106
  loopIteration,
@@ -103,6 +108,7 @@ export default class TurnExecutor {
103
108
  const initial = await materializeContext({
104
109
  db: this.#db,
105
110
  hooks: this.#hooks,
111
+ entries: this.#entries,
106
112
  contextSize,
107
113
  ...budgetCtx,
108
114
  });
@@ -113,18 +119,22 @@ export default class TurnExecutor {
113
119
  rowCount: initial.rows.length,
114
120
  });
115
121
 
116
- const budgetResult = await this.#hooks.budget.enforce({
117
- contextSize,
118
- messages: initial.messages,
119
- rows: initial.rows,
120
- lastPromptTokens: initial.lastContextTokens,
121
- ctx: budgetCtx,
122
- rummy,
123
- });
124
- const messages = budgetResult.messages;
125
- const assembledTokens = budgetResult.assembledTokens;
122
+ const dispatchPacket = await this.#hooks.turn.beforeDispatch.filter(
123
+ {
124
+ contextSize,
125
+ messages: initial.messages,
126
+ rows: initial.rows,
127
+ lastPromptTokens: initial.lastContextTokens,
128
+ assembledTokens: 0,
129
+ ok: true,
130
+ overflow: null,
131
+ },
132
+ { rummy, ctx: budgetCtx },
133
+ );
134
+ const messages = dispatchPacket.messages;
135
+ const assembledTokens = dispatchPacket.assembledTokens;
126
136
 
127
- if (!budgetResult.ok) {
137
+ if (!dispatchPacket.ok) {
128
138
  return {
129
139
  turn,
130
140
  turnId: turnRow.id,
@@ -133,11 +143,10 @@ export default class TurnExecutor {
133
143
  updateText: null,
134
144
  assembledTokens,
135
145
  contextSize,
136
- overflow: budgetResult.overflow,
146
+ overflow: dispatchPacket.overflow,
137
147
  };
138
148
  }
139
149
 
140
- const runRow = await this.#db.get_run_by_id.get({ id: currentRunId });
141
150
  const filteredMessages = await this.#hooks.llm.messages.filter(messages, {
142
151
  model: requestedModel,
143
152
  projectId,
@@ -180,6 +189,35 @@ export default class TurnExecutor {
180
189
  contextSize,
181
190
  };
182
191
  }
192
+ // LLM fetch hit its per-call ceiling (provider's
193
+ // AbortSignal.timeout(FETCH_TIMEOUT) fired). Convert to a
194
+ // 504 strike so the loop continues — one timed-out turn is
195
+ // recoverable; MAX_STRIKES in a row abandon at 499. Without
196
+ // this catch the AbortError escapes to AgentLoop's outer
197
+ // catch and the run dies at status=500, losing all prior
198
+ // productive turns. signal.aborted being true means OUR
199
+ // controller fired (drain), not a fetch timeout — re-throw
200
+ // so AgentLoop ends the run cleanly at 499.
201
+ if (err?.name === "TimeoutError" || err?.name === "AbortError") {
202
+ if (signal?.aborted) throw err;
203
+ await this.#hooks.error.log.emit({
204
+ store: this.#entries,
205
+ runId: currentRunId,
206
+ turn,
207
+ loopId: currentLoopId,
208
+ message: `LLM call timed out: ${err.message}`,
209
+ status: 504,
210
+ });
211
+ return {
212
+ turn,
213
+ turnId: turnRow.id,
214
+ recorded: [],
215
+ summaryText: null,
216
+ updateText: null,
217
+ assembledTokens,
218
+ contextSize,
219
+ };
220
+ }
183
221
  throw err;
184
222
  }
185
223
  const result = await this.#hooks.llm.response.filter(rawResult, {
@@ -196,6 +234,10 @@ export default class TurnExecutor {
196
234
  const content = responseMessage?.content ? responseMessage.content : "";
197
235
 
198
236
  const { commands, warnings, unparsed } = XmlParser.parse(content);
237
+ // Parser warnings are recovered emissions — the parser already
238
+ // corrected a mismatched/unclosed tag and produced commands. Log
239
+ // them so the model sees what happened, but don't strike: the
240
+ // turn's productive work is intact.
199
241
  for (const w of warnings) {
200
242
  await this.#hooks.error.log.emit({
201
243
  store: this.#entries,
@@ -204,6 +246,7 @@ export default class TurnExecutor {
204
246
  message: w,
205
247
  loopId: currentLoopId,
206
248
  status: 422,
249
+ soft: true,
207
250
  });
208
251
  }
209
252
  if (commands.length === 0 && unparsed?.trim() && warnings.length === 0) {
@@ -217,6 +260,27 @@ export default class TurnExecutor {
217
260
  });
218
261
  }
219
262
 
263
+ // Contract floor: a turn without <update> is malformed; refuse to
264
+ // honor its side effects. Repetition loops, partial outputs, and
265
+ // other broken responses commonly emit actions without closure;
266
+ // dispatching them anyway lets a broken turn corrupt state. Skip
267
+ // recording AND dispatching when commands are present but no
268
+ // <update> closes the turn — the strike system still fires via
269
+ // turnErrors, model retries cleanly next turn.
270
+ const hasUpdate = commands.some((c) => c.name === "update");
271
+ const skipDispatch = commands.length > 0 && !hasUpdate;
272
+ if (skipDispatch) {
273
+ await this.#hooks.error.log.emit({
274
+ store: this.#entries,
275
+ runId: currentRunId,
276
+ turn,
277
+ loopId: currentLoopId,
278
+ message:
279
+ "Turn rejected: no <update> emitted. Actions are not honored unless the turn ends with an <update>.",
280
+ status: 422,
281
+ });
282
+ }
283
+
220
284
  // Layer plugin reasoning contributions onto the API-provided seed.
221
285
  if (responseMessage) {
222
286
  const seed = responseMessage.reasoning_content
@@ -242,17 +306,19 @@ export default class TurnExecutor {
242
306
  userMsg: userMsg?.content,
243
307
  });
244
308
 
245
- // PHASE 1: RECORD
309
+ // PHASE 1: RECORD (skipped when skipDispatch — broken turn, no side effects)
246
310
  const recorded = [];
247
- for (const cmd of commands) {
248
- const entry = await this.#record(
249
- currentRunId,
250
- currentLoopId,
251
- turn,
252
- mode,
253
- cmd,
254
- );
255
- if (entry) recorded.push(entry);
311
+ if (!skipDispatch) {
312
+ for (const cmd of commands) {
313
+ const entry = await this.#record(
314
+ currentRunId,
315
+ currentLoopId,
316
+ turn,
317
+ mode,
318
+ cmd,
319
+ );
320
+ if (entry) recorded.push(entry);
321
+ }
256
322
  }
257
323
 
258
324
  // PHASE 2: DISPATCH — sequential; abort-after-failure; proposals notify-and-await.
@@ -334,7 +400,7 @@ export default class TurnExecutor {
334
400
  }
335
401
  }
336
402
 
337
- await this.#hooks.budget.postDispatch({
403
+ await this.#hooks.turn.dispatched.emit({
338
404
  contextSize,
339
405
  ctx: budgetCtx,
340
406
  rummy,
@@ -379,8 +445,11 @@ export default class TurnExecutor {
379
445
  if (cmd.path) rawTarget = cmd.path;
380
446
  else if (cmd.command) rawTarget = cmd.command;
381
447
  else if (cmd.question) rawTarget = cmd.question;
382
- // Reject likely reasoning bleed: oversize or control chars in target.
383
- if (rawTarget.length > 512 || /\p{Cc}/u.test(rawTarget)) {
448
+ // Reject reasoning-bleed in path-shaped fields only. cmd.command
449
+ // (sh/env shell scripts) and cmd.question (ask_user prose) are
450
+ // content fields where newlines/tabs/length are legitimate; the
451
+ // slugifier sanitizes them downstream when deriving the log path.
452
+ if (cmd.path && (cmd.path.length > 2048 || /\p{Cc}/u.test(cmd.path))) {
384
453
  const rejectPath = await this.#entries.logPath(
385
454
  runId,
386
455
  turn,
@@ -391,7 +460,7 @@ export default class TurnExecutor {
391
460
  runId,
392
461
  turn,
393
462
  path: rejectPath,
394
- body: `Invalid path: too long or contains non-printing characters`,
463
+ body: "Invalid path.",
395
464
  state: "failed",
396
465
  outcome: "validation",
397
466
  attributes: { action: scheme },
@@ -1,6 +1,47 @@
1
- import { parseEditContent } from "../plugins/hedberg/edits.js";
2
- import { parseJsonEdit } from "../plugins/hedberg/normalize.js";
3
- import { parseSed } from "../plugins/hedberg/sed.js";
1
+ import {
2
+ extractSingleHeredoc,
3
+ parseMarkerBody,
4
+ } from "../lib/hedberg/marker.js";
5
+
6
+ // Edit-marker body opacity. When `#findBodyEnd` is scanning a `<set>`
7
+ // body and hits an opener, jump past the matching closer so tag-shaped
8
+ // content inside the marker (`</set>`, `<get/>`, etc.) doesn't trigger
9
+ // structural recovery.
10
+ //
11
+ // Two opener shapes are recognized for opacity:
12
+ // - `<<IDENT` — current edit syntax (parsed by marker.js).
13
+ // - `<<:::IDENT` — packet-rendering shape (engine emits via
14
+ // plugins/helpers.js). A model copy-pasting the packet shape into
15
+ // a `<set>` body should still get clean opacity even though
16
+ // marker.js routes such bodies to plain-body REPLACE.
17
+ function skipBareMarker(s, pos) {
18
+ const m = s.slice(pos).match(/^<<([A-Z][A-Za-z0-9_]*)/);
19
+ if (!m) return null;
20
+ const ident = m[1];
21
+ const openerEnd = pos + m[0].length;
22
+ const escIdent = ident.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
23
+ const closerRe = new RegExp(`(?<=^|\\s)${escIdent}(?=[\\s<>]|$)`);
24
+ const cm = s.slice(openerEnd).match(closerRe);
25
+ if (!cm) return null;
26
+ return openerEnd + cm.index + cm[0].length;
27
+ }
28
+
29
+ function skipPacketMarker(s, pos) {
30
+ const m = s.slice(pos).match(/^<<:::([A-Za-z_][A-Za-z0-9_./-]*)/);
31
+ if (!m) return null;
32
+ const ident = m[1];
33
+ const openerEnd = pos + m[0].length;
34
+ const escIdent = ident.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
35
+ const closerRe = new RegExp(`:::${escIdent}(?![A-Za-z0-9_])`);
36
+ const cm = s.slice(openerEnd).match(closerRe);
37
+ if (!cm) return null;
38
+ return openerEnd + cm.index + cm[0].length;
39
+ }
40
+
41
+ function skipEditMarker(s, pos) {
42
+ if (s.startsWith("<<:::", pos)) return skipPacketMarker(s, pos);
43
+ return skipBareMarker(s, pos);
44
+ }
4
45
 
5
46
  const STORE_TOOLS = new Set(["get", "rm", "set", "mv", "cp", "search"]);
6
47
  export const ALL_TOOLS = new Set([
@@ -14,69 +55,43 @@ export const ALL_TOOLS = new Set([
14
55
 
15
56
  // Per-tool resolution: missing canonical attribute is filled silently from the body.
16
57
  function resolveCommand(name, a, rawBody) {
58
+ // Generic heredoc affordance: any non-`<set>` plugin's body may be
59
+ // wrapped in a single `<<IDENT...IDENT` heredoc to opaquely contain
60
+ // multi-line scripts, tag-shaped prose, or content with special
61
+ // characters. Plugins consume the unwrapped inner body verbatim;
62
+ // the IDENT is exposed as `heredocIdent` on the command for plugins
63
+ // that want to act on the label. `<set>` is exempt because it does
64
+ // its own multi-op heredoc parsing via `parseMarkerBody`.
65
+ if (name !== "set") {
66
+ const heredoc = extractSingleHeredoc(rawBody);
67
+ if (heredoc) {
68
+ rawBody = heredoc.content;
69
+ a = { ...a, heredocIdent: heredoc.ident };
70
+ }
71
+ }
17
72
  const trimmed = rawBody.trim();
18
73
 
19
74
  if (name === "set") {
20
- const hasEdit =
21
- /<{3,12} SEARCH/.test(trimmed) ||
22
- />{3,12} REPLACE/.test(trimmed) ||
23
- (trimmed.includes("@@") &&
24
- (trimmed.includes("\n-") || trimmed.includes("\n+"))) ||
25
- trimmed.includes("<old_text>");
26
- if (hasEdit) {
27
- const blocks = parseEditContent(rawBody);
28
- if (blocks.length > 0) {
29
- return {
30
- name,
31
- path: a.path,
32
- body: a.body,
33
- manifest: a.manifest,
34
- blocks,
35
- };
36
- }
37
- }
38
- const jsonEdit = parseJsonEdit(trimmed);
39
- if (jsonEdit) {
40
- return { name, path: a.path, ...jsonEdit };
41
- }
42
- if (trimmed.startsWith("s/")) {
43
- const blocks = parseSed(trimmed);
44
- if (blocks?.length === 1) {
45
- return {
46
- name,
47
- path: a.path,
48
- search: blocks[0].search,
49
- replace: blocks[0].replace,
50
- flags: blocks[0].flags,
51
- sed: true,
52
- };
53
- }
54
- if (blocks?.length > 1) {
55
- return { name, path: a.path, blocks };
56
- }
57
- }
58
- if (a.search) {
59
- const replace = a.replace ?? trimmed;
60
- return {
61
- name,
62
- path: a.path,
63
- body: a.body,
64
- manifest: a.manifest,
65
- search: a.search,
66
- replace,
67
- };
68
- }
69
- if (trimmed && a.body) {
70
- return {
71
- name,
72
- path: a.path,
73
- search: a.body,
74
- replace: trimmed,
75
- manifest: a.manifest,
76
- };
77
- }
78
- const body = trimmed || a.body || "";
79
- return { name, ...a, body };
75
+ // `search`/`replace` as attributes is no longer in the grammar;
76
+ // strip them so they can't sneak past via the attribute spread.
77
+ const { search: _s, replace: _r, ...rest } = a;
78
+ a = rest;
79
+
80
+ // Self-close / no-body: visibility/metadata op.
81
+ if (!trimmed) return { name, ...a, body: a.body || "" };
82
+
83
+ // Edit syntax (SPEC.md "Edit Syntax"): walks the body for
84
+ // `<<:::IDENT...:::IDENT` markers and returns an ordered op
85
+ // list. No markers → plain body, treated as full-replace.
86
+ // Non-keyword IDENTs (path-flavored, identifier-flavored)
87
+ // route to REPLACE so the model gets a working write whatever
88
+ // IDENT it picks.
89
+ const { ops, error } = parseMarkerBody(rawBody);
90
+ if (error) return { name, ...a, error };
91
+ if (ops) return { name, ...a, operations: ops };
92
+
93
+ // No markers — plain body, full-replace.
94
+ return { name, ...a, body: trimmed };
80
95
  }
81
96
 
82
97
  if (name === "update") {
@@ -85,55 +100,80 @@ function resolveCommand(name, a, rawBody) {
85
100
  return { name, ...a, body, status };
86
101
  }
87
102
 
103
+ // Body shorthand fallback: when the attribute is unset (undefined),
104
+ // fall back to the trimmed body. Empty-string attrs are preserved
105
+ // as-is — handlers validate. `||` would conflate the two cases.
106
+ const fromBody = trimmed === "" ? null : trimmed;
107
+
88
108
  if (name === "get" || name === "rm") {
89
- return { name, ...a, path: a.path || trimmed || null };
109
+ return { name, ...a, path: a.path ?? fromBody };
90
110
  }
91
111
 
92
112
  if (name === "search") {
93
- const path = a.path || trimmed || null;
113
+ const path = a.path ?? fromBody;
94
114
  const results = a.results ? Number(a.results) : null;
95
115
  return { name, ...a, path, results };
96
116
  }
97
117
 
98
118
  if (name === "mv" || name === "cp") {
99
- return { name, ...a, path: a.path, to: a.to || trimmed || null };
119
+ return { name, ...a, path: a.path, to: a.to ?? fromBody };
100
120
  }
101
121
 
102
122
  if (name === "sh" || name === "env") {
103
- const command = a.command || trimmed || null;
123
+ const command = a.command ?? fromBody;
104
124
  return { name, ...a, command };
105
125
  }
106
126
 
107
127
  if (name === "ask_user") {
108
- const question = a.question || null;
109
- const options = a.options || trimmed || null;
128
+ const question = a.question ?? null;
129
+ const options = a.options ?? fromBody;
110
130
  return { name, ...a, question, options };
111
131
  }
112
132
 
113
- return { name, ...a, body: trimmed || a.body };
133
+ return { name, ...a, body: trimmed === "" ? a.body : trimmed };
114
134
  }
115
135
 
116
136
  const NAME_CHAR = /[a-zA-Z0-9_]/;
117
137
  const ATTR_KEY_CHAR = /[a-zA-Z0-9_:-]/;
118
138
  const WS = /\s/;
119
139
 
120
- // Recovery-tolerant tokenizer for rummy's closed set of tool tags.
140
+ // Tokenizer for rummy's closed set of tool tags. Body opacity for closed
141
+ // bodies; tail recovery for unclosed bodies.
121
142
  //
122
143
  // Design contract:
123
144
  // - Tool tags (<get>, <set>, <sh>, ...) are the only syntactic special tags.
124
145
  // Any other "<...>" sequence in OUTER text is treated as literal text.
125
- // - Inside a tool tag's body, content is OPAQUE: only the matching close
126
- // tag is recognized. Body may contain regex (`(?<!`), generics (`Vec<u8>`),
127
- // HTML, XML, heredocs, comparison operators none of it affects parsing.
128
- // - Backtick spans (`...`) and triple-backtick fences (```...```) at the
129
- // OUTER level neutralize tag-like content, mirroring the markdown
130
- // convention that documentation about a tool isn't a tool call.
131
- // Inside tool bodies this tracking does NOT apply (body opacity wins).
146
+ // - Inside a tool tag's body, content is OPAQUE: only the matching
147
+ // `</tagname>` close (depth-counted for same-name nesting) ends the
148
+ // body. Mismatched closes of OTHER tag names `</env>`, `</mv>`,
149
+ // `</foo>` inside a `<set>` body are body content, not structural
150
+ // signals.
151
+ // - Backtick spans (`...`) and triple-backtick fences (```...```)
152
+ // suppress tag recognition AT THE OUTER LEVEL ONLY (between tool
153
+ // calls). Documentation prose with backticked tag examples doesn't
154
+ // get parsed as commands. Inside tool bodies backticks are content;
155
+ // bodies that need opacity for tag-like content use the edit-syntax
156
+ // marker family (see SPEC.md "Edit Syntax"), which has no
157
+ // false-positive failure modes (unlike inside-body backtick
158
+ // tracking, which would suppress closing tags on bodies with stray
159
+ // unbalanced backticks).
160
+ // - Edit-syntax marker opacity (set only): `<<:::IDENT...:::IDENT`
161
+ // spans inside a `<set>` body are skipped during tag detection so
162
+ // content with `</set>` literals or marker-shaped text stays as
163
+ // body. Multiple markers per body supported; see marker.js.
132
164
  // - Same-name nesting (`<set>...<set/>...</set>`) is depth-counted so
133
- // nested examples don't prematurely close the outer.
134
- // - Recovery: unclosed openers capture body to EOF + emit a warning.
135
- // Orphan closes at outer level become text, no warning (body opacity
136
- // means models legitimately write `</set>` in prose / summaries).
165
+ // nested examples don't prematurely close the outer. Same-name
166
+ // nesting also disables tail recovery the model's intent is clearly
167
+ // opaque body content.
168
+ // - Unclosed openers (no matching close, no same-name nesting) try
169
+ // tail recovery: scan the captured body for the leftmost position
170
+ // whose suffix tokenizes cleanly into ≥1 well-formed tool calls
171
+ // with zero leftover text. If found, end the unclosed body there
172
+ // and let the trailing tags parse as proper siblings. The warning
173
+ // surfaces "Unclosed <name> — recovered N trailing tool call(s)"
174
+ // so the model can see what happened. If recovery finds nothing,
175
+ // capture body to EOF and emit "Unclosed <name> — content captured
176
+ // anyway".
137
177
  export default class XmlParser {
138
178
  static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS);
139
179
 
@@ -197,11 +237,13 @@ export default class XmlParser {
197
237
  const result = XmlParser.#findBodyEnd(s, name, openerEnd);
198
238
  const body = s.slice(openerEnd, result.bodyEnd);
199
239
  if (result.unclosed) {
200
- warnings.push(`Unclosed <${name}> tag — content captured anyway`);
201
- } else if (result.mismatchedCloseName) {
202
- warnings.push(
203
- `Mismatched </${result.mismatchedCloseName}> closing <${name}> — corrected to </${name}>`,
204
- );
240
+ if (result.recoveredTailCount) {
241
+ warnings.push(
242
+ `Unclosed <${name}> tag — recovered ${result.recoveredTailCount} trailing tool call(s)`,
243
+ );
244
+ } else {
245
+ warnings.push(`Unclosed <${name}> tag — content captured anyway`);
246
+ }
205
247
  }
206
248
  commands.push(resolveCommand(name, attrs, body));
207
249
  i = result.afterClose;
@@ -327,18 +369,42 @@ export default class XmlParser {
327
369
 
328
370
  // Scans body content from `fromPos` until the matching `</name>` closer,
329
371
  // counting depth so same-name nested examples don't prematurely close.
330
- // Returns { bodyEnd, afterClose, unclosed, mismatchedCloseName }.
372
+ // Returns { bodyEnd, afterClose, unclosed }.
373
+ //
374
+ // Strict body opacity: only `</name>` (matching the open) and same-name
375
+ // nested opens affect parsing. Mismatched closes of OTHER tag names are
376
+ // body content, period.
377
+ //
378
+ // Backtick fences (`…`, ```…```) inside the body suppress all tag
379
+ // recognition — a markdown table cell containing `<set>` examples
380
+ // stays as content, not interpreted as a nested tag. This matches
381
+ // the outer-level convention and is the load-bearing reason a model
382
+ // can write documentation about rummy commands inside a deliverable
383
+ // body without breaking parsing.
331
384
  //
332
- // Mismatched-close recovery: if we encounter `</X>` where X != name and X
333
- // is not a depth-counted nested tag, we use a balance heuristic to decide
334
- // whether the orphan close was a typo (recover here) or legitimate body
335
- // content (continue scanning). Specifically: count `</name>` minus
336
- // `<name` in the rest of the string; if non-positive, no real close
337
- // exists ahead and the orphan must be the intended close.
385
+ // If the matching close never arrives, emit "Unclosed" so the model
386
+ // sees a clear failure and corrects on the next turn.
338
387
  static #findBodyEnd(s, name, fromPos) {
339
388
  let depth = 1;
389
+ let sameNameNested = false;
340
390
  let i = fromPos;
341
391
  while (i < s.length) {
392
+ // Edit-syntax marker opacity: marker spans (bare `<<IDENT` or
393
+ // packet-shaped `<<:::IDENT`) are opaque — tag detection
394
+ // skips them so inner `</set>` and other tag-shaped content
395
+ // stays as body. Multiple markers per `<set>` body are
396
+ // supported; check on every iteration.
397
+ if (
398
+ name === "set" &&
399
+ (s.startsWith("<<:::", i) ||
400
+ (s.startsWith("<<", i) && /^[A-Z]/.test(s[i + 2] ?? "")))
401
+ ) {
402
+ const skipTo = skipEditMarker(s, i);
403
+ if (skipTo != null) {
404
+ i = skipTo;
405
+ continue;
406
+ }
407
+ }
342
408
  if (s[i] !== "<") {
343
409
  i++;
344
410
  continue;
@@ -360,35 +426,64 @@ export default class XmlParser {
360
426
  i = k + 1;
361
427
  continue;
362
428
  }
363
-
364
- if (isCloseTag && closeName.length > 0) {
365
- const rest = s.slice(k + 1);
366
- const closesAhead = (
367
- rest.match(new RegExp(`<\\/${name}\\b\\s*>`, "g")) || []
368
- ).length;
369
- const opensAhead = (rest.match(new RegExp(`<${name}\\b`, "g")) || [])
370
- .length;
371
- if (closesAhead - opensAhead < 1) {
372
- return {
373
- bodyEnd: i,
374
- afterClose: k + 1,
375
- unclosed: false,
376
- mismatchedCloseName: closeName,
377
- };
378
- }
379
- }
380
429
  }
381
430
  const opener = XmlParser.#matchOpener(s, i);
382
431
  if (opener && opener.name === name && !opener.selfClose) {
383
432
  depth++;
433
+ sameNameNested = true;
384
434
  i = opener.end;
385
435
  continue;
386
436
  }
387
437
  i++;
388
438
  }
439
+ // Unclosed: try tail recovery, but only if the body never
440
+ // nested a same-name opener. Same-name nesting is the model
441
+ // deliberately using opaque body for examples (`<set>` writing
442
+ // docs about `<set>`); we trust the body content as authored.
443
+ // No nesting means a plain botched `</set>` — recovery is safe.
444
+ // If the body's tail is a clean sequence of one or more
445
+ // well-formed tool calls (zero leftover text), end the body
446
+ // at the start of that tail and let the outer tokenizer parse
447
+ // those calls as proper siblings. Closes the silent-swallow
448
+ // gap when a model botches `</set>` after SEARCH/REPLACE and
449
+ // emits trailing `<sh>` / `<update>`.
450
+ if (sameNameNested) {
451
+ return { bodyEnd: s.length, afterClose: s.length, unclosed: true };
452
+ }
453
+ const recovery = XmlParser.#findTailRecovery(s, fromPos);
454
+ if (recovery) {
455
+ return {
456
+ bodyEnd: recovery.tailStart,
457
+ afterClose: recovery.tailStart,
458
+ unclosed: true,
459
+ recoveredTailCount: recovery.commandCount,
460
+ };
461
+ }
389
462
  return { bodyEnd: s.length, afterClose: s.length, unclosed: true };
390
463
  }
391
464
 
465
+ // Scan body content for the leftmost position whose suffix tokenizes
466
+ // cleanly into ≥1 commands with no leftover non-whitespace text.
467
+ // Returns { tailStart, commandCount } or null. Only considers opener
468
+ // positions; treats the suffix as outer-level so backtick fences and
469
+ // tag recognition match the parent tokenizer's behavior.
470
+ static #findTailRecovery(s, fromPos) {
471
+ let best = null;
472
+ let i = fromPos;
473
+ while (i < s.length) {
474
+ if (s[i] === "<" && XmlParser.#matchOpener(s, i)) {
475
+ const suffix = s.slice(i);
476
+ const result = XmlParser.#tokenize(suffix, []);
477
+ if (result.commands.length > 0 && result.unparsed === "") {
478
+ best = { tailStart: i, commandCount: result.commands.length };
479
+ break;
480
+ }
481
+ }
482
+ i++;
483
+ }
484
+ return best;
485
+ }
486
+
392
487
  // Translate native training-format tool calls into rummy XML silently.
393
488
  static #normalizeToolCalls(content) {
394
489
  // Gemma code-fenced XML.
@@ -1,3 +1,21 @@
1
+ // Outcomes that record a failure but don't strike — findings the model
2
+ // adapts to, not contract violations. `not_found` (model acted on an
3
+ // entry that doesn't exist) and `conflict` (SEARCH text didn't match
4
+ // current body) are recoverable: read the new state, try again.
5
+ // `unparsed` (free text outside any tool tag — comments, "thinking
6
+ // out loud" between tool calls) is non-actionable but not malicious;
7
+ // the empty-turn failure mode is already caught by update plugin's
8
+ // 422 "Missing update", so striking unparsed too is duplicative.
9
+ // Hard outcomes (validation, permission, exit:N) DO strike. Shared
10
+ // between error.js's verdict accumulator (recordedFailed gate) and
11
+ // Entries' auto-failure hook (passes soft=true so error.log.emit
12
+ // skips turn errors increment when the outcome is soft).
13
+ export const SOFT_FAILURE_OUTCOMES = new Set([
14
+ "not_found",
15
+ "conflict",
16
+ "unparsed",
17
+ ]);
18
+
1
19
  // Writer tier excluded from scheme.writable_by; see SPEC writer_tiers.
2
20
  export class PermissionError extends Error {
3
21
  constructor(scheme, writer, allowed) {
@@ -14,3 +32,20 @@ export class PermissionError extends Error {
14
32
  this.allowed = [...allowed];
15
33
  }
16
34
  }
35
+
36
+ // Body length exceeded the entries.body CHECK constraint (RUMMY_ENTRY_SIZE_MAX
37
+ // at create-time). Surfaced as a 413 strike. The cap value lives only in the
38
+ // schema — JS does not duplicate it, because the database persists across
39
+ // rummy invocations and the env var that built the schema may differ from
40
+ // the env var seen by the running instance. Reporting body size is enough
41
+ // for the model to adapt; operators can read the cap from the schema.
42
+ export class EntryOverflowError extends Error {
43
+ constructor(path, size) {
44
+ super(
45
+ `413: entry "${path}" body ${size} bytes exceeds RUMMY_ENTRY_SIZE_MAX`,
46
+ );
47
+ this.name = "EntryOverflowError";
48
+ this.path = path;
49
+ this.size = size;
50
+ }
51
+ }