@possumtech/rummy 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/.env.example +12 -0
  2. package/FIDELITY_CONTRACT.md +172 -0
  3. package/README.md +5 -1
  4. package/SPEC.md +31 -17
  5. package/migrations/001_initial_schema.sql +3 -4
  6. package/package.json +1 -1
  7. package/src/agent/AgentLoop.js +51 -153
  8. package/src/agent/ContextAssembler.js +2 -0
  9. package/src/agent/KnownStore.js +16 -9
  10. package/src/agent/ResponseHealer.js +54 -1
  11. package/src/agent/TurnExecutor.js +125 -323
  12. package/src/agent/XmlParser.js +172 -42
  13. package/src/agent/known_queries.sql +1 -1
  14. package/src/agent/known_store.sql +29 -72
  15. package/src/agent/runs.sql +2 -2
  16. package/src/hooks/Hooks.js +1 -0
  17. package/src/hooks/PluginContext.js +8 -2
  18. package/src/hooks/RummyContext.js +6 -3
  19. package/src/hooks/ToolRegistry.js +29 -32
  20. package/src/plugins/ask_user/ask_user.js +2 -2
  21. package/src/plugins/ask_user/ask_userDoc.js +7 -10
  22. package/src/plugins/budget/README.md +28 -18
  23. package/src/plugins/budget/budget.js +80 -3
  24. package/src/plugins/budget/recovery.js +47 -0
  25. package/src/plugins/cp/cp.js +5 -5
  26. package/src/plugins/cp/cpDoc.js +1 -14
  27. package/src/plugins/engine/engine.sql +1 -1
  28. package/src/plugins/env/env.js +4 -4
  29. package/src/plugins/env/envDoc.js +4 -9
  30. package/src/plugins/file/file.js +2 -7
  31. package/src/plugins/get/get.js +32 -13
  32. package/src/plugins/get/getDoc.js +26 -44
  33. package/src/plugins/helpers.js +4 -4
  34. package/src/plugins/instructions/instructions.js +9 -7
  35. package/src/plugins/instructions/preamble.md +45 -26
  36. package/src/plugins/known/known.js +71 -15
  37. package/src/plugins/known/knownDoc.js +4 -20
  38. package/src/plugins/mv/mv.js +6 -6
  39. package/src/plugins/mv/mvDoc.js +4 -30
  40. package/src/plugins/policy/policy.js +47 -0
  41. package/src/plugins/previous/previous.js +10 -14
  42. package/src/plugins/progress/progress.js +29 -48
  43. package/src/plugins/prompt/prompt.js +18 -6
  44. package/src/plugins/rm/rm.js +4 -4
  45. package/src/plugins/rm/rmDoc.js +5 -14
  46. package/src/plugins/rpc/rpc.js +4 -2
  47. package/src/plugins/set/set.js +86 -91
  48. package/src/plugins/set/setDoc.js +28 -41
  49. package/src/plugins/sh/sh.js +4 -4
  50. package/src/plugins/sh/shDoc.js +4 -9
  51. package/src/plugins/skill/skill.js +2 -1
  52. package/src/plugins/summarize/summarize.js +9 -2
  53. package/src/plugins/summarize/summarizeDoc.js +10 -16
  54. package/src/plugins/telemetry/telemetry.js +36 -11
  55. package/src/plugins/think/think.js +13 -0
  56. package/src/plugins/think/thinkDoc.js +16 -0
  57. package/src/plugins/unknown/unknown.js +37 -9
  58. package/src/plugins/unknown/unknownDoc.js +7 -16
  59. package/src/plugins/update/update.js +9 -2
  60. package/src/plugins/update/updateDoc.js +12 -14
  61. package/src/server/ClientConnection.js +11 -1
  62. package/src/sql/functions/slugify.js +13 -1
  63. package/src/sql/v_model_context.sql +6 -6
@@ -7,8 +7,8 @@ export default class AskUser {
7
7
  this.#core = core;
8
8
  core.registerScheme();
9
9
  core.on("handler", this.handler.bind(this));
10
- core.on("full", this.full.bind(this));
11
- core.on("summary", this.summary.bind(this));
10
+ core.on("promoted", this.full.bind(this));
11
+ core.on("demoted", this.summary.bind(this));
12
12
  core.filter("instructions.toolDocs", async (docsMap) => {
13
13
  docsMap.ask_user = docs;
14
14
  return docsMap;
@@ -2,27 +2,24 @@
2
2
  // Text goes to the model. Rationale stays in source.
3
3
  // Changing ANY line requires reading ALL rationales first.
4
4
  const LINES = [
5
- // --- Syntax: question attr + options in body
6
- ['## <ask_user question="[Question?]">[option1; option2; ...]</ask_user>'],
7
-
8
- // --- Constraints FIRST: frames correct usage before examples
5
+ [
6
+ '## <ask_user question="[Question?]">[option1; option2; ...]</ask_user> - Ask the user a question',
7
+ ],
9
8
  [
10
9
  "* YOU SHOULD use for decisions, preferences, or approvals the user must make",
11
- "Positive framing. Shows what ask_user IS for, not just what it isn't.",
10
+ "Positive framing. Shows what ask_user IS for.",
12
11
  ],
13
12
  [
14
- "* YOU SHOULD use <get> to find information before asking the user",
15
- "Gentle redirect. Encourages self-sufficiency without forbidding interaction.",
13
+ "* YOU SHOULD use <get></get> to find information before asking the user",
14
+ "Gentle redirect. Encourages self-sufficiency.",
16
15
  ],
17
-
18
- // --- Examples: genuine decision points where user input is valuable
19
16
  [
20
17
  'Example: <ask_user question="Which test framework?">Mocha; Jest; Node Native</ask_user>',
21
18
  "Preference decision. Model truly cannot know this without asking.",
22
19
  ],
23
20
  [
24
21
  'Example: <ask_user question="Deploy to staging or production?">staging; production</ask_user>',
25
- "Consequential action. Shows ask_user for high-stakes choices.",
22
+ "Consequential action. High-stakes choice.",
26
23
  ],
27
24
  ];
28
25
 
@@ -2,30 +2,40 @@
2
2
 
3
3
  Context ceiling enforcement.
4
4
 
5
- ## Files
5
+ ## Design
6
6
 
7
- - **budget.js** Plugin. Pre-LLM enforce, BudgetGuard activation.
8
- - **BudgetGuard.js** Write-layer gate. Installed on KnownStore during
9
- dispatch. Checks token delta on every upsert, promote, and body update.
7
+ Ceiling = `floor(contextSize × 0.9)`. The 10% headroom is the system's
8
+ operating room for graceful overflow handling. No per-write gating
9
+ tools run uninterrupted. Enforcement happens at boundaries.
10
10
 
11
- ## Registration
11
+ ## Enforcement Points
12
12
 
13
- - **Hook**: `hooks.budget.enforce` pre-LLM ceiling check.
14
- - **Hook**: `hooks.budget.activate(store, contextSize, assembledTokens)` install guard.
15
- - **Hook**: `hooks.budget.deactivate(store)` remove guard.
13
+ 1. **Pre-LLM enforce** (`budget.enforce`): checks assembled context
14
+ before the LLM call. If over ceiling Prompt Demotion (summarize
15
+ the incoming prompt). Model runs in the headroom.
16
16
 
17
- ## Budget Contract
17
+ 2. **Post-dispatch Turn Demotion**: after all tools dispatch, check
18
+ context. If over ceiling → demote ALL entries from this turn
19
+ (every scheme except `budget`/`system`/`prompt`/`instructions`,
20
+ and 4xx error states stay promoted). Write `budget://` entry with
21
+ directive to demote irrelevant entries and promote fewer next time.
22
+ Model sees it next turn and adapts.
18
23
 
19
- `contextSize` is the ceiling. `countTokens()` is the measurement.
20
- Over = 413. Under = 200. No margins.
24
+ 3. **LLM rejection** (`isContextExceeded`): turn-1 token estimate
25
+ drift causes LLM to reject. Same demotion pattern.
21
26
 
22
- ## BudgetGuard
27
+ 4. **AgentLoop recovery**: pre-LLM 413 that Prompt Demotion can't
28
+ resolve. Batch-demote all full entries, budget entry, model gets
29
+ recovery turns. 3 strikes without progress → hard 413 to client.
30
+ Only path where 413 reaches the client.
23
31
 
24
- Installed on KnownStore by TurnExecutor before dispatch, cleared in
25
- `finally`. Gates `upsert()`, `promoteByPattern()`, `updateBodyByPattern()`.
32
+ ## Files
26
33
 
27
- Exemptions: `status >= 400` (error entries), `model_visible = 0` (audit),
28
- `fidelity = "archive"` (not in context).
34
+ - **budget.js** Plugin. Pre-LLM enforce hook.
35
+ - **BudgetGuard.js** `BudgetExceeded` error type, `delta` utility.
29
36
 
30
- On first violation: `BudgetExceeded` thrown, guard trips, all subsequent
31
- writes fail. TurnExecutor catches per-tool, writes 413 result entry.
37
+ ## Registration
38
+
39
+ - **Hook**: `hooks.budget.enforce` — pre-LLM ceiling check.
40
+ - **Scheme**: `budget://` — logging category, model-visible. `onView`
41
+ renders body at all fidelity levels (demoted shows full content).
@@ -1,5 +1,8 @@
1
1
  import { countTokens } from "../../agent/tokens.js";
2
2
 
3
+ const CEILING_RATIO = Number(process.env.RUMMY_BUDGET_CEILING);
4
+ if (!CEILING_RATIO) throw new Error("RUMMY_BUDGET_CEILING must be set");
5
+
3
6
  function measureMessages(messages) {
4
7
  return messages.reduce((sum, m) => sum + countTokens(m.content), 0);
5
8
  }
@@ -17,6 +20,7 @@ export default class Budget {
17
20
  core.hooks.tools.onView("budget", (entry) => entry.body);
18
21
  core.hooks.budget = {
19
22
  enforce: this.enforce.bind(this),
23
+ postDispatch: this.postDispatch.bind(this),
20
24
  };
21
25
  }
22
26
 
@@ -25,8 +29,6 @@ export default class Budget {
25
29
  return { messages, rows, demoted: [], assembledTokens: 0, status: 200 };
26
30
  }
27
31
 
28
- // Prefer actual prompt_tokens from the last API response — the estimate
29
- // from measureMessages can be wildly off for structured/XML-heavy content.
30
32
  const assembledTokens =
31
33
  lastPromptTokens > 0 ? lastPromptTokens : measureMessages(messages);
32
34
 
@@ -34,7 +36,7 @@ export default class Budget {
34
36
  `[RUMMY] Budget enforce: ${assembledTokens} tokens (${lastPromptTokens > 0 ? "actual" : "estimated"}), ceiling ${contextSize}, ${rows.length} rows`,
35
37
  );
36
38
 
37
- const ceiling = Math.floor(contextSize * 0.9);
39
+ const ceiling = Math.floor(contextSize * CEILING_RATIO);
38
40
  if (assembledTokens > ceiling) {
39
41
  const overflow = assembledTokens - ceiling;
40
42
  console.warn(
@@ -52,4 +54,79 @@ export default class Budget {
52
54
 
53
55
  return { messages, rows, demoted: [], assembledTokens, status: 200 };
54
56
  }
57
+
58
+ async postDispatch({
59
+ contextSize,
60
+ messages,
61
+ rows,
62
+ runId,
63
+ loopId,
64
+ turn,
65
+ db,
66
+ store,
67
+ }) {
68
+ if (!contextSize) return null;
69
+
70
+ const postBudget = await this.enforce({
71
+ contextSize,
72
+ messages,
73
+ rows,
74
+ lastPromptTokens: 0,
75
+ });
76
+
77
+ if (postBudget.status !== 413) return null;
78
+
79
+ // Demote this turn's entries
80
+ const demotedEntries = await db.demote_turn_entries.all({
81
+ run_id: runId,
82
+ turn,
83
+ });
84
+
85
+ // Also demote the prompt
86
+ const promptRow = rows.find((r) => r.scheme === "prompt");
87
+ if (promptRow) {
88
+ await store.setFidelity(runId, promptRow.path, "demoted");
89
+ }
90
+
91
+ // Rewrite get-result bodies — the get handler claimed "promoted" success
92
+ // before this panic ran. Without rewriting, the model reads conflicting
93
+ // signals next turn (status=413 but body says "promoted").
94
+ for (const entry of demotedEntries) {
95
+ if (!entry.path.startsWith("get://")) continue;
96
+ await db.resolve_known_entry.run({
97
+ run_id: runId,
98
+ path: entry.path,
99
+ body: `Demoted by budget. See budget://${loopId}/${turn}.`,
100
+ status: 413,
101
+ });
102
+ }
103
+
104
+ // Write budget entry — terse, actionable. Path list dropped since
105
+ // demoted entries already render at fidelity="demoted" in <knowns>/<files>.
106
+ // "tokens remaining" dropped too — the number was over-optimistic (it
107
+ // treated re-demoted files as freeing their full-body tokens when their
108
+ // demoted-view renderings return to baseline). Model reads the truthful
109
+ // remaining in next turn's progress line.
110
+ //
111
+ // The 50% rule is the key directive: it forces the model to sum
112
+ // promotion costs (which is the behavior we want), and the threshold
113
+ // gives a concrete ceiling for the next try. Twofer — abiding by the
114
+ // rule requires budget awareness as a side effect.
115
+ const ceiling = Math.floor(contextSize * CEILING_RATIO);
116
+ const totalDemoted = demotedEntries.reduce((s, r) => s + r.tokens, 0);
117
+ const body = [
118
+ `413 Token Budget Error: overflowed by ${postBudget.overflow} tokens. Token Budget: ${ceiling}.`,
119
+ `Your ${demotedEntries.length} promotions from last turn (${totalDemoted} tokens total) were demoted to fit.`,
120
+ `Required: sum the tokens="N" of your promotions and new entries before emitting. A single turn must add no more than 50% of remaining Token Budget.`,
121
+ ].join("\n");
122
+
123
+ await store.upsert(runId, turn, `budget://${loopId}/${turn}`, body, 413, {
124
+ loopId,
125
+ });
126
+
127
+ return {
128
+ target: ceiling,
129
+ promptPath: promptRow?.path ?? null,
130
+ };
131
+ }
55
132
  }
@@ -0,0 +1,47 @@
1
+ /**
2
+ * Pure recovery state transition — exported for testing.
3
+ *
4
+ * @param {object|null} recovery Current recovery state.
5
+ * @param {{ assembledTokens: number, budgetRecovery?: { target: number, promptPath: string|null } }} result
6
+ * @returns {{ next: object|null, action: null|'restore'|'hard413', promptPath: string|null }}
7
+ */
8
+ export function advanceRecovery(recovery, result) {
9
+ // Initialise or update recovery state from a new Turn Demotion event.
10
+ if (result.budgetRecovery) {
11
+ if (!recovery) {
12
+ recovery = {
13
+ target: result.budgetRecovery.target,
14
+ promptPath: result.budgetRecovery.promptPath,
15
+ strikes: 0,
16
+ lastTokens: result.assembledTokens,
17
+ };
18
+ } else {
19
+ // Re-overflow during recovery: tighten target, don't count as strike.
20
+ recovery = {
21
+ ...recovery,
22
+ target: Math.min(recovery.target, result.budgetRecovery.target),
23
+ };
24
+ }
25
+ }
26
+
27
+ if (recovery === null) return { next: null, action: null, promptPath: null };
28
+
29
+ const current = result.assembledTokens;
30
+
31
+ if (current <= recovery.target) {
32
+ return { next: null, action: "restore", promptPath: recovery.promptPath };
33
+ }
34
+
35
+ const noProgress = current >= recovery.lastTokens && !result.budgetRecovery;
36
+ const strikes = noProgress ? recovery.strikes + 1 : 0;
37
+
38
+ if (strikes >= 3) {
39
+ return { next: null, action: "hard413", promptPath: null };
40
+ }
41
+
42
+ return {
43
+ next: { ...recovery, strikes, lastTokens: current },
44
+ action: null,
45
+ promptPath: null,
46
+ };
47
+ }
@@ -8,8 +8,8 @@ export default class Cp {
8
8
  this.#core = core;
9
9
  core.registerScheme();
10
10
  core.on("handler", this.handler.bind(this));
11
- core.on("full", this.full.bind(this));
12
- core.on("summary", this.summary.bind(this));
11
+ core.on("promoted", this.full.bind(this));
12
+ core.on("demoted", this.summary.bind(this));
13
13
  core.filter("instructions.toolDocs", async (docsMap) => {
14
14
  docsMap.cp = docs;
15
15
  return docsMap;
@@ -19,7 +19,7 @@ export default class Cp {
19
19
  async handler(entry, rummy) {
20
20
  const { entries: store, sequence: turn, runId, loopId } = rummy;
21
21
  const { path, to } = entry.attributes;
22
- const VALID = { stored: 1, summary: 1, index: 1, full: 1, archive: 1 };
22
+ const VALID = { promoted: 1, demoted: 1, archived: 1 };
23
23
  const fidelity = VALID[entry.attributes.fidelity]
24
24
  ? entry.attributes.fidelity
25
25
  : undefined;
@@ -53,7 +53,7 @@ export default class Cp {
53
53
  return `# cp ${entry.attributes.from || ""} ${entry.attributes.to || ""}`;
54
54
  }
55
55
 
56
- summary(entry) {
57
- return this.full(entry);
56
+ summary() {
57
+ return "";
58
58
  }
59
59
  }
@@ -2,27 +2,14 @@
2
2
  // Text goes to the model. Rationale stays in source.
3
3
  // Changing ANY line requires reading ALL rationales first.
4
4
  const LINES = [
5
- // --- Syntax: path attr = source, body = destination
6
5
  ['## <cp path="[source]">[destination]</cp> - Copy a file or entry'],
7
-
8
- // --- Examples: single copy, glob batch, cross-scheme
9
6
  [
10
7
  'Example: <cp path="src/config.js">src/config.backup.js</cp>',
11
8
  "Simple file copy. Path = source, body = destination.",
12
9
  ],
13
10
  [
14
11
  'Example: <cp path="known://plan_*">known://archive_</cp>',
15
- "Glob batch copy across known entries. Shows pattern operations on cp.",
16
- ],
17
-
18
- // --- Constraints
19
- [
20
- "* Source path accepts patterns: `src/*.js`, `known://draft_*`",
21
- "Pattern support. Distributes glob teaching beyond get.",
22
- ],
23
- [
24
- "* Use `preview` to check matches before bulk copy",
25
- "Safety pattern consistent with get and rm preview.",
12
+ "Glob batch copy across known entries.",
26
13
  ],
27
14
  ];
28
15
 
@@ -6,7 +6,7 @@ FROM known_entries AS ke
6
6
  JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
7
7
  WHERE
8
8
  ke.run_id = :run_id
9
- AND ke.fidelity IN ('full', 'summary')
9
+ AND ke.fidelity IN ('promoted', 'demoted')
10
10
  AND s.model_visible = 1
11
11
  ORDER BY ke.turn, ke.refs, ke.tokens DESC;
12
12
 
@@ -7,8 +7,8 @@ export default class Env {
7
7
  this.#core = core;
8
8
  core.registerScheme();
9
9
  core.on("handler", this.handler.bind(this));
10
- core.on("full", this.full.bind(this));
11
- core.on("summary", this.summary.bind(this));
10
+ core.on("promoted", this.full.bind(this));
11
+ core.on("demoted", this.summary.bind(this));
12
12
  core.filter("instructions.toolDocs", async (docsMap) => {
13
13
  docsMap.env = docs;
14
14
  return docsMap;
@@ -27,7 +27,7 @@ export default class Env {
27
27
  return `# env ${entry.attributes.command || ""}\n${entry.body}`;
28
28
  }
29
29
 
30
- summary(entry) {
31
- return entry.attributes.command || "";
30
+ summary() {
31
+ return "";
32
32
  }
33
33
  }
@@ -2,10 +2,7 @@
2
2
  // Text goes to the model. Rationale stays in source.
3
3
  // Changing ANY line requires reading ALL rationales first.
4
4
  const LINES = [
5
- // --- Syntax
6
5
  ["## <env>[command]</env> - Run an exploratory shell command"],
7
-
8
- // --- Examples: version check and git status — safe, read-only commands
9
6
  [
10
7
  "Example: <env>npm --version</env>",
11
8
  "Version check. Safe, no side effects.",
@@ -14,15 +11,13 @@ const LINES = [
14
11
  "Example: <env>git log --oneline -5</env>",
15
12
  "Git history. Shows env for read-only investigation.",
16
13
  ],
17
-
18
- // --- Constraints: hard boundaries
19
14
  [
20
- '* YOU MUST NOT use <env/> to read or list files — use <get path="*" preview/> instead',
21
- "Prevents cat/ls through shell. Forces file access through get for proper tracking.",
15
+ '* YOU MUST NOT use <env></env> to read or list files — use <get path="*"/> instead',
16
+ "Prevents cat/ls through shell. Forces file access through get.",
22
17
  ],
23
18
  [
24
- "* YOU MUST use <sh/> for commands with side effects",
25
- "Separates exploration from action. env = observe, sh = mutate.",
19
+ "* YOU MUST NOT use <env></env> for commands with side effects",
20
+ "Separates exploration from action. env = observe only.",
26
21
  ],
27
22
  ];
28
23
 
@@ -16,13 +16,8 @@ export default class File {
16
16
  this.#core = core;
17
17
  // "file" scheme covers bare paths (scheme IS NULL in DB)
18
18
  core.registerScheme({ category: "data" });
19
- core.registerScheme({ name: "http", category: "data" });
20
- core.registerScheme({ name: "https", category: "data" });
21
- core.on("full", this.full.bind(this));
22
- core.on("summary", this.summary.bind(this));
23
- // Default identity views for http/https — rummy.web overrides these
24
- core.hooks.tools.onView("http", (entry) => entry.body);
25
- core.hooks.tools.onView("https", (entry) => entry.body);
19
+ core.on("promoted", this.full.bind(this));
20
+ core.on("demoted", this.summary.bind(this));
26
21
  }
27
22
 
28
23
  full(entry) {
@@ -9,8 +9,8 @@ export default class Get {
9
9
  this.#core = core;
10
10
  core.registerScheme();
11
11
  core.on("handler", this.handler.bind(this));
12
- core.on("full", this.full.bind(this));
13
- core.on("summary", this.summary.bind(this));
12
+ core.on("promoted", this.full.bind(this));
13
+ core.on("demoted", this.summary.bind(this));
14
14
  core.filter("instructions.toolDocs", async (docsMap) => {
15
15
  docsMap.get = docs;
16
16
  return docsMap;
@@ -29,6 +29,7 @@ export default class Get {
29
29
  }
30
30
  const normalized = KnownStore.normalizePath(target);
31
31
  const bodyFilter = entry.attributes.body || null;
32
+ const preview = entry.attributes.preview !== undefined;
32
33
  const isPattern = bodyFilter || normalized.includes("*");
33
34
 
34
35
  const line =
@@ -46,6 +47,25 @@ export default class Get {
46
47
  bodyFilter,
47
48
  );
48
49
 
50
+ // Preview — list matches with their full-body token costs. No promotion,
51
+ // no fidelity change, no Token Budget spent. Model uses this to plan
52
+ // which entries to actually promote. getDoc promises this behavior; the
53
+ // prior implementation silently promoted anyway, burning the Token Budget
54
+ // on entries the model thought it was only inspecting.
55
+ if (preview) {
56
+ await storePatternResult(
57
+ store,
58
+ runId,
59
+ turn,
60
+ "get",
61
+ target,
62
+ bodyFilter,
63
+ matches,
64
+ { preview: true, loopId, attributes: { path: target } },
65
+ );
66
+ return;
67
+ }
68
+
49
69
  // Partial read — no fidelity promotion, returns a line slice as the log item.
50
70
  if (line !== null || limit !== null) {
51
71
  if (isPattern) {
@@ -55,7 +75,7 @@ export default class Get {
55
75
  entry.resultPath,
56
76
  "line/limit requires a single path, not a glob or body filter",
57
77
  400,
58
- { loopId },
78
+ { loopId, attributes: { path: target } },
59
79
  );
60
80
  return;
61
81
  }
@@ -66,7 +86,7 @@ export default class Get {
66
86
  entry.resultPath,
67
87
  `${target} not found`,
68
88
  200,
69
- { loopId },
89
+ { loopId, attributes: { path: target } },
70
90
  );
71
91
  return;
72
92
  }
@@ -84,17 +104,15 @@ export default class Get {
84
104
  entry.resultPath,
85
105
  `${header}\n${slice}`,
86
106
  200,
87
- { loopId },
107
+ { loopId, attributes: { path: target } },
88
108
  );
89
109
  return;
90
110
  }
91
111
 
92
112
  const VALID_FIDELITY = {
93
- stored: 1,
94
- summary: 1,
95
- index: 1,
96
- full: 1,
97
- archive: 1,
113
+ demoted: 1,
114
+ promoted: 1,
115
+ archived: 1,
98
116
  };
99
117
  const fidelityAttr = VALID_FIDELITY[entry.attributes.fidelity]
100
118
  ? entry.attributes.fidelity
@@ -115,17 +133,18 @@ export default class Get {
115
133
  target,
116
134
  bodyFilter,
117
135
  matches,
118
- { loopId },
136
+ { loopId, attributes: { path: target } },
119
137
  );
120
138
  } else {
121
- const total = matches.reduce((s, m) => s + m.tokens_full, 0);
139
+ const total = matches.reduce((s, m) => s + m.tokens, 0);
122
140
  const paths = matches.map((m) => m.path).join(", ");
123
141
  const body =
124
142
  matches.length > 0
125
- ? `${paths} loaded into <knowns> (${total} tokens)`
143
+ ? `${paths} promoted (${total} tokens)`
126
144
  : `${target} not found`;
127
145
  await store.upsert(runId, turn, entry.resultPath, body, 200, {
128
146
  loopId,
147
+ attributes: { path: target },
129
148
  });
130
149
  }
131
150
  }
@@ -2,50 +2,32 @@
2
2
  // Text goes to the model. Rationale stays in source.
3
3
  // Changing ANY line requires reading ALL rationales first.
4
4
  const LINES = [
5
- // --- Syntax: body-form is the primary invocation (simplest)
6
- ["## <get>[path/to/file]</get> - Load a file or entry into context"],
7
-
8
- // --- Examples: 3 examples covering single file, known recall, and content search
9
- [
10
- "Example: <get>src/app.js</get>",
11
- "Simplest form. Body = path. Teaches that get is the default read tool.",
12
- ],
13
- [
14
- 'Example: <get path="known://*">auth</get>',
15
- "Keyword recall: glob in path, search term in body. Cross-scheme hedberg pattern.",
16
- ],
17
- [
18
- 'Example: <get path="src/**/*.js" preview>authentication</get>',
19
- "Full pattern: recursive glob + preview + content filter. Shows all 3 features at once. Body is a filter keyword, never file content.",
20
- ],
21
-
22
- // --- Partial read: line/limit — show before constraints so model sees it as a first-class pattern
23
- [
24
- 'Example: <get path="src/agent/AgentLoop.js" line="644" limit="80"/>',
25
- "Partial read. Returns lines 644–723 as the log item without promoting the entry to full. Use summary fidelity to find line numbers, then target the symbol directly.",
26
- ],
27
-
28
- // --- Constraints: RFC-style. Each prevents a specific failure mode.
29
- [
30
- "* Paths accept patterns: `src/**/*.js`, `known://api_*`",
31
- "Reinforces picomatch patterns work everywhere, not just in examples.",
32
- ],
33
- [
34
- "* `preview` shows matches without loading into context",
35
- "Budget-awareness. Without this, models load everything and blow context.",
36
- ],
37
- [
38
- "* Body text filters results by content match",
39
- "Generalizes examples 2-3. Body = filter, not just path.",
40
- ],
41
- [
42
- "* `line` and `limit` read a slice without promoting — patterns not allowed",
43
- "The no-promotion constraint is what makes partial read safe: context budget is unaffected.",
44
- ],
45
- [
46
- '* Use <set path="..." fidelity="archive"/> to remove loaded content from context',
47
- "Lifecycle: get→set. Load, read, archive. Prevents context hoarding.",
48
- ],
5
+ ["## <get>[path/to/file]</get> - Promote an entry"],
6
+ ["Example: <get>src/app.js</get>", "Simplest form. Body = path."],
7
+ [
8
+ 'Example: <get path="known://*">auth</get>',
9
+ "Keyword recall: glob in path, search term in body.",
10
+ ],
11
+ [
12
+ 'Example: <get path="src/**/*.js">authentication</get>',
13
+ "Full pattern: recursive glob + content filter.",
14
+ ],
15
+ [
16
+ 'Example: <get path="src/agent/AgentLoop.js" line="644" limit="80"/>',
17
+ "Partial read. Returns lines 644–723 without promoting.",
18
+ ],
19
+ [
20
+ "* Paths accept patterns: `src/**/*.js`, `known://api_*`",
21
+ "Reinforces picomatch patterns work everywhere.",
22
+ ],
23
+ [
24
+ "* Body text filters results by content match",
25
+ "Body = filter, not just path.",
26
+ ],
27
+ [
28
+ "* `line` and `limit` read a slice without promoting the entry, which costs as many tokens as the slice contains.",
29
+ "Partial read is safe: context budget unaffected.",
30
+ ],
49
31
  ];
50
32
 
51
33
  export default LINES.map(([text]) => text).join("\n");
@@ -10,13 +10,13 @@ export async function storePatternResult(
10
10
  path,
11
11
  bodyFilter,
12
12
  matches,
13
- { preview = false, loopId = null } = {},
13
+ { preview = false, loopId = null, attributes = null } = {},
14
14
  ) {
15
15
  const slug = await store.slugPath(runId, scheme, path);
16
16
  const filter = bodyFilter ? ` body="${bodyFilter}"` : "";
17
- const total = matches.reduce((s, m) => s + m.tokens_full, 0);
18
- const listing = matches.map((m) => `${m.path} (${m.tokens_full})`).join("\n");
17
+ const total = matches.reduce((s, m) => s + m.tokens, 0);
18
+ const listing = matches.map((m) => `${m.path} (${m.tokens})`).join("\n");
19
19
  const prefix = preview ? "PREVIEW " : "";
20
20
  const body = `${prefix}${scheme} path="${path}"${filter}: ${matches.length} matched (${total} tokens)\n${listing}`;
21
- await store.upsert(runId, turn, slug, body, 200, { loopId });
21
+ await store.upsert(runId, turn, slug, body, 200, { loopId, attributes });
22
22
  }
@@ -10,7 +10,7 @@ export default class Instructions {
10
10
 
11
11
  constructor(core) {
12
12
  this.#core = core;
13
- core.on("full", this.full.bind(this));
13
+ core.on("promoted", this.full.bind(this));
14
14
  core.on("turn.started", this.onTurnStarted.bind(this));
15
15
  }
16
16
 
@@ -33,20 +33,22 @@ export default class Instructions {
33
33
  const activeTools = attrs.toolSet
34
34
  ? new Set(attrs.toolSet)
35
35
  : new Set(this.#core.hooks.tools.names);
36
- const sorted = this.#core.hooks.tools.names.filter((n) =>
37
- activeTools.has(n),
38
- );
39
- const tools = sorted.join(", ");
40
- let prompt = preamble.replace("[%TOOLS%]", tools);
41
36
  const toolDocs = await this.#core.hooks.instructions.toolDocs.filter(
42
37
  {},
43
38
  { toolSet: activeTools },
44
39
  );
40
+ // Hidden tools are excluded at the registry level (see ToolRegistry).
41
+ const sorted = this.#core.hooks.tools.advertisedNames.filter((n) =>
42
+ activeTools.has(n),
43
+ );
44
+ const tools = sorted.join(", ");
45
45
  const docsText = sorted
46
46
  .filter((key) => toolDocs[key])
47
47
  .map((key) => toolDocs[key])
48
48
  .join("\n\n");
49
- if (docsText) prompt += `\n\n${docsText}`;
49
+ let prompt = preamble
50
+ .replace("[%TOOLS%]", tools)
51
+ .replace("[%TOOLDOCS%]", docsText);
50
52
  if (attrs.persona) prompt += `\n\n## Persona\n\n${attrs.persona}`;
51
53
  return prompt;
52
54
  }