@possumtech/rummy 0.5.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/.env.example +21 -5
  2. package/PLUGINS.md +389 -194
  3. package/README.md +25 -8
  4. package/SPEC.md +850 -373
  5. package/bin/demo.js +166 -0
  6. package/bin/rummy.js +9 -3
  7. package/biome/no-fallbacks.grit +50 -0
  8. package/lang/en.json +2 -2
  9. package/migrations/001_initial_schema.sql +88 -37
  10. package/package.json +6 -4
  11. package/service.js +50 -9
  12. package/src/agent/AgentLoop.js +460 -330
  13. package/src/agent/ContextAssembler.js +4 -4
  14. package/src/agent/Entries.js +655 -0
  15. package/src/agent/ProjectAgent.js +30 -18
  16. package/src/agent/TurnExecutor.js +229 -421
  17. package/src/agent/XmlParser.js +99 -33
  18. package/src/agent/budget.js +56 -0
  19. package/src/agent/errors.js +22 -0
  20. package/src/agent/httpStatus.js +39 -0
  21. package/src/agent/known_checks.sql +8 -4
  22. package/src/agent/known_queries.sql +9 -13
  23. package/src/agent/known_store.sql +275 -125
  24. package/src/agent/materializeContext.js +102 -0
  25. package/src/agent/runs.sql +10 -7
  26. package/src/agent/schemes.sql +14 -3
  27. package/src/agent/turns.sql +9 -9
  28. package/src/hooks/HookRegistry.js +6 -5
  29. package/src/hooks/Hooks.js +44 -3
  30. package/src/hooks/PluginContext.js +29 -21
  31. package/src/{server → hooks}/RpcRegistry.js +2 -1
  32. package/src/hooks/RummyContext.js +135 -35
  33. package/src/hooks/ToolRegistry.js +21 -16
  34. package/src/llm/LlmProvider.js +64 -90
  35. package/src/llm/errors.js +21 -0
  36. package/src/plugins/ask_user/README.md +1 -1
  37. package/src/plugins/ask_user/ask_user.js +37 -12
  38. package/src/plugins/ask_user/ask_userDoc.js +2 -25
  39. package/src/plugins/ask_user/ask_userDoc.md +10 -0
  40. package/src/plugins/budget/README.md +27 -25
  41. package/src/plugins/budget/budget.js +260 -88
  42. package/src/plugins/cp/README.md +2 -2
  43. package/src/plugins/cp/cp.js +29 -11
  44. package/src/plugins/cp/cpDoc.js +2 -15
  45. package/src/plugins/cp/cpDoc.md +7 -0
  46. package/src/plugins/engine/README.md +2 -2
  47. package/src/plugins/engine/engine.sql +4 -4
  48. package/src/plugins/engine/turn_context.sql +10 -10
  49. package/src/plugins/env/README.md +20 -5
  50. package/src/plugins/env/env.js +45 -6
  51. package/src/plugins/env/envDoc.js +2 -23
  52. package/src/plugins/env/envDoc.md +13 -0
  53. package/src/plugins/error/README.md +16 -0
  54. package/src/plugins/error/error.js +151 -0
  55. package/src/plugins/file/README.md +6 -6
  56. package/src/plugins/file/file.js +15 -2
  57. package/src/plugins/get/README.md +1 -1
  58. package/src/plugins/get/get.js +103 -48
  59. package/src/plugins/get/getDoc.js +2 -32
  60. package/src/plugins/get/getDoc.md +36 -0
  61. package/src/plugins/hedberg/README.md +1 -2
  62. package/src/plugins/hedberg/hedberg.js +8 -4
  63. package/src/plugins/hedberg/matcher.js +16 -17
  64. package/src/plugins/hedberg/normalize.js +0 -48
  65. package/src/plugins/helpers.js +42 -2
  66. package/src/plugins/index.js +146 -123
  67. package/src/plugins/instructions/README.md +35 -9
  68. package/src/plugins/instructions/instructions.js +122 -9
  69. package/src/plugins/instructions/instructions.md +25 -0
  70. package/src/plugins/instructions/instructions_104.md +7 -0
  71. package/src/plugins/instructions/instructions_105.md +46 -0
  72. package/src/plugins/instructions/instructions_106.md +0 -0
  73. package/src/plugins/instructions/instructions_107.md +0 -0
  74. package/src/plugins/instructions/instructions_108.md +8 -0
  75. package/src/plugins/instructions/protocol.js +12 -0
  76. package/src/plugins/known/README.md +2 -2
  77. package/src/plugins/known/known.js +67 -36
  78. package/src/plugins/known/knownDoc.js +2 -17
  79. package/src/plugins/known/knownDoc.md +8 -0
  80. package/src/plugins/log/README.md +48 -0
  81. package/src/plugins/log/log.js +109 -0
  82. package/src/plugins/mv/README.md +2 -2
  83. package/src/plugins/mv/mv.js +55 -22
  84. package/src/plugins/mv/mvDoc.js +2 -18
  85. package/src/plugins/mv/mvDoc.md +10 -0
  86. package/src/plugins/ollama/README.md +15 -0
  87. package/src/{llm/OllamaClient.js → plugins/ollama/ollama.js} +40 -18
  88. package/src/plugins/openai/README.md +17 -0
  89. package/src/plugins/openai/openai.js +120 -0
  90. package/src/plugins/openrouter/README.md +27 -0
  91. package/src/plugins/openrouter/openrouter.js +121 -0
  92. package/src/plugins/persona/README.md +20 -0
  93. package/src/plugins/persona/persona.js +9 -16
  94. package/src/plugins/policy/README.md +21 -0
  95. package/src/plugins/policy/policy.js +29 -14
  96. package/src/plugins/prompt/README.md +1 -1
  97. package/src/plugins/prompt/prompt.js +58 -16
  98. package/src/plugins/rm/README.md +1 -1
  99. package/src/plugins/rm/rm.js +56 -12
  100. package/src/plugins/rm/rmDoc.js +2 -20
  101. package/src/plugins/rm/rmDoc.md +13 -0
  102. package/src/plugins/rpc/README.md +2 -2
  103. package/src/plugins/rpc/rpc.js +515 -296
  104. package/src/plugins/set/README.md +1 -1
  105. package/src/plugins/set/set.js +318 -75
  106. package/src/plugins/set/setDoc.js +2 -35
  107. package/src/plugins/set/setDoc.md +22 -0
  108. package/src/plugins/sh/README.md +28 -5
  109. package/src/plugins/sh/sh.js +50 -6
  110. package/src/plugins/sh/shDoc.js +2 -23
  111. package/src/plugins/sh/shDoc.md +13 -0
  112. package/src/plugins/skill/README.md +23 -0
  113. package/src/plugins/skill/skill.js +14 -18
  114. package/src/plugins/stream/README.md +101 -0
  115. package/src/plugins/stream/stream.js +290 -0
  116. package/src/plugins/telemetry/README.md +1 -1
  117. package/src/plugins/telemetry/telemetry.js +129 -80
  118. package/src/plugins/think/README.md +1 -1
  119. package/src/plugins/think/think.js +12 -0
  120. package/src/plugins/think/thinkDoc.js +2 -15
  121. package/src/plugins/think/thinkDoc.md +7 -0
  122. package/src/plugins/unknown/README.md +3 -3
  123. package/src/plugins/unknown/unknown.js +47 -19
  124. package/src/plugins/unknown/unknownDoc.js +2 -21
  125. package/src/plugins/unknown/unknownDoc.md +11 -0
  126. package/src/plugins/update/README.md +1 -1
  127. package/src/plugins/update/update.js +67 -5
  128. package/src/plugins/update/updateDoc.js +2 -30
  129. package/src/plugins/update/updateDoc.md +8 -0
  130. package/src/plugins/xai/README.md +23 -0
  131. package/src/{llm/XaiClient.js → plugins/xai/xai.js} +58 -37
  132. package/src/server/ClientConnection.js +64 -37
  133. package/src/server/SocketServer.js +23 -10
  134. package/src/server/protocol.js +11 -0
  135. package/src/sql/v_model_context.sql +27 -31
  136. package/src/sql/v_run_log.sql +9 -14
  137. package/EXCEPTIONS.md +0 -46
  138. package/FIDELITY_CONTRACT.md +0 -172
  139. package/src/agent/KnownStore.js +0 -337
  140. package/src/agent/ResponseHealer.js +0 -241
  141. package/src/llm/OpenAiClient.js +0 -100
  142. package/src/llm/OpenRouterClient.js +0 -100
  143. package/src/plugins/budget/recovery.js +0 -47
  144. package/src/plugins/instructions/preamble.md +0 -45
  145. package/src/plugins/performed/README.md +0 -15
  146. package/src/plugins/performed/performed.js +0 -45
  147. package/src/plugins/previous/README.md +0 -16
  148. package/src/plugins/previous/previous.js +0 -56
  149. package/src/plugins/progress/README.md +0 -16
  150. package/src/plugins/progress/progress.js +0 -43
  151. package/src/plugins/summarize/README.md +0 -19
  152. package/src/plugins/summarize/summarize.js +0 -32
  153. package/src/plugins/summarize/summarizeDoc.js +0 -27
@@ -1,28 +1,128 @@
1
- import { readFileSync } from "node:fs";
1
+ import { existsSync, readFileSync } from "node:fs";
2
+ import Protocol from "./protocol.js";
2
3
 
3
- const preamble = readFileSync(
4
- new URL("./preamble.md", import.meta.url),
4
+ const baseInstructions = readFileSync(
5
+ new URL("./instructions.md", import.meta.url),
5
6
  "utf8",
6
7
  );
7
8
 
9
+ // 1XY status encoding: X=current phase, Y=next phase. Y routes through
10
+ // phaseForStatus to select next turn's <instructions>. Phases 4–9 are
11
+ // reserved (status codes 1X4..1X9); add new phases by dropping in
12
+ // `instructions_10N.md`. Absent files render no <instructions> block —
13
+ // the model runs on base instructions only. This lets you route ahead
14
+ // of writing the prose (e.g. an upcoming "ask lite" phase 9).
15
+ const PHASES = [4, 5, 6, 7, 8, 9];
16
+ const phaseInstructions = Object.fromEntries(
17
+ PHASES.flatMap((p) => {
18
+ const url = new URL(`./instructions_10${p}.md`, import.meta.url);
19
+ return existsSync(url) ? [[p, readFileSync(url, "utf8").trim()]] : [];
20
+ }),
21
+ );
22
+ const TURN_FROM_PATH = /^log:\/\/turn_(\d+)\/update\//;
23
+
24
+ function phaseForStatus(status) {
25
+ if (status == null) return 4;
26
+ if (status === 200) return 8;
27
+ const last = status % 10;
28
+ return PHASES.includes(last) ? last : 4;
29
+ }
30
+
31
+ // Scan an already-materialized row set for the most recent update
32
+ // emission's status. Used by the assembly.user filter so the phase
33
+ // instructions ride with the user message (dynamic, expected to
34
+ // change every turn) instead of the system prompt (stable, cached).
35
+ // Validation is upstream (update.js isValidStatus + 422 error log) so
36
+ // we trust the status and route on it directly — a whitelist here
37
+ // silently drops advertised completion codes whose contracts drift,
38
+ // which is worse than a noisy fallback.
39
+ function latestUpdateStatusFromRows(rows) {
40
+ let bestTurn = -1;
41
+ let bestStatus = null;
42
+ for (const r of rows) {
43
+ const m = TURN_FROM_PATH.exec(r.path);
44
+ if (!m) continue;
45
+ const turn = Number(m[1]);
46
+ const attrs =
47
+ typeof r.attributes === "string"
48
+ ? JSON.parse(r.attributes)
49
+ : r.attributes;
50
+ const status = attrs?.status;
51
+ if (status == null) continue;
52
+ if (turn > bestTurn || (turn === bestTurn && status > bestStatus)) {
53
+ bestTurn = turn;
54
+ bestStatus = status;
55
+ }
56
+ }
57
+ return bestStatus;
58
+ }
59
+
8
60
  export default class Instructions {
9
61
  #core;
10
62
 
11
63
  constructor(core) {
12
64
  this.#core = core;
13
- core.on("promoted", this.full.bind(this));
65
+ core.on("visible", this.full.bind(this));
14
66
  core.on("turn.started", this.onTurnStarted.bind(this));
67
+ core.hooks.instructions.resolveSystemPrompt =
68
+ this.resolveSystemPrompt.bind(this);
69
+ // Dynamic phase instructions live in the user message (above
70
+ // <prompt>) so the system message stays cache-stable across turns.
71
+ // Priority 250 puts us between <log> (100), <unknowns> (200),
72
+ // and <prompt> (300).
73
+ core.filter("assembly.user", this.assembleInstructions.bind(this), 250);
74
+ new Protocol(core);
75
+ }
76
+
77
+ /**
78
+ * Materialize the system prompt for a run: look up the
79
+ * instructions://system entry, project it through the promoted view.
80
+ * TurnExecutor calls this once per turn before context assembly.
81
+ */
82
+ async resolveSystemPrompt(rummy) {
83
+ const { entries: store, runId, hooks } = rummy;
84
+ const entries = await store.getEntriesByPattern(
85
+ runId,
86
+ "instructions://system",
87
+ null,
88
+ );
89
+ // The entry is always written by onTurnStarted before this runs.
90
+ const entry = entries[0];
91
+ const attributes = await store.getAttributes(
92
+ runId,
93
+ "instructions://system",
94
+ );
95
+ return hooks.tools.view("instructions", {
96
+ path: "instructions://system",
97
+ scheme: "instructions",
98
+ body: entry.body,
99
+ attributes,
100
+ visibility: "visible",
101
+ category: "system",
102
+ });
15
103
  }
16
104
 
17
105
  async onTurnStarted({ rummy }) {
18
106
  const { entries: store, sequence: turn, runId } = rummy;
19
- const runRow = await rummy.db.get_run_by_id.get({ id: runId });
107
+ const runRow = await store.getRun(runId);
20
108
  const toolSet = rummy.toolSet
21
109
  ? [...rummy.toolSet]
22
110
  : this.#core.hooks.tools.names;
23
- await store.upsert(runId, turn, "instructions://system", "", 200, {
111
+ // instructions:// is an audit scheme (writable_by: ["system"]).
112
+ // No per-turn phase state on this entry — keeps the system
113
+ // prompt cache-stable across turns. Phase selection happens at
114
+ // assembly.user time from the current row set.
115
+ await store.set({
116
+ runId,
117
+ turn,
118
+ path: "instructions://system",
119
+ body: "",
120
+ state: "resolved",
121
+ writer: "system",
24
122
  attributes: {
25
- persona: runRow?.persona || null,
123
+ // runRow.persona is a nullable TEXT column; absent row is
124
+ // a system bug — let the null propagate if runRow exists.
125
+ persona: runRow.persona,
26
126
  toolSet,
27
127
  },
28
128
  });
@@ -41,15 +141,28 @@ export default class Instructions {
41
141
  const sorted = this.#core.hooks.tools.advertisedNames.filter((n) =>
42
142
  activeTools.has(n),
43
143
  );
44
- const tools = sorted.join(", ");
144
+ const tools = sorted.map((n) => `<${n}/>`).join(", ");
45
145
  const docsText = sorted
46
146
  .filter((key) => toolDocs[key])
47
147
  .map((key) => toolDocs[key])
48
148
  .join("\n\n");
49
- let prompt = preamble
149
+ let prompt = baseInstructions
50
150
  .replace("[%TOOLS%]", tools)
51
151
  .replace("[%TOOLDOCS%]", docsText);
52
152
  if (attrs.persona) prompt += `\n\n## Persona\n\n${attrs.persona}`;
53
153
  return prompt;
54
154
  }
155
+
156
+ // Renders the current phase's instructions as an <instructions>
157
+ // block in the user message. Runs at priority 250 — after <log>
158
+ // and <unknowns>, immediately before <prompt>. System prompt stays
159
+ // static so prompt caching keeps its prefix intact across turns.
160
+ // A routed phase without an instructions_10N.md file emits nothing —
161
+ // the model proceeds on base instructions alone.
162
+ assembleInstructions(content, ctx) {
163
+ const status = latestUpdateStatusFromRows(ctx.rows);
164
+ const step = phaseInstructions[phaseForStatus(status)];
165
+ if (!step) return content;
166
+ return `${content}<instructions>\n${step}\n</instructions>\n`;
167
+ }
55
168
  }
@@ -0,0 +1,25 @@
1
+ XML Commands Available: [%TOOLS%]
2
+
3
+ # FCRM Engine
4
+
5
+ You are a Folksonomic Context Relevance Maximization (FCRM) engine with a **Primary Directive** of Context Relevance Maximization.
6
+ * Definition Stage: Register everything unknown about the prompt request.
7
+ * Discovery Stage: Discover, Distill, and Demote source entries to resolve unknowns into knowns.
8
+ * Deployment Stage: Act on the prompt.
9
+
10
+ Warning: YOU MUST NOT allow the `tokens="N"` sum of irrelevant source entries or log events to exceed `tokensFree` budget.
11
+
12
+ Tip: The `tokens="N"` shows how much context memory is consumed if "visible". Entries only consume tokens when at "visible" visibility.
13
+ Tip: The "summarized" and "archived" entries and log events use no context memory (`tokensFree="N"`).
14
+ Tip: You can use <get path="..." preview/> to preview the potential `tokens="N"` budget impact of bulk operations.
15
+ Tip: You can use <get path="..." line="X" limit="Y"/> to read subsets of entries that would exceed your `tokensFree` budget.
16
+ Tip: Log items are demotable just like context entries. Demote their visibility to "summarized" or "archived" as needed.
17
+ Tip: Entries and log events that have been archived are fully hidden (no memory used, no summary), but can be retrieved later by path.
18
+
19
+ # Commands
20
+
21
+ Warning: YOU MUST NOT use shell commands for project file operations. Project files are entries that require XML Command operations.
22
+ Example: <set path="src/file.txt">new file content</set>
23
+ Example: <get path="src/*.txt" preview/>
24
+
25
+ [%TOOLDOCS%]
@@ -0,0 +1,7 @@
1
+ Definition Stage: YOU MUST ONLY define relevant information you do not know in this stage.
2
+
3
+ YOU MUST create topical, taxonomized, and tagged unknown:// entries for missing information you need to discover.
4
+ Example: <set path="unknown://countries/france/capital" summary="countries,france,capital,geography,trivia">What is the capital of France?</set>
5
+
6
+ Definition Stage Continuation: <update status="144">identifying more unknowns</update>
7
+ Definition Stage Completion: <update status="145">unknowns identified</update>
@@ -0,0 +1,46 @@
1
+ # Discovery Stage
2
+
3
+ YOU MUST ONLY perform discovery actions (Discover -> Distill -> Demote) during the Discovery Stage.
4
+ YOU MUST discover and get source entries with information relevant to the unknown:// entries.
5
+ YOU MUST create topical, taxonomized, and tagged known:// entries to resolve unknown:// entries.
6
+ YOU MUST include at least 1 link to a relevant unknown and at least 1 link to a relevant source entry in every known:// entry.
7
+ YOU MUST demote source entries to "summarized" after extracting and decomposing their relevant information into known:// entries.
8
+ YOU MUST demote the unknown:// entries to "summarized" after they are referenced or resolved by known:// entries.
9
+ YOU MUST demote all irrelevant source entries and log events to maximize FCRM.
10
+ Tip: Source entry "summarized" information is not reliable. Only place "visible" source entry information in known:// entries.
11
+ Tip: A "relevant" source entry that has been successfully distilled into known:// entries is no longer relevant.
12
+ Tip: Discover, Distill, and Demote per source entry, not globally, to maximize FCRM.
13
+
14
+ ## Discovery Lifecycle: Promoting a source entry, creating a known entry, demoting the source entry, then archiving the resolved unknown
15
+
16
+ ### Discover
17
+
18
+ <set path="trivia/capitals.csv" visibility="visible"/>
19
+
20
+ ### Distill
21
+ <set path="known://countries/france/capital" summary="countries,france,capital,geography,trivia">
22
+ # Capital of France
23
+ The capital of France is Paris.
24
+
25
+ { ... }
26
+
27
+ # References
28
+ [unknown resolving](unknown://countries/france/capital)
29
+ [source entry](trivia/capitals.csv)
30
+ </set>
31
+
32
+ ### Demote
33
+
34
+ <set path="trivia/capitals.csv" visibility="summarized"/>
35
+ <set path="unknown://countries/france/capital" visibility="summarized"/>
36
+ <set path="unknown://countries/poland/capital" summary="REJECTED: Irrelevant" visibility="summarized"/>
37
+ <set path="https://en.wikipedia.org/wiki/Paris,_Texas" summary="REJECTED: Wrong Paris" visibility="summarized"/>
38
+ <set path="log://turn_1/set/*" visibility="archived"/>
39
+ <set path="log://turn_1/get/trivia/*" visibility="archived"/>
40
+ <set path="log://turn_2/get/capital%20of%20france" visibility="archived"/>
41
+
42
+ ## Turn Termination (CHOOSE ONLY ONE):
43
+
44
+ Definition Stage Return: <update status="154">returning to definition stage</update>
45
+ Discovery Stage Continuation: <update status="155">referencing and resolving more unknowns</update>
46
+ Discovery Stage Completion: <update status="158">all unknowns (if any) referenced or resolved by known entries</update>
File without changes
File without changes
@@ -0,0 +1,8 @@
1
+ YOU MUST act on the prompt.
2
+
3
+ Turn Termination (CHOOSE ONLY ONE):
4
+
5
+ Definition Stage Return: <update status="184">returning to definition stage</update>
6
+ Discovery Stage Return: <update status="185">returning to discovery stage</update>
7
+ Deployment Stage Continuation: <update status="188">performing more actions</update>
8
+ Deployment Stage Completion: <update status="200">summary of actions performed, or direct answer</update>
@@ -0,0 +1,12 @@
1
+ export default class Protocol {
2
+ #core;
3
+
4
+ constructor(core) {
5
+ this.#core = core;
6
+ core.filter("entry.recording", this.#onRecording.bind(this), 1);
7
+ }
8
+
9
+ async #onRecording(entry, _ctx) {
10
+ return entry;
11
+ }
12
+ }
@@ -1,6 +1,6 @@
1
- # known
1
+ # known {#known_plugin}
2
2
 
3
- Writes knowledge entries into the store at full fidelity.
3
+ Writes knowledge entries into the store at full visibility.
4
4
 
5
5
  ## Registration
6
6
 
@@ -1,6 +1,7 @@
1
+ import { stateToStatus } from "../../agent/httpStatus.js";
1
2
  import { countTokens } from "../../agent/tokens.js";
2
3
 
3
- const MAX_ENTRY_TOKENS = Number(process.env.RUMMY_MAX_ENTRY_TOKENS) || 512;
4
+ const MAX_ENTRY_TOKENS = Number(process.env.RUMMY_MAX_ENTRY_TOKENS);
4
5
 
5
6
  export default class Known {
6
7
  #core;
@@ -9,9 +10,9 @@ export default class Known {
9
10
  this.#core = core;
10
11
  core.registerScheme({ category: "data" });
11
12
  core.on("handler", this.handler.bind(this));
12
- core.on("promoted", this.full.bind(this));
13
- core.on("demoted", this.summary.bind(this));
14
- core.filter("assembly.system", this.assembleKnown.bind(this), 100);
13
+ core.on("visible", this.full.bind(this));
14
+ core.on("summarized", this.summary.bind(this));
15
+ core.filter("assembly.system", this.assembleContext.bind(this), 100);
15
16
  // <known> is internal — written via <set path="known://...">. Hidden
16
17
  // from all model-facing tool lists. Handler still dispatches if the
17
18
  // model emits <known> directly out of habit.
@@ -26,19 +27,20 @@ export default class Known {
26
27
  const entryTokens = countTokens(entry.body);
27
28
  if (entryTokens > MAX_ENTRY_TOKENS) {
28
29
  const rejectPath = await store.slugPath(runId, "known", entry.body);
29
- await store.upsert(
30
+ await store.set({
30
31
  runId,
31
32
  turn,
32
- rejectPath,
33
- `Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
34
- 413,
35
- { loopId },
36
- );
33
+ path: rejectPath,
34
+ body: `Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
35
+ state: "failed",
36
+ outcome: `overflow:${entryTokens}`,
37
+ loopId,
38
+ });
37
39
  return;
38
40
  }
39
41
 
40
42
  // Resolve path: explicit or auto-generated slug
41
- let knownPath = entry.attributes?.path || null;
43
+ let knownPath = entry.attributes?.path;
42
44
  if (knownPath && !knownPath.includes("://")) {
43
45
  knownPath = `known://${knownPath}`;
44
46
  }
@@ -51,21 +53,30 @@ export default class Known {
51
53
  );
52
54
  }
53
55
 
54
- // Dedup: if path exists, update rather than duplicate
56
+ // Dedup: if path exists, update rather than duplicate. An empty
57
+ // new body means "preserve the existing entry's body" (e.g. the
58
+ // model is updating attributes only).
55
59
  const existing = await store.getEntriesByPattern(runId, knownPath, null);
56
60
  if (existing.length > 0) {
57
- await store.upsert(
61
+ const nextBody = entry.body === "" ? existing[0].body : entry.body;
62
+ await store.set({
58
63
  runId,
59
64
  turn,
60
- existing[0].path,
61
- entry.body || existing[0].body,
62
- 200,
63
- { attributes: entry.attributes, loopId },
64
- );
65
+ path: existing[0].path,
66
+ body: nextBody,
67
+ state: "resolved",
68
+ attributes: entry.attributes,
69
+ loopId,
70
+ });
65
71
  return;
66
72
  }
67
73
 
68
- await store.upsert(runId, turn, knownPath, entry.body, 200, {
74
+ await store.set({
75
+ runId,
76
+ turn,
77
+ path: knownPath,
78
+ body: entry.body,
79
+ state: "resolved",
69
80
  attributes: entry.attributes,
70
81
  loopId,
71
82
  });
@@ -75,33 +86,52 @@ export default class Known {
75
86
  return entry.body;
76
87
  }
77
88
 
78
- summary() {
79
- return "";
89
+ // Summarized knowns keep the first 500 characters so the model
90
+ // doesn't lose the plot when budget auto-demotion kicks in on its
91
+ // own work. Anything larger gets capped so a pathologically big
92
+ // known doesn't saturate the packet at summarized visibility
93
+ // either. Matches the pattern on `<prompt>` summarized view.
94
+ summary(entry) {
95
+ if (!entry.body) return "";
96
+ if (entry.body.length <= 500) return entry.body;
97
+ return `${entry.body.slice(0, 500)}\n[truncated — promote to see the full body]`;
80
98
  }
81
99
 
82
- async assembleKnown(content, ctx) {
100
+ async assembleContext(content, ctx) {
83
101
  const entries = ctx.rows.filter((r) => r.category === "data");
84
102
  if (entries.length === 0) return content;
85
-
86
- // Rows arrive pre-sorted by SQL: summary → full, then by recency
87
- const demotedSet = new Set(ctx.demoted || []);
88
- const lines = entries.map((e) => renderKnownTag(e, demotedSet));
89
- return `${content}\n\n<knowns>\n${lines.join("\n")}\n</knowns>`;
103
+ const demotedSet = new Set(ctx.demoted);
104
+ const lines = entries.map((e) => renderContextTag(e, demotedSet));
105
+ return `${content}\n\n<context>\n${lines.join("\n")}\n</context>`;
90
106
  }
91
107
  }
92
108
 
93
- function renderKnownTag(entry, demotedSet) {
94
- const tag = entry.scheme || "file";
109
+ function renderContextTag(entry, demotedSet) {
110
+ // schemeOf() returns NULL / "" for bare file paths; translate for the tag.
111
+ const tag = entry.scheme ? entry.scheme : "file";
95
112
  const turn = entry.source_turn ? ` turn="${entry.source_turn}"` : "";
96
- const tokens = entry.tokens ? ` tokens="${entry.tokens}"` : "";
97
- const status = entry.status ? ` status="${entry.status}"` : "";
98
- const fidelity = entry.fidelity ? ` fidelity="${entry.fidelity}"` : "";
99
- const flag = demotedSet?.has(entry.path) ? " demoted" : "";
100
-
113
+ const tokens = entry.aTokens != null ? ` tokens="${entry.aTokens}"` : "";
101
114
  const attrs =
102
115
  typeof entry.attributes === "string"
103
116
  ? JSON.parse(entry.attributes)
104
117
  : entry.attributes;
118
+ const statusValue =
119
+ attrs?.status != null
120
+ ? attrs.status
121
+ : entry.state
122
+ ? stateToStatus(entry.state, entry.outcome)
123
+ : null;
124
+ const status =
125
+ statusValue != null && statusValue !== 200
126
+ ? ` status="${statusValue}"`
127
+ : "";
128
+ const stateAttr =
129
+ entry.state && entry.state !== "resolved" ? ` state="${entry.state}"` : "";
130
+ const outcomeAttr = entry.outcome ? ` outcome="${entry.outcome}"` : "";
131
+ const visibility = entry.visibility
132
+ ? ` visibility="${entry.visibility}"`
133
+ : "";
134
+ const flag = demotedSet?.has(entry.path) ? " demoted" : "";
105
135
  // Always render summary attribute on knowns — empty value hints the model
106
136
  // it forgot to add searchable keywords.
107
137
  const summaryText =
@@ -110,8 +140,9 @@ function renderKnownTag(entry, demotedSet) {
110
140
  : "";
111
141
  const summary = ` summary="${summaryText}"`;
112
142
 
143
+ const attrStr = `${turn}${status}${stateAttr}${outcomeAttr}${summary}${visibility}${tokens}${flag}`;
113
144
  if (entry.body) {
114
- return `<${tag} path="${entry.path}"${turn}${status}${summary}${fidelity}${tokens}${flag}>${entry.body}</${tag}>`;
145
+ return `<${tag} path="${entry.path}"${attrStr}>${entry.body}</${tag}>`;
115
146
  }
116
- return `<${tag} path="${entry.path}"${turn}${status}${summary}${fidelity}${tokens}${flag}/>`;
147
+ return `<${tag} path="${entry.path}"${attrStr}/>`;
117
148
  }
@@ -1,18 +1,3 @@
1
- // Tool doc for <known/>. Each entry: [text, rationale].
2
- // Text goes to the model. Rationale stays in source.
3
- // Changing ANY line requires reading ALL rationales first.
4
- const LINES = [
5
- [
6
- '## <known path="known://topic/subtopic" summary="keyword,keyword,keyword">[specific facts, decisions, or plans]</known> - Sort and save what you learn for later recall',
7
- ],
8
- [
9
- 'Example: <known path="known://people/rumsfeld" summary="defense,secretary,born,1932">Donald Rumsfeld was born in 1932 and served as Secretary of Defense</known>',
10
- "Explicit path form: slashed path=category/key, summary=keywords.",
11
- ],
12
- [
13
- '* Recall with <get path="known://people/*">keyword</get>',
14
- "Cross-tool lifecycle: pattern by category, filter by keyword.",
15
- ],
16
- ];
1
+ import { loadDoc } from "../helpers.js";
17
2
 
18
- export default LINES.map(([text]) => text).join("\n");
3
+ export default loadDoc(import.meta.url, "knownDoc.md");
@@ -0,0 +1,8 @@
1
+ ## <set path="known://topic/subtopic" summary="keyword,keyword,keyword">[specific facts, decisions, or plans]</set> - Sort and save what you learn for later recall
2
+ <!-- Use <set> to write known entries (not <known>). Matches instructions examples. -->
3
+
4
+ Example: <set path="known://people/rumsfeld" summary="defense,secretary,born,1932">Donald Rumsfeld was born in 1932 and served as Secretary of Defense</set>
5
+ <!-- Explicit path form: slashed path=category/key, summary=keywords. -->
6
+
7
+ * Recall with <get path="known://people/*">keyword</get>
8
+ <!-- Cross-tool lifecycle: pattern by category, filter by keyword. -->
@@ -0,0 +1,48 @@
1
+ # log {#log_plugin}
2
+
3
+ Assembles the `<log>` block in the user message: every
4
+ `category="logging"` entry across the entire run, rendered as XML tool
5
+ tags in v_model_context sort order.
6
+
7
+ ## Registration
8
+
9
+ - **Filter**: `assembly.user` (priority 100) — contributes the `<log>`
10
+ block to the user packet.
11
+
12
+ ## Rendering
13
+
14
+ Each logging entry renders with its scheme as the tag name (`<get>`,
15
+ `<set>`, `<search>`, `<rm>`, `<cp>`, `<mv>`, `<sh>`, `<env>`,
16
+ `<update>`, `<ask_user>`, `<error>`, `<budget>`). Attributes:
17
+ `path`, `turn`, `status`, `state`, `outcome`, `summary`, `visibility`,
18
+ `tokens`.
19
+
20
+ **`tokens=` invariant.** The value is always the full-visibility cost
21
+ of the thing the tag represents — never the log entry's own stub body
22
+ size. Resolution:
23
+
24
+ - If the log entry has `attrs.path` referencing a data entry (`get`,
25
+ `set`, `mv`, `cp`): `tokens=` is that target's tokens. Promotes the
26
+ audit record into a cost-accurate signal the model can plan against.
27
+ - If the action's log body itself IS the cost-bearing content
28
+ (`search`, `update`, `error`, `ask_user`): `tokens=` is the entry's
29
+ own body tokens.
30
+ - `sh` and `env` own multiple streaming channels (`sh://turn_N/{slug}_N`)
31
+ — no single target to point at. `tokens=` is omitted; the channels
32
+ render their own tokens in `<context>`.
33
+
34
+ ## Behavior
35
+
36
+ No loop-boundary split. The `turn` attribute on every entry carries
37
+ when it happened; the model derives loop membership from the data if
38
+ it matters. One chronological log from turn 1 to now.
39
+
40
+ ## Scheme invariant
41
+
42
+ Log entries (`log://turn_N/{action}/{slug}`) are audit records —
43
+ summary, exit status, references to where the data lives — and never
44
+ carry the payload itself. Payload for streaming actions lives under the
45
+ producer's own scheme (`sh://`, `env://`, future `search://`, etc.) at
46
+ `category=data`, and is rendered inside `<context>` by the known
47
+ plugin. Scheme determines category; data and logging never share a
48
+ scheme. See [scheme_category_split](#scheme_category_split).
@@ -0,0 +1,109 @@
1
+ import { stateToStatus } from "../../agent/httpStatus.js";
2
+
3
+ // Schemes whose log body is an action summary, not the cost-bearing
4
+ // content. For these, the action's cost lives on a separate data entry
5
+ // (sh/env: streaming channels; set/mv/cp: the target entry). Report
6
+ // tokens from the target when we can resolve it (set/mv/cp via
7
+ // attrs.path); omit entirely for sh/env (multiple channels, no single
8
+ // target to point at).
9
+ const STREAM_NO_TOKENS = new Set(["sh", "env"]);
10
+
11
+ export default class Log {
12
+ #core;
13
+
14
+ constructor(core) {
15
+ this.#core = core;
16
+ core.filter("assembly.user", this.assembleLog.bind(this), 100);
17
+ }
18
+
19
+ async assembleLog(content, ctx) {
20
+ // Log includes action entries (scheme=log) AND prior prompts. The
21
+ // most recent prompt is rendered separately by the prompt plugin
22
+ // as `<prompt>`; everything older lives in the log so the model
23
+ // can see the full question history across a sustained run.
24
+ const latestPrompt = ctx.rows.findLast(
25
+ (r) => r.category === "prompt" && r.scheme === "prompt",
26
+ );
27
+ const entries = ctx.rows.filter((r) => {
28
+ if (r.category === "logging" && r.scheme === "log") return true;
29
+ if (r.category === "prompt" && r.scheme === "prompt") {
30
+ return r !== latestPrompt;
31
+ }
32
+ return false;
33
+ });
34
+ if (entries.length === 0) return content;
35
+ const rowsByPath = new Map();
36
+ for (const r of ctx.rows) rowsByPath.set(r.path, r);
37
+ const lines = entries.map((e) => renderLogTag(e, rowsByPath));
38
+ return `${content}<log>\n${lines.join("\n")}\n</log>\n`;
39
+ }
40
+ }
41
+
42
+ // Log paths are log://turn_N/action/slug. The second segment is the
43
+ // action — the plugin/tool that produced this log entry (set, get,
44
+ // search, update, error, etc.). Used as the XML tag name. Prompt
45
+ // entries live at prompt://N; they render as <prompt> in history.
46
+ function actionFromPath(path) {
47
+ if (path?.startsWith("prompt://")) return "prompt";
48
+ const match = path?.match(/^log:\/\/turn_\d+\/([^/]+)\//);
49
+ return match ? match[1] : "log";
50
+ }
51
+
52
+ function renderLogTag(entry, rowsByPath) {
53
+ const attrs =
54
+ typeof entry.attributes === "string"
55
+ ? JSON.parse(entry.attributes)
56
+ : entry.attributes;
57
+
58
+ const action = actionFromPath(entry.path);
59
+
60
+ const statusValue =
61
+ attrs?.status != null
62
+ ? attrs.status
63
+ : entry.state
64
+ ? stateToStatus(entry.state, entry.outcome)
65
+ : null;
66
+ const status = statusValue != null ? ` status="${statusValue}"` : "";
67
+ const outcomeAttr = entry.outcome ? ` outcome="${entry.outcome}"` : "";
68
+ // `tokens=` is the promotion premium (aTokens) of the thing this tag
69
+ // represents — what the model would free by demoting it. For actions
70
+ // that reference a separate data entry (get/set/mv/cp), resolve via
71
+ // attrs.path and report the target's aTokens. For actions whose log
72
+ // body IS the cost-bearing content (search/update/error/ask_user,
73
+ // plus <get> slice reads), use the log entry's own aTokens. sh/env
74
+ // span multiple channel entries and are omitted — the channels
75
+ // render their own tokens in <context>.
76
+ const isSlice = attrs?.lineStart != null;
77
+ const targetEntry = attrs?.path ? rowsByPath.get(attrs.path) : null;
78
+ let tokenSource = null;
79
+ if (STREAM_NO_TOKENS.has(action)) tokenSource = null;
80
+ else if (isSlice) tokenSource = entry.aTokens;
81
+ else if (targetEntry) tokenSource = targetEntry.aTokens;
82
+ else tokenSource = entry.aTokens;
83
+ const tokens = tokenSource != null ? ` tokens="${tokenSource}"` : "";
84
+ const summary =
85
+ typeof attrs?.summary === "string"
86
+ ? ` summary="${attrs.summary.slice(0, 80)}"`
87
+ : "";
88
+ const query =
89
+ typeof attrs?.query === "string" ? ` query="${attrs.query}"` : "";
90
+ const command =
91
+ typeof attrs?.command === "string" ? ` command="${attrs.command}"` : "";
92
+ // target= is the path the action touched (e.g. the file/known that was
93
+ // set, the URL that was fetched). Plugins store it in attrs.path when
94
+ // they write the log entry.
95
+ const target = attrs?.path ? ` target="${attrs.path}"` : "";
96
+ // Slice reads tag the log entry with lineStart/lineEnd/totalLines so
97
+ // the <get> tag surfaces `lines="a-b/total"` — a concrete handle for
98
+ // the model to re-issue or compare against another slice.
99
+ const lines = isSlice
100
+ ? ` lines="${attrs.lineStart}-${attrs.lineEnd}/${attrs.totalLines}"`
101
+ : "";
102
+
103
+ const attrStr = `${target}${status}${outcomeAttr}${query}${command}${summary}${lines}${tokens}`;
104
+
105
+ if (entry.body) {
106
+ return `<${action} path="${entry.path}"${attrStr}>${entry.body}</${action}>`;
107
+ }
108
+ return `<${action} path="${entry.path}"${attrStr}/>`;
109
+ }
@@ -1,4 +1,4 @@
1
- # mv
1
+ # mv {#mv_plugin}
2
2
 
3
3
  Moves (renames) an entry from one path to another within the K/V store.
4
4
 
@@ -15,5 +15,5 @@ Shows `mv {from} {to}`.
15
15
  ## Behavior
16
16
 
17
17
  Warns if the destination already exists and will be overwritten. Uses
18
- `KnownStore.scheme()` to determine scheme vs file paths. Source entry
18
+ `Entries.scheme()` to determine scheme vs file paths. Source entry
19
19
  is removed on successful scheme moves.