@possumtech/rummy 0.5.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/.env.example +42 -5
  2. package/PLUGINS.md +389 -194
  3. package/README.md +25 -8
  4. package/SPEC.md +934 -373
  5. package/bin/demo.js +166 -0
  6. package/bin/rummy.js +9 -3
  7. package/biome/no-fallbacks.grit +50 -0
  8. package/lang/en.json +2 -2
  9. package/migrations/001_initial_schema.sql +88 -37
  10. package/package.json +13 -11
  11. package/scriptify/ask_run.js +77 -0
  12. package/service.js +50 -9
  13. package/src/agent/AgentLoop.js +476 -335
  14. package/src/agent/ContextAssembler.js +4 -4
  15. package/src/agent/Entries.js +676 -0
  16. package/src/agent/ProjectAgent.js +30 -18
  17. package/src/agent/TurnExecutor.js +232 -421
  18. package/src/agent/XmlParser.js +99 -33
  19. package/src/agent/budget.js +56 -0
  20. package/src/agent/errors.js +22 -0
  21. package/src/agent/httpStatus.js +39 -0
  22. package/src/agent/known_checks.sql +8 -4
  23. package/src/agent/known_queries.sql +9 -13
  24. package/src/agent/known_store.sql +280 -125
  25. package/src/agent/materializeContext.js +104 -0
  26. package/src/agent/runs.sql +29 -7
  27. package/src/agent/schemes.sql +14 -3
  28. package/src/agent/tokens.js +6 -0
  29. package/src/agent/turns.sql +9 -9
  30. package/src/hooks/HookRegistry.js +6 -5
  31. package/src/hooks/Hooks.js +44 -3
  32. package/src/hooks/PluginContext.js +29 -21
  33. package/src/{server → hooks}/RpcRegistry.js +2 -1
  34. package/src/hooks/RummyContext.js +139 -35
  35. package/src/hooks/ToolRegistry.js +21 -16
  36. package/src/llm/LlmProvider.js +66 -89
  37. package/src/llm/errors.js +21 -0
  38. package/src/llm/retry.js +63 -0
  39. package/src/plugins/ask_user/README.md +1 -1
  40. package/src/plugins/ask_user/ask_user.js +37 -12
  41. package/src/plugins/ask_user/ask_userDoc.js +2 -25
  42. package/src/plugins/ask_user/ask_userDoc.md +10 -0
  43. package/src/plugins/budget/README.md +27 -25
  44. package/src/plugins/budget/budget.js +306 -88
  45. package/src/plugins/cp/README.md +2 -2
  46. package/src/plugins/cp/cp.js +29 -11
  47. package/src/plugins/cp/cpDoc.js +2 -15
  48. package/src/plugins/cp/cpDoc.md +7 -0
  49. package/src/plugins/engine/README.md +2 -2
  50. package/src/plugins/engine/engine.sql +4 -4
  51. package/src/plugins/engine/turn_context.sql +10 -10
  52. package/src/plugins/env/README.md +20 -5
  53. package/src/plugins/env/env.js +45 -6
  54. package/src/plugins/env/envDoc.js +2 -23
  55. package/src/plugins/env/envDoc.md +13 -0
  56. package/src/plugins/error/README.md +16 -0
  57. package/src/plugins/error/error.js +151 -0
  58. package/src/plugins/file/README.md +6 -6
  59. package/src/plugins/file/file.js +15 -2
  60. package/src/plugins/get/README.md +1 -1
  61. package/src/plugins/get/get.js +103 -48
  62. package/src/plugins/get/getDoc.js +2 -32
  63. package/src/plugins/get/getDoc.md +36 -0
  64. package/src/plugins/hedberg/README.md +1 -2
  65. package/src/plugins/hedberg/hedberg.js +8 -4
  66. package/src/plugins/hedberg/matcher.js +16 -17
  67. package/src/plugins/hedberg/normalize.js +0 -48
  68. package/src/plugins/helpers.js +42 -2
  69. package/src/plugins/index.js +146 -123
  70. package/src/plugins/instructions/README.md +35 -9
  71. package/src/plugins/instructions/instructions.js +244 -9
  72. package/src/plugins/instructions/instructions.md +33 -0
  73. package/src/plugins/instructions/instructions_104.md +7 -0
  74. package/src/plugins/instructions/instructions_105.md +38 -0
  75. package/src/plugins/instructions/instructions_106.md +21 -0
  76. package/src/plugins/instructions/instructions_107.md +10 -0
  77. package/src/plugins/instructions/instructions_108.md +0 -0
  78. package/src/plugins/instructions/protocol.js +12 -0
  79. package/src/plugins/known/README.md +2 -2
  80. package/src/plugins/known/known.js +68 -36
  81. package/src/plugins/known/knownDoc.js +2 -17
  82. package/src/plugins/known/knownDoc.md +8 -0
  83. package/src/plugins/log/README.md +48 -0
  84. package/src/plugins/log/log.js +129 -0
  85. package/src/plugins/mv/README.md +2 -2
  86. package/src/plugins/mv/mv.js +55 -22
  87. package/src/plugins/mv/mvDoc.js +2 -18
  88. package/src/plugins/mv/mvDoc.md +10 -0
  89. package/src/plugins/ollama/README.md +15 -0
  90. package/src/{llm/OllamaClient.js → plugins/ollama/ollama.js} +40 -18
  91. package/src/plugins/openai/README.md +17 -0
  92. package/src/plugins/openai/openai.js +120 -0
  93. package/src/plugins/openrouter/README.md +27 -0
  94. package/src/plugins/openrouter/openrouter.js +121 -0
  95. package/src/plugins/persona/README.md +20 -0
  96. package/src/plugins/persona/persona.js +9 -16
  97. package/src/plugins/policy/README.md +21 -0
  98. package/src/plugins/policy/policy.js +29 -14
  99. package/src/plugins/prompt/README.md +1 -1
  100. package/src/plugins/prompt/prompt.js +64 -16
  101. package/src/plugins/rm/README.md +1 -1
  102. package/src/plugins/rm/rm.js +56 -12
  103. package/src/plugins/rm/rmDoc.js +2 -20
  104. package/src/plugins/rm/rmDoc.md +13 -0
  105. package/src/plugins/rpc/README.md +2 -2
  106. package/src/plugins/rpc/rpc.js +525 -296
  107. package/src/plugins/set/README.md +1 -1
  108. package/src/plugins/set/set.js +318 -75
  109. package/src/plugins/set/setDoc.js +2 -35
  110. package/src/plugins/set/setDoc.md +22 -0
  111. package/src/plugins/sh/README.md +28 -5
  112. package/src/plugins/sh/sh.js +50 -6
  113. package/src/plugins/sh/shDoc.js +2 -23
  114. package/src/plugins/sh/shDoc.md +13 -0
  115. package/src/plugins/skill/README.md +23 -0
  116. package/src/plugins/skill/skill.js +14 -18
  117. package/src/plugins/stream/README.md +101 -0
  118. package/src/plugins/stream/stream.js +290 -0
  119. package/src/plugins/telemetry/README.md +1 -1
  120. package/src/plugins/telemetry/telemetry.js +129 -80
  121. package/src/plugins/think/README.md +1 -1
  122. package/src/plugins/think/think.js +12 -0
  123. package/src/plugins/think/thinkDoc.js +2 -15
  124. package/src/plugins/think/thinkDoc.md +7 -0
  125. package/src/plugins/unknown/README.md +3 -3
  126. package/src/plugins/unknown/unknown.js +47 -19
  127. package/src/plugins/unknown/unknownDoc.js +2 -21
  128. package/src/plugins/unknown/unknownDoc.md +11 -0
  129. package/src/plugins/update/README.md +1 -1
  130. package/src/plugins/update/update.js +83 -5
  131. package/src/plugins/update/updateDoc.js +2 -30
  132. package/src/plugins/update/updateDoc.md +8 -0
  133. package/src/plugins/xai/README.md +23 -0
  134. package/src/{llm/XaiClient.js → plugins/xai/xai.js} +58 -37
  135. package/src/plugins/yolo/yolo.js +192 -0
  136. package/src/server/ClientConnection.js +64 -37
  137. package/src/server/SocketServer.js +23 -10
  138. package/src/server/protocol.js +11 -0
  139. package/src/sql/v_model_context.sql +27 -31
  140. package/src/sql/v_run_log.sql +9 -14
  141. package/EXCEPTIONS.md +0 -46
  142. package/FIDELITY_CONTRACT.md +0 -172
  143. package/src/agent/KnownStore.js +0 -337
  144. package/src/agent/ResponseHealer.js +0 -241
  145. package/src/llm/OpenAiClient.js +0 -100
  146. package/src/llm/OpenRouterClient.js +0 -100
  147. package/src/plugins/budget/recovery.js +0 -47
  148. package/src/plugins/instructions/preamble.md +0 -45
  149. package/src/plugins/performed/README.md +0 -15
  150. package/src/plugins/performed/performed.js +0 -45
  151. package/src/plugins/previous/README.md +0 -16
  152. package/src/plugins/previous/previous.js +0 -56
  153. package/src/plugins/progress/README.md +0 -16
  154. package/src/plugins/progress/progress.js +0 -43
  155. package/src/plugins/summarize/README.md +0 -19
  156. package/src/plugins/summarize/summarize.js +0 -32
  157. package/src/plugins/summarize/summarizeDoc.js +0 -27
@@ -1,28 +1,250 @@
1
- import { readFileSync } from "node:fs";
1
+ import { existsSync, readFileSync } from "node:fs";
2
+ import Protocol from "./protocol.js";
2
3
 
3
- const preamble = readFileSync(
4
- new URL("./preamble.md", import.meta.url),
4
+ const baseInstructions = readFileSync(
5
+ new URL("./instructions.md", import.meta.url),
5
6
  "utf8",
6
7
  );
7
8
 
9
+ // 1XY status encoding: X=current phase, Y=next phase. Y routes through
10
+ // phaseForStatus to select next turn's <instructions>. Phases 4–9 are
11
+ // reserved (status codes 1X4..1X9); add new phases by dropping in
12
+ // `instructions_10N.md`. Absent files render no <instructions> block —
13
+ // the model runs on base instructions only. This lets you route ahead
14
+ // of writing the prose (e.g. an upcoming "ask lite" phase 9).
15
+ const PHASES = [4, 5, 6, 7, 8, 9];
16
+ const phaseInstructions = Object.fromEntries(
17
+ PHASES.flatMap((p) => {
18
+ const url = new URL(`./instructions_10${p}.md`, import.meta.url);
19
+ return existsSync(url) ? [[p, readFileSync(url, "utf8").trim()]] : [];
20
+ }),
21
+ );
22
+ const TURN_FROM_PATH = /^log:\/\/turn_(\d+)\/update\//;
23
+
24
+ function phaseForStatus(status) {
25
+ if (status == null) return 4;
26
+ if (status === 200) return 7;
27
+ const last = status % 10;
28
+ return PHASES.includes(last) ? last : 4;
29
+ }
30
+
31
+ // Scan an already-materialized row set for the most recent update
32
+ // emission's status. Used by the assembly.user filter so the phase
33
+ // instructions ride with the user message (dynamic, expected to
34
+ // change every turn) instead of the system prompt (stable, cached).
35
+ // Validation is upstream (update.js isValidStatus + 422 error log) so
36
+ // we trust the status and route on it directly — a whitelist here
37
+ // silently drops advertised completion codes whose contracts drift,
38
+ // which is worse than a noisy fallback.
39
+ function latestUpdateStatusFromRows(rows) {
40
+ let bestTurn = -1;
41
+ let bestStatus = null;
42
+ for (const r of rows) {
43
+ const m = TURN_FROM_PATH.exec(r.path);
44
+ if (!m) continue;
45
+ const turn = Number(m[1]);
46
+ const attrs =
47
+ typeof r.attributes === "string"
48
+ ? JSON.parse(r.attributes)
49
+ : r.attributes;
50
+ const status = attrs?.status;
51
+ if (status == null) continue;
52
+ // Rejected updates are written for the model's audit trail but are
53
+ // not navigation events — phase router skips them so the model
54
+ // stays in the stage it was already in.
55
+ if (attrs?.rejected) continue;
56
+ if (turn > bestTurn || (turn === bestTurn && status > bestStatus)) {
57
+ bestTurn = turn;
58
+ bestStatus = status;
59
+ }
60
+ }
61
+ return bestStatus;
62
+ }
63
+
8
64
  export default class Instructions {
9
65
  #core;
10
66
 
11
67
  constructor(core) {
12
68
  this.#core = core;
13
- core.on("promoted", this.full.bind(this));
69
+ core.on("visible", this.full.bind(this));
14
70
  core.on("turn.started", this.onTurnStarted.bind(this));
71
+ core.hooks.instructions.resolveSystemPrompt =
72
+ this.resolveSystemPrompt.bind(this);
73
+ core.hooks.instructions.validateNavigation =
74
+ this.validateNavigation.bind(this);
75
+ core.hooks.instructions.findLatestSummary =
76
+ this.findLatestSummary.bind(this);
77
+ // Dynamic phase instructions live in the user message (above
78
+ // <prompt>) so the system message stays cache-stable across turns.
79
+ // Priority 250 puts us between <log> (100), <unknowns> (200),
80
+ // and <prompt> (300).
81
+ core.filter("assembly.user", this.assembleInstructions.bind(this), 250);
82
+ new Protocol(core);
83
+ }
84
+
85
+ /**
86
+ * Materialize the system prompt for a run: look up the
87
+ * instructions://system entry, project it through the promoted view.
88
+ * TurnExecutor calls this once per turn before context assembly.
89
+ */
90
+ async resolveSystemPrompt(rummy) {
91
+ const { entries: store, runId, hooks } = rummy;
92
+ const entries = await store.getEntriesByPattern(
93
+ runId,
94
+ "instructions://system",
95
+ null,
96
+ { includeAuditSchemes: true },
97
+ );
98
+ // The entry is always written by onTurnStarted before this runs.
99
+ const entry = entries[0];
100
+ const attributes = await store.getAttributes(
101
+ runId,
102
+ "instructions://system",
103
+ );
104
+ return hooks.tools.view("instructions", {
105
+ path: "instructions://system",
106
+ scheme: "instructions",
107
+ body: entry.body,
108
+ attributes,
109
+ visibility: "visible",
110
+ category: "system",
111
+ });
112
+ }
113
+
114
+ /**
115
+ * Reject illegal stage navigation. Two checks:
116
+ *
117
+ * 1. Forward skip — `nextPhase > currentPhase + 1`. Models advancing
118
+ * more than one stage at a time are jumping past required work.
119
+ * Returns and continuations (nextPhase ≤ currentPhase) always pass.
120
+ *
121
+ * 2. Deployment with prior prompts — any status landing the model in
122
+ * Deployment (phase 7) requires zero visible PRIOR prompts. State-
123
+ * property rule covering both entry (167) and continuation (177,
124
+ * 200) — once in Deployment, the model still can't claim it with
125
+ * undemoted prior prompts. The current (latest) prompt always
126
+ * stays visible since Deployment must act on it.
127
+ *
128
+ * On rejection the caller marks the update entry rejected (so the
129
+ * phase router skips it) and emits an error log; navigation rejections
130
+ * count as normal strikes.
131
+ */
132
+ async validateNavigation(status, rummy) {
133
+ const currentPhase = await this.#getCurrentPhase(rummy);
134
+ const nextPhase = phaseForStatus(status);
135
+ if (nextPhase > currentPhase + 1) {
136
+ return { ok: false, reason: "Illegal navigation attempt" };
137
+ }
138
+ if (nextPhase === 7) {
139
+ const visible = await this.#countVisiblePriorPrompts(rummy);
140
+ if (visible > 0) {
141
+ return {
142
+ ok: false,
143
+ reason: `Illegal navigation attempt: ${visible} visible prior prompts`,
144
+ };
145
+ }
146
+ }
147
+ return { ok: true };
148
+ }
149
+
150
+ async #getCurrentPhase(rummy) {
151
+ // `**` (not `*`) for the slug position — update slugs are derived
152
+ // from the model's update body and can contain URL-encoded `/`
153
+ // characters (e.g. `known%3A//foo/bar` in a "ready for deployment"
154
+ // summary). Single `*` doesn't cross those embedded slashes and
155
+ // silently misses the prior turn's update.
156
+ const updates = await rummy.entries.getEntriesByPattern(
157
+ rummy.runId,
158
+ "log://*/update/**",
159
+ null,
160
+ );
161
+ let bestTurn = -1;
162
+ let bestStatus = null;
163
+ for (const e of updates) {
164
+ const m = TURN_FROM_PATH.exec(e.path);
165
+ if (!m) continue;
166
+ const turn = Number(m[1]);
167
+ if (turn >= rummy.sequence) continue;
168
+ const attrs =
169
+ typeof e.attributes === "string"
170
+ ? JSON.parse(e.attributes)
171
+ : e.attributes;
172
+ if (attrs?.rejected) continue;
173
+ if (attrs?.status == null) continue;
174
+ if (turn > bestTurn) {
175
+ bestTurn = turn;
176
+ bestStatus = attrs.status;
177
+ }
178
+ }
179
+ return phaseForStatus(bestStatus);
180
+ }
181
+
182
+ /**
183
+ * Find the latest successful Deployment summary from a log-entry list.
184
+ * Matches `log://turn_N/update/...` entries with status=200 (successful
185
+ * Deployment completion) and returns the most recent. Used by
186
+ * AgentLoop telemetry to surface the model's latest delivery.
187
+ *
188
+ * Lives here, not in AgentLoop, because "what counts as a summary" is
189
+ * state-machine knowledge — phase 7's success status (200) is the
190
+ * definition. AgentLoop just consumes the result.
191
+ */
192
+ findLatestSummary(logEntries) {
193
+ return logEntries
194
+ .filter((e) => {
195
+ if (!TURN_FROM_PATH.test(e.path)) return false;
196
+ const attrs =
197
+ typeof e.attributes === "string"
198
+ ? JSON.parse(e.attributes)
199
+ : e.attributes;
200
+ return attrs?.status === 200;
201
+ })
202
+ .at(-1);
203
+ }
204
+
205
+ async #countVisiblePriorPrompts(rummy) {
206
+ const prompts = await rummy.entries.getEntriesByPattern(
207
+ rummy.runId,
208
+ "prompt://*",
209
+ null,
210
+ );
211
+ const visible = prompts.filter((p) => p.visibility === "visible");
212
+ if (visible.length === 0) return 0;
213
+ // Exclude the current (latest) prompt — that's what Deployment acts on.
214
+ // Demoting it would force the model to deliver on content it hid from
215
+ // itself. Only PRIOR prompts are subject to demote-before-Deployment.
216
+ let maxNum = -1;
217
+ for (const p of visible) {
218
+ const m = /^prompt:\/\/(\d+)$/.exec(p.path);
219
+ if (m && Number(m[1]) > maxNum) maxNum = Number(m[1]);
220
+ }
221
+ return visible.filter((p) => {
222
+ const m = /^prompt:\/\/(\d+)$/.exec(p.path);
223
+ return !m || Number(m[1]) !== maxNum;
224
+ }).length;
15
225
  }
16
226
 
17
227
  async onTurnStarted({ rummy }) {
18
228
  const { entries: store, sequence: turn, runId } = rummy;
19
- const runRow = await rummy.db.get_run_by_id.get({ id: runId });
229
+ const runRow = await store.getRun(runId);
20
230
  const toolSet = rummy.toolSet
21
231
  ? [...rummy.toolSet]
22
232
  : this.#core.hooks.tools.names;
23
- await store.upsert(runId, turn, "instructions://system", "", 200, {
233
+ // instructions:// is an audit scheme (writable_by: ["system"]).
234
+ // No per-turn phase state on this entry — keeps the system
235
+ // prompt cache-stable across turns. Phase selection happens at
236
+ // assembly.user time from the current row set.
237
+ await store.set({
238
+ runId,
239
+ turn,
240
+ path: "instructions://system",
241
+ body: "",
242
+ state: "resolved",
243
+ writer: "system",
24
244
  attributes: {
25
- persona: runRow?.persona || null,
245
+ // runRow.persona is a nullable TEXT column; absent row is
246
+ // a system bug — let the null propagate if runRow exists.
247
+ persona: runRow.persona,
26
248
  toolSet,
27
249
  },
28
250
  });
@@ -41,15 +263,28 @@ export default class Instructions {
41
263
  const sorted = this.#core.hooks.tools.advertisedNames.filter((n) =>
42
264
  activeTools.has(n),
43
265
  );
44
- const tools = sorted.join(", ");
266
+ const tools = sorted.map((n) => `<${n}/>`).join(", ");
45
267
  const docsText = sorted
46
268
  .filter((key) => toolDocs[key])
47
269
  .map((key) => toolDocs[key])
48
270
  .join("\n\n");
49
- let prompt = preamble
271
+ let prompt = baseInstructions
50
272
  .replace("[%TOOLS%]", tools)
51
273
  .replace("[%TOOLDOCS%]", docsText);
52
274
  if (attrs.persona) prompt += `\n\n## Persona\n\n${attrs.persona}`;
53
275
  return prompt;
54
276
  }
277
+
278
+ // Renders the current phase's instructions as an <instructions>
279
+ // block in the user message. Runs at priority 250 — after <log>
280
+ // and <unknowns>, immediately before <prompt>. System prompt stays
281
+ // static so prompt caching keeps its prefix intact across turns.
282
+ // A routed phase without an instructions_10N.md file emits nothing —
283
+ // the model proceeds on base instructions alone.
284
+ assembleInstructions(content, ctx) {
285
+ const status = latestUpdateStatusFromRows(ctx.rows);
286
+ const step = phaseInstructions[phaseForStatus(status)];
287
+ if (!step) return content;
288
+ return `${content}<instructions>\n${step}\n</instructions>\n`;
289
+ }
55
290
  }
@@ -0,0 +1,33 @@
1
+ XML Commands Available: [%TOOLS%]
2
+
3
+ # FCRM State Machine
4
+
5
+ You are a Folksonomic Context Relevance Maximization (FCRM) State Machine
6
+
7
+ YOU MUST perform the actions corresponding with your current stage:
8
+ * Definition Stage: Defining what's unknown into unknown:// entries
9
+ * Discovery Stage: Selecting an unknown, discovering relevant source entries and prompts, then distilling them into known:// entries
10
+ * Demotion Stage: Demoting the unknown entries, source entries, prompts, and log events after distillation is completed
11
+ * Deployment Stage: Acting on the current prompt
12
+ * Resolution Stage: Multi-prompt benchmark final `fcrmScore`
13
+
14
+ ## Visibility States: Promote and Demote Visibility State to Control Context Relevance
15
+ * visible: Fully visible, but uses `tokens="N"` context budget
16
+ * summarized: Approximate, summary information, very small context budget penalty
17
+ * archived: Hidden from Context, but can be retrieved later with <get path="..."/>
18
+
19
+ Tip: You can leverage the FCRM's Visibility States with folksonomic taxonomies and tags to store and recall unlimited information.
20
+ Tip: The `tokens="N"` shows how much context memory is consumed if "visible". Entries only consume tokens when at "visible" visibility.
21
+
22
+ Warning: YOU MUST NOT allow the `tokens="N"` sum of irrelevant source entries, prompts, or log events to exceed `tokensFree` budget.
23
+ Warning: YOU MUST NOT skip or avoid state machine steps or the Resolution Stage will fail.
24
+
25
+ # Commands
26
+
27
+ Warning: YOU MUST NOT use shell commands for project file operations. Project files are entries that require XML Commands.
28
+ Example: <set path="src/file.txt">new file content</set>
29
+ Example: <get path="src/*.txt" preview/>
30
+
31
+ Tip: Project files, entries, prompts, and log events are all accessible with the XML Commands.
32
+
33
+ [%TOOLDOCS%]
@@ -0,0 +1,7 @@
1
+ # Definition Stage: YOU MUST ONLY create topical, taxonomized, and tagged unknown:// entries for missing information
2
+
3
+ Example: <set path="unknown://countries/france/capital" summary="countries,france,capital,geography,trivia">What is the capital of France?</set>
4
+
5
+
6
+ ## Turn Termination:
7
+ * Definition Stage Completion: <update status="145">unknowns identified</update>
@@ -0,0 +1,38 @@
1
+ # Discovery Stage: YOU MUST select an unknown:// entry, then discover its source entries and distill them into known:// entries
2
+
3
+ YOU MUST create topical, taxonomized, and tagged known:// entries to resolve the selected unknown:// entry.
4
+ YOU MUST reference all related source entries and prompts.
5
+ YOU MUST ONLY populate known entries with promoted information, NOT from your own training data or opinion.
6
+ YOU MUST immediately demote unknowns, source entries, prompts, and log events after they are distilled, irrelevant, or resolved.
7
+
8
+ Tip: Check the `tokens="N"` of the source entries against the `tokensFree="N"` constraint before promoting entries.
9
+ Tip: You can use <get path="..." preview/> to preview the potential `tokens="N"` budget impact of bulk operations.
10
+ Tip: You can use <get path="..." line="X" limit="Y"/> to read subsets of entries that would exceed your `tokensFree` budget.
11
+
12
+ ## Example:
13
+ <get path="**" preview>capital</get>
14
+ <get path="prompt://3" line="1" limit="100"/>
15
+
16
+ <set path="trivia/capitals.csv" visibility="visible"/>
17
+
18
+ <set path="known://countries/france/capital" summary="countries,france,capital,geography,trivia">
19
+ # Capital of France
20
+ The capital of France is Paris.
21
+
22
+ {...}
23
+
24
+ ## Related
25
+ [trivia question](prompt://3)
26
+ [unknown resolving](unknown://countries/france/capital)
27
+ [source entry](trivia/capitals.csv)
28
+ </set>
29
+
30
+ <set path="prompt://3" visibility="summarized"/>
31
+ <set path="unknown://countries/france/capital" visibility="summarized"/>
32
+ <set path="unknown://countries/france/seat_of_government" summary="RESOLVED: Not necessary" visibility="summarized"/>
33
+ <set path="trivia/capitals.csv" visibility="summarized"/>
34
+
35
+ ## Turn Termination (CHOOSE ONLY ONE):
36
+ * Definition Stage Return: <update status="154">returning to Definition Stage</update>
37
+ * Discovery Stage Continuation: <update status="155">discovering and distilling more for the selected unknown</update>
38
+ * Discovery Stage Completion: <update status="156">this unknown's known entries written</update>
@@ -0,0 +1,21 @@
1
+ # Demotion Stage: YOU MUST demote all source entries, prompts, and log events that are now distilled or no longer relevant
2
+
3
+ Examples:
4
+ <set path="prompt://2" summary="All information distilled into knowns" visibility="summarized"/>
5
+ <set path="trivia/capitals.csv" visibility="summarized"/>
6
+ <set path="unknown://countries/france/capital" visibility="summarized"/>
7
+ <set path="unknown://countries/poland/capital" summary="REJECTED: Irrelevant" visibility="summarized"/>
8
+ <set path="https://en.wikipedia.org/wiki/Paris,_Texas" summary="REJECTED: Wrong Paris" visibility="summarized"/>
9
+ <set path="log://turn_1/**" visibility="archived"/>
10
+ <set path="log://turn_2/**" visibility="archived"/>
11
+ <set path="log://turn_3/set/**" visibility="archived"/>
12
+ <set path="log://turn_3/get/**" visibility="archived"/>
13
+ <set path="log://turn_3/search/**" visibility="archived"/>
14
+
15
+ Tip: You need room to think. Demote large prompts and source entries, then iterate them with <get path="..." line="N" limit="N"/> as necessary.
16
+
17
+ ## Turn Termination (CHOOSE ONLY ONE):
18
+ * Definition Stage Return: <update status="164">returning to Definition Stage</update>
19
+ * Discovery Stage Return: <update status="165">more unknowns remain; returning to Discovery Stage</update>
20
+ * Demotion Stage Continuation: <update status="166">demoting more distilled or irrelevant entries, prompts, and log events</update>
21
+ * Demotion Stage Completion: <update status="167">all unknowns resolved and demoted; ready for Deployment Stage</update>
@@ -0,0 +1,10 @@
1
+ # Deployment Stage
2
+
3
+ YOU MUST act on the prompt.
4
+
5
+ ## Turn Termination (CHOOSE ONLY ONE):
6
+ * Definition Stage Return: <update status="174">returning to Definition Stage</update>
7
+ * Discovery Stage Return: <update status="175">returning to Discovery Stage</update>
8
+ * Demotion Stage Return: <update status="176">returning to Demotion Stage</update>
9
+ * Deployment Stage Continuation: <update status="177">performing more actions</update>
10
+ * Deployment Stage Completion: <update status="200">{direct answer if prompt asked a question, summary of actions if not}</update>
File without changes
@@ -0,0 +1,12 @@
1
+ export default class Protocol {
2
+ #core;
3
+
4
+ constructor(core) {
5
+ this.#core = core;
6
+ core.filter("entry.recording", this.#onRecording.bind(this), 1);
7
+ }
8
+
9
+ async #onRecording(entry, _ctx) {
10
+ return entry;
11
+ }
12
+ }
@@ -1,6 +1,6 @@
1
- # known
1
+ # known {#known_plugin}
2
2
 
3
- Writes knowledge entries into the store at full fidelity.
3
+ Writes knowledge entries into the store at full visibility.
4
4
 
5
5
  ## Registration
6
6
 
@@ -1,6 +1,7 @@
1
+ import { stateToStatus } from "../../agent/httpStatus.js";
1
2
  import { countTokens } from "../../agent/tokens.js";
2
3
 
3
- const MAX_ENTRY_TOKENS = Number(process.env.RUMMY_MAX_ENTRY_TOKENS) || 512;
4
+ const MAX_ENTRY_TOKENS = Number(process.env.RUMMY_MAX_ENTRY_TOKENS);
4
5
 
5
6
  export default class Known {
6
7
  #core;
@@ -9,9 +10,9 @@ export default class Known {
9
10
  this.#core = core;
10
11
  core.registerScheme({ category: "data" });
11
12
  core.on("handler", this.handler.bind(this));
12
- core.on("promoted", this.full.bind(this));
13
- core.on("demoted", this.summary.bind(this));
14
- core.filter("assembly.system", this.assembleKnown.bind(this), 100);
13
+ core.on("visible", this.full.bind(this));
14
+ core.on("summarized", this.summary.bind(this));
15
+ core.filter("assembly.system", this.assembleContext.bind(this), 100);
15
16
  // <known> is internal — written via <set path="known://...">. Hidden
16
17
  // from all model-facing tool lists. Handler still dispatches if the
17
18
  // model emits <known> directly out of habit.
@@ -26,19 +27,20 @@ export default class Known {
26
27
  const entryTokens = countTokens(entry.body);
27
28
  if (entryTokens > MAX_ENTRY_TOKENS) {
28
29
  const rejectPath = await store.slugPath(runId, "known", entry.body);
29
- await store.upsert(
30
+ await store.set({
30
31
  runId,
31
32
  turn,
32
- rejectPath,
33
- `Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
34
- 413,
35
- { loopId },
36
- );
33
+ path: rejectPath,
34
+ body: `Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
35
+ state: "failed",
36
+ outcome: `overflow:${entryTokens}`,
37
+ loopId,
38
+ });
37
39
  return;
38
40
  }
39
41
 
40
42
  // Resolve path: explicit or auto-generated slug
41
- let knownPath = entry.attributes?.path || null;
43
+ let knownPath = entry.attributes?.path;
42
44
  if (knownPath && !knownPath.includes("://")) {
43
45
  knownPath = `known://${knownPath}`;
44
46
  }
@@ -51,21 +53,30 @@ export default class Known {
51
53
  );
52
54
  }
53
55
 
54
- // Dedup: if path exists, update rather than duplicate
56
+ // Dedup: if path exists, update rather than duplicate. An empty
57
+ // new body means "preserve the existing entry's body" (e.g. the
58
+ // model is updating attributes only).
55
59
  const existing = await store.getEntriesByPattern(runId, knownPath, null);
56
60
  if (existing.length > 0) {
57
- await store.upsert(
61
+ const nextBody = entry.body === "" ? existing[0].body : entry.body;
62
+ await store.set({
58
63
  runId,
59
64
  turn,
60
- existing[0].path,
61
- entry.body || existing[0].body,
62
- 200,
63
- { attributes: entry.attributes, loopId },
64
- );
65
+ path: existing[0].path,
66
+ body: nextBody,
67
+ state: "resolved",
68
+ attributes: entry.attributes,
69
+ loopId,
70
+ });
65
71
  return;
66
72
  }
67
73
 
68
- await store.upsert(runId, turn, knownPath, entry.body, 200, {
74
+ await store.set({
75
+ runId,
76
+ turn,
77
+ path: knownPath,
78
+ body: entry.body,
79
+ state: "resolved",
69
80
  attributes: entry.attributes,
70
81
  loopId,
71
82
  });
@@ -75,33 +86,53 @@ export default class Known {
75
86
  return entry.body;
76
87
  }
77
88
 
78
- summary() {
79
- return "";
89
+ // Summarized knowns keep the first 500 characters so the model
90
+ // doesn't lose the plot when budget auto-demotion kicks in on its
91
+ // own work. Anything larger gets capped so a pathologically big
92
+ // known doesn't saturate the packet at summarized visibility
93
+ // either. Matches the pattern on `<prompt>` summarized view.
94
+ summary(entry) {
95
+ if (!entry.body) return "";
96
+ if (entry.body.length <= 500) return entry.body;
97
+ return `${entry.body.slice(0, 500)}\n[truncated — promote to see the full body]`;
80
98
  }
81
99
 
82
- async assembleKnown(content, ctx) {
100
+ async assembleContext(content, ctx) {
83
101
  const entries = ctx.rows.filter((r) => r.category === "data");
84
102
  if (entries.length === 0) return content;
85
-
86
- // Rows arrive pre-sorted by SQL: summary → full, then by recency
87
- const demotedSet = new Set(ctx.demoted || []);
88
- const lines = entries.map((e) => renderKnownTag(e, demotedSet));
89
- return `${content}\n\n<knowns>\n${lines.join("\n")}\n</knowns>`;
103
+ const demotedSet = new Set(ctx.demoted);
104
+ const lines = entries.map((e) => renderContextTag(e, demotedSet));
105
+ return `${content}\n\n<context>\n${lines.join("\n")}\n</context>`;
90
106
  }
91
107
  }
92
108
 
93
- function renderKnownTag(entry, demotedSet) {
94
- const tag = entry.scheme || "file";
109
+ function renderContextTag(entry, demotedSet) {
110
+ // schemeOf() returns NULL / "" for bare file paths; translate for the tag.
111
+ const tag = entry.scheme ? entry.scheme : "file";
95
112
  const turn = entry.source_turn ? ` turn="${entry.source_turn}"` : "";
96
- const tokens = entry.tokens ? ` tokens="${entry.tokens}"` : "";
97
- const status = entry.status ? ` status="${entry.status}"` : "";
98
- const fidelity = entry.fidelity ? ` fidelity="${entry.fidelity}"` : "";
99
- const flag = demotedSet?.has(entry.path) ? " demoted" : "";
100
-
113
+ const tokens = entry.aTokens != null ? ` tokens="${entry.aTokens}"` : "";
114
+ const lines = entry.vLines != null ? ` lines="${entry.vLines}"` : "";
101
115
  const attrs =
102
116
  typeof entry.attributes === "string"
103
117
  ? JSON.parse(entry.attributes)
104
118
  : entry.attributes;
119
+ const statusValue =
120
+ attrs?.status != null
121
+ ? attrs.status
122
+ : entry.state
123
+ ? stateToStatus(entry.state, entry.outcome)
124
+ : null;
125
+ const status =
126
+ statusValue != null && statusValue !== 200
127
+ ? ` status="${statusValue}"`
128
+ : "";
129
+ const stateAttr =
130
+ entry.state && entry.state !== "resolved" ? ` state="${entry.state}"` : "";
131
+ const outcomeAttr = entry.outcome ? ` outcome="${entry.outcome}"` : "";
132
+ const visibility = entry.visibility
133
+ ? ` visibility="${entry.visibility}"`
134
+ : "";
135
+ const flag = demotedSet?.has(entry.path) ? " demoted" : "";
105
136
  // Always render summary attribute on knowns — empty value hints the model
106
137
  // it forgot to add searchable keywords.
107
138
  const summaryText =
@@ -110,8 +141,9 @@ function renderKnownTag(entry, demotedSet) {
110
141
  : "";
111
142
  const summary = ` summary="${summaryText}"`;
112
143
 
144
+ const attrStr = `${turn}${status}${stateAttr}${outcomeAttr}${summary}${visibility}${tokens}${lines}${flag}`;
113
145
  if (entry.body) {
114
- return `<${tag} path="${entry.path}"${turn}${status}${summary}${fidelity}${tokens}${flag}>${entry.body}</${tag}>`;
146
+ return `<${tag} path="${entry.path}"${attrStr}>${entry.body}</${tag}>`;
115
147
  }
116
- return `<${tag} path="${entry.path}"${turn}${status}${summary}${fidelity}${tokens}${flag}/>`;
148
+ return `<${tag} path="${entry.path}"${attrStr}/>`;
117
149
  }
@@ -1,18 +1,3 @@
1
- // Tool doc for <known/>. Each entry: [text, rationale].
2
- // Text goes to the model. Rationale stays in source.
3
- // Changing ANY line requires reading ALL rationales first.
4
- const LINES = [
5
- [
6
- '## <known path="known://topic/subtopic" summary="keyword,keyword,keyword">[specific facts, decisions, or plans]</known> - Sort and save what you learn for later recall',
7
- ],
8
- [
9
- 'Example: <known path="known://people/rumsfeld" summary="defense,secretary,born,1932">Donald Rumsfeld was born in 1932 and served as Secretary of Defense</known>',
10
- "Explicit path form: slashed path=category/key, summary=keywords.",
11
- ],
12
- [
13
- '* Recall with <get path="known://people/*">keyword</get>',
14
- "Cross-tool lifecycle: pattern by category, filter by keyword.",
15
- ],
16
- ];
1
+ import { loadDoc } from "../helpers.js";
17
2
 
18
- export default LINES.map(([text]) => text).join("\n");
3
+ export default loadDoc(import.meta.url, "knownDoc.md");
@@ -0,0 +1,8 @@
1
+ ## <set path="known://topic/subtopic" summary="keyword,keyword,keyword">[specific facts, decisions, or plans]</set> - Sort and save what you learn for later recall
2
+ <!-- Use <set> to write known entries (not <known>). Matches instructions examples. -->
3
+
4
+ Example: <set path="known://people/rumsfeld" summary="defense,secretary,born,1932">Donald Rumsfeld was born in 1932 and served as Secretary of Defense</set>
5
+ <!-- Explicit path form: slashed path=category/key, summary=keywords. -->
6
+
7
+ * Recall with <get path="known://people/*">keyword</get>
8
+ <!-- Cross-tool lifecycle: pattern by category, filter by keyword. -->