@possumtech/rummy 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/.env.example +12 -7
  2. package/BENCH_ENVIRONMENT.md +230 -0
  3. package/CLIENT_INTERFACE.md +396 -0
  4. package/PLUGINS.md +93 -1
  5. package/SPEC.md +305 -28
  6. package/bin/postinstall.js +2 -2
  7. package/bin/rummy.js +2 -2
  8. package/last_run.txt +5617 -0
  9. package/migrations/001_initial_schema.sql +2 -1
  10. package/package.json +6 -2
  11. package/scriptify/cache_probe.js +66 -0
  12. package/scriptify/cache_probe_grok.js +74 -0
  13. package/service.js +22 -11
  14. package/src/agent/AgentLoop.js +33 -139
  15. package/src/agent/ContextAssembler.js +2 -9
  16. package/src/agent/Entries.js +36 -101
  17. package/src/agent/ProjectAgent.js +2 -9
  18. package/src/agent/TurnExecutor.js +45 -83
  19. package/src/agent/XmlParser.js +247 -273
  20. package/src/agent/budget.js +5 -28
  21. package/src/agent/config.js +38 -0
  22. package/src/agent/errors.js +7 -13
  23. package/src/agent/httpStatus.js +1 -19
  24. package/src/agent/known_store.sql +7 -2
  25. package/src/agent/materializeContext.js +12 -17
  26. package/src/agent/pathEncode.js +5 -0
  27. package/src/agent/rummyHome.js +9 -0
  28. package/src/agent/runs.sql +18 -0
  29. package/src/agent/tokens.js +2 -8
  30. package/src/hooks/HookRegistry.js +1 -16
  31. package/src/hooks/Hooks.js +8 -33
  32. package/src/hooks/PluginContext.js +3 -21
  33. package/src/hooks/RpcRegistry.js +1 -4
  34. package/src/hooks/RummyContext.js +2 -16
  35. package/src/hooks/ToolRegistry.js +5 -15
  36. package/src/llm/LlmProvider.js +28 -23
  37. package/src/llm/errors.js +41 -4
  38. package/src/llm/openaiStream.js +125 -0
  39. package/src/llm/retry.js +61 -15
  40. package/src/plugins/budget/budget.js +14 -81
  41. package/src/plugins/cli/README.md +87 -0
  42. package/src/plugins/cli/bin.js +61 -0
  43. package/src/plugins/cli/cli.js +120 -0
  44. package/src/plugins/env/README.md +2 -1
  45. package/src/plugins/env/env.js +4 -6
  46. package/src/plugins/env/envDoc.md +2 -2
  47. package/src/plugins/error/error.js +23 -23
  48. package/src/plugins/file/file.js +2 -22
  49. package/src/plugins/get/get.js +12 -34
  50. package/src/plugins/get/getDoc.md +5 -3
  51. package/src/plugins/hedberg/edits.js +1 -11
  52. package/src/plugins/hedberg/hedberg.js +3 -26
  53. package/src/plugins/hedberg/normalize.js +1 -5
  54. package/src/plugins/hedberg/patterns.js +4 -15
  55. package/src/plugins/hedberg/sed.js +1 -7
  56. package/src/plugins/helpers.js +28 -20
  57. package/src/plugins/index.js +25 -41
  58. package/src/plugins/instructions/README.md +18 -0
  59. package/src/plugins/instructions/instructions.js +13 -76
  60. package/src/plugins/instructions/instructions.md +19 -18
  61. package/src/plugins/instructions/instructions_104.md +5 -4
  62. package/src/plugins/instructions/instructions_105.md +16 -15
  63. package/src/plugins/instructions/instructions_106.md +15 -14
  64. package/src/plugins/instructions/instructions_107.md +13 -6
  65. package/src/plugins/known/README.md +26 -6
  66. package/src/plugins/known/known.js +36 -34
  67. package/src/plugins/log/README.md +2 -2
  68. package/src/plugins/log/log.js +6 -33
  69. package/src/plugins/ollama/ollama.js +50 -66
  70. package/src/plugins/openai/openai.js +26 -44
  71. package/src/plugins/openrouter/openrouter.js +28 -52
  72. package/src/plugins/policy/README.md +8 -2
  73. package/src/plugins/policy/policy.js +8 -21
  74. package/src/plugins/prompt/README.md +22 -0
  75. package/src/plugins/prompt/prompt.js +8 -16
  76. package/src/plugins/rm/rm.js +5 -2
  77. package/src/plugins/rm/rmDoc.md +4 -4
  78. package/src/plugins/rpc/README.md +2 -1
  79. package/src/plugins/rpc/rpc.js +51 -47
  80. package/src/plugins/set/README.md +5 -1
  81. package/src/plugins/set/set.js +23 -33
  82. package/src/plugins/set/setDoc.md +1 -1
  83. package/src/plugins/sh/README.md +2 -1
  84. package/src/plugins/sh/sh.js +5 -11
  85. package/src/plugins/sh/shDoc.md +2 -2
  86. package/src/plugins/stream/README.md +6 -5
  87. package/src/plugins/stream/stream.js +6 -35
  88. package/src/plugins/telemetry/telemetry.js +26 -19
  89. package/src/plugins/think/think.js +4 -7
  90. package/src/plugins/unknown/unknown.js +8 -13
  91. package/src/plugins/update/update.js +36 -35
  92. package/src/plugins/update/updateDoc.md +3 -3
  93. package/src/plugins/xai/xai.js +30 -20
  94. package/src/plugins/yolo/yolo.js +8 -41
  95. package/src/server/ClientConnection.js +17 -47
  96. package/src/server/SocketServer.js +14 -14
  97. package/src/server/protocol.js +1 -10
  98. package/src/sql/functions/slugify.js +5 -7
  99. package/src/sql/v_model_context.sql +4 -11
  100. package/turns/cli_1777462658211/turn_001.txt +772 -0
  101. package/turns/cli_1777462658211/turn_002.txt +606 -0
  102. package/turns/cli_1777462658211/turn_003.txt +667 -0
  103. package/turns/cli_1777462658211/turn_004.txt +297 -0
  104. package/turns/cli_1777462658211/turn_005.txt +301 -0
  105. package/turns/cli_1777462658211/turn_006.txt +262 -0
  106. package/turns/cli_1777465095132/turn_001.txt +715 -0
  107. package/turns/cli_1777465095132/turn_002.txt +236 -0
  108. package/turns/cli_1777465095132/turn_003.txt +287 -0
  109. package/turns/cli_1777465095132/turn_004.txt +694 -0
  110. package/turns/cli_1777465095132/turn_005.txt +422 -0
  111. package/turns/cli_1777465095132/turn_006.txt +365 -0
  112. package/turns/cli_1777465095132/turn_007.txt +885 -0
  113. package/turns/cli_1777465095132/turn_008.txt +1277 -0
  114. package/turns/cli_1777465095132/turn_009.txt +736 -0
@@ -3,28 +3,18 @@ import { existsSync } from "node:fs";
3
3
  import { readdir, stat } from "node:fs/promises";
4
4
  import { basename, isAbsolute, join } from "node:path";
5
5
  import { pathToFileURL } from "node:url";
6
+ import config from "../agent/config.js";
6
7
  import PluginContext from "../hooks/PluginContext.js";
7
8
 
9
+ const { PLUGINS_LOAD_TIMEOUT } = config;
10
+
8
11
  let globalPrefix;
9
12
  function getGlobalPrefix() {
10
13
  globalPrefix ??= execSync("npm prefix -g", { encoding: "utf8" }).trim();
11
14
  return globalPrefix;
12
15
  }
13
16
 
14
- /**
15
- * Plugin loader:
16
- * 1. Walk filesystem + env vars to collect plugin descriptors.
17
- * 2. Import each and instantiate with a fresh PluginContext.
18
- *
19
- * Returns a Map of name → PluginContext for the caller to pass to
20
- * initPlugins. No module-global state — each caller owns its set.
21
- *
22
- * Plugin constructors must be declarative (SPEC surfaces): they
23
- * register schemes, hooks, filters, RPC methods — but don't dereference
24
- * infrastructure that might not be ready yet. Because the plugin
25
- * contract makes constructors side-effect-free on each other, load
26
- * order doesn't matter and there is no dependency system.
27
- */
17
+ // Walk filesystem + env vars, import, instantiate; constructors must stay declarative.
28
18
  export async function registerPlugins(dirs = [], hooks) {
29
19
  const uniqueDirs = [...new Set(dirs.map((d) => join(d)))];
30
20
 
@@ -39,12 +29,22 @@ export async function registerPlugins(dirs = [], hooks) {
39
29
  try {
40
30
  const module = await withTimeout(
41
31
  import(d.url),
42
- PLUGIN_LOAD_TIMEOUT,
32
+ PLUGINS_LOAD_TIMEOUT,
43
33
  `Plugin import timed out: ${d.source}`,
44
34
  );
45
35
  resolved.push({ ...d, Plugin: module.default });
46
36
  } catch (err) {
47
- console.warn(`[RUMMY] Plugin import failed: ${d.name} ${err.message}`);
37
+ // Core plugins live on disk and are part of rummy's contract;
38
+ // their failure is structural and must crash. Third-party
39
+ // plugins (RUMMY_PLUGIN_<x>) are user-installed and may be
40
+ // busted; we log loudly and continue without them.
41
+ if (d.source.startsWith("env:")) {
42
+ console.error(
43
+ `[RUMMY] Plugin import failed: ${d.name} — ${err.message}`,
44
+ );
45
+ continue;
46
+ }
47
+ throw new Error(`Core plugin '${d.name}' import failed`, { cause: err });
48
48
  }
49
49
  }
50
50
 
@@ -53,7 +53,11 @@ export async function registerPlugins(dirs = [], hooks) {
53
53
  try {
54
54
  await instantiatePlugin(r, hooks, instances);
55
55
  } catch (err) {
56
- console.warn(`[RUMMY] Plugin load failed: ${r.name} — ${err.message}`);
56
+ if (r.source.startsWith("env:")) {
57
+ console.error(`[RUMMY] Plugin load failed: ${r.name} — ${err.message}`);
58
+ continue;
59
+ }
60
+ throw new Error(`Core plugin '${r.name}' load failed`, { cause: err });
57
61
  }
58
62
  }
59
63
  return instances;
@@ -63,7 +67,7 @@ async function instantiatePlugin({ name, Plugin, source }, hooks, instances) {
63
67
  if (typeof Plugin?.register === "function") {
64
68
  await withTimeout(
65
69
  Plugin.register(hooks),
66
- PLUGIN_LOAD_TIMEOUT,
70
+ PLUGINS_LOAD_TIMEOUT,
67
71
  `Plugin register timed out: ${source}`,
68
72
  );
69
73
  return;
@@ -89,26 +93,14 @@ const AUDIT_SCHEMES = [
89
93
 
90
94
  const PROMPT_SCHEMES = ["prompt"];
91
95
 
92
- // Lifecycle schemes: client-addressable entries that reflect server
93
- // state. Writable by system (internal bookkeeping), plugin (extensions),
94
- // and client (RPC in Phase 4).
96
+ // Lifecycle entries mirror server state; writable by system/plugin/client.
95
97
  const LIFECYCLE_SCHEMES = ["run"];
96
98
 
97
- // Unified log namespace for action history entries under
98
- // log://turn_N/scheme/slug.
99
99
  const LOG_SCHEMES = ["log"];
100
100
 
101
- /**
102
- * After DB is ready, upsert declared schemes and bootstrap audit/prompt
103
- * schemes. Takes the plugin collection returned by registerPlugins.
104
- * Per-plugin store/db access is provided per-turn via RummyContext;
105
- * PluginContext itself holds only name + hooks.
106
- */
101
+ // Bootstraps audit/prompt/log/lifecycle schemes; called after DB is ready.
107
102
  export async function initPlugins(db, hooks, instances) {
108
103
  for (const name of AUDIT_SCHEMES) {
109
- // Audit schemes are written only by system-level code (reasoning,
110
- // user/assistant/model messages, etc.). Closing the door on model
111
- // writes and plugin writes here.
112
104
  await db.upsert_scheme.run({
113
105
  name,
114
106
  model_visible: 0,
@@ -118,8 +110,6 @@ export async function initPlugins(db, hooks, instances) {
118
110
  });
119
111
  }
120
112
  for (const name of PROMPT_SCHEMES) {
121
- // Prompt entries are created by the prompt plugin on user input;
122
- // model doesn't emit <set path="prompt://...">.
123
113
  await db.upsert_scheme.run({
124
114
  name,
125
115
  model_visible: 1,
@@ -138,9 +128,6 @@ export async function initPlugins(db, hooks, instances) {
138
128
  });
139
129
  }
140
130
  for (const name of LIFECYCLE_SCHEMES) {
141
- // Lifecycle entries are client-addressable mirrors of server state.
142
- // Not model-visible. System writes internally; plugins and clients
143
- // write via the 6 primitives.
144
131
  await db.upsert_scheme.run({
145
132
  name,
146
133
  model_visible: 0,
@@ -156,7 +143,7 @@ export async function initPlugins(db, hooks, instances) {
156
143
  }
157
144
  }
158
145
 
159
- // Register default schemes for tools that plugins ensured but didn't registerScheme for
146
+ // Default scheme for tools that ensureTool'd but didn't registerScheme.
160
147
  const registered = new Set();
161
148
  for (const ctx of instances.values()) {
162
149
  for (const s of ctx.schemes) registered.add(s.name);
@@ -177,7 +164,6 @@ export async function initPlugins(db, hooks, instances) {
177
164
  }
178
165
 
179
166
  function resolvePlugin(packageName) {
180
- // Check local node_modules first, then global
181
167
  const localDir = join(process.cwd(), "node_modules", packageName);
182
168
  if (existsSync(join(localDir, "package.json"))) return localDir;
183
169
  const globalDir = join(getGlobalPrefix(), "lib", "node_modules", packageName);
@@ -256,8 +242,6 @@ async function collectFromDir(dir, isRoot, descriptors) {
256
242
  }
257
243
  }
258
244
 
259
- const PLUGIN_LOAD_TIMEOUT = 10000;
260
-
261
245
  function withTimeout(promise, ms, message) {
262
246
  return Promise.race([
263
247
  promise,
@@ -29,6 +29,24 @@ phase directive so prompt caching holds across turns within a run.
29
29
  - `protocol.js` — placeholder module reserved for deterministic
30
30
  protocol rule enforcement. Currently pass-through.
31
31
 
32
+ ## Navigation validation
33
+
34
+ `validateNavigation(status, rummy)` rejects illegal stage transitions
35
+ emitted via `<update status="N">`:
36
+
37
+ - **Forward skip** — `nextPhase > currentPhase + 1`. Models advancing
38
+ more than one stage at once are jumping past required work. Returns
39
+ and continuations (`nextPhase ≤ currentPhase`) always pass.
40
+ - **Status 200 outside Deployment** — 200 is Deployment Completion.
41
+ Emitting it from earlier phases skips the actual Deployment work.
42
+ - **Deployment with prior prompts** — entering or remaining in
43
+ Deployment (phase 7) requires zero visible PRIOR prompts. Covers
44
+ 167 (entry), 177 / 200 (continuation, completion).
45
+
46
+ On rejection the update entry is marked `rejected` (the phase router
47
+ skips it) and an error log is emitted; rejections count as normal
48
+ strikes.
49
+
32
50
  ## Cache shape
33
51
 
34
52
  - System message includes the base template + tool docs + persona.
@@ -6,12 +6,7 @@ const baseInstructions = readFileSync(
6
6
  "utf8",
7
7
  );
8
8
 
9
- // 1XY status encoding: X=current phase, Y=next phase. Y routes through
10
- // phaseForStatus to select next turn's <instructions>. Phases 4–9 are
11
- // reserved (status codes 1X4..1X9); add new phases by dropping in
12
- // `instructions_10N.md`. Absent files render no <instructions> block —
13
- // the model runs on base instructions only. This lets you route ahead
14
- // of writing the prose (e.g. an upcoming "ask lite" phase 9).
9
+ // 1XY phase routing; see plugin README.
15
10
  const PHASES = [4, 5, 6, 7, 8, 9];
16
11
  const phaseInstructions = Object.fromEntries(
17
12
  PHASES.flatMap((p) => {
@@ -28,14 +23,7 @@ function phaseForStatus(status) {
28
23
  return PHASES.includes(last) ? last : 4;
29
24
  }
30
25
 
31
- // Scan an already-materialized row set for the most recent update
32
- // emission's status. Used by the assembly.user filter so the phase
33
- // instructions ride with the user message (dynamic, expected to
34
- // change every turn) instead of the system prompt (stable, cached).
35
- // Validation is upstream (update.js isValidStatus + 422 error log) so
36
- // we trust the status and route on it directly — a whitelist here
37
- // silently drops advertised completion codes whose contracts drift,
38
- // which is worse than a noisy fallback.
26
+ // Latest non-rejected update status from materialized rows.
39
27
  function latestUpdateStatusFromRows(rows) {
40
28
  let bestTurn = -1;
41
29
  let bestStatus = null;
@@ -49,9 +37,6 @@ function latestUpdateStatusFromRows(rows) {
49
37
  : r.attributes;
50
38
  const status = attrs?.status;
51
39
  if (status == null) continue;
52
- // Rejected updates are written for the model's audit trail but are
53
- // not navigation events — phase router skips them so the model
54
- // stays in the stage it was already in.
55
40
  if (attrs?.rejected) continue;
56
41
  if (turn > bestTurn || (turn === bestTurn && status > bestStatus)) {
57
42
  bestTurn = turn;
@@ -74,19 +59,11 @@ export default class Instructions {
74
59
  this.validateNavigation.bind(this);
75
60
  core.hooks.instructions.findLatestSummary =
76
61
  this.findLatestSummary.bind(this);
77
- // Dynamic phase instructions live in the user message (above
78
- // <prompt>) so the system message stays cache-stable across turns.
79
- // Priority 250 puts us between <log> (100), <unknowns> (200),
80
- // and <prompt> (300).
81
- core.filter("assembly.user", this.assembleInstructions.bind(this), 250);
62
+ core.filter("assembly.user", this.assembleInstructions.bind(this), 200);
82
63
  new Protocol(core);
83
64
  }
84
65
 
85
- /**
86
- * Materialize the system prompt for a run: look up the
87
- * instructions://system entry, project it through the promoted view.
88
- * TurnExecutor calls this once per turn before context assembly.
89
- */
66
+ // Project instructions://system through the visible view; called once per turn.
90
67
  async resolveSystemPrompt(rummy) {
91
68
  const { entries: store, runId, hooks } = rummy;
92
69
  const entries = await store.getEntriesByPattern(
@@ -111,30 +88,16 @@ export default class Instructions {
111
88
  });
112
89
  }
113
90
 
114
- /**
115
- * Reject illegal stage navigation. Two checks:
116
- *
117
- * 1. Forward skip — `nextPhase > currentPhase + 1`. Models advancing
118
- * more than one stage at a time are jumping past required work.
119
- * Returns and continuations (nextPhase ≤ currentPhase) always pass.
120
- *
121
- * 2. Deployment with prior prompts — any status landing the model in
122
- * Deployment (phase 7) requires zero visible PRIOR prompts. State-
123
- * property rule covering both entry (167) and continuation (177,
124
- * 200) — once in Deployment, the model still can't claim it with
125
- * undemoted prior prompts. The current (latest) prompt always
126
- * stays visible since Deployment must act on it.
127
- *
128
- * On rejection the caller marks the update entry rejected (so the
129
- * phase router skips it) and emits an error log; navigation rejections
130
- * count as normal strikes.
131
- */
91
+ // Reject illegal stage navigation; see plugin README.
132
92
  async validateNavigation(status, rummy) {
133
93
  const currentPhase = await this.#getCurrentPhase(rummy);
134
94
  const nextPhase = phaseForStatus(status);
135
95
  if (nextPhase > currentPhase + 1) {
136
96
  return { ok: false, reason: "Illegal navigation attempt" };
137
97
  }
98
+ if (status === 200 && currentPhase !== 7) {
99
+ return { ok: false, reason: "Illegal navigation attempt" };
100
+ }
138
101
  if (nextPhase === 7) {
139
102
  const visible = await this.#countVisiblePriorPrompts(rummy);
140
103
  if (visible > 0) {
@@ -148,11 +111,7 @@ export default class Instructions {
148
111
  }
149
112
 
150
113
  async #getCurrentPhase(rummy) {
151
- // `**` (not `*`) for the slug position update slugs are derived
152
- // from the model's update body and can contain URL-encoded `/`
153
- // characters (e.g. `known%3A//foo/bar` in a "ready for deployment"
154
- // summary). Single `*` doesn't cross those embedded slashes and
155
- // silently misses the prior turn's update.
114
+ // `**` not `*`: update slugs may contain URL-encoded `/`.
156
115
  const updates = await rummy.entries.getEntriesByPattern(
157
116
  rummy.runId,
158
117
  "log://*/update/**",
@@ -179,16 +138,7 @@ export default class Instructions {
179
138
  return phaseForStatus(bestStatus);
180
139
  }
181
140
 
182
- /**
183
- * Find the latest successful Deployment summary from a log-entry list.
184
- * Matches `log://turn_N/update/...` entries with status=200 (successful
185
- * Deployment completion) and returns the most recent. Used by
186
- * AgentLoop telemetry to surface the model's latest delivery.
187
- *
188
- * Lives here, not in AgentLoop, because "what counts as a summary" is
189
- * state-machine knowledge — phase 7's success status (200) is the
190
- * definition. AgentLoop just consumes the result.
191
- */
141
+ // Latest phase-7 success (status=200); state-machine knowledge lives here, not AgentLoop.
192
142
  findLatestSummary(logEntries) {
193
143
  return logEntries
194
144
  .filter((e) => {
@@ -210,9 +160,7 @@ export default class Instructions {
210
160
  );
211
161
  const visible = prompts.filter((p) => p.visibility === "visible");
212
162
  if (visible.length === 0) return 0;
213
- // Exclude the current (latest) prompt that's what Deployment acts on.
214
- // Demoting it would force the model to deliver on content it hid from
215
- // itself. Only PRIOR prompts are subject to demote-before-Deployment.
163
+ // Exclude the latest prompt; only PRIOR prompts trigger demote-before-Deployment.
216
164
  let maxNum = -1;
217
165
  for (const p of visible) {
218
166
  const m = /^prompt:\/\/(\d+)$/.exec(p.path);
@@ -230,10 +178,7 @@ export default class Instructions {
230
178
  const toolSet = rummy.toolSet
231
179
  ? [...rummy.toolSet]
232
180
  : this.#core.hooks.tools.names;
233
- // instructions:// is an audit scheme (writable_by: ["system"]).
234
- // No per-turn phase state on this entry — keeps the system
235
- // prompt cache-stable across turns. Phase selection happens at
236
- // assembly.user time from the current row set.
181
+ // instructions://system stays cache-stable; phase selection at assembly.user.
237
182
  await store.set({
238
183
  runId,
239
184
  turn,
@@ -242,8 +187,6 @@ export default class Instructions {
242
187
  state: "resolved",
243
188
  writer: "system",
244
189
  attributes: {
245
- // runRow.persona is a nullable TEXT column; absent row is
246
- // a system bug — let the null propagate if runRow exists.
247
190
  persona: runRow.persona,
248
191
  toolSet,
249
192
  },
@@ -259,7 +202,6 @@ export default class Instructions {
259
202
  {},
260
203
  { toolSet: activeTools },
261
204
  );
262
- // Hidden tools are excluded at the registry level (see ToolRegistry).
263
205
  const sorted = this.#core.hooks.tools.advertisedNames.filter((n) =>
264
206
  activeTools.has(n),
265
207
  );
@@ -275,12 +217,7 @@ export default class Instructions {
275
217
  return prompt;
276
218
  }
277
219
 
278
- // Renders the current phase's instructions as an <instructions>
279
- // block in the user message. Runs at priority 250 — after <log>
280
- // and <unknowns>, immediately before <prompt>. System prompt stays
281
- // static so prompt caching keeps its prefix intact across turns.
282
- // A routed phase without an instructions_10N.md file emits nothing —
283
- // the model proceeds on base instructions alone.
220
+ // Render <instructions> for current phase; absent phase file → no block.
284
221
  assembleInstructions(content, ctx) {
285
222
  const status = latestUpdateStatusFromRows(ctx.rows);
286
223
  const step = phaseInstructions[phaseForStatus(status)];
@@ -2,32 +2,33 @@ XML Commands Available: [%TOOLS%]
2
2
 
3
3
  # FCRM State Machine
4
4
 
5
- You are a Folksonomic Context Relevance Maximization (FCRM) State Machine
5
+ You are a Folksonomic Context Relevance Maximization (FCRM) State Machine.
6
6
 
7
- YOU MUST perform the actions corresponding with your current stage:
8
- * Definition Stage: Defining what's unknown into unknown:// entries
9
- * Discovery Stage: Selecting an unknown, discovering relevant source entries and prompts, then distilling them into known:// entries
10
- * Demotion Stage: Demoting the unknown entries, source entries, prompts, and log events after distillation is completed
11
- * Deployment Stage: Acting on the current prompt
12
- * Resolution Stage: Multi-prompt benchmark final `fcrmScore`
7
+ YOU MUST ONLY perform the actions corresponding with your current stage:
8
+ * Decomposition Stage: Determine, define, and decompose key unknown and unresolved into unknown:// entries
9
+ * Distillation Stage: discovering relevant source entries, then distilling into known:// entries to resolve unknowns
10
+ * Demotion Stage: Demote the unknown entries, source entries, prompts, and log events after distillation is completed
11
+ * Deployment Stage: Act on the current prompt after relevant context is distilled and irrelevant context is demoted
12
+ * Resolution Stage: Evaluation of context relevance maximization, state machine compliance, and prompt resolution.
13
13
 
14
14
  ## Visibility States: Promote and Demote Visibility State to Control Context Relevance
15
- * visible: Fully visible, but uses `tokens="N"` context budget
16
- * summarized: Approximate, summary information, very small context budget penalty
17
- * archived: Hidden from Context, but can be retrieved later with <get path="..."/>
15
+ * visible: Full entry body in context, uses `tokens="N"` context budget
16
+ * summarized: Short summary in context, very small context budget penalty
17
+ * archived: Hidden from context, recallable later by path reference or pattern search
18
18
 
19
- Tip: You can leverage the FCRM's Visibility States with folksonomic taxonomies and tags to store and recall unlimited information.
20
- Tip: The `tokens="N"` shows how much context memory is consumed if "visible". Entries only consume tokens when at "visible" visibility.
19
+ * Leverage the FCRM's Visibility States with folksonomic taxonomies and tags to store and recall unlimited information.
20
+ * When an entry is "visible", it will appear in both the summarized and visible sections.
21
+ * The `tokens="N"` shows how much context is consumed if "visible". Entries consume very few tokens when summarized.
21
22
 
22
- Warning: YOU MUST NOT allow the `tokens="N"` sum of irrelevant source entries, prompts, or log events to exceed `tokensFree` budget.
23
- Warning: YOU MUST NOT skip or avoid state machine steps or the Resolution Stage will fail.
23
+ YOU MUST NOT allow the `tokens="N"` sum of source entries, prompts, or log events to exceed `tokensFree` budget.
24
24
 
25
25
  # Commands
26
26
 
27
- Warning: YOU MUST NOT use shell commands for project file operations. Project files are entries that require XML Commands.
28
- Example: <set path="src/file.txt">new file content</set>
29
- Example: <get path="src/*.txt" preview/>
27
+ YOU MUST NOT use shell commands for file operations. Files are also entries that require XML Commands.
28
+ Example: <set path="projectFile.txt">new file content</set>
29
+ Example: <get path="src/*.txt" manifest/>
30
30
 
31
- Tip: Project files, entries, prompts, and log events are all accessible with the XML Commands.
31
+ * Files, entries, prompts, and log events are all accessible with the XML Commands.
32
+ * Entries without a `{scheme}://` are files. They can be read and modified through the unified XML Commands interface.
32
33
 
33
34
  [%TOOLDOCS%]
@@ -1,7 +1,8 @@
1
- # Definition Stage: YOU MUST ONLY create topical, taxonomized, and tagged unknown:// entries for missing information
1
+ # Decomposition Stage: YOU MUST ONLY create topical, taxonomized, and tagged unknown:// entries
2
2
 
3
- Example: <set path="unknown://countries/france/capital" summary="countries,france,capital,geography,trivia">What is the capital of France?</set>
3
+ YOU MUST decompose the prompt into the key information, issues, and items that are unknown and/or unresolved.
4
4
 
5
+ Example: <set path="unknown://countries/france/capital" summary="countries,france,capital,geography,trivia">What is the capital of France?</set>
5
6
 
6
- ## Turn Termination:
7
- * Definition Stage Completion: <update status="145">unknowns identified</update>
7
+ ## Turn Termination (CHOOSE ONLY ONE):
8
+ * Decomposition Stage Completion: <update status="145">prompt decomposed</update>
@@ -1,30 +1,31 @@
1
- # Discovery Stage: YOU MUST select an unknown:// entry, then discover its source entries and distill them into known:// entries
1
+ # Distillation Stage: YOU MUST select an unknown:// entry, then discover its source entries and distill them into known:// entries
2
2
 
3
3
  YOU MUST create topical, taxonomized, and tagged known:// entries to resolve the selected unknown:// entry.
4
- YOU MUST reference all related source entries and prompts.
4
+ YOU MUST reference all related source entries and prompts in the `# Related` list
5
5
  YOU MUST ONLY populate known entries with promoted information, NOT from your own training data or opinion.
6
6
  YOU MUST immediately demote unknowns, source entries, prompts, and log events after they are distilled, irrelevant, or resolved.
7
7
 
8
- Tip: Check the `tokens="N"` of the source entries against the `tokensFree="N"` constraint before promoting entries.
9
- Tip: You can use <get path="..." preview/> to preview the potential `tokens="N"` budget impact of bulk operations.
10
- Tip: You can use <get path="..." line="X" limit="Y"/> to read subsets of entries that would exceed your `tokensFree` budget.
8
+ * Check the `tokens="N"` of the source entries against the `tokensFree="N"` constraint before promoting entries.
9
+ * You can use <get path="..." manifest/> to list paths and their token amounts for bulk operations without performing them.
10
+ * You can use <get path="..." line="X" limit="Y"/> to read subsets of entries that would exceed your `tokensFree` budget.
11
+ * Don't accidentally set the current prompt to `archived`.
11
12
 
12
- ## Example:
13
- <get path="**" preview>capital</get>
13
+ Example:
14
+ <get path="**" manifest>capital</get>
14
15
  <get path="prompt://3" line="1" limit="100"/>
15
16
 
16
17
  <set path="trivia/capitals.csv" visibility="visible"/>
17
18
 
18
19
  <set path="known://countries/france/capital" summary="countries,france,capital,geography,trivia">
20
+ # Related
21
+ [trivia question](prompt://3)
22
+ [unknown resolving](unknown://countries/france/capital)
23
+ [source entry](trivia/capitals.csv)
24
+
19
25
  # Capital of France
20
26
  The capital of France is Paris.
21
27
 
22
28
  {...}
23
-
24
- ## Related
25
- [trivia question](prompt://3)
26
- [unknown resolving](unknown://countries/france/capital)
27
- [source entry](trivia/capitals.csv)
28
29
  </set>
29
30
 
30
31
  <set path="prompt://3" visibility="summarized"/>
@@ -33,6 +34,6 @@ Tip: You can use <get path="..." line="X" limit="Y"/> to read subsets of entries
33
34
  <set path="trivia/capitals.csv" visibility="summarized"/>
34
35
 
35
36
  ## Turn Termination (CHOOSE ONLY ONE):
36
- * Definition Stage Return: <update status="154">returning to Definition Stage</update>
37
- * Discovery Stage Continuation: <update status="155">discovering and distilling more for the selected unknown</update>
38
- * Discovery Stage Completion: <update status="156">this unknown's known entries written</update>
37
+ * Decomposition Stage Return: <update status="154">additional unknowns identified; returning to Decomposition Stage</update>
38
+ * Distillation Stage Continuation: <update status="155">discovering and distilling more for the selected unknown</update>
39
+ * Distillation Stage Completion: <update status="156">this unknown's known entries written</update>
@@ -1,21 +1,22 @@
1
1
  # Demotion Stage: YOU MUST demote all source entries, prompts, and log events that are now distilled or no longer relevant
2
2
 
3
- Examples:
4
- <set path="prompt://2" summary="All information distilled into knowns" visibility="summarized"/>
5
- <set path="trivia/capitals.csv" visibility="summarized"/>
6
- <set path="unknown://countries/france/capital" visibility="summarized"/>
7
- <set path="unknown://countries/poland/capital" summary="REJECTED: Irrelevant" visibility="summarized"/>
8
- <set path="https://en.wikipedia.org/wiki/Paris,_Texas" summary="REJECTED: Wrong Paris" visibility="summarized"/>
9
- <set path="log://turn_1/**" visibility="archived"/>
10
- <set path="log://turn_2/**" visibility="archived"/>
11
- <set path="log://turn_3/set/**" visibility="archived"/>
12
- <set path="log://turn_3/get/**" visibility="archived"/>
13
- <set path="log://turn_3/search/**" visibility="archived"/>
3
+ Example:
4
+ <set path="prompt://2" summary="All information distilled into knowns" visibility="summarized"/>
5
+ <set path="trivia/capitals.csv" visibility="summarized"/>
6
+ <set path="unknown://countries/france/capital" visibility="summarized"/>
7
+ <set path="unknown://countries/poland/capital" summary="REJECTED: Irrelevant" visibility="summarized"/>
8
+ <set path="https://en.wikipedia.org/wiki/Paris,_Texas" summary="REJECTED: Wrong Paris" visibility="summarized"/>
9
+ <set path="log://turn_1/**" visibility="archived"/>
10
+ <set path="log://turn_2/**" visibility="archived"/>
11
+ <set path="log://turn_3/set/**" visibility="archived"/>
12
+ <set path="log://turn_3/get/**" visibility="archived"/>
13
+ <set path="log://turn_3/search/**" visibility="archived"/>
14
14
 
15
- Tip: You need room to think. Demote large prompts and source entries, then iterate them with <get path="..." line="N" limit="N"/> as necessary.
15
+ * You need room to think. Demote large prompts and source entries, then iterate them with <get path="..." line="N" limit="N"/> as necessary.
16
+ * When demoting prompts, prefer "summarized" to "archived" to avoid losing necessary context.
16
17
 
17
18
  ## Turn Termination (CHOOSE ONLY ONE):
18
- * Definition Stage Return: <update status="164">returning to Definition Stage</update>
19
- * Discovery Stage Return: <update status="165">more unknowns remain; returning to Discovery Stage</update>
19
+ * Decomposition Stage Return: <update status="164">additional unknowns identified; returning to Decomposition Stage</update>
20
+ * Distillation Stage Return: <update status="165">more unknowns remain; returning to Distillation Stage</update>
20
21
  * Demotion Stage Continuation: <update status="166">demoting more distilled or irrelevant entries, prompts, and log events</update>
21
22
  * Demotion Stage Completion: <update status="167">all unknowns resolved and demoted; ready for Deployment Stage</update>
@@ -1,10 +1,17 @@
1
- # Deployment Stage
1
+ # Deployment Stage: YOU MUST act on the prompt.
2
2
 
3
- YOU MUST act on the prompt.
3
+ YOU MUST attempt to deterministically verify your actions, outputs, or answers before declaring completion, if possible.
4
+
5
+ Example: verifying deliverable before completion
6
+ <set path="sum.js">console.log(process.argv.slice(2).reduce((a, b) => a + Number(b), 0));</set>
7
+ <sh>[ -f sum.js ] && node --version && node sum.js 2 2 | grep -qx 4</sh>
8
+ <update status="177">sum.js written, node available, ran cleanly, correct output?</update>
9
+
10
+ Example: <update status="200">Paris</update>
4
11
 
5
12
  ## Turn Termination (CHOOSE ONLY ONE):
6
- * Definition Stage Return: <update status="174">returning to Definition Stage</update>
7
- * Discovery Stage Return: <update status="175">returning to Discovery Stage</update>
8
- * Demotion Stage Return: <update status="176">returning to Demotion Stage</update>
13
+ * Decomposition Stage Return: <update status="174">additional unknowns identified; returning to Decomposition Stage</update>
14
+ * Distillation Stage Return: <update status="175">selected unknown not yet resolved; returning to Distillation Stage</update>
15
+ * Demotion Stage Return: <update status="176">context not yet sufficiently demoted; returning to Demotion Stage</update>
9
16
  * Deployment Stage Continuation: <update status="177">performing more actions</update>
10
- * Deployment Stage Completion: <update status="200">{direct answer if prompt asked a question, summary of actions if not}</update>
17
+ * Deployment Stage Completion: <update status="200">{direct answer (summary of actions performed if prompt not a question)}</update>
@@ -1,13 +1,17 @@
1
1
  # known {#known_plugin}
2
2
 
3
- Writes knowledge entries into the store at full visibility.
3
+ Writes knowledge entries into the store at full visibility, and renders
4
+ the project's data surface as the bifurcated `<summarized>` /
5
+ `<visible>` blocks at the top of the user message.
4
6
 
5
7
  ## Registration
6
8
 
7
9
  - **Tool**: `known`
8
10
  - **Category**: `data`
9
11
  - **Handler**: Upserts the entry body at the target path with status 200.
10
- - **Filter**: `assembly.system` at priority 100 — renders `<knowns>` section.
12
+ - **Filters**:
13
+ - `assembly.user` priority 50 — renders `<summarized>`.
14
+ - `assembly.user` priority 75 — renders `<visible>`.
11
15
 
12
16
  ## Projection
13
17
 
@@ -15,7 +19,23 @@ Shows `# known {path}` followed by the entry body.
15
19
 
16
20
  ## Assembly
17
21
 
18
- Filters turn_context rows where `category === "data"`. Renders all
19
- data entries (files, knowledge, skills, URLs) into the `<knowns>` section
20
- of the system message. Third-party plugins that register with
21
- `category: "data"` automatically appear here.
22
+ Filters `ctx.rows` where `category === "data"`. Two separate blocks
23
+ emit at the top of the user message in this order:
24
+
25
+ - `<summarized>` — each data entry whose visibility is `visible` or
26
+ `summarized`, rendered under its scheme tag with the plugin's
27
+ summary projection as body (truncated knowns, code symbols,
28
+ page abstracts — whatever the plugin's `summary()` hook produces).
29
+ Plus the named carve-out: archived prompts pass through
30
+ (visibility="archived") so the model can `<get>` the active prompt
31
+ back after demotion.
32
+ - `<visible>` — each data entry whose visibility is `visible`,
33
+ rendered with the plugin's visible projection (full body) as the
34
+ tag body. A visible entry appears in *both* blocks: summary
35
+ projection up top, full body below.
36
+
37
+ This split lets `<summarized>` stay cache-stable across promote/demote
38
+ operations — only `<visible>` mutates when the model promotes a
39
+ summary or demotes a visible entry. Third-party plugins that register
40
+ with `category: "data"` automatically appear in both blocks under
41
+ their scheme tag.