akm-cli 0.9.0-beta.45 → 0.9.0-beta.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +1 -2
- package/dist/commands/env/env-cli.js +16 -24
- package/dist/commands/env/secret-cli.js +12 -20
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +3 -3
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +64 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/improve/consolidate/types.js +4 -0
- package/dist/commands/improve/consolidate.js +20 -571
- package/dist/commands/improve/distill.js +5 -9
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/extract-cli.js +9 -1
- package/dist/commands/improve/extract.js +5 -19
- package/dist/commands/improve/improve-auto-accept.js +4 -8
- package/dist/commands/improve/improve-cli.js +35 -60
- package/dist/commands/improve/improve-result-file.js +5 -23
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +107 -3606
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1079 -0
- package/dist/commands/improve/preparation.js +1963 -0
- package/dist/commands/improve/recombine.js +6 -12
- package/dist/commands/improve/reflect.js +29 -34
- package/dist/commands/proposal/drain.js +25 -48
- package/dist/commands/proposal/proposal-cli.js +21 -31
- package/dist/commands/proposal/validators/proposals.js +3 -7
- package/dist/commands/read/curate.js +70 -14
- package/dist/commands/read/knowledge.js +2 -2
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/stash-cli.js +9 -37
- package/dist/commands/tasks/tasks-cli.js +19 -27
- package/dist/commands/wiki-cli.js +21 -35
- package/dist/core/config/config.js +18 -2
- package/dist/core/events.js +3 -7
- package/dist/core/logs-db.js +6 -63
- package/dist/core/state/migrations.js +714 -0
- package/dist/core/state-db.js +28 -779
- package/dist/indexer/db/db.js +82 -216
- package/dist/indexer/indexer.js +11 -112
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/search/search-source.js +10 -24
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/llm/client.js +22 -11
- package/dist/llm/graph-extract.js +28 -39
- package/dist/llm/memory-infer.js +34 -22
- package/dist/llm/metadata-enhance.js +35 -30
- package/dist/llm/structured-call.js +49 -0
- package/dist/output/shapes/passthrough.js +0 -1
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +22 -9
- package/dist/scripts/migrate-storage.js +892 -1186
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +214 -179
- package/dist/setup/setup.js +26 -5
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +206 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -671
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/website.js +2 -3
- package/dist/sources/website-ingest.js +51 -9
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +181 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/registry-cache.js +92 -0
- package/dist/tasks/runner.js +5 -13
- package/dist/workflows/runtime/runs.js +1 -117
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/package.json +5 -5
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
package/dist/cli/shared.js
CHANGED
|
@@ -15,6 +15,7 @@ import { getOutputMode } from "../output/context.js";
|
|
|
15
15
|
import { DEFAULT_TEMPLATE, deliverRendered, escapeHtml, renderHtml, resolveTemplatePath } from "../output/html-render.js";
|
|
16
16
|
import { shapeForCommand } from "../output/shapes.js";
|
|
17
17
|
import { formatPlain, outputJsonl } from "../output/text.js";
|
|
18
|
+
import { hasSubcommand } from "./parse-args.js";
|
|
18
19
|
// ── Exit codes ───────────────────────────────────────────────────────────────
|
|
19
20
|
/**
|
|
20
21
|
* Canonical process exit-code table for the akm CLI. Single source of truth —
|
|
@@ -111,6 +112,33 @@ export function defineJsonCommand(def) {
|
|
|
111
112
|
run: (context) => runWithJsonErrors(() => run(context)),
|
|
112
113
|
});
|
|
113
114
|
}
|
|
115
|
+
/**
|
|
116
|
+
* Define a citty subcommand-group command (env, secret, proposal, tasks, wiki,
|
|
117
|
+
* graph, …) that shares one wiring shape: a `subCommands` map, a routing set
|
|
118
|
+
* DERIVED from that map's keys (so the set can never silently desync from the
|
|
119
|
+
* registered subcommands), and a default body that fires ONLY for the bare
|
|
120
|
+
* group invocation — citty still runs the group body after dispatching a
|
|
121
|
+
* subcommand, so the shared guard short-circuits when `args._[0]` names a
|
|
122
|
+
* registered subcommand.
|
|
123
|
+
*
|
|
124
|
+
* The `defaultRun` body is wrapped in `runWithJsonErrors`, so it emits a
|
|
125
|
+
* byte-identical JSON error envelope on throw — exactly the per-site
|
|
126
|
+
* `run() { return runWithJsonErrors(() => { if (hasSubcommand(...)) return; … }); }`
|
|
127
|
+
* boilerplate this replaces.
|
|
128
|
+
*/
|
|
129
|
+
export function defineGroupCommand(def) {
|
|
130
|
+
const subcommandSet = new Set(Object.keys(def.subCommands));
|
|
131
|
+
return defineCommand({
|
|
132
|
+
meta: def.meta,
|
|
133
|
+
...(def.args ? { args: def.args } : {}),
|
|
134
|
+
subCommands: def.subCommands,
|
|
135
|
+
run: (context) => runWithJsonErrors(() => {
|
|
136
|
+
if (hasSubcommand(context.args, subcommandSet))
|
|
137
|
+
return;
|
|
138
|
+
return def.defaultRun(context);
|
|
139
|
+
}),
|
|
140
|
+
});
|
|
141
|
+
}
|
|
114
142
|
/**
|
|
115
143
|
* Render a command result according to the active output mode
|
|
116
144
|
* (json/jsonl/yaml/text/md/html). When `--output <path>` is set, the rendered
|
package/dist/cli.js
CHANGED
|
@@ -77,7 +77,7 @@ import { registryCommand } from "./commands/registry-cli.js";
|
|
|
77
77
|
import { addCommand } from "./commands/sources/add-cli.js";
|
|
78
78
|
import { renderMigrationHelp } from "./commands/sources/migration-help.js";
|
|
79
79
|
import { cloneCommand, historyCommand, listCommand, removeCommand, syncCommand, updateCommand, upgradeCommand, } from "./commands/sources/sources-cli.js";
|
|
80
|
-
import {
|
|
80
|
+
import { importKnowledgeCommand, indexCommand, infoCommand, initCommand } from "./commands/sources/stash-cli.js";
|
|
81
81
|
import { tasksCommand } from "./commands/tasks/tasks-cli.js";
|
|
82
82
|
import { wikiCommand } from "./commands/wiki-cli.js";
|
|
83
83
|
import { workflowCommand } from "./commands/workflow-cli.js";
|
|
@@ -486,7 +486,6 @@ export const main = defineCommand({
|
|
|
486
486
|
health: healthCommand,
|
|
487
487
|
info: infoCommand,
|
|
488
488
|
graph: graphCommand,
|
|
489
|
-
db: dbCommand,
|
|
490
489
|
add: addCommand,
|
|
491
490
|
list: listCommand,
|
|
492
491
|
remove: removeCommand,
|
|
@@ -19,8 +19,8 @@ import { spawnSync } from "node:child_process";
|
|
|
19
19
|
import fs from "node:fs";
|
|
20
20
|
import path from "node:path";
|
|
21
21
|
import { defineCommand } from "citty";
|
|
22
|
-
import { getStringArg
|
|
23
|
-
import { output, runWithJsonErrors } from "../../cli/shared.js";
|
|
22
|
+
import { getStringArg } from "../../cli/parse-args.js";
|
|
23
|
+
import { defineGroupCommand, output, runWithJsonErrors } from "../../cli/shared.js";
|
|
24
24
|
import { assertFlatAssetName, combineCreatePath, normalizeCreateSubPath } from "../../core/asset/asset-create.js";
|
|
25
25
|
import { deriveCanonicalAssetName, resolveAssetPathFromName } from "../../core/asset/asset-spec.js";
|
|
26
26
|
import { isWithin, writeFileAtomic } from "../../core/common.js";
|
|
@@ -499,31 +499,23 @@ const envUnsetCommand = defineCommand({
|
|
|
499
499
|
});
|
|
500
500
|
},
|
|
501
501
|
});
|
|
502
|
-
|
|
503
|
-
// (M10) so adding a subcommand can never silently desync from `hasSubcommand`.
|
|
504
|
-
const envSubCommands = {
|
|
505
|
-
list: envListCommand,
|
|
506
|
-
path: envPathCommand,
|
|
507
|
-
export: envExportCommand,
|
|
508
|
-
run: envRunCommand,
|
|
509
|
-
create: envCreateCommand,
|
|
510
|
-
set: envSetCommand,
|
|
511
|
-
unset: envUnsetCommand,
|
|
512
|
-
remove: envRemoveCommand,
|
|
513
|
-
};
|
|
514
|
-
const ENV_SUBCOMMAND_SET = new Set(Object.keys(envSubCommands));
|
|
515
|
-
export const envCommand = defineCommand({
|
|
502
|
+
export const envCommand = defineGroupCommand({
|
|
516
503
|
meta: {
|
|
517
504
|
name: "env",
|
|
518
505
|
description: "Manage `.env` files — a group of related CONFIGURATION values for an app or service (URLs, flags, plus any credentials it needs), loaded together. Values may or may not be sensitive; akm protects them all the same (key names visible, values never in structured output). For a single sensitive value used on its own (an auth token, key, or cert), use `akm secret`.",
|
|
519
506
|
},
|
|
520
|
-
subCommands:
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
507
|
+
subCommands: {
|
|
508
|
+
list: envListCommand,
|
|
509
|
+
path: envPathCommand,
|
|
510
|
+
export: envExportCommand,
|
|
511
|
+
run: envRunCommand,
|
|
512
|
+
create: envCreateCommand,
|
|
513
|
+
set: envSetCommand,
|
|
514
|
+
unset: envUnsetCommand,
|
|
515
|
+
remove: envRemoveCommand,
|
|
516
|
+
},
|
|
517
|
+
async defaultRun() {
|
|
518
|
+
const { listKeys } = await import("./env.js");
|
|
519
|
+
output("env-list", { envs: listEnvsRecursive(listKeys) });
|
|
528
520
|
},
|
|
529
521
|
});
|
|
@@ -19,8 +19,8 @@ import { spawnSync } from "node:child_process";
|
|
|
19
19
|
import fs from "node:fs";
|
|
20
20
|
import path from "node:path";
|
|
21
21
|
import { defineCommand } from "citty";
|
|
22
|
-
import { getStringArg
|
|
23
|
-
import { output, runWithJsonErrors } from "../../cli/shared.js";
|
|
22
|
+
import { getStringArg } from "../../cli/parse-args.js";
|
|
23
|
+
import { defineGroupCommand, output, runWithJsonErrors } from "../../cli/shared.js";
|
|
24
24
|
import { deriveCanonicalAssetName } from "../../core/asset/asset-spec.js";
|
|
25
25
|
import { loadConfig } from "../../core/config/config.js";
|
|
26
26
|
import { makeSecretRef, resolveSecretPath } from "../../core/env-secret-ref.js";
|
|
@@ -233,27 +233,19 @@ const secretRemoveCommand = defineCommand({
|
|
|
233
233
|
});
|
|
234
234
|
},
|
|
235
235
|
});
|
|
236
|
-
|
|
237
|
-
// (M10) so adding a subcommand can never silently desync from `hasSubcommand`.
|
|
238
|
-
const secretSubCommands = {
|
|
239
|
-
list: secretListCommand,
|
|
240
|
-
path: secretPathCommand,
|
|
241
|
-
run: secretRunCommand,
|
|
242
|
-
set: secretSetCommand,
|
|
243
|
-
remove: secretRemoveCommand,
|
|
244
|
-
};
|
|
245
|
-
const SECRET_SUBCOMMAND_SET = new Set(Object.keys(secretSubCommands));
|
|
246
|
-
export const secretCommand = defineCommand({
|
|
236
|
+
export const secretCommand = defineGroupCommand({
|
|
247
237
|
meta: {
|
|
248
238
|
name: "secret",
|
|
249
239
|
description: "Manage secrets — a single sensitive value used on its own for authentication (an API token, a PEM private key, a TLS cert), one value per file. Names are visible; the file contents are the value and never appear in structured output. For a group of related configuration loaded together, use `akm env`.",
|
|
250
240
|
},
|
|
251
|
-
subCommands:
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
241
|
+
subCommands: {
|
|
242
|
+
list: secretListCommand,
|
|
243
|
+
path: secretPathCommand,
|
|
244
|
+
run: secretRunCommand,
|
|
245
|
+
set: secretSetCommand,
|
|
246
|
+
remove: secretRemoveCommand,
|
|
247
|
+
},
|
|
248
|
+
defaultRun() {
|
|
249
|
+
output("secret-list", { secrets: listSecretsRecursive() });
|
|
258
250
|
},
|
|
259
251
|
});
|
|
@@ -9,12 +9,9 @@
|
|
|
9
9
|
* same JSON envelope (stdout/stderr/exit-code) as the inline `runWithJsonErrors`
|
|
10
10
|
* form it replaces.
|
|
11
11
|
*/
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import { defineJsonCommand, output, runWithJsonErrors } from "../../cli/shared.js";
|
|
12
|
+
import { parsePositiveIntFlag } from "../../cli/parse-args.js";
|
|
13
|
+
import { defineGroupCommand, defineJsonCommand, output } from "../../cli/shared.js";
|
|
15
14
|
import { akmGraphEntities, akmGraphEntity, akmGraphExport, akmGraphOrphans, akmGraphRelated, akmGraphRelations, akmGraphSummary, akmGraphUpdate, } from "./graph.js";
|
|
16
|
-
// Single source of truth: the routing set is derived from the subCommands keys
|
|
17
|
-
// (M10) so adding a subcommand can never silently desync from `hasSubcommand`.
|
|
18
15
|
const graphSubCommands = {
|
|
19
16
|
summary: defineJsonCommand({
|
|
20
17
|
meta: { name: "summary", description: "Show entity-graph counts and quality telemetry" },
|
|
@@ -118,15 +115,10 @@ const graphSubCommands = {
|
|
|
118
115
|
},
|
|
119
116
|
}),
|
|
120
117
|
};
|
|
121
|
-
const
|
|
122
|
-
export const graphCommand = defineCommand({
|
|
118
|
+
export const graphCommand = defineGroupCommand({
|
|
123
119
|
meta: { name: "graph", description: "Inspect the indexed entity graph stored in SQLite" },
|
|
124
120
|
subCommands: graphSubCommands,
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
if (hasSubcommand(args, GRAPH_SUBCOMMAND_SET))
|
|
128
|
-
return;
|
|
129
|
-
output("graph-summary", akmGraphSummary());
|
|
130
|
-
});
|
|
121
|
+
defaultRun() {
|
|
122
|
+
output("graph-summary", akmGraphSummary());
|
|
131
123
|
},
|
|
132
124
|
});
|
|
@@ -8,7 +8,7 @@ import { loadConfig } from "../../core/config/config.js";
|
|
|
8
8
|
import { NotFoundError, UsageError } from "../../core/errors.js";
|
|
9
9
|
import { getDbPath } from "../../core/paths.js";
|
|
10
10
|
import { warn } from "../../core/warn.js";
|
|
11
|
-
import { closeDatabase, findEntryIdByRef, getEntryById, getEntryRefRowsForStashRoot,
|
|
11
|
+
import { closeDatabase, findEntryIdByRef, getEntryById, getEntryRefRowsForStashRoot, openExistingDatabase, openIndexDatabase, } from "../../indexer/db/db.js";
|
|
12
12
|
import { loadStoredGraphSnapshot } from "../../indexer/db/graph-db.js";
|
|
13
13
|
import { listRelatedPathsForFile } from "../../indexer/graph/graph-boost.js";
|
|
14
14
|
import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
|
|
@@ -385,7 +385,7 @@ export async function akmGraphUpdate(options) {
|
|
|
385
385
|
let db;
|
|
386
386
|
const resolvedPaths = new Set();
|
|
387
387
|
try {
|
|
388
|
-
db =
|
|
388
|
+
db = openIndexDatabase(dbPath);
|
|
389
389
|
for (const ref of options.refs) {
|
|
390
390
|
const trimmed = ref.trim();
|
|
391
391
|
if (!trimmed)
|
|
@@ -426,7 +426,7 @@ export async function akmGraphUpdate(options) {
|
|
|
426
426
|
let db;
|
|
427
427
|
const startMs = Date.now();
|
|
428
428
|
try {
|
|
429
|
-
db =
|
|
429
|
+
db = openIndexDatabase(getDbPath());
|
|
430
430
|
const onProgress = (event) => {
|
|
431
431
|
if (!event.currentPath)
|
|
432
432
|
return;
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
// Chunk sizing + per-chunk prompt assembly for consolidate. Pure token math and
|
|
5
|
+
// prompt-string construction over MemoryEntry inputs — no LLM call, no embedder,
|
|
6
|
+
// no orchestrator coupling.
|
|
7
|
+
import fs from "node:fs";
|
|
8
|
+
import { parseFrontmatter } from "../../../core/asset/frontmatter.js";
|
|
9
|
+
import { cacheHash } from "../dedup.js";
|
|
10
|
+
/**
|
|
11
|
+
* Conservative chars-per-token estimate used when computing prompt budgets.
|
|
12
|
+
* English text averages roughly 4 chars/token for most LLM tokenizers. We use
|
|
13
|
+
* 3 to stay conservative (shorter tokens = more tokens per char).
|
|
14
|
+
*/
|
|
15
|
+
const CHARS_PER_TOKEN = 3;
|
|
16
|
+
/**
|
|
17
|
+
* Overhead budget reserved for the system prompt, chunk header lines, and per-
|
|
18
|
+
* memory metadata lines (name, description, tags, separator). Measured at
|
|
19
|
+
* roughly 600 chars for the system prompt + ~100 chars of header + ~50 chars
|
|
20
|
+
* per memory × chunk size. We round up to 2 000 tokens to leave room for the
|
|
21
|
+
* model's own output.
|
|
22
|
+
*/
|
|
23
|
+
const PROMPT_OVERHEAD_TOKENS = 2_000;
|
|
24
|
+
/**
|
|
25
|
+
* Default effective token budget used when the default LLM profile's
|
|
26
|
+
* `contextLength` is not set. This is intentionally conservative (4 096)
|
|
27
|
+
* rather than being set to the model's actual context window, because:
|
|
28
|
+
*
|
|
29
|
+
* - When the agent path is used, the agent CLI (e.g. opencode)
|
|
30
|
+
* prepends its own large system prompt + conversation history before
|
|
31
|
+
* forwarding to the model. That overhead easily consumes 30K+ tokens on
|
|
32
|
+
* a model with a 16K context window, leaving very little room for
|
|
33
|
+
* chunk content.
|
|
34
|
+
* - When the HTTP path is used (an LLM profile is selected), only the akm
|
|
35
|
+
* system prompt and user prompt are sent, so the budget can be set to the
|
|
36
|
+
* model's actual context length via profiles.llm[defaults.llm].contextLength.
|
|
37
|
+
*
|
|
38
|
+
* Set profiles.llm[defaults.llm].contextLength in your config file to the
|
|
39
|
+
* model's actual context window to allow larger chunks on the HTTP path.
|
|
40
|
+
*/
|
|
41
|
+
export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
|
|
42
|
+
/**
|
|
43
|
+
* Given the model's context window and the per-memory body truncation limit,
|
|
44
|
+
* return the maximum number of memories that can safely fit in one chunk
|
|
45
|
+
* without the prompt overflowing the context window.
|
|
46
|
+
*
|
|
47
|
+
* The formula is:
|
|
48
|
+
* usableTokens = contextLength - PROMPT_OVERHEAD_TOKENS
|
|
49
|
+
* tokensPerMemory = ceil(bodyTruncation / CHARS_PER_TOKEN)
|
|
50
|
+
* chunkSize = floor(usableTokens / tokensPerMemory)
|
|
51
|
+
*
|
|
52
|
+
* Result is clamped between 1 and 50 to avoid degenerate values.
|
|
53
|
+
*
|
|
54
|
+
* @param contextLength - Model context window in tokens.
|
|
55
|
+
* @param bodyTruncation - Max chars per memory body included in the prompt.
|
|
56
|
+
* @param maxChunkSize - Optional override for the hardcoded cap of 50 (1–50).
|
|
57
|
+
*/
|
|
58
|
+
export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize) {
|
|
59
|
+
const usableTokens = Math.max(contextLength - PROMPT_OVERHEAD_TOKENS, 0);
|
|
60
|
+
const tokensPerMemory = Math.max(Math.ceil(bodyTruncation / CHARS_PER_TOKEN), 1);
|
|
61
|
+
const raw = Math.floor(usableTokens / tokensPerMemory);
|
|
62
|
+
return Math.max(1, Math.min(maxChunkSize ?? 50, raw));
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Build the per-chunk user prompt fed to the consolidate LLM.
|
|
66
|
+
*
|
|
67
|
+
* Each memory is annotated with two flags that drive the system-prompt
|
|
68
|
+
* rules at lines 181-186:
|
|
69
|
+
* - `(captureMode: hot)` — user-explicit memory; system prompt rule 2
|
|
70
|
+
* forbids proposing delete. ~60 wasted LLM verdicts/4h on this user's
|
|
71
|
+
* stack before this annotation.
|
|
72
|
+
* - `(already queued)` — the memory's body hash matches a pending
|
|
73
|
+
* consolidate proposal; system prompt rule 3 forbids proposing
|
|
74
|
+
* promote/merge/contradict. ~107/4h before this annotation.
|
|
75
|
+
*
|
|
76
|
+
* Both annotations are visible to the LLM. `pendingProposalBodyHashes`
|
|
77
|
+
* is precomputed once per run by `loadPendingConsolidateProposalHashes`
|
|
78
|
+
* so the cost stays O(memories) inside the chunk loop.
|
|
79
|
+
*/
|
|
80
|
+
export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation, pendingProposalBodyHashes = new Set(), standardsContext = "") {
|
|
81
|
+
const start = memories[0] ? `memory:${memories[0].name}` : "";
|
|
82
|
+
const end = memories[memories.length - 1] ? `memory:${memories[memories.length - 1].name}` : "";
|
|
83
|
+
const annotationsByIndex = [];
|
|
84
|
+
const hotRefs = [];
|
|
85
|
+
for (const m of memories) {
|
|
86
|
+
let body = "";
|
|
87
|
+
try {
|
|
88
|
+
body = fs.readFileSync(m.filePath, "utf8");
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
body = "(unreadable)";
|
|
92
|
+
}
|
|
93
|
+
const parsed = parseFrontmatter(body);
|
|
94
|
+
const isHot = parsed.data.captureMode === "hot";
|
|
95
|
+
// Use cacheHash (case-preserving stripped body) to match the domain used
|
|
96
|
+
// by loadPendingConsolidateProposalHashes and the body-embedding cache.
|
|
97
|
+
const bodyHash = cacheHash(body);
|
|
98
|
+
const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
|
|
99
|
+
annotationsByIndex.push({ isHot, isAlreadyQueued, body });
|
|
100
|
+
if (isHot)
|
|
101
|
+
hotRefs.push(`memory:${m.name}`);
|
|
102
|
+
}
|
|
103
|
+
const lines = [
|
|
104
|
+
`Source: ${sourceName}`,
|
|
105
|
+
`Chunk ${chunkIndex + 1} of ${totalChunks}, memories ${start}–${end}:`,
|
|
106
|
+
"",
|
|
107
|
+
];
|
|
108
|
+
if (standardsContext.trim()) {
|
|
109
|
+
lines.push("Standards to follow (the rulebook for this target):");
|
|
110
|
+
lines.push(standardsContext.trim());
|
|
111
|
+
lines.push("");
|
|
112
|
+
}
|
|
113
|
+
// Top-of-prompt protection block for hot refs. Neutral phrasing — avoid
|
|
114
|
+
// op-words like "promote", "merge", "contradict" so the model doesn't
|
|
115
|
+
// accidentally treat the warning as a hint to use that op elsewhere
|
|
116
|
+
// (variant B leaked the word "contradict" into the control sample
|
|
117
|
+
// during the diagnostic).
|
|
118
|
+
if (hotRefs.length > 0) {
|
|
119
|
+
lines.push("⛔ DO NOT propose any `delete` operation for these refs — they are user-explicit (captureMode: hot) and the downstream guard refuses them regardless. Proposing delete for any of these only wastes tokens.");
|
|
120
|
+
for (const ref of hotRefs)
|
|
121
|
+
lines.push(` - ${ref}`);
|
|
122
|
+
lines.push("");
|
|
123
|
+
}
|
|
124
|
+
for (let i = 0; i < memories.length; i++) {
|
|
125
|
+
const m = memories[i];
|
|
126
|
+
const { isHot, isAlreadyQueued, body } = annotationsByIndex[i];
|
|
127
|
+
const annotations = [];
|
|
128
|
+
if (isHot)
|
|
129
|
+
annotations.push("captureMode: hot");
|
|
130
|
+
if (isAlreadyQueued)
|
|
131
|
+
annotations.push("already queued");
|
|
132
|
+
const annotationSuffix = annotations.length > 0 ? ` (${annotations.join("; ")})` : "";
|
|
133
|
+
lines.push(`[${i + 1}] memory:${m.name}${annotationSuffix}`);
|
|
134
|
+
lines.push(`Description: ${m.description || "(none)"}`);
|
|
135
|
+
lines.push(`Tags: ${m.tags.length > 0 ? m.tags.join(", ") : "(none)"}`);
|
|
136
|
+
lines.push("---");
|
|
137
|
+
lines.push(body.slice(0, bodyTruncation));
|
|
138
|
+
lines.push("");
|
|
139
|
+
}
|
|
140
|
+
return lines.join("\n");
|
|
141
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
// Eligibility / safety predicates for consolidate: "may we touch this memory?"
|
|
5
|
+
// One reason to change — the policy for what consolidate is allowed to act on.
|
|
6
|
+
import fs from "node:fs";
|
|
7
|
+
import { parseFrontmatter } from "../../../core/asset/frontmatter.js";
|
|
8
|
+
import { hasHotCaptureMode } from "../../proposal/validators/proposal-quality-validators.js";
|
|
9
|
+
export function isConsolidationEligibleMemoryName(name) {
|
|
10
|
+
return !name.endsWith(".derived");
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Returns true when the memory file has `captureMode: hot` in its frontmatter.
|
|
14
|
+
*
|
|
15
|
+
* Hot memories are USER-EXPLICIT (written via `akm remember` on the hot path).
|
|
16
|
+
* The consolidate LLM is forbidden from deleting or auto-merging them — the
|
|
17
|
+
* user wrote them on purpose and only the user can decide to retire them.
|
|
18
|
+
*
|
|
19
|
+
* Reads the file once per check; consolidate runs against ~10 memories per
|
|
20
|
+
* chunk so the IO cost is trivial. Returns false on any read/parse error
|
|
21
|
+
* (fail-safe: an unparseable file is treated as not-hot, but the broader
|
|
22
|
+
* consolidate flow already guards against unparseable memories elsewhere).
|
|
23
|
+
*
|
|
24
|
+
* Defends against four observed defect classes (see
|
|
25
|
+
* `memory:akm-improve-critical-review-2026-05-20`):
|
|
26
|
+
* - LLM marks a memory contradicted then deletes (dangling contradictedBy)
|
|
27
|
+
* - LLM merges two unrelated memories sharing a topic keyword
|
|
28
|
+
* - LLM judges a recent durable design memo as "redundant"
|
|
29
|
+
* - Cascade deletes (LLM uses ref:X as `contradictedBy` for ref:Y then deletes both)
|
|
30
|
+
*/
|
|
31
|
+
export function isHotCapturedMemory(filePath) {
|
|
32
|
+
try {
|
|
33
|
+
if (!fs.existsSync(filePath))
|
|
34
|
+
return false;
|
|
35
|
+
const content = fs.readFileSync(filePath, "utf8");
|
|
36
|
+
const parsed = parseFrontmatter(content);
|
|
37
|
+
return hasHotCaptureMode(parsed.data);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export function consolidateGuardStatus(filePath) {
|
|
44
|
+
if (!fs.existsSync(filePath))
|
|
45
|
+
return "missing";
|
|
46
|
+
let content;
|
|
47
|
+
try {
|
|
48
|
+
content = fs.readFileSync(filePath, "utf8");
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
return "unparseable";
|
|
52
|
+
}
|
|
53
|
+
let parsed;
|
|
54
|
+
try {
|
|
55
|
+
parsed = parseFrontmatter(content);
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
return "unparseable";
|
|
59
|
+
}
|
|
60
|
+
const data = parsed.data;
|
|
61
|
+
if (!data || Object.keys(data).length === 0)
|
|
62
|
+
return "unparseable";
|
|
63
|
+
return hasHotCaptureMode(data) ? "hot" : "safe";
|
|
64
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
export function isValidOp(op) {
|
|
5
|
+
if (typeof op !== "object" || op === null)
|
|
6
|
+
return false;
|
|
7
|
+
const o = op;
|
|
8
|
+
if (o.op === "merge") {
|
|
9
|
+
return typeof o.primary === "string" && Array.isArray(o.secondaries);
|
|
10
|
+
}
|
|
11
|
+
if (o.op === "delete") {
|
|
12
|
+
return typeof o.ref === "string";
|
|
13
|
+
}
|
|
14
|
+
if (o.op === "promote") {
|
|
15
|
+
return typeof o.ref === "string" && typeof o.knowledgeRef === "string";
|
|
16
|
+
}
|
|
17
|
+
if (o.op === "contradict") {
|
|
18
|
+
return typeof o.ref === "string" && typeof o.contradictedByRef === "string";
|
|
19
|
+
}
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
export function mergePlans(chunks, knownRefs) {
|
|
23
|
+
const mergeOps = new Map();
|
|
24
|
+
const deleteOps = new Map();
|
|
25
|
+
const promoteOps = new Map();
|
|
26
|
+
// C-3 / #382: contradict ops keyed by `ref|contradictedByRef` to deduplicate.
|
|
27
|
+
const contradictOps = new Map();
|
|
28
|
+
const warnings = [];
|
|
29
|
+
for (const chunk of chunks) {
|
|
30
|
+
for (const op of chunk) {
|
|
31
|
+
if (op.op === "merge") {
|
|
32
|
+
// Drop ops whose primary the LLM hallucinated (not in the loaded memory
|
|
33
|
+
// pool). Without this guard, a hallucinated primary flows all the way to
|
|
34
|
+
// Phase B where !memoryByRef.has(primary) fires and charges every real
|
|
35
|
+
// secondary with merge_primary_missing — masking LLM hallucinations as
|
|
36
|
+
// filter regressions in health metrics.
|
|
37
|
+
if (knownRefs && !knownRefs.has(op.primary)) {
|
|
38
|
+
warnings.push(`mergePlans: primary ${op.primary} not in loaded memory pool (LLM hallucination) — dropping op before execution.`);
|
|
39
|
+
// Use a dedicated skip reason so dashboards can distinguish
|
|
40
|
+
// hallucinated primaries from stale-DB regressions.
|
|
41
|
+
// Secondaries are real refs; they are NOT charged here — they remain
|
|
42
|
+
// available for other ops to claim.
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
// Filter hallucinated secondaries while preserving real ones.
|
|
46
|
+
let mergeOp = op;
|
|
47
|
+
if (knownRefs) {
|
|
48
|
+
const filteredSecondaries = op.secondaries.filter((sec) => {
|
|
49
|
+
if (!knownRefs.has(sec)) {
|
|
50
|
+
warnings.push(`mergePlans: secondary ${sec} not in loaded memory pool (LLM hallucination) — dropping from op.`);
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
return true;
|
|
54
|
+
});
|
|
55
|
+
if (filteredSecondaries.length !== op.secondaries.length) {
|
|
56
|
+
mergeOp = { ...op, secondaries: filteredSecondaries };
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// merge wins over delete
|
|
60
|
+
if (deleteOps.has(mergeOp.primary)) {
|
|
61
|
+
deleteOps.delete(mergeOp.primary);
|
|
62
|
+
}
|
|
63
|
+
for (const sec of mergeOp.secondaries) {
|
|
64
|
+
if (deleteOps.has(sec))
|
|
65
|
+
deleteOps.delete(sec);
|
|
66
|
+
}
|
|
67
|
+
mergeOps.set(mergeOp.primary, mergeOp);
|
|
68
|
+
}
|
|
69
|
+
else if (op.op === "delete") {
|
|
70
|
+
// merge and promote both win over delete. A promote is non-destructive
|
|
71
|
+
// (creates a proposal) but the source memory is counted in `promoted`;
|
|
72
|
+
// if a delete also fires, the ref lands in both `promoted` and
|
|
73
|
+
// `skipReasons`, breaking the invariant by +1.
|
|
74
|
+
if (!mergeOps.has(op.ref) && !promoteOps.has(op.ref)) {
|
|
75
|
+
deleteOps.set(op.ref, op);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
else if (op.op === "promote") {
|
|
79
|
+
// C-2 / #381: when both a promote and a merge target the same ref,
|
|
80
|
+
// queue the promote FIRST rather than discarding it. The promote op
|
|
81
|
+
// routes through createProposal (the human-gated proposal queue), so
|
|
82
|
+
// it is non-destructive. The merge follows after the proposal is
|
|
83
|
+
// created. This preserves the human reviewer's ability to inspect the
|
|
84
|
+
// promotion before the source memory is merged/deleted.
|
|
85
|
+
// AGM K*8 — retain the maximally informative consistent subset.
|
|
86
|
+
promoteOps.set(op.ref, op);
|
|
87
|
+
}
|
|
88
|
+
else if (op.op === "contradict") {
|
|
89
|
+
// Deduplicate by ref+contradictedByRef pair.
|
|
90
|
+
const key = `${op.ref}|${op.contradictedByRef}`;
|
|
91
|
+
if (!contradictOps.has(key)) {
|
|
92
|
+
contradictOps.set(key, op);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
// Second pass: enforce merge-wins-over-delete and deduplicate secondaries.
|
|
98
|
+
//
|
|
99
|
+
// 1. Delete/secondary ordering bug: the per-chunk loop removes delete ops
|
|
100
|
+
// for secondaries that were already in deleteOps, but misses the case
|
|
101
|
+
// where the delete chunk came first. A full sweep here fixes both orders.
|
|
102
|
+
//
|
|
103
|
+
// 2. Cross-merge secondary dedup: if ref A is a secondary in two merge ops,
|
|
104
|
+
// only the first (insertion-order) retains it. Without this, a successful
|
|
105
|
+
// merge credits A to mergedSecondaries and a later merge's emitMerge-
|
|
106
|
+
// FailureSkips also charges A to skipReasons — double-counting A while
|
|
107
|
+
// processed has it only once.
|
|
108
|
+
//
|
|
109
|
+
// 3. Primary-as-secondary dedup: if ref A is a primary in one merge op and
|
|
110
|
+
// a secondary in another, remove A from the secondary list. Both merges
|
|
111
|
+
// would otherwise claim A (merged++ for A, then mergedSecondaries++ for A)
|
|
112
|
+
// breaking the invariant the same way.
|
|
113
|
+
// Also remove delete ops for any ref claimed by a promote op (handles the
|
|
114
|
+
// case where the delete chunk appeared before the promote chunk).
|
|
115
|
+
for (const ref of promoteOps.keys()) {
|
|
116
|
+
deleteOps.delete(ref);
|
|
117
|
+
}
|
|
118
|
+
const claimedSecondaries = new Set();
|
|
119
|
+
for (const mergeOp of mergeOps.values()) {
|
|
120
|
+
deleteOps.delete(mergeOp.primary);
|
|
121
|
+
mergeOp.secondaries = mergeOp.secondaries.filter((sec) => {
|
|
122
|
+
if (mergeOps.has(sec)) {
|
|
123
|
+
warnings.push(`Merge: secondary ${sec} is also a merge primary — removing from secondary list to avoid double-count.`);
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
if (claimedSecondaries.has(sec)) {
|
|
127
|
+
warnings.push(`Merge: secondary ${sec} appears in multiple merge ops — retaining in first op only.`);
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
claimedSecondaries.add(sec);
|
|
131
|
+
deleteOps.delete(sec);
|
|
132
|
+
return true;
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
// C-2 / #381: promote ops are ordered BEFORE merge ops so that the
|
|
136
|
+
// human-gated proposal queue entry is created before any destructive merge.
|
|
137
|
+
// Phase B processes ops in array order, so promote executes first.
|
|
138
|
+
const ops = [
|
|
139
|
+
...promoteOps.values(),
|
|
140
|
+
...mergeOps.values(),
|
|
141
|
+
...deleteOps.values(),
|
|
142
|
+
...contradictOps.values(),
|
|
143
|
+
];
|
|
144
|
+
return { ops, warnings };
|
|
145
|
+
}
|