akm-cli 0.9.14 → 0.9.15-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +559 -0
  2. package/STABILITY.md +6 -3
  3. package/dist/akm +54 -1
  4. package/dist/akm-migrate +34 -1
  5. package/dist/assets/prompts/reflect-feedback-framing.md +1 -0
  6. package/dist/assets/prompts/reflect-llm-framed-contract.md +2 -0
  7. package/dist/assets/prompts/reflect-llm-schema-contract.md +2 -0
  8. package/dist/assets/tasks/core/improve.yml +1 -1
  9. package/dist/assets/tasks/core/index-refresh.yml +1 -1
  10. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +1 -1
  11. package/dist/assets/tasks/improve/akm-improve-catchup.yml +1 -1
  12. package/dist/assets/tasks/improve/akm-improve-consolidate.yml +1 -1
  13. package/dist/assets/tasks/improve/akm-improve-frequent.yml +1 -1
  14. package/dist/assets/tasks/improve/akm-improve-nightly.yml +1 -1
  15. package/dist/cli/retired-commands.js +0 -1
  16. package/dist/cli/shared.js +9 -0
  17. package/dist/cli/unknown-flags.js +1 -0
  18. package/dist/cli.js +40 -3
  19. package/dist/commands/config-cli.js +85 -3
  20. package/dist/commands/env/env-cli.js +1 -42
  21. package/dist/commands/env/env.js +1 -1
  22. package/dist/commands/env/secret-cli.js +1 -2
  23. package/dist/commands/health/checks.js +357 -63
  24. package/dist/commands/health/engine-usage.js +45 -0
  25. package/dist/commands/health/improve-metrics.js +18 -0
  26. package/dist/commands/health/llm-usage.js +41 -1
  27. package/dist/commands/health/plugin-staleness.js +7 -3
  28. package/dist/commands/health/version-drift.js +93 -0
  29. package/dist/commands/health/windows.js +3 -1
  30. package/dist/commands/health.js +44 -9
  31. package/dist/commands/improve/consolidate/chunking.js +4 -2
  32. package/dist/commands/improve/improve-cli.js +99 -5
  33. package/dist/commands/improve/improve-report.js +154 -0
  34. package/dist/commands/improve/improve-result-file.js +45 -33
  35. package/dist/commands/improve/improve-strategies.js +133 -3
  36. package/dist/commands/improve/improve-usage-report.js +182 -0
  37. package/dist/commands/improve/improve.js +40 -3
  38. package/dist/commands/improve/locks.js +28 -78
  39. package/dist/commands/improve/planner.js +1 -0
  40. package/dist/commands/improve/preparation.js +9 -1
  41. package/dist/commands/improve/reflect.js +44 -4
  42. package/dist/commands/models-cli.js +50 -1
  43. package/dist/commands/proposal/repository.js +8 -3
  44. package/dist/commands/proposal/validators/proposal-quality-validators.js +41 -6
  45. package/dist/commands/proposal/validators/proposal-validators.js +24 -0
  46. package/dist/commands/read/search-cli.js +38 -2
  47. package/dist/commands/read/show.js +103 -4
  48. package/dist/commands/sources/info.js +5 -1
  49. package/dist/commands/sources/installed-stashes.js +58 -16
  50. package/dist/commands/sources/self-update.js +2 -2
  51. package/dist/commands/sources/stash-cli.js +48 -0
  52. package/dist/commands/tasks/tasks-cli.js +49 -2
  53. package/dist/commands/workflow-cli.js +86 -12
  54. package/dist/core/asset/markdown-fragments.js +35 -0
  55. package/dist/core/config/config-schema.js +14 -0
  56. package/dist/core/config/config.js +302 -24
  57. package/dist/core/config/schema/embedding.js +41 -0
  58. package/dist/core/env-secret-ref.js +58 -5
  59. package/dist/core/errors.js +30 -0
  60. package/dist/core/file-lock.js +49 -15
  61. package/dist/core/improve-result.js +51 -0
  62. package/dist/core/loopback.js +17 -0
  63. package/dist/core/parent-watchdog.js +64 -0
  64. package/dist/core/paths.js +11 -0
  65. package/dist/core/run-lock.js +107 -0
  66. package/dist/core/sensitive-marker-path.js +19 -0
  67. package/dist/core/state-db.js +74 -14
  68. package/dist/indexer/index-rebuild-lock.js +73 -0
  69. package/dist/indexer/index-writer-lock.js +40 -1
  70. package/dist/indexer/index-written-assets.js +29 -1
  71. package/dist/indexer/indexer.js +93 -29
  72. package/dist/indexer/materialize-embeddings.js +564 -48
  73. package/dist/indexer/search/db-search.js +49 -2
  74. package/dist/indexer/search/search-source.js +23 -1
  75. package/dist/integrations/agent/engine-resolution.js +96 -6
  76. package/dist/integrations/agent/execution-definitions.js +6 -15
  77. package/dist/integrations/agent/execution-lowering.js +6 -1
  78. package/dist/integrations/agent/execution-preparation.js +1 -1
  79. package/dist/integrations/agent/model-map.js +123 -20
  80. package/dist/integrations/agent/prompts.js +40 -8
  81. package/dist/integrations/agent/runner-dispatch.js +9 -3
  82. package/dist/integrations/agent/runner.js +2 -0
  83. package/dist/llm/client.js +8 -3
  84. package/dist/llm/embedder.js +20 -8
  85. package/dist/llm/embedders/local.js +10 -2
  86. package/dist/llm/embedders/remote.js +497 -32
  87. package/dist/output/shapes/helpers.js +38 -2
  88. package/dist/output/shapes/models-list.js +16 -0
  89. package/dist/output/shapes/passthrough.js +2 -0
  90. package/dist/output/shapes.js +4 -0
  91. package/dist/output/text/command-format.js +29 -0
  92. package/dist/output/text/helpers.js +1 -1
  93. package/dist/output/text/improve-report.js +27 -0
  94. package/dist/{commands/env/marker-path.js → output/text/models.js} +4 -3
  95. package/dist/output/text/show-format.js +4 -0
  96. package/dist/output/text.js +4 -0
  97. package/dist/scripts/akm-migrate-node.js +25146 -21759
  98. package/dist/scripts/akm-migrate.js +24271 -20885
  99. package/dist/storage/repositories/embedding-salvage-repository.js +184 -0
  100. package/dist/storage/repositories/improve-runs-repository.js +34 -0
  101. package/dist/storage/repositories/index-fts-repository.js +49 -6
  102. package/dist/storage/repositories/index-schema.js +16 -0
  103. package/dist/storage/repositories/index-vec-repository.js +30 -0
  104. package/dist/storage/repositories/workflow-runs-repository.js +55 -18
  105. package/dist/tasks/backends/cron.js +14 -7
  106. package/dist/tasks/run/run-native-task.js +23 -1
  107. package/dist/tasks/run/run-workflow-task.js +16 -0
  108. package/dist/workflows/exec/child-workflow.js +2 -2
  109. package/dist/workflows/exec/dispatch-redaction.js +21 -9
  110. package/dist/workflows/exec/run-workflow.js +6 -5
  111. package/dist/workflows/runtime/runs.js +33 -5
  112. package/docs/migration/release-notes/0.9.15.md +133 -0
  113. package/docs/migration/release-notes/README.md +5 -0
  114. package/docs/reference/cli.md +271 -30
  115. package/docs/reference/configuration.md +234 -21
  116. package/docs/reference/data-and-telemetry.md +8 -0
  117. package/docs/reference/tasks.md +16 -1
  118. package/docs/reference/workflow-schema.md +5 -1
  119. package/package.json +1 -1
  120. package/schemas/akm-config.json +47 -0
@@ -0,0 +1,184 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * `index.db` embedding salvage (#955) — a transient, self-emptying table
6
+ * that lets a full rebuild or an index-generation bump reuse vectors instead
7
+ * of re-embedding a corpus whose content did not change.
8
+ *
9
+ * Zero steady-state cost by design: this is NOT a second embedding cache.
10
+ * Rows are copied aside only at the moment they would otherwise be discarded
11
+ * wholesale — a full-index wipe (`persistDirRecords`) or a generation bump
12
+ * (`rebuildIncompatibleIndexGeneration`) — and are consumed by the very next
13
+ * embedding pass (`generateEmbeddingsForDb`). A pass that completes without
14
+ * abort or circuit-break purges whatever is left; an interrupted pass leaves
15
+ * the table for the next attempt to pick up.
16
+ *
17
+ * Reuse is keyed on `sha256(search_text)` plus the fingerprint the vector was
18
+ * generated under — a fingerprint mismatch or a single-byte content change
19
+ * both correctly fall through to a real provider call. `content_hash` is the
20
+ * PRIMARY KEY (not `(content_hash, fingerprint)`) so relabeling a whole
21
+ * generation's fingerprint after a canary "keep" verdict is one UPDATE, and a
22
+ * hash colliding across two discards simply keeps the most recent copy —
23
+ * salvage is a best-effort optimization, not a durable multi-generation
24
+ * archive.
25
+ */
26
+ import { createHash } from "node:crypto";
27
+ import { blobToEmbedding } from "./embeddings-repository.js";
28
+ import { getMeta } from "./index-meta-repository.js";
29
+ import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
30
+ /**
31
+ * Create the salvage table. Additive-only DDL: it carries no bearing on the
32
+ * `entries` generation fingerprint (`hasCanonicalEntrySchema`), so adding it
33
+ * does not require an index-generation bump.
34
+ */
35
+ export function ensureEmbeddingSalvageTable(db) {
36
+ db.exec(`
37
+ CREATE TABLE IF NOT EXISTS embedding_salvage (
38
+ content_hash TEXT PRIMARY KEY,
39
+ fingerprint TEXT NOT NULL,
40
+ embedding BLOB NOT NULL,
41
+ salvaged_at TEXT NOT NULL
42
+ );
43
+ `);
44
+ }
45
+ /** The one hash function salvage writes and reuse lookups must agree on. */
46
+ export function hashEmbeddableText(searchText) {
47
+ return createHash("sha256").update(searchText, "utf8").digest("hex");
48
+ }
49
+ function tableExists(db, name) {
50
+ return db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name) != null;
51
+ }
52
+ function tableHasColumn(db, table, column) {
53
+ const columns = db.prepare(`PRAGMA table_info(${table})`).all();
54
+ return columns.some((c) => c.name === column);
55
+ }
56
+ /**
57
+ * Copy every (hash of search_text, embedding) pair about to be discarded
58
+ * wholesale into `embedding_salvage`, tagged with the `embeddingFingerprint`
59
+ * the discarded vectors were generated under. The caller MUST run this
60
+ * inside the same transaction as the discard that follows it, so the copy
61
+ * and the delete commit or roll back together.
62
+ *
63
+ * Streams `entries JOIN embeddings` in id-ordered pages of
64
+ * {@link SQLITE_CHUNK_SIZE} instead of loading every row into memory before
65
+ * hashing anything — a full rebuild of a large stash otherwise held the
66
+ * entire corpus's search text and vectors in memory at once just to copy
67
+ * them aside (#955, field-report follow-up).
68
+ *
69
+ * A no-op (returns 0) when there is no stored `embeddingFingerprint` to tag
70
+ * rows with (nothing was ever verified against a provider, so there is
71
+ * nothing worth reusing later) or the generation being discarded predates
72
+ * the `entries.search_text` column or has no `embeddings` table at all — an
73
+ * older generation than that has nothing this can safely read.
74
+ */
75
+ export function salvageEmbeddingsBeforeDiscard(db) {
76
+ const fingerprint = getMeta(db, "embeddingFingerprint");
77
+ if (!fingerprint)
78
+ return 0;
79
+ if (!tableExists(db, "entries") || !tableExists(db, "embeddings"))
80
+ return 0;
81
+ if (!tableHasColumn(db, "entries", "search_text"))
82
+ return 0;
83
+ const page = db.prepare("SELECT e.id AS id, e.search_text AS searchText, em.embedding AS embedding " +
84
+ "FROM entries e JOIN embeddings em ON em.id = e.id WHERE e.id > ? ORDER BY e.id LIMIT ?");
85
+ const insert = db.prepare("INSERT OR REPLACE INTO embedding_salvage (content_hash, fingerprint, embedding, salvaged_at) VALUES (?, ?, ?, ?)");
86
+ const salvagedAt = new Date().toISOString();
87
+ let lastId = 0;
88
+ let total = 0;
89
+ for (;;) {
90
+ const rows = page.all(lastId, SQLITE_CHUNK_SIZE);
91
+ if (rows.length === 0)
92
+ break;
93
+ for (const row of rows) {
94
+ insert.run(hashEmbeddableText(row.searchText), fingerprint, row.embedding, salvagedAt);
95
+ }
96
+ total += rows.length;
97
+ lastId = rows[rows.length - 1]?.id ?? lastId;
98
+ if (rows.length < SQLITE_CHUNK_SIZE)
99
+ break;
100
+ }
101
+ return total;
102
+ }
103
+ /**
104
+ * Remove every salvage row. Called after an embedding pass completes without
105
+ * abort or circuit-break (the salvaged generation has now either been reused
106
+ * or superseded), and by `--reembed` / a canary "rebuild" verdict (the
107
+ * salvaged vectors belong to a different model and are never reusable).
108
+ */
109
+ export function purgeEmbeddingSalvage(db) {
110
+ db.exec("DELETE FROM embedding_salvage");
111
+ }
112
+ /**
113
+ * A canary "keep" verdict means the model did not actually change — only its
114
+ * fingerprint STRING did (e.g. a gateway rename). Salvage rows tagged with
115
+ * the old string are still valid vectors; rewrite them to the new string so
116
+ * they remain reusable instead of silently going stale.
117
+ */
118
+ export function relabelEmbeddingSalvageFingerprint(db, fromFingerprint, toFingerprint) {
119
+ db.prepare("UPDATE embedding_salvage SET fingerprint = ? WHERE fingerprint = ?").run(toFingerprint, fromFingerprint);
120
+ }
121
+ /**
122
+ * Reuse salvaged vectors for `entries` whose `searchText` hash matches a
123
+ * salvage row tagged with the CURRENT `fingerprint` — never across
124
+ * fingerprints, and never when `search_text` differs by even one byte (the
125
+ * hash is exact-match only, by design). Matches are written via
126
+ * `writeReused` in chunks of {@link SQLITE_CHUNK_SIZE}, each its own
127
+ * transaction, mirroring the main pass's per-batch commit (#955) so an
128
+ * interruption partway through the reuse step keeps whatever already wrote.
129
+ *
130
+ * The steady state of every ordinary run is an EMPTY salvage table (nothing
131
+ * was just discarded), so this checks that first with one indexed lookup —
132
+ * `SELECT 1 ... LIMIT 1` — before hashing a single pending entry. Hashing
133
+ * every entry up front to look up a table that is empty 100% of the time
134
+ * outside a rebuild was pure wasted work on the common path (#955,
135
+ * field-report follow-up).
136
+ */
137
+ export function reuseSalvagedEmbeddings(db, entries, fingerprint, writeReused) {
138
+ if (entries.length === 0)
139
+ return { reusedCount: 0, remaining: [] };
140
+ const anySalvageForFingerprint = db
141
+ .prepare("SELECT 1 FROM embedding_salvage WHERE fingerprint = ? LIMIT 1")
142
+ .get(fingerprint);
143
+ if (!anySalvageForFingerprint)
144
+ return { reusedCount: 0, remaining: [...entries] };
145
+ const hashes = entries.map((entry) => hashEmbeddableText(entry.searchText));
146
+ const salvageByHash = new Map();
147
+ const uniqueHashes = [...new Set(hashes)];
148
+ for (let offset = 0; offset < uniqueHashes.length; offset += SQLITE_CHUNK_SIZE) {
149
+ const chunk = uniqueHashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
150
+ const placeholders = chunk.map(() => "?").join(",");
151
+ const rows = db
152
+ .prepare(`SELECT content_hash AS contentHash, embedding FROM embedding_salvage WHERE fingerprint = ? AND content_hash IN (${placeholders})`)
153
+ .all(fingerprint, ...chunk);
154
+ for (const row of rows)
155
+ salvageByHash.set(row.contentHash, row.embedding);
156
+ }
157
+ if (salvageByHash.size === 0)
158
+ return { reusedCount: 0, remaining: [...entries] };
159
+ let reusedCount = 0;
160
+ const remaining = [];
161
+ for (let offset = 0; offset < entries.length; offset += SQLITE_CHUNK_SIZE) {
162
+ const end = Math.min(offset + SQLITE_CHUNK_SIZE, entries.length);
163
+ const chunkMatches = [];
164
+ for (let i = offset; i < end; i++) {
165
+ const entry = entries[i];
166
+ const blob = salvageByHash.get(hashes[i]);
167
+ if (blob)
168
+ chunkMatches.push({ entry, blob });
169
+ else
170
+ remaining.push(entry);
171
+ }
172
+ if (chunkMatches.length === 0)
173
+ continue;
174
+ db.transaction(() => {
175
+ for (const { entry, blob } of chunkMatches) {
176
+ if (writeReused(entry, blobToEmbedding(blob)))
177
+ reusedCount++;
178
+ else
179
+ remaining.push(entry);
180
+ }
181
+ })();
182
+ }
183
+ return { reusedCount, remaining };
184
+ }
@@ -102,6 +102,40 @@ export function queryImproveRuns(db, since, until) {
102
102
  : "SELECT id, started_at, completed_at, ok, scope_mode, scope_value, strategy, result_json FROM improve_runs WHERE started_at >= ? AND dry_run = 0 ORDER BY started_at DESC";
103
103
  return (until ? db.prepare(sql).all(since, until) : db.prepare(sql).all(since));
104
104
  }
105
+ const IMPROVE_RUN_ROW_COLUMNS = "id, started_at, completed_at, stash_dir, dry_run, strategy, scope_mode, scope_value, guidance, ok, result_json, metrics_json, metadata_json";
106
+ /**
107
+ * Look up a single `improve_runs` row by its id (#944 — `akm improve report --run <id>`).
108
+ * `id` is the PRIMARY KEY, so at most one row matches. Returns `undefined`
109
+ * for an unknown id rather than throwing — the caller decides how to report
110
+ * "no such run".
111
+ */
112
+ export function getImproveRunById(db, id) {
113
+ return db.prepare(`SELECT ${IMPROVE_RUN_ROW_COLUMNS} FROM improve_runs WHERE id = ?`).get(id);
114
+ }
115
+ /**
116
+ * Look up the most recent real (non-dry-run) `improve_runs` row (#944 —
117
+ * `akm improve report`'s default target, and `--last`). Same `dry_run = 0`
118
+ * filter as {@link queryImproveRuns} so a dry-run preview never becomes the
119
+ * implicit "last run" a report reads.
120
+ */
121
+ export function getLatestImproveRun(db) {
122
+ return db
123
+ .prepare(`SELECT ${IMPROVE_RUN_ROW_COLUMNS} FROM improve_runs WHERE dry_run = 0 ORDER BY started_at DESC LIMIT 1`)
124
+ .get();
125
+ }
126
+ /**
127
+ * #950: count real (non-dry-run) improve_runs rows whose `started_at` falls
128
+ * in `[since, now)` — same filter as {@link queryImproveRuns}, but a `SELECT
129
+ * COUNT(*)` for a caller (the `engine-last-used` gate) that only needs to
130
+ * know whether any run happened, not the rows themselves (which would pull
131
+ * every `result_json` blob in the window just to test for zero).
132
+ */
133
+ export function countImproveRunsSince(db, since) {
134
+ const row = db
135
+ .prepare("SELECT COUNT(*) AS cnt FROM improve_runs WHERE started_at >= ? AND dry_run = 0")
136
+ .get(since);
137
+ return row.cnt;
138
+ }
105
139
  /**
106
140
  * Delete improve_runs rows older than `retentionDays` (default: 90). Mirrors
107
141
  * {@link purgeOldEvents} — same default, same return shape (number of rows
@@ -7,7 +7,7 @@
7
7
  * Owns the `entries_fts` full-text query path, per-entry projections, and the
8
8
  * explicit full recovery rebuild.
9
9
  */
10
- import { fragmentForSelector, splitMarkdownFragments } from "../../core/asset/markdown-fragments.js";
10
+ import { splitMarkdownFragments } from "../../core/asset/markdown-fragments.js";
11
11
  import { stableFtsScore } from "../../core/lexical-score.js";
12
12
  import { warn } from "../../core/warn.js";
13
13
  import { buildLexicalQueryPlan } from "../../indexer/search/fts-query.js";
@@ -87,11 +87,54 @@ export function searchFts(db, query, limit, entryType, excludeTypes) {
87
87
  * file edit; the next index refresh atomically publishes the new revision.
88
88
  */
89
89
  export function getIndexedMarkdownFragment(db, itemRef, fragmentId) {
90
- const row = db
91
- .prepare("SELECT s.safe_markdown FROM entry_fragments_fts f JOIN entry_fragments s ON s.entry_id = f.entry_id JOIN entries e ON e.id = f.entry_id WHERE e.item_ref = ? AND f.fragment_id = ?")
92
- .get(itemRef, fragmentId);
93
- const fragment = row ? fragmentForSelector(row.safe_markdown, fragmentId) : undefined;
94
- return fragment ? { content: fragment.text } : undefined;
90
+ return getIndexedMarkdownFragments(db, [{ itemRef, fragmentId }])[0];
91
+ }
92
+ /**
93
+ * Batch the selected-hit projection read. Search commonly enriches several
94
+ * fragment hits at once; reading all indexed-safe parents in chunks avoids an
95
+ * N-query loop, while grouping selectors by parent ensures each safe revision
96
+ * is split at most once.
97
+ */
98
+ export function getIndexedMarkdownFragments(db, selections) {
99
+ if (selections.length === 0)
100
+ return [];
101
+ const itemRefs = [...new Set(selections.map((selection) => selection.itemRef))];
102
+ const sourceByRef = new Map();
103
+ for (let offset = 0; offset < itemRefs.length; offset += SQLITE_CHUNK_SIZE) {
104
+ const chunk = itemRefs.slice(offset, offset + SQLITE_CHUNK_SIZE);
105
+ const placeholders = chunk.map(() => "?").join(",");
106
+ const rows = db
107
+ .prepare(`SELECT e.item_ref, s.safe_markdown FROM entry_fragments s JOIN entries e ON e.id = s.entry_id WHERE e.item_ref IN (${placeholders})`)
108
+ .all(...chunk);
109
+ for (const row of rows)
110
+ sourceByRef.set(row.item_ref, row.safe_markdown);
111
+ }
112
+ const fragmentsByRef = new Map();
113
+ for (const [itemRef, safeMarkdown] of sourceByRef) {
114
+ fragmentsByRef.set(itemRef, splitMarkdownFragments(safeMarkdown));
115
+ }
116
+ return selections.map((selection) => {
117
+ const safeMarkdown = sourceByRef.get(selection.itemRef);
118
+ const fragments = fragmentsByRef.get(selection.itemRef);
119
+ if (safeMarkdown === undefined || !fragments)
120
+ return undefined;
121
+ const fragment = fragments.find((candidate) => candidate.fragmentId === selection.fragmentId || candidate.headingSlug === selection.fragmentId);
122
+ return fragment ? materializeIndexedMarkdownFragment(fragment, fragments, safeMarkdown.length) : undefined;
123
+ });
124
+ }
125
+ function materializeIndexedMarkdownFragment(fragment, fragments, parentChars) {
126
+ return {
127
+ content: fragment.text,
128
+ ordinal: fragment.ordinal,
129
+ count: fragments.length,
130
+ startLine: fragment.startLine,
131
+ endLine: fragment.endLine,
132
+ previousFragmentId: fragments[fragment.ordinal - 1]?.fragmentId,
133
+ nextFragmentId: fragments[fragment.ordinal + 1]?.fragmentId,
134
+ fragmentChars: fragment.text.length,
135
+ parentChars,
136
+ fragments,
137
+ };
95
138
  }
96
139
  function runFtsQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludeTypes) {
97
140
  // Preserve the repository's ordinary limit contract for direct callers.
@@ -12,6 +12,7 @@
12
12
  */
13
13
  import { ConfigError } from "../../core/errors.js";
14
14
  import { warn } from "../../core/warn.js";
15
+ import { ensureEmbeddingSalvageTable, salvageEmbeddingsBeforeDiscard } from "./embedding-salvage-repository.js";
15
16
  import { CANONICAL_ENTRY_SCHEMA_SQL, CANONICAL_INDEX_DB_VERSION, classifyIndexGeneration, isCanonicalIndexGeneration, } from "./index-entry-schema.js";
16
17
  import { getMeta, setMeta } from "./index-meta-repository.js";
17
18
  import { isVecAvailable, purgeEmbeddings } from "./index-vec-repository.js";
@@ -196,6 +197,13 @@ function rebuildIncompatibleIndexGeneration(db) {
196
197
  vecResetPending = true;
197
198
  }
198
199
  db.transaction(() => {
200
+ // #955: copy embeddings about to be discarded wholesale into
201
+ // `embedding_salvage` (keyed by content hash + the fingerprint they were
202
+ // generated under) BEFORE dropping `embeddings`, in the same transaction
203
+ // as the drop, so the copy and the discard commit or roll back together.
204
+ // The next embedding pass hands salvaged vectors back to unchanged
205
+ // content instead of re-embedding the whole corpus after this bump.
206
+ salvageEmbeddingsBeforeDiscard(db);
199
207
  db.exec("DROP TABLE IF EXISTS graph_file_relations");
200
208
  db.exec("DROP TABLE IF EXISTS graph_file_entities");
201
209
  db.exec("DROP TABLE IF EXISTS graph_files");
@@ -212,6 +220,9 @@ function rebuildIncompatibleIndexGeneration(db) {
212
220
  db.exec("DROP TABLE IF EXISTS index_dir_state");
213
221
  db.exec("DROP TABLE IF EXISTS entries");
214
222
  db.exec("DELETE FROM index_meta");
223
+ // embedding_salvage is deliberately absent from the drop list above —
224
+ // it is the ONE piece of derived state a generation rebuild must not
225
+ // discard.
215
226
  })();
216
227
  if (vecResetPending)
217
228
  setMeta(db, "vecResetPending", "1");
@@ -224,6 +235,11 @@ export function ensureSchema(db, embeddingDim) {
224
235
  value TEXT NOT NULL
225
236
  );
226
237
  `);
238
+ // #955: created before the generation-rebuild check below so a discard
239
+ // has somewhere to copy vectors to. Additive-only — it carries no bearing
240
+ // on the `entries` generation fingerprint (`hasCanonicalEntrySchema`), so
241
+ // adding it does not require a `CANONICAL_INDEX_DB_VERSION` bump.
242
+ ensureEmbeddingSalvageTable(db);
227
243
  rebuildIncompatibleIndexGeneration(db);
228
244
  db.exec(CANONICAL_ENTRY_SCHEMA_SQL);
229
245
  // Workflow source is compiled directly into source IR at each command
@@ -333,3 +333,33 @@ export function getEmbeddingCount(db) {
333
333
  const row = db.prepare("SELECT COUNT(*) AS cnt FROM embeddings").get();
334
334
  return row.cnt;
335
335
  }
336
+ /**
337
+ * Sample up to `limit` already-embedded entries (id, search text, and the
338
+ * stored vector) for the embedding-fingerprint canary check: re-embedding
339
+ * these texts with the CURRENT config and comparing against `vector` is how
340
+ * a model-string rename is told apart from a genuine model/dimension change
341
+ * (#955), without trusting the config string alone.
342
+ *
343
+ * Ordered by `id` for a deterministic, cheap sample (no `ORDER BY RANDOM()`)
344
+ * — the canary only needs "some" already-verified vectors, not a
345
+ * statistically representative one. A corrupt stored BLOB (see
346
+ * `bufferToFloat32`) is skipped rather than failing the whole sample.
347
+ */
348
+ export function sampleEmbeddedEntriesForCanary(db, limit) {
349
+ const rows = db
350
+ .prepare(`
351
+ SELECT e.id, e.search_text AS searchText, em.embedding AS embedding
352
+ FROM entries e
353
+ JOIN embeddings em ON em.id = e.id
354
+ ORDER BY e.id
355
+ LIMIT ?
356
+ `)
357
+ .all(limit);
358
+ const samples = [];
359
+ for (const row of rows) {
360
+ const vector = bufferToFloat32(row.embedding, Math.floor(row.embedding.byteLength / 4));
361
+ if (vector)
362
+ samples.push({ id: row.id, searchText: row.searchText, vector });
363
+ }
364
+ return samples;
365
+ }
@@ -2,8 +2,8 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { randomUUID } from "node:crypto";
5
- import { NotFoundError, UsageError } from "../../core/errors.js";
6
- import { openStateDatabase, withImmediateTransaction } from "../../core/state-db.js";
5
+ import { NotFoundError, TransientError, UsageError } from "../../core/errors.js";
6
+ import { isSqliteContentionError, openStateDatabase, withImmediateTransaction } from "../../core/state-db.js";
7
7
  import { borrowScopedStateDb, withStateDbScope } from "../../core/state-db-scope.js";
8
8
  import { sleepSync } from "../../runtime.js";
9
9
  import { escapeLikePattern } from "../like-pattern.js";
@@ -25,22 +25,21 @@ function assertAttemptReservationLease(input, run) {
25
25
  /**
26
26
  * Whether `error` is one of the specific SQLite conditions a run-lease
27
27
  * statement can throw under real cross-process contention on the same row:
28
- * `SQLITE_BUSY`/`SQLITE_LOCKED` (both drivers), or the message text a
29
- * transient contention blip has been observed producing, "database is
30
- * locked", "disk I/O error", or "database disk image is malformed".
31
- * Matching on this set alone is never sufficient to call something lease
32
- * contention see {@link WorkflowRunsRepository.acquireEngineLease}, which
33
- * additionally requires a fresh read confirming a live lease before
34
- * substituting the lease-held message for the original error.
28
+ * the shared {@link isSqliteContentionError} classifier (SQLITE_BUSY/LOCKED,
29
+ * "database is locked", "database table is locked", the phantom-BEGIN
30
+ * marker) plus two corruption-shaped message texts a transient contention
31
+ * blip has been observed producing on this specific race, "disk I/O error"
32
+ * and "database disk image is malformed". Matching on this set alone is
33
+ * never sufficient to call something lease contention see
34
+ * {@link WorkflowRunsRepository.acquireEngineLease}, which additionally
35
+ * requires a fresh read confirming a live lease before substituting the
36
+ * lease-held message for the original error.
35
37
  */
36
38
  function isLeaseContentionSqliteError(error) {
37
- const code = error?.code;
38
- if (code === "SQLITE_BUSY" || code === "SQLITE_LOCKED")
39
+ if (isSqliteContentionError(error))
39
40
  return true;
40
41
  const message = error instanceof Error ? error.message : String(error);
41
- return (message.includes("database is locked") ||
42
- message.includes("disk I/O error") ||
43
- message.includes("database disk image is malformed"));
42
+ return message.includes("disk I/O error") || message.includes("database disk image is malformed");
44
43
  }
45
44
  const LEASE_RETRY_ATTEMPTS = 4;
46
45
  const LEASE_RETRY_BASE_DELAY_MS = 15;
@@ -110,6 +109,9 @@ export class WorkflowRunsRepository {
110
109
  * operator to `akm workflow abandon` a child a parent is actively driving.
111
110
  * For any database with no child rows the result is byte-identical, same
112
111
  * as the other three B-N10 sites below.
112
+ *
113
+ * Always scope-local: `scopeKey` is a real scope, never "every scope" — see
114
+ * {@link findActiveRunOutsideScope} for the cross-scope warning's query.
113
115
  */
114
116
  findActiveRunForScope(workflowRefs, scopeKey) {
115
117
  const refs = typeof workflowRefs === "string" ? [workflowRefs] : [...workflowRefs];
@@ -119,6 +121,26 @@ export class WorkflowRunsRepository {
119
121
  .prepare(`SELECT id, current_step_id FROM workflow_runs WHERE workflow_ref IN (${refs.map(() => "?").join(", ")}) AND scope_key = ? AND status = 'active' AND parent_run_id IS NULL ORDER BY updated_at DESC, created_at DESC LIMIT 1`)
120
122
  .get(...refs, scopeKey);
121
123
  }
124
+ /**
125
+ * The cross-scope start warning's query (#942): the most recently active
126
+ * run of these refs OUTSIDE `scopeKey` — i.e. `scope_key IS NULL OR
127
+ * scope_key != scopeKey`, not merely "the most recent active run of these
128
+ * refs anywhere". A same-scope active row must never win the `LIMIT 1` and
129
+ * mask a DIFFERENT scope's run: with `--new`/`--force`, the caller's own
130
+ * active run could otherwise sort first (most recently updated) and hide a
131
+ * third scope's run entirely, so `startWorkflowRun` silently warned about
132
+ * nothing while a genuinely stale run in another scope went unreported.
133
+ * `findActiveRunForScope` deliberately stays scope-local and is never used
134
+ * for this purpose.
135
+ */
136
+ findActiveRunOutsideScope(workflowRefs, scopeKey) {
137
+ const refs = typeof workflowRefs === "string" ? [workflowRefs] : [...workflowRefs];
138
+ if (refs.length === 0)
139
+ return undefined;
140
+ return (this.db
141
+ .prepare(`SELECT * FROM workflow_runs WHERE workflow_ref IN (${refs.map(() => "?").join(", ")}) AND (scope_key IS NULL OR scope_key != ?) AND status = 'active' AND parent_run_id IS NULL ORDER BY updated_at DESC, created_at DESC LIMIT 1`)
142
+ .get(...refs, scopeKey) ?? undefined);
143
+ }
122
144
  getRunById(runId) {
123
145
  return (this.db.prepare("SELECT * FROM workflow_runs WHERE id = ?").get(runId) ??
124
146
  undefined);
@@ -130,6 +152,9 @@ export class WorkflowRunsRepository {
130
152
  * to directly through this path — a parent-driven child would then have
131
153
  * TWO drivers. For any database with no child rows the result is
132
154
  * byte-identical.
155
+ *
156
+ * Always scope-local: `scopeKey` is a real scope, never "every scope" — see
157
+ * {@link findActiveRunOutsideScope} for the cross-scope warning's query.
133
158
  */
134
159
  getActiveRunRowForScope(workflowRefs, scopeKey) {
135
160
  const refs = typeof workflowRefs === "string" ? [workflowRefs] : [...workflowRefs];
@@ -159,8 +184,13 @@ export class WorkflowRunsRepository {
159
184
  listRuns(filter) {
160
185
  const filters = [];
161
186
  const params = [];
162
- filters.push("scope_key = ?");
163
- params.push(filter.scopeKey);
187
+ // `scopeKey: null` (#942) is "every scope" — the predicate is omitted
188
+ // rather than bound as SQL NULL (which would match nothing, since a real
189
+ // `scope_key` column value is never NULL for a fresh run).
190
+ if (filter.scopeKey !== null) {
191
+ filters.push("scope_key = ?");
192
+ params.push(filter.scopeKey);
193
+ }
164
194
  if (filter.workflowRef) {
165
195
  filters.push("workflow_ref = ?");
166
196
  params.push(filter.workflowRef);
@@ -282,10 +312,17 @@ export class WorkflowRunsRepository {
282
312
  this.immediateTransaction((db) => {
283
313
  input.revalidateSources();
284
314
  if (!input.force) {
315
+ // The uniqueness guard must never silently skip its scope predicate
316
+ // (#942) — every real caller (`startWorkflowRun`) stamps a concrete
317
+ // scope key, so a null one here means the caller is misusing this
318
+ // top-level guard, not that scoping should be waived.
319
+ if (input.run.scopeKey === null) {
320
+ throw new Error("publishWorkflowRunV4: run.scopeKey must not be null for the scope uniqueness guard.");
321
+ }
285
322
  const existing = this.findActiveRunForScope(input.workflowRefs, input.run.scopeKey);
286
323
  if (existing) {
287
324
  throw new UsageError(`Workflow ${input.run.workflowRef} already has an active run in this scope ` +
288
- `(id=${existing.id}, step=${existing.current_step_id ?? "—"}). ` +
325
+ `(id=${existing.id}, scope=${input.run.scopeKey}, step=${existing.current_step_id ?? "—"}). ` +
289
326
  `Use 'akm workflow run ${input.run.workflowRef}' to resume it or ` +
290
327
  `'akm workflow abandon ${existing.id}' to give up on it.`, "RESOURCE_ALREADY_EXISTS");
291
328
  }
@@ -443,7 +480,7 @@ export class WorkflowRunsRepository {
443
480
  throw error;
444
481
  const row = this.tryReadLeaseColumns(runId);
445
482
  if (row?.engine_lease_holder && row.engine_lease_until && row.engine_lease_until >= now) {
446
- throw new UsageError(`Workflow run ${runId} is already being driven by engine ${row.engine_lease_holder} ` +
483
+ throw new TransientError(`Workflow run ${runId} is already being driven by engine ${row.engine_lease_holder} ` +
447
484
  `(run lease expires ${row.engine_lease_until}). A second \`akm workflow run\` would race it — ` +
448
485
  `wait for that invocation to finish or for the lease to expire.`, "RUN_LEASE_HELD");
449
486
  }
@@ -7,7 +7,7 @@
7
7
  // its other lines untouched:
8
8
  //
9
9
  // # akm:task <id> BEGIN
10
- // [SCHED] /abs/akm task run <id> >> /home/.../tasks/logs/<id>.log 2>&1
10
+ // [SCHED] /abs/akm task run <id> > /home/.../tasks/logs/<id>.log 2>&1
11
11
  // # akm:task <id> END
12
12
  //
13
13
  // The backend reads/writes the user's crontab via `crontab -l` and
@@ -58,9 +58,9 @@ export function CRON_BACKEND(options = {}) {
58
58
  install(task, opts, expected) {
59
59
  if (expected)
60
60
  assertSchedulerExpectationIdentity(expected, task);
61
- // Create the log directory before writing the crontab line — cron
62
- // appends with `>>` and the surrounding shell will fail the entire
63
- // entry if the parent directory doesn't exist.
61
+ // Create the log directory before writing the crontab line — the
62
+ // redirect target's parent directory must exist or the surrounding
63
+ // shell will fail the entire entry.
64
64
  const cronLineParts = buildCronLineParts(task, [...(opts?.binding ?? akmArgv)], logDir, opts?.contextPath ?? defaultContextPath, opts?.target);
65
65
  const cronLine = cronLineParts.line;
66
66
  assertPortableCronLine(cronLine);
@@ -260,14 +260,17 @@ function buildCronLineParts(task, akmArgv, logDir, contextPath, _target) {
260
260
  const logPath = path.join(logDir, `${nativeId}.log`);
261
261
  const invocation = buildScheduledBindingInvocation(akmArgv, contextPath, task.invocation);
262
262
  const cmd = invocation.argv.map((part) => quoteForCron(part)).join(" ");
263
- const directLine = `${cronExpr} ${cmd} >> ${quoteForCron(logPath)} 2>&1`;
263
+ // #951: truncate (not append) so this bootstrap safety-net file always
264
+ // holds exactly the latest run's raw output rather than growing forever.
265
+ // akm's own per-run log (src/tasks/run/task-log.ts) already keeps history.
266
+ const directLine = `${cronExpr} ${cmd} > ${quoteForCron(logPath)} 2>&1`;
264
267
  if (Buffer.byteLength(directLine, "utf8") <= PORTABLE_CRON_LINE_LIMIT) {
265
268
  return { line: directLine };
266
269
  }
267
270
  const content = cronWrapperScriptContent(invocation.argv);
268
271
  const contentHash = createHash("sha256").update(content).digest("hex").slice(0, 16);
269
272
  const wrapperPath = path.join(logDir, `${CRON_WRAPPER_PREFIX}${nativeId}-${contentHash}.sh`);
270
- const line = `${cronExpr} sh ${quoteForCron(wrapperPath)} >> ${quoteForCron(logPath)} 2>&1`;
273
+ const line = `${cronExpr} sh ${quoteForCron(wrapperPath)} > ${quoteForCron(logPath)} 2>&1`;
271
274
  return { line, wrapper: { path: wrapperPath, content } };
272
275
  }
273
276
  const CRON_WRAPPER_PREFIX = ".akm-cron-wrapper-";
@@ -360,7 +363,11 @@ export function extractCronInvocation(body) {
360
363
  let commandStart = 5;
361
364
  while (commandStart < fields.length && /^[A-Za-z_][A-Za-z0-9_]*=/.test(fields[commandStart]))
362
365
  commandStart += 1;
363
- const redirectIndex = fields.indexOf(">>", commandStart);
366
+ // #951: newly-installed rows redirect with `>` (truncate); tolerate `>>`
367
+ // (append) too, since that's what every row written before this change —
368
+ // including ones a still-running older akm binary installs during a
369
+ // rolling upgrade — looks like on disk.
370
+ const redirectIndex = fields.findIndex((field, index) => index >= commandStart && (field === ">" || field === ">>"));
364
371
  if (redirectIndex === -1)
365
372
  return undefined;
366
373
  const tail = fields.slice(commandStart, redirectIndex);
@@ -145,6 +145,23 @@ export async function runNativeTask(input) {
145
145
  const logLines = [header];
146
146
  const dbLines = [{ line: header }];
147
147
  let exitCode = null;
148
+ // #956: the task runner's OWN process (`akm task run`, launched by cron /
149
+ // launchd / schtasks, or forwarded a SIGTERM by the published launcher)
150
+ // had no way to end its detached, own-process-group
151
+ // child (spawned by `runManagedSubprocess` for the group-kill guarantee
152
+ // above) — a timeout or SIGTERM to the direct child would reap the whole
153
+ // group, but nothing tied THIS process's own termination to that same
154
+ // ladder, so a signal to the runner left its child running as an orphan.
155
+ // Aborting `runManagedSubprocess` here runs its normal SIGTERM→SIGKILL
156
+ // kill ladder against the child's process group.
157
+ const abortController = new AbortController();
158
+ const forwardTerminationSignal = (signal) => {
159
+ abortController.abort(new Error(`task runner received ${signal}`));
160
+ };
161
+ const onSigterm = () => forwardTerminationSignal("SIGTERM");
162
+ const onSigint = () => forwardTerminationSignal("SIGINT");
163
+ process.once("SIGTERM", onSigterm);
164
+ process.once("SIGINT", onSigint);
148
165
  try {
149
166
  // Re-resolve the authored root/cwd immediately before spawn so a
150
167
  // symlink, ancestor, bundle-root, or directory/file swap cannot redirect
@@ -159,7 +176,9 @@ export async function runNativeTask(input) {
159
176
  }
160
177
  // Managed spawn (src/core/subprocess.ts): process-GROUP kill so a timeout
161
178
  // reaps the whole command tree (no orphans), and a SIGTERM→SIGKILL ladder
162
- // so a child that ignores SIGTERM can't wedge the run forever.
179
+ // so a child that ignores SIGTERM can't wedge the run forever. `signal`
180
+ // ties that same ladder to a termination signal received by this
181
+ // process itself (#956), not only to the timeout.
163
182
  const result = await runManagedSubprocess(cmd, {
164
183
  capture: true,
165
184
  cwd: task.cwd,
@@ -178,6 +197,7 @@ export async function runNativeTask(input) {
178
197
  // layer on top and break any resolved path containing a space.
179
198
  windowsVerbatimArguments: task.kind === "shell" && task.shell === "cmd",
180
199
  timeoutMs,
200
+ signal: abortController.signal,
181
201
  ...(input.spawnFn ? { spawnFn: input.spawnFn } : {}),
182
202
  ...(input.setTimeoutFn ? { setTimeoutFn: input.setTimeoutFn } : {}),
183
203
  ...(input.clearTimeoutFn ? { clearTimeoutFn: input.clearTimeoutFn } : {}),
@@ -221,6 +241,8 @@ export async function runNativeTask(input) {
221
241
  exitCode = 1;
222
242
  }
223
243
  finally {
244
+ process.off("SIGTERM", onSigterm);
245
+ process.off("SIGINT", onSigint);
224
246
  if (materialized)
225
247
  cleanupFrozenScript(materialized);
226
248
  }
@@ -187,6 +187,22 @@ export async function runWorkflowTask(input) {
187
187
  * here is a *compile* error rather than silently collapsing to "completed".
188
188
  * The previous silent `default: "completed"` is preserved only for the
189
189
  * `undefined` (no-detail) case, which is handled up front.
190
+ *
191
+ * #943: is `undefined` reachable from a timeout (i.e. can a wedged/killed run
192
+ * produce `status: "completed"`)? No. This function is only ever called as
193
+ * `mapWorkflowStatus(detail?.status)` behind `failure ? "failed" : ...` in
194
+ * {@link runWorkflowTask} — `failure` is `error ?? gateError ?? timeoutError`,
195
+ * and `detail` is left `undefined` only in the `catch` branch that also sets
196
+ * `error` (making `failure` truthy). So whenever this function runs, either
197
+ * `failure` already short-circuited the caller to `"failed"` without
198
+ * consulting it, or the run awaited successfully and `detail` (thus
199
+ * `detail.status`) is guaranteed set by `RunWorkflowResult["run"]`, a
200
+ * required field. `status === undefined` is therefore unreachable today; it
201
+ * stays mapped to `"completed"` defensively rather than thrown on, since a
202
+ * literal `undefined` can only mean "the runtime told us nothing went
203
+ * wrong" — never observed from `runWorkflowStepsImpl`, but if some future
204
+ * caller ever passes it, failing loudly on "no evidence of failure" would be
205
+ * a worse trade than the status quo.
190
206
  */
191
207
  function mapWorkflowStatus(status) {
192
208
  // No run detail → treat as completed (unchanged from the prior silent default).