akm-cli 0.9.16-alpha.1 → 0.9.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/CHANGELOG.md +56 -132
  2. package/dist/assets/hints/cli-hints-full.md +13 -6
  3. package/dist/assets/tasks/core/index-refresh.yml +1 -1
  4. package/dist/assets/tasks/improve/akm-improve-catchup.yml +3 -6
  5. package/dist/cli/retired-commands.js +0 -4
  6. package/dist/cli/unknown-flags.js +3 -36
  7. package/dist/commands/env/env-binding.js +4 -4
  8. package/dist/commands/env/env-cli.js +3 -3
  9. package/dist/commands/improve/collapse-detector.js +2 -2
  10. package/dist/commands/improve/consolidate.js +4 -6
  11. package/dist/commands/improve/improve-cli.js +20 -15
  12. package/dist/commands/improve/reflect.js +23 -2
  13. package/dist/commands/lint/base-linter.js +9 -0
  14. package/dist/commands/lint/env-key-rules.js +2 -2
  15. package/dist/commands/proposal/propose.js +15 -1
  16. package/dist/commands/proposal/repository.js +3 -12
  17. package/dist/commands/proposal/validators/proposal-quality-validators.js +40 -3
  18. package/dist/commands/proposal/validators/proposal-validators.js +5 -4
  19. package/dist/commands/read/curate.js +44 -34
  20. package/dist/commands/read/search.js +35 -54
  21. package/dist/commands/read/show.js +21 -2
  22. package/dist/commands/registry-cli.js +5 -5
  23. package/dist/commands/sources/add-cli.js +59 -16
  24. package/dist/commands/sources/bundle-cli.js +35 -11
  25. package/dist/commands/sources/bundle-config-ops.js +30 -0
  26. package/dist/commands/sources/dangerous-env-audit.js +4 -4
  27. package/dist/commands/sources/info.js +8 -8
  28. package/dist/commands/sources/installed-stashes.js +55 -61
  29. package/dist/commands/sources/source-add.js +39 -38
  30. package/dist/commands/sources/source-manage.js +34 -12
  31. package/dist/commands/sources/stash-cli.js +111 -119
  32. package/dist/commands/sources/stash-skeleton.js +6 -3
  33. package/dist/commands/tasks/explain.js +4 -1
  34. package/dist/commands/tasks/tasks-cli.js +31 -9
  35. package/dist/commands/tasks/tasks.js +239 -194
  36. package/dist/commands/tasks/validate.js +20 -32
  37. package/dist/core/activation-policy.js +4 -4
  38. package/dist/core/adapter/adapters/akm-adapter.js +8 -35
  39. package/dist/core/adapter/adapters/akm-metadata.js +1 -11
  40. package/dist/core/adapter/execution-source.js +10 -29
  41. package/dist/core/asset/asset-placement.js +0 -35
  42. package/dist/core/config/config-schema.js +64 -8
  43. package/dist/core/config/config-sources.js +96 -2
  44. package/dist/core/config/config.js +190 -24
  45. package/dist/core/config/legacy-source-shape-shim.js +9 -0
  46. package/dist/core/config/schema/embedding.js +30 -7
  47. package/dist/core/config/schema/execution.js +23 -0
  48. package/dist/core/config/schema/experimental.js +1 -1
  49. package/dist/core/config/schema/scheduler.js +20 -0
  50. package/dist/core/config/schema/search.js +10 -12
  51. package/dist/core/config/schema/sources-bundles.js +32 -1
  52. package/dist/core/content-safety.js +52 -0
  53. package/dist/core/errors.js +2 -5
  54. package/dist/core/maintenance-barrier.js +11 -13
  55. package/dist/core/paths.js +11 -0
  56. package/dist/core/run-lock.js +2 -5
  57. package/dist/core/state/migrations.js +1 -26
  58. package/dist/core/state-db.js +27 -63
  59. package/dist/core/type-presentation.js +1 -1
  60. package/dist/core/write-source.js +13 -8
  61. package/dist/indexer/bundle-identity-guard.js +45 -8
  62. package/dist/indexer/ensure-index.js +0 -5
  63. package/dist/indexer/index-db-contention.js +56 -0
  64. package/dist/indexer/index-rebuild-lock.js +73 -0
  65. package/dist/indexer/index-written-assets.js +171 -133
  66. package/dist/indexer/indexer.js +1621 -458
  67. package/dist/indexer/lookup/adapter-concept-owner.js +5 -19
  68. package/dist/indexer/materialize-embeddings.js +785 -0
  69. package/dist/indexer/passes/dir-staleness.js +161 -0
  70. package/dist/indexer/passes/metadata.js +1 -18
  71. package/dist/indexer/scan/drain-dir.js +70 -27
  72. package/dist/indexer/search/db-search.js +89 -373
  73. package/dist/indexer/search/ranking-contributors.js +16 -21
  74. package/dist/indexer/search/ranking.js +57 -135
  75. package/dist/indexer/search/search-source.js +29 -11
  76. package/dist/integrations/agent/execution-lowering.js +3 -2
  77. package/dist/integrations/agent/execution-preparation.js +32 -1
  78. package/dist/integrations/agent/prompts.js +1 -1
  79. package/dist/integrations/agent/request-lowering.js +3 -2
  80. package/dist/llm/client.js +3 -11
  81. package/dist/llm/embedder.js +3 -10
  82. package/dist/llm/embedders/remote.js +104 -133
  83. package/dist/llm/feature-gate.js +2 -4
  84. package/dist/llm/rerank-client.js +3 -3
  85. package/dist/output/html-render.js +2 -1
  86. package/dist/output/shapes/passthrough.js +2 -1
  87. package/dist/output/stdout.js +24 -0
  88. package/dist/output/text/command-format.js +13 -19
  89. package/dist/output/text/helpers.js +1 -1
  90. package/dist/output/text/index.js +2 -5
  91. package/dist/output/text.js +4 -3
  92. package/dist/registry/resolve.js +37 -10
  93. package/dist/scripts/akm-migrate-node.js +15197 -11351
  94. package/dist/scripts/akm-migrate.js +15514 -11668
  95. package/dist/setup/semantic-assets.js +2 -2
  96. package/dist/setup/setup.js +3 -3
  97. package/dist/setup/steps/connection.js +2 -3
  98. package/dist/setup/steps/tasks.js +29 -36
  99. package/dist/sources/providers/git-install.js +17 -11
  100. package/dist/sources/providers/git-provider.js +12 -5
  101. package/dist/sources/providers/git-stash.js +38 -16
  102. package/dist/sources/snapshot-fetchers/website-ingest.js +3 -3
  103. package/dist/storage/repositories/embedding-salvage-repository.js +184 -0
  104. package/dist/storage/repositories/index-connection.js +3 -1
  105. package/dist/storage/repositories/index-entries-repository.js +68 -77
  106. package/dist/storage/repositories/index-entry-schema.js +25 -16
  107. package/dist/storage/repositories/index-fts-repository.js +263 -29
  108. package/dist/storage/repositories/index-meta-repository.js +29 -0
  109. package/dist/storage/repositories/index-schema.js +122 -115
  110. package/dist/storage/repositories/index-utility-repository.js +1 -1
  111. package/dist/storage/repositories/index-vec-repository.js +435 -22
  112. package/dist/tasks/activation-config.js +90 -0
  113. package/dist/tasks/backends/cron.js +9 -0
  114. package/dist/tasks/backends/launchd.js +1 -0
  115. package/dist/tasks/backends/schtasks.js +2 -0
  116. package/dist/tasks/embedded.js +4 -5
  117. package/dist/tasks/scheduler-binding.js +2 -2
  118. package/dist/tasks/scheduler-sync-preview.js +8 -1
  119. package/dist/tasks/scheduler-sync.js +19 -10
  120. package/dist/tasks/source/parse-task-source.js +10 -113
  121. package/dist/tasks/source/project-v4.js +2 -2
  122. package/dist/tasks/source/task-source-v4.js +4 -12
  123. package/dist/tasks/source/task-to-v3.js +4 -12
  124. package/dist/tasks/source/task-to-v4.js +40 -7
  125. package/docs/migration/README.md +1 -0
  126. package/docs/migration/release-notes/0.9.15.md +36 -34
  127. package/docs/migration/release-notes/0.9.16.md +60 -98
  128. package/docs/migration/release-notes/README.md +0 -5
  129. package/docs/migration/v0.9.1-to-v0.9.2.md +6 -9
  130. package/docs/reference/cli.md +124 -122
  131. package/docs/reference/configuration.md +137 -133
  132. package/docs/reference/data-and-telemetry.md +1 -2
  133. package/docs/reference/tasks.md +34 -29
  134. package/package.json +1 -1
  135. package/schemas/akm-config.json +170 -6
  136. package/schemas/akm-task.json +1 -2
  137. package/dist/commands/sources/index-status.js +0 -99
  138. package/dist/core/hash.js +0 -18
  139. package/dist/indexer/drain.js +0 -306
  140. package/dist/indexer/embedding-identity.js +0 -20
  141. package/dist/indexer/enrich.js +0 -260
  142. package/dist/indexer/reconcile.js +0 -890
  143. package/dist/indexer/scan/parse-file.js +0 -66
  144. package/dist/indexer/units/unit.js +0 -159
  145. package/dist/llm/embedders/provider-limits.js +0 -288
  146. package/dist/storage/repositories/files-repository.js +0 -181
  147. package/dist/storage/repositories/units-repository.js +0 -510
@@ -2,37 +2,24 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  /**
5
- * `index.db` sqlite-vec extension load/availability probe.
6
- *
7
- * This module used to also own the legacy per-entry vector store (a BLOB
8
- * `embeddings` table, an `entries_vec` vec0 mirror, and a JS-cosine fallback
9
- * over the BLOB rows) — all of it dead code once the write-time indexing
10
- * pipeline moved to the content-addressed `units`/`units_vec` store
11
- * (`units-repository.ts`, docs/plans/index-fragment-vectors.md) and nothing
12
- * wrote to `embeddings`/`entries_vec` any more. The tables themselves, and
13
- * every function whose only purpose was reading or writing them, were
14
- * deleted in the index-redesign's final cleanup (B5h) — see
15
- * `docs/architecture/internals/storage-locations.md` for what replaced them.
16
- * What remains here — loading the extension and reporting whether it loaded
17
- * — is still shared by both the units store and every other vec0 consumer.
5
+ * `index.db` vector + embedding repository.
6
+ *
7
+ * Owns the sqlite-vec extension load/availability probe, the BLOB `embeddings`
8
+ * table, the `entries_vec` virtual table, and the JS-cosine fallback path.
18
9
  */
19
10
  import { createRequire } from "node:module";
11
+ import { bestEffort } from "../../core/best-effort.js";
12
+ import { warn } from "../../core/warn.js";
13
+ import { cosineSimilarity } from "../../llm/embedders/types.js";
14
+ import { getMeta, setMeta } from "./index-meta-repository.js";
15
+ import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
20
16
  // ── sqlite-vec extension ────────────────────────────────────────────────────
21
17
  const vecStatus = new WeakMap();
22
- let forceVecUnavailableForTests = false;
23
- /** TEST-ONLY. Force `loadVecExtension` to record unavailable without touching the package; pass false to restore. */
24
- export function _setVecUnavailableForTests(unavailable) {
25
- forceVecUnavailableForTests = unavailable;
26
- }
27
18
  /**
28
19
  * Attempt to load the sqlite-vec extension into `db`, recording availability.
29
20
  * Exported so the connection lifecycle can arm it at open time.
30
21
  */
31
22
  export function loadVecExtension(db) {
32
- if (forceVecUnavailableForTests) {
33
- vecStatus.set(db, false);
34
- return;
35
- }
36
23
  try {
37
24
  const esmRequire = createRequire(import.meta.url);
38
25
  const sqliteVec = esmRequire("sqlite-vec");
@@ -50,3 +37,429 @@ export function loadVecExtension(db) {
50
37
  export function isVecAvailable(db) {
51
38
  return vecStatus.get(db) ?? false;
52
39
  }
40
+ /**
41
+ * Meta key persisting whether the sqlite-vec fast-path table (`entries_vec`) is
42
+ * fully populated and trustworthy for this index. Set to "0" by the embedding
43
+ * phase when one or more vec inserts FAILED (e.g. a vec0 dimension mismatch)
44
+ * while their BLOB rows still wrote — so semantic search reads the complete
45
+ * BLOB table via the JS-cosine fallback instead of a partial/mismatched vec
46
+ * table. Absent (legacy indexes) and "1" both mean the fast path is trusted.
47
+ */
48
+ const VEC_FAST_PATH_READY_META = "vecFastPathReady";
49
+ /** Persist whether the sqlite-vec fast path is trustworthy (see the meta doc). */
50
+ export function setVecFastPathReady(db, ready) {
51
+ setMeta(db, VEC_FAST_PATH_READY_META, ready ? "1" : "0");
52
+ }
53
+ /**
54
+ * True unless the embedding phase recorded a vec insert failure. Reflects the
55
+ * ACTUAL insert outcomes recorded at index time — not an inference from how many
56
+ * BLOB rows exist — so a degraded vec table routes search to the JS fallback
57
+ * rather than silently returning partial fast-path results.
58
+ */
59
+ export function isVecFastPathReady(db) {
60
+ if (getMeta(db, VEC_FAST_PATH_READY_META) === "0")
61
+ return false;
62
+ // The meta flag alone is not sufficient. An index built while sqlite-vec was
63
+ // unavailable wrote only BLOB rows, and because "unavailable" outcomes were
64
+ // not counted as failures the flag was still set to "1" against a table that
65
+ // is empty or absent. If sqlite-vec later becomes loadable — the user installs
66
+ // it, or the same index is opened under the other runtime — the fast path
67
+ // would then be trusted and return zero neighbours while the BLOB table holds
68
+ // every embedding. Indexes written by earlier versions still carry that stale
69
+ // flag, so the read path has to verify the table really exists.
70
+ return hasVecTable(db);
71
+ }
72
+ /**
73
+ * Verify that the vec fast-path table mirrors the complete durable BLOB set.
74
+ *
75
+ * A targeted embedding write preserves the prior readiness decision because
76
+ * its subset cannot prove an older degraded generation is healed. Global
77
+ * materialization uses this aggregate check before promoting the persisted
78
+ * flag; search itself still reads the cheap flag and does not repeat the check
79
+ * per query.
80
+ */
81
+ export function isVecFastPathComplete(db) {
82
+ if (!isVecAvailable(db) || !hasVecTable(db))
83
+ return false;
84
+ try {
85
+ const missingVecRows = db
86
+ .prepare(`
87
+ SELECT id FROM embeddings
88
+ EXCEPT
89
+ SELECT id FROM entries_vec
90
+ LIMIT 1
91
+ `)
92
+ .all();
93
+ if (missingVecRows.length > 0)
94
+ return false;
95
+ const orphanVecRows = db
96
+ .prepare(`
97
+ SELECT id FROM entries_vec
98
+ EXCEPT
99
+ SELECT id FROM embeddings
100
+ LIMIT 1
101
+ `)
102
+ .all();
103
+ return orphanVecRows.length === 0;
104
+ }
105
+ catch {
106
+ return false;
107
+ }
108
+ }
109
+ /**
110
+ * Reconcile sqlite-vec's derived mirror from the durable BLOB embeddings.
111
+ *
112
+ * This never calls an embedding provider and never mutates the BLOB table.
113
+ * The readiness flag is lowered before the first mutation and is promoted only
114
+ * after a bidirectional aggregate check proves both ID sets match exactly.
115
+ */
116
+ export function repairVecFastPath(db, embeddingDim) {
117
+ let repaired = 0;
118
+ let removedOrphans = 0;
119
+ let rejected = 0;
120
+ setVecFastPathReady(db, false);
121
+ if (!isVecAvailable(db) || !hasVecTable(db)) {
122
+ return { available: false, repaired, removedOrphans, rejected, complete: false };
123
+ }
124
+ if (!Number.isInteger(embeddingDim) || embeddingDim <= 0) {
125
+ return {
126
+ available: true,
127
+ repaired,
128
+ removedOrphans,
129
+ rejected,
130
+ complete: false,
131
+ error: `Invalid embedding dimension ${embeddingDim}.`,
132
+ };
133
+ }
134
+ try {
135
+ while (true) {
136
+ const orphanIds = db
137
+ .prepare(`
138
+ SELECT id FROM entries_vec
139
+ EXCEPT
140
+ SELECT id FROM embeddings
141
+ ORDER BY id
142
+ LIMIT ?
143
+ `)
144
+ .all(SQLITE_CHUNK_SIZE);
145
+ if (orphanIds.length === 0)
146
+ break;
147
+ db.transaction(() => {
148
+ const remove = db.prepare("DELETE FROM entries_vec WHERE id = ?");
149
+ for (const { id } of orphanIds) {
150
+ remove.run(id);
151
+ removedOrphans++;
152
+ }
153
+ })();
154
+ }
155
+ let afterId = -1;
156
+ while (true) {
157
+ const missingIds = db
158
+ .prepare(`
159
+ SELECT id FROM (
160
+ SELECT id FROM embeddings
161
+ EXCEPT
162
+ SELECT id FROM entries_vec
163
+ ) AS missing
164
+ WHERE id > ?
165
+ ORDER BY id
166
+ LIMIT ?
167
+ `)
168
+ .all(afterId, SQLITE_CHUNK_SIZE);
169
+ if (missingIds.length === 0)
170
+ break;
171
+ afterId = missingIds[missingIds.length - 1].id;
172
+ const placeholders = missingIds.map(() => "?").join(",");
173
+ const rows = db
174
+ .prepare(`SELECT id, embedding FROM embeddings WHERE id IN (${placeholders}) ORDER BY id`)
175
+ .all(...missingIds.map(({ id }) => id));
176
+ db.transaction(() => {
177
+ const insert = db.prepare("INSERT INTO entries_vec (id, embedding) VALUES (?, ?)");
178
+ for (const row of rows) {
179
+ if (row.embedding.byteLength !== embeddingDim * 4) {
180
+ rejected++;
181
+ continue;
182
+ }
183
+ try {
184
+ insert.run(row.id, Buffer.from(row.embedding));
185
+ repaired++;
186
+ }
187
+ catch {
188
+ rejected++;
189
+ }
190
+ }
191
+ })();
192
+ }
193
+ const complete = rejected === 0 && isVecFastPathComplete(db);
194
+ setVecFastPathReady(db, complete);
195
+ return { available: true, repaired, removedOrphans, rejected, complete };
196
+ }
197
+ catch (error) {
198
+ setVecFastPathReady(db, false);
199
+ return {
200
+ available: true,
201
+ repaired,
202
+ removedOrphans,
203
+ rejected,
204
+ complete: false,
205
+ error: error instanceof Error ? error.message : String(error),
206
+ };
207
+ }
208
+ }
209
+ const vecTablePresent = new WeakMap();
210
+ /**
211
+ * Whether `entries_vec` exists on this connection, memoized per handle.
212
+ *
213
+ * openExistingDatabase loads the vec extension but deliberately does not run
214
+ * ensureSchema, so the table is not created on read paths — its absence is a
215
+ * normal state, not an error.
216
+ */
217
+ function hasVecTable(db) {
218
+ // Only a POSITIVE result is memoized. The table cannot vanish from a live
219
+ // connection, but it CAN appear — ensureSchema creates it partway through an
220
+ // index run — so caching "absent" would pin a stale answer for the rest of
221
+ // the handle's life.
222
+ if (vecTablePresent.get(db) === true)
223
+ return true;
224
+ let present = false;
225
+ try {
226
+ present =
227
+ db.prepare("SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name = 'entries_vec'").get() !==
228
+ undefined;
229
+ }
230
+ catch {
231
+ present = false;
232
+ }
233
+ if (present)
234
+ vecTablePresent.set(db, true);
235
+ return present;
236
+ }
237
+ /** Remove both vector representations for an entry whose embedding input changed. */
238
+ export function deleteEntryVectors(db, id) {
239
+ db.prepare("DELETE FROM embeddings WHERE id = ?").run(id);
240
+ if (isVecAvailable(db))
241
+ db.prepare("DELETE FROM entries_vec WHERE id = ?").run(id);
242
+ }
243
+ const VEC_DOCS_URL = "https://github.com/itlackey/akm/blob/main/docs/reference/configuration.md#sqlite-vec-extension";
244
+ const VEC_FALLBACK_THRESHOLD = 10_000;
245
+ // Per-database warning state: tracks which databases have already emitted the
246
+ // vec-missing warning so we don't spam on every openDatabase() call.
247
+ const vecInitWarnedDbs = new WeakSet();
248
+ /**
249
+ * Warn if sqlite-vec is unavailable and embedding count exceeds threshold.
250
+ * Called from openDatabase (once at init) and from indexer (each run).
251
+ */
252
+ export function warnIfVecMissing(db, { once } = { once: false }) {
253
+ if (isVecAvailable(db))
254
+ return;
255
+ if (once && vecInitWarnedDbs.has(db))
256
+ return;
257
+ bestEffort(() => {
258
+ const row = db.prepare("SELECT COUNT(*) AS cnt FROM embeddings").get();
259
+ const count = row?.cnt ?? 0;
260
+ if (count >= VEC_FALLBACK_THRESHOLD) {
261
+ warn("Semantic search is using JS fallback for %d entries. Install sqlite-vec for faster performance.\n See: %s", count, VEC_DOCS_URL);
262
+ if (once)
263
+ vecInitWarnedDbs.add(db);
264
+ }
265
+ }, "embeddings table may not exist yet during init");
266
+ }
267
+ /**
268
+ * Purge stored embeddings (BLOB rows in `embeddings`, plus the `entries_vec`
269
+ * virtual table) and mark the index as embedding-free. The single place that
270
+ * invalidates embeddings — used on a dimension change, a model/provider change,
271
+ * and a full rebuild.
272
+ *
273
+ * No backup: embeddings are a derived cache, fully regenerable from the markdown
274
+ * by the next `akm index`. (Recovery model decided 2026-06-25.)
275
+ *
276
+ * `dropVecTable: true` DROPs `entries_vec` — used on a DIMENSION change, where
277
+ * the vec0 table must be recreated at the new width by the caller. The default
278
+ * clears its rows in place (same dimension, stale vectors).
279
+ */
280
+ export function purgeEmbeddings(db, opts) {
281
+ bestEffort(() => db.exec("DELETE FROM embeddings"), "purge embeddings");
282
+ if (isVecAvailable(db)) {
283
+ bestEffort(() => db.exec(opts?.dropVecTable ? "DROP TABLE IF EXISTS entries_vec" : "DELETE FROM entries_vec"), "purge entries_vec");
284
+ }
285
+ setMeta(db, "hasEmbeddings", "0");
286
+ }
287
+ export function upsertEmbedding(db, entryId, embedding) {
288
+ // Pre-flight FK guard: when an entry is deleted between when its id is queued
289
+ // for embedding and when this INSERT runs (e.g. consolidation deletes during
290
+ // a concurrent improve cycle), the INSERT throws "FOREIGN KEY constraint failed"
291
+ // and rolls back the entire batch transaction in the caller, losing every
292
+ // embedding for that run. A cheap SELECT here turns the race into a clean skip.
293
+ const exists = db.prepare("SELECT 1 FROM entries WHERE id = ?").get(entryId);
294
+ if (!exists)
295
+ return { stored: false, vec: "unavailable" };
296
+ const buf = float32Buffer(embedding);
297
+ // Always write to BLOB table (works without sqlite-vec; the JS-cosine fallback
298
+ // reads it, so semantic search survives a vec fast-path failure).
299
+ db.prepare("INSERT OR REPLACE INTO embeddings (id, embedding) VALUES (?, ?)").run(entryId, buf);
300
+ if (!isVecAvailable(db))
301
+ return { stored: true, vec: "unavailable" };
302
+ // Fast path: mirror into the sqlite-vec table. Wrapped in a transaction so a
303
+ // crash between DELETE and INSERT does not leave the entry missing. A THROW
304
+ // here — previously swallowed silently by bestEffort — is now surfaced to the
305
+ // caller so the embedding phase can count it, warn, and route search to the
306
+ // (complete) BLOB table rather than a partial/mismatched vec table.
307
+ try {
308
+ db.transaction(() => {
309
+ db.prepare("DELETE FROM entries_vec WHERE id = ?").run(entryId);
310
+ db.prepare("INSERT INTO entries_vec (id, embedding) VALUES (?, ?)").run(entryId, buf);
311
+ })();
312
+ return { stored: true, vec: "ok" };
313
+ }
314
+ catch {
315
+ return { stored: true, vec: "failed" };
316
+ }
317
+ }
318
+ export function searchVec(db, queryEmbedding, k) {
319
+ // Fast path: sqlite-vec, but ONLY when the extension is loaded AND the
320
+ // embedding phase did not record a vec insert failure. A degraded fast-path
321
+ // table (partial or dimension-mismatched) would return wrong or missing
322
+ // neighbours, so we honestly fall back to the JS-cosine scan over the
323
+ // complete BLOB table instead.
324
+ if (isVecAvailable(db) && isVecFastPathReady(db)) {
325
+ const buf = float32Buffer(queryEmbedding);
326
+ try {
327
+ return db
328
+ .prepare("SELECT id, distance FROM entries_vec WHERE embedding MATCH ? AND k = ?")
329
+ .all(buf, k);
330
+ }
331
+ catch (err) {
332
+ // A dimension mismatch (e.g. the embedding provider/model changed since
333
+ // the fast-path table was built) is a real, expected reason this query
334
+ // specifically cannot use the vec table — the complete BLOB table below
335
+ // is unaffected, so fall back to it rather than either silently
336
+ // returning [] (masking a genuinely corrupt index) or failing the whole
337
+ // search over one degraded index.
338
+ warn("[db] searchVec (sqlite-vec path) failed, falling back to JS-cosine scan:", err instanceof Error ? err.message : String(err));
339
+ return searchBlobVec(db, queryEmbedding, k);
340
+ }
341
+ }
342
+ // Fallback: JS-based cosine similarity over BLOB table
343
+ return searchBlobVec(db, queryEmbedding, k);
344
+ }
345
+ /**
346
+ * Return the k nearest neighbours of an already-indexed entry using its
347
+ * persisted embedding — no re-embedding, no network. Decodes the stored BLOB by
348
+ * byte length (dim = bytes / 4) and reuses searchVec (sqlite-vec fast path or
349
+ * JS-cosine fallback). Returns [] when the entry has no stored embedding or the
350
+ * BLOB is corrupt. The query entry itself is typically returned with distance
351
+ * ~0 — callers should filter it out by id.
352
+ */
353
+ export function getNeighborsByEntryId(db, id, k) {
354
+ const row = db.prepare("SELECT embedding FROM embeddings WHERE id = ?").get(id);
355
+ if (!row)
356
+ return [];
357
+ const queryEmbedding = bufferToFloat32(row.embedding, Math.floor(row.embedding.byteLength / 4));
358
+ if (!queryEmbedding)
359
+ return [];
360
+ return searchVec(db, queryEmbedding, k);
361
+ }
362
+ function float32Buffer(vec) {
363
+ const f32 = new Float32Array(vec);
364
+ return Buffer.from(f32.buffer);
365
+ }
366
+ /**
367
+ * Decode a stored embedding BLOB into a Float32 array of `expectedDim`
368
+ * dimensions. Returns `null` (and emits a warning) when the byte length does
369
+ * not exactly match `expectedDim * 4`, including the legacy partial-trailing
370
+ * float case the previous truncating-divide silently swallowed.
371
+ *
372
+ * BUG-M2: the previous `buf.byteLength / 4` divide would truncate any
373
+ * trailing partial float and a misaligned `byteOffset` would throw — both
374
+ * surfaced as opaque generic errors caught upstream.
375
+ */
376
+ function bufferToFloat32(buf, expectedDim) {
377
+ if (buf.byteLength !== expectedDim * 4) {
378
+ warn("[db] bufferToFloat32: skipping embedding row — expected %d bytes (%d dim x 4), got %d", expectedDim * 4, expectedDim, buf.byteLength);
379
+ return null;
380
+ }
381
+ // Copy into a fresh ArrayBuffer to sidestep any byteOffset alignment
382
+ // requirements imposed by Float32Array's typed-array view contract.
383
+ const aligned = new ArrayBuffer(buf.byteLength);
384
+ new Uint8Array(aligned).set(buf);
385
+ const f32 = new Float32Array(aligned);
386
+ return Array.from(f32);
387
+ }
388
+ function searchBlobVec(db, queryEmbedding, k) {
389
+ const rows = db.prepare("SELECT id, embedding FROM embeddings").all();
390
+ if (rows.length === 0)
391
+ return [];
392
+ const expectedDim = queryEmbedding.length;
393
+ const scored = [];
394
+ for (const row of rows) {
395
+ const embedding = bufferToFloat32(row.embedding, expectedDim);
396
+ if (embedding === null)
397
+ continue;
398
+ const similarity = cosineSimilarity(queryEmbedding, embedding);
399
+ scored.push({ id: row.id, similarity });
400
+ }
401
+ scored.sort((a, b) => b.similarity - a.similarity);
402
+ // Convert cosine similarity to L2 distance for compatibility with sqlite-vec interface
403
+ // For normalized vectors: L2² = 2(1 - cos_sim)
404
+ return scored.slice(0, k).map(({ id, similarity }) => ({
405
+ id,
406
+ distance: Math.sqrt(2 * Math.max(0, 1 - similarity)),
407
+ }));
408
+ }
409
+ /**
410
+ * Return all entries that do not yet have an embedding row.
411
+ * Used by the embedding phase to determine which entries need vectors generated.
412
+ */
413
+ export function getAllEntriesForEmbedding(db, entryIds) {
414
+ const select = `
415
+ SELECT e.id, e.search_text AS searchText, e.item_ref AS itemRef, e.file_path AS filePath FROM entries e
416
+ `;
417
+ const missing = "NOT EXISTS (SELECT 1 FROM embeddings b WHERE b.id = e.id)";
418
+ if (entryIds === undefined) {
419
+ return db.prepare(`${select} WHERE ${missing} ORDER BY e.id`).all();
420
+ }
421
+ const targets = [...new Set(entryIds)].sort((left, right) => left - right);
422
+ const rows = [];
423
+ for (let offset = 0; offset < targets.length; offset += SQLITE_CHUNK_SIZE) {
424
+ const chunk = targets.slice(offset, offset + SQLITE_CHUNK_SIZE);
425
+ if (chunk.length === 0)
426
+ continue;
427
+ const placeholders = chunk.map(() => "?").join(",");
428
+ rows.push(...db.prepare(`${select} WHERE e.id IN (${placeholders}) AND ${missing} ORDER BY e.id`).all(...chunk));
429
+ }
430
+ return rows;
431
+ }
432
+ export function getEmbeddingCount(db) {
433
+ const row = db.prepare("SELECT COUNT(*) AS cnt FROM embeddings").get();
434
+ return row.cnt;
435
+ }
436
+ /**
437
+ * Sample up to `limit` already-embedded entries (id, search text, and the
438
+ * stored vector) for the embedding-fingerprint canary check: re-embedding
439
+ * these texts with the CURRENT config and comparing against `vector` is how
440
+ * a model-string rename is told apart from a genuine model/dimension change
441
+ * (#955), without trusting the config string alone.
442
+ *
443
+ * Ordered by `id` for a deterministic, cheap sample (no `ORDER BY RANDOM()`)
444
+ * — the canary only needs "some" already-verified vectors, not a
445
+ * statistically representative one. A corrupt stored BLOB (see
446
+ * `bufferToFloat32`) is skipped rather than failing the whole sample.
447
+ */
448
+ export function sampleEmbeddedEntriesForCanary(db, limit) {
449
+ const rows = db
450
+ .prepare(`
451
+ SELECT e.id, e.search_text AS searchText, em.embedding AS embedding
452
+ FROM entries e
453
+ JOIN embeddings em ON em.id = e.id
454
+ ORDER BY e.id
455
+ LIMIT ?
456
+ `)
457
+ .all(limit);
458
+ const samples = [];
459
+ for (const row of rows) {
460
+ const vector = bufferToFloat32(row.embedding, Math.floor(row.embedding.byteLength / 4));
461
+ if (vector)
462
+ samples.push({ id: row.id, searchText: row.searchText, vector });
463
+ }
464
+ return samples;
465
+ }
@@ -0,0 +1,90 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { bundleRefToString, parseBundleRef } from "../core/asset/asset-ref.js";
5
+ import { deriveBundleId } from "../core/bundle-id.js";
6
+ import { resolveStashDir } from "../core/common.js";
7
+ import { mutateConfig } from "../core/config/config.js";
8
+ import { bundleSourceId, filesystemBundleSourceId, isBundleEnabled } from "../core/config/config-sources.js";
9
+ import { UsageError } from "../core/errors.js";
10
+ export function canonicalSchedulerActivationRef(ref) {
11
+ const parsed = parseBundleRef(ref);
12
+ if (!parsed.bundle || parsed.fragment !== undefined || bundleRefToString(parsed) !== ref) {
13
+ throw new UsageError(`Scheduler activation requires a canonical fully-qualified ref, got ${JSON.stringify(ref)}.`, "INVALID_FLAG_VALUE");
14
+ }
15
+ return ref;
16
+ }
17
+ export function schedulerActivations(config) {
18
+ return Object.freeze((config.scheduler?.enabled ?? []).map((activation) => Object.freeze({
19
+ kind: activation.kind,
20
+ ref: canonicalSchedulerActivationRef(activation.ref),
21
+ sourceId: activation.sourceId,
22
+ })));
23
+ }
24
+ export function isSchedulerRefEnabled(config, kind, ref) {
25
+ const canonicalRef = canonicalSchedulerActivationRef(ref);
26
+ const bundle = parseBundleRef(canonicalRef).bundle;
27
+ const currentSourceId = schedulerActivationSourceId(config, bundle);
28
+ if (!currentSourceId)
29
+ return false;
30
+ return schedulerActivations(config).some((activation) => activation.kind === kind && activation.ref === canonicalRef && activation.sourceId === currentSourceId);
31
+ }
32
+ /** Grants that still name the same active source installation they approved. */
33
+ export function activeSchedulerActivations(config) {
34
+ return schedulerActivations(config).filter((activation) => {
35
+ const bundle = parseBundleRef(activation.ref).bundle;
36
+ return activation.sourceId === schedulerActivationSourceId(config, bundle);
37
+ });
38
+ }
39
+ /** Pure lifecycle helper used when a bundle is removed or replaced. */
40
+ export function revokeSchedulerActivationsForBundle(config, bundleId) {
41
+ const enabled = schedulerActivations(config).filter((activation) => parseBundleRef(activation.ref).bundle !== bundleId);
42
+ if (enabled.length === (config.scheduler?.enabled ?? []).length)
43
+ return config;
44
+ return { ...config, scheduler: { ...config.scheduler, enabled } };
45
+ }
46
+ export function setSchedulerRefEnabled(kind, ref, enabled) {
47
+ let canonicalRef = ref;
48
+ const result = mutateConfig((current) => {
49
+ canonicalRef = canonicalSchedulerActivationRef(ref);
50
+ const existing = schedulerActivations(current);
51
+ const bundle = parseBundleRef(canonicalRef).bundle;
52
+ const sourceId = enabled ? schedulerActivationSourceId(current, bundle) : undefined;
53
+ if (enabled && !sourceId) {
54
+ throw new UsageError(`Cannot enable scheduled execution from inactive or unconfigured bundle ${JSON.stringify(bundle)}.`, "INVALID_FLAG_VALUE");
55
+ }
56
+ const present = existing.some((activation) => activation.kind === kind && activation.ref === canonicalRef && (!enabled || activation.sourceId === sourceId));
57
+ if (present === enabled)
58
+ return current;
59
+ const next = enabled
60
+ ? [
61
+ ...existing.filter((activation) => activation.kind !== kind || activation.ref !== canonicalRef),
62
+ { kind, ref: canonicalRef, sourceId: sourceId },
63
+ ]
64
+ : existing.filter((activation) => activation.kind !== kind || activation.ref !== canonicalRef);
65
+ next.sort((left, right) => left.ref.localeCompare(right.ref) || left.kind.localeCompare(right.kind));
66
+ return {
67
+ ...current,
68
+ scheduler: {
69
+ ...current.scheduler,
70
+ enabled: next,
71
+ },
72
+ };
73
+ });
74
+ return { config: result.config, changed: result.written, ref: canonicalRef };
75
+ }
76
+ /** Current active source identity for a configured or environment-only bundle. */
77
+ export function schedulerActivationSourceId(config, bundleId) {
78
+ if (isBundleEnabled(config, bundleId))
79
+ return bundleSourceId(config, bundleId);
80
+ if (config.bundles?.[bundleId] !== undefined || !process.env.AKM_BUNDLE_DIR?.trim())
81
+ return undefined;
82
+ try {
83
+ const root = resolveStashDir();
84
+ const implicitId = deriveBundleId(undefined, root, new Set(Object.keys(config.bundles ?? {})));
85
+ return implicitId === bundleId ? filesystemBundleSourceId(root) : undefined;
86
+ }
87
+ catch {
88
+ return undefined;
89
+ }
90
+ }
@@ -41,6 +41,7 @@ import { nodeFs, throwIfNotOk } from "./exec-utils.js";
41
41
  const BEGIN = (id) => `# akm:task ${assertCronValue(id)} BEGIN`;
42
42
  const END = (id) => `# akm:task ${assertCronValue(id)} END`;
43
43
  const DISABLED_PREFIX = "# akm:disabled ";
44
+ const SOURCE_ENABLED_PREFIX = "# akm:source-enabled ";
44
45
  const BLOCK_RE = /^# akm:task ([\w.@:_-]+) BEGIN$/;
45
46
  const BLOCK_END_RE = /^# akm:task ([\w.@:_-]+) END$/;
46
47
  export const PORTABLE_CRON_LINE_LIMIT = 1000;
@@ -216,6 +217,7 @@ function inspectCronState(crontab, fallbackContextPath) {
216
217
  continue;
217
218
  const ref = {
218
219
  id: schedulerLogicalBindingId(id, parsed.invocation),
220
+ enabled: !cronIsDisabled(body),
219
221
  signature: fingerprint,
220
222
  ...(parsed.target !== undefined ? { target: parsed.target } : {}),
221
223
  binding: parsed.binding,
@@ -438,9 +440,16 @@ function normalizeSignature(body) {
438
440
  return body
439
441
  .split(/\r?\n/)
440
442
  .map((l) => l.trim())
443
+ .filter((l) => !stripDisabledPrefix(l).startsWith(SOURCE_ENABLED_PREFIX))
441
444
  .filter((l) => l.length > 0)
442
445
  .join("\n");
443
446
  }
447
+ function stripDisabledPrefix(line) {
448
+ return line.startsWith(DISABLED_PREFIX) ? line.slice(DISABLED_PREFIX.length) : line;
449
+ }
450
+ function cronIsDisabled(body) {
451
+ return body.trimStart().startsWith(DISABLED_PREFIX);
452
+ }
444
453
  export function upsertBlock(existing, id, block) {
445
454
  const trimmed = existing.replace(/\s+$/g, "");
446
455
  const removed = removeBlock(trimmed, id);
@@ -557,6 +557,7 @@ function inspectStableLaunchdNamespace(seedIds, context) {
557
557
  continue;
558
558
  installed.push(withInstalledInvocation({
559
559
  id: schedulerLogicalBindingId(entry.nativeId, parsed.invocation),
560
+ enabled: entry.enabled && entry.loaded,
560
561
  ...(entry.loaded ? { signature: entry.artifact.fingerprint } : {}),
561
562
  ...(parsed.target !== undefined ? { target: parsed.target } : {}),
562
563
  binding: parsed.binding,
@@ -257,8 +257,10 @@ function inspectSchtasksState(exec, folder, taskName) {
257
257
  const parsed = extractSchtasksInvocation(query.stdout);
258
258
  if (!parsed)
259
259
  continue;
260
+ const enabled = taskXmlEnabled(query.stdout);
260
261
  const ref = {
261
262
  id: schedulerLogicalBindingId(nativeId, parsed.invocation),
263
+ ...(enabled !== undefined ? { enabled } : {}),
262
264
  ...(artifact.fingerprint !== undefined ? { signature: artifact.fingerprint } : {}),
263
265
  ...(parsed.target !== undefined ? { target: parsed.target } : {}),
264
266
  binding: parsed.binding,
@@ -25,6 +25,7 @@ import { getDirname } from "../runtime.js";
25
25
  import { parseTaskSource } from "./source/parse-task-source.js";
26
26
  /** Directory holding the bundled task template categories. */
27
27
  const TASKS_ASSETS_DIR = path.join(getDirname(import.meta.url), "../assets/tasks");
28
+ const DEFAULT_DISABLED_TASKS = new Set(["improve/akm-improve-catchup"]);
28
29
  /**
29
30
  * Enumerate the embedded task templates from every category subdirectory of
30
31
  * the bundled assets directory. Sorted by category then id for deterministic
@@ -72,10 +73,8 @@ export function listEmbeddedTasks() {
72
73
  catch {
73
74
  continue;
74
75
  }
75
- // Shipped templates are task source v4 (spec docs/plans/specs/p4-deletions-closeout.md
76
- // §3.2.6, row B-24): a template's `enabled` is per schedule-binding, so
77
- // the single-cron display shape here reads the FIRST schedule entry —
78
- // every shipped template authors exactly one.
76
+ // Setup defaults are trusted application metadata, not bundle-authored
77
+ // task data. Task source can describe a schedule but cannot activate it.
79
78
  const task = parsed.v4;
80
79
  const [firstSchedule] = task.schedule;
81
80
  if (task.target.kind !== "run" || !firstSchedule)
@@ -86,7 +85,7 @@ export function listEmbeddedTasks() {
86
85
  command: task.target.run,
87
86
  schedule: firstSchedule.cron,
88
87
  description: task.description ?? "",
89
- enabled: firstSchedule.enabled,
88
+ enabled: !DEFAULT_DISABLED_TASKS.has(`${category}/${id}`),
90
89
  yaml,
91
90
  });
92
91
  }
@@ -41,7 +41,7 @@ export function compileTaskSchedulerBindings(input) {
41
41
  cron: schedule.cron,
42
42
  source: schedule.source,
43
43
  ordinal: schedule.ordinal,
44
- enabled: schedule.enabled ?? input.enabled,
44
+ enabled: true,
45
45
  invocation,
46
46
  });
47
47
  }));
@@ -286,7 +286,7 @@ export function canonicalSchedulerIdentity(logicalSource, ordinal, invocation) {
286
286
  }
287
287
  const canonicalInvocation = ["task", "run", taskId, "--bundle", parsed.bundle, "--scheduled"];
288
288
  if (!sameInvocation(invocation, canonicalInvocation)) {
289
- throw new UsageError("Task scheduler expectation invocation does not match its qualified source.", "INVALID_FLAG_VALUE");
289
+ throw new UsageError(`Task scheduler expectation invocation does not match its qualified source: expected ${JSON.stringify(canonicalInvocation)}, got ${JSON.stringify(invocation)}.`, "INVALID_FLAG_VALUE");
290
290
  }
291
291
  if (parsed.conceptId !== taskId && parsed.conceptId !== `tasks/${taskId}`) {
292
292
  throw new UsageError("Task scheduler expectation id does not match its qualified source.", "INVALID_FLAG_VALUE");