akm-cli 0.9.0-beta.45 → 0.9.0-beta.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/assets/wiki/ingest-workflow-template.md +17 -10
  2. package/dist/cli/shared.js +28 -0
  3. package/dist/cli.js +1 -2
  4. package/dist/commands/env/env-cli.js +16 -24
  5. package/dist/commands/env/secret-cli.js +12 -20
  6. package/dist/commands/graph/graph-cli.js +5 -13
  7. package/dist/commands/graph/graph.js +3 -3
  8. package/dist/commands/improve/consolidate/chunking.js +141 -0
  9. package/dist/commands/improve/consolidate/eligibility.js +64 -0
  10. package/dist/commands/improve/consolidate/merge.js +145 -0
  11. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  12. package/dist/commands/improve/consolidate/types.js +4 -0
  13. package/dist/commands/improve/consolidate.js +20 -571
  14. package/dist/commands/improve/distill.js +5 -9
  15. package/dist/commands/improve/eligibility.js +434 -0
  16. package/dist/commands/improve/extract-cli.js +9 -1
  17. package/dist/commands/improve/extract.js +5 -19
  18. package/dist/commands/improve/improve-auto-accept.js +4 -8
  19. package/dist/commands/improve/improve-cli.js +35 -60
  20. package/dist/commands/improve/improve-result-file.js +5 -23
  21. package/dist/commands/improve/improve-session.js +58 -0
  22. package/dist/commands/improve/improve.js +107 -3606
  23. package/dist/commands/improve/locks.js +154 -0
  24. package/dist/commands/improve/loop-stages.js +1079 -0
  25. package/dist/commands/improve/preparation.js +1963 -0
  26. package/dist/commands/improve/recombine.js +6 -12
  27. package/dist/commands/improve/reflect.js +29 -34
  28. package/dist/commands/proposal/drain.js +25 -48
  29. package/dist/commands/proposal/proposal-cli.js +21 -31
  30. package/dist/commands/proposal/validators/proposals.js +3 -7
  31. package/dist/commands/read/curate.js +70 -14
  32. package/dist/commands/read/knowledge.js +2 -2
  33. package/dist/commands/sources/self-update.js +2 -2
  34. package/dist/commands/sources/stash-cli.js +9 -37
  35. package/dist/commands/tasks/tasks-cli.js +19 -27
  36. package/dist/commands/wiki-cli.js +21 -35
  37. package/dist/core/config/config.js +18 -2
  38. package/dist/core/events.js +3 -7
  39. package/dist/core/logs-db.js +6 -63
  40. package/dist/core/state/migrations.js +714 -0
  41. package/dist/core/state-db.js +28 -779
  42. package/dist/indexer/db/db.js +82 -216
  43. package/dist/indexer/indexer.js +11 -112
  44. package/dist/indexer/passes/dir-staleness.js +114 -0
  45. package/dist/indexer/search/search-source.js +10 -24
  46. package/dist/indexer/search/semantic-status.js +4 -0
  47. package/dist/integrations/agent/runner-dispatch.js +59 -0
  48. package/dist/llm/client.js +22 -11
  49. package/dist/llm/embedder.js +15 -0
  50. package/dist/llm/embedders/deterministic.js +66 -0
  51. package/dist/llm/graph-extract.js +28 -39
  52. package/dist/llm/memory-infer.js +34 -22
  53. package/dist/llm/metadata-enhance.js +35 -30
  54. package/dist/llm/structured-call.js +49 -0
  55. package/dist/output/shapes/passthrough.js +0 -1
  56. package/dist/registry/providers/skills-sh.js +21 -147
  57. package/dist/registry/providers/static-index.js +15 -157
  58. package/dist/registry/resolve.js +22 -9
  59. package/dist/scripts/migrate-storage.js +892 -1186
  60. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +214 -179
  61. package/dist/setup/setup.js +26 -5
  62. package/dist/sources/providers/filesystem.js +0 -1
  63. package/dist/sources/providers/git-install.js +206 -0
  64. package/dist/sources/providers/git-provider.js +234 -0
  65. package/dist/sources/providers/git-stash.js +248 -0
  66. package/dist/sources/providers/git.js +10 -671
  67. package/dist/sources/providers/npm.js +2 -6
  68. package/dist/sources/providers/sync-from-ref.js +9 -1
  69. package/dist/sources/providers/website.js +2 -3
  70. package/dist/sources/website-ingest.js +51 -9
  71. package/dist/sources/wiki-fetchers/registry.js +53 -0
  72. package/dist/sources/wiki-fetchers/youtube.js +185 -0
  73. package/dist/storage/database.js +45 -10
  74. package/dist/storage/managed-db.js +82 -0
  75. package/dist/storage/repositories/registry-cache.js +92 -0
  76. package/dist/tasks/runner.js +5 -13
  77. package/dist/workflows/runtime/runs.js +1 -117
  78. package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
  79. package/package.json +5 -5
  80. package/dist/commands/db-cli.js +0 -23
  81. package/dist/indexer/db/db-backup.js +0 -376
@@ -7,35 +7,33 @@ import path from "node:path";
7
7
  import { parseAssetRef } from "../../core/asset/asset-ref.js";
8
8
  import { bestEffort } from "../../core/best-effort.js";
9
9
  import { getDbPath } from "../../core/paths.js";
10
- import { REGISTRY_INDEX_CACHE_DDL } from "../../core/state-db.js";
11
10
  import { warn } from "../../core/warn.js";
12
11
  import { cosineSimilarity } from "../../llm/embedders/types.js";
13
12
  import { sha256Hex } from "../../runtime.js";
14
- import { openDatabase as openSqlite } from "../../storage/database.js";
13
+ import { openDatabase } from "../../storage/database.js";
15
14
  import { applyStandardPragmas } from "../../storage/sqlite-pragmas.js";
16
15
  import { buildSearchFields } from "../search/search-fields.js";
17
16
  import { ensureUsageEventsSchema } from "../usage/usage-events.js";
18
- import { backupDataDir, EMBEDDING_DIM_CHANGE_REASON } from "./db-backup.js";
19
17
  // ── Constants ───────────────────────────────────────────────────────────────
20
- // NOTE: do NOT bump DB_VERSION for graph-schema changes. A DB_VERSION mismatch
21
- // triggers handleVersionUpgrade()'s NUCLEAR drop of the ENTIRE index entries,
22
- // embeddings, FTS, and the llm_enrichment_cache forcing every user to re-embed
23
- // their whole corpus on upgrade. The graph tables are derived and cheap to
24
- // rebuild, so graph re-keying is migrated in a TARGETED, graph-only path
25
- // (migrateGraphFilesSchema) that leaves entries + embeddings untouched.
18
+ // NOTE: schema changes are additive. DB_VERSION is a forensic stamp only — it
19
+ // no longer gates any destructive path (the old nuclear drop-and-rebuild was
20
+ // removed; index.db's idempotent CREATE IF NOT EXISTS schema converges any
21
+ // older/partial DB forward without dropping data). Graph re-keying uses a
22
+ // TARGETED, graph-only migration (migrateGraphFilesSchema) the model for any
23
+ // incompatible change: migrate in place, never wipe the whole index.
26
24
  export const DB_VERSION = 17;
27
25
  export const EMBEDDING_DIM = 384;
28
26
  // #624-P1: graph_files re-keyed to (stash_root, file_path, body_hash). Bumped 3→4
29
27
  // as a marker; the actual migration is the targeted drop in migrateGraphFilesSchema.
30
28
  export const GRAPH_SCHEMA_VERSION = 4;
31
29
  // ── Database lifecycle ──────────────────────────────────────────────────────
32
- export function openDatabase(dbPath, options) {
30
+ export function openIndexDatabase(dbPath, options) {
33
31
  const resolvedPath = dbPath ?? getDbPath();
34
32
  const dir = path.dirname(resolvedPath);
35
33
  if (!fs.existsSync(dir)) {
36
34
  fs.mkdirSync(dir, { recursive: true });
37
35
  }
38
- const db = openSqlite(resolvedPath);
36
+ const db = openDatabase(resolvedPath);
39
37
  applyStandardPragmas(db, { dataDir: dir });
40
38
  // Try to load sqlite-vec extension
41
39
  loadVecExtension(db);
@@ -45,7 +43,7 @@ export function openDatabase(dbPath, options) {
45
43
  // both are absent do we fall through to the no-clobber path, which keeps
46
44
  // ensureSchema from touching `index_meta.embeddingDim` at all.
47
45
  const resolvedDim = options?.embeddingDim ?? resolveConfiguredEmbeddingDim();
48
- ensureSchema(db, resolvedDim, { dataDir: dir });
46
+ ensureSchema(db, resolvedDim);
49
47
  // Warn once at init if using JS fallback with many entries
50
48
  warnIfVecMissing(db, { once: true });
51
49
  return db;
@@ -75,7 +73,7 @@ function resolveConfiguredEmbeddingDim() {
75
73
  export function openExistingDatabase(dbPath) {
76
74
  const resolvedPath = dbPath ?? getDbPath();
77
75
  const dir = path.dirname(resolvedPath);
78
- const db = openSqlite(resolvedPath);
76
+ const db = openDatabase(resolvedPath);
79
77
  applyStandardPragmas(db, { dataDir: dir });
80
78
  // Existing-DB callers must not mutate schema or embedding metadata on open,
81
79
  // but some paths still need write access to usage_events and other tables.
@@ -129,7 +127,44 @@ export function warnIfVecMissing(db, { once } = { once: false }) {
129
127
  }
130
128
  }, "embeddings table may not exist yet during init");
131
129
  }
132
- function ensureSchema(db, embeddingDim, options) {
130
+ // ── Schema ──────────────────────────────────────────────────────────────────
131
+ /**
132
+ * DDL for the `registry_index_cache` table. This table lives in index.db
133
+ * (managed by this module), so its DDL belongs here next to the `ensureSchema`
134
+ * that applies it — not in state-db.ts.
135
+ *
136
+ * Created with CREATE TABLE IF NOT EXISTS so it is safe to call inside
137
+ * `ensureSchema()`. Caches the result of resolving and fetching remote registry
138
+ * stash indexes so `akm search` does not hit the network on every invocation.
139
+ *
140
+ * Indexed (query) columns:
141
+ * registry_url TEXT PK — canonical URL of the registry; cache key.
142
+ * fetched_at TEXT — ISO-8601; used to detect stale entries (TTL).
143
+ * etag TEXT — HTTP ETag for conditional GET (If-None-Match).
144
+ * last_modified TEXT — HTTP Last-Modified for conditional GET.
145
+ *
146
+ * Non-indexed payload:
147
+ * index_json TEXT — JSON blob of the fetched registry index document.
148
+ *
149
+ * ADD COLUMN extension points (future migrations):
150
+ * ALTER TABLE registry_index_cache ADD COLUMN schema_version INTEGER DEFAULT 1;
151
+ * ALTER TABLE registry_index_cache ADD COLUMN kit_count INTEGER DEFAULT NULL;
152
+ * ALTER TABLE registry_index_cache ADD COLUMN error_message TEXT DEFAULT NULL;
153
+ */
154
+ const REGISTRY_INDEX_CACHE_DDL = `
155
+ CREATE TABLE IF NOT EXISTS registry_index_cache (
156
+ registry_url TEXT PRIMARY KEY,
157
+ fetched_at TEXT NOT NULL,
158
+ etag TEXT,
159
+ last_modified TEXT,
160
+ index_json TEXT NOT NULL DEFAULT '{}'
161
+ );
162
+
163
+ CREATE INDEX IF NOT EXISTS idx_registry_cache_fetched
164
+ ON registry_index_cache(fetched_at);
165
+ `;
166
+ /** A row backed up out of the legacy `usage_events` table during a version upgrade. */
167
+ function ensureSchema(db, embeddingDim) {
133
168
  // Create meta table first so we can check version
134
169
  db.exec(`
135
170
  CREATE TABLE IF NOT EXISTS index_meta (
@@ -137,46 +172,16 @@ function ensureSchema(db, embeddingDim, options) {
137
172
  value TEXT NOT NULL
138
173
  );
139
174
  `);
140
- // MVP DB-backup hook (0.8.x): when the stored DB version differs from the
141
- // running binary's DB_VERSION, snapshot the data directory BEFORE
142
- // `handleVersionUpgrade()` drops tables. This is best-effort
143
- // `backupDataDir` returns null on opt-out, missing data dir, low free
144
- // space, or copy errors, and we proceed with the upgrade in all cases.
145
- // The proper migration framework lands in 0.9.0; until then this lets
146
- // operators recover with `scripts/migrations/restore-data-dir.sh`.
147
- if (options?.dataDir) {
148
- const storedVersionRaw = getMeta(db, "version");
149
- const storedVersion = storedVersionRaw !== undefined && storedVersionRaw !== "" ? Number.parseInt(storedVersionRaw, 10) : null;
150
- const willUpgrade = storedVersionRaw !== undefined && storedVersionRaw !== "" && storedVersionRaw !== String(DB_VERSION);
151
- if (willUpgrade) {
152
- try {
153
- // Pass env explicitly so tests can override AKM_DB_BACKUP / AKM_DB_BACKUP_RETAIN
154
- // without mutating process.env. Production callers default to process.env.
155
- const result = backupDataDir({
156
- dataDir: options.dataDir,
157
- sourceVersion: storedVersion !== null && !Number.isNaN(storedVersion) ? storedVersion : null,
158
- targetVersion: DB_VERSION,
159
- env: process.env,
160
- });
161
- if (result) {
162
- warn("[akm] data directory backed up to %s before v%s→v%d upgrade", result.path, storedVersionRaw, DB_VERSION);
163
- }
164
- }
165
- catch (err) {
166
- // Defensive — backupDataDir already swallows most errors, but if it
167
- // throws for an unexpected reason we must still proceed with the
168
- // upgrade so the user isn't locked out of their binary.
169
- warn("[akm] pre-upgrade data dir backup raised an unexpected error — %s; upgrade will proceed without a snapshot", err instanceof Error ? err.message : String(err));
170
- }
171
- }
172
- }
173
- // Check stored version — if it differs from DB_VERSION, drop and recreate all tables.
174
- // Usage events are preserved across version upgrades so that utility score
175
- // history is not silently lost. The backup is captured here and threaded
176
- // explicitly to `restoreUsageEventsBackup` below — the previous version
177
- // attached `__usageBackup` to the Database instance via a typeless property
178
- // injection, which was a source of fragile coupling.
179
- const usageBackup = handleVersionUpgrade(db);
175
+ // index.db is a fully regenerable derived cache, so its schema is built
176
+ // idempotently below: every table is CREATE IF NOT EXISTS and column
177
+ // additions go through guarded ALTERs (ensureDerivedFromColumn) and targeted
178
+ // migrations (migrateGraphFilesSchema / migrateGraphDataFromLegacy). Opening a
179
+ // database with an older or partial schema converges it forward WITHOUT ever
180
+ // dropping data there is intentionally no "nuclear drop the whole index on a
181
+ // DB_VERSION mismatch" path (a destructive design the regenerable index never
182
+ // needed, and whose pre-drop data-dir backup it required). A genuinely
183
+ // incompatible change is handled by an additive/targeted migration; the few
184
+ // derived tables that ever must be rebuilt are regenerated by `akm index`.
180
185
  db.exec(`
181
186
  CREATE TABLE IF NOT EXISTS entries (
182
187
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -446,17 +451,10 @@ function ensureSchema(db, embeddingDim, options) {
446
451
  if (dimExplicit) {
447
452
  const storedDim = getMeta(db, "embeddingDim");
448
453
  if (storedDim && storedDim !== String(embeddingDim)) {
449
- // Re-embedding the whole stash is expensive (LLM API calls + cache
450
- // misses), so snapshot the data dir before we drop the vec table and
451
- // wipe `embeddings`. This is the SAME hook the version-upgrade path
452
- // uses earlier in this function, just gated on embedding-dim mismatch
453
- // and tagged so operators can tell the two backup kinds apart.
454
- backupBeforeEmbeddingDimChange(options?.dataDir, storedDim, String(embeddingDim));
455
- bestEffort(() => db.exec("DROP TABLE IF EXISTS entries_vec"), "drop entries_vec on dim change");
456
- // Delete stale BLOB embeddings so they don't produce silently wrong
457
- // similarity scores against the new-dimension vec table.
458
- bestEffort(() => db.exec("DELETE FROM embeddings"), "delete stale embeddings on dim change");
459
- setMeta(db, "hasEmbeddings", "0");
454
+ // Stored vectors are incompatible with the new dimension. Drop the vec
455
+ // table so the block below recreates it at the new width; the BLOB rows
456
+ // go too. Regenerable from markdown re-embedded by the next index.
457
+ purgeEmbeddings(db, { dropVecTable: true });
460
458
  }
461
459
  }
462
460
  const vecExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_vec'").get();
@@ -483,9 +481,8 @@ function ensureSchema(db, embeddingDim, options) {
483
481
  if (dimExplicit) {
484
482
  const storedDim = getMeta(db, "embeddingDim");
485
483
  if (storedDim && storedDim !== String(embeddingDim)) {
486
- backupBeforeEmbeddingDimChange(options?.dataDir, storedDim, String(embeddingDim));
487
- bestEffort(() => db.exec("DELETE FROM embeddings"), "delete embeddings on explicit dim change");
488
- setMeta(db, "hasEmbeddings", "0");
484
+ // JS-fallback path: no vec table, just clear the stale BLOB vectors.
485
+ purgeEmbeddings(db);
489
486
  }
490
487
  setMeta(db, "embeddingDim", String(embeddingDim));
491
488
  }
@@ -493,158 +490,28 @@ function ensureSchema(db, embeddingDim, options) {
493
490
  // Usage telemetry table
494
491
  ensureUsageEventsSchema(db);
495
492
  // Registry index cache table — caches remote registry index documents so
496
- // `akm search` does not hit the network on every invocation. The DDL is
497
- // defined in state-db.ts and shared here to avoid duplication.
493
+ // `akm search` does not hit the network on every invocation.
498
494
  db.exec(REGISTRY_INDEX_CACHE_DDL);
499
- // Restore usage_events backed up by the version-upgrade path above.
500
- restoreUsageEventsBackup(db, usageBackup);
501
- }
502
- /**
503
- * Detect a stored DB version that differs from {@link DB_VERSION}, drop the
504
- * old schema, and return a backup of the previous `usage_events` rows so the
505
- * rest of `ensureSchema()` can restore them once the new table exists.
506
- *
507
- * Returns an empty array when no upgrade is needed or when the previous
508
- * `usage_events` table is unreadable.
509
- */
510
- function handleVersionUpgrade(db) {
511
- const storedVersion = getMeta(db, "version");
512
- // BUG-L4: distinguish "missing" (undefined) from "present but empty" — both
513
- // were previously coerced through `!storedVersion` and treated as "no
514
- // upgrade needed", which caused fresh databases (with no version row) to
515
- // skip the upgrade path correctly, but also caused the upgrade path to be
516
- // taken when a corrupted/empty version string was persisted. The current
517
- // tables get dropped only when the stored version exists AND differs from
518
- // DB_VERSION; missing or empty version means a fresh DB and no upgrade.
519
- if (storedVersion === undefined || storedVersion === "" || storedVersion === String(DB_VERSION))
520
- return [];
521
- let usageBackup = [];
522
- bestEffort(() => {
523
- usageBackup = db.prepare("SELECT * FROM usage_events").all();
524
- }, "usage_events table may not exist in older versions");
525
- db.exec("DROP TABLE IF EXISTS utility_scores");
526
- db.exec("DROP TABLE IF EXISTS utility_scores_scoped");
527
- db.exec("DROP INDEX IF EXISTS idx_utility_scores_scoped_entry_id");
528
- db.exec("DROP TABLE IF EXISTS usage_events");
529
- db.exec("DROP TABLE IF EXISTS embeddings");
530
- db.exec("DROP TABLE IF EXISTS entries_vec");
531
- db.exec("DROP TABLE IF EXISTS entries_fts");
532
- db.exec("DROP TABLE IF EXISTS index_dir_state");
533
- db.exec("DROP TABLE IF EXISTS llm_enrichment_cache");
534
- db.exec("DROP INDEX IF EXISTS idx_llm_cache_updated");
535
- db.exec("DROP TABLE IF EXISTS graph_extraction_queue");
536
- db.exec("DROP TABLE IF EXISTS graph_file_relations");
537
- db.exec("DROP TABLE IF EXISTS graph_file_entities");
538
- db.exec("DROP TABLE IF EXISTS graph_files");
539
- db.exec("DROP TABLE IF EXISTS graph_meta");
540
- db.exec("DROP TABLE IF EXISTS graph_relations");
541
- db.exec("DROP TABLE IF EXISTS graph_entities");
542
- db.exec("DROP TABLE IF EXISTS graph_nodes");
543
- db.exec("DROP TABLE IF EXISTS graph_stashes");
544
- db.exec("DROP INDEX IF EXISTS idx_entries_dir");
545
- db.exec("DROP INDEX IF EXISTS idx_entries_type");
546
- db.exec("DROP TABLE IF EXISTS entries");
547
- db.exec("DELETE FROM index_meta");
548
- warn("[akm] Index rebuilt due to version upgrade. Run 'akm index' to repopulate.");
549
- return usageBackup;
550
495
  }
551
496
  /**
552
- * Snapshot the data directory before the embedding-dimension drop path wipes
553
- * `embeddings` and recreates `entries_vec`. Re-embedding a real-world stash
554
- * is expensive (LLM calls + cache misses), so we capture the pre-drop state
555
- * here using the same MVP backup helper the version-upgrade hook uses
556
- * earlier in {@link ensureSchema}.
497
+ * Purge stored embeddings (BLOB rows in `embeddings`, plus the `entries_vec`
498
+ * virtual table) and mark the index as embedding-free. The single place that
499
+ * invalidates embeddings used on a dimension change, a model/provider change,
500
+ * and a full rebuild.
557
501
  *
558
- * The backup is tagged with the `embedding-dim-change` reason so it lands in
559
- * `<dataDir>/backups/<timestamp>-embedding-dim-change/` instead of the
560
- * version-upgrade-flavored `<timestamp>-pre-v<N>/` directory. Restoration
561
- * works identically via `scripts/migrations/restore-data-dir.sh`.
502
+ * No backup: embeddings are a derived cache, fully regenerable from the markdown
503
+ * by the next `akm index`. (Recovery model decided 2026-06-25.)
562
504
  *
563
- * Failures are non-fatalthey downgrade to a warning and the destructive
564
- * ops run anyway, matching the version-upgrade hook's behavior so a broken
565
- * backup cannot brick a binary that bumped the configured dim. Likewise,
566
- * `AKM_DB_BACKUP=0` opts out via the same path.
505
+ * `dropVecTable: true` DROPs `entries_vec` used on a DIMENSION change, where
506
+ * the vec0 table must be recreated at the new width by the caller. The default
507
+ * clears its rows in place (same dimension, stale vectors).
567
508
  */
568
- function backupBeforeEmbeddingDimChange(dataDir, fromDim, toDim) {
569
- if (!dataDir)
570
- return;
571
- try {
572
- const result = backupDataDir({
573
- dataDir,
574
- // The DB version isn't changing here — pass the current DB_VERSION for
575
- // both source and target so the metadata sidecar still records the
576
- // running binary's version for forensic context.
577
- sourceVersion: DB_VERSION,
578
- targetVersion: DB_VERSION,
579
- reason: EMBEDDING_DIM_CHANGE_REASON,
580
- env: process.env,
581
- });
582
- if (result) {
583
- warn("[akm] embedding dimension changed %s→%s; data directory backed up to %s; embeddings will be regenerated", fromDim, toDim, result.path);
584
- }
585
- }
586
- catch (err) {
587
- // Defensive — backupDataDir already swallows most errors, but if it
588
- // throws for an unexpected reason we must still proceed with the drop
589
- // so the user isn't locked out of their binary on a changed dim.
590
- warn("[akm] pre-embedding-dim-change data dir backup raised an unexpected error — %s; embeddings will be regenerated without a snapshot", err instanceof Error ? err.message : String(err));
591
- }
592
- }
593
- /**
594
- * Re-insert backed-up `usage_events` rows into the freshly-created table.
595
- *
596
- * Wrapped in an outer try/catch because schema changes across versions may
597
- * make the backup incompatible with the new table definition; in that case
598
- * the backup is discarded silently rather than blocking startup.
599
- */
600
- function restoreUsageEventsBackup(db, backup) {
601
- if (backup.length === 0)
602
- return;
603
- try {
604
- // BUG-H4: introspect the *target* table's columns rather than relying on
605
- // `row[0]`'s keys. The backup may carry columns the new schema dropped,
606
- // and the new schema may have NOT-NULL columns without DEFAULT that the
607
- // old backup never carried. Project the backup onto the intersection so
608
- // we don't silently lose every row to per-row INSERT errors, and warn
609
- // once if any backup column was dropped from the new schema.
610
- const targetCols = db.prepare("PRAGMA table_info(usage_events)").all().map((c) => c.name);
611
- if (targetCols.length === 0) {
612
- warn("[db] restoreUsageEventsBackup: usage_events table missing — discarding %d backup row(s)", backup.length);
613
- return;
614
- }
615
- const targetSet = new Set(targetCols);
616
- const backupCols = Object.keys(backup[0] ?? {});
617
- const projectedCols = backupCols.filter((c) => targetSet.has(c));
618
- const droppedCols = backupCols.filter((c) => !targetSet.has(c));
619
- if (projectedCols.length === 0) {
620
- warn("[db] restoreUsageEventsBackup: no overlapping columns between backup and current schema — discarding %d row(s); dropped: %s", backup.length, droppedCols.join(", ") || "(none)");
621
- return;
622
- }
623
- if (droppedCols.length > 0) {
624
- warn("[db] restoreUsageEventsBackup: dropping columns no longer in usage_events schema: %s", droppedCols.join(", "));
625
- }
626
- let restored = 0;
627
- let failed = 0;
628
- db.transaction(() => {
629
- const placeholders = projectedCols.map(() => "?").join(", ");
630
- const insert = db.prepare(`INSERT INTO usage_events (${projectedCols.join(", ")}) VALUES (${placeholders})`);
631
- for (const row of backup) {
632
- try {
633
- insert.run(...projectedCols.map((c) => row[c]));
634
- restored++;
635
- }
636
- catch {
637
- failed++;
638
- }
639
- }
640
- })();
641
- if (failed > 0) {
642
- warn("[db] restoreUsageEventsBackup: restored %d row(s); skipped %d incompatible row(s)", restored, failed);
643
- }
644
- }
645
- catch (err) {
646
- warn("[db] restoreUsageEventsBackup: discarded %d backup row(s) — %s", backup.length, err instanceof Error ? err.message : String(err));
509
+ export function purgeEmbeddings(db, opts) {
510
+ bestEffort(() => db.exec("DELETE FROM embeddings"), "purge embeddings");
511
+ if (isVecAvailable(db)) {
512
+ bestEffort(() => db.exec(opts?.dropVecTable ? "DROP TABLE IF EXISTS entries_vec" : "DELETE FROM entries_vec"), "purge entries_vec");
647
513
  }
514
+ setMeta(db, "hasEmbeddings", "0");
648
515
  }
649
516
  // ── Meta helpers ────────────────────────────────────────────────────────────
650
517
  export function getMeta(db, key) {
@@ -748,9 +615,8 @@ function getUpsertStmts(db) {
748
615
  *
749
616
  * Ensures the `entries.derived_from` column + index exist on the open
750
617
  * connection. Called from `ensureSchema()` after the entries CREATE so that
751
- * legacy databases (created against a pre-v17 binary but reopened without
752
- * triggering `handleVersionUpgrade()`) still gain the new column without
753
- * data loss. Idempotent: a `PRAGMA table_info` lookup gates the ALTER.
618
+ * legacy databases (created against a pre-v17 binary) still gain the new column
619
+ * without data loss. Idempotent: a `PRAGMA table_info` lookup gates the ALTER.
754
620
  */
755
621
  function ensureDerivedFromColumn(db) {
756
622
  bestEffort(() => {
@@ -10,9 +10,10 @@ import { getDbPath } from "../core/paths.js";
10
10
  import { isVerbose, warn, warnVerbose } from "../core/warn.js";
11
11
  import { resolveIndexPassLLM } from "../llm/index-passes.js";
12
12
  import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
13
- import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
13
+ import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntryCount, getMeta, isVecAvailable, openExistingDatabase, openIndexDatabase, purgeEmbeddings, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
14
14
  import { deleteStoredGraph } from "./db/graph-db.js";
15
15
  import { withIndexWriterLease } from "./index-writer-lock.js";
16
+ import { canUseIncrementalSkip, computeDirFingerprint, getCachedZeroRowDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
16
17
  import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./passes/metadata.js";
17
18
  import { buildSearchText } from "./search/search-fields.js";
18
19
  import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
@@ -196,8 +197,8 @@ async function runFinalizePhase(ctx) {
196
197
  }
197
198
  onProgress({ phase: "verify", message: verification.message });
198
199
  // Store verification result and totalEntries on ctx for the caller to use
199
- ctx._verification = verification;
200
- ctx._totalEntries = totalEntries;
200
+ ctx.verification = verification;
201
+ ctx.totalEntries = totalEntries;
201
202
  // suppress unused warning — sources was previously used inline
202
203
  void sources;
203
204
  }
@@ -253,7 +254,7 @@ export async function akmIndex(options) {
253
254
  // Open database — pass embedding dimension from config if available
254
255
  const dbPath = getDbPath();
255
256
  const embeddingDim = config.embedding?.dimension;
256
- const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
257
+ const db = openIndexDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
257
258
  try {
258
259
  // Determine incremental vs full mode
259
260
  const prevStashDir = getMeta(db, "stashDir");
@@ -307,7 +308,9 @@ export async function akmIndex(options) {
307
308
  await runEmbeddingPhase(ctx);
308
309
  await runFinalizePhase(ctx);
309
310
  // ────────────────────────────────────────────────────────────────────────
310
- const { _verification: verification, _totalEntries: totalEntries } = ctx;
311
+ // runFinalizePhase always populates these before returning.
312
+ const verification = ctx.verification;
313
+ const totalEntries = ctx.totalEntries;
311
314
  const { timing } = ctx;
312
315
  // ── Clean pass ───────────────────────────────────────────────────────────
313
316
  // After the normal index completes, remove entries whose source files no
@@ -637,98 +640,6 @@ async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadR
637
640
  insertTransaction();
638
641
  return { scannedDirs, skippedDirs, generatedCount, warnings, dirsNeedingLlm };
639
642
  }
640
- function getDirIndexState(db, dirPath, files, builtAtMs) {
641
- const prevEntries = getEntriesByDir(db, dirPath);
642
- const fingerprint = computeDirFingerprint(dirPath, files);
643
- if (prevEntries.length > 0) {
644
- const staleReason = getDirStaleReason(dirPath, files, prevEntries, builtAtMs);
645
- if (!staleReason) {
646
- return { stale: false, reason: { kind: "unchanged" }, persistedRowCount: prevEntries.length };
647
- }
648
- return { stale: true, reason: staleReason, persistedRowCount: prevEntries.length };
649
- }
650
- const cachedState = getIndexDirState(db, dirPath);
651
- if (cachedState &&
652
- cachedState.fileSetHash === fingerprint.fileSetHash &&
653
- cachedState.fileMtimeMaxMs === fingerprint.fileMtimeMaxMs) {
654
- return {
655
- stale: false,
656
- reason: { kind: "cached-zero-row-state", detail: cachedState.reason },
657
- persistedRowCount: 0,
658
- };
659
- }
660
- return {
661
- stale: true,
662
- reason: { kind: "no-previous-rows", detail: cachedState ? `cached=${cachedState.reason}` : undefined },
663
- persistedRowCount: 0,
664
- };
665
- }
666
- function getCachedZeroRowDirState(db, dirPath, files, builtAtMs, priorDirsChanged) {
667
- const state = getDirIndexState(db, dirPath, files, builtAtMs);
668
- if (state.stale || state.reason.kind !== "cached-zero-row-state")
669
- return undefined;
670
- if (!canUseIncrementalSkip(state, priorDirsChanged))
671
- return undefined;
672
- return state;
673
- }
674
- function canUseIncrementalSkip(state, priorDirsChanged) {
675
- return !(priorDirsChanged &&
676
- state.reason.kind === "cached-zero-row-state" &&
677
- state.reason.detail === "deduped-zero-row");
678
- }
679
- function computeDirFingerprint(_dirPath, files) {
680
- const normalizedFiles = [...new Set(files.map((file) => path.basename(file)))].sort();
681
- let fileMtimeMaxMs = 0;
682
- for (const file of files) {
683
- try {
684
- fileMtimeMaxMs = Math.max(fileMtimeMaxMs, fs.statSync(file).mtimeMs);
685
- }
686
- catch {
687
- fileMtimeMaxMs = Number.POSITIVE_INFINITY;
688
- break;
689
- }
690
- }
691
- return {
692
- fileSetHash: normalizedFiles.join("\0"),
693
- fileMtimeMaxMs,
694
- };
695
- }
696
- function getDirStaleReason(_dirPath, currentFiles, previousEntries, builtAtMs) {
697
- const prevFileNames = new Set(previousEntries
698
- .map((ie) => {
699
- const fromPath = path.basename(ie.filePath);
700
- return fromPath || ie.entry.filename;
701
- })
702
- .filter((e) => !!e));
703
- const currFileNames = new Set(currentFiles.map((f) => path.basename(f)));
704
- if (prevFileNames.size !== currFileNames.size) {
705
- return { kind: "file-set-changed", detail: `${prevFileNames.size} -> ${currFileNames.size} files` };
706
- }
707
- for (const name of currFileNames) {
708
- if (!prevFileNames.has(name))
709
- return { kind: "file-set-changed", detail: name };
710
- }
711
- for (const file of currentFiles) {
712
- try {
713
- if (fs.statSync(file).mtimeMs > builtAtMs)
714
- return { kind: "mtime-changed", detail: path.basename(file) };
715
- }
716
- catch {
717
- return { kind: "missing-file", detail: path.basename(file) };
718
- }
719
- }
720
- return undefined;
721
- }
722
- function inferZeroRowReason(stash, priorReason, warnings, dirPath, dedupedRows) {
723
- if (dedupedRows > 0)
724
- return "deduped-zero-row";
725
- const workflowNoise = warnings.some((warning) => warning.startsWith("Skipped workflow ") && warning.includes(dirPath));
726
- if (workflowNoise)
727
- return "workflow-noise";
728
- if (!stash || stash.entries.length === 0)
729
- return "empty-generated-set";
730
- return `zero-row:${priorReason?.kind ?? "unknown"}`;
731
- }
732
643
  async function enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal, _enrich = false, reEnrich = false) {
733
644
  // Resolve per-pass LLM config via the unified shim. Returns undefined when
734
645
  // either no `akm.llm` is configured or the user opted this pass out via
@@ -899,21 +810,9 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal) {
899
810
  const currentFingerprint = deriveSemanticProviderFingerprint(config.embedding);
900
811
  const storedFingerprint = getMeta(db, "embeddingFingerprint");
901
812
  if (storedFingerprint && storedFingerprint !== currentFingerprint) {
902
- try {
903
- db.exec("DELETE FROM embeddings");
904
- }
905
- catch {
906
- /* ignore */
907
- }
908
- if (isVecAvailable(db)) {
909
- try {
910
- db.exec("DELETE FROM entries_vec");
911
- }
912
- catch {
913
- /* ignore */
914
- }
915
- }
916
- setMeta(db, "hasEmbeddings", "0");
813
+ // Model/provider changed → stored vectors are incompatible. Clear them
814
+ // (same dimension, so keep the vec table); re-embedded by this index run.
815
+ purgeEmbeddings(db);
917
816
  }
918
817
  try {
919
818
  const { embedBatch } = await import("../llm/embedder.js");