akm-cli 0.9.0-beta.0 → 0.9.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +511 -0
  2. package/dist/assets/profiles/quick.json +2 -1
  3. package/dist/assets/templates/html/default.html +78 -0
  4. package/dist/assets/templates/html/health.html +732 -0
  5. package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
  6. package/dist/cli/shared.js +21 -5
  7. package/dist/cli.js +47 -5
  8. package/dist/commands/config-cli.js +0 -10
  9. package/dist/commands/feedback-cli.js +42 -37
  10. package/dist/commands/graph/graph.js +75 -71
  11. package/dist/commands/health/checks.js +48 -0
  12. package/dist/commands/health/html-report.js +666 -0
  13. package/dist/commands/health.js +186 -13
  14. package/dist/commands/improve/consolidate.js +39 -6
  15. package/dist/commands/improve/distill.js +26 -5
  16. package/dist/commands/improve/extract-prompt.js +1 -1
  17. package/dist/commands/improve/extract.js +52 -8
  18. package/dist/commands/improve/improve-auto-accept.js +33 -1
  19. package/dist/commands/improve/improve-cli.js +7 -0
  20. package/dist/commands/improve/improve-profiles.js +4 -0
  21. package/dist/commands/improve/improve.js +877 -433
  22. package/dist/commands/improve/proactive-maintenance.js +113 -0
  23. package/dist/commands/improve/reflect-noise.js +0 -0
  24. package/dist/commands/improve/reflect.js +31 -0
  25. package/dist/commands/proposal/drain.js +73 -6
  26. package/dist/commands/proposal/proposal-cli.js +22 -10
  27. package/dist/commands/proposal/proposal.js +17 -1
  28. package/dist/commands/proposal/validators/proposals.js +365 -329
  29. package/dist/commands/read/curate.js +17 -0
  30. package/dist/commands/remember.js +6 -2
  31. package/dist/commands/sources/stash-cli.js +10 -2
  32. package/dist/commands/tasks/tasks.js +32 -8
  33. package/dist/core/config/config-schema.js +30 -0
  34. package/dist/core/file-lock.js +22 -0
  35. package/dist/core/logs-db.js +304 -0
  36. package/dist/core/paths.js +3 -0
  37. package/dist/core/state-db.js +152 -14
  38. package/dist/indexer/db/db.js +99 -13
  39. package/dist/indexer/ensure-index.js +152 -17
  40. package/dist/indexer/index-writer-lock.js +99 -0
  41. package/dist/indexer/indexer.js +114 -111
  42. package/dist/indexer/passes/memory-inference.js +61 -22
  43. package/dist/integrations/harnesses/claude/session-log.js +17 -5
  44. package/dist/llm/client.js +38 -4
  45. package/dist/llm/usage-persist.js +77 -0
  46. package/dist/llm/usage-telemetry.js +103 -0
  47. package/dist/output/context.js +3 -2
  48. package/dist/output/html-render.js +73 -0
  49. package/dist/output/shapes/helpers.js +17 -1
  50. package/dist/output/text/helpers.js +69 -1
  51. package/dist/scripts/migrate-storage.js +153 -25
  52. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +21 -2
  53. package/dist/sources/providers/tar-utils.js +16 -8
  54. package/dist/tasks/backends/cron.js +46 -9
  55. package/dist/tasks/runner.js +99 -16
  56. package/dist/workflows/db.js +4 -0
  57. package/package.json +3 -2
  58. package/dist/commands/config-edit.js +0 -344
@@ -86,11 +86,12 @@ export function getStateDbPath() {
86
86
  * backwards compatibility; enabling them prevents orphaned rows in tables
87
87
  * that reference each other (not used in v1 schema but guards future ones).
88
88
  *
89
- * busy_timeout = 5000
89
+ * busy_timeout = 30000
90
90
  * When another connection holds a write lock, SQLite retries for up to
91
- * 5 000 ms before returning SQLITE_BUSY. Without this, the default timeout
92
- * is 0 ms — any concurrent writer causes an immediate error. 5 s matches
93
- * the same value used in openDatabase() for index.db.
91
+ * 30 000 ms before returning SQLITE_BUSY. Without this, the default timeout
92
+ * is 0 ms — any concurrent writer causes an immediate error. 30 s (#589)
93
+ * matches the value used in openDatabase() for index.db; 5 s proved too
94
+ * narrow when a post-inference reindex overlapped a parallel event write.
94
95
  */
95
96
  export function openStateDatabase(dbPath) {
96
97
  const resolvedPath = dbPath ?? getStateDbPath();
@@ -102,7 +103,7 @@ export function openStateDatabase(dbPath) {
102
103
  // PRAGMAs must run before any DDL or DML.
103
104
  db.exec("PRAGMA journal_mode = WAL");
104
105
  db.exec("PRAGMA foreign_keys = ON");
105
- db.exec("PRAGMA busy_timeout = 5000");
106
+ db.exec("PRAGMA busy_timeout = 30000");
106
107
  runMigrations(db);
107
108
  return db;
108
109
  }
@@ -190,7 +191,9 @@ const MIGRATIONS = [
190
191
  --
191
192
  -- Extensible (metadata_json) columns:
192
193
  -- metadata_json TEXT — JSON object for future proposal fields.
193
- -- Current fields stored here: sourceRun, review.
194
+ -- Current fields stored here: sourceRun,
195
+ -- review, confidence, gateDecision (#577),
196
+ -- backupContent, eligibilitySource.
194
197
  --
195
198
  -- ADD COLUMN extension points (future migrations):
196
199
  -- ALTER TABLE proposals ADD COLUMN source_run TEXT DEFAULT NULL;
@@ -458,6 +461,47 @@ const MIGRATIONS = [
458
461
  ON extract_sessions_seen(processed_at);
459
462
  `,
460
463
  },
464
+ // ── Migration 005 — proposal_fs_imports ─────────────────────────────────────
465
+ //
466
+ // One-shot ledger for the legacy filesystem→SQLite proposal import (#578).
467
+ //
468
+ // Before 0.9.0 the proposal queue lived as per-uuid JSON directories under
469
+ // `<stashDir>/.akm/proposals/` and the `proposals` table (created in 001) was
470
+ // dead weight. 0.9.0 makes the table canonical; the first proposal operation
471
+ // against a stash imports any legacy `proposal.json` files it finds (INSERT
472
+ // OR IGNORE, so re-runs never duplicate) and records the stash here so later
473
+ // invocations skip the directory walk entirely.
474
+ //
475
+ // Indexed (query) columns:
476
+ // stash_dir TEXT PK — absolute stash root the import ran against.
477
+ //
478
+ // Non-indexed columns:
479
+ // imported_at TEXT — ISO-8601 UTC; when the import completed.
480
+ // imported_count INTEGER — rows actually inserted by the import.
481
+ {
482
+ id: "005-proposal-fs-imports",
483
+ up: `
484
+ CREATE TABLE IF NOT EXISTS proposal_fs_imports (
485
+ stash_dir TEXT PRIMARY KEY,
486
+ imported_at TEXT NOT NULL,
487
+ imported_count INTEGER NOT NULL DEFAULT 0
488
+ );
489
+ `,
490
+ },
491
+ // ── Migration 006 — pending proposal lookup index ──────────────────────────
492
+ //
493
+ // Supports the transaction-scoped dedup / queue-mutation hardening added in
494
+ // 0.9.x. The queue now acquires an IMMEDIATE write transaction before it
495
+ // reads pending proposals, so the hot path is a stash-scoped `status='pending'
496
+ // AND ref=?` probe followed by an update/insert. This composite index keeps
497
+ // that lookup index-covered under contention.
498
+ {
499
+ id: "006-proposals-pending-ref-source",
500
+ up: `
501
+ CREATE INDEX IF NOT EXISTS idx_proposals_stash_status_ref_source
502
+ ON proposals(stash_dir, status, ref, source);
503
+ `,
504
+ },
461
505
  ];
462
506
  /**
463
507
  * Apply every pending migration in a single transaction per migration.
@@ -529,6 +573,12 @@ export function proposalRowToProposal(row) {
529
573
  ...(frontmatter !== undefined ? { frontmatter } : {}),
530
574
  },
531
575
  ...(meta.review !== undefined ? { review: meta.review } : {}),
576
+ ...(typeof meta.confidence === "number" ? { confidence: meta.confidence } : {}),
577
+ ...(meta.gateDecision !== undefined ? { gateDecision: meta.gateDecision } : {}),
578
+ ...(typeof meta.backupContent === "string" ? { backupContent: meta.backupContent } : {}),
579
+ ...(typeof meta.eligibilitySource === "string"
580
+ ? { eligibilitySource: meta.eligibilitySource }
581
+ : {}),
532
582
  };
533
583
  }
534
584
  /**
@@ -542,6 +592,14 @@ export function proposalToRowValues(proposal, stashDir) {
542
592
  metaObj.sourceRun = proposal.sourceRun;
543
593
  if (proposal.review !== undefined)
544
594
  metaObj.review = proposal.review;
595
+ if (proposal.confidence !== undefined)
596
+ metaObj.confidence = proposal.confidence;
597
+ if (proposal.gateDecision !== undefined)
598
+ metaObj.gateDecision = proposal.gateDecision;
599
+ if (proposal.backupContent !== undefined)
600
+ metaObj.backupContent = proposal.backupContent;
601
+ if (proposal.eligibilitySource !== undefined)
602
+ metaObj.eligibilitySource = proposal.eligibilitySource;
545
603
  return {
546
604
  id: proposal.id,
547
605
  stash_dir: stashDir,
@@ -656,7 +714,10 @@ export function upsertProposal(db, proposal, stashDir) {
656
714
  }
657
715
  /**
658
716
  * List proposals, optionally filtered by stashDir, status, and/or ref.
659
- * Results are sorted by created_at ASC to match the existing listProposals() behaviour.
717
+ *
718
+ * Results are ordered by `created_at ASC` (matching the historical
719
+ * `listProposals()` sort), with `rowid` as a deterministic tiebreak so two
720
+ * proposals created in the same millisecond list in insertion order.
660
721
  */
661
722
  export function listStateProposals(db, options = {}) {
662
723
  const conditions = [];
@@ -677,21 +738,98 @@ export function listStateProposals(db, options = {}) {
677
738
  const rows = db
678
739
  .prepare(`SELECT id, stash_dir, ref, status, source, created_at, updated_at,
679
740
  content, frontmatter_json, metadata_json
680
- FROM proposals ${where} ORDER BY created_at ASC`)
741
+ FROM proposals ${where} ORDER BY created_at ASC, rowid ASC`)
681
742
  .all(...params);
682
743
  return rows.map(proposalRowToProposal);
683
744
  }
684
745
  /**
685
- * Look up a single proposal by id. Returns undefined when not found.
746
+ * Look up a single proposal by id, optionally scoped to one stash root.
747
+ * Returns undefined when not found.
686
748
  */
687
- export function getStateProposal(db, id) {
688
- const row = db
689
- .prepare(`SELECT id, stash_dir, ref, status, source, created_at, updated_at,
749
+ export function getStateProposal(db, id, stashDir) {
750
+ const sql = `SELECT id, stash_dir, ref, status, source, created_at, updated_at,
690
751
  content, frontmatter_json, metadata_json
691
- FROM proposals WHERE id = ?`)
692
- .get(id);
752
+ FROM proposals WHERE id = ?${stashDir ? " AND stash_dir = ?" : ""}`;
753
+ const row = (stashDir ? db.prepare(sql).get(id, stashDir) : db.prepare(sql).get(id));
693
754
  return row ? proposalRowToProposal(row) : undefined;
694
755
  }
756
+ /**
757
+ * Find PENDING proposal ids in one stash whose id starts with `idPrefix`.
758
+ * Backs the UUID-prefix form of `akm proposal show/accept/... <prefix>` —
759
+ * prefix resolution is deliberately scoped to the live (pending) queue,
760
+ * mirroring the historical behaviour of scanning only the live directory.
761
+ *
762
+ * `%` / `_` / `\` in the prefix are escaped so the LIKE pattern is literal.
763
+ */
764
+ export function listStateProposalIdsByPrefix(db, stashDir, idPrefix) {
765
+ const escaped = idPrefix.replace(/[\\%_]/g, (ch) => `\\${ch}`);
766
+ const rows = db
767
+ .prepare(`SELECT id FROM proposals
768
+ WHERE stash_dir = ? AND status = 'pending' AND id LIKE ? ESCAPE '\\'
769
+ ORDER BY id ASC`)
770
+ .all(stashDir, `${escaped}%`);
771
+ return rows.map((r) => r.id);
772
+ }
773
+ /**
774
+ * Whether the legacy filesystem proposal import has already run for `stashDir`.
775
+ * See migration 005 (`proposal_fs_imports`).
776
+ */
777
+ export function hasImportedFsProposals(db, stashDir) {
778
+ // Drivers disagree on the no-row sentinel (bun:sqlite → null,
779
+ // better-sqlite3 → undefined) — Boolean() covers both.
780
+ return Boolean(db.prepare("SELECT 1 FROM proposal_fs_imports WHERE stash_dir = ?").get(stashDir));
781
+ }
782
+ /**
783
+ * Record that the legacy filesystem proposal import completed for `stashDir`
784
+ * so subsequent invocations skip the directory walk. INSERT OR REPLACE keeps
785
+ * the call idempotent.
786
+ */
787
+ export function recordFsProposalsImport(db, stashDir, importedCount) {
788
+ db.prepare("INSERT OR REPLACE INTO proposal_fs_imports (stash_dir, imported_at, imported_count) VALUES (?, ?, ?)").run(stashDir, new Date().toISOString(), importedCount);
789
+ }
790
+ /**
791
+ * Insert a proposal row ONLY when the id is not already present (used by the
792
+ * legacy filesystem import so re-runs never clobber rows that have since been
793
+ * mutated through the canonical store). Returns true when a row was inserted.
794
+ */
795
+ export function insertProposalIfAbsent(db, proposal, stashDir) {
796
+ const v = proposalToRowValues(proposal, stashDir);
797
+ const result = db
798
+ .prepare(`
799
+ INSERT OR IGNORE INTO proposals
800
+ (id, stash_dir, ref, status, source, created_at, updated_at, content, frontmatter_json, metadata_json)
801
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
802
+ `)
803
+ .run(v.id, v.stash_dir, v.ref, v.status, v.source, v.created_at, v.updated_at, v.content, v.frontmatter_json, v.metadata_json);
804
+ const changes = result.changes ?? 0;
805
+ return Number(changes) > 0;
806
+ }
807
+ /**
808
+ * Run `fn` inside a `BEGIN IMMEDIATE` transaction.
809
+ *
810
+ * `db.transaction()` is DEFERRED by default on both Bun and better-sqlite3,
811
+ * which means two writers can both perform stale preflight reads and only race
812
+ * when they finally attempt the write. Proposal creation and queue mutation
813
+ * need the write lock BEFORE those reads so concurrent processes serialize on
814
+ * the live queue state rather than clobbering each other.
815
+ */
816
+ export function withImmediateTransaction(db, fn) {
817
+ db.exec("BEGIN IMMEDIATE");
818
+ try {
819
+ const result = fn();
820
+ db.exec("COMMIT");
821
+ return result;
822
+ }
823
+ catch (err) {
824
+ try {
825
+ db.exec("ROLLBACK");
826
+ }
827
+ catch {
828
+ // Ignore rollback failures so the original error is preserved.
829
+ }
830
+ throw err;
831
+ }
832
+ }
695
833
  // ── task_history table helpers ───────────────────────────────────────────────
696
834
  /**
697
835
  * Upsert a task history row.
@@ -28,7 +28,7 @@ export function openDatabase(dbPath, options) {
28
28
  }
29
29
  const db = openSqlite(resolvedPath);
30
30
  db.exec("PRAGMA journal_mode = WAL");
31
- db.exec("PRAGMA busy_timeout = 5000");
31
+ db.exec("PRAGMA busy_timeout = 30000");
32
32
  db.exec("PRAGMA foreign_keys = ON");
33
33
  // Try to load sqlite-vec extension
34
34
  loadVecExtension(db);
@@ -69,7 +69,7 @@ export function openExistingDatabase(dbPath) {
69
69
  const resolvedPath = dbPath ?? getDbPath();
70
70
  const db = openSqlite(resolvedPath);
71
71
  db.exec("PRAGMA journal_mode = WAL");
72
- db.exec("PRAGMA busy_timeout = 5000");
72
+ db.exec("PRAGMA busy_timeout = 30000");
73
73
  db.exec("PRAGMA foreign_keys = ON");
74
74
  // Existing-DB callers must not mutate schema or embedding metadata on open,
75
75
  // but some paths still need write access to usage_events and other tables.
@@ -1262,6 +1262,19 @@ export function getEntryIdByFilePath(db, filePath) {
1262
1262
  const row = db.prepare("SELECT id FROM entries WHERE file_path = ? LIMIT 1").get(filePath);
1263
1263
  return row?.id;
1264
1264
  }
1265
+ /**
1266
+ * Set of every non-empty `entries.file_path` currently indexed (across all
1267
+ * stashes/sources). Used by staleness detection to spot files that exist on
1268
+ * disk but were never indexed — a clock-independent signal for newly-added
1269
+ * assets that an mtime-vs-builtAt comparison can miss when the two clocks
1270
+ * (filesystem vs wall-clock) are skewed within the same millisecond.
1271
+ */
1272
+ export function getIndexedFilePaths(db) {
1273
+ const rows = db
1274
+ .prepare("SELECT DISTINCT file_path FROM entries WHERE file_path IS NOT NULL AND file_path <> ''")
1275
+ .all();
1276
+ return new Set(rows.map((r) => r.file_path));
1277
+ }
1265
1278
  /**
1266
1279
  * Resolve a single `entries.file_path` by primary key, or `undefined` if no
1267
1280
  * row matches.
@@ -1477,25 +1490,98 @@ export function computeBodyHash(body) {
1477
1490
  return sha256Hex(body);
1478
1491
  }
1479
1492
  /**
1480
- * Count search and show events for the given entry refs.
1481
- * Returns a Map<ref, count> with only refs that have at least one event.
1482
- * Used by the improve loop to find high-retrieval assets without feedback.
1493
+ * Reduce a ref to its bare `type:name` form, dropping any `origin//` prefix.
1494
+ *
1495
+ * usage_events store entry_ref inconsistently: search/show writers persist
1496
+ * whatever ref the result carried, which is sometimes stash-prefixed
1497
+ * (`origin//type:name`) and sometimes bare (`type:name`). Retrieval counting
1498
+ * keys on the bare form so both spellings of the same asset collapse together.
1499
+ *
1500
+ * Returns the bare form, or the original string when it cannot be parsed (best
1501
+ * effort — never throws so a malformed stored ref can't break counting).
1502
+ */
1503
+ function bareRef(ref) {
1504
+ try {
1505
+ const parsed = parseAssetRef(ref);
1506
+ return `${parsed.type}:${parsed.name}`;
1507
+ }
1508
+ catch {
1509
+ return ref;
1510
+ }
1511
+ }
1512
+ /**
1513
+ * Count retrieval events for the given entry refs.
1514
+ *
1515
+ * Counts `search`, `show`, and `curate` usage events. Returns a
1516
+ * Map<inputRef, count> keyed by the *input* ref strings (only those with at
1517
+ * least one matching event appear). Used by the improve loop to find
1518
+ * high-retrieval assets without feedback.
1519
+ *
1520
+ * Matching is normalization-aware: each stored `entry_ref` is reduced to its
1521
+ * bare `type:name` form before comparison, so a stash-prefixed stored ref
1522
+ * (`origin//type:name`) still matches a bare input ref (`type:name`) and vice
1523
+ * versa. Previously the raw `entry_ref IN (...)` comparison silently dropped
1524
+ * roughly half the signal whenever the two spellings disagreed.
1525
+ *
1526
+ * `curate` events are included: their per-item rows are written with
1527
+ * entry_ref populated (see logCurateEvent), so curation is a real retrieval
1528
+ * signal here. Legacy summary-only curate rows with a NULL entry_ref simply
1529
+ * contribute nothing.
1483
1530
  */
1484
1531
  export function getRetrievalCounts(db, refs) {
1485
1532
  if (refs.length === 0)
1486
1533
  return new Map();
1487
- const result = new Map();
1534
+ // Map each distinct bare form back to the input ref(s) that produced it so we
1535
+ // can re-key DB results (grouped by bare form) onto the caller's ref strings.
1536
+ const bareToInputs = new Map();
1537
+ for (const ref of refs) {
1538
+ const bare = bareRef(ref);
1539
+ const existing = bareToInputs.get(bare);
1540
+ if (existing)
1541
+ existing.push(ref);
1542
+ else
1543
+ bareToInputs.set(bare, [ref]);
1544
+ }
1545
+ const bareForms = [...bareToInputs.keys()];
1546
+ // Accumulate counts per bare form across chunks before re-keying.
1547
+ const countsByBare = new Map();
1488
1548
  // Chunk to stay within SQLITE_MAX_VARIABLE_NUMBER (same pattern as getUtilityScoresByIds).
1489
- for (let i = 0; i < refs.length; i += SQLITE_CHUNK_SIZE) {
1490
- const chunk = refs.slice(i, i + SQLITE_CHUNK_SIZE);
1549
+ for (let i = 0; i < bareForms.length; i += SQLITE_CHUNK_SIZE) {
1550
+ const chunk = bareForms.slice(i, i + SQLITE_CHUNK_SIZE);
1491
1551
  const placeholders = chunk.map(() => "?").join(", ");
1552
+ // Normalize the stored entry_ref to its bare form inside SQL by stripping
1553
+ // everything up to and including the last `//` separator. SQLite has no
1554
+ // rfind, but stored origins never themselves contain `//`, so a stash ref
1555
+ // has exactly one `//` and `substr(... instr ...)` is exact; bare refs have
1556
+ // no `//` and pass through unchanged.
1492
1557
  const rows = db
1493
- .prepare(`SELECT entry_ref, COUNT(*) AS cnt FROM usage_events
1494
- WHERE event_type IN ('search','show') AND entry_ref IN (${placeholders})
1495
- GROUP BY entry_ref`)
1558
+ .prepare(`SELECT
1559
+ CASE
1560
+ WHEN instr(entry_ref, '//') > 0
1561
+ THEN substr(entry_ref, instr(entry_ref, '//') + 2)
1562
+ ELSE entry_ref
1563
+ END AS bare_ref,
1564
+ COUNT(*) AS cnt
1565
+ FROM usage_events
1566
+ WHERE event_type IN ('search','show','curate')
1567
+ AND entry_ref IS NOT NULL
1568
+ AND CASE
1569
+ WHEN instr(entry_ref, '//') > 0
1570
+ THEN substr(entry_ref, instr(entry_ref, '//') + 2)
1571
+ ELSE entry_ref
1572
+ END IN (${placeholders})
1573
+ GROUP BY bare_ref`)
1496
1574
  .all(...chunk);
1497
- for (const r of rows)
1498
- result.set(r.entry_ref, r.cnt);
1575
+ for (const r of rows) {
1576
+ countsByBare.set(r.bare_ref, (countsByBare.get(r.bare_ref) ?? 0) + r.cnt);
1577
+ }
1578
+ }
1579
+ // Re-key bare-form counts onto every input ref that maps to that bare form.
1580
+ const result = new Map();
1581
+ for (const [bare, count] of countsByBare) {
1582
+ for (const input of bareToInputs.get(bare) ?? []) {
1583
+ result.set(input, count);
1584
+ }
1499
1585
  }
1500
1586
  return result;
1501
1587
  }
@@ -11,12 +11,14 @@
11
11
  * `searchLocal()` and `show.ts`, centralizing the "indexed yet?" gap handling
12
12
  * behind a single entry point.
13
13
  */
14
+ import { spawn } from "node:child_process";
14
15
  import fs from "node:fs";
15
16
  import path from "node:path";
16
17
  import { ASSET_SPECS, TYPE_DIRS } from "../core/asset/asset-spec.js";
17
- import { getDbPath } from "../core/paths.js";
18
+ import { getDataDir, getDbPath } from "../core/paths.js";
18
19
  import { warn } from "../core/warn.js";
19
- import { closeDatabase, getEntryCount, getMeta, openExistingDatabase } from "./db/db.js";
20
+ import { closeDatabase, getEntryCount, getIndexedFilePaths, getMeta, openExistingDatabase } from "./db/db.js";
21
+ import { acquireIndexWriterLease, handoffIndexWriterLeaseToPid } from "./index-writer-lock.js";
20
22
  function getIndexableFiles(root, spec) {
21
23
  if (!fs.existsSync(root))
22
24
  return [];
@@ -52,16 +54,34 @@ function getIndexableFiles(root, spec) {
52
54
  }
53
55
  return files;
54
56
  }
55
- function hasNewerIndexableFiles(stashDir, builtAt) {
56
- if (!builtAt)
57
- return true;
58
- const builtAtMs = new Date(builtAt).getTime();
59
- if (!Number.isFinite(builtAtMs))
60
- return true;
57
+ /**
58
+ * Whether any indexable file under `stashDir` is newer than the last build, or
59
+ * has never been indexed at all.
60
+ *
61
+ * Two independent signals, because neither alone is sufficient:
62
+ * 1. **mtime > builtAt** — catches in-place *edits* of already-indexed files.
63
+ * 2. **path not in `indexedPaths`** — catches *newly added* files. This is
64
+ * clock-independent on purpose: a freshly-written file can have a
65
+ * filesystem mtime that compares as *older* than the wall-clock `builtAt`
66
+ * (the two clocks are not perfectly synchronized and `builtAt` is
67
+ * millisecond-truncated), so the mtime test alone silently misses
68
+ * additions made within ~a millisecond of the previous build.
69
+ *
70
+ * `getIndexableFiles` applies each asset type's own relevance filter, so
71
+ * non-indexed companion files (e.g. `package.json` next to a knowledge doc) are
72
+ * never considered and do not produce false "new file" positives.
73
+ */
74
+ function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths) {
75
+ const builtAtMs = builtAt ? new Date(builtAt).getTime() : Number.NaN;
76
+ const builtAtUsable = Number.isFinite(builtAtMs);
61
77
  for (const [type, spec] of Object.entries(ASSET_SPECS)) {
62
78
  const typeRoot = path.join(stashDir, TYPE_DIRS[type] ?? spec.stashDir);
63
79
  const files = getIndexableFiles(typeRoot, spec);
64
80
  for (const file of files) {
81
+ if (!indexedPaths.has(file))
82
+ return true;
83
+ if (!builtAtUsable)
84
+ return true;
65
85
  try {
66
86
  if (fs.statSync(file).mtimeMs > builtAtMs)
67
87
  return true;
@@ -89,7 +109,7 @@ export function isIndexStale(stashDir) {
89
109
  if (entryCount === 0)
90
110
  return true;
91
111
  const builtAt = getMeta(db, "builtAt");
92
- if (hasNewerIndexableFiles(stashDir, builtAt))
112
+ if (hasNewerIndexableFiles(stashDir, builtAt, getIndexedFilePaths(db)))
93
113
  return true;
94
114
  const storedStashDir = getMeta(db, "stashDir");
95
115
  if (storedStashDir !== stashDir) {
@@ -114,16 +134,84 @@ export function isIndexStale(stashDir) {
114
134
  }
115
135
  }
116
136
  /**
117
- * Run an incremental index when the local index is stale. Best-effort
118
- * failures are logged as warnings but never thrown, so the caller can
119
- * proceed (and surface a proper "not in index" error if the index is
120
- * still unusable).
121
- *
122
- * Returns `true` if an index run was attempted.
137
+ * Whether the existing index can serve queries for `stashDir` *right now*
138
+ * i.e. the DB file exists, the `entries` table holds rows, and those rows were
139
+ * built for this stash (it is the stored primary stash or appears in the
140
+ * stored `stashDirs` set). When this is true the index is at worst
141
+ * content-stale, so the `#607` background-reindex optimization is safe: the
142
+ * caller gets slightly-stale-but-relevant results immediately. When it is
143
+ * false the existing index has nothing relevant to return (no DB, no `entries`
144
+ * table, zero rows, or built for a different stash), so a background reindex
145
+ * would leave the caller empty until the next read — those cases must rebuild
146
+ * inline.
123
147
  */
124
- export async function ensureIndex(stashDir) {
125
- if (!isIndexStale(stashDir))
148
+ function indexCanServeStash(stashDir) {
149
+ const dbPath = getDbPath();
150
+ if (!fs.existsSync(dbPath))
151
+ return false;
152
+ let db;
153
+ try {
154
+ db = openExistingDatabase(dbPath);
155
+ if (getEntryCount(db) === 0)
156
+ return false;
157
+ const storedStashDir = getMeta(db, "stashDir");
158
+ if (storedStashDir === stashDir)
159
+ return true;
160
+ try {
161
+ const storedDirs = JSON.parse(getMeta(db, "stashDirs") ?? "[]");
162
+ return storedDirs.includes(stashDir);
163
+ }
164
+ catch {
165
+ return false;
166
+ }
167
+ }
168
+ catch {
169
+ // No `entries` table (or otherwise unreadable) — cannot serve.
126
170
  return false;
171
+ }
172
+ finally {
173
+ if (db)
174
+ closeDatabase(db);
175
+ }
176
+ }
177
+ /**
178
+ * Spawn a background `akm index` process. Non-blocking — returns immediately.
179
+ * Background callers share the same global index-writer lease as foreground
180
+ * writers, so stale-read-triggered auto-index attempts coalesce safely.
181
+ */
182
+ async function spawnBackgroundReindex(_stashDir) {
183
+ const dataDir = getDataDir();
184
+ const logFile = path.join(dataDir, "logs", "index-background.log");
185
+ fs.mkdirSync(path.dirname(logFile), { recursive: true });
186
+ const lease = await acquireIndexWriterLease({ mode: "try", purpose: "background-reindex-spawn" });
187
+ if (!lease)
188
+ return;
189
+ const akmBin = process.argv[0];
190
+ const akmScript = process.argv[1];
191
+ try {
192
+ const child = spawn(akmBin, [akmScript, "index", "--background"], {
193
+ detached: true,
194
+ stdio: ["ignore", fs.openSync(logFile, "a"), fs.openSync(logFile, "a")],
195
+ env: { ...process.env },
196
+ });
197
+ if (!child.pid) {
198
+ lease.release();
199
+ return;
200
+ }
201
+ handoffIndexWriterLeaseToPid(lease, child.pid, "background-reindex");
202
+ try {
203
+ child.unref();
204
+ }
205
+ catch {
206
+ // ignore
207
+ }
208
+ }
209
+ catch (error) {
210
+ lease.release();
211
+ throw error;
212
+ }
213
+ }
214
+ async function runInlineReindex(stashDir) {
127
215
  try {
128
216
  const { akmIndex } = await import("./indexer.js");
129
217
  await akmIndex({ stashDir });
@@ -134,3 +222,50 @@ export async function ensureIndex(stashDir) {
134
222
  return true;
135
223
  }
136
224
  }
225
+ /**
226
+ * Ensure the local index exists and is fresh enough for the caller's needs.
227
+ *
228
+ * Default mode is `background`, which preserves the low-latency behavior used
229
+ * by read paths (`search`, `show`, `feedback`): when a populated index is
230
+ * merely stale, spawn a detached reindex and proceed against the existing
231
+ * index. When the index is entirely absent (no DB / no `entries` table / zero
232
+ * rows) the rebuild runs inline regardless of mode, since there is nothing to
233
+ * proceed against.
234
+ *
235
+ * `mode: "blocking"` waits for the rebuild to finish before returning. Use
236
+ * this for callers like `improve` whose planning logic depends on a populated
237
+ * `entries` table in the same process.
238
+ *
239
+ * Returns `true` if an index run was attempted.
240
+ */
241
+ export async function ensureIndex(stashDir, options = {}) {
242
+ if (!isIndexStale(stashDir))
243
+ return false;
244
+ // Blocking when explicitly requested, or whenever the existing index cannot
245
+ // serve this stash (absent DB, no `entries` table, zero rows, or built for a
246
+ // different stash): a background reindex returns immediately and would leave
247
+ // a first-time caller (search, curate, wiki, show, feedback) with empty
248
+ // results. Building inline is a one-off cost; a populated index for this
249
+ // stash that is merely content-stale still refreshes in the background.
250
+ if (options.mode === "blocking" || !indexCanServeStash(stashDir)) {
251
+ return runInlineReindex(stashDir);
252
+ }
253
+ // The background path re-invokes the akm CLI as a detached child via
254
+ // `process.argv[1]`. That is only the akm entrypoint when THIS process is the
255
+ // akm CLI itself — which the CLI startup block signals with AKM_CLI_ENTRY=1.
256
+ // In any other host (the in-process test runner, a library embedding akm),
257
+ // argv[1] points at the host (e.g. the test runner), so spawning it would
258
+ // launch the wrong program and orphan it. Build inline there instead — same
259
+ // resulting index, no detached process.
260
+ if (process.env.AKM_CLI_ENTRY !== "1") {
261
+ return runInlineReindex(stashDir);
262
+ }
263
+ try {
264
+ await spawnBackgroundReindex(stashDir);
265
+ return true;
266
+ }
267
+ catch (error) {
268
+ warn("Background reindex spawn failed, proceeding with existing index:", error instanceof Error ? error.message : String(error));
269
+ return true;
270
+ }
271
+ }