akm-cli 0.9.0-beta.0 → 0.9.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +511 -0
- package/dist/assets/profiles/quick.json +2 -1
- package/dist/assets/templates/html/default.html +78 -0
- package/dist/assets/templates/html/health.html +732 -0
- package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
- package/dist/cli/shared.js +21 -5
- package/dist/cli.js +47 -5
- package/dist/commands/config-cli.js +0 -10
- package/dist/commands/feedback-cli.js +42 -37
- package/dist/commands/graph/graph.js +75 -71
- package/dist/commands/health/checks.js +48 -0
- package/dist/commands/health/html-report.js +666 -0
- package/dist/commands/health.js +186 -13
- package/dist/commands/improve/consolidate.js +39 -6
- package/dist/commands/improve/distill.js +26 -5
- package/dist/commands/improve/extract-prompt.js +1 -1
- package/dist/commands/improve/extract.js +52 -8
- package/dist/commands/improve/improve-auto-accept.js +33 -1
- package/dist/commands/improve/improve-cli.js +7 -0
- package/dist/commands/improve/improve-profiles.js +4 -0
- package/dist/commands/improve/improve.js +877 -433
- package/dist/commands/improve/proactive-maintenance.js +113 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +31 -0
- package/dist/commands/proposal/drain.js +73 -6
- package/dist/commands/proposal/proposal-cli.js +22 -10
- package/dist/commands/proposal/proposal.js +17 -1
- package/dist/commands/proposal/validators/proposals.js +365 -329
- package/dist/commands/read/curate.js +17 -0
- package/dist/commands/remember.js +6 -2
- package/dist/commands/sources/stash-cli.js +10 -2
- package/dist/commands/tasks/tasks.js +32 -8
- package/dist/core/config/config-schema.js +30 -0
- package/dist/core/file-lock.js +22 -0
- package/dist/core/logs-db.js +304 -0
- package/dist/core/paths.js +3 -0
- package/dist/core/state-db.js +152 -14
- package/dist/indexer/db/db.js +99 -13
- package/dist/indexer/ensure-index.js +152 -17
- package/dist/indexer/index-writer-lock.js +99 -0
- package/dist/indexer/indexer.js +114 -111
- package/dist/indexer/passes/memory-inference.js +61 -22
- package/dist/integrations/harnesses/claude/session-log.js +17 -5
- package/dist/llm/client.js +38 -4
- package/dist/llm/usage-persist.js +77 -0
- package/dist/llm/usage-telemetry.js +103 -0
- package/dist/output/context.js +3 -2
- package/dist/output/html-render.js +73 -0
- package/dist/output/shapes/helpers.js +17 -1
- package/dist/output/text/helpers.js +69 -1
- package/dist/scripts/migrate-storage.js +153 -25
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +21 -2
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/tasks/backends/cron.js +46 -9
- package/dist/tasks/runner.js +99 -16
- package/dist/workflows/db.js +4 -0
- package/package.json +3 -2
- package/dist/commands/config-edit.js +0 -344
package/dist/core/state-db.js
CHANGED
|
@@ -86,11 +86,12 @@ export function getStateDbPath() {
|
|
|
86
86
|
* backwards compatibility; enabling them prevents orphaned rows in tables
|
|
87
87
|
* that reference each other (not used in v1 schema but guards future ones).
|
|
88
88
|
*
|
|
89
|
-
* busy_timeout =
|
|
89
|
+
* busy_timeout = 30000
|
|
90
90
|
* When another connection holds a write lock, SQLite retries for up to
|
|
91
|
-
*
|
|
92
|
-
* is 0 ms — any concurrent writer causes an immediate error.
|
|
93
|
-
* the
|
|
91
|
+
* 30 000 ms before returning SQLITE_BUSY. Without this, the default timeout
|
|
92
|
+
* is 0 ms — any concurrent writer causes an immediate error. 30 s (#589)
|
|
93
|
+
* matches the value used in openDatabase() for index.db; 5 s proved too
|
|
94
|
+
* narrow when a post-inference reindex overlapped a parallel event write.
|
|
94
95
|
*/
|
|
95
96
|
export function openStateDatabase(dbPath) {
|
|
96
97
|
const resolvedPath = dbPath ?? getStateDbPath();
|
|
@@ -102,7 +103,7 @@ export function openStateDatabase(dbPath) {
|
|
|
102
103
|
// PRAGMAs must run before any DDL or DML.
|
|
103
104
|
db.exec("PRAGMA journal_mode = WAL");
|
|
104
105
|
db.exec("PRAGMA foreign_keys = ON");
|
|
105
|
-
db.exec("PRAGMA busy_timeout =
|
|
106
|
+
db.exec("PRAGMA busy_timeout = 30000");
|
|
106
107
|
runMigrations(db);
|
|
107
108
|
return db;
|
|
108
109
|
}
|
|
@@ -190,7 +191,9 @@ const MIGRATIONS = [
|
|
|
190
191
|
--
|
|
191
192
|
-- Extensible (metadata_json) columns:
|
|
192
193
|
-- metadata_json TEXT — JSON object for future proposal fields.
|
|
193
|
-
-- Current fields stored here: sourceRun,
|
|
194
|
+
-- Current fields stored here: sourceRun,
|
|
195
|
+
-- review, confidence, gateDecision (#577),
|
|
196
|
+
-- backupContent, eligibilitySource.
|
|
194
197
|
--
|
|
195
198
|
-- ADD COLUMN extension points (future migrations):
|
|
196
199
|
-- ALTER TABLE proposals ADD COLUMN source_run TEXT DEFAULT NULL;
|
|
@@ -458,6 +461,47 @@ const MIGRATIONS = [
|
|
|
458
461
|
ON extract_sessions_seen(processed_at);
|
|
459
462
|
`,
|
|
460
463
|
},
|
|
464
|
+
// ── Migration 005 — proposal_fs_imports ─────────────────────────────────────
|
|
465
|
+
//
|
|
466
|
+
// One-shot ledger for the legacy filesystem→SQLite proposal import (#578).
|
|
467
|
+
//
|
|
468
|
+
// Before 0.9.0 the proposal queue lived as per-uuid JSON directories under
|
|
469
|
+
// `<stashDir>/.akm/proposals/` and the `proposals` table (created in 001) was
|
|
470
|
+
// dead weight. 0.9.0 makes the table canonical; the first proposal operation
|
|
471
|
+
// against a stash imports any legacy `proposal.json` files it finds (INSERT
|
|
472
|
+
// OR IGNORE, so re-runs never duplicate) and records the stash here so later
|
|
473
|
+
// invocations skip the directory walk entirely.
|
|
474
|
+
//
|
|
475
|
+
// Indexed (query) columns:
|
|
476
|
+
// stash_dir TEXT PK — absolute stash root the import ran against.
|
|
477
|
+
//
|
|
478
|
+
// Non-indexed columns:
|
|
479
|
+
// imported_at TEXT — ISO-8601 UTC; when the import completed.
|
|
480
|
+
// imported_count INTEGER — rows actually inserted by the import.
|
|
481
|
+
{
|
|
482
|
+
id: "005-proposal-fs-imports",
|
|
483
|
+
up: `
|
|
484
|
+
CREATE TABLE IF NOT EXISTS proposal_fs_imports (
|
|
485
|
+
stash_dir TEXT PRIMARY KEY,
|
|
486
|
+
imported_at TEXT NOT NULL,
|
|
487
|
+
imported_count INTEGER NOT NULL DEFAULT 0
|
|
488
|
+
);
|
|
489
|
+
`,
|
|
490
|
+
},
|
|
491
|
+
// ── Migration 006 — pending proposal lookup index ──────────────────────────
|
|
492
|
+
//
|
|
493
|
+
// Supports the transaction-scoped dedup / queue-mutation hardening added in
|
|
494
|
+
// 0.9.x. The queue now acquires an IMMEDIATE write transaction before it
|
|
495
|
+
// reads pending proposals, so the hot path is a stash-scoped `status='pending'
|
|
496
|
+
// AND ref=?` probe followed by an update/insert. This composite index keeps
|
|
497
|
+
// that lookup index-covered under contention.
|
|
498
|
+
{
|
|
499
|
+
id: "006-proposals-pending-ref-source",
|
|
500
|
+
up: `
|
|
501
|
+
CREATE INDEX IF NOT EXISTS idx_proposals_stash_status_ref_source
|
|
502
|
+
ON proposals(stash_dir, status, ref, source);
|
|
503
|
+
`,
|
|
504
|
+
},
|
|
461
505
|
];
|
|
462
506
|
/**
|
|
463
507
|
* Apply every pending migration in a single transaction per migration.
|
|
@@ -529,6 +573,12 @@ export function proposalRowToProposal(row) {
|
|
|
529
573
|
...(frontmatter !== undefined ? { frontmatter } : {}),
|
|
530
574
|
},
|
|
531
575
|
...(meta.review !== undefined ? { review: meta.review } : {}),
|
|
576
|
+
...(typeof meta.confidence === "number" ? { confidence: meta.confidence } : {}),
|
|
577
|
+
...(meta.gateDecision !== undefined ? { gateDecision: meta.gateDecision } : {}),
|
|
578
|
+
...(typeof meta.backupContent === "string" ? { backupContent: meta.backupContent } : {}),
|
|
579
|
+
...(typeof meta.eligibilitySource === "string"
|
|
580
|
+
? { eligibilitySource: meta.eligibilitySource }
|
|
581
|
+
: {}),
|
|
532
582
|
};
|
|
533
583
|
}
|
|
534
584
|
/**
|
|
@@ -542,6 +592,14 @@ export function proposalToRowValues(proposal, stashDir) {
|
|
|
542
592
|
metaObj.sourceRun = proposal.sourceRun;
|
|
543
593
|
if (proposal.review !== undefined)
|
|
544
594
|
metaObj.review = proposal.review;
|
|
595
|
+
if (proposal.confidence !== undefined)
|
|
596
|
+
metaObj.confidence = proposal.confidence;
|
|
597
|
+
if (proposal.gateDecision !== undefined)
|
|
598
|
+
metaObj.gateDecision = proposal.gateDecision;
|
|
599
|
+
if (proposal.backupContent !== undefined)
|
|
600
|
+
metaObj.backupContent = proposal.backupContent;
|
|
601
|
+
if (proposal.eligibilitySource !== undefined)
|
|
602
|
+
metaObj.eligibilitySource = proposal.eligibilitySource;
|
|
545
603
|
return {
|
|
546
604
|
id: proposal.id,
|
|
547
605
|
stash_dir: stashDir,
|
|
@@ -656,7 +714,10 @@ export function upsertProposal(db, proposal, stashDir) {
|
|
|
656
714
|
}
|
|
657
715
|
/**
|
|
658
716
|
* List proposals, optionally filtered by stashDir, status, and/or ref.
|
|
659
|
-
*
|
|
717
|
+
*
|
|
718
|
+
* Results are ordered by `created_at ASC` (matching the historical
|
|
719
|
+
* `listProposals()` sort), with `rowid` as a deterministic tiebreak so two
|
|
720
|
+
* proposals created in the same millisecond list in insertion order.
|
|
660
721
|
*/
|
|
661
722
|
export function listStateProposals(db, options = {}) {
|
|
662
723
|
const conditions = [];
|
|
@@ -677,21 +738,98 @@ export function listStateProposals(db, options = {}) {
|
|
|
677
738
|
const rows = db
|
|
678
739
|
.prepare(`SELECT id, stash_dir, ref, status, source, created_at, updated_at,
|
|
679
740
|
content, frontmatter_json, metadata_json
|
|
680
|
-
FROM proposals ${where} ORDER BY created_at ASC`)
|
|
741
|
+
FROM proposals ${where} ORDER BY created_at ASC, rowid ASC`)
|
|
681
742
|
.all(...params);
|
|
682
743
|
return rows.map(proposalRowToProposal);
|
|
683
744
|
}
|
|
684
745
|
/**
|
|
685
|
-
* Look up a single proposal by id
|
|
746
|
+
* Look up a single proposal by id, optionally scoped to one stash root.
|
|
747
|
+
* Returns undefined when not found.
|
|
686
748
|
*/
|
|
687
|
-
export function getStateProposal(db, id) {
|
|
688
|
-
const
|
|
689
|
-
.prepare(`SELECT id, stash_dir, ref, status, source, created_at, updated_at,
|
|
749
|
+
export function getStateProposal(db, id, stashDir) {
|
|
750
|
+
const sql = `SELECT id, stash_dir, ref, status, source, created_at, updated_at,
|
|
690
751
|
content, frontmatter_json, metadata_json
|
|
691
|
-
FROM proposals WHERE id =
|
|
692
|
-
|
|
752
|
+
FROM proposals WHERE id = ?${stashDir ? " AND stash_dir = ?" : ""}`;
|
|
753
|
+
const row = (stashDir ? db.prepare(sql).get(id, stashDir) : db.prepare(sql).get(id));
|
|
693
754
|
return row ? proposalRowToProposal(row) : undefined;
|
|
694
755
|
}
|
|
756
|
+
/**
|
|
757
|
+
* Find PENDING proposal ids in one stash whose id starts with `idPrefix`.
|
|
758
|
+
* Backs the UUID-prefix form of `akm proposal show/accept/... <prefix>` —
|
|
759
|
+
* prefix resolution is deliberately scoped to the live (pending) queue,
|
|
760
|
+
* mirroring the historical behaviour of scanning only the live directory.
|
|
761
|
+
*
|
|
762
|
+
* `%` / `_` / `\` in the prefix are escaped so the LIKE pattern is literal.
|
|
763
|
+
*/
|
|
764
|
+
export function listStateProposalIdsByPrefix(db, stashDir, idPrefix) {
|
|
765
|
+
const escaped = idPrefix.replace(/[\\%_]/g, (ch) => `\\${ch}`);
|
|
766
|
+
const rows = db
|
|
767
|
+
.prepare(`SELECT id FROM proposals
|
|
768
|
+
WHERE stash_dir = ? AND status = 'pending' AND id LIKE ? ESCAPE '\\'
|
|
769
|
+
ORDER BY id ASC`)
|
|
770
|
+
.all(stashDir, `${escaped}%`);
|
|
771
|
+
return rows.map((r) => r.id);
|
|
772
|
+
}
|
|
773
|
+
/**
|
|
774
|
+
* Whether the legacy filesystem proposal import has already run for `stashDir`.
|
|
775
|
+
* See migration 005 (`proposal_fs_imports`).
|
|
776
|
+
*/
|
|
777
|
+
export function hasImportedFsProposals(db, stashDir) {
|
|
778
|
+
// Drivers disagree on the no-row sentinel (bun:sqlite → null,
|
|
779
|
+
// better-sqlite3 → undefined) — Boolean() covers both.
|
|
780
|
+
return Boolean(db.prepare("SELECT 1 FROM proposal_fs_imports WHERE stash_dir = ?").get(stashDir));
|
|
781
|
+
}
|
|
782
|
+
/**
|
|
783
|
+
* Record that the legacy filesystem proposal import completed for `stashDir`
|
|
784
|
+
* so subsequent invocations skip the directory walk. INSERT OR REPLACE keeps
|
|
785
|
+
* the call idempotent.
|
|
786
|
+
*/
|
|
787
|
+
export function recordFsProposalsImport(db, stashDir, importedCount) {
|
|
788
|
+
db.prepare("INSERT OR REPLACE INTO proposal_fs_imports (stash_dir, imported_at, imported_count) VALUES (?, ?, ?)").run(stashDir, new Date().toISOString(), importedCount);
|
|
789
|
+
}
|
|
790
|
+
/**
|
|
791
|
+
* Insert a proposal row ONLY when the id is not already present (used by the
|
|
792
|
+
* legacy filesystem import so re-runs never clobber rows that have since been
|
|
793
|
+
* mutated through the canonical store). Returns true when a row was inserted.
|
|
794
|
+
*/
|
|
795
|
+
export function insertProposalIfAbsent(db, proposal, stashDir) {
|
|
796
|
+
const v = proposalToRowValues(proposal, stashDir);
|
|
797
|
+
const result = db
|
|
798
|
+
.prepare(`
|
|
799
|
+
INSERT OR IGNORE INTO proposals
|
|
800
|
+
(id, stash_dir, ref, status, source, created_at, updated_at, content, frontmatter_json, metadata_json)
|
|
801
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
802
|
+
`)
|
|
803
|
+
.run(v.id, v.stash_dir, v.ref, v.status, v.source, v.created_at, v.updated_at, v.content, v.frontmatter_json, v.metadata_json);
|
|
804
|
+
const changes = result.changes ?? 0;
|
|
805
|
+
return Number(changes) > 0;
|
|
806
|
+
}
|
|
807
|
+
/**
|
|
808
|
+
* Run `fn` inside a `BEGIN IMMEDIATE` transaction.
|
|
809
|
+
*
|
|
810
|
+
* `db.transaction()` is DEFERRED by default on both Bun and better-sqlite3,
|
|
811
|
+
* which means two writers can both perform stale preflight reads and only race
|
|
812
|
+
* when they finally attempt the write. Proposal creation and queue mutation
|
|
813
|
+
* need the write lock BEFORE those reads so concurrent processes serialize on
|
|
814
|
+
* the live queue state rather than clobbering each other.
|
|
815
|
+
*/
|
|
816
|
+
export function withImmediateTransaction(db, fn) {
|
|
817
|
+
db.exec("BEGIN IMMEDIATE");
|
|
818
|
+
try {
|
|
819
|
+
const result = fn();
|
|
820
|
+
db.exec("COMMIT");
|
|
821
|
+
return result;
|
|
822
|
+
}
|
|
823
|
+
catch (err) {
|
|
824
|
+
try {
|
|
825
|
+
db.exec("ROLLBACK");
|
|
826
|
+
}
|
|
827
|
+
catch {
|
|
828
|
+
// Ignore rollback failures so the original error is preserved.
|
|
829
|
+
}
|
|
830
|
+
throw err;
|
|
831
|
+
}
|
|
832
|
+
}
|
|
695
833
|
// ── task_history table helpers ───────────────────────────────────────────────
|
|
696
834
|
/**
|
|
697
835
|
* Upsert a task history row.
|
package/dist/indexer/db/db.js
CHANGED
|
@@ -28,7 +28,7 @@ export function openDatabase(dbPath, options) {
|
|
|
28
28
|
}
|
|
29
29
|
const db = openSqlite(resolvedPath);
|
|
30
30
|
db.exec("PRAGMA journal_mode = WAL");
|
|
31
|
-
db.exec("PRAGMA busy_timeout =
|
|
31
|
+
db.exec("PRAGMA busy_timeout = 30000");
|
|
32
32
|
db.exec("PRAGMA foreign_keys = ON");
|
|
33
33
|
// Try to load sqlite-vec extension
|
|
34
34
|
loadVecExtension(db);
|
|
@@ -69,7 +69,7 @@ export function openExistingDatabase(dbPath) {
|
|
|
69
69
|
const resolvedPath = dbPath ?? getDbPath();
|
|
70
70
|
const db = openSqlite(resolvedPath);
|
|
71
71
|
db.exec("PRAGMA journal_mode = WAL");
|
|
72
|
-
db.exec("PRAGMA busy_timeout =
|
|
72
|
+
db.exec("PRAGMA busy_timeout = 30000");
|
|
73
73
|
db.exec("PRAGMA foreign_keys = ON");
|
|
74
74
|
// Existing-DB callers must not mutate schema or embedding metadata on open,
|
|
75
75
|
// but some paths still need write access to usage_events and other tables.
|
|
@@ -1262,6 +1262,19 @@ export function getEntryIdByFilePath(db, filePath) {
|
|
|
1262
1262
|
const row = db.prepare("SELECT id FROM entries WHERE file_path = ? LIMIT 1").get(filePath);
|
|
1263
1263
|
return row?.id;
|
|
1264
1264
|
}
|
|
1265
|
+
/**
|
|
1266
|
+
* Set of every non-empty `entries.file_path` currently indexed (across all
|
|
1267
|
+
* stashes/sources). Used by staleness detection to spot files that exist on
|
|
1268
|
+
* disk but were never indexed — a clock-independent signal for newly-added
|
|
1269
|
+
* assets that an mtime-vs-builtAt comparison can miss when the two clocks
|
|
1270
|
+
* (filesystem vs wall-clock) are skewed within the same millisecond.
|
|
1271
|
+
*/
|
|
1272
|
+
export function getIndexedFilePaths(db) {
|
|
1273
|
+
const rows = db
|
|
1274
|
+
.prepare("SELECT DISTINCT file_path FROM entries WHERE file_path IS NOT NULL AND file_path <> ''")
|
|
1275
|
+
.all();
|
|
1276
|
+
return new Set(rows.map((r) => r.file_path));
|
|
1277
|
+
}
|
|
1265
1278
|
/**
|
|
1266
1279
|
* Resolve a single `entries.file_path` by primary key, or `undefined` if no
|
|
1267
1280
|
* row matches.
|
|
@@ -1477,25 +1490,98 @@ export function computeBodyHash(body) {
|
|
|
1477
1490
|
return sha256Hex(body);
|
|
1478
1491
|
}
|
|
1479
1492
|
/**
|
|
1480
|
-
*
|
|
1481
|
-
*
|
|
1482
|
-
*
|
|
1493
|
+
* Reduce a ref to its bare `type:name` form, dropping any `origin//` prefix.
|
|
1494
|
+
*
|
|
1495
|
+
* usage_events store entry_ref inconsistently: search/show writers persist
|
|
1496
|
+
* whatever ref the result carried, which is sometimes stash-prefixed
|
|
1497
|
+
* (`origin//type:name`) and sometimes bare (`type:name`). Retrieval counting
|
|
1498
|
+
* keys on the bare form so both spellings of the same asset collapse together.
|
|
1499
|
+
*
|
|
1500
|
+
* Returns the bare form, or the original string when it cannot be parsed (best
|
|
1501
|
+
* effort — never throws so a malformed stored ref can't break counting).
|
|
1502
|
+
*/
|
|
1503
|
+
function bareRef(ref) {
|
|
1504
|
+
try {
|
|
1505
|
+
const parsed = parseAssetRef(ref);
|
|
1506
|
+
return `${parsed.type}:${parsed.name}`;
|
|
1507
|
+
}
|
|
1508
|
+
catch {
|
|
1509
|
+
return ref;
|
|
1510
|
+
}
|
|
1511
|
+
}
|
|
1512
|
+
/**
|
|
1513
|
+
* Count retrieval events for the given entry refs.
|
|
1514
|
+
*
|
|
1515
|
+
* Counts `search`, `show`, and `curate` usage events. Returns a
|
|
1516
|
+
* Map<inputRef, count> keyed by the *input* ref strings (only those with at
|
|
1517
|
+
* least one matching event appear). Used by the improve loop to find
|
|
1518
|
+
* high-retrieval assets without feedback.
|
|
1519
|
+
*
|
|
1520
|
+
* Matching is normalization-aware: each stored `entry_ref` is reduced to its
|
|
1521
|
+
* bare `type:name` form before comparison, so a stash-prefixed stored ref
|
|
1522
|
+
* (`origin//type:name`) still matches a bare input ref (`type:name`) and vice
|
|
1523
|
+
* versa. Previously the raw `entry_ref IN (...)` comparison silently dropped
|
|
1524
|
+
* roughly half the signal whenever the two spellings disagreed.
|
|
1525
|
+
*
|
|
1526
|
+
* `curate` events are included: their per-item rows are written with
|
|
1527
|
+
* entry_ref populated (see logCurateEvent), so curation is a real retrieval
|
|
1528
|
+
* signal here. Legacy summary-only curate rows with a NULL entry_ref simply
|
|
1529
|
+
* contribute nothing.
|
|
1483
1530
|
*/
|
|
1484
1531
|
export function getRetrievalCounts(db, refs) {
|
|
1485
1532
|
if (refs.length === 0)
|
|
1486
1533
|
return new Map();
|
|
1487
|
-
|
|
1534
|
+
// Map each distinct bare form back to the input ref(s) that produced it so we
|
|
1535
|
+
// can re-key DB results (grouped by bare form) onto the caller's ref strings.
|
|
1536
|
+
const bareToInputs = new Map();
|
|
1537
|
+
for (const ref of refs) {
|
|
1538
|
+
const bare = bareRef(ref);
|
|
1539
|
+
const existing = bareToInputs.get(bare);
|
|
1540
|
+
if (existing)
|
|
1541
|
+
existing.push(ref);
|
|
1542
|
+
else
|
|
1543
|
+
bareToInputs.set(bare, [ref]);
|
|
1544
|
+
}
|
|
1545
|
+
const bareForms = [...bareToInputs.keys()];
|
|
1546
|
+
// Accumulate counts per bare form across chunks before re-keying.
|
|
1547
|
+
const countsByBare = new Map();
|
|
1488
1548
|
// Chunk to stay within SQLITE_MAX_VARIABLE_NUMBER (same pattern as getUtilityScoresByIds).
|
|
1489
|
-
for (let i = 0; i <
|
|
1490
|
-
const chunk =
|
|
1549
|
+
for (let i = 0; i < bareForms.length; i += SQLITE_CHUNK_SIZE) {
|
|
1550
|
+
const chunk = bareForms.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
1491
1551
|
const placeholders = chunk.map(() => "?").join(", ");
|
|
1552
|
+
// Normalize the stored entry_ref to its bare form inside SQL by stripping
|
|
1553
|
+
// everything up to and including the last `//` separator. SQLite has no
|
|
1554
|
+
// rfind, but stored origins never themselves contain `//`, so a stash ref
|
|
1555
|
+
// has exactly one `//` and `substr(... instr ...)` is exact; bare refs have
|
|
1556
|
+
// no `//` and pass through unchanged.
|
|
1492
1557
|
const rows = db
|
|
1493
|
-
.prepare(`SELECT
|
|
1494
|
-
|
|
1495
|
-
|
|
1558
|
+
.prepare(`SELECT
|
|
1559
|
+
CASE
|
|
1560
|
+
WHEN instr(entry_ref, '//') > 0
|
|
1561
|
+
THEN substr(entry_ref, instr(entry_ref, '//') + 2)
|
|
1562
|
+
ELSE entry_ref
|
|
1563
|
+
END AS bare_ref,
|
|
1564
|
+
COUNT(*) AS cnt
|
|
1565
|
+
FROM usage_events
|
|
1566
|
+
WHERE event_type IN ('search','show','curate')
|
|
1567
|
+
AND entry_ref IS NOT NULL
|
|
1568
|
+
AND CASE
|
|
1569
|
+
WHEN instr(entry_ref, '//') > 0
|
|
1570
|
+
THEN substr(entry_ref, instr(entry_ref, '//') + 2)
|
|
1571
|
+
ELSE entry_ref
|
|
1572
|
+
END IN (${placeholders})
|
|
1573
|
+
GROUP BY bare_ref`)
|
|
1496
1574
|
.all(...chunk);
|
|
1497
|
-
for (const r of rows)
|
|
1498
|
-
|
|
1575
|
+
for (const r of rows) {
|
|
1576
|
+
countsByBare.set(r.bare_ref, (countsByBare.get(r.bare_ref) ?? 0) + r.cnt);
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
// Re-key bare-form counts onto every input ref that maps to that bare form.
|
|
1580
|
+
const result = new Map();
|
|
1581
|
+
for (const [bare, count] of countsByBare) {
|
|
1582
|
+
for (const input of bareToInputs.get(bare) ?? []) {
|
|
1583
|
+
result.set(input, count);
|
|
1584
|
+
}
|
|
1499
1585
|
}
|
|
1500
1586
|
return result;
|
|
1501
1587
|
}
|
|
@@ -11,12 +11,14 @@
|
|
|
11
11
|
* `searchLocal()` and `show.ts`, centralizing the "indexed yet?" gap handling
|
|
12
12
|
* behind a single entry point.
|
|
13
13
|
*/
|
|
14
|
+
import { spawn } from "node:child_process";
|
|
14
15
|
import fs from "node:fs";
|
|
15
16
|
import path from "node:path";
|
|
16
17
|
import { ASSET_SPECS, TYPE_DIRS } from "../core/asset/asset-spec.js";
|
|
17
|
-
import { getDbPath } from "../core/paths.js";
|
|
18
|
+
import { getDataDir, getDbPath } from "../core/paths.js";
|
|
18
19
|
import { warn } from "../core/warn.js";
|
|
19
|
-
import { closeDatabase, getEntryCount, getMeta, openExistingDatabase } from "./db/db.js";
|
|
20
|
+
import { closeDatabase, getEntryCount, getIndexedFilePaths, getMeta, openExistingDatabase } from "./db/db.js";
|
|
21
|
+
import { acquireIndexWriterLease, handoffIndexWriterLeaseToPid } from "./index-writer-lock.js";
|
|
20
22
|
function getIndexableFiles(root, spec) {
|
|
21
23
|
if (!fs.existsSync(root))
|
|
22
24
|
return [];
|
|
@@ -52,16 +54,34 @@ function getIndexableFiles(root, spec) {
|
|
|
52
54
|
}
|
|
53
55
|
return files;
|
|
54
56
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Whether any indexable file under `stashDir` is newer than the last build, or
|
|
59
|
+
* has never been indexed at all.
|
|
60
|
+
*
|
|
61
|
+
* Two independent signals, because neither alone is sufficient:
|
|
62
|
+
* 1. **mtime > builtAt** — catches in-place *edits* of already-indexed files.
|
|
63
|
+
* 2. **path not in `indexedPaths`** — catches *newly added* files. This is
|
|
64
|
+
* clock-independent on purpose: a freshly-written file can have a
|
|
65
|
+
* filesystem mtime that compares as *older* than the wall-clock `builtAt`
|
|
66
|
+
* (the two clocks are not perfectly synchronized and `builtAt` is
|
|
67
|
+
* millisecond-truncated), so the mtime test alone silently misses
|
|
68
|
+
* additions made within ~a millisecond of the previous build.
|
|
69
|
+
*
|
|
70
|
+
* `getIndexableFiles` applies each asset type's own relevance filter, so
|
|
71
|
+
* non-indexed companion files (e.g. `package.json` next to a knowledge doc) are
|
|
72
|
+
* never considered and do not produce false "new file" positives.
|
|
73
|
+
*/
|
|
74
|
+
function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths) {
|
|
75
|
+
const builtAtMs = builtAt ? new Date(builtAt).getTime() : Number.NaN;
|
|
76
|
+
const builtAtUsable = Number.isFinite(builtAtMs);
|
|
61
77
|
for (const [type, spec] of Object.entries(ASSET_SPECS)) {
|
|
62
78
|
const typeRoot = path.join(stashDir, TYPE_DIRS[type] ?? spec.stashDir);
|
|
63
79
|
const files = getIndexableFiles(typeRoot, spec);
|
|
64
80
|
for (const file of files) {
|
|
81
|
+
if (!indexedPaths.has(file))
|
|
82
|
+
return true;
|
|
83
|
+
if (!builtAtUsable)
|
|
84
|
+
return true;
|
|
65
85
|
try {
|
|
66
86
|
if (fs.statSync(file).mtimeMs > builtAtMs)
|
|
67
87
|
return true;
|
|
@@ -89,7 +109,7 @@ export function isIndexStale(stashDir) {
|
|
|
89
109
|
if (entryCount === 0)
|
|
90
110
|
return true;
|
|
91
111
|
const builtAt = getMeta(db, "builtAt");
|
|
92
|
-
if (hasNewerIndexableFiles(stashDir, builtAt))
|
|
112
|
+
if (hasNewerIndexableFiles(stashDir, builtAt, getIndexedFilePaths(db)))
|
|
93
113
|
return true;
|
|
94
114
|
const storedStashDir = getMeta(db, "stashDir");
|
|
95
115
|
if (storedStashDir !== stashDir) {
|
|
@@ -114,16 +134,84 @@ export function isIndexStale(stashDir) {
|
|
|
114
134
|
}
|
|
115
135
|
}
|
|
116
136
|
/**
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
137
|
+
* Whether the existing index can serve queries for `stashDir` *right now* —
|
|
138
|
+
* i.e. the DB file exists, the `entries` table holds rows, and those rows were
|
|
139
|
+
* built for this stash (it is the stored primary stash or appears in the
|
|
140
|
+
* stored `stashDirs` set). When this is true the index is at worst
|
|
141
|
+
* content-stale, so the `#607` background-reindex optimization is safe: the
|
|
142
|
+
* caller gets slightly-stale-but-relevant results immediately. When it is
|
|
143
|
+
* false the existing index has nothing relevant to return (no DB, no `entries`
|
|
144
|
+
* table, zero rows, or built for a different stash), so a background reindex
|
|
145
|
+
* would leave the caller empty until the next read — those cases must rebuild
|
|
146
|
+
* inline.
|
|
123
147
|
*/
|
|
124
|
-
|
|
125
|
-
|
|
148
|
+
function indexCanServeStash(stashDir) {
|
|
149
|
+
const dbPath = getDbPath();
|
|
150
|
+
if (!fs.existsSync(dbPath))
|
|
151
|
+
return false;
|
|
152
|
+
let db;
|
|
153
|
+
try {
|
|
154
|
+
db = openExistingDatabase(dbPath);
|
|
155
|
+
if (getEntryCount(db) === 0)
|
|
156
|
+
return false;
|
|
157
|
+
const storedStashDir = getMeta(db, "stashDir");
|
|
158
|
+
if (storedStashDir === stashDir)
|
|
159
|
+
return true;
|
|
160
|
+
try {
|
|
161
|
+
const storedDirs = JSON.parse(getMeta(db, "stashDirs") ?? "[]");
|
|
162
|
+
return storedDirs.includes(stashDir);
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
// No `entries` table (or otherwise unreadable) — cannot serve.
|
|
126
170
|
return false;
|
|
171
|
+
}
|
|
172
|
+
finally {
|
|
173
|
+
if (db)
|
|
174
|
+
closeDatabase(db);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Spawn a background `akm index` process. Non-blocking — returns immediately.
|
|
179
|
+
* Background callers share the same global index-writer lease as foreground
|
|
180
|
+
* writers, so stale-read-triggered auto-index attempts coalesce safely.
|
|
181
|
+
*/
|
|
182
|
+
async function spawnBackgroundReindex(_stashDir) {
|
|
183
|
+
const dataDir = getDataDir();
|
|
184
|
+
const logFile = path.join(dataDir, "logs", "index-background.log");
|
|
185
|
+
fs.mkdirSync(path.dirname(logFile), { recursive: true });
|
|
186
|
+
const lease = await acquireIndexWriterLease({ mode: "try", purpose: "background-reindex-spawn" });
|
|
187
|
+
if (!lease)
|
|
188
|
+
return;
|
|
189
|
+
const akmBin = process.argv[0];
|
|
190
|
+
const akmScript = process.argv[1];
|
|
191
|
+
try {
|
|
192
|
+
const child = spawn(akmBin, [akmScript, "index", "--background"], {
|
|
193
|
+
detached: true,
|
|
194
|
+
stdio: ["ignore", fs.openSync(logFile, "a"), fs.openSync(logFile, "a")],
|
|
195
|
+
env: { ...process.env },
|
|
196
|
+
});
|
|
197
|
+
if (!child.pid) {
|
|
198
|
+
lease.release();
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
handoffIndexWriterLeaseToPid(lease, child.pid, "background-reindex");
|
|
202
|
+
try {
|
|
203
|
+
child.unref();
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
// ignore
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
catch (error) {
|
|
210
|
+
lease.release();
|
|
211
|
+
throw error;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
async function runInlineReindex(stashDir) {
|
|
127
215
|
try {
|
|
128
216
|
const { akmIndex } = await import("./indexer.js");
|
|
129
217
|
await akmIndex({ stashDir });
|
|
@@ -134,3 +222,50 @@ export async function ensureIndex(stashDir) {
|
|
|
134
222
|
return true;
|
|
135
223
|
}
|
|
136
224
|
}
|
|
225
|
+
/**
|
|
226
|
+
* Ensure the local index exists and is fresh enough for the caller's needs.
|
|
227
|
+
*
|
|
228
|
+
* Default mode is `background`, which preserves the low-latency behavior used
|
|
229
|
+
* by read paths (`search`, `show`, `feedback`): when a populated index is
|
|
230
|
+
* merely stale, spawn a detached reindex and proceed against the existing
|
|
231
|
+
* index. When the index is entirely absent (no DB / no `entries` table / zero
|
|
232
|
+
* rows) the rebuild runs inline regardless of mode, since there is nothing to
|
|
233
|
+
* proceed against.
|
|
234
|
+
*
|
|
235
|
+
* `mode: "blocking"` waits for the rebuild to finish before returning. Use
|
|
236
|
+
* this for callers like `improve` whose planning logic depends on a populated
|
|
237
|
+
* `entries` table in the same process.
|
|
238
|
+
*
|
|
239
|
+
* Returns `true` if an index run was attempted.
|
|
240
|
+
*/
|
|
241
|
+
export async function ensureIndex(stashDir, options = {}) {
|
|
242
|
+
if (!isIndexStale(stashDir))
|
|
243
|
+
return false;
|
|
244
|
+
// Blocking when explicitly requested, or whenever the existing index cannot
|
|
245
|
+
// serve this stash (absent DB, no `entries` table, zero rows, or built for a
|
|
246
|
+
// different stash): a background reindex returns immediately and would leave
|
|
247
|
+
// a first-time caller (search, curate, wiki, show, feedback) with empty
|
|
248
|
+
// results. Building inline is a one-off cost; a populated index for this
|
|
249
|
+
// stash that is merely content-stale still refreshes in the background.
|
|
250
|
+
if (options.mode === "blocking" || !indexCanServeStash(stashDir)) {
|
|
251
|
+
return runInlineReindex(stashDir);
|
|
252
|
+
}
|
|
253
|
+
// The background path re-invokes the akm CLI as a detached child via
|
|
254
|
+
// `process.argv[1]`. That is only the akm entrypoint when THIS process is the
|
|
255
|
+
// akm CLI itself — which the CLI startup block signals with AKM_CLI_ENTRY=1.
|
|
256
|
+
// In any other host (the in-process test runner, a library embedding akm),
|
|
257
|
+
// argv[1] points at the host (e.g. the test runner), so spawning it would
|
|
258
|
+
// launch the wrong program and orphan it. Build inline there instead — same
|
|
259
|
+
// resulting index, no detached process.
|
|
260
|
+
if (process.env.AKM_CLI_ENTRY !== "1") {
|
|
261
|
+
return runInlineReindex(stashDir);
|
|
262
|
+
}
|
|
263
|
+
try {
|
|
264
|
+
await spawnBackgroundReindex(stashDir);
|
|
265
|
+
return true;
|
|
266
|
+
}
|
|
267
|
+
catch (error) {
|
|
268
|
+
warn("Background reindex spawn failed, proceeding with existing index:", error instanceof Error ? error.message : String(error));
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
271
|
+
}
|