akm-cli 0.9.2-alpha.1 → 0.9.2-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -28
- package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +3 -4
- package/dist/assets/stash-skeleton/facts/conventions/organization.md +1 -3
- package/dist/commands/improve/collapse-detector.js +3 -4
- package/dist/commands/improve/extract-prompt.js +64 -22
- package/dist/commands/improve/extract.js +122 -53
- package/dist/commands/read/curate.js +43 -22
- package/dist/commands/sources/bundle-cli.js +1 -1
- package/dist/commands/sources/installed-stashes.js +67 -26
- package/dist/core/adapter/adapters/akm-adapter.js +2 -1
- package/dist/core/config/config.js +2 -6
- package/dist/core/config/schema/index-config.js +0 -27
- package/dist/indexer/index-written-assets.js +17 -9
- package/dist/indexer/indexer.js +32 -214
- package/dist/indexer/materialize-embeddings.js +155 -0
- package/dist/indexer/passes/metadata.js +263 -118
- package/dist/indexer/scan/doc-to-entry.js +0 -1
- package/dist/indexer/search/db-search.js +58 -28
- package/dist/indexer/search/fts-query.js +40 -40
- package/dist/indexer/search/ranking.js +36 -1
- package/dist/indexer/search/search-attribution.js +3 -1
- package/dist/indexer/search/search-fields.js +23 -14
- package/dist/output/text/command-format.js +3 -1
- package/dist/scripts/akm-migrate-node.js +12892 -12731
- package/dist/scripts/akm-migrate.js +12892 -12731
- package/dist/storage/repositories/index-entries-repository.js +40 -26
- package/dist/storage/repositories/index-entry-schema.js +1 -1
- package/dist/storage/repositories/index-fts-repository.js +56 -63
- package/dist/storage/repositories/index-schema.js +4 -9
- package/dist/storage/repositories/index-vec-repository.js +55 -6
- package/docs/reference/cli.md +4 -4
- package/docs/reference/configuration.md +6 -9
- package/package.json +1 -1
- package/schemas/akm-config.json +0 -8
|
@@ -18,21 +18,22 @@ import { getStateDbPath, withStateDb } from "../../core/state-db.js";
|
|
|
18
18
|
import { warn } from "../../core/warn.js";
|
|
19
19
|
import { buildSearchText } from "../../indexer/search/search-fields.js";
|
|
20
20
|
import { ENTRY_COLUMNS, rowToIndexedEntry } from "./index-entry-mapper.js";
|
|
21
|
+
import { deleteFtsEntries, replaceFtsEntry } from "./index-fts-repository.js";
|
|
21
22
|
import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
|
|
22
23
|
import { deleteEntryVectors, isVecAvailable } from "./index-vec-repository.js";
|
|
23
24
|
// ── Entry operations ────────────────────────────────────────────────────────
|
|
24
25
|
/**
|
|
25
|
-
* Insert or update
|
|
26
|
+
* Insert or update one canonical entry and all synchronously derived search
|
|
27
|
+
* state. Returns the stable row id.
|
|
26
28
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
29
|
+
* The entries row, FTS projection, and stale-vector invalidation commit as one
|
|
30
|
+
* SQLite transaction. Callers therefore cannot publish an entry and forget a
|
|
31
|
+
* second FTS maintenance step.
|
|
30
32
|
*/
|
|
31
33
|
export function upsertEntry(db, filePath, entry, searchText, provenance, contentHash) {
|
|
32
34
|
// Hot path during indexing — cache prepared statements per database
|
|
33
35
|
// connection so we don't pay the SQL parse/compile cost on every call.
|
|
34
36
|
const stmts = getUpsertStmts(db);
|
|
35
|
-
const previous = stmts.findByItemRef.get(provenance.itemRef);
|
|
36
37
|
// Phase 5A / Advantage D5: surface derived memory parent ref into the
|
|
37
38
|
// dedicated `derived_from` column so retrieval-time lookup (parent→child)
|
|
38
39
|
// does not have to scan + JSON-decode every memory row.
|
|
@@ -40,18 +41,20 @@ export function upsertEntry(db, filePath, entry, searchText, provenance, content
|
|
|
40
41
|
// `content_hash` is optional on the LLM-enrichment re-upsert; a missing hash
|
|
41
42
|
// preserves the scan writer's current value.
|
|
42
43
|
const apply = () => {
|
|
44
|
+
const previous = stmts.findByItemRef.get(provenance.itemRef);
|
|
43
45
|
const result = stmts.upsert.get(provenance.itemRef, provenance.bundleId, provenance.componentId, provenance.conceptId, provenance.adapterId, entry.type, filePath, contentHash ?? null, JSON.stringify(entry), searchText, derivedFrom);
|
|
44
46
|
if (!result)
|
|
45
47
|
throw new Error("upsertEntry: item_ref not found after upsert");
|
|
46
48
|
if (previous?.id === result.id && previous.search_text !== searchText)
|
|
47
49
|
deleteEntryVectors(db, result.id);
|
|
48
|
-
|
|
49
|
-
// only revisits entries that actually changed. INSERT OR IGNORE is
|
|
50
|
-
// idempotent across multiple upserts of the same row.
|
|
51
|
-
stmts.markDirty.run(result.id);
|
|
50
|
+
replaceFtsEntry(db, result.id, entry);
|
|
52
51
|
return result.id;
|
|
53
52
|
};
|
|
54
|
-
|
|
53
|
+
// Always enter the driver's transaction wrapper. Both supported SQLite
|
|
54
|
+
// drivers lower a transaction opened inside another transaction to a
|
|
55
|
+
// savepoint, so a caller that catches this mutation's error cannot commit a
|
|
56
|
+
// partial entries row through its outer transaction.
|
|
57
|
+
return db.transaction(apply)();
|
|
55
58
|
}
|
|
56
59
|
const upsertStmtsByDb = new WeakMap();
|
|
57
60
|
// item_ref is the sole durable conflict target. `content_hash` COALESCEs so a
|
|
@@ -84,7 +87,6 @@ function getUpsertStmts(db) {
|
|
|
84
87
|
ON CONFLICT(item_ref) DO UPDATE ${UPSERT_SET_CLAUSE}
|
|
85
88
|
RETURNING id
|
|
86
89
|
`),
|
|
87
|
-
markDirty: db.prepare("INSERT OR IGNORE INTO entries_fts_dirty (entry_id) VALUES (?)"),
|
|
88
90
|
findByItemRef: db.prepare("SELECT id, search_text FROM entries WHERE item_ref = ?"),
|
|
89
91
|
};
|
|
90
92
|
upsertStmtsByDb.set(db, stmts);
|
|
@@ -169,8 +171,8 @@ export function getBaseBeliefStatesForDerivedTwins(db, twinIds) {
|
|
|
169
171
|
* `asset_ref` TEXT and are re-keyed separately by `akm mv` — see
|
|
170
172
|
* the state rekey helper.) `document_json.name` (and `filename`, when
|
|
171
173
|
* present) is patched and `search_text` rebuilt so search reflects the new
|
|
172
|
-
* name
|
|
173
|
-
*
|
|
174
|
+
* name. Its FTS projection and stale vector are updated in the same
|
|
175
|
+
* transaction as the canonical identity.
|
|
174
176
|
*
|
|
175
177
|
* Bundle-qualified `usage_events.entry_ref` rows for the old conceptId are
|
|
176
178
|
* rewritten to the new item ref. Without this, events keep the old
|
|
@@ -209,6 +211,7 @@ export function rekeyEntryInPlace(db, opts) {
|
|
|
209
211
|
// the utility history survives; the next full index heals the JSON.
|
|
210
212
|
let documentJson = row.document_json;
|
|
211
213
|
let searchText = row.search_text;
|
|
214
|
+
let document;
|
|
212
215
|
try {
|
|
213
216
|
const entry = JSON.parse(row.document_json);
|
|
214
217
|
entry.name = opts.newName;
|
|
@@ -218,6 +221,7 @@ export function rekeyEntryInPlace(db, opts) {
|
|
|
218
221
|
entry.derivedFrom = opts.newDerivedFrom;
|
|
219
222
|
documentJson = JSON.stringify(entry);
|
|
220
223
|
searchText = buildSearchText(entry);
|
|
224
|
+
document = entry;
|
|
221
225
|
}
|
|
222
226
|
catch {
|
|
223
227
|
/* corrupt document_json — identity/path-only re-key */
|
|
@@ -242,7 +246,12 @@ export function rekeyEntryInPlace(db, opts) {
|
|
|
242
246
|
if (opts.newDerivedFrom !== undefined) {
|
|
243
247
|
db.prepare("UPDATE entries SET derived_from = ? WHERE id = ?").run(opts.newDerivedFrom, row.id);
|
|
244
248
|
}
|
|
245
|
-
|
|
249
|
+
if (row.search_text !== searchText)
|
|
250
|
+
deleteEntryVectors(db, row.id);
|
|
251
|
+
if (document)
|
|
252
|
+
replaceFtsEntry(db, row.id, document);
|
|
253
|
+
else
|
|
254
|
+
deleteFtsEntries(db, [row.id]);
|
|
246
255
|
})();
|
|
247
256
|
// Re-point usage history at the new ref. Chunk-8 WI-8.3: usage_events lives in
|
|
248
257
|
// state.db now, so this is a SEPARATE cross-DB transaction (best-effort — the
|
|
@@ -358,6 +367,17 @@ export function deleteEntriesByBundle(db, bundleId) {
|
|
|
358
367
|
deleteEntryRows(db, rows);
|
|
359
368
|
})();
|
|
360
369
|
}
|
|
370
|
+
/**
|
|
371
|
+
* Delete the complete regenerable entry generation through the same child-row
|
|
372
|
+
* authority used by targeted deletes. The caller may retain cross-database
|
|
373
|
+
* usage events so the finalize pass can relink them to the new row ids.
|
|
374
|
+
*/
|
|
375
|
+
export function deleteAllEntries(db, options = {}) {
|
|
376
|
+
return db.transaction(() => {
|
|
377
|
+
const rows = db.prepare("SELECT id FROM entries").all();
|
|
378
|
+
return deleteEntryRows(db, rows, options);
|
|
379
|
+
})();
|
|
380
|
+
}
|
|
361
381
|
/**
|
|
362
382
|
* Diff-persist orphan delete: remove every entry under `dirPath` whose durable
|
|
363
383
|
* `item_ref` is not in `keepRefs`.
|
|
@@ -382,15 +402,9 @@ function deleteRelatedRows(db, ids, options = {}) {
|
|
|
382
402
|
return;
|
|
383
403
|
const numericIds = ids.map((r) => r.id);
|
|
384
404
|
const vecAvail = isVecAvailable(db);
|
|
385
|
-
//
|
|
386
|
-
//
|
|
387
|
-
|
|
388
|
-
for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
|
|
389
|
-
const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
390
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
391
|
-
bestEffort(() => db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk), "fts table may not exist on a brand-new db");
|
|
392
|
-
bestEffort(() => db.prepare(`DELETE FROM entries_fts_dirty WHERE entry_id IN (${placeholders})`).run(...chunk), "fts dirty table is created lazily by upsertEntry");
|
|
393
|
-
}
|
|
405
|
+
// FTS is part of the canonical mutation boundary, not a caller-maintained
|
|
406
|
+
// dirty queue. Delete it before the parent row inside this transaction.
|
|
407
|
+
deleteFtsEntries(db, numericIds);
|
|
394
408
|
// Process in chunks to stay within SQLITE_MAX_VARIABLE_NUMBER
|
|
395
409
|
for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
|
|
396
410
|
const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
@@ -453,10 +467,10 @@ export function deleteUsageEventsByEntryIds(entryIds) {
|
|
|
453
467
|
}
|
|
454
468
|
/**
|
|
455
469
|
* Delete entries by their primary key IDs, along with all related rows
|
|
456
|
-
* (embeddings, entries_vec, entries_fts,
|
|
470
|
+
* (embeddings, entries_vec, entries_fts, utility scores, usage_events).
|
|
457
471
|
*
|
|
458
|
-
* Used by
|
|
459
|
-
* no longer exist
|
|
472
|
+
* Used by explicit `--clean` reconciliation before embeddings and final
|
|
473
|
+
* verification to remove stale entries whose source files no longer exist.
|
|
460
474
|
*/
|
|
461
475
|
export function deleteEntriesByIds(db, ids) {
|
|
462
476
|
if (ids.length === 0)
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* serving preflights, and read-only evaluator tooling cannot drift into
|
|
11
11
|
* separate definitions of "current".
|
|
12
12
|
*/
|
|
13
|
-
export const CANONICAL_INDEX_DB_VERSION =
|
|
13
|
+
export const CANONICAL_INDEX_DB_VERSION = 22;
|
|
14
14
|
export const CANONICAL_ENTRY_SCHEMA_SQL = `
|
|
15
15
|
CREATE TABLE IF NOT EXISTS entries (
|
|
16
16
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -2,33 +2,63 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
/**
|
|
5
|
-
* `index.db` FTS5 search +
|
|
5
|
+
* `index.db` FTS5 search + materialization repository.
|
|
6
6
|
*
|
|
7
|
-
* Owns the `entries_fts` full-text query path and the
|
|
8
|
-
* rebuild.
|
|
7
|
+
* Owns the `entries_fts` full-text query path, per-entry projections, and the
|
|
8
|
+
* explicit full recovery rebuild.
|
|
9
9
|
*/
|
|
10
10
|
import { warn } from "../../core/warn.js";
|
|
11
|
-
import {
|
|
11
|
+
import { buildLexicalQueryPlan } from "../../indexer/search/fts-query.js";
|
|
12
12
|
import { buildSearchFields } from "../../indexer/search/search-fields.js";
|
|
13
13
|
import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
|
|
14
|
+
const INSERT_FTS_SQL = "INSERT INTO entries_fts (entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?)";
|
|
15
|
+
const ftsMutationStatementsByDb = new WeakMap();
|
|
16
|
+
function getFtsMutationStatements(db) {
|
|
17
|
+
const existing = ftsMutationStatementsByDb.get(db);
|
|
18
|
+
if (existing)
|
|
19
|
+
return existing;
|
|
20
|
+
const statements = {
|
|
21
|
+
deleteOne: db.prepare("DELETE FROM entries_fts WHERE entry_id = ?"),
|
|
22
|
+
insert: db.prepare(INSERT_FTS_SQL),
|
|
23
|
+
};
|
|
24
|
+
ftsMutationStatementsByDb.set(db, statements);
|
|
25
|
+
return statements;
|
|
26
|
+
}
|
|
27
|
+
/** Replace one entry's derived FTS projection inside the caller's transaction. */
|
|
28
|
+
export function replaceFtsEntry(db, entryId, entry) {
|
|
29
|
+
const fields = buildSearchFields(entry);
|
|
30
|
+
const statements = getFtsMutationStatements(db);
|
|
31
|
+
statements.deleteOne.run(entryId);
|
|
32
|
+
statements.insert.run(entryId, fields.name, fields.description, fields.tags, fields.hints, fields.content);
|
|
33
|
+
}
|
|
34
|
+
/** Delete derived FTS projections for canonical entries that are being removed. */
|
|
35
|
+
export function deleteFtsEntries(db, entryIds) {
|
|
36
|
+
for (let i = 0; i < entryIds.length; i += SQLITE_CHUNK_SIZE) {
|
|
37
|
+
const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
38
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
39
|
+
db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
14
42
|
export function searchFts(db, query, limit, entryType, excludeTypes) {
|
|
15
|
-
const
|
|
16
|
-
if (!
|
|
43
|
+
const plan = buildLexicalQueryPlan(query);
|
|
44
|
+
if (!plan.exact)
|
|
17
45
|
return [];
|
|
18
46
|
// Try the exact AND query first
|
|
19
|
-
const exactResults = runFtsQuery(db,
|
|
47
|
+
const exactResults = runFtsQuery(db, plan.exact, "exact", limit, entryType, excludeTypes);
|
|
20
48
|
if (exactResults.length > 0)
|
|
21
49
|
return exactResults;
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
50
|
+
if (plan.exactPrefix) {
|
|
51
|
+
const prefixResults = runFtsQuery(db, plan.exactPrefix, "prefix", limit, entryType, excludeTypes);
|
|
52
|
+
if (prefixResults.length > 0)
|
|
53
|
+
return prefixResults;
|
|
54
|
+
}
|
|
55
|
+
// One measured relaxation only after both conjunctive forms miss. This is
|
|
56
|
+
// still the same FTS table, BM25 weights, candidate collection, and
|
|
57
|
+
// downstream ranker — merely an OR candidate query for sentence-shaped
|
|
58
|
+
// input whose filler terms prevented a strict hit.
|
|
59
|
+
return plan.relaxed ? runFtsQuery(db, plan.relaxed, "relaxed", limit, entryType, excludeTypes) : [];
|
|
30
60
|
}
|
|
31
|
-
function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
|
|
61
|
+
function runFtsQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludeTypes) {
|
|
32
62
|
// #627 — exclude-type clause. Only applies on the untyped ('any') path; an
|
|
33
63
|
// explicit include filter (entryType) already narrows to a single type, so
|
|
34
64
|
// exclusion is redundant there. An empty list skips the clause entirely
|
|
@@ -84,6 +114,7 @@ function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
|
|
|
84
114
|
bundleId: row.bundleId,
|
|
85
115
|
conceptId: row.conceptId,
|
|
86
116
|
adapterId: row.adapterId,
|
|
117
|
+
lexicalMatch,
|
|
87
118
|
});
|
|
88
119
|
}
|
|
89
120
|
return results;
|
|
@@ -94,49 +125,20 @@ function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
|
|
|
94
125
|
}
|
|
95
126
|
}
|
|
96
127
|
/**
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
* appropriate for `akm index --full` and version-upgrade rebuilds.
|
|
103
|
-
*
|
|
104
|
-
* Both paths are wrapped in a single transaction so the FTS table is never
|
|
105
|
-
* left in a half-rebuilt state.
|
|
128
|
+
* Explicitly rebuild the complete FTS5 projection from canonical entries.
|
|
129
|
+
* Ordinary entry mutations do not call this: `upsertEntry` and the delete
|
|
130
|
+
* operations publish their FTS state in the same transaction as `entries`.
|
|
131
|
+
* This remains a recovery/schema-verification primitive for regenerable
|
|
132
|
+
* `index.db` state.
|
|
106
133
|
*
|
|
107
134
|
* Skipped corrupt-JSON rows are aggregated into one warning instead of
|
|
108
135
|
* spamming stderr per-entry.
|
|
109
136
|
*/
|
|
110
|
-
export function rebuildFts(db
|
|
111
|
-
const incremental = options?.incremental === true;
|
|
137
|
+
export function rebuildFts(db) {
|
|
112
138
|
db.transaction(() => {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
// Then drop the matching rows from entries_fts so the INSERT below
|
|
117
|
-
// doesn't double-up. The dirty list is drained at the end.
|
|
118
|
-
rows = db
|
|
119
|
-
.prepare(`SELECT e.id AS id, e.document_json AS document_json
|
|
120
|
-
FROM entries_fts_dirty d
|
|
121
|
-
JOIN entries e ON e.id = d.entry_id`)
|
|
122
|
-
.all();
|
|
123
|
-
if (rows.length === 0)
|
|
124
|
-
return;
|
|
125
|
-
const ids = rows.map((r) => r.id);
|
|
126
|
-
// Delete only the dirty FTS rows — chunk to stay under
|
|
127
|
-
// SQLITE_MAX_VARIABLE_NUMBER on large dirty queues.
|
|
128
|
-
for (let i = 0; i < ids.length; i += SQLITE_CHUNK_SIZE) {
|
|
129
|
-
const chunk = ids.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
130
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
131
|
-
db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
else {
|
|
135
|
-
// Full path: wipe and re-read every row.
|
|
136
|
-
db.exec("DELETE FROM entries_fts");
|
|
137
|
-
rows = db.prepare("SELECT id, document_json FROM entries").all();
|
|
138
|
-
}
|
|
139
|
-
const insertStmt = db.prepare("INSERT INTO entries_fts (entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?)");
|
|
139
|
+
db.exec("DELETE FROM entries_fts");
|
|
140
|
+
const rows = db.prepare("SELECT id, document_json FROM entries").all();
|
|
141
|
+
const insertStmt = db.prepare(INSERT_FTS_SQL);
|
|
140
142
|
let skipped = 0;
|
|
141
143
|
for (const row of rows) {
|
|
142
144
|
let entry;
|
|
@@ -154,14 +156,5 @@ export function rebuildFts(db, options) {
|
|
|
154
156
|
if (skipped > 0) {
|
|
155
157
|
warn(`[db] rebuildFts: skipped ${skipped} entr${skipped === 1 ? "y" : "ies"} with invalid document_json`);
|
|
156
158
|
}
|
|
157
|
-
// Always drain the dirty queue — both paths converge here. The
|
|
158
|
-
// incremental path drains it because we just consumed every dirty row;
|
|
159
|
-
// the full path drains it because a full rebuild covers everything the
|
|
160
|
-
// dirty list tracks. The table is guaranteed to exist (created by
|
|
161
|
-
// ensureSchema()).
|
|
162
|
-
//
|
|
163
|
-
// BUG-L1: previously the if/else arms ran identical statements — the
|
|
164
|
-
// duplication has been collapsed.
|
|
165
|
-
db.exec("DELETE FROM entries_fts_dirty");
|
|
166
159
|
})();
|
|
167
160
|
}
|
|
@@ -14,6 +14,10 @@ import { isVecAvailable, purgeEmbeddings } from "./index-vec-repository.js";
|
|
|
14
14
|
// entry_type columns. item_ref is the sole conflict key; document_json is the
|
|
15
15
|
// sole stored document projection; bundle provenance and file_path provide the
|
|
16
16
|
// current identity and materialized read path.
|
|
17
|
+
//
|
|
18
|
+
// v21→v22: entry mutations publish FTS synchronously and no dirty queue exists.
|
|
19
|
+
// Discard the old derived generation so stale FTS rows and caller-managed dirty
|
|
20
|
+
// state cannot cross the mutation-authority boundary.
|
|
17
21
|
export const DB_VERSION = CANONICAL_INDEX_DB_VERSION;
|
|
18
22
|
export const EMBEDDING_DIM = 384;
|
|
19
23
|
// #624-P1: graph_files is keyed to (stash_root, file_path, body_hash).
|
|
@@ -301,15 +305,6 @@ export function ensureSchema(db, embeddingDim) {
|
|
|
301
305
|
// child rows are removed when a graph_files row is replaced.
|
|
302
306
|
//
|
|
303
307
|
ensureGraphTables(db);
|
|
304
|
-
// FTS-dirty queue. Created here (not lazily on first upsert) so the
|
|
305
|
-
// per-entry write path doesn't issue a CREATE TABLE IF NOT EXISTS on
|
|
306
|
-
// every call — that DDL would fire thousands of times during a full
|
|
307
|
-
// index. See `markFtsDirty` and `rebuildFts({ incremental: true })`.
|
|
308
|
-
db.exec(`
|
|
309
|
-
CREATE TABLE IF NOT EXISTS entries_fts_dirty (
|
|
310
|
-
entry_id INTEGER PRIMARY KEY
|
|
311
|
-
);
|
|
312
|
-
`);
|
|
313
308
|
// If a generation rebuild could not drop a vec0 table while the extension
|
|
314
309
|
// was unavailable, finish that reset as soon as vec0 can be loaded again.
|
|
315
310
|
if (isVecAvailable(db) && getMeta(db, "vecResetPending") === "1") {
|
|
@@ -12,6 +12,7 @@ import { bestEffort } from "../../core/best-effort.js";
|
|
|
12
12
|
import { warn } from "../../core/warn.js";
|
|
13
13
|
import { cosineSimilarity } from "../../llm/embedders/types.js";
|
|
14
14
|
import { getMeta, setMeta } from "./index-meta-repository.js";
|
|
15
|
+
import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
|
|
15
16
|
// ── sqlite-vec extension ────────────────────────────────────────────────────
|
|
16
17
|
const vecStatus = new WeakMap();
|
|
17
18
|
/**
|
|
@@ -68,6 +69,43 @@ export function isVecFastPathReady(db) {
|
|
|
68
69
|
// flag, so the read path has to verify the table really exists.
|
|
69
70
|
return hasVecTable(db);
|
|
70
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Verify that the vec fast-path table mirrors the complete durable BLOB set.
|
|
74
|
+
*
|
|
75
|
+
* A targeted embedding write preserves the prior readiness decision because
|
|
76
|
+
* its subset cannot prove an older degraded generation is healed. Global
|
|
77
|
+
* materialization uses this aggregate check before promoting the persisted
|
|
78
|
+
* flag; search itself still reads the cheap flag and does not repeat the check
|
|
79
|
+
* per query.
|
|
80
|
+
*/
|
|
81
|
+
export function isVecFastPathComplete(db) {
|
|
82
|
+
if (!isVecAvailable(db) || !hasVecTable(db))
|
|
83
|
+
return false;
|
|
84
|
+
try {
|
|
85
|
+
const missingVecRows = db
|
|
86
|
+
.prepare(`
|
|
87
|
+
SELECT id FROM embeddings
|
|
88
|
+
EXCEPT
|
|
89
|
+
SELECT id FROM entries_vec
|
|
90
|
+
LIMIT 1
|
|
91
|
+
`)
|
|
92
|
+
.all();
|
|
93
|
+
if (missingVecRows.length > 0)
|
|
94
|
+
return false;
|
|
95
|
+
const orphanVecRows = db
|
|
96
|
+
.prepare(`
|
|
97
|
+
SELECT id FROM entries_vec
|
|
98
|
+
EXCEPT
|
|
99
|
+
SELECT id FROM embeddings
|
|
100
|
+
LIMIT 1
|
|
101
|
+
`)
|
|
102
|
+
.all();
|
|
103
|
+
return orphanVecRows.length === 0;
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
71
109
|
const vecTablePresent = new WeakMap();
|
|
72
110
|
/**
|
|
73
111
|
* Whether `entries_vec` exists on this connection, memoized per handle.
|
|
@@ -274,13 +312,24 @@ function searchBlobVec(db, queryEmbedding, k) {
|
|
|
274
312
|
* Return all entries that do not yet have an embedding row.
|
|
275
313
|
* Used by the embedding phase to determine which entries need vectors generated.
|
|
276
314
|
*/
|
|
277
|
-
export function getAllEntriesForEmbedding(db) {
|
|
278
|
-
|
|
279
|
-
.prepare(`
|
|
315
|
+
export function getAllEntriesForEmbedding(db, entryIds) {
|
|
316
|
+
const select = `
|
|
280
317
|
SELECT e.id, e.search_text AS searchText, e.item_ref AS itemRef, e.file_path AS filePath FROM entries e
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
318
|
+
`;
|
|
319
|
+
const missing = "NOT EXISTS (SELECT 1 FROM embeddings b WHERE b.id = e.id)";
|
|
320
|
+
if (entryIds === undefined) {
|
|
321
|
+
return db.prepare(`${select} WHERE ${missing} ORDER BY e.id`).all();
|
|
322
|
+
}
|
|
323
|
+
const targets = [...new Set(entryIds)].sort((left, right) => left - right);
|
|
324
|
+
const rows = [];
|
|
325
|
+
for (let offset = 0; offset < targets.length; offset += SQLITE_CHUNK_SIZE) {
|
|
326
|
+
const chunk = targets.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
327
|
+
if (chunk.length === 0)
|
|
328
|
+
continue;
|
|
329
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
330
|
+
rows.push(...db.prepare(`${select} WHERE e.id IN (${placeholders}) AND ${missing} ORDER BY e.id`).all(...chunk));
|
|
331
|
+
}
|
|
332
|
+
return rows;
|
|
284
333
|
}
|
|
285
334
|
export function getEmbeddingCount(db) {
|
|
286
335
|
const row = db.prepare("SELECT COUNT(*) AS cnt FROM embeddings").get();
|
package/docs/reference/cli.md
CHANGED
|
@@ -869,10 +869,10 @@ akm bundle remove my-provider --yes # Skip the confirmation prompt
|
|
|
869
869
|
|
|
870
870
|
### bundle update
|
|
871
871
|
|
|
872
|
-
Update one
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
872
|
+
Update one bundle, or refresh every configured bundle with `--all`. Git, npm,
|
|
873
|
+
and website candidates are staged and audited before they replace the active
|
|
874
|
+
generation. Filesystem bundles require no hydration; update reconciles their
|
|
875
|
+
current files into the index immediately.
|
|
876
876
|
|
|
877
877
|
```sh
|
|
878
878
|
akm bundle update npm:@scope/pkg
|
|
@@ -288,15 +288,12 @@ bundled.
|
|
|
288
288
|
|
|
289
289
|
## Indexing
|
|
290
290
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
toggling it so all entries and embeddings are rebuilt consistently. If the
|
|
298
|
-
setting differs from the state used to build the current index, AKM warns until
|
|
299
|
-
that full rebuild completes.
|
|
291
|
+
AKM-native Markdown contributes a normalized body projection to the
|
|
292
|
+
lowest-weight `content` search field. The projection is capped at 16,384
|
|
293
|
+
characters, removes frontmatter, comments, fenced code, and link destinations,
|
|
294
|
+
and is never produced for secret, env, session, or session-checkpoint assets.
|
|
295
|
+
Embedding input is separately capped at 8,192 characters with structured
|
|
296
|
+
metadata placed before body content.
|
|
300
297
|
|
|
301
298
|
## Semantic search
|
|
302
299
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "akm-cli",
|
|
3
|
-
"version": "0.9.2-alpha.
|
|
3
|
+
"version": "0.9.2-alpha.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cursor.",
|
|
6
6
|
"keywords": [
|
package/schemas/akm-config.json
CHANGED
|
@@ -305,10 +305,6 @@
|
|
|
305
305
|
}
|
|
306
306
|
},
|
|
307
307
|
"additionalProperties": true
|
|
308
|
-
},
|
|
309
|
-
"indexBodyOpening": {
|
|
310
|
-
"type": "boolean",
|
|
311
|
-
"description": "Index the first prose paragraph of each markdown asset body (capped at 280 chars) into the lowest-weight `content` search column and the embedding text (default false). Secret/env files and session-kind memories are never captured. Toggling the flag changes indexed text: run `akm index --full` afterwards to re-extract every entry and regenerate embeddings, and re-mint collapse-detector canary baselines via `bun scripts/refresh-canary-set.ts --refresh`."
|
|
312
308
|
}
|
|
313
309
|
},
|
|
314
310
|
"additionalProperties": {
|
|
@@ -1999,10 +1995,6 @@
|
|
|
1999
1995
|
}
|
|
2000
1996
|
},
|
|
2001
1997
|
"additionalProperties": true
|
|
2002
|
-
},
|
|
2003
|
-
"indexBodyOpening": {
|
|
2004
|
-
"type": "boolean",
|
|
2005
|
-
"description": "Index the first prose paragraph of each markdown asset body (capped at 280 chars) into the lowest-weight `content` search column and the embedding text (default false). Secret/env files and session-kind memories are never captured. Toggling the flag changes indexed text: run `akm index --full` afterwards to re-extract every entry and regenerate embeddings, and re-mint collapse-detector canary baselines via `bun scripts/refresh-canary-set.ts --refresh`."
|
|
2006
1998
|
}
|
|
2007
1999
|
},
|
|
2008
2000
|
"additionalProperties": {
|