akm-cli 0.9.2-alpha.1 → 0.9.2-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +15 -28
  2. package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +3 -4
  3. package/dist/assets/stash-skeleton/facts/conventions/organization.md +1 -3
  4. package/dist/commands/improve/collapse-detector.js +3 -4
  5. package/dist/commands/improve/extract-prompt.js +64 -22
  6. package/dist/commands/improve/extract.js +122 -53
  7. package/dist/commands/read/curate.js +43 -22
  8. package/dist/commands/sources/bundle-cli.js +1 -1
  9. package/dist/commands/sources/installed-stashes.js +67 -26
  10. package/dist/core/adapter/adapters/akm-adapter.js +2 -1
  11. package/dist/core/config/config.js +2 -6
  12. package/dist/core/config/schema/index-config.js +0 -27
  13. package/dist/indexer/index-written-assets.js +17 -9
  14. package/dist/indexer/indexer.js +32 -214
  15. package/dist/indexer/materialize-embeddings.js +155 -0
  16. package/dist/indexer/passes/metadata.js +263 -118
  17. package/dist/indexer/scan/doc-to-entry.js +0 -1
  18. package/dist/indexer/search/db-search.js +58 -28
  19. package/dist/indexer/search/fts-query.js +40 -40
  20. package/dist/indexer/search/ranking.js +36 -1
  21. package/dist/indexer/search/search-attribution.js +3 -1
  22. package/dist/indexer/search/search-fields.js +23 -14
  23. package/dist/output/text/command-format.js +3 -1
  24. package/dist/scripts/akm-migrate-node.js +12892 -12731
  25. package/dist/scripts/akm-migrate.js +12892 -12731
  26. package/dist/storage/repositories/index-entries-repository.js +40 -26
  27. package/dist/storage/repositories/index-entry-schema.js +1 -1
  28. package/dist/storage/repositories/index-fts-repository.js +56 -63
  29. package/dist/storage/repositories/index-schema.js +4 -9
  30. package/dist/storage/repositories/index-vec-repository.js +55 -6
  31. package/docs/reference/cli.md +4 -4
  32. package/docs/reference/configuration.md +6 -9
  33. package/package.json +1 -1
  34. package/schemas/akm-config.json +0 -8
@@ -18,21 +18,22 @@ import { getStateDbPath, withStateDb } from "../../core/state-db.js";
18
18
  import { warn } from "../../core/warn.js";
19
19
  import { buildSearchText } from "../../indexer/search/search-fields.js";
20
20
  import { ENTRY_COLUMNS, rowToIndexedEntry } from "./index-entry-mapper.js";
21
+ import { deleteFtsEntries, replaceFtsEntry } from "./index-fts-repository.js";
21
22
  import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
22
23
  import { deleteEntryVectors, isVecAvailable } from "./index-vec-repository.js";
23
24
  // ── Entry operations ────────────────────────────────────────────────────────
24
25
  /**
25
- * Insert or update an entry in the `entries` table. Returns the row id.
26
+ * Insert or update one canonical entry and all synchronously derived search
27
+ * state. Returns the stable row id.
26
28
  *
27
- * **Important:** This does not update the FTS index. Callers must call
28
- * `rebuildFts()` after all upserts are complete for full-text search to
29
- * reflect the changes.
29
+ * The entries row, FTS projection, and stale-vector invalidation commit as one
30
+ * SQLite transaction. Callers therefore cannot publish an entry and forget a
31
+ * second FTS maintenance step.
30
32
  */
31
33
  export function upsertEntry(db, filePath, entry, searchText, provenance, contentHash) {
32
34
  // Hot path during indexing — cache prepared statements per database
33
35
  // connection so we don't pay the SQL parse/compile cost on every call.
34
36
  const stmts = getUpsertStmts(db);
35
- const previous = stmts.findByItemRef.get(provenance.itemRef);
36
37
  // Phase 5A / Advantage D5: surface derived memory parent ref into the
37
38
  // dedicated `derived_from` column so retrieval-time lookup (parent→child)
38
39
  // does not have to scan + JSON-decode every memory row.
@@ -40,18 +41,20 @@ export function upsertEntry(db, filePath, entry, searchText, provenance, content
40
41
  // `content_hash` is optional on the LLM-enrichment re-upsert; a missing hash
41
42
  // preserves the scan writer's current value.
42
43
  const apply = () => {
44
+ const previous = stmts.findByItemRef.get(provenance.itemRef);
43
45
  const result = stmts.upsert.get(provenance.itemRef, provenance.bundleId, provenance.componentId, provenance.conceptId, provenance.adapterId, entry.type, filePath, contentHash ?? null, JSON.stringify(entry), searchText, derivedFrom);
44
46
  if (!result)
45
47
  throw new Error("upsertEntry: item_ref not found after upsert");
46
48
  if (previous?.id === result.id && previous.search_text !== searchText)
47
49
  deleteEntryVectors(db, result.id);
48
- // Mark this entry as FTS-dirty so `rebuildFts({ incremental: true })`
49
- // only revisits entries that actually changed. INSERT OR IGNORE is
50
- // idempotent across multiple upserts of the same row.
51
- stmts.markDirty.run(result.id);
50
+ replaceFtsEntry(db, result.id, entry);
52
51
  return result.id;
53
52
  };
54
- return previous && previous.search_text !== searchText ? db.transaction(apply)() : apply();
53
+ // Always enter the driver's transaction wrapper. Both supported SQLite
54
+ // drivers lower a transaction opened inside another transaction to a
55
+ // savepoint, so a caller that catches this mutation's error cannot commit a
56
+ // partial entries row through its outer transaction.
57
+ return db.transaction(apply)();
55
58
  }
56
59
  const upsertStmtsByDb = new WeakMap();
57
60
  // item_ref is the sole durable conflict target. `content_hash` COALESCEs so a
@@ -84,7 +87,6 @@ function getUpsertStmts(db) {
84
87
  ON CONFLICT(item_ref) DO UPDATE ${UPSERT_SET_CLAUSE}
85
88
  RETURNING id
86
89
  `),
87
- markDirty: db.prepare("INSERT OR IGNORE INTO entries_fts_dirty (entry_id) VALUES (?)"),
88
90
  findByItemRef: db.prepare("SELECT id, search_text FROM entries WHERE item_ref = ?"),
89
91
  };
90
92
  upsertStmtsByDb.set(db, stmts);
@@ -169,8 +171,8 @@ export function getBaseBeliefStatesForDerivedTwins(db, twinIds) {
169
171
  * `asset_ref` TEXT and are re-keyed separately by `akm mv` — see
170
172
  * the state rekey helper.) `document_json.name` (and `filename`, when
171
173
  * present) is patched and `search_text` rebuilt so search reflects the new
172
- * name; the row is marked FTS-dirty for the caller's
173
- * `rebuildFts({incremental: true})`.
174
+ * name. Its FTS projection and stale vector are updated in the same
175
+ * transaction as the canonical identity.
174
176
  *
175
177
  * Bundle-qualified `usage_events.entry_ref` rows for the old conceptId are
176
178
  * rewritten to the new item ref. Without this, events keep the old
@@ -209,6 +211,7 @@ export function rekeyEntryInPlace(db, opts) {
209
211
  // the utility history survives; the next full index heals the JSON.
210
212
  let documentJson = row.document_json;
211
213
  let searchText = row.search_text;
214
+ let document;
212
215
  try {
213
216
  const entry = JSON.parse(row.document_json);
214
217
  entry.name = opts.newName;
@@ -218,6 +221,7 @@ export function rekeyEntryInPlace(db, opts) {
218
221
  entry.derivedFrom = opts.newDerivedFrom;
219
222
  documentJson = JSON.stringify(entry);
220
223
  searchText = buildSearchText(entry);
224
+ document = entry;
221
225
  }
222
226
  catch {
223
227
  /* corrupt document_json — identity/path-only re-key */
@@ -242,7 +246,12 @@ export function rekeyEntryInPlace(db, opts) {
242
246
  if (opts.newDerivedFrom !== undefined) {
243
247
  db.prepare("UPDATE entries SET derived_from = ? WHERE id = ?").run(opts.newDerivedFrom, row.id);
244
248
  }
245
- db.prepare("INSERT OR IGNORE INTO entries_fts_dirty (entry_id) VALUES (?)").run(row.id);
249
+ if (row.search_text !== searchText)
250
+ deleteEntryVectors(db, row.id);
251
+ if (document)
252
+ replaceFtsEntry(db, row.id, document);
253
+ else
254
+ deleteFtsEntries(db, [row.id]);
246
255
  })();
247
256
  // Re-point usage history at the new ref. Chunk-8 WI-8.3: usage_events lives in
248
257
  // state.db now, so this is a SEPARATE cross-DB transaction (best-effort — the
@@ -358,6 +367,17 @@ export function deleteEntriesByBundle(db, bundleId) {
358
367
  deleteEntryRows(db, rows);
359
368
  })();
360
369
  }
370
+ /**
371
+ * Delete the complete regenerable entry generation through the same child-row
372
+ * authority used by targeted deletes. The caller may retain cross-database
373
+ * usage events so the finalize pass can relink them to the new row ids.
374
+ */
375
+ export function deleteAllEntries(db, options = {}) {
376
+ return db.transaction(() => {
377
+ const rows = db.prepare("SELECT id FROM entries").all();
378
+ return deleteEntryRows(db, rows, options);
379
+ })();
380
+ }
361
381
  /**
362
382
  * Diff-persist orphan delete: remove every entry under `dirPath` whose durable
363
383
  * `item_ref` is not in `keepRefs`.
@@ -382,15 +402,9 @@ function deleteRelatedRows(db, ids, options = {}) {
382
402
  return;
383
403
  const numericIds = ids.map((r) => r.id);
384
404
  const vecAvail = isVecAvailable(db);
385
- // Drop matching FTS rows + dirty markers immediately so an incremental
386
- // rebuild after a deletion doesn't try to re-index entries that no longer
387
- // exist (and so a full scan after deletion sees a consistent FTS).
388
- for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
389
- const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
390
- const placeholders = chunk.map(() => "?").join(",");
391
- bestEffort(() => db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk), "fts table may not exist on a brand-new db");
392
- bestEffort(() => db.prepare(`DELETE FROM entries_fts_dirty WHERE entry_id IN (${placeholders})`).run(...chunk), "fts dirty table is created lazily by upsertEntry");
393
- }
405
+ // FTS is part of the canonical mutation boundary, not a caller-maintained
406
+ // dirty queue. Delete it before the parent row inside this transaction.
407
+ deleteFtsEntries(db, numericIds);
394
408
  // Process in chunks to stay within SQLITE_MAX_VARIABLE_NUMBER
395
409
  for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
396
410
  const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
@@ -453,10 +467,10 @@ export function deleteUsageEventsByEntryIds(entryIds) {
453
467
  }
454
468
  /**
455
469
  * Delete entries by their primary key IDs, along with all related rows
456
- * (embeddings, entries_vec, entries_fts, utility_scores, usage_events).
470
+ * (embeddings, entries_vec, entries_fts, utility scores, usage_events).
457
471
  *
458
- * Used by the `--clean` post-pass to remove stale entries whose source files
459
- * no longer exist on disk.
472
+ * Used by explicit `--clean` reconciliation before embeddings and final
473
+ * verification to remove stale entries whose source files no longer exist.
460
474
  */
461
475
  export function deleteEntriesByIds(db, ids) {
462
476
  if (ids.length === 0)
@@ -10,7 +10,7 @@
10
10
  * serving preflights, and read-only evaluator tooling cannot drift into
11
11
  * separate definitions of "current".
12
12
  */
13
- export const CANONICAL_INDEX_DB_VERSION = 21;
13
+ export const CANONICAL_INDEX_DB_VERSION = 22;
14
14
  export const CANONICAL_ENTRY_SCHEMA_SQL = `
15
15
  CREATE TABLE IF NOT EXISTS entries (
16
16
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -2,33 +2,63 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  /**
5
- * `index.db` FTS5 search + rebuild repository.
5
+ * `index.db` FTS5 search + materialization repository.
6
6
  *
7
- * Owns the `entries_fts` full-text query path and the incremental/full FTS
8
- * rebuild.
7
+ * Owns the `entries_fts` full-text query path, per-entry projections, and the
8
+ * explicit full recovery rebuild.
9
9
  */
10
10
  import { warn } from "../../core/warn.js";
11
- import { buildPrefixQuery, sanitizeFtsQuery } from "../../indexer/search/fts-query.js";
11
+ import { buildLexicalQueryPlan } from "../../indexer/search/fts-query.js";
12
12
  import { buildSearchFields } from "../../indexer/search/search-fields.js";
13
13
  import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
14
+ const INSERT_FTS_SQL = "INSERT INTO entries_fts (entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?)";
15
+ const ftsMutationStatementsByDb = new WeakMap();
16
+ function getFtsMutationStatements(db) {
17
+ const existing = ftsMutationStatementsByDb.get(db);
18
+ if (existing)
19
+ return existing;
20
+ const statements = {
21
+ deleteOne: db.prepare("DELETE FROM entries_fts WHERE entry_id = ?"),
22
+ insert: db.prepare(INSERT_FTS_SQL),
23
+ };
24
+ ftsMutationStatementsByDb.set(db, statements);
25
+ return statements;
26
+ }
27
+ /** Replace one entry's derived FTS projection inside the caller's transaction. */
28
+ export function replaceFtsEntry(db, entryId, entry) {
29
+ const fields = buildSearchFields(entry);
30
+ const statements = getFtsMutationStatements(db);
31
+ statements.deleteOne.run(entryId);
32
+ statements.insert.run(entryId, fields.name, fields.description, fields.tags, fields.hints, fields.content);
33
+ }
34
+ /** Delete derived FTS projections for canonical entries that are being removed. */
35
+ export function deleteFtsEntries(db, entryIds) {
36
+ for (let i = 0; i < entryIds.length; i += SQLITE_CHUNK_SIZE) {
37
+ const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
38
+ const placeholders = chunk.map(() => "?").join(",");
39
+ db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
40
+ }
41
+ }
14
42
  export function searchFts(db, query, limit, entryType, excludeTypes) {
15
- const ftsQuery = sanitizeFtsQuery(query);
16
- if (!ftsQuery)
43
+ const plan = buildLexicalQueryPlan(query);
44
+ if (!plan.exact)
17
45
  return [];
18
46
  // Try the exact AND query first
19
- const exactResults = runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes);
47
+ const exactResults = runFtsQuery(db, plan.exact, "exact", limit, entryType, excludeTypes);
20
48
  if (exactResults.length > 0)
21
49
  return exactResults;
22
- // Exact match returned zero results — try prefix fallback.
23
- // Append FTS5 `*` suffix to each token that is >= 3 characters long.
24
- // Short tokens (1-2 chars) are excluded from prefix expansion because
25
- // they produce too many false positives.
26
- const prefixQuery = buildPrefixQuery(ftsQuery);
27
- if (!prefixQuery)
28
- return [];
29
- return runFtsQuery(db, prefixQuery, limit, entryType, excludeTypes);
50
+ if (plan.exactPrefix) {
51
+ const prefixResults = runFtsQuery(db, plan.exactPrefix, "prefix", limit, entryType, excludeTypes);
52
+ if (prefixResults.length > 0)
53
+ return prefixResults;
54
+ }
55
+ // One measured relaxation only after both conjunctive forms miss. This is
56
+ // still the same FTS table, BM25 weights, candidate collection, and
57
+ // downstream ranker merely an OR candidate query for sentence-shaped
58
+ // input whose filler terms prevented a strict hit.
59
+ return plan.relaxed ? runFtsQuery(db, plan.relaxed, "relaxed", limit, entryType, excludeTypes) : [];
30
60
  }
31
- function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
61
+ function runFtsQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludeTypes) {
32
62
  // #627 — exclude-type clause. Only applies on the untyped ('any') path; an
33
63
  // explicit include filter (entryType) already narrows to a single type, so
34
64
  // exclusion is redundant there. An empty list skips the clause entirely
@@ -84,6 +114,7 @@ function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
84
114
  bundleId: row.bundleId,
85
115
  conceptId: row.conceptId,
86
116
  adapterId: row.adapterId,
117
+ lexicalMatch,
87
118
  });
88
119
  }
89
120
  return results;
@@ -94,49 +125,20 @@ function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
94
125
  }
95
126
  }
96
127
  /**
97
- * Rebuild the FTS5 search index.
98
- *
99
- * `incremental` (default `false`): when true, only rebuild rows that
100
- * `upsertEntry` marked dirty since the last `rebuildFts` call. The full path
101
- * (default) wipes `entries_fts` and re-inserts every row from `entries` —
102
- * appropriate for `akm index --full` and version-upgrade rebuilds.
103
- *
104
- * Both paths are wrapped in a single transaction so the FTS table is never
105
- * left in a half-rebuilt state.
128
+ * Explicitly rebuild the complete FTS5 projection from canonical entries.
129
+ * Ordinary entry mutations do not call this: `upsertEntry` and the delete
130
+ * operations publish their FTS state in the same transaction as `entries`.
131
+ * This remains a recovery/schema-verification primitive for regenerable
132
+ * `index.db` state.
106
133
  *
107
134
  * Skipped corrupt-JSON rows are aggregated into one warning instead of
108
135
  * spamming stderr per-entry.
109
136
  */
110
- export function rebuildFts(db, options) {
111
- const incremental = options?.incremental === true;
137
+ export function rebuildFts(db) {
112
138
  db.transaction(() => {
113
- let rows;
114
- if (incremental) {
115
- // Read the dirty queue and join against entries to get the JSON.
116
- // Then drop the matching rows from entries_fts so the INSERT below
117
- // doesn't double-up. The dirty list is drained at the end.
118
- rows = db
119
- .prepare(`SELECT e.id AS id, e.document_json AS document_json
120
- FROM entries_fts_dirty d
121
- JOIN entries e ON e.id = d.entry_id`)
122
- .all();
123
- if (rows.length === 0)
124
- return;
125
- const ids = rows.map((r) => r.id);
126
- // Delete only the dirty FTS rows — chunk to stay under
127
- // SQLITE_MAX_VARIABLE_NUMBER on large dirty queues.
128
- for (let i = 0; i < ids.length; i += SQLITE_CHUNK_SIZE) {
129
- const chunk = ids.slice(i, i + SQLITE_CHUNK_SIZE);
130
- const placeholders = chunk.map(() => "?").join(",");
131
- db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
132
- }
133
- }
134
- else {
135
- // Full path: wipe and re-read every row.
136
- db.exec("DELETE FROM entries_fts");
137
- rows = db.prepare("SELECT id, document_json FROM entries").all();
138
- }
139
- const insertStmt = db.prepare("INSERT INTO entries_fts (entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?)");
139
+ db.exec("DELETE FROM entries_fts");
140
+ const rows = db.prepare("SELECT id, document_json FROM entries").all();
141
+ const insertStmt = db.prepare(INSERT_FTS_SQL);
140
142
  let skipped = 0;
141
143
  for (const row of rows) {
142
144
  let entry;
@@ -154,14 +156,5 @@ export function rebuildFts(db, options) {
154
156
  if (skipped > 0) {
155
157
  warn(`[db] rebuildFts: skipped ${skipped} entr${skipped === 1 ? "y" : "ies"} with invalid document_json`);
156
158
  }
157
- // Always drain the dirty queue — both paths converge here. The
158
- // incremental path drains it because we just consumed every dirty row;
159
- // the full path drains it because a full rebuild covers everything the
160
- // dirty list tracks. The table is guaranteed to exist (created by
161
- // ensureSchema()).
162
- //
163
- // BUG-L1: previously the if/else arms ran identical statements — the
164
- // duplication has been collapsed.
165
- db.exec("DELETE FROM entries_fts_dirty");
166
159
  })();
167
160
  }
@@ -14,6 +14,10 @@ import { isVecAvailable, purgeEmbeddings } from "./index-vec-repository.js";
14
14
  // entry_type columns. item_ref is the sole conflict key; document_json is the
15
15
  // sole stored document projection; bundle provenance and file_path provide the
16
16
  // current identity and materialized read path.
17
+ //
18
+ // v21→v22: entry mutations publish FTS synchronously and no dirty queue exists.
19
+ // Discard the old derived generation so stale FTS rows and caller-managed dirty
20
+ // state cannot cross the mutation-authority boundary.
17
21
  export const DB_VERSION = CANONICAL_INDEX_DB_VERSION;
18
22
  export const EMBEDDING_DIM = 384;
19
23
  // #624-P1: graph_files is keyed to (stash_root, file_path, body_hash).
@@ -301,15 +305,6 @@ export function ensureSchema(db, embeddingDim) {
301
305
  // child rows are removed when a graph_files row is replaced.
302
306
  //
303
307
  ensureGraphTables(db);
304
- // FTS-dirty queue. Created here (not lazily on first upsert) so the
305
- // per-entry write path doesn't issue a CREATE TABLE IF NOT EXISTS on
306
- // every call — that DDL would fire thousands of times during a full
307
- // index. See `markFtsDirty` and `rebuildFts({ incremental: true })`.
308
- db.exec(`
309
- CREATE TABLE IF NOT EXISTS entries_fts_dirty (
310
- entry_id INTEGER PRIMARY KEY
311
- );
312
- `);
313
308
  // If a generation rebuild could not drop a vec0 table while the extension
314
309
  // was unavailable, finish that reset as soon as vec0 can be loaded again.
315
310
  if (isVecAvailable(db) && getMeta(db, "vecResetPending") === "1") {
@@ -12,6 +12,7 @@ import { bestEffort } from "../../core/best-effort.js";
12
12
  import { warn } from "../../core/warn.js";
13
13
  import { cosineSimilarity } from "../../llm/embedders/types.js";
14
14
  import { getMeta, setMeta } from "./index-meta-repository.js";
15
+ import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
15
16
  // ── sqlite-vec extension ────────────────────────────────────────────────────
16
17
  const vecStatus = new WeakMap();
17
18
  /**
@@ -68,6 +69,43 @@ export function isVecFastPathReady(db) {
68
69
  // flag, so the read path has to verify the table really exists.
69
70
  return hasVecTable(db);
70
71
  }
72
+ /**
73
+ * Verify that the vec fast-path table mirrors the complete durable BLOB set.
74
+ *
75
+ * A targeted embedding write preserves the prior readiness decision because
76
+ * its subset cannot prove an older degraded generation is healed. Global
77
+ * materialization uses this aggregate check before promoting the persisted
78
+ * flag; search itself still reads the cheap flag and does not repeat the check
79
+ * per query.
80
+ */
81
+ export function isVecFastPathComplete(db) {
82
+ if (!isVecAvailable(db) || !hasVecTable(db))
83
+ return false;
84
+ try {
85
+ const missingVecRows = db
86
+ .prepare(`
87
+ SELECT id FROM embeddings
88
+ EXCEPT
89
+ SELECT id FROM entries_vec
90
+ LIMIT 1
91
+ `)
92
+ .all();
93
+ if (missingVecRows.length > 0)
94
+ return false;
95
+ const orphanVecRows = db
96
+ .prepare(`
97
+ SELECT id FROM entries_vec
98
+ EXCEPT
99
+ SELECT id FROM embeddings
100
+ LIMIT 1
101
+ `)
102
+ .all();
103
+ return orphanVecRows.length === 0;
104
+ }
105
+ catch {
106
+ return false;
107
+ }
108
+ }
71
109
  const vecTablePresent = new WeakMap();
72
110
  /**
73
111
  * Whether `entries_vec` exists on this connection, memoized per handle.
@@ -274,13 +312,24 @@ function searchBlobVec(db, queryEmbedding, k) {
274
312
  * Return all entries that do not yet have an embedding row.
275
313
  * Used by the embedding phase to determine which entries need vectors generated.
276
314
  */
277
- export function getAllEntriesForEmbedding(db) {
278
- return db
279
- .prepare(`
315
+ export function getAllEntriesForEmbedding(db, entryIds) {
316
+ const select = `
280
317
  SELECT e.id, e.search_text AS searchText, e.item_ref AS itemRef, e.file_path AS filePath FROM entries e
281
- WHERE NOT EXISTS (SELECT 1 FROM embeddings b WHERE b.id = e.id)
282
- `)
283
- .all();
318
+ `;
319
+ const missing = "NOT EXISTS (SELECT 1 FROM embeddings b WHERE b.id = e.id)";
320
+ if (entryIds === undefined) {
321
+ return db.prepare(`${select} WHERE ${missing} ORDER BY e.id`).all();
322
+ }
323
+ const targets = [...new Set(entryIds)].sort((left, right) => left - right);
324
+ const rows = [];
325
+ for (let offset = 0; offset < targets.length; offset += SQLITE_CHUNK_SIZE) {
326
+ const chunk = targets.slice(offset, offset + SQLITE_CHUNK_SIZE);
327
+ if (chunk.length === 0)
328
+ continue;
329
+ const placeholders = chunk.map(() => "?").join(",");
330
+ rows.push(...db.prepare(`${select} WHERE e.id IN (${placeholders}) AND ${missing} ORDER BY e.id`).all(...chunk));
331
+ }
332
+ return rows;
284
333
  }
285
334
  export function getEmbeddingCount(db) {
286
335
  const row = db.prepare("SELECT COUNT(*) AS cnt FROM embeddings").get();
@@ -869,10 +869,10 @@ akm bundle remove my-provider --yes # Skip the confirmation prompt
869
869
 
870
870
  ### bundle update
871
871
 
872
- Update one remote bundle, or refresh every configured bundle with `--all`.
873
- Git, npm, and website candidates are staged and audited before they replace the
874
- active generation; filesystem bundles are reported as skipped because they
875
- already reflect local files in place.
872
+ Update one bundle, or refresh every configured bundle with `--all`. Git, npm,
873
+ and website candidates are staged and audited before they replace the active
874
+ generation. Filesystem bundles require no hydration; update reconciles their
875
+ current files into the index immediately.
876
876
 
877
877
  ```sh
878
878
  akm bundle update npm:@scope/pkg
@@ -288,15 +288,12 @@ bundled.
288
288
 
289
289
  ## Indexing
290
290
 
291
- `index.indexBodyOpening` defaults to `false`. When enabled, AKM captures the
292
- first prose paragraph of each Markdown asset body, capped at 280 characters,
293
- into the lowest-weight search content and embedding text. Secret and env files
294
- are never read for this field, and session-kind memories are excluded.
295
-
296
- Changing this option changes indexed text. Run `akm index --full` after
297
- toggling it so all entries and embeddings are rebuilt consistently. If the
298
- setting differs from the state used to build the current index, AKM warns until
299
- that full rebuild completes.
291
+ AKM-native Markdown contributes a normalized body projection to the
292
+ lowest-weight `content` search field. The projection is capped at 16,384
293
+ characters, removes frontmatter, comments, fenced code, and link destinations,
294
+ and is never produced for secret, env, session, or session-checkpoint assets.
295
+ Embedding input is separately capped at 8,192 characters with structured
296
+ metadata placed before body content.
300
297
 
301
298
  ## Semantic search
302
299
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "akm-cli",
3
- "version": "0.9.2-alpha.1",
3
+ "version": "0.9.2-alpha.2",
4
4
  "type": "module",
5
5
  "description": "akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cursor.",
6
6
  "keywords": [
@@ -305,10 +305,6 @@
305
305
  }
306
306
  },
307
307
  "additionalProperties": true
308
- },
309
- "indexBodyOpening": {
310
- "type": "boolean",
311
- "description": "Index the first prose paragraph of each markdown asset body (capped at 280 chars) into the lowest-weight `content` search column and the embedding text (default false). Secret/env files and session-kind memories are never captured. Toggling the flag changes indexed text: run `akm index --full` afterwards to re-extract every entry and regenerate embeddings, and re-mint collapse-detector canary baselines via `bun scripts/refresh-canary-set.ts --refresh`."
312
308
  }
313
309
  },
314
310
  "additionalProperties": {
@@ -1999,10 +1995,6 @@
1999
1995
  }
2000
1996
  },
2001
1997
  "additionalProperties": true
2002
- },
2003
- "indexBodyOpening": {
2004
- "type": "boolean",
2005
- "description": "Index the first prose paragraph of each markdown asset body (capped at 280 chars) into the lowest-weight `content` search column and the embedding text (default false). Secret/env files and session-kind memories are never captured. Toggling the flag changes indexed text: run `akm index --full` afterwards to re-extract every entry and regenerate embeddings, and re-mint collapse-detector canary baselines via `bun scripts/refresh-canary-set.ts --refresh`."
2006
1998
  }
2007
1999
  },
2008
2000
  "additionalProperties": {