akm-cli 0.9.15-beta.5 → 0.9.16-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +145 -1
  2. package/dist/assets/tasks/core/index-refresh.yml +1 -1
  3. package/dist/cli/retired-commands.js +2 -0
  4. package/dist/cli/unknown-flags.js +36 -3
  5. package/dist/commands/improve/collapse-detector.js +2 -2
  6. package/dist/commands/improve/consolidate.js +6 -4
  7. package/dist/commands/improve/improve-cli.js +1 -1
  8. package/dist/commands/proposal/repository.js +12 -3
  9. package/dist/commands/read/curate.js +34 -44
  10. package/dist/commands/read/search.js +50 -2
  11. package/dist/commands/sources/index-status.js +99 -0
  12. package/dist/commands/sources/info.js +8 -8
  13. package/dist/commands/sources/installed-stashes.js +33 -12
  14. package/dist/commands/sources/source-add.js +21 -6
  15. package/dist/commands/sources/stash-cli.js +119 -111
  16. package/dist/core/adapter/adapters/akm-adapter.js +35 -3
  17. package/dist/core/adapter/adapters/akm-metadata.js +11 -1
  18. package/dist/core/asset/asset-placement.js +35 -0
  19. package/dist/core/config/schema/embedding.js +7 -30
  20. package/dist/core/config/schema/search.js +11 -9
  21. package/dist/core/errors.js +5 -2
  22. package/dist/core/hash.js +18 -0
  23. package/dist/core/maintenance-barrier.js +8 -6
  24. package/dist/core/paths.js +0 -11
  25. package/dist/core/run-lock.js +5 -2
  26. package/dist/core/state/migrations.js +26 -1
  27. package/dist/core/state-db.js +63 -27
  28. package/dist/indexer/drain.js +306 -0
  29. package/dist/indexer/embedding-identity.js +20 -0
  30. package/dist/indexer/enrich.js +260 -0
  31. package/dist/indexer/ensure-index.js +5 -0
  32. package/dist/indexer/index-written-assets.js +133 -171
  33. package/dist/indexer/indexer.js +458 -1621
  34. package/dist/indexer/lookup/adapter-concept-owner.js +19 -5
  35. package/dist/indexer/passes/metadata.js +18 -1
  36. package/dist/indexer/reconcile.js +890 -0
  37. package/dist/indexer/scan/drain-dir.js +27 -70
  38. package/dist/indexer/scan/parse-file.js +66 -0
  39. package/dist/indexer/search/db-search.js +373 -89
  40. package/dist/indexer/search/ranking-contributors.js +21 -16
  41. package/dist/indexer/search/ranking.js +135 -57
  42. package/dist/indexer/units/unit.js +159 -0
  43. package/dist/llm/client.js +10 -1
  44. package/dist/llm/embedder.js +10 -3
  45. package/dist/llm/embedders/provider-limits.js +288 -0
  46. package/dist/llm/embedders/remote.js +133 -104
  47. package/dist/llm/feature-gate.js +4 -2
  48. package/dist/llm/rerank-client.js +3 -3
  49. package/dist/output/shapes/passthrough.js +1 -0
  50. package/dist/output/text/command-format.js +19 -13
  51. package/dist/output/text/helpers.js +1 -1
  52. package/dist/output/text/index.js +5 -2
  53. package/dist/scripts/akm-migrate-node.js +1141 -1237
  54. package/dist/scripts/akm-migrate.js +1141 -1237
  55. package/dist/setup/semantic-assets.js +2 -2
  56. package/dist/setup/steps/connection.js +3 -2
  57. package/dist/storage/repositories/files-repository.js +181 -0
  58. package/dist/storage/repositories/index-connection.js +1 -3
  59. package/dist/storage/repositories/index-entries-repository.js +77 -68
  60. package/dist/storage/repositories/index-entry-schema.js +16 -25
  61. package/dist/storage/repositories/index-fts-repository.js +29 -263
  62. package/dist/storage/repositories/index-meta-repository.js +0 -29
  63. package/dist/storage/repositories/index-schema.js +115 -122
  64. package/dist/storage/repositories/index-utility-repository.js +1 -1
  65. package/dist/storage/repositories/index-vec-repository.js +21 -334
  66. package/dist/storage/repositories/units-repository.js +510 -0
  67. package/docs/migration/release-notes/0.9.15.md +34 -36
  68. package/docs/migration/release-notes/0.9.16.md +110 -0
  69. package/docs/migration/release-notes/README.md +5 -0
  70. package/docs/reference/cli.md +93 -87
  71. package/docs/reference/configuration.md +128 -89
  72. package/docs/reference/data-and-telemetry.md +2 -1
  73. package/package.json +1 -1
  74. package/schemas/akm-config.json +2 -58
  75. package/dist/indexer/index-db-contention.js +0 -56
  76. package/dist/indexer/index-rebuild-lock.js +0 -73
  77. package/dist/indexer/materialize-embeddings.js +0 -771
  78. package/dist/indexer/passes/dir-staleness.js +0 -161
  79. package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
package/CHANGELOG.md CHANGED
@@ -4,7 +4,151 @@ All notable changes to this project will be documented in this file.
4
4
 
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
6
6
 
7
- ## [0.9.15-beta.5] - 2026-09-10
7
+ ## [Unreleased]
8
+
9
+ ## [0.9.16-alpha.1] - 2026-09-11
10
+
11
+ ### Added
12
+
13
+ - **`akm index status`.** A cheap, read-only snapshot of `index.db`'s
14
+ current state — files tracked, entries, distinct units, how many have a
15
+ vector for the active embedding identity (and therefore the embedding
16
+ queue's remaining depth), the active identity itself, and the last
17
+ reconcile/build times — with no writes. Mirrors `akm info`'s
18
+ absent/inaccessible handling: a missing index reads as the ordinary
19
+ first-run state, an unreadable one is reported, never silently presented
20
+ as empty.
21
+ - **A credential diagnostic on the embedding queue.** Before the first
22
+ provider request `akm index`'s drain makes, one default-level line names
23
+ the embedding endpoint, model, and the credential's SOURCE (never the
24
+ resolved value) whenever a remote endpoint is configured — so a field run
25
+ can compare it against what the gateway actually saw (#953).
26
+
27
+ ### Changed
28
+
29
+ - **`akm index` is redesigned end to end
30
+ (`docs/plans/index-redesign.md`).** The old walk/clean/embed/finalize
31
+ phase pipeline, per-directory fingerprint cache, and duplicated storage
32
+ (entry text held three times, every vector held twice plus a salvage
33
+ copy) are replaced by two steps: reconcile, then drain. Reconcile
34
+ (`src/indexer/reconcile.ts`) stat-walks every configured root against a
35
+ `files` cache, hashing and re-parsing only what actually changed, and
36
+ derives content-addressed units — one "card" unit per entry
37
+ (name/description/tags/hints) and one "fragment" unit per Markdown
38
+ section — into a single text table (`unit_texts`/`units_fts`). Drain
39
+ (`src/indexer/drain.ts`) treats embedding as a queue, not a phase: the
40
+ pending set is exactly the unit hashes with no vector under the active
41
+ embedding identity, packed against the provider's own probed context
42
+ window and slot count, written into a single vector table
43
+ (`units`/`units_vec`) keyed by `(unit_hash, identity)`. An unchanged file
44
+ or unit is never re-derived or re-embedded again — not on a rename, not
45
+ on `akm index --full`, not on a future generation bump — because
46
+ everything is keyed by content hash and observed provider identity, never
47
+ by a row id or a config string.
48
+ - **Every write path indexes what it just wrote, inline.** `remember`,
49
+ `import`, extract's session-asset capture, `source clone`, and proposal
50
+ accept each reconcile and drain exactly the paths/units they touched, in
51
+ the same call as the write, with no lock probe and no background reindex
52
+ spawn.
53
+ - **Search is one query over `units`**, scoring lexical evidence by BM25
54
+ magnitude through the calibrated transform the repository already had and
55
+ combining it with semantic distance on the proven 0.7/0.3 split. Reciprocal
56
+ rank fusion was tried first and measured worse than the path it replaced
57
+ (0.918 against 0.933 on the `curate-golden` fixture, unchanged by splitting
58
+ or weighting the lists), because a rank cannot tell a strong match from a
59
+ weak one; the shipped scoring measures 0.936 with no banned-above-required
60
+ hits. The semantic-only `minScore` floor is gone, type filters now apply in
61
+ SQL before the candidate cap, and the exact/prefix/relaxed ladder tops up to
62
+ the candidate budget instead of stopping at the first non-empty tier. The
63
+ tier a hit came from is also ranking evidence: a unit matching every query
64
+ token outranks one matching a subset, because the calibrated BM25 transform
65
+ compresses even a sixfold magnitude difference into a few thousandths — far
66
+ less than any single ranking contributor — so tier decides across tiers and
67
+ magnitude decides within one.
68
+ - **`akm index --full`** no longer drops anything first: it forces every
69
+ walked file to be re-parsed (skipping the unchanged-file shortcut) but
70
+ updates each file's existing row in place, keeping its id, vectors, and
71
+ learned utility scores. **`akm index --reembed`** now means "drop the
72
+ active embedding identity's vectors, then re-embed every unit from
73
+ scratch."
74
+ - **`akm index --skip-if-locked` is deprecated and does nothing.** Index
75
+ runs no longer take a rebuild lock — every write is a short, idempotent,
76
+ content-addressed transaction, so two concurrent runs converge instead of
77
+ contending. Passing the flag prints one deprecation warning; kept only so
78
+ an existing script does not fail on an unknown flag.
79
+ - **The derived `index.db` generation changes from v23 to v24.** The first
80
+ read (or explicit `akm index`) after upgrade re-derives `entries` and
81
+ every unit from files, and embeds the full corpus once against the
82
+ configured provider, since the new unit vector store does not carry
83
+ forward the superseded per-entry vector tables. See the [0.9.16 migration
84
+ note](docs/migration/release-notes/0.9.16.md) for the cost, stated
85
+ plainly.
86
+
87
+ ### Removed
88
+
89
+ - **The index phase pipeline, directory-fingerprint staleness cache, the
90
+ index rebuild lock, and the index-path use of the maintenance barrier** —
91
+ roughly 4,800 lines across the files that implemented the old index core,
92
+ replaced by the reconcile/drain design above (`docs/plans/index-redesign.md`).
93
+ - **`entries_fts`, `entry_fragments_fts`, the legacy per-entry `embeddings`
94
+ materializer (`materialize-embeddings.ts`), and `embedding_salvage`** —
95
+ superseded by the single `unit_texts`/`units_fts` and `units`/`units_vec`
96
+ tables, which never need a salvage-before-discard step because they are
97
+ content-addressed and never wholesale-discarded.
98
+ - **`akm index --enrich`, `--re-enrich`, `--clean`, and `--dry-run`.**
99
+ Plain `akm index` now always performs metadata enrichment when an engine
100
+ is configured, and every run already removes stale entries as part of
101
+ reconcile — the work these flags used to separately opt into. All four
102
+ now fail with a `UsageError` naming the replacement, instead of silently
103
+ doing nothing or being silently accepted.
104
+ - **Config keys `embedding.maxInputTokens`, `embedding.maxTokens`,
105
+ `embedding.batchSize`, `embedding.contextLength`, and
106
+ `search.minScore`.** Embedding request packing is sourced from the
107
+ provider's own probed limits (unchanged from 0.9.15 packing, applied to
108
+ units instead of whole entries); the fused score has no comparable 0–1
109
+ threshold to tune. A config that still sets any of them loads without
110
+ error and is simply ignored.
111
+ - **The `"ready-js"` semantic-search status.** Named a pure-JS
112
+ cosine-similarity fallback for when `sqlite-vec` was unavailable; the new
113
+ unit vector store has no BLOB-table fallback to fall back to, so nothing
114
+ produces that value any more.
115
+
116
+ ### Fixed
117
+
118
+ - **Two akm processes starting at the same moment against a database neither
119
+ has created yet no longer fail.** `state.db` and `index.db` each had a
120
+ first-open race. `index.db` created `entries` about twenty statements
121
+ before it stamped the generation, so a second opener read
122
+ entries-without-a-generation as a stale index and dropped the table out
123
+ from under the first process, which then exited 70 with `no such table:
124
+ entries` — measured at 8 of 440 racing child processes. `state.db` read its
125
+ migration ledger and its "does this file have any other tables" check as
126
+ two separate statements, so a sibling's bootstrap committing between them
127
+ looked exactly like a legacy unversioned database and was refused outright
128
+ — 2 of 1200 racing trials. Both initializations are now single atomic
129
+ units, measured at zero failures in 840 and 1200 trials respectively, idle
130
+ and under load. Only a genuinely contended run still fails, as
131
+ `INDEX_DB_CONTENDED`/`STATE_DB_CONTENDED` at exit 75, the documented
132
+ retry-shortly contract. An already-initialized database takes the same
133
+ unlocked path it always did.
134
+ - **`akm show <memory>` no longer fails when that memory has an inferred
135
+ `.derived` twin.** It exited 2 with `RESOURCE_ALREADY_EXISTS` ("multiple
136
+ physical owners"), so once `akm improve` derived a memory — its ordinary
137
+ output — the base ref stopped being usable, and the read path that did not
138
+ fail served the twin's content instead of the memory's. `.derived` is a
139
+ provenance marker on the same identity and the placement layer always
140
+ declared that the plain file wins; the physical-owner lookup now honours
141
+ that instead of discarding it. Genuinely ambiguous cases, including two
142
+ case-only spellings of one name and `env`'s co-equal `.env`/`default.env`
143
+ pair, still fail loudly and unchanged. Present since before 0.9.15.
144
+ - **A derived memory no longer outranks the memory it was derived from.** A
145
+ twin's own filename contributed a `derived` tag that minted a synthetic
146
+ alias, and the machine-written `source:` provenance backref was folded into
147
+ search hints, together handing the twin a flat 0.42 of ranking credit for
148
+ bookkeeping no author wrote — enough to beat a memory whose description
149
+ matched the query verbatim. Neither field earns ranking credit any more.
150
+
151
+ ## [0.9.15] - 2026-09-10
8
152
 
9
153
  ### Added
10
154
 
@@ -1,4 +1,4 @@
1
1
  version: 4
2
- run: akm index --skip-if-locked
2
+ run: akm index
3
3
  description: Nightly incremental index refresh
4
4
  schedule: "0 4 * * *"
@@ -104,6 +104,8 @@ export function retiredCommandHint(parentPath, attempted) {
104
104
  */
105
105
  const RETIRED_FLAG_HINTS = {
106
106
  "index --background": "`--background` was removed in 0.9 — the flag never actually backgrounded; use `--quiet`.",
107
+ "index --clean": "`--clean` was removed in the index redesign — every `akm index` run now removes stale entries as part of reconcile, the same work `--clean` used to opt into.",
108
+ "index --dry-run": "`--dry-run` was removed in the index redesign along with `--clean`, the only flag it ever modified.",
107
109
  "setup --detect-only": "`--detect-only` was removed in 0.9 — environment detection runs inside `akm setup`; `akm info` reports the configured capabilities.",
108
110
  "setup --reset-recommended": "`--reset-recommended` was removed in 0.9 — `akm setup` now offers to apply recommended defaults interactively.",
109
111
  "proposal extract --watch": "`--watch` was removed in 0.9 — schedule `akm proposal extract --auto` as a task instead.",
@@ -30,6 +30,28 @@ import { cittyComparableName, findCittyTopLevelCommandIndex, toAliasArray, } fro
30
30
  import { retiredFlagHint } from "./retired-commands.js";
31
31
  /** Flags citty implements itself, which no command declares. */
32
32
  const IMPLICIT_FLAGS = ["help", "h", "version", "v"];
33
+ /**
34
+ * Keys `GLOBAL_OUTPUT_ARGS` (cli/shared.ts) contributes. Every leaf AND every
35
+ * group re-declares these so citty's parser consumes their values (see that
36
+ * constant's own doc) — so their mere presence in a group's own `args` never
37
+ * means the group has a real business use for its own flags; only a key
38
+ * beyond this set does. Duplicated here (not imported) to avoid coupling this
39
+ * shared scanner to `cli/shared.ts`'s own dependency surface; keep in sync if
40
+ * `GLOBAL_OUTPUT_ARGS` gains or loses a key.
41
+ */
42
+ const GLOBAL_OUTPUT_ARG_KEYS = new Set(["format", "detail", "shape", "output", "quiet", "verbose"]);
43
+ /**
44
+ * Whether a group declares any flag of its own beyond the global output
45
+ * scaffold — i.e. whether its bare invocation (no subcommand token) runs a
46
+ * REAL default body that reads those flags (`akm index`'s `full`/`reembed`),
47
+ * as opposed to the canonical bare-group usage error every other group falls
48
+ * through to (`defineGroupCommand`, cli/shared.ts) — a `UsageError` either
49
+ * way, whose own declared args (if any) exist only so `--help` documents them,
50
+ * never so a body reads them.
51
+ */
52
+ function hasOwnBusinessArgs(cmd) {
53
+ return Object.keys(cmd.args ?? {}).some((key) => !GLOBAL_OUTPUT_ARG_KEYS.has(key));
54
+ }
33
55
  /**
34
56
  * Retired flags whose commands still diagnose them THEMSELVES, with a message
35
57
  * that names the replacement ("`--scope` was removed, use `--filter`",
@@ -102,9 +124,20 @@ function collectKnownArgs(root, rawArgs) {
102
124
  break;
103
125
  const idx = findCittyTopLevelCommandIndex(args, (cmd.args ?? {}));
104
126
  const token = idx >= 0 ? args[idx] : undefined;
105
- // A group with no subcommand token: citty reports "no command specified".
106
- if (token === undefined)
107
- return { names, valueFlags, booleanFlags, displayNames, path, resolved: false };
127
+ // A group with no subcommand token: citty always calls the group's own
128
+ // `run` regardless (a `defineGroupCommand` group's `run` is never
129
+ // undefined see its doc in cli/shared.ts). For most groups that `run`
130
+ // is the canonical bare-group usage error, and its own declared args (if
131
+ // any) exist only for `--help`, so a flag meant for the subcommand the
132
+ // caller forgot to type must not be misreported as "unknown" — stand
133
+ // down, exactly as before. `akm index` is the one group with a REAL
134
+ // default body that reads its own flags (`full`/`reembed`), so `akm index
135
+ // --background` must still be rejected even though `index` also carries a
136
+ // real subcommand (`status`) — `hasOwnBusinessArgs` is what tells the two
137
+ // apart.
138
+ if (token === undefined) {
139
+ return { names, valueFlags, booleanFlags, displayNames, path, resolved: hasOwnBusinessArgs(cmd) };
140
+ }
108
141
  const sub = subCommands[token];
109
142
  // An unrecognized token: citty reports the unknown command, which is the
110
143
  // real problem — its flags are beside the point.
@@ -36,10 +36,10 @@ import { getImproveProcessConfig } from "../../core/config/config.js";
36
36
  import { appendEvent } from "../../core/events.js";
37
37
  import { withStateDb } from "../../core/state-db.js";
38
38
  import { warn } from "../../core/warn.js";
39
+ import { searchEntriesLexical } from "../../indexer/search/db-search.js";
39
40
  import { deactivateCanarySet, getActiveCanaries, getCanariesBySetId, insertCanaries, insertCycleMetrics, listActiveCanarySetIds, queryRecentCycleMetrics, } from "../../storage/repositories/canaries-repository.js";
40
41
  import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
41
42
  import { getAllEntries } from "../../storage/repositories/index-entries-repository.js";
42
- import { searchFts } from "../../storage/repositories/index-fts-repository.js";
43
43
  import { computeBigramDiversity, DEFAULT_MAX_GENERATION } from "./anti-collapse.js";
44
44
  import { getAllRankScores } from "./salience.js";
45
45
  // ── Defaults (mirrored in config-schema.ts ImproveCollapseDetectorSchema) ────
@@ -189,7 +189,7 @@ export function normHash(text) {
189
189
  * Returns the 0-based rank of the first hit, or -1.
190
190
  */
191
191
  function scoreCanary(indexDb, canary, k) {
192
- const results = searchFts(indexDb, canary.query, k);
192
+ const results = searchEntriesLexical(indexDb, canary.query, k);
193
193
  const anchorConceptId = canaryConceptId(canary.anchor_ref);
194
194
  for (let i = 0; i < Math.min(results.length, k); i++) {
195
195
  const r = results[i];
@@ -24,7 +24,7 @@ import { callStructured, preflightStructuredLlmRunner } from "../../llm/structur
24
24
  import { getBodyEmbeddings, upsertBodyEmbeddings } from "../../storage/repositories/embeddings-repository.js";
25
25
  import { closeDatabase, openExistingDatabase, openReadonlyExistingDatabase, } from "../../storage/repositories/index-connection.js";
26
26
  import { findEntryIdByRef, getAllEntries, getEntryById } from "../../storage/repositories/index-entries-repository.js";
27
- import { getNeighborsByEntryId } from "../../storage/repositories/index-vec-repository.js";
27
+ import { getNeighborsByEntryId } from "../../storage/repositories/units-repository.js";
28
28
  import { isProposalSkipped, listProposals, listProposalsReadOnly, proposalContent, } from "../proposal/repository.js";
29
29
  import { hasSupersededStatus, validateProposalFrontmatter } from "../proposal/validators/proposal-quality-validators.js";
30
30
  import { DEFAULT_RANDOM_CLUSTER_FRACTION } from "./anti-collapse.js";
@@ -1394,9 +1394,11 @@ export function narrowToIncrementalCandidates(memories, since, warnings, neighbo
1394
1394
  const id = findEntryIdByRef(db, conceptIdFromTypeName("memory", m.name));
1395
1395
  if (id === undefined)
1396
1396
  continue;
1397
- for (const hit of getNeighborsByEntryId(db, id, neighborsPerChanged + 1)) {
1398
- if (hit.id === id)
1399
- continue;
1397
+ // index-redesign-contract.md B5f item 4 `getNeighborsByEntryId`
1398
+ // (units-repository.ts) already excludes the querying entry itself, so
1399
+ // `neighborsPerChanged` genuinely OTHER neighbours are requested
1400
+ // directly (no `+ 1` for self, no self-filter here).
1401
+ for (const hit of getNeighborsByEntryId(db, id, neighborsPerChanged)) {
1400
1402
  const entry = getEntryById(db, hit.id);
1401
1403
  if (!entry)
1402
1404
  continue;
@@ -249,7 +249,7 @@ export const improveCommand = defineCommand({
249
249
  },
250
250
  "skip-if-locked": {
251
251
  type: "boolean",
252
- description: "If another improve run already holds the lock, skip gracefully (exit 0) instead of failing with 'already running' (exit 78). Use for high-frequency scheduled runs so they don't pile up failures while a longer run is in progress.",
252
+ description: "If another improve run already holds the lock, skip gracefully (exit 0) instead of failing with 'already running' (exit 75). Use for high-frequency scheduled runs so they don't pile up failures while a longer run is in progress.",
253
253
  default: false,
254
254
  },
255
255
  "require-engines": {
@@ -1027,10 +1027,19 @@ async function finalizeProposalTransaction(txn, target, proposal, ctx) {
1027
1027
  }
1028
1028
  let accepted = getProposal(p.stashDir, p.proposalId, ctx);
1029
1029
  if (txn.journal.phase === "proposal-persisted") {
1030
- if (!(await indexWrittenAssets(txn.journal.root, [p.assetPath], { bundleId: target.source.name }))) {
1031
- throw new Error(`Proposal ${p.proposalId} index finalization failed.`);
1030
+ if (await indexWrittenAssets(txn.journal.root, [p.assetPath], { bundleId: target.source.name })) {
1031
+ advanceTxn(txn, "index-finalized");
1032
+ }
1033
+ else {
1034
+ // `indexWrittenAssets`'s contract (index-written-assets.ts): `false`
1035
+ // means the asset write already stands, but the index itself needs a
1036
+ // manual `akm index` — warn and continue, the same as `source clone`
1037
+ // treats the same return, instead of failing the whole accept/revert.
1038
+ // Leave the phase at "proposal-persisted" (do not advance) so a later
1039
+ // recovery of this journal retries the index update rather than
1040
+ // skipping it as already done.
1041
+ warn(`${p.operation === "accept" ? "Accept" : "Revert"} of ${p.ref} succeeded, but its index update failed; run \`akm index\` to refresh it.`);
1032
1042
  }
1033
- advanceTxn(txn, "index-finalized");
1034
1043
  }
1035
1044
  if (txn.journal.phase === "index-finalized") {
1036
1045
  accepted = getProposal(p.stashDir, p.proposalId, ctx);
@@ -25,8 +25,6 @@ import { copySearchHitAttribution, getSearchHitAttribution, usageEventAttributio
25
25
  import { findSourceForPath, resolveSourceEntries } from "../../indexer/search/search-source.js";
26
26
  import { insertUsageEvent } from "../../indexer/usage/usage-events.js";
27
27
  import { estimateTokenCount } from "../../llm/embedders/remote.js";
28
- import { tryLlmFeature } from "../../llm/feature-gate.js";
29
- import { rerankDocuments } from "../../llm/rerank-client.js";
30
28
  import { truncateDescription } from "../../output/shapes/helpers.js";
31
29
  import { TELEMETRY_BUSY_TIMEOUT_MS, withIndexDb } from "../../storage/repositories/index-db.js";
32
30
  import { findEntryIdByRef, getItemRefById } from "../../storage/repositories/index-entries-repository.js";
@@ -147,7 +145,7 @@ export async function curateSearchResults(query, result, limit, selectedType, ev
147
145
  // fixtures) with a `SearchResponse` that was never type-filtered.
148
146
  const stashHits = selectedType && selectedType !== "any" ? allStashHits.filter((hit) => hit.type === selectedType) : allStashHits;
149
147
  const selected = selectCuratedStashHits(query, stashHits, limit);
150
- const selectedStashHits = await maybeRerankCuratedStashHits(query, selected.selected);
148
+ const selectedStashHits = selected.selected;
151
149
  const supportRefsByRef = selected.supportRefsByRef;
152
150
  // F4/R-019: respect `--limit` for registry fill instead of hard-capping it
153
151
  // at a bare literal 2 — the remaining slots after stash hits ARE the cap.
@@ -171,10 +169,16 @@ export async function curateSearchResults(query, result, limit, selectedType, ev
171
169
  /**
172
170
  * Pack a curate result's stash hits into a single token-budgeted blob:
173
171
  * resolve each hit's content via the SAME path `akm show` uses
174
- * (`akmShowUnified` this also means a `ref#fragment` hit packs just the
175
- * matched section), then greedily accumulate hits, in the ranking order
176
- * `curateSearchResults` already produced, until the next hit would exceed
177
- * `budgetTokens`.
172
+ * (`akmShowUnified`, called with `item.selectedRef ?? item.ref` this also
173
+ * means a hit whose search match was a Markdown fragment packs just the
174
+ * matched section, not the whole entry `item.ref` now always addresses; see
175
+ * index-redesign-contract.md B5f item 1), then greedily accumulate hits, in
176
+ * the ranking order `curateSearchResults` already produced, until the next
177
+ * hit would exceed `budgetTokens`. Each packed item's `ref` is that SAME
178
+ * fetched ref (`item.selectedRef ?? item.ref`), mirroring
179
+ * `enrichCuratedStashHit`'s own `contentRef` convention — labelling a packed
180
+ * fragment with the bare entry ref would let a later `akm show <ref>` return
181
+ * a different, larger document than what was actually packed and budgeted.
178
182
  *
179
183
  * Registry hits are never packed — only `CuratedStashItem`s (locked
180
184
  * contract, AGENTS.md: registry results stay separate/opt-in).
@@ -188,9 +192,17 @@ export async function packCuratedHits(result, budgetTokens) {
188
192
  const packed = [];
189
193
  let used = 0;
190
194
  for (const item of stashItems) {
195
+ // item 4 — record the ref actually FETCHED (`contentRef`), mirroring
196
+ // `enrichCuratedStashHit`'s own convention: `item.ref` is now always the
197
+ // bare entry ref, but a body-only match's content came from the
198
+ // fragment-qualified `selectedRef`. Recording `item.ref` here labelled a
199
+ // packed fragment with the whole entry's ref, so a consumer that later
200
+ // ran `akm show <that ref>` got a different, larger document than what
201
+ // was actually packed and budgeted.
202
+ const contentRef = item.selectedRef ?? item.ref;
191
203
  let shown;
192
204
  try {
193
- shown = await akmShowUnified({ ref: item.ref, skipLogging: true });
205
+ shown = await akmShowUnified({ ref: contentRef, skipLogging: true });
194
206
  }
195
207
  catch {
196
208
  continue;
@@ -198,7 +210,7 @@ export async function packCuratedHits(result, budgetTokens) {
198
210
  const content = shown.content ?? shown.template ?? shown.prompt ?? "";
199
211
  const tokens = estimateTokenCount(content);
200
212
  if (used + tokens <= budgetTokens) {
201
- packed.push({ ref: item.ref, tokens, content });
213
+ packed.push({ ref: contentRef, tokens, content });
202
214
  used += tokens;
203
215
  continue;
204
216
  }
@@ -206,7 +218,7 @@ export async function packCuratedHits(result, budgetTokens) {
206
218
  const remaining = budgetTokens - used;
207
219
  if (remaining > 0) {
208
220
  const truncated = content.slice(0, remaining * 4);
209
- packed.push({ ref: item.ref, tokens: estimateTokenCount(truncated), content: truncated });
221
+ packed.push({ ref: contentRef, tokens: estimateTokenCount(truncated), content: truncated });
210
222
  used += estimateTokenCount(truncated);
211
223
  }
212
224
  }
@@ -215,9 +227,15 @@ export async function packCuratedHits(result, budgetTokens) {
215
227
  return { query: result.query, budget: budgetTokens, tokens: used, items: packed };
216
228
  }
217
229
  async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs, eventSource) {
230
+ // index-redesign-contract.md B5f item 1 — `hit.ref` is always the bare entry
231
+ // ref now; `contentRef` is the fragment-qualified ref when the search match
232
+ // was a Markdown fragment (`hit.selectedRef`), so the preview/description
233
+ // resolved below and the curated item's own `followUp` still land on the
234
+ // matched section instead of regressing to the whole entry.
235
+ const contentRef = hit.selectedRef ?? hit.ref;
218
236
  let shown;
219
237
  try {
220
- shown = await akmShowUnified({ ref: hit.ref, eventSource, skipLogging: true });
238
+ shown = await akmShowUnified({ ref: contentRef, eventSource, skipLogging: true });
221
239
  }
222
240
  catch {
223
241
  shown = undefined;
@@ -235,10 +253,13 @@ async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs, even
235
253
  type: shown?.type ?? hit.type,
236
254
  name: shown?.name ?? hit.name,
237
255
  ref: hit.ref,
256
+ ...(hit.selectedRef ? { selectedRef: hit.selectedRef } : {}),
238
257
  path: shown?.path ?? hit.path,
239
258
  editable: shown?.editable ?? hit.editable ?? false,
240
259
  ...((shown?.editable ?? hit.editable ?? false) === false
241
- ? { editHint: shown?.editHint ?? hit.editHint ?? `This asset is read-only. Inspect it with: akm show ${hit.ref}` }
260
+ ? {
261
+ editHint: shown?.editHint ?? hit.editHint ?? `This asset is read-only. Inspect it with: akm show ${hit.ref}`,
262
+ }
242
263
  : {}),
243
264
  ...(description ? { description } : {}),
244
265
  ...(preview ? { preview } : {}),
@@ -246,7 +267,7 @@ async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs, even
246
267
  ...(shown?.parameters?.length ? { parameters: shown.parameters } : {}),
247
268
  ...(shown?.run ? { run: shown.run } : {}),
248
269
  ...(mergedSupportRefs.length > 0 ? { supportRefs: mergedSupportRefs } : {}),
249
- followUp: `akm show ${hit.ref}`,
270
+ followUp: `akm show ${contentRef}`,
250
271
  reason: buildCuratedReason(query, shown?.type ?? hit.type),
251
272
  ...(hit.score !== undefined ? { score: hit.score } : {}),
252
273
  };
@@ -596,37 +617,6 @@ function appendCurateSupportRef(supportRefsByRef, ownerRef, supportRef) {
596
617
  return;
597
618
  supportRefsByRef.set(ownerRef, [...existing, supportRef]);
598
619
  }
599
- /** Default number of `selectCuratedStashHits` candidates sent to the reranker when `search.curateRerank.topN` isn't set. */
600
- const DEFAULT_CURATE_RERANK_TOP_N = 8;
601
- /**
602
- * Optional cross-encoder rerank pass over curate's already-selected, already-
603
- * ranked candidates (#951). Disabled by default (`search.curateRerank.enabled`
604
- * is falsy) and, when enabled, best-effort: any failure (misconfigured
605
- * endpoint, network error, timeout, malformed response) falls back to
606
- * `selectCuratedStashHits`'s own ranking unchanged — a reranker outage must
607
- * never turn into a curate failure.
608
- *
609
- * Only the top `topN` (default {@link DEFAULT_CURATE_RERANK_TOP_N}) already-
610
- * selected hits are sent (bounded request size); anything past that keeps its
611
- * original position appended after the reranked prefix.
612
- */
613
- async function maybeRerankCuratedStashHits(query, hits) {
614
- if (hits.length <= 1)
615
- return hits;
616
- const config = loadConfig();
617
- const rerankConfig = config.search?.curateRerank;
618
- return tryLlmFeature("curate_rerank", config, async () => {
619
- const topN = rerankConfig?.topN ?? DEFAULT_CURATE_RERANK_TOP_N;
620
- const head = hits.slice(0, topN);
621
- const tail = hits.slice(topN);
622
- const documents = head.map((hit) => [hit.name, hit.description].filter(Boolean).join(" — "));
623
- const ranked = await rerankDocuments(rerankConfig ?? {}, query, documents);
624
- const rerankedHead = ranked
625
- .map(({ index }) => head[index])
626
- .filter((hit) => hit !== undefined);
627
- return [...rerankedHead, ...tail];
628
- }, hits, { timeoutMs: rerankConfig?.timeoutMs ?? null });
629
- }
630
620
  function selectCuratedStashHits(query, hits, limit) {
631
621
  const intent = parseCurateIntent(query);
632
622
  const collapsed = collapseCurateFamilies(query, hits);
@@ -17,6 +17,8 @@ import { appendEvent } from "../../core/events.js";
17
17
  import { resolveReadSources } from "../../indexer/read-preflight.js";
18
18
  import { searchLocal } from "../../indexer/search/db-search.js";
19
19
  import { getSearchHitAttribution, usageEventAttributionMetadata, } from "../../indexer/search/search-attribution.js";
20
+ import { tryLlmFeature } from "../../llm/feature-gate.js";
21
+ import { rerankDocuments } from "../../llm/rerank-client.js";
20
22
  import { getEntryIdByFilePath, getItemRefById } from "../../storage/repositories/index-entries-repository.js";
21
23
  // Eagerly import source providers to trigger self-registration before the
22
24
  // indexer or path-resolution code runs.
@@ -111,11 +113,21 @@ export async function akmSearch(input) {
111
113
  disableProjectContext: input.disableProjectContext === true,
112
114
  disableScopedUtility: input.disableScopedUtility === true,
113
115
  });
116
+ // #951 (moved from curate in 0.9.16 — the pass was always meant for
117
+ // search). Applied ONCE here, to LOCAL hits only, before the source
118
+ // branches below divide the same `localResult.hits` between the "local"
119
+ // and "all" responses — so both get the rerank and neither double-applies
120
+ // it. `localResult` is `undefined` for `source === "registry"`, so a
121
+ // registry-only search never reaches this call and never pays for the
122
+ // reranker's HTTP request; registry hits are never reranked (registry
123
+ // results staying separate from stash hits is a locked contract,
124
+ // AGENTS.md).
125
+ const rerankedLocalHits = localResult ? await maybeRerankSearchHits(query, localResult.hits, config) : undefined;
114
126
  const registryResult = source === "local"
115
127
  ? undefined
116
128
  : await searchRegistry(query, { limit, includeAssets: input.assets === true, registries: config.registries });
117
129
  if (source === "local") {
118
- const localHits = localResult?.hits ?? [];
130
+ const localHits = rerankedLocalHits ?? [];
119
131
  const hasResults = localHits.length > 0;
120
132
  const response = {
121
133
  schemaVersion: 1,
@@ -161,7 +173,7 @@ export async function akmSearch(input) {
161
173
  return response;
162
174
  }
163
175
  // source === "all"
164
- const allStashHits = (localResult?.hits ?? []).slice(0, limit);
176
+ const allStashHits = (rerankedLocalHits ?? []).slice(0, limit);
165
177
  const warnings = [...(localResult?.warnings ?? []), ...(registryResult?.warnings ?? [])];
166
178
  const hasResults = allStashHits.length > 0 || registryHits.length > 0;
167
179
  const response = {
@@ -182,6 +194,42 @@ export async function akmSearch(input) {
182
194
  function usageSearchMode(mode) {
183
195
  return mode === "semantic" ? "semantic" : "keyword";
184
196
  }
197
+ /** Default number of `searchLocal`'s already-ranked LOCAL hits sent to the reranker when `search.rerank.topN` isn't set. */
198
+ const DEFAULT_SEARCH_RERANK_TOP_N = 8;
199
+ /**
200
+ * Optional cross-encoder rerank pass over `akm search`'s already-ranked LOCAL
201
+ * hits (#951, moved from `akm curate` in 0.9.16 — the pass was always meant
202
+ * for search). Disabled by default (`search.rerank.enabled` is falsy) and,
203
+ * when enabled, best-effort: any failure (misconfigured endpoint, network
204
+ * error, timeout, malformed response, an out-of-range or duplicate index in
205
+ * the response) falls back to `searchLocal`'s own ranking unchanged — a
206
+ * reranker outage must never turn into a search failure.
207
+ *
208
+ * Only the top `topN` (default {@link DEFAULT_SEARCH_RERANK_TOP_N}) hits are
209
+ * sent (bounded request size); anything past that keeps its original
210
+ * position appended after the reranked prefix. The reranker changes ARRAY
211
+ * ORDER only — each hit's own `score` is left untouched as the retrieval
212
+ * score (see docs/reference/cli.md and docs/reference/configuration.md for
213
+ * why: `SearchHit.score` is a locked `[0,1]` contract downstream consumers
214
+ * compare and threshold on, while ordering is the field a rerank-aware
215
+ * consumer reads).
216
+ */
217
+ async function maybeRerankSearchHits(query, hits, config) {
218
+ if (hits.length <= 1)
219
+ return hits;
220
+ const rerankConfig = config.search?.rerank;
221
+ return tryLlmFeature("search_rerank", config, async () => {
222
+ const topN = rerankConfig?.topN ?? DEFAULT_SEARCH_RERANK_TOP_N;
223
+ const head = hits.slice(0, topN);
224
+ const tail = hits.slice(topN);
225
+ const documents = head.map((hit) => [hit.name, hit.description].filter(Boolean).join(" — "));
226
+ const ranked = await rerankDocuments(rerankConfig ?? {}, query, documents);
227
+ const rerankedHead = ranked
228
+ .map(({ index }) => head[index])
229
+ .filter((hit) => hit !== undefined);
230
+ return [...rerankedHead, ...tail];
231
+ }, hits, { timeoutMs: rerankConfig?.timeoutMs ?? null });
232
+ }
185
233
  function maybeLogSearchEvent(input, query, response, mode) {
186
234
  if (input.skipLogging)
187
235
  return;
@@ -0,0 +1,99 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * `akm index status` — a cheap, read-only snapshot of `index.db`'s current
6
+ * state: files tracked, entries, unit coverage for the active embedding
7
+ * identity, and the last reconcile time. Mirrors `assembleInfo`'s
8
+ * absent/inaccessible handling (`src/commands/sources/info.ts`) so a missing
9
+ * index reads as the ordinary first-run state and an unreadable one is
10
+ * reported, never silently presented as empty (#791).
11
+ */
12
+ import { classifyPathAccess, describeInaccessiblePath } from "../../core/path-access.js";
13
+ import { getDbPath } from "../../core/paths.js";
14
+ import { error } from "../../core/warn.js";
15
+ import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
16
+ import { getEntryCount } from "../../storage/repositories/index-entries-repository.js";
17
+ import { getMeta } from "../../storage/repositories/index-meta-repository.js";
18
+ function emptyStatus(indexPath) {
19
+ return {
20
+ indexPath,
21
+ files: 0,
22
+ entries: 0,
23
+ units: { total: 0, withVector: 0, pending: 0 },
24
+ activeIdentity: null,
25
+ lastReconcileAt: null,
26
+ builtAt: null,
27
+ };
28
+ }
29
+ function readUnitsStatus(db, identity) {
30
+ // Mirrors `drainEmbeddingQueue`'s own candidate set (`selectAllUnitHashes`,
31
+ // src/indexer/drain.ts) — `unit_texts`, not `entry_units` — so a stale/
32
+ // orphaned unit_texts row is counted here exactly as it will be by the
33
+ // next drain, instead of understating the backlog. (Reconcile prunes the
34
+ // hashes each write itself replaced, and sweeps the whole table at the end
35
+ // of a full run, so orphans are bounded — but "bounded" is not "none", and
36
+ // this count must match the drain either way.)
37
+ const total = db.prepare("SELECT COUNT(DISTINCT unit_hash) AS n FROM unit_texts").get().n;
38
+ const withVector = identity
39
+ ? db
40
+ .prepare("SELECT COUNT(DISTINCT ut.unit_hash) AS n FROM unit_texts ut " +
41
+ "JOIN units u ON u.unit_hash = ut.unit_hash AND u.identity = ?")
42
+ .get(identity).n
43
+ : 0;
44
+ return { total, withVector, pending: Math.max(0, total - withVector) };
45
+ }
46
+ /**
47
+ * Assemble `akm index status`'s envelope.
48
+ *
49
+ * @param options.dbPath - Override the database path (useful for testing)
50
+ */
51
+ export function assembleIndexStatus(options) {
52
+ const resolvedDbPath = options?.dbPath ?? getDbPath();
53
+ const empty = emptyStatus(resolvedDbPath);
54
+ // "Absent" is the ordinary first-run state; "inaccessible" is a fault that
55
+ // must not present as an empty index (#791).
56
+ const { access, code } = classifyPathAccess(resolvedDbPath);
57
+ if (access === "absent")
58
+ return empty;
59
+ if (access === "inaccessible") {
60
+ const detail = describeInaccessiblePath(resolvedDbPath, code);
61
+ error(`[akm index status] index database is not readable: ${detail}`);
62
+ return { ...empty, unreadable: detail };
63
+ }
64
+ let db;
65
+ try {
66
+ db = openExistingDatabase(resolvedDbPath);
67
+ const files = db.prepare("SELECT COUNT(*) AS n FROM files").get().n;
68
+ const identity = getMeta(db, "embeddingIdentity") ?? null;
69
+ return {
70
+ indexPath: resolvedDbPath,
71
+ files,
72
+ entries: getEntryCount(db),
73
+ units: readUnitsStatus(db, identity),
74
+ activeIdentity: identity,
75
+ lastReconcileAt: getMeta(db, "lastReconcileAt") ?? null,
76
+ builtAt: getMeta(db, "builtAt") ?? null,
77
+ };
78
+ }
79
+ catch (err) {
80
+ const detail = String(err instanceof Error ? err.message : err);
81
+ error(`[akm index status] failed to read index status from ${resolvedDbPath}: ${detail}`);
82
+ // A path that classified as accessible can still fail to open as a
83
+ // database (corrupt content, a truncated file, an ABI mismatch) — that
84
+ // is exactly the "unreadable, not silently empty" case #791 exists for,
85
+ // just discovered a step later than the access-classification check
86
+ // above rather than by it.
87
+ return { ...empty, unreadable: detail };
88
+ }
89
+ finally {
90
+ if (db) {
91
+ try {
92
+ closeDatabase(db);
93
+ }
94
+ catch {
95
+ // Best-effort close; the read already happened.
96
+ }
97
+ }
98
+ }
99
+ }