akm-cli 0.9.15 → 0.9.16-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +144 -0
  2. package/dist/assets/tasks/core/index-refresh.yml +1 -1
  3. package/dist/cli/retired-commands.js +2 -0
  4. package/dist/cli/unknown-flags.js +36 -3
  5. package/dist/commands/improve/collapse-detector.js +2 -2
  6. package/dist/commands/improve/consolidate.js +6 -4
  7. package/dist/commands/improve/improve-cli.js +1 -1
  8. package/dist/commands/proposal/repository.js +12 -3
  9. package/dist/commands/read/curate.js +34 -44
  10. package/dist/commands/read/search.js +50 -2
  11. package/dist/commands/sources/index-status.js +99 -0
  12. package/dist/commands/sources/info.js +8 -8
  13. package/dist/commands/sources/installed-stashes.js +33 -12
  14. package/dist/commands/sources/source-add.js +21 -6
  15. package/dist/commands/sources/stash-cli.js +119 -111
  16. package/dist/core/adapter/adapters/akm-adapter.js +35 -3
  17. package/dist/core/adapter/adapters/akm-metadata.js +11 -1
  18. package/dist/core/asset/asset-placement.js +35 -0
  19. package/dist/core/config/schema/embedding.js +7 -30
  20. package/dist/core/config/schema/search.js +11 -9
  21. package/dist/core/errors.js +5 -2
  22. package/dist/core/hash.js +18 -0
  23. package/dist/core/maintenance-barrier.js +8 -6
  24. package/dist/core/paths.js +0 -11
  25. package/dist/core/run-lock.js +5 -2
  26. package/dist/core/state/migrations.js +26 -1
  27. package/dist/core/state-db.js +63 -27
  28. package/dist/indexer/drain.js +306 -0
  29. package/dist/indexer/embedding-identity.js +20 -0
  30. package/dist/indexer/enrich.js +260 -0
  31. package/dist/indexer/ensure-index.js +5 -0
  32. package/dist/indexer/index-written-assets.js +133 -171
  33. package/dist/indexer/indexer.js +458 -1621
  34. package/dist/indexer/lookup/adapter-concept-owner.js +19 -5
  35. package/dist/indexer/passes/metadata.js +18 -1
  36. package/dist/indexer/reconcile.js +890 -0
  37. package/dist/indexer/scan/drain-dir.js +27 -70
  38. package/dist/indexer/scan/parse-file.js +66 -0
  39. package/dist/indexer/search/db-search.js +373 -89
  40. package/dist/indexer/search/ranking-contributors.js +21 -16
  41. package/dist/indexer/search/ranking.js +135 -57
  42. package/dist/indexer/units/unit.js +159 -0
  43. package/dist/llm/client.js +10 -1
  44. package/dist/llm/embedder.js +10 -3
  45. package/dist/llm/embedders/provider-limits.js +288 -0
  46. package/dist/llm/embedders/remote.js +133 -104
  47. package/dist/llm/feature-gate.js +4 -2
  48. package/dist/llm/rerank-client.js +3 -3
  49. package/dist/output/shapes/passthrough.js +1 -0
  50. package/dist/output/text/command-format.js +19 -13
  51. package/dist/output/text/helpers.js +1 -1
  52. package/dist/output/text/index.js +5 -2
  53. package/dist/scripts/akm-migrate-node.js +1141 -1237
  54. package/dist/scripts/akm-migrate.js +1141 -1237
  55. package/dist/setup/semantic-assets.js +2 -2
  56. package/dist/setup/steps/connection.js +3 -2
  57. package/dist/storage/repositories/files-repository.js +181 -0
  58. package/dist/storage/repositories/index-connection.js +1 -3
  59. package/dist/storage/repositories/index-entries-repository.js +77 -68
  60. package/dist/storage/repositories/index-entry-schema.js +16 -25
  61. package/dist/storage/repositories/index-fts-repository.js +29 -263
  62. package/dist/storage/repositories/index-meta-repository.js +0 -29
  63. package/dist/storage/repositories/index-schema.js +115 -122
  64. package/dist/storage/repositories/index-utility-repository.js +1 -1
  65. package/dist/storage/repositories/index-vec-repository.js +21 -334
  66. package/dist/storage/repositories/units-repository.js +510 -0
  67. package/docs/migration/release-notes/0.9.15.md +34 -36
  68. package/docs/migration/release-notes/0.9.16.md +110 -0
  69. package/docs/migration/release-notes/README.md +5 -0
  70. package/docs/reference/cli.md +93 -87
  71. package/docs/reference/configuration.md +128 -89
  72. package/docs/reference/data-and-telemetry.md +2 -1
  73. package/package.json +1 -1
  74. package/schemas/akm-config.json +2 -58
  75. package/dist/indexer/index-db-contention.js +0 -56
  76. package/dist/indexer/index-rebuild-lock.js +0 -73
  77. package/dist/indexer/materialize-embeddings.js +0 -771
  78. package/dist/indexer/passes/dir-staleness.js +0 -161
  79. package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
@@ -53,15 +53,15 @@ export function assembleInfo(options) {
53
53
  // Semantic status is read live from the index's own state, not a cached
54
54
  // verdict — a failed embed attempt at search time falls back to FTS and
55
55
  // reports that in the search response, it never disables the mode here.
56
- const semanticStatus = config.semanticSearchMode === "off"
57
- ? "disabled"
58
- : !indexStats.hasEmbeddings
59
- ? "pending"
60
- : indexStats.vecAvailable
61
- ? "ready-vec"
62
- : "ready-js";
56
+ // "ready-js" (a JS-computed cosine-similarity fallback for when the
57
+ // sqlite-vec extension is unavailable) is retired (index redesign, B5):
58
+ // the units vector store (`units_vec`) is a vec0 virtual table with no
59
+ // BLOB fallback, so there is nothing to have embeddings without also
60
+ // having the extension available — `vecAvailable` false and
61
+ // `hasEmbeddings` true together is no longer a reachable combination.
62
+ const semanticStatus = config.semanticSearchMode === "off" ? "disabled" : !indexStats.hasEmbeddings ? "pending" : "ready-vec";
63
63
  const searchModes = ["fts"];
64
- if (semanticStatus === "ready-js" || semanticStatus === "ready-vec") {
64
+ if (semanticStatus === "ready-vec") {
65
65
  searchModes.push("semantic", "hybrid");
66
66
  }
67
67
  return {
@@ -26,7 +26,7 @@ import { beginImmediateTransaction, getStateDbPath, openStateDatabase } from "..
26
26
  import { warn } from "../../core/warn.js";
27
27
  import { resolveGitContentRoot } from "../../core/write-source.js";
28
28
  import { withAssetMutationLease } from "../../indexer/index-writer-lock.js";
29
- import { akmIndex, runEmbeddingPass } from "../../indexer/indexer.js";
29
+ import { akmIndex, reclassifyIndexDbContention, runEmbeddingPass } from "../../indexer/indexer.js";
30
30
  import { compareAndSwapLockfileSnapshot, publishLockfileUpdate, readLockfile, readLockfileForUpdate, } from "../../integrations/lockfile.js";
31
31
  import { parseRegistryRef } from "../../registry/resolve.js";
32
32
  import { sha256Hex } from "../../runtime.js";
@@ -312,11 +312,15 @@ export async function akmRemove(input) {
312
312
  config: {
313
313
  sourceCount: getSources(updatedConfig).length,
314
314
  },
315
+ // `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
316
+ // removed (#index-redesign W6) — this response's own `index` shape is
317
+ // unchanged, so bridge from the renamed source field;
318
+ // `directoriesSkipped` was already always 0.
315
319
  index: {
316
320
  mode: index.mode,
317
321
  totalEntries: index.totalEntries,
318
- directoriesScanned: index.directoriesScanned,
319
- directoriesSkipped: index.directoriesSkipped,
322
+ directoriesScanned: index.sourcesScanned,
323
+ directoriesSkipped: 0,
320
324
  },
321
325
  };
322
326
  }
@@ -342,11 +346,15 @@ export async function akmRemove(input) {
342
346
  config: {
343
347
  sourceCount: getSources(updatedConfig).length,
344
348
  },
349
+ // `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
350
+ // removed (#index-redesign W6) — this response's own `index` shape is
351
+ // unchanged, so bridge from the renamed source field;
352
+ // `directoriesSkipped` was already always 0.
345
353
  index: {
346
354
  mode: index.mode,
347
355
  totalEntries: index.totalEntries,
348
- directoriesScanned: index.directoriesScanned,
349
- directoriesSkipped: index.directoriesSkipped,
356
+ directoriesScanned: index.sourcesScanned,
357
+ directoriesSkipped: 0,
350
358
  },
351
359
  };
352
360
  }
@@ -361,14 +369,13 @@ export async function akmRemove(input) {
361
369
  function readCurrentIndexSummary() {
362
370
  const db = openReadonlyExistingDatabase(getDbPath());
363
371
  if (!db) {
364
- return { mode: "incremental", totalEntries: 0, directoriesScanned: 0, directoriesSkipped: 0 };
372
+ return { mode: "incremental", totalEntries: 0, sourcesScanned: 0 };
365
373
  }
366
374
  try {
367
375
  return {
368
376
  mode: "incremental",
369
377
  totalEntries: getAllEntries(db).length,
370
- directoriesScanned: 0,
371
- directoriesSkipped: 0,
378
+ sourcesScanned: 0,
372
379
  };
373
380
  }
374
381
  finally {
@@ -390,11 +397,15 @@ function buildUpdateResponse(stashDir, target, all, processed, opts) {
390
397
  config: {
391
398
  sourceCount: getSources(finalConfig).length,
392
399
  },
400
+ // `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
401
+ // removed (#index-redesign W6) — `UpdateResponse["index"]`'s own shape is
402
+ // unchanged, so bridge from the renamed source field;
403
+ // `directoriesSkipped` was already always 0.
393
404
  index: {
394
405
  mode: index.mode,
395
406
  totalEntries: index.totalEntries,
396
- directoriesScanned: index.directoriesScanned,
397
- directoriesSkipped: index.directoriesSkipped,
407
+ directoriesScanned: index.sourcesScanned,
408
+ directoriesSkipped: 0,
398
409
  ...(index.scanComplete !== undefined ? { scanComplete: index.scanComplete } : {}),
399
410
  // A real embedding pass (`akmIndex`/`runEmbeddingPass`) reports its own
400
411
  // verified `semanticStatus`. When no pass ran this update (the
@@ -462,7 +473,9 @@ function openUnifiedUpdateTransaction() {
462
473
  candidate.exec(`ATTACH DATABASE ${sqliteStringLiteral(statePath)} AS "${UPDATE_STATE_SCHEMA}"`);
463
474
  // openIndexDatabase invokes this before ensureSchema, so the outer
464
475
  // transaction begins before the first update-owned index mutation.
465
- beginImmediateTransaction(candidate);
476
+ // `candidate` is the index.db connection, so contention here must
477
+ // report as INDEX_DB_CONTENDED, not the state.db default.
478
+ beginImmediateTransaction(candidate, "index");
466
479
  },
467
480
  });
468
481
  return {
@@ -540,7 +553,15 @@ async function runPostCommitEmbeddingPass(index) {
540
553
  return { ...index, verification };
541
554
  }
542
555
  catch (error) {
543
- const message = error instanceof Error ? error.message : String(error);
556
+ // Run the failure through the same index.db contention classifier the
557
+ // drain catch uses (indexer.ts's reclassifyIndexDbContention) before
558
+ // building the message: this catch's first statement, openIndexDatabase
559
+ // (init runs ensureSchema and writes), can throw the raw SQLite driver
560
+ // error under contention, and this was the last surviving non-fatal
561
+ // catch that printed that raw text verbatim to the operator instead of
562
+ // reporting it as index.db contention.
563
+ const reportedError = reclassifyIndexDbContention(error);
564
+ const message = reportedError instanceof Error ? reportedError.message : String(reportedError);
544
565
  warn(`[akm bundle update] post-commit embedding pass failed: ${message}`);
545
566
  return {
546
567
  ...index,
@@ -86,8 +86,13 @@ async function addLocalSource(ref, sourcePath, stashDir, explicitName, explicitA
86
86
  index: {
87
87
  mode: index.mode,
88
88
  totalEntries: index.totalEntries,
89
- directoriesScanned: index.directoriesScanned,
90
- directoriesSkipped: index.directoriesSkipped,
89
+ // `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
90
+ // removed (#index-redesign W6: reconcile is a flat per-file stat walk,
91
+ // never a directory walk) — this response's own `directoriesScanned`/
92
+ // `directoriesSkipped` shape is unchanged, so bridge from the renamed
93
+ // source field; `directoriesSkipped` was already always 0.
94
+ directoriesScanned: index.sourcesScanned,
95
+ directoriesSkipped: 0,
91
96
  ...(index.warnings?.length ? { warnings: index.warnings } : {}),
92
97
  },
93
98
  };
@@ -151,8 +156,13 @@ async function addWebsiteSource(ref, stashDir, name, options) {
151
156
  index: {
152
157
  mode: index.mode,
153
158
  totalEntries: index.totalEntries,
154
- directoriesScanned: index.directoriesScanned,
155
- directoriesSkipped: index.directoriesSkipped,
159
+ // `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
160
+ // removed (#index-redesign W6: reconcile is a flat per-file stat walk,
161
+ // never a directory walk) — this response's own `directoriesScanned`/
162
+ // `directoriesSkipped` shape is unchanged, so bridge from the renamed
163
+ // source field; `directoriesSkipped` was already always 0.
164
+ directoriesScanned: index.sourcesScanned,
165
+ directoriesSkipped: 0,
156
166
  ...(index.warnings?.length ? { warnings: index.warnings } : {}),
157
167
  },
158
168
  };
@@ -238,8 +248,13 @@ async function addRegistryStash(ref, stashDir, writable) {
238
248
  index: {
239
249
  mode: index.mode,
240
250
  totalEntries: index.totalEntries,
241
- directoriesScanned: index.directoriesScanned,
242
- directoriesSkipped: index.directoriesSkipped,
251
+ // `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
252
+ // removed (#index-redesign W6: reconcile is a flat per-file stat walk,
253
+ // never a directory walk) — this response's own `directoriesScanned`/
254
+ // `directoriesSkipped` shape is unchanged, so bridge from the renamed
255
+ // source field; `directoriesSkipped` was already always 0.
256
+ directoriesScanned: index.sourcesScanned,
257
+ directoriesSkipped: 0,
243
258
  ...(index.warnings?.length ? { warnings: index.warnings } : {}),
244
259
  },
245
260
  };
@@ -28,10 +28,9 @@
28
28
  * SIGINT/SIGTERM handlers in a try/finally — left byte-for-byte untouched.
29
29
  */
30
30
  import path from "node:path";
31
- import { defineCommand } from "citty";
32
31
  import * as p from "../../cli/clack.js";
33
32
  import { getParsedInvocation } from "../../cli/invocation.js";
34
- import { defineJsonCommand, GLOBAL_OUTPUT_ARGS, output, parseAllFlagValues, runWithJsonErrors } from "../../cli/shared.js";
33
+ import { defineGroupCommand, defineJsonCommand, GLOBAL_OUTPUT_ARGS, output, parseAllFlagValues, } from "../../cli/shared.js";
35
34
  import { assertFlatAssetName } from "../../core/asset/asset-create.js";
36
35
  import { parseFrontmatter } from "../../core/asset/frontmatter.js";
37
36
  import { isHttpUrl, resolveStashDir } from "../../core/common.js";
@@ -40,17 +39,51 @@ import { UsageError } from "../../core/errors.js";
40
39
  import { appendEvent } from "../../core/events.js";
41
40
  import { resolveBundleWriteTarget } from "../../core/mutation-target.js";
42
41
  import { getCacheDir } from "../../core/paths.js";
43
- import { clearLogFile, info, isVerbose, setLogFile } from "../../core/warn.js";
42
+ import { clearLogFile, info, isVerbose, setLogFile, warn } from "../../core/warn.js";
44
43
  import { resolveWriteTarget } from "../../core/write-source.js";
45
- import { releaseIndexRebuildLock, tryAcquireIndexRebuildLock } from "../../indexer/index-rebuild-lock.js";
44
+ import { DRAIN_BATCH_PROGRESS_PREFIX } from "../../indexer/drain.js";
46
45
  import { akmIndex } from "../../indexer/indexer.js";
46
+ import { RECONCILE_ROOT_PROGRESS_PREFIX } from "../../indexer/reconcile.js";
47
47
  import { getHyphenatedBoolean, getOutputMode } from "../../output/context.js";
48
48
  import { inferAssetName, mergeXrefsIntoContent, readKnowledgeInput, resolveSupersedesForWrite, resolveSupersedesWriteTarget, resolveXrefsForWrite, writeMarkdownAsset, } from "../read/knowledge.js";
49
+ import { assembleIndexStatus } from "./index-status.js";
49
50
  import { assembleInfo } from "./info.js";
50
- /** Matches the high-frequency per-committed-batch progress line (#954), excluded from non-verbose JSON-mode stderr. */
51
- const EMBEDDED_BATCH_PROGRESS_PATTERN = /^Embedded \d+\/\d+ entries\.$/;
52
- export const indexCommand = defineCommand({
53
- meta: { name: "index", description: "Build search index (incremental by default; --full forces full reindex)" },
51
+ /**
52
+ * The two high-frequency, one-line-per-unit-of-work progress lines (#954) —
53
+ * drain's per-batch commit line and reconcile's per-root "done" line —
54
+ * excluded from non-verbose, non-text (JSON/yaml/etc) stderr. Matched by the
55
+ * exact prefix each producer exports, not a re-derived regex, so the two
56
+ * never drift apart (index-redesign B5g): this used to be a regex tuned to
57
+ * the deleted per-entry pipeline's `Embedded N/M entries.` line, which never
58
+ * matched either replacement line, so every progress line reached stderr
59
+ * regardless of `--verbose`.
60
+ */
61
+ function isDetailProgressLine(message) {
62
+ return message.startsWith(DRAIN_BATCH_PROGRESS_PREFIX) || message.startsWith(RECONCILE_ROOT_PROGRESS_PREFIX);
63
+ }
64
+ export const indexStatusCommand = defineJsonCommand({
65
+ meta: {
66
+ name: "status",
67
+ description: "Show index.db's current state: files, entries, unit coverage, and the last reconcile time.",
68
+ },
69
+ run() {
70
+ output("index-status", assembleIndexStatus());
71
+ },
72
+ });
73
+ /**
74
+ * `akm index` = reconcile + drain (docs/plans/index-redesign.md). Still a raw
75
+ * group command (not `defineJsonCommand`) because its default body owns a
76
+ * spinner, an AbortController, and SIGINT/SIGTERM handlers in a try/finally;
77
+ * `defineGroupCommand` gives it `status` as a real subcommand while keeping
78
+ * that default body as plain `akm index`'s behavior (S-057 canonical
79
+ * bare-group rule does not apply here — a bare `akm index` has always run the
80
+ * indexer, and that stays).
81
+ */
82
+ export const indexCommand = defineGroupCommand({
83
+ meta: {
84
+ name: "index",
85
+ description: "Reconcile the search index and drain the embedding queue (--full forces a full re-derivation)",
86
+ },
54
87
  args: {
55
88
  // R-051: `index` is a raw `defineCommand` (not `defineJsonCommand`), so it
56
89
  // does not get `GLOBAL_OUTPUT_ARGS` for free. `--format`/`--detail`/
@@ -58,124 +91,99 @@ export const indexCommand = defineCommand({
58
91
  // extra positional for a stray value to fall into), so this is purely a
59
92
  // `--help` visibility / consistency fix, not a behavior change.
60
93
  ...GLOBAL_OUTPUT_ARGS,
61
- full: { type: "boolean", description: "Force full reindex", default: false },
62
- clean: {
63
- type: "boolean",
64
- description: "After indexing, remove any entries whose source file no longer exists on disk.",
65
- default: false,
66
- },
67
- "dry-run": {
94
+ full: {
68
95
  type: "boolean",
69
- description: "When combined with --clean, report stale entries without deleting them.",
96
+ description: "Force every file to be re-derived (ignore the unchanged-file shortcut), reconciling in place " +
97
+ "existing rows keep their id/embeddings/utility scores; nothing is dropped first.",
70
98
  default: false,
71
99
  },
72
100
  reembed: {
73
101
  type: "boolean",
74
- description: "Force re-embedding of every entry, bypassing the embedding-model-rename compatibility check.",
102
+ description: "Drop the active embedding identity's vectors, then re-embed every unit from scratch.",
75
103
  default: false,
76
104
  },
77
105
  "skip-if-locked": {
78
106
  type: "boolean",
79
- description: "If another `akm index` run already holds the rebuild lock, skip gracefully (exit 0) instead of contending with it. Use for scheduled/opportunistic index runs so they don't pile up against a longer run in progress.",
107
+ description: "Deprecated, no effect. Index runs no longer take a rebuild lock (docs/plans/index-redesign.md) " +
108
+ "every write is a short, idempotent, content-addressed transaction, so two concurrent index runs " +
109
+ "converge instead of contending. Kept only so existing scripts do not fail on an unknown flag.",
80
110
  default: false,
81
111
  },
82
112
  },
83
- async run({ args }) {
84
- await runWithJsonErrors(async () => {
85
- if (getHyphenatedBoolean(args, "enrich") || getParsedInvocation().getFlagValue("--enrich") !== undefined) {
86
- throw new UsageError("`akm index --enrich` has been removed. Plain `akm index` now performs metadata enrichment by default.");
87
- }
88
- if (getHyphenatedBoolean(args, "re-enrich") || getParsedInvocation().getFlagValue("--re-enrich") !== undefined) {
89
- throw new UsageError("`akm index --re-enrich` has been removed. Re-enrichment of index-time LLM passes is not exposed in this slice.");
90
- }
91
- // #956: opt-in, non-blocking rebuild lock — never gates a human-typed
92
- // `akm index` (it only warns and contends), but a scheduled/opportunistic
93
- // caller can pass --skip-if-locked to step aside instead of piling up
94
- // behind a run already in progress. Acquired before any other side
95
- // effect (log file, spinner) so a skip does neither.
96
- const lockAcquisition = tryAcquireIndexRebuildLock(args["skip-if-locked"]);
97
- if (lockAcquisition.state === "skipped") {
98
- output("index", {
99
- ok: true,
100
- skipped: {
101
- reason: "lock-held",
102
- pid: lockAcquisition.holder.pid,
103
- // #956: the launcher pid (when known) alongside the pid that
104
- // actually holds the lock every process listing and task log
105
- // shows the launcher pid, not the bun/node child's.
106
- launcherPid: lockAcquisition.holder.launcherPid,
107
- startedAt: lockAcquisition.holder.startedAt,
108
- },
109
- });
110
- return;
111
- }
112
- const outputMode = getOutputMode();
113
- const controller = new AbortController();
114
- const abort = () => controller.abort(new Error("index interrupted"));
115
- process.once("SIGINT", abort);
116
- process.once("SIGTERM", abort);
117
- const indexLogFile = path.join(getCacheDir(), "logs", "index", `${new Date().toISOString().replace(/[:.]/g, "-")}.log`);
118
- setLogFile(indexLogFile);
119
- const verbose = isVerbose();
120
- const spin = !verbose && outputMode.format === "text" ? p.spinner() : null;
113
+ subCommands: { status: indexStatusCommand },
114
+ async defaultRun({ args }) {
115
+ if (getHyphenatedBoolean(args, "enrich") || getParsedInvocation().getFlagValue("--enrich") !== undefined) {
116
+ throw new UsageError("`akm index --enrich` has been removed. Plain `akm index` now performs metadata enrichment by default.");
117
+ }
118
+ if (getHyphenatedBoolean(args, "re-enrich") || getParsedInvocation().getFlagValue("--re-enrich") !== undefined) {
119
+ throw new UsageError("`akm index --re-enrich` has been removed. Re-enrichment of index-time LLM passes is not exposed in this slice.");
120
+ }
121
+ if (args["skip-if-locked"]) {
122
+ warn("[index] --skip-if-locked is deprecated and has no effect — index runs no longer take a rebuild lock.");
123
+ }
124
+ const outputMode = getOutputMode();
125
+ const controller = new AbortController();
126
+ const abort = () => controller.abort(new Error("index interrupted"));
127
+ process.once("SIGINT", abort);
128
+ process.once("SIGTERM", abort);
129
+ const indexLogFile = path.join(getCacheDir(), "logs", "index", `${new Date().toISOString().replace(/[:.]/g, "-")}.log`);
130
+ setLogFile(indexLogFile);
131
+ const verbose = isVerbose();
132
+ const spin = !verbose && outputMode.format === "text" ? p.spinner() : null;
133
+ if (spin) {
134
+ spin.start(`Building search index${args.full ? " (full rebuild)" : ""}...`);
135
+ }
136
+ let latestMessage = "";
137
+ // Resolve the stash dir once at the `akm index` command boundary and
138
+ // thread it into the indexer (WI-9.10 CLI-wide sweep) — the indexer leaf
139
+ // no longer reads the ambient `resolveStashDir()`.
140
+ const stashDir = resolveStashDir();
141
+ try {
142
+ const result = await akmIndex({
143
+ stashDir,
144
+ full: args.full,
145
+ reembed: args.reembed,
146
+ onProgress: ({ phase, message, processed, total }) => {
147
+ latestMessage = message;
148
+ const progressPrefix = processed !== undefined && total !== undefined ? `[${processed}/${total}] ` : "";
149
+ if (verbose) {
150
+ info(`[index:${phase}] ${progressPrefix}${message}`);
151
+ }
152
+ else if (spin) {
153
+ spin.stop(`${progressPrefix}${message}`);
154
+ spin.start(`${progressPrefix}${message}`);
155
+ }
156
+ else if (!isDetailProgressLine(message)) {
157
+ // Non-verbose, non-text (JSON/yaml/etc) mode: silence used to be
158
+ // total until the run finished (#954) — a stalled
159
+ // run looked identical to "nothing written". Phase-start
160
+ // messages, the credential diagnostic, and the reconcile/drain
161
+ // totals now reach stderr here too; the high-frequency
162
+ // per-root `Reconciled "…"` and per-batch `[drain] batch N: …`
163
+ // lines are deliberately excluded — that would be spam, not a
164
+ // heartbeat. `--verbose` (the `if` branch above) still gets
165
+ // every one of them.
166
+ info(`[index:${phase}] ${progressPrefix}${message}`);
167
+ }
168
+ },
169
+ signal: controller.signal,
170
+ });
121
171
  if (spin) {
122
- spin.start(`Building search index${args.full ? " (full rebuild)" : ""}...`);
123
- }
124
- let latestMessage = "";
125
- // Resolve the stash dir once at the `akm index` command boundary and
126
- // thread it into the indexer (WI-9.10 CLI-wide sweep) — the indexer leaf
127
- // no longer reads the ambient `resolveStashDir()`.
128
- const stashDir = resolveStashDir();
129
- try {
130
- const result = await akmIndex({
131
- stashDir,
132
- full: args.full,
133
- clean: args.clean,
134
- dryRun: args["dry-run"],
135
- reembed: args.reembed,
136
- onProgress: ({ phase, message, processed, total }) => {
137
- latestMessage = message;
138
- const progressPrefix = processed !== undefined && total !== undefined ? `[${processed}/${total}] ` : "";
139
- if (verbose) {
140
- info(`[index:${phase}] ${progressPrefix}${message}`);
141
- }
142
- else if (spin) {
143
- spin.stop(`${progressPrefix}${message}`);
144
- spin.start(`${progressPrefix}${message}`);
145
- }
146
- else if (!EMBEDDED_BATCH_PROGRESS_PATTERN.test(message)) {
147
- // Non-verbose, non-text (JSON/yaml/etc) mode: silence used to be
148
- // total until the run finished (#954) — a stalled
149
- // run looked identical to "nothing written". Phase-start
150
- // messages and the embedding heartbeat now reach stderr here
151
- // too; the high-frequency per-batch `Embedded N/M entries.`
152
- // line (emitted after every committed batch)
153
- // is deliberately excluded — that would be spam, not a
154
- // heartbeat.
155
- info(`[index:${phase}] ${progressPrefix}${message}`);
156
- }
157
- },
158
- signal: controller.signal,
159
- });
160
- if (spin) {
161
- spin.stop(`Indexed ${result.totalEntries} assets.`);
162
- }
163
- output("index", result);
172
+ spin.stop(`Indexed ${result.totalEntries} assets.`);
164
173
  }
165
- catch (error) {
166
- if (spin) {
167
- spin.stop(latestMessage ? `Indexing failed after: ${latestMessage}` : "Indexing failed.");
168
- }
169
- throw error;
170
- }
171
- finally {
172
- clearLogFile();
173
- process.off("SIGINT", abort);
174
- process.off("SIGTERM", abort);
175
- if (lockAcquisition.state === "acquired")
176
- releaseIndexRebuildLock(lockAcquisition.ownership);
174
+ output("index", result);
175
+ }
176
+ catch (error) {
177
+ if (spin) {
178
+ spin.stop(latestMessage ? `Indexing failed after: ${latestMessage}` : "Indexing failed.");
177
179
  }
178
- });
180
+ throw error;
181
+ }
182
+ finally {
183
+ clearLogFile();
184
+ process.off("SIGINT", abort);
185
+ process.off("SIGTERM", abort);
186
+ }
179
187
  },
180
188
  });
181
189
  export const infoCommand = defineJsonCommand({
@@ -82,7 +82,7 @@
82
82
  import fs from "node:fs";
83
83
  import path from "node:path";
84
84
  import { applyPostContributorFields, applyPreContributorFields, extractPackageMetadata, getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "../../../indexer/passes/metadata.js";
85
- import { assetPathCandidatesForName, assetPathForName, deriveCanonicalAssetNameFromStashRoot, placementTypes, stashDirFor, stashDirNames, } from "../../asset/asset-placement.js";
85
+ import { assetPathCandidatesAreOrderedByPreference, assetPathCandidatesForName, assetPathForName, deriveCanonicalAssetNameFromStashRoot, placementTypes, stashDirFor, stashDirNames, } from "../../asset/asset-placement.js";
86
86
  import { parseFrontmatter } from "../../asset/frontmatter.js";
87
87
  import { executionDefaultsFromFrontmatter, renderMarkdownExecutionSource } from "../execution-source.js";
88
88
  import { recognizeMatch } from "../recognize-match.js";
@@ -489,7 +489,25 @@ export const akmAdapter = {
489
489
  * type's own stash subdir) and the LOOSE FALLBACK (authored anywhere else
490
490
  * in the bundle, so the canonical name is the file's full path relative to
491
491
  * the bundle root instead of the stash subdir). `assetPathCandidatesForName`
492
- * additionally expands `env`'s `.env`/`<name>.env` duality on each.
492
+ * additionally expands `env`'s `.env`/`<name>.env` duality, and memory's
493
+ * `<name>`/`<name>.derived` twin duality, on each.
494
+ *
495
+ * `priority` (#882 fix) carries each candidate's rank WITHIN its own root's
496
+ * list, but ONLY for a type whose duality is a declared, ORDERED
497
+ * preference per `assetPathCandidatesAreOrderedByPreference` (only
498
+ * `memory`, today) — `assetPathCandidatesForName` returns primary before
499
+ * derived-twin for that type, so index 0 is the declared winner when both
500
+ * exist. CANONICAL and LOOSE are separate lists whose ranks both start
501
+ * back at 0: two candidates that tie on rank (e.g. the canonical and loose
502
+ * spellings both being the primary, rank-0, form) are a genuine collision
503
+ * between two independently-authored files, not a declared preference —
504
+ * only a rank difference WITHIN one root's own list resolves silently.
505
+ * Every other type's candidates (including `env`'s `.env`/`default.env`
506
+ * pair — co-equal spellings, not an ordered preference, per that same
507
+ * predicate's doc comment) carry NO `priority`, unchanged from before
508
+ * #882: `resolveAdapterConceptOwner` treats priority-less candidates as
509
+ * tied, so more than one existing together still collides. See
510
+ * `AdapterReadCandidate.priority`'s doc comment.
493
511
  */
494
512
  readCandidates(c, conceptId) {
495
513
  const posix = conceptId.replace(/\\/g, "/");
@@ -503,9 +521,23 @@ export const akmAdapter = {
503
521
  return [];
504
522
  const canonical = assetPathCandidatesForName(type, path.join(c.root, head), rest);
505
523
  const loose = assetPathCandidatesForName(type, c.root, rest);
506
- return [...new Set([...canonical, ...loose])].map((candidatePath) => ({
524
+ if (!assetPathCandidatesAreOrderedByPreference(type)) {
525
+ return [...new Set([...canonical, ...loose])].map((candidatePath) => ({
526
+ path: candidatePath,
527
+ conceptId: posix,
528
+ }));
529
+ }
530
+ const priorityByPath = new Map();
531
+ for (const list of [canonical, loose]) {
532
+ list.forEach((candidatePath, rank) => {
533
+ if (!priorityByPath.has(candidatePath))
534
+ priorityByPath.set(candidatePath, rank);
535
+ });
536
+ }
537
+ return [...priorityByPath.keys()].map((candidatePath) => ({
507
538
  path: candidatePath,
508
539
  conceptId: posix,
540
+ priority: priorityByPath.get(candidatePath),
509
541
  }));
510
542
  },
511
543
  /**
@@ -172,7 +172,17 @@ export function foldRecognizedMetadata(rendererName, file) {
172
172
  const fm = parseFrontmatter(file.content()).data;
173
173
  applyFrontmatterDescriptionAndTags(fm, out);
174
174
  const hints = new Set();
175
- const source = nonEmptyString(fm.source);
175
+ // fix-ranking-derived-outranks-primary: `source:` on an ordinary
176
+ // memory is an author-written citation worth indexing as a hint, but
177
+ // on an inferred (`.derived`) twin it is ALWAYS the machine-written
178
+ // provenance backref `memory-inference.ts` writes (`memories/<parent>`
179
+ // — see its `FM_SOURCE`), already captured properly as
180
+ // `entry.derivedFrom` (metadata.ts). Folding that backref into
181
+ // searchHints too means the base memory's own name — almost always a
182
+ // query token whenever the base is relevant — auto-credits the twin
183
+ // via `search-hint-ranking`'s substring match, independent of whether
184
+ // the twin's own content actually matches the query.
185
+ const source = fm.inferred === true ? undefined : nonEmptyString(fm.source);
176
186
  if (source)
177
187
  hints.add(source);
178
188
  const fmObservedAt = nonEmptyString(fm.observed_at);
@@ -269,3 +269,38 @@ export function assetPathCandidatesForName(assetType, typeRoot, name) {
269
269
  const namedForm = path.join(typeRoot, base, "default.env");
270
270
  return [...new Set([primary, dotForm, namedForm])];
271
271
  }
272
+ /**
273
+ * Whether {@link assetPathCandidatesForName}'s returned list for `assetType`
274
+ * is an ORDERED preference — earlier candidates are a declared winner over
275
+ * later ones, so a physical-owner resolver may pick the earliest that exists
276
+ * with no throw when more than one does — or a set of CO-EQUAL spellings,
277
+ * where more than one existing together is a genuine authoring collision to
278
+ * report, not a preference to resolve silently. #882's fix (the memory
279
+ * `.derived`-twin bug) needs this distinction: attaching a declared rank to
280
+ * every multi-candidate type's list turned `env`'s co-equal collision into a
281
+ * silently-resolved false negative (#882 follow-up) — the two dualities this
282
+ * module documents are NOT the same kind of thing.
283
+ *
284
+ * Only `memory` is ordered. Its own doc comment above states a winner in so
285
+ * many words: "The plain `.md` file wins when both exist, so it stays
286
+ * `primary`" — `.derived` is a provenance marker on the SAME identity, not a
287
+ * second, independently-authored file.
288
+ *
289
+ * `env`'s `.env`/`default.env` duality is explicitly NOT ordered — the doc
290
+ * comment above says only that both spellings "derive the same canonical
291
+ * name" and a lookup "must consider both", never that one wins over the
292
+ * other. They are two independently authored files that happen to collide
293
+ * on one ref; both existing together is exactly the ambiguity
294
+ * `AdapterConceptCollisionError` exists to report.
295
+ *
296
+ * Callers that attach a preference rank to distinguish a declared duality
297
+ * from a genuine collision (`AdapterReadCandidate.priority`,
298
+ * `resolveAdapterConceptOwner` in `indexer/lookup/adapter-concept-owner.ts`)
299
+ * must consult this predicate per asset type rather than assuming every
300
+ * multi-candidate type behaves like `memory` — and must NOT special-case the
301
+ * literal `.derived` suffix or `assetType === "memory"` themselves; this is
302
+ * the one place that knowledge lives.
303
+ */
304
+ export function assetPathCandidatesAreOrderedByPreference(assetType) {
305
+ return assetType === "memory";
306
+ }
@@ -33,32 +33,6 @@ export const EmbeddingConnectionConfigSchema = z
33
33
  // `akm index` when ensureSchema rejects it (§24.2 "Semantic" gate).
34
34
  dimension: positiveInt.max(4096).optional(),
35
35
  localModel: z.string().min(1).optional(),
36
- /**
37
- * Per-document token cap applied BEFORE batching (default 512,
38
- * `DEFAULT_MAX_INPUT_TOKENS` in `src/llm/embedders/remote.ts`, #956).
39
- * The materializer truncates a document's embedded text to
40
- * this cap (head only, unicode-safe) instead of skipping it outright, so
41
- * one oversized entry can no longer fail a whole batch. Distinct from
42
- * `maxTokens` below, which bounds a whole HTTP REQUEST (many documents);
43
- * this bounds one DOCUMENT.
44
- */
45
- maxInputTokens: positiveInt.optional(),
46
- /**
47
- * Client-side per-request token budget — how many documents' estimated
48
- * tokens fit in one HTTP request (default `DEFAULT_TOKEN_BUDGET` = 6000
49
- * in `src/llm/embedders/remote.ts`). With the 512-token `maxInputTokens`
50
- * cap above, a request carries about 11 documents by default.
51
- */
52
- maxTokens: positiveInt.optional(),
53
- batchSize: positiveInt.optional(),
54
- /**
55
- * Ollama's `num_ctx` ONLY (#956) — sent verbatim as
56
- * `options.num_ctx` on the native `/api/embed` request. It no longer also
57
- * feeds the client-side request token budget (`maxTokens` above): the two
58
- * used to share this one field, so setting it for the server's context
59
- * window silently changed request batching too.
60
- */
61
- contextLength: positiveInt.optional(),
62
36
  ollamaOptions: EmbeddingOllamaOptionsSchema.optional(),
63
37
  /**
64
38
  * Per-request timeout in milliseconds for a remote embedding request
@@ -72,10 +46,13 @@ export const EmbeddingConnectionConfigSchema = z
72
46
  * Overrides the fixed in-flight request window (#954, added after field
73
47
  * evidence from multi-slot local servers). Bounded 1-16. Unset keeps
74
48
  * today's default: 1 for a loopback endpoint, 2 for a remote one
75
- * (`resolveEmbeddingConcurrency`, `src/llm/embedders/remote.ts`). Set it
76
- * only for an endpoint that genuinely serves parallel requests (llama.cpp
77
- * `--parallel N`, vLLM) — request SIZE (`batchSize`, `maxTokens`/
78
- * `contextLength`) remains the first throughput lever.
49
+ * (`resolveEmbeddingConcurrency`, `src/llm/embedders/remote.ts`), unless
50
+ * the provider's own probed slot count overrides it
51
+ * (`probeProviderLimits`, `src/llm/embedders/provider-limits.ts`, used by
52
+ * `akm index`'s drain queue). Set it only for an endpoint that genuinely
53
+ * serves parallel requests (llama.cpp `--parallel N`, vLLM) — request
54
+ * SIZE, packed against the provider's own probed context window, remains
55
+ * the first throughput lever.
79
56
  */
80
57
  concurrency: positiveInt.max(16).optional(),
81
58
  })