akm-cli 0.9.15 → 0.9.16-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +144 -0
- package/dist/assets/tasks/core/index-refresh.yml +1 -1
- package/dist/cli/retired-commands.js +2 -0
- package/dist/cli/unknown-flags.js +36 -3
- package/dist/commands/improve/collapse-detector.js +2 -2
- package/dist/commands/improve/consolidate.js +6 -4
- package/dist/commands/improve/improve-cli.js +1 -1
- package/dist/commands/proposal/repository.js +12 -3
- package/dist/commands/read/curate.js +34 -44
- package/dist/commands/read/search.js +50 -2
- package/dist/commands/sources/index-status.js +99 -0
- package/dist/commands/sources/info.js +8 -8
- package/dist/commands/sources/installed-stashes.js +33 -12
- package/dist/commands/sources/source-add.js +21 -6
- package/dist/commands/sources/stash-cli.js +119 -111
- package/dist/core/adapter/adapters/akm-adapter.js +35 -3
- package/dist/core/adapter/adapters/akm-metadata.js +11 -1
- package/dist/core/asset/asset-placement.js +35 -0
- package/dist/core/config/schema/embedding.js +7 -30
- package/dist/core/config/schema/search.js +11 -9
- package/dist/core/errors.js +5 -2
- package/dist/core/hash.js +18 -0
- package/dist/core/maintenance-barrier.js +8 -6
- package/dist/core/paths.js +0 -11
- package/dist/core/run-lock.js +5 -2
- package/dist/core/state/migrations.js +26 -1
- package/dist/core/state-db.js +63 -27
- package/dist/indexer/drain.js +306 -0
- package/dist/indexer/embedding-identity.js +20 -0
- package/dist/indexer/enrich.js +260 -0
- package/dist/indexer/ensure-index.js +5 -0
- package/dist/indexer/index-written-assets.js +133 -171
- package/dist/indexer/indexer.js +458 -1621
- package/dist/indexer/lookup/adapter-concept-owner.js +19 -5
- package/dist/indexer/passes/metadata.js +18 -1
- package/dist/indexer/reconcile.js +890 -0
- package/dist/indexer/scan/drain-dir.js +27 -70
- package/dist/indexer/scan/parse-file.js +66 -0
- package/dist/indexer/search/db-search.js +373 -89
- package/dist/indexer/search/ranking-contributors.js +21 -16
- package/dist/indexer/search/ranking.js +135 -57
- package/dist/indexer/units/unit.js +159 -0
- package/dist/llm/client.js +10 -1
- package/dist/llm/embedder.js +10 -3
- package/dist/llm/embedders/provider-limits.js +288 -0
- package/dist/llm/embedders/remote.js +133 -104
- package/dist/llm/feature-gate.js +4 -2
- package/dist/llm/rerank-client.js +3 -3
- package/dist/output/shapes/passthrough.js +1 -0
- package/dist/output/text/command-format.js +19 -13
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/index.js +5 -2
- package/dist/scripts/akm-migrate-node.js +1141 -1237
- package/dist/scripts/akm-migrate.js +1141 -1237
- package/dist/setup/semantic-assets.js +2 -2
- package/dist/setup/steps/connection.js +3 -2
- package/dist/storage/repositories/files-repository.js +181 -0
- package/dist/storage/repositories/index-connection.js +1 -3
- package/dist/storage/repositories/index-entries-repository.js +77 -68
- package/dist/storage/repositories/index-entry-schema.js +16 -25
- package/dist/storage/repositories/index-fts-repository.js +29 -263
- package/dist/storage/repositories/index-meta-repository.js +0 -29
- package/dist/storage/repositories/index-schema.js +115 -122
- package/dist/storage/repositories/index-utility-repository.js +1 -1
- package/dist/storage/repositories/index-vec-repository.js +21 -334
- package/dist/storage/repositories/units-repository.js +510 -0
- package/docs/migration/release-notes/0.9.15.md +34 -36
- package/docs/migration/release-notes/0.9.16.md +110 -0
- package/docs/migration/release-notes/README.md +5 -0
- package/docs/reference/cli.md +93 -87
- package/docs/reference/configuration.md +128 -89
- package/docs/reference/data-and-telemetry.md +2 -1
- package/package.json +1 -1
- package/schemas/akm-config.json +2 -58
- package/dist/indexer/index-db-contention.js +0 -56
- package/dist/indexer/index-rebuild-lock.js +0 -73
- package/dist/indexer/materialize-embeddings.js +0 -771
- package/dist/indexer/passes/dir-staleness.js +0 -161
- package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
|
@@ -53,15 +53,15 @@ export function assembleInfo(options) {
|
|
|
53
53
|
// Semantic status is read live from the index's own state, not a cached
|
|
54
54
|
// verdict — a failed embed attempt at search time falls back to FTS and
|
|
55
55
|
// reports that in the search response, it never disables the mode here.
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
56
|
+
// "ready-js" (a JS-computed cosine-similarity fallback for when the
|
|
57
|
+
// sqlite-vec extension is unavailable) is retired (index redesign, B5):
|
|
58
|
+
// the units vector store (`units_vec`) is a vec0 virtual table with no
|
|
59
|
+
// BLOB fallback, so there is nothing to have embeddings without also
|
|
60
|
+
// having the extension available — `vecAvailable` false and
|
|
61
|
+
// `hasEmbeddings` true together is no longer a reachable combination.
|
|
62
|
+
const semanticStatus = config.semanticSearchMode === "off" ? "disabled" : !indexStats.hasEmbeddings ? "pending" : "ready-vec";
|
|
63
63
|
const searchModes = ["fts"];
|
|
64
|
-
if (semanticStatus === "ready-
|
|
64
|
+
if (semanticStatus === "ready-vec") {
|
|
65
65
|
searchModes.push("semantic", "hybrid");
|
|
66
66
|
}
|
|
67
67
|
return {
|
|
@@ -26,7 +26,7 @@ import { beginImmediateTransaction, getStateDbPath, openStateDatabase } from "..
|
|
|
26
26
|
import { warn } from "../../core/warn.js";
|
|
27
27
|
import { resolveGitContentRoot } from "../../core/write-source.js";
|
|
28
28
|
import { withAssetMutationLease } from "../../indexer/index-writer-lock.js";
|
|
29
|
-
import { akmIndex, runEmbeddingPass } from "../../indexer/indexer.js";
|
|
29
|
+
import { akmIndex, reclassifyIndexDbContention, runEmbeddingPass } from "../../indexer/indexer.js";
|
|
30
30
|
import { compareAndSwapLockfileSnapshot, publishLockfileUpdate, readLockfile, readLockfileForUpdate, } from "../../integrations/lockfile.js";
|
|
31
31
|
import { parseRegistryRef } from "../../registry/resolve.js";
|
|
32
32
|
import { sha256Hex } from "../../runtime.js";
|
|
@@ -312,11 +312,15 @@ export async function akmRemove(input) {
|
|
|
312
312
|
config: {
|
|
313
313
|
sourceCount: getSources(updatedConfig).length,
|
|
314
314
|
},
|
|
315
|
+
// `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
|
|
316
|
+
// removed (#index-redesign W6) — this response's own `index` shape is
|
|
317
|
+
// unchanged, so bridge from the renamed source field;
|
|
318
|
+
// `directoriesSkipped` was already always 0.
|
|
315
319
|
index: {
|
|
316
320
|
mode: index.mode,
|
|
317
321
|
totalEntries: index.totalEntries,
|
|
318
|
-
directoriesScanned: index.
|
|
319
|
-
directoriesSkipped:
|
|
322
|
+
directoriesScanned: index.sourcesScanned,
|
|
323
|
+
directoriesSkipped: 0,
|
|
320
324
|
},
|
|
321
325
|
};
|
|
322
326
|
}
|
|
@@ -342,11 +346,15 @@ export async function akmRemove(input) {
|
|
|
342
346
|
config: {
|
|
343
347
|
sourceCount: getSources(updatedConfig).length,
|
|
344
348
|
},
|
|
349
|
+
// `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
|
|
350
|
+
// removed (#index-redesign W6) — this response's own `index` shape is
|
|
351
|
+
// unchanged, so bridge from the renamed source field;
|
|
352
|
+
// `directoriesSkipped` was already always 0.
|
|
345
353
|
index: {
|
|
346
354
|
mode: index.mode,
|
|
347
355
|
totalEntries: index.totalEntries,
|
|
348
|
-
directoriesScanned: index.
|
|
349
|
-
directoriesSkipped:
|
|
356
|
+
directoriesScanned: index.sourcesScanned,
|
|
357
|
+
directoriesSkipped: 0,
|
|
350
358
|
},
|
|
351
359
|
};
|
|
352
360
|
}
|
|
@@ -361,14 +369,13 @@ export async function akmRemove(input) {
|
|
|
361
369
|
function readCurrentIndexSummary() {
|
|
362
370
|
const db = openReadonlyExistingDatabase(getDbPath());
|
|
363
371
|
if (!db) {
|
|
364
|
-
return { mode: "incremental", totalEntries: 0,
|
|
372
|
+
return { mode: "incremental", totalEntries: 0, sourcesScanned: 0 };
|
|
365
373
|
}
|
|
366
374
|
try {
|
|
367
375
|
return {
|
|
368
376
|
mode: "incremental",
|
|
369
377
|
totalEntries: getAllEntries(db).length,
|
|
370
|
-
|
|
371
|
-
directoriesSkipped: 0,
|
|
378
|
+
sourcesScanned: 0,
|
|
372
379
|
};
|
|
373
380
|
}
|
|
374
381
|
finally {
|
|
@@ -390,11 +397,15 @@ function buildUpdateResponse(stashDir, target, all, processed, opts) {
|
|
|
390
397
|
config: {
|
|
391
398
|
sourceCount: getSources(finalConfig).length,
|
|
392
399
|
},
|
|
400
|
+
// `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
|
|
401
|
+
// removed (#index-redesign W6) — `UpdateResponse["index"]`'s own shape is
|
|
402
|
+
// unchanged, so bridge from the renamed source field;
|
|
403
|
+
// `directoriesSkipped` was already always 0.
|
|
393
404
|
index: {
|
|
394
405
|
mode: index.mode,
|
|
395
406
|
totalEntries: index.totalEntries,
|
|
396
|
-
directoriesScanned: index.
|
|
397
|
-
directoriesSkipped:
|
|
407
|
+
directoriesScanned: index.sourcesScanned,
|
|
408
|
+
directoriesSkipped: 0,
|
|
398
409
|
...(index.scanComplete !== undefined ? { scanComplete: index.scanComplete } : {}),
|
|
399
410
|
// A real embedding pass (`akmIndex`/`runEmbeddingPass`) reports its own
|
|
400
411
|
// verified `semanticStatus`. When no pass ran this update (the
|
|
@@ -462,7 +473,9 @@ function openUnifiedUpdateTransaction() {
|
|
|
462
473
|
candidate.exec(`ATTACH DATABASE ${sqliteStringLiteral(statePath)} AS "${UPDATE_STATE_SCHEMA}"`);
|
|
463
474
|
// openIndexDatabase invokes this before ensureSchema, so the outer
|
|
464
475
|
// transaction begins before the first update-owned index mutation.
|
|
465
|
-
|
|
476
|
+
// `candidate` is the index.db connection, so contention here must
|
|
477
|
+
// report as INDEX_DB_CONTENDED, not the state.db default.
|
|
478
|
+
beginImmediateTransaction(candidate, "index");
|
|
466
479
|
},
|
|
467
480
|
});
|
|
468
481
|
return {
|
|
@@ -540,7 +553,15 @@ async function runPostCommitEmbeddingPass(index) {
|
|
|
540
553
|
return { ...index, verification };
|
|
541
554
|
}
|
|
542
555
|
catch (error) {
|
|
543
|
-
|
|
556
|
+
// Run the failure through the same index.db contention classifier the
|
|
557
|
+
// drain catch uses (indexer.ts's reclassifyIndexDbContention) before
|
|
558
|
+
// building the message: this catch's first statement, openIndexDatabase
|
|
559
|
+
// (init runs ensureSchema and writes), can throw the raw SQLite driver
|
|
560
|
+
// error under contention, and this was the last surviving non-fatal
|
|
561
|
+
// catch that printed that raw text verbatim to the operator instead of
|
|
562
|
+
// reporting it as index.db contention.
|
|
563
|
+
const reportedError = reclassifyIndexDbContention(error);
|
|
564
|
+
const message = reportedError instanceof Error ? reportedError.message : String(reportedError);
|
|
544
565
|
warn(`[akm bundle update] post-commit embedding pass failed: ${message}`);
|
|
545
566
|
return {
|
|
546
567
|
...index,
|
|
@@ -86,8 +86,13 @@ async function addLocalSource(ref, sourcePath, stashDir, explicitName, explicitA
|
|
|
86
86
|
index: {
|
|
87
87
|
mode: index.mode,
|
|
88
88
|
totalEntries: index.totalEntries,
|
|
89
|
-
|
|
90
|
-
|
|
89
|
+
// `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
|
|
90
|
+
// removed (#index-redesign W6: reconcile is a flat per-file stat walk,
|
|
91
|
+
// never a directory walk) — this response's own `directoriesScanned`/
|
|
92
|
+
// `directoriesSkipped` shape is unchanged, so bridge from the renamed
|
|
93
|
+
// source field; `directoriesSkipped` was already always 0.
|
|
94
|
+
directoriesScanned: index.sourcesScanned,
|
|
95
|
+
directoriesSkipped: 0,
|
|
91
96
|
...(index.warnings?.length ? { warnings: index.warnings } : {}),
|
|
92
97
|
},
|
|
93
98
|
};
|
|
@@ -151,8 +156,13 @@ async function addWebsiteSource(ref, stashDir, name, options) {
|
|
|
151
156
|
index: {
|
|
152
157
|
mode: index.mode,
|
|
153
158
|
totalEntries: index.totalEntries,
|
|
154
|
-
|
|
155
|
-
|
|
159
|
+
// `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
|
|
160
|
+
// removed (#index-redesign W6: reconcile is a flat per-file stat walk,
|
|
161
|
+
// never a directory walk) — this response's own `directoriesScanned`/
|
|
162
|
+
// `directoriesSkipped` shape is unchanged, so bridge from the renamed
|
|
163
|
+
// source field; `directoriesSkipped` was already always 0.
|
|
164
|
+
directoriesScanned: index.sourcesScanned,
|
|
165
|
+
directoriesSkipped: 0,
|
|
156
166
|
...(index.warnings?.length ? { warnings: index.warnings } : {}),
|
|
157
167
|
},
|
|
158
168
|
};
|
|
@@ -238,8 +248,13 @@ async function addRegistryStash(ref, stashDir, writable) {
|
|
|
238
248
|
index: {
|
|
239
249
|
mode: index.mode,
|
|
240
250
|
totalEntries: index.totalEntries,
|
|
241
|
-
|
|
242
|
-
|
|
251
|
+
// `IndexResponse.directoriesScanned`/`directoriesSkipped` were renamed/
|
|
252
|
+
// removed (#index-redesign W6: reconcile is a flat per-file stat walk,
|
|
253
|
+
// never a directory walk) — this response's own `directoriesScanned`/
|
|
254
|
+
// `directoriesSkipped` shape is unchanged, so bridge from the renamed
|
|
255
|
+
// source field; `directoriesSkipped` was already always 0.
|
|
256
|
+
directoriesScanned: index.sourcesScanned,
|
|
257
|
+
directoriesSkipped: 0,
|
|
243
258
|
...(index.warnings?.length ? { warnings: index.warnings } : {}),
|
|
244
259
|
},
|
|
245
260
|
};
|
|
@@ -28,10 +28,9 @@
|
|
|
28
28
|
* SIGINT/SIGTERM handlers in a try/finally — left byte-for-byte untouched.
|
|
29
29
|
*/
|
|
30
30
|
import path from "node:path";
|
|
31
|
-
import { defineCommand } from "citty";
|
|
32
31
|
import * as p from "../../cli/clack.js";
|
|
33
32
|
import { getParsedInvocation } from "../../cli/invocation.js";
|
|
34
|
-
import { defineJsonCommand, GLOBAL_OUTPUT_ARGS, output, parseAllFlagValues,
|
|
33
|
+
import { defineGroupCommand, defineJsonCommand, GLOBAL_OUTPUT_ARGS, output, parseAllFlagValues, } from "../../cli/shared.js";
|
|
35
34
|
import { assertFlatAssetName } from "../../core/asset/asset-create.js";
|
|
36
35
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
37
36
|
import { isHttpUrl, resolveStashDir } from "../../core/common.js";
|
|
@@ -40,17 +39,51 @@ import { UsageError } from "../../core/errors.js";
|
|
|
40
39
|
import { appendEvent } from "../../core/events.js";
|
|
41
40
|
import { resolveBundleWriteTarget } from "../../core/mutation-target.js";
|
|
42
41
|
import { getCacheDir } from "../../core/paths.js";
|
|
43
|
-
import { clearLogFile, info, isVerbose, setLogFile } from "../../core/warn.js";
|
|
42
|
+
import { clearLogFile, info, isVerbose, setLogFile, warn } from "../../core/warn.js";
|
|
44
43
|
import { resolveWriteTarget } from "../../core/write-source.js";
|
|
45
|
-
import {
|
|
44
|
+
import { DRAIN_BATCH_PROGRESS_PREFIX } from "../../indexer/drain.js";
|
|
46
45
|
import { akmIndex } from "../../indexer/indexer.js";
|
|
46
|
+
import { RECONCILE_ROOT_PROGRESS_PREFIX } from "../../indexer/reconcile.js";
|
|
47
47
|
import { getHyphenatedBoolean, getOutputMode } from "../../output/context.js";
|
|
48
48
|
import { inferAssetName, mergeXrefsIntoContent, readKnowledgeInput, resolveSupersedesForWrite, resolveSupersedesWriteTarget, resolveXrefsForWrite, writeMarkdownAsset, } from "../read/knowledge.js";
|
|
49
|
+
import { assembleIndexStatus } from "./index-status.js";
|
|
49
50
|
import { assembleInfo } from "./info.js";
|
|
50
|
-
/**
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
51
|
+
/**
|
|
52
|
+
* The two high-frequency, one-line-per-unit-of-work progress lines (#954) —
|
|
53
|
+
* drain's per-batch commit line and reconcile's per-root "done" line —
|
|
54
|
+
* excluded from non-verbose, non-text (JSON/yaml/etc) stderr. Matched by the
|
|
55
|
+
* exact prefix each producer exports, not a re-derived regex, so the two
|
|
56
|
+
* never drift apart (index-redesign B5g): this used to be a regex tuned to
|
|
57
|
+
* the deleted per-entry pipeline's `Embedded N/M entries.` line, which never
|
|
58
|
+
* matched either replacement line, so every progress line reached stderr
|
|
59
|
+
* regardless of `--verbose`.
|
|
60
|
+
*/
|
|
61
|
+
function isDetailProgressLine(message) {
|
|
62
|
+
return message.startsWith(DRAIN_BATCH_PROGRESS_PREFIX) || message.startsWith(RECONCILE_ROOT_PROGRESS_PREFIX);
|
|
63
|
+
}
|
|
64
|
+
export const indexStatusCommand = defineJsonCommand({
|
|
65
|
+
meta: {
|
|
66
|
+
name: "status",
|
|
67
|
+
description: "Show index.db's current state: files, entries, unit coverage, and the last reconcile time.",
|
|
68
|
+
},
|
|
69
|
+
run() {
|
|
70
|
+
output("index-status", assembleIndexStatus());
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
/**
|
|
74
|
+
* `akm index` = reconcile + drain (docs/plans/index-redesign.md). Still a raw
|
|
75
|
+
* group command (not `defineJsonCommand`) because its default body owns a
|
|
76
|
+
* spinner, an AbortController, and SIGINT/SIGTERM handlers in a try/finally;
|
|
77
|
+
* `defineGroupCommand` gives it `status` as a real subcommand while keeping
|
|
78
|
+
* that default body as plain `akm index`'s behavior (S-057 canonical
|
|
79
|
+
* bare-group rule does not apply here — a bare `akm index` has always run the
|
|
80
|
+
* indexer, and that stays).
|
|
81
|
+
*/
|
|
82
|
+
export const indexCommand = defineGroupCommand({
|
|
83
|
+
meta: {
|
|
84
|
+
name: "index",
|
|
85
|
+
description: "Reconcile the search index and drain the embedding queue (--full forces a full re-derivation)",
|
|
86
|
+
},
|
|
54
87
|
args: {
|
|
55
88
|
// R-051: `index` is a raw `defineCommand` (not `defineJsonCommand`), so it
|
|
56
89
|
// does not get `GLOBAL_OUTPUT_ARGS` for free. `--format`/`--detail`/
|
|
@@ -58,124 +91,99 @@ export const indexCommand = defineCommand({
|
|
|
58
91
|
// extra positional for a stray value to fall into), so this is purely a
|
|
59
92
|
// `--help` visibility / consistency fix, not a behavior change.
|
|
60
93
|
...GLOBAL_OUTPUT_ARGS,
|
|
61
|
-
full: {
|
|
62
|
-
clean: {
|
|
63
|
-
type: "boolean",
|
|
64
|
-
description: "After indexing, remove any entries whose source file no longer exists on disk.",
|
|
65
|
-
default: false,
|
|
66
|
-
},
|
|
67
|
-
"dry-run": {
|
|
94
|
+
full: {
|
|
68
95
|
type: "boolean",
|
|
69
|
-
description: "
|
|
96
|
+
description: "Force every file to be re-derived (ignore the unchanged-file shortcut), reconciling in place — " +
|
|
97
|
+
"existing rows keep their id/embeddings/utility scores; nothing is dropped first.",
|
|
70
98
|
default: false,
|
|
71
99
|
},
|
|
72
100
|
reembed: {
|
|
73
101
|
type: "boolean",
|
|
74
|
-
description: "
|
|
102
|
+
description: "Drop the active embedding identity's vectors, then re-embed every unit from scratch.",
|
|
75
103
|
default: false,
|
|
76
104
|
},
|
|
77
105
|
"skip-if-locked": {
|
|
78
106
|
type: "boolean",
|
|
79
|
-
description: "
|
|
107
|
+
description: "Deprecated, no effect. Index runs no longer take a rebuild lock (docs/plans/index-redesign.md) — " +
|
|
108
|
+
"every write is a short, idempotent, content-addressed transaction, so two concurrent index runs " +
|
|
109
|
+
"converge instead of contending. Kept only so existing scripts do not fail on an unknown flag.",
|
|
80
110
|
default: false,
|
|
81
111
|
},
|
|
82
112
|
},
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
113
|
+
subCommands: { status: indexStatusCommand },
|
|
114
|
+
async defaultRun({ args }) {
|
|
115
|
+
if (getHyphenatedBoolean(args, "enrich") || getParsedInvocation().getFlagValue("--enrich") !== undefined) {
|
|
116
|
+
throw new UsageError("`akm index --enrich` has been removed. Plain `akm index` now performs metadata enrichment by default.");
|
|
117
|
+
}
|
|
118
|
+
if (getHyphenatedBoolean(args, "re-enrich") || getParsedInvocation().getFlagValue("--re-enrich") !== undefined) {
|
|
119
|
+
throw new UsageError("`akm index --re-enrich` has been removed. Re-enrichment of index-time LLM passes is not exposed in this slice.");
|
|
120
|
+
}
|
|
121
|
+
if (args["skip-if-locked"]) {
|
|
122
|
+
warn("[index] --skip-if-locked is deprecated and has no effect — index runs no longer take a rebuild lock.");
|
|
123
|
+
}
|
|
124
|
+
const outputMode = getOutputMode();
|
|
125
|
+
const controller = new AbortController();
|
|
126
|
+
const abort = () => controller.abort(new Error("index interrupted"));
|
|
127
|
+
process.once("SIGINT", abort);
|
|
128
|
+
process.once("SIGTERM", abort);
|
|
129
|
+
const indexLogFile = path.join(getCacheDir(), "logs", "index", `${new Date().toISOString().replace(/[:.]/g, "-")}.log`);
|
|
130
|
+
setLogFile(indexLogFile);
|
|
131
|
+
const verbose = isVerbose();
|
|
132
|
+
const spin = !verbose && outputMode.format === "text" ? p.spinner() : null;
|
|
133
|
+
if (spin) {
|
|
134
|
+
spin.start(`Building search index${args.full ? " (full rebuild)" : ""}...`);
|
|
135
|
+
}
|
|
136
|
+
let latestMessage = "";
|
|
137
|
+
// Resolve the stash dir once at the `akm index` command boundary and
|
|
138
|
+
// thread it into the indexer (WI-9.10 CLI-wide sweep) — the indexer leaf
|
|
139
|
+
// no longer reads the ambient `resolveStashDir()`.
|
|
140
|
+
const stashDir = resolveStashDir();
|
|
141
|
+
try {
|
|
142
|
+
const result = await akmIndex({
|
|
143
|
+
stashDir,
|
|
144
|
+
full: args.full,
|
|
145
|
+
reembed: args.reembed,
|
|
146
|
+
onProgress: ({ phase, message, processed, total }) => {
|
|
147
|
+
latestMessage = message;
|
|
148
|
+
const progressPrefix = processed !== undefined && total !== undefined ? `[${processed}/${total}] ` : "";
|
|
149
|
+
if (verbose) {
|
|
150
|
+
info(`[index:${phase}] ${progressPrefix}${message}`);
|
|
151
|
+
}
|
|
152
|
+
else if (spin) {
|
|
153
|
+
spin.stop(`${progressPrefix}${message}`);
|
|
154
|
+
spin.start(`${progressPrefix}${message}`);
|
|
155
|
+
}
|
|
156
|
+
else if (!isDetailProgressLine(message)) {
|
|
157
|
+
// Non-verbose, non-text (JSON/yaml/etc) mode: silence used to be
|
|
158
|
+
// total until the run finished (#954) — a stalled
|
|
159
|
+
// run looked identical to "nothing written". Phase-start
|
|
160
|
+
// messages, the credential diagnostic, and the reconcile/drain
|
|
161
|
+
// totals now reach stderr here too; the high-frequency
|
|
162
|
+
// per-root `Reconciled "…"` and per-batch `[drain] batch N: …`
|
|
163
|
+
// lines are deliberately excluded — that would be spam, not a
|
|
164
|
+
// heartbeat. `--verbose` (the `if` branch above) still gets
|
|
165
|
+
// every one of them.
|
|
166
|
+
info(`[index:${phase}] ${progressPrefix}${message}`);
|
|
167
|
+
}
|
|
168
|
+
},
|
|
169
|
+
signal: controller.signal,
|
|
170
|
+
});
|
|
121
171
|
if (spin) {
|
|
122
|
-
spin.
|
|
123
|
-
}
|
|
124
|
-
let latestMessage = "";
|
|
125
|
-
// Resolve the stash dir once at the `akm index` command boundary and
|
|
126
|
-
// thread it into the indexer (WI-9.10 CLI-wide sweep) — the indexer leaf
|
|
127
|
-
// no longer reads the ambient `resolveStashDir()`.
|
|
128
|
-
const stashDir = resolveStashDir();
|
|
129
|
-
try {
|
|
130
|
-
const result = await akmIndex({
|
|
131
|
-
stashDir,
|
|
132
|
-
full: args.full,
|
|
133
|
-
clean: args.clean,
|
|
134
|
-
dryRun: args["dry-run"],
|
|
135
|
-
reembed: args.reembed,
|
|
136
|
-
onProgress: ({ phase, message, processed, total }) => {
|
|
137
|
-
latestMessage = message;
|
|
138
|
-
const progressPrefix = processed !== undefined && total !== undefined ? `[${processed}/${total}] ` : "";
|
|
139
|
-
if (verbose) {
|
|
140
|
-
info(`[index:${phase}] ${progressPrefix}${message}`);
|
|
141
|
-
}
|
|
142
|
-
else if (spin) {
|
|
143
|
-
spin.stop(`${progressPrefix}${message}`);
|
|
144
|
-
spin.start(`${progressPrefix}${message}`);
|
|
145
|
-
}
|
|
146
|
-
else if (!EMBEDDED_BATCH_PROGRESS_PATTERN.test(message)) {
|
|
147
|
-
// Non-verbose, non-text (JSON/yaml/etc) mode: silence used to be
|
|
148
|
-
// total until the run finished (#954) — a stalled
|
|
149
|
-
// run looked identical to "nothing written". Phase-start
|
|
150
|
-
// messages and the embedding heartbeat now reach stderr here
|
|
151
|
-
// too; the high-frequency per-batch `Embedded N/M entries.`
|
|
152
|
-
// line (emitted after every committed batch)
|
|
153
|
-
// is deliberately excluded — that would be spam, not a
|
|
154
|
-
// heartbeat.
|
|
155
|
-
info(`[index:${phase}] ${progressPrefix}${message}`);
|
|
156
|
-
}
|
|
157
|
-
},
|
|
158
|
-
signal: controller.signal,
|
|
159
|
-
});
|
|
160
|
-
if (spin) {
|
|
161
|
-
spin.stop(`Indexed ${result.totalEntries} assets.`);
|
|
162
|
-
}
|
|
163
|
-
output("index", result);
|
|
172
|
+
spin.stop(`Indexed ${result.totalEntries} assets.`);
|
|
164
173
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
}
|
|
171
|
-
finally {
|
|
172
|
-
clearLogFile();
|
|
173
|
-
process.off("SIGINT", abort);
|
|
174
|
-
process.off("SIGTERM", abort);
|
|
175
|
-
if (lockAcquisition.state === "acquired")
|
|
176
|
-
releaseIndexRebuildLock(lockAcquisition.ownership);
|
|
174
|
+
output("index", result);
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
if (spin) {
|
|
178
|
+
spin.stop(latestMessage ? `Indexing failed after: ${latestMessage}` : "Indexing failed.");
|
|
177
179
|
}
|
|
178
|
-
|
|
180
|
+
throw error;
|
|
181
|
+
}
|
|
182
|
+
finally {
|
|
183
|
+
clearLogFile();
|
|
184
|
+
process.off("SIGINT", abort);
|
|
185
|
+
process.off("SIGTERM", abort);
|
|
186
|
+
}
|
|
179
187
|
},
|
|
180
188
|
});
|
|
181
189
|
export const infoCommand = defineJsonCommand({
|
|
@@ -82,7 +82,7 @@
|
|
|
82
82
|
import fs from "node:fs";
|
|
83
83
|
import path from "node:path";
|
|
84
84
|
import { applyPostContributorFields, applyPreContributorFields, extractPackageMetadata, getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "../../../indexer/passes/metadata.js";
|
|
85
|
-
import { assetPathCandidatesForName, assetPathForName, deriveCanonicalAssetNameFromStashRoot, placementTypes, stashDirFor, stashDirNames, } from "../../asset/asset-placement.js";
|
|
85
|
+
import { assetPathCandidatesAreOrderedByPreference, assetPathCandidatesForName, assetPathForName, deriveCanonicalAssetNameFromStashRoot, placementTypes, stashDirFor, stashDirNames, } from "../../asset/asset-placement.js";
|
|
86
86
|
import { parseFrontmatter } from "../../asset/frontmatter.js";
|
|
87
87
|
import { executionDefaultsFromFrontmatter, renderMarkdownExecutionSource } from "../execution-source.js";
|
|
88
88
|
import { recognizeMatch } from "../recognize-match.js";
|
|
@@ -489,7 +489,25 @@ export const akmAdapter = {
|
|
|
489
489
|
* type's own stash subdir) and the LOOSE FALLBACK (authored anywhere else
|
|
490
490
|
* in the bundle, so the canonical name is the file's full path relative to
|
|
491
491
|
* the bundle root instead of the stash subdir). `assetPathCandidatesForName`
|
|
492
|
-
* additionally expands `env`'s `.env`/`<name>.env` duality
|
|
492
|
+
* additionally expands `env`'s `.env`/`<name>.env` duality, and memory's
|
|
493
|
+
* `<name>`/`<name>.derived` twin duality, on each.
|
|
494
|
+
*
|
|
495
|
+
* `priority` (#882 fix) carries each candidate's rank WITHIN its own root's
|
|
496
|
+
* list, but ONLY for a type whose duality is a declared, ORDERED
|
|
497
|
+
* preference per `assetPathCandidatesAreOrderedByPreference` (only
|
|
498
|
+
* `memory`, today) — `assetPathCandidatesForName` returns primary before
|
|
499
|
+
* derived-twin for that type, so index 0 is the declared winner when both
|
|
500
|
+
* exist. CANONICAL and LOOSE are separate lists whose ranks both start
|
|
501
|
+
* back at 0: two candidates that tie on rank (e.g. the canonical and loose
|
|
502
|
+
* spellings both being the primary, rank-0, form) are a genuine collision
|
|
503
|
+
* between two independently-authored files, not a declared preference —
|
|
504
|
+
* only a rank difference WITHIN one root's own list resolves silently.
|
|
505
|
+
* Every other type's candidates (including `env`'s `.env`/`default.env`
|
|
506
|
+
* pair — co-equal spellings, not an ordered preference, per that same
|
|
507
|
+
* predicate's doc comment) carry NO `priority`, unchanged from before
|
|
508
|
+
* #882: `resolveAdapterConceptOwner` treats priority-less candidates as
|
|
509
|
+
* tied, so more than one existing together still collides. See
|
|
510
|
+
* `AdapterReadCandidate.priority`'s doc comment.
|
|
493
511
|
*/
|
|
494
512
|
readCandidates(c, conceptId) {
|
|
495
513
|
const posix = conceptId.replace(/\\/g, "/");
|
|
@@ -503,9 +521,23 @@ export const akmAdapter = {
|
|
|
503
521
|
return [];
|
|
504
522
|
const canonical = assetPathCandidatesForName(type, path.join(c.root, head), rest);
|
|
505
523
|
const loose = assetPathCandidatesForName(type, c.root, rest);
|
|
506
|
-
|
|
524
|
+
if (!assetPathCandidatesAreOrderedByPreference(type)) {
|
|
525
|
+
return [...new Set([...canonical, ...loose])].map((candidatePath) => ({
|
|
526
|
+
path: candidatePath,
|
|
527
|
+
conceptId: posix,
|
|
528
|
+
}));
|
|
529
|
+
}
|
|
530
|
+
const priorityByPath = new Map();
|
|
531
|
+
for (const list of [canonical, loose]) {
|
|
532
|
+
list.forEach((candidatePath, rank) => {
|
|
533
|
+
if (!priorityByPath.has(candidatePath))
|
|
534
|
+
priorityByPath.set(candidatePath, rank);
|
|
535
|
+
});
|
|
536
|
+
}
|
|
537
|
+
return [...priorityByPath.keys()].map((candidatePath) => ({
|
|
507
538
|
path: candidatePath,
|
|
508
539
|
conceptId: posix,
|
|
540
|
+
priority: priorityByPath.get(candidatePath),
|
|
509
541
|
}));
|
|
510
542
|
},
|
|
511
543
|
/**
|
|
@@ -172,7 +172,17 @@ export function foldRecognizedMetadata(rendererName, file) {
|
|
|
172
172
|
const fm = parseFrontmatter(file.content()).data;
|
|
173
173
|
applyFrontmatterDescriptionAndTags(fm, out);
|
|
174
174
|
const hints = new Set();
|
|
175
|
-
|
|
175
|
+
// fix-ranking-derived-outranks-primary: `source:` on an ordinary
|
|
176
|
+
// memory is an author-written citation worth indexing as a hint, but
|
|
177
|
+
// on an inferred (`.derived`) twin it is ALWAYS the machine-written
|
|
178
|
+
// provenance backref `memory-inference.ts` writes (`memories/<parent>`
|
|
179
|
+
// — see its `FM_SOURCE`), already captured properly as
|
|
180
|
+
// `entry.derivedFrom` (metadata.ts). Folding that backref into
|
|
181
|
+
// searchHints too means the base memory's own name — almost always a
|
|
182
|
+
// query token whenever the base is relevant — auto-credits the twin
|
|
183
|
+
// via `search-hint-ranking`'s substring match, independent of whether
|
|
184
|
+
// the twin's own content actually matches the query.
|
|
185
|
+
const source = fm.inferred === true ? undefined : nonEmptyString(fm.source);
|
|
176
186
|
if (source)
|
|
177
187
|
hints.add(source);
|
|
178
188
|
const fmObservedAt = nonEmptyString(fm.observed_at);
|
|
@@ -269,3 +269,38 @@ export function assetPathCandidatesForName(assetType, typeRoot, name) {
|
|
|
269
269
|
const namedForm = path.join(typeRoot, base, "default.env");
|
|
270
270
|
return [...new Set([primary, dotForm, namedForm])];
|
|
271
271
|
}
|
|
272
|
+
/**
|
|
273
|
+
* Whether {@link assetPathCandidatesForName}'s returned list for `assetType`
|
|
274
|
+
* is an ORDERED preference — earlier candidates are a declared winner over
|
|
275
|
+
* later ones, so a physical-owner resolver may pick the earliest that exists
|
|
276
|
+
* with no throw when more than one does — or a set of CO-EQUAL spellings,
|
|
277
|
+
* where more than one existing together is a genuine authoring collision to
|
|
278
|
+
* report, not a preference to resolve silently. #882's fix (the memory
|
|
279
|
+
* `.derived`-twin bug) needs this distinction: attaching a declared rank to
|
|
280
|
+
* every multi-candidate type's list turned `env`'s co-equal collision into a
|
|
281
|
+
* silently-resolved false negative (#882 follow-up) — the two dualities this
|
|
282
|
+
* module documents are NOT the same kind of thing.
|
|
283
|
+
*
|
|
284
|
+
* Only `memory` is ordered. Its own doc comment above states a winner in so
|
|
285
|
+
* many words: "The plain `.md` file wins when both exist, so it stays
|
|
286
|
+
* `primary`" — `.derived` is a provenance marker on the SAME identity, not a
|
|
287
|
+
* second, independently-authored file.
|
|
288
|
+
*
|
|
289
|
+
* `env`'s `.env`/`default.env` duality is explicitly NOT ordered — the doc
|
|
290
|
+
* comment above says only that both spellings "derive the same canonical
|
|
291
|
+
* name" and a lookup "must consider both", never that one wins over the
|
|
292
|
+
* other. They are two independently authored files that happen to collide
|
|
293
|
+
* on one ref; both existing together is exactly the ambiguity
|
|
294
|
+
* `AdapterConceptCollisionError` exists to report.
|
|
295
|
+
*
|
|
296
|
+
* Callers that attach a preference rank to distinguish a declared duality
|
|
297
|
+
* from a genuine collision (`AdapterReadCandidate.priority`,
|
|
298
|
+
* `resolveAdapterConceptOwner` in `indexer/lookup/adapter-concept-owner.ts`)
|
|
299
|
+
* must consult this predicate per asset type rather than assuming every
|
|
300
|
+
* multi-candidate type behaves like `memory` — and must NOT special-case the
|
|
301
|
+
* literal `.derived` suffix or `assetType === "memory"` themselves; this is
|
|
302
|
+
* the one place that knowledge lives.
|
|
303
|
+
*/
|
|
304
|
+
export function assetPathCandidatesAreOrderedByPreference(assetType) {
|
|
305
|
+
return assetType === "memory";
|
|
306
|
+
}
|
|
@@ -33,32 +33,6 @@ export const EmbeddingConnectionConfigSchema = z
|
|
|
33
33
|
// `akm index` when ensureSchema rejects it (§24.2 "Semantic" gate).
|
|
34
34
|
dimension: positiveInt.max(4096).optional(),
|
|
35
35
|
localModel: z.string().min(1).optional(),
|
|
36
|
-
/**
|
|
37
|
-
* Per-document token cap applied BEFORE batching (default 512,
|
|
38
|
-
* `DEFAULT_MAX_INPUT_TOKENS` in `src/llm/embedders/remote.ts`, #956).
|
|
39
|
-
* The materializer truncates a document's embedded text to
|
|
40
|
-
* this cap (head only, unicode-safe) instead of skipping it outright, so
|
|
41
|
-
* one oversized entry can no longer fail a whole batch. Distinct from
|
|
42
|
-
* `maxTokens` below, which bounds a whole HTTP REQUEST (many documents);
|
|
43
|
-
* this bounds one DOCUMENT.
|
|
44
|
-
*/
|
|
45
|
-
maxInputTokens: positiveInt.optional(),
|
|
46
|
-
/**
|
|
47
|
-
* Client-side per-request token budget — how many documents' estimated
|
|
48
|
-
* tokens fit in one HTTP request (default `DEFAULT_TOKEN_BUDGET` = 6000
|
|
49
|
-
* in `src/llm/embedders/remote.ts`). With the 512-token `maxInputTokens`
|
|
50
|
-
* cap above, a request carries about 11 documents by default.
|
|
51
|
-
*/
|
|
52
|
-
maxTokens: positiveInt.optional(),
|
|
53
|
-
batchSize: positiveInt.optional(),
|
|
54
|
-
/**
|
|
55
|
-
* Ollama's `num_ctx` ONLY (#956) — sent verbatim as
|
|
56
|
-
* `options.num_ctx` on the native `/api/embed` request. It no longer also
|
|
57
|
-
* feeds the client-side request token budget (`maxTokens` above): the two
|
|
58
|
-
* used to share this one field, so setting it for the server's context
|
|
59
|
-
* window silently changed request batching too.
|
|
60
|
-
*/
|
|
61
|
-
contextLength: positiveInt.optional(),
|
|
62
36
|
ollamaOptions: EmbeddingOllamaOptionsSchema.optional(),
|
|
63
37
|
/**
|
|
64
38
|
* Per-request timeout in milliseconds for a remote embedding request
|
|
@@ -72,10 +46,13 @@ export const EmbeddingConnectionConfigSchema = z
|
|
|
72
46
|
* Overrides the fixed in-flight request window (#954, added after field
|
|
73
47
|
* evidence from multi-slot local servers). Bounded 1-16. Unset keeps
|
|
74
48
|
* today's default: 1 for a loopback endpoint, 2 for a remote one
|
|
75
|
-
* (`resolveEmbeddingConcurrency`, `src/llm/embedders/remote.ts`)
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
* `
|
|
49
|
+
* (`resolveEmbeddingConcurrency`, `src/llm/embedders/remote.ts`), unless
|
|
50
|
+
* the provider's own probed slot count overrides it
|
|
51
|
+
* (`probeProviderLimits`, `src/llm/embedders/provider-limits.ts`, used by
|
|
52
|
+
* `akm index`'s drain queue). Set it only for an endpoint that genuinely
|
|
53
|
+
* serves parallel requests (llama.cpp `--parallel N`, vLLM) — request
|
|
54
|
+
* SIZE, packed against the provider's own probed context window, remains
|
|
55
|
+
* the first throughput lever.
|
|
79
56
|
*/
|
|
80
57
|
concurrency: positiveInt.max(16).optional(),
|
|
81
58
|
})
|