akm-cli 0.9.8-beta.1 → 0.9.8-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +134 -0
- package/dist/commands/health/data-dir-usage.js +182 -0
- package/dist/commands/health.js +19 -4
- package/dist/commands/migrate-cli.js +24 -5
- package/dist/commands/workflow-cli.js +11 -61
- package/dist/core/config/config-io.js +21 -8
- package/dist/core/state/migrations.js +22 -0
- package/dist/core/warn.js +15 -0
- package/dist/indexer/indexer.js +42 -21
- package/dist/indexer/passes/dir-staleness.js +60 -21
- package/dist/scripts/akm-migrate-node.js +37 -4
- package/dist/scripts/akm-migrate.js +37 -4
- package/dist/storage/repositories/index-entries-repository.js +24 -11
- package/dist/storage/repositories/index-meta-repository.js +6 -4
- package/dist/storage/repositories/index-schema.js +16 -1
- package/dist/storage/repositories/proposals-repository.js +4 -1
- package/dist/tasks/resolve-akm-bin.js +15 -0
- package/dist/tasks/source/task-to-v3.js +10 -2
- package/docs/migration/v0.9.1-to-v0.9.2.md +18 -4
- package/docs/reference/cli.md +1 -1
- package/docs/reference/data-and-telemetry.md +1 -0
- package/docs/reference/tasks.md +11 -0
- package/package.json +1 -5
package/dist/indexer/indexer.js
CHANGED
|
@@ -30,7 +30,7 @@ import { deleteStoredGraph } from "./db/graph-db.js";
|
|
|
30
30
|
import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
|
|
31
31
|
import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
|
|
32
32
|
import { generateEmbeddingsForDb } from "./materialize-embeddings.js";
|
|
33
|
-
import { canUseIncrementalSkip, computeDirFingerprint,
|
|
33
|
+
import { canUseIncrementalSkip, computeDirFingerprint, getCachedDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
|
|
34
34
|
import { isEnrichmentComplete, isWorkflowSkipWarning } from "./passes/metadata.js";
|
|
35
35
|
import { drainDirDocuments } from "./scan/drain-dir.js";
|
|
36
36
|
import { buildSearchText } from "./search/search-fields.js";
|
|
@@ -360,6 +360,19 @@ export function _setIndexTransactionHookForTests(hook) {
|
|
|
360
360
|
function indexTransactionHook(point) {
|
|
361
361
|
indexTransactionHookForTests?.(point);
|
|
362
362
|
}
|
|
363
|
+
let drainObserverForTests;
|
|
364
|
+
/**
|
|
365
|
+
* TEST-ONLY. Observe every directory that actually reaches
|
|
366
|
+
* `drainDirDocuments` — the per-file read/sha256-hash/frontmatter-parse step
|
|
367
|
+
* (#900) — with the directory path and its walked file count. `undefined`
|
|
368
|
+
* restores. A directory the pre-drain gate (`getCachedDirState`) skips never
|
|
369
|
+
* fires this observer, so it is the
|
|
370
|
+
* seam #900's own tests use to assert an unchanged directory's files are
|
|
371
|
+
* never read on a no-op incremental run.
|
|
372
|
+
*/
|
|
373
|
+
export function _setDrainObserverForTests(observer) {
|
|
374
|
+
drainObserverForTests = observer;
|
|
375
|
+
}
|
|
363
376
|
/**
|
|
364
377
|
* Detect an adapter for every resolvable source that does not declare one, and
|
|
365
378
|
* persist each detection into `config.json`.
|
|
@@ -883,17 +896,19 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
883
896
|
// Incremental freshness gate shared by both branches: consult the persisted
|
|
884
897
|
// dir state and record either a skip (unchanged + eligible for incremental
|
|
885
898
|
// skip) or a scan record carrying the candidate stash.
|
|
886
|
-
const recordFreshnessDecision = (dirPath, currentStashDir, stateFiles, stash, hashByFile, conceptIdByFile, indexVariant, forceScan, pruneMissing) => {
|
|
887
|
-
const previousState = getDirIndexState(db, dirPath, stateFiles, builtAtMs, indexVariant);
|
|
899
|
+
const recordFreshnessDecision = (dirPath, currentStashDir, stateFiles, fingerprint, stash, hashByFile, conceptIdByFile, indexVariant, forceScan, pruneMissing) => {
|
|
900
|
+
const previousState = getDirIndexState(db, dirPath, stateFiles, builtAtMs, indexVariant, fingerprint);
|
|
888
901
|
if (isIncremental && !forceScan && !previousState.stale && canUseIncrementalSkip(previousState, priorDirsChanged)) {
|
|
889
902
|
skippedDirs++;
|
|
890
903
|
dirRecords.push({
|
|
891
904
|
dirPath,
|
|
892
905
|
currentStashDir,
|
|
893
906
|
files: stateFiles,
|
|
907
|
+
fingerprint,
|
|
894
908
|
stash: null,
|
|
895
909
|
skip: true,
|
|
896
910
|
reason: previousState.reason,
|
|
911
|
+
persistedRowCount: previousState.persistedRowCount,
|
|
897
912
|
indexVariant,
|
|
898
913
|
});
|
|
899
914
|
reportDirDecision("skip", dirPath, currentStashDir, previousState.reason, previousState.persistedRowCount);
|
|
@@ -906,6 +921,7 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
906
921
|
dirPath,
|
|
907
922
|
currentStashDir,
|
|
908
923
|
files: stateFiles,
|
|
924
|
+
fingerprint,
|
|
909
925
|
stash,
|
|
910
926
|
skip: false,
|
|
911
927
|
reason,
|
|
@@ -959,10 +975,13 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
959
975
|
reportDirDecision("skip", dirPath, currentStashDir, reason);
|
|
960
976
|
continue;
|
|
961
977
|
}
|
|
962
|
-
|
|
978
|
+
// #900: decide from stat data alone whether the directory can be skipped,
|
|
979
|
+
// before drainDirDocuments reads, hashes, and parses every file.
|
|
980
|
+
const fingerprint = computeDirFingerprint(dirPath, indexableFiles, indexVariant);
|
|
981
|
+
const cachedState = isIncremental &&
|
|
963
982
|
!forceScan &&
|
|
964
|
-
|
|
965
|
-
if (
|
|
983
|
+
getCachedDirState(db, dirPath, indexableFiles, builtAtMs, priorDirsChanged, indexVariant, fingerprint);
|
|
984
|
+
if (cachedState) {
|
|
966
985
|
skippedDirs++;
|
|
967
986
|
dirRecords.push({
|
|
968
987
|
dirPath,
|
|
@@ -970,15 +989,16 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
970
989
|
files: indexableFiles,
|
|
971
990
|
stash: null,
|
|
972
991
|
skip: true,
|
|
973
|
-
reason:
|
|
992
|
+
reason: cachedState.reason,
|
|
974
993
|
indexVariant,
|
|
975
994
|
});
|
|
976
|
-
reportDirDecision("skip", dirPath, currentStashDir,
|
|
995
|
+
reportDirDecision("skip", dirPath, currentStashDir, cachedState.reason, cachedState.persistedRowCount);
|
|
977
996
|
continue;
|
|
978
997
|
}
|
|
979
998
|
// F4a M-core-2 (the flip): drain the dir's `IndexDocument` stream via the
|
|
980
999
|
// component's dispatched `adapter.recognize` (broken workflows dropped-with-
|
|
981
1000
|
// warning at the drain layer) and reconstruct the durable `IndexDocument`s.
|
|
1001
|
+
drainObserverForTests?.(dirPath, ctxs.length);
|
|
982
1002
|
const drained = drainDirDocuments(adapter, component, ctxs);
|
|
983
1003
|
if (drained.warnings.length)
|
|
984
1004
|
warnings.push(...drained.warnings);
|
|
@@ -992,7 +1012,7 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
992
1012
|
if (generated.entries.length > 0) {
|
|
993
1013
|
generatedCount += generated.entries.length;
|
|
994
1014
|
}
|
|
995
|
-
recordFreshnessDecision(dirPath, currentStashDir, staleFiles, stash, drained.hashByFile, drained.conceptIdByFile, indexVariant, forceScan, walkComplete);
|
|
1015
|
+
recordFreshnessDecision(dirPath, currentStashDir, staleFiles, fingerprint, stash, drained.hashByFile, drained.conceptIdByFile, indexVariant, forceScan, walkComplete);
|
|
996
1016
|
}
|
|
997
1017
|
}
|
|
998
1018
|
return {
|
|
@@ -1089,7 +1109,7 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1089
1109
|
// tests/integration/indexer/reindex-generation-atomicity.test.ts.
|
|
1090
1110
|
indexTransactionHook("full-delete-applied");
|
|
1091
1111
|
}
|
|
1092
|
-
for (const { dirPath, currentStashDir, files, stash, skip, reason, hashByFile, conceptIdByFile, indexVariant, remove, pruneMissing, } of dirRecords) {
|
|
1112
|
+
for (const { dirPath, currentStashDir, files, fingerprint, stash, skip, reason, persistedRowCount, hashByFile, conceptIdByFile, indexVariant, remove, pruneMissing, } of dirRecords) {
|
|
1093
1113
|
const bundle = bundleByRoot.get(path.resolve(currentStashDir));
|
|
1094
1114
|
if (!bundle)
|
|
1095
1115
|
throw new Error(`Missing bundle provenance for indexed source ${currentStashDir}`);
|
|
@@ -1102,14 +1122,11 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1102
1122
|
continue;
|
|
1103
1123
|
}
|
|
1104
1124
|
if (skip) {
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
fileMtimeMaxMs: fingerprint.fileMtimeMaxMs,
|
|
1111
|
-
reason: reason.kind,
|
|
1112
|
-
});
|
|
1125
|
+
// "unchanged" is the post-drain verdict: re-persist so the row carries
|
|
1126
|
+
// row_count and the gate skips this directory before draining next
|
|
1127
|
+
// time. "unchanged-precheck" already matched the stored row.
|
|
1128
|
+
if (reason?.kind === "unchanged" && fingerprint) {
|
|
1129
|
+
upsertIndexDirState(db, { dirPath, ...fingerprint, reason: reason.kind, rowCount: persistedRowCount });
|
|
1113
1130
|
}
|
|
1114
1131
|
continue;
|
|
1115
1132
|
}
|
|
@@ -1165,7 +1182,7 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1165
1182
|
if (pruneMissing !== false) {
|
|
1166
1183
|
addEntryIds(deletedUsageEntryIds, deleteEntriesByDirExceptRefs(db, dirPath, bundle.bundleId, keptItemRefs, { cleanupUsageEvents: false }));
|
|
1167
1184
|
}
|
|
1168
|
-
const
|
|
1185
|
+
const persistedFingerprint = fingerprint ?? computeDirFingerprint(dirPath, files, indexVariant);
|
|
1169
1186
|
const persistedReason = persistedRows === 0
|
|
1170
1187
|
? inferZeroRowReason(stash, reason, warnings, dirPath, dedupedRows)
|
|
1171
1188
|
: reason?.kind === "full-rebuild"
|
|
@@ -1173,9 +1190,13 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1173
1190
|
: (reason?.kind ?? "updated");
|
|
1174
1191
|
upsertIndexDirState(db, {
|
|
1175
1192
|
dirPath,
|
|
1176
|
-
|
|
1177
|
-
fileMtimeMaxMs: fingerprint.fileMtimeMaxMs,
|
|
1193
|
+
...persistedFingerprint,
|
|
1178
1194
|
reason: persistedReason,
|
|
1195
|
+
// A directory that lost rows to per-source dedup depends on the
|
|
1196
|
+
// directories persisted before it, not only on its own files, so it
|
|
1197
|
+
// must keep draining every run (as it did before the gate) until a
|
|
1198
|
+
// drain persists it without dedup. NULL keeps the gate closed.
|
|
1199
|
+
rowCount: dedupedRows === 0 ? persistedRows : undefined,
|
|
1179
1200
|
});
|
|
1180
1201
|
if (persistedRows === 0) {
|
|
1181
1202
|
// Warn only when the dir had files that *could* produce entries (.md or
|
|
@@ -10,20 +10,27 @@
|
|
|
10
10
|
*
|
|
11
11
|
* Two persisted signals back the decision:
|
|
12
12
|
* 1. The `entries` rows already indexed for the directory (`getEntriesByDir`).
|
|
13
|
-
* 2. The `index_dir_state`
|
|
14
|
-
*
|
|
15
|
-
*
|
|
13
|
+
* 2. The `index_dir_state` row (`getIndexDirState`): the fingerprint of the
|
|
14
|
+
* directory's walked file set (basename set + max mtime, `computeDirFingerprint`)
|
|
15
|
+
* as of its last drain, plus the row count that drain persisted.
|
|
16
16
|
*
|
|
17
|
-
* `
|
|
18
|
-
*
|
|
17
|
+
* `getCachedDirState` is the pre-drain gate (#900): a directory whose walked
|
|
18
|
+
* fingerprint still matches its row is skipped before any file is read.
|
|
19
19
|
*/
|
|
20
|
+
import { createHash } from "node:crypto";
|
|
20
21
|
import fs from "node:fs";
|
|
21
22
|
import path from "node:path";
|
|
23
|
+
import { compareCodePoints } from "../../core/common.js";
|
|
22
24
|
import { getEntriesByDir } from "../../storage/repositories/index-entries-repository.js";
|
|
23
25
|
import { getIndexDirState } from "../../storage/repositories/index-meta-repository.js";
|
|
24
|
-
|
|
26
|
+
/**
|
|
27
|
+
* Post-drain freshness verdict. `files` is the recognized file set the drain
|
|
28
|
+
* produced (compared against the persisted entries); `fingerprint` is the
|
|
29
|
+
* walked-set fingerprint compared against the persisted row and defaults to
|
|
30
|
+
* one computed over `files`.
|
|
31
|
+
*/
|
|
32
|
+
export function getDirIndexState(db, dirPath, files, builtAtMs, indexVariant = "", fingerprint = computeDirFingerprint(dirPath, files, indexVariant)) {
|
|
25
33
|
const prevEntries = getEntriesByDir(db, dirPath);
|
|
26
|
-
const fingerprint = computeDirFingerprint(dirPath, files, indexVariant);
|
|
27
34
|
if (prevEntries.length > 0) {
|
|
28
35
|
const staleReason = getDirStaleReason(dirPath, files, prevEntries, builtAtMs);
|
|
29
36
|
if (staleReason)
|
|
@@ -39,9 +46,7 @@ export function getDirIndexState(db, dirPath, files, builtAtMs, indexVariant = "
|
|
|
39
46
|
return { stale: false, reason: { kind: "unchanged" }, persistedRowCount: prevEntries.length };
|
|
40
47
|
}
|
|
41
48
|
const cachedState = getIndexDirState(db, dirPath);
|
|
42
|
-
if (cachedState &&
|
|
43
|
-
cachedState.fileSetHash === fingerprint.fileSetHash &&
|
|
44
|
-
cachedState.fileMtimeMaxMs === fingerprint.fileMtimeMaxMs) {
|
|
49
|
+
if (cachedState && cachedState.fileSetHash === fingerprint.fileSetHash) {
|
|
45
50
|
return {
|
|
46
51
|
stale: false,
|
|
47
52
|
reason: { kind: "cached-zero-row-state", detail: cachedState.reason },
|
|
@@ -54,8 +59,21 @@ export function getDirIndexState(db, dirPath, files, builtAtMs, indexVariant = "
|
|
|
54
59
|
persistedRowCount: 0,
|
|
55
60
|
};
|
|
56
61
|
}
|
|
57
|
-
|
|
58
|
-
|
|
62
|
+
/**
|
|
63
|
+
* Pre-drain gate (#900). A directory whose walked-set fingerprint matches its
|
|
64
|
+
* persisted row cannot recognize differently than last time, so it is skipped
|
|
65
|
+
* before `drainDirDocuments` reads a single file. A row that recorded a real
|
|
66
|
+
* generation (`rowCount > 0`) is skipped outright; a zero-row or pre-#900 row
|
|
67
|
+
* goes through the entries-aware check so the dedup-order guard still applies.
|
|
68
|
+
*/
|
|
69
|
+
export function getCachedDirState(db, dirPath, files, builtAtMs, priorDirsChanged, indexVariant, fingerprint) {
|
|
70
|
+
const cached = getIndexDirState(db, dirPath);
|
|
71
|
+
if (!cached || cached.fileSetHash !== fingerprint.fileSetHash)
|
|
72
|
+
return undefined;
|
|
73
|
+
if (cached.rowCount !== undefined && cached.rowCount > 0) {
|
|
74
|
+
return { stale: false, reason: { kind: "unchanged-precheck" }, persistedRowCount: cached.rowCount };
|
|
75
|
+
}
|
|
76
|
+
const state = getDirIndexState(db, dirPath, files, builtAtMs, indexVariant, fingerprint);
|
|
59
77
|
if (state.stale || state.reason.kind !== "cached-zero-row-state")
|
|
60
78
|
return undefined;
|
|
61
79
|
if (!canUseIncrementalSkip(state, priorDirsChanged))
|
|
@@ -68,21 +86,42 @@ export function canUseIncrementalSkip(state, priorDirsChanged) {
|
|
|
68
86
|
state.reason.detail === "deduped-zero-row");
|
|
69
87
|
}
|
|
70
88
|
export function computeDirFingerprint(_dirPath, files, indexVariant = "") {
|
|
71
|
-
|
|
89
|
+
// One `statSync` per file — the same call this function has always made — but
|
|
90
|
+
// every field it returns that can witness a change is kept, per file, instead
|
|
91
|
+
// of being collapsed into a single max.
|
|
92
|
+
//
|
|
93
|
+
// `Math.max` over mtimes discarded everything except the newest file, so an
|
|
94
|
+
// edit to any other file landed below the max and was invisible; and mtime
|
|
95
|
+
// alone is writable by ordinary tooling (`touch -r`, `rsync --times`,
|
|
96
|
+
// `cp -p`, archive extraction), so a restored timestamp hid an edit outright.
|
|
97
|
+
// Size catches any length-changing edit; ctime catches the rest, because
|
|
98
|
+
// utimes(2) cannot hold the inode's change time back.
|
|
99
|
+
//
|
|
100
|
+
// This is still a heuristic: ctime also moves on metadata-only changes
|
|
101
|
+
// (chmod/chown) and after copying a tree, which costs an unnecessary rescan.
|
|
102
|
+
// That direction is safe — extra work, never stale content.
|
|
103
|
+
const entries = [];
|
|
72
104
|
let fileMtimeMaxMs = 0;
|
|
73
|
-
for (const file of files) {
|
|
105
|
+
for (const file of [...new Set(files)].sort(compareCodePoints)) {
|
|
106
|
+
const name = path.basename(file);
|
|
74
107
|
try {
|
|
75
|
-
|
|
108
|
+
// `bigint: true` is the same syscall but reports nanoseconds. Millisecond
|
|
109
|
+
// floats would let an edit made inside the same millisecond as the last
|
|
110
|
+
// run's stat land on an identical digest.
|
|
111
|
+
const stat = fs.statSync(file, { bigint: true });
|
|
112
|
+
fileMtimeMaxMs = Math.max(fileMtimeMaxMs, Number(stat.mtimeMs));
|
|
113
|
+
entries.push(`${name}\0${stat.size}\0${stat.mtimeNs}\0${stat.ctimeNs}`);
|
|
76
114
|
}
|
|
77
115
|
catch {
|
|
78
|
-
|
|
79
|
-
|
|
116
|
+
// Unreadable or vanished: record it as such so the digest differs from
|
|
117
|
+
// any run where the file could be read, forcing a rescan.
|
|
118
|
+
entries.push(`${name}\0unreadable`);
|
|
80
119
|
}
|
|
81
120
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
};
|
|
121
|
+
const digest = createHash("sha256")
|
|
122
|
+
.update([indexVariant, ...entries].join("\n"), "utf8")
|
|
123
|
+
.digest("hex");
|
|
124
|
+
return { fileSetHash: digest, fileMtimeMaxMs };
|
|
86
125
|
}
|
|
87
126
|
function getDirStaleReason(_dirPath, currentFiles, previousEntries, builtAtMs) {
|
|
88
127
|
const prevFileNames = new Set(previousEntries
|
|
@@ -13104,6 +13104,7 @@ function warn(...args) {
|
|
|
13104
13104
|
console.warn(...args);
|
|
13105
13105
|
}
|
|
13106
13106
|
}
|
|
13107
|
+
var warnedOnceKeys = new Set;
|
|
13107
13108
|
function warnVerbose(...args) {
|
|
13108
13109
|
if (sinkOverride) {
|
|
13109
13110
|
sinkOverride("warnVerbose", args);
|
|
@@ -16855,6 +16856,7 @@ function base(input) {
|
|
|
16855
16856
|
...input.containmentRoot ? { containmentRoot: input.containmentRoot } : {}
|
|
16856
16857
|
};
|
|
16857
16858
|
}
|
|
16859
|
+
var ARGV_ARRAY_BLOCK_DETAIL = "Manual conversion required: an array `command:` has no safe v3 `run:` string. Rewrite it by hand as " + "`run:` (string) plus `shell:` — see docs/migration/v0.9.1-to-v0.9.2.md for the full v2 to v4 field mapping.";
|
|
16858
16860
|
function blocked(input, reason, detail) {
|
|
16859
16861
|
return Object.freeze({ status: "blocked", ...base(input), reason, ...detail ? { detail } : {} });
|
|
16860
16862
|
}
|
|
@@ -17117,8 +17119,10 @@ function planLegacyTaskDataToV3(input, data) {
|
|
|
17117
17119
|
} catch (cause) {
|
|
17118
17120
|
return blocked(input, "invalid-v2-task", cause instanceof Error ? cause.message : String(cause));
|
|
17119
17121
|
}
|
|
17120
|
-
if (isReason(migrated))
|
|
17121
|
-
|
|
17122
|
+
if (isReason(migrated)) {
|
|
17123
|
+
const detail = migrated === "argv-array-has-no-portable-shell-string" ? ARGV_ARRAY_BLOCK_DETAIL : undefined;
|
|
17124
|
+
return blocked(input, migrated, detail);
|
|
17125
|
+
}
|
|
17122
17126
|
const after = Buffer.from($stringify(migrated), "utf8");
|
|
17123
17127
|
try {
|
|
17124
17128
|
parseTaskV3Yaml({
|
|
@@ -23871,6 +23875,27 @@ function describeJsonRoot(value) {
|
|
|
23871
23875
|
return "a boolean";
|
|
23872
23876
|
return typeof value;
|
|
23873
23877
|
}
|
|
23878
|
+
function pruneToNewest(dir, keep, select) {
|
|
23879
|
+
let entries;
|
|
23880
|
+
try {
|
|
23881
|
+
entries = fs18.readdirSync(dir, { withFileTypes: true });
|
|
23882
|
+
} catch {
|
|
23883
|
+
return;
|
|
23884
|
+
}
|
|
23885
|
+
const candidates = entries.filter(select).map((entry) => {
|
|
23886
|
+
const full = path27.join(dir, entry.name);
|
|
23887
|
+
let mtime = 0;
|
|
23888
|
+
try {
|
|
23889
|
+
mtime = fs18.statSync(full).mtimeMs;
|
|
23890
|
+
} catch {}
|
|
23891
|
+
return { path: full, mtime };
|
|
23892
|
+
}).sort((a, b) => b.mtime - a.mtime);
|
|
23893
|
+
for (const stale of candidates.slice(keep)) {
|
|
23894
|
+
try {
|
|
23895
|
+
fs18.rmSync(stale.path, { recursive: true, force: true });
|
|
23896
|
+
} catch {}
|
|
23897
|
+
}
|
|
23898
|
+
}
|
|
23874
23899
|
function getConfigLockPath() {
|
|
23875
23900
|
return path27.join(getConfigDir(), "config.json.lck");
|
|
23876
23901
|
}
|
|
@@ -29844,6 +29869,10 @@ function inspectCurrentTaskPlan() {
|
|
|
29844
29869
|
function inspectMigrationPlan() {
|
|
29845
29870
|
return inspectCurrentTaskPlan().result;
|
|
29846
29871
|
}
|
|
29872
|
+
var MAX_TASK_MIGRATION_BACKUPS = 5;
|
|
29873
|
+
function pruneTaskMigrationBackups(generationBackupDir) {
|
|
29874
|
+
pruneToNewest(generationBackupDir, MAX_TASK_MIGRATION_BACKUPS, (entry) => entry.isDirectory());
|
|
29875
|
+
}
|
|
29847
29876
|
function printPlan(plan) {
|
|
29848
29877
|
console.log(JSON.stringify(plan));
|
|
29849
29878
|
if (plan.status === "blocked")
|
|
@@ -29861,12 +29890,14 @@ async function runMigrationApply(options = {}) {
|
|
|
29861
29890
|
const before = inspectCurrentTaskPlan();
|
|
29862
29891
|
if (before.result.taskV3Migration.changed === 0)
|
|
29863
29892
|
return before.result;
|
|
29864
|
-
const
|
|
29893
|
+
const backupRoot = path33.join(getDataDir(), "backups", "task-v3");
|
|
29894
|
+
const backupPath = path33.join(backupRoot, `${Date.now()}-${randomUUID2()}`);
|
|
29865
29895
|
const applied = applyTaskToV3MigrationPlan(before.plan, { backupRoot: backupPath });
|
|
29866
29896
|
const after = inspectCurrentTaskPlan().result;
|
|
29867
29897
|
if (after.taskV3Migration.changed > 0) {
|
|
29868
29898
|
throw new ConfigError("Task migration did not converge to task v3.", "INVALID_CONFIG_FILE");
|
|
29869
29899
|
}
|
|
29900
|
+
pruneTaskMigrationBackups(backupRoot);
|
|
29870
29901
|
return { ...after, backupPath, applied: applied.changed.length };
|
|
29871
29902
|
}));
|
|
29872
29903
|
printPlan(result);
|
|
@@ -29924,12 +29955,14 @@ async function runTaskV4MigrationApply(options = {}) {
|
|
|
29924
29955
|
const before = inspectCurrentTaskV4Plan();
|
|
29925
29956
|
if (before.result.taskV4Migration.changed === 0)
|
|
29926
29957
|
return before.result;
|
|
29927
|
-
const
|
|
29958
|
+
const backupRoot = path33.join(getDataDir(), "backups", "task-v4");
|
|
29959
|
+
const backupPath = path33.join(backupRoot, `${Date.now()}-${randomUUID2()}`);
|
|
29928
29960
|
const applied = applyTaskToV4MigrationPlan(before.plan, { backupRoot: backupPath });
|
|
29929
29961
|
const after = inspectCurrentTaskV4Plan().result;
|
|
29930
29962
|
if (after.taskV4Migration.changed > 0) {
|
|
29931
29963
|
throw new ConfigError("Task migration did not converge to task source v4.", "INVALID_CONFIG_FILE");
|
|
29932
29964
|
}
|
|
29965
|
+
pruneTaskMigrationBackups(backupRoot);
|
|
29933
29966
|
return { ...after, backupPath, applied: applied.changed.length };
|
|
29934
29967
|
}));
|
|
29935
29968
|
printPlan(result);
|
|
@@ -13026,6 +13026,7 @@ function warn(...args) {
|
|
|
13026
13026
|
console.warn(...args);
|
|
13027
13027
|
}
|
|
13028
13028
|
}
|
|
13029
|
+
var warnedOnceKeys = new Set;
|
|
13029
13030
|
function warnVerbose(...args) {
|
|
13030
13031
|
if (sinkOverride) {
|
|
13031
13032
|
sinkOverride("warnVerbose", args);
|
|
@@ -16777,6 +16778,7 @@ function base(input) {
|
|
|
16777
16778
|
...input.containmentRoot ? { containmentRoot: input.containmentRoot } : {}
|
|
16778
16779
|
};
|
|
16779
16780
|
}
|
|
16781
|
+
var ARGV_ARRAY_BLOCK_DETAIL = "Manual conversion required: an array `command:` has no safe v3 `run:` string. Rewrite it by hand as " + "`run:` (string) plus `shell:` \u2014 see docs/migration/v0.9.1-to-v0.9.2.md for the full v2 to v4 field mapping.";
|
|
16780
16782
|
function blocked(input, reason, detail) {
|
|
16781
16783
|
return Object.freeze({ status: "blocked", ...base(input), reason, ...detail ? { detail } : {} });
|
|
16782
16784
|
}
|
|
@@ -17039,8 +17041,10 @@ function planLegacyTaskDataToV3(input, data) {
|
|
|
17039
17041
|
} catch (cause) {
|
|
17040
17042
|
return blocked(input, "invalid-v2-task", cause instanceof Error ? cause.message : String(cause));
|
|
17041
17043
|
}
|
|
17042
|
-
if (isReason(migrated))
|
|
17043
|
-
|
|
17044
|
+
if (isReason(migrated)) {
|
|
17045
|
+
const detail = migrated === "argv-array-has-no-portable-shell-string" ? ARGV_ARRAY_BLOCK_DETAIL : undefined;
|
|
17046
|
+
return blocked(input, migrated, detail);
|
|
17047
|
+
}
|
|
17044
17048
|
const after = Buffer.from($stringify(migrated), "utf8");
|
|
17045
17049
|
try {
|
|
17046
17050
|
parseTaskV3Yaml({
|
|
@@ -23793,6 +23797,27 @@ function describeJsonRoot(value) {
|
|
|
23793
23797
|
return "a boolean";
|
|
23794
23798
|
return typeof value;
|
|
23795
23799
|
}
|
|
23800
|
+
function pruneToNewest(dir, keep, select) {
|
|
23801
|
+
let entries;
|
|
23802
|
+
try {
|
|
23803
|
+
entries = fs18.readdirSync(dir, { withFileTypes: true });
|
|
23804
|
+
} catch {
|
|
23805
|
+
return;
|
|
23806
|
+
}
|
|
23807
|
+
const candidates = entries.filter(select).map((entry) => {
|
|
23808
|
+
const full = path27.join(dir, entry.name);
|
|
23809
|
+
let mtime = 0;
|
|
23810
|
+
try {
|
|
23811
|
+
mtime = fs18.statSync(full).mtimeMs;
|
|
23812
|
+
} catch {}
|
|
23813
|
+
return { path: full, mtime };
|
|
23814
|
+
}).sort((a, b) => b.mtime - a.mtime);
|
|
23815
|
+
for (const stale of candidates.slice(keep)) {
|
|
23816
|
+
try {
|
|
23817
|
+
fs18.rmSync(stale.path, { recursive: true, force: true });
|
|
23818
|
+
} catch {}
|
|
23819
|
+
}
|
|
23820
|
+
}
|
|
23796
23821
|
function getConfigLockPath() {
|
|
23797
23822
|
return path27.join(getConfigDir(), "config.json.lck");
|
|
23798
23823
|
}
|
|
@@ -29766,6 +29791,10 @@ function inspectCurrentTaskPlan() {
|
|
|
29766
29791
|
function inspectMigrationPlan() {
|
|
29767
29792
|
return inspectCurrentTaskPlan().result;
|
|
29768
29793
|
}
|
|
29794
|
+
var MAX_TASK_MIGRATION_BACKUPS = 5;
|
|
29795
|
+
function pruneTaskMigrationBackups(generationBackupDir) {
|
|
29796
|
+
pruneToNewest(generationBackupDir, MAX_TASK_MIGRATION_BACKUPS, (entry) => entry.isDirectory());
|
|
29797
|
+
}
|
|
29769
29798
|
function printPlan(plan) {
|
|
29770
29799
|
console.log(JSON.stringify(plan));
|
|
29771
29800
|
if (plan.status === "blocked")
|
|
@@ -29783,12 +29812,14 @@ async function runMigrationApply(options = {}) {
|
|
|
29783
29812
|
const before = inspectCurrentTaskPlan();
|
|
29784
29813
|
if (before.result.taskV3Migration.changed === 0)
|
|
29785
29814
|
return before.result;
|
|
29786
|
-
const
|
|
29815
|
+
const backupRoot = path33.join(getDataDir(), "backups", "task-v3");
|
|
29816
|
+
const backupPath = path33.join(backupRoot, `${Date.now()}-${randomUUID2()}`);
|
|
29787
29817
|
const applied = applyTaskToV3MigrationPlan(before.plan, { backupRoot: backupPath });
|
|
29788
29818
|
const after = inspectCurrentTaskPlan().result;
|
|
29789
29819
|
if (after.taskV3Migration.changed > 0) {
|
|
29790
29820
|
throw new ConfigError("Task migration did not converge to task v3.", "INVALID_CONFIG_FILE");
|
|
29791
29821
|
}
|
|
29822
|
+
pruneTaskMigrationBackups(backupRoot);
|
|
29792
29823
|
return { ...after, backupPath, applied: applied.changed.length };
|
|
29793
29824
|
}));
|
|
29794
29825
|
printPlan(result);
|
|
@@ -29846,12 +29877,14 @@ async function runTaskV4MigrationApply(options = {}) {
|
|
|
29846
29877
|
const before = inspectCurrentTaskV4Plan();
|
|
29847
29878
|
if (before.result.taskV4Migration.changed === 0)
|
|
29848
29879
|
return before.result;
|
|
29849
|
-
const
|
|
29880
|
+
const backupRoot = path33.join(getDataDir(), "backups", "task-v4");
|
|
29881
|
+
const backupPath = path33.join(backupRoot, `${Date.now()}-${randomUUID2()}`);
|
|
29850
29882
|
const applied = applyTaskToV4MigrationPlan(before.plan, { backupRoot: backupPath });
|
|
29851
29883
|
const after = inspectCurrentTaskV4Plan().result;
|
|
29852
29884
|
if (after.taskV4Migration.changed > 0) {
|
|
29853
29885
|
throw new ConfigError("Task migration did not converge to task source v4.", "INVALID_CONFIG_FILE");
|
|
29854
29886
|
}
|
|
29887
|
+
pruneTaskMigrationBackups(backupRoot);
|
|
29855
29888
|
return { ...after, backupPath, applied: applied.changed.length };
|
|
29856
29889
|
}));
|
|
29857
29890
|
printPlan(result);
|
|
@@ -336,12 +336,28 @@ export function getPositiveFeedbackCountsByIds(ids) {
|
|
|
336
336
|
}, "positive feedback counts are best-effort");
|
|
337
337
|
return result;
|
|
338
338
|
}
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
339
|
+
/**
|
|
340
|
+
* Rows whose `file_path` sits directly in `dirPath`. A half-open byte range
|
|
341
|
+
* over `idx_entries_file_path` (`[dir + sep, dir + sep + 1)`) turns the lookup
|
|
342
|
+
* into an index seek; the range is exact for "starts with `dir/`" but also
|
|
343
|
+
* admits nested subdirectories (`/a/b/c/x.md` for `/a/b`), so the dirname
|
|
344
|
+
* post-filter stays.
|
|
345
|
+
*/
|
|
346
|
+
function selectRowsInDirectory(db, dirPath, columns, bundleId) {
|
|
343
347
|
const resolvedDir = path.resolve(dirPath);
|
|
344
|
-
|
|
348
|
+
const prefix = resolvedDir + path.sep;
|
|
349
|
+
const upperBound = resolvedDir + String.fromCharCode(path.sep.charCodeAt(0) + 1);
|
|
350
|
+
const params = [prefix, upperBound];
|
|
351
|
+
let sql = `SELECT ${columns} FROM entries WHERE file_path >= ? AND file_path < ?`;
|
|
352
|
+
if (bundleId) {
|
|
353
|
+
sql += " AND bundle_id = ?";
|
|
354
|
+
params.push(bundleId);
|
|
355
|
+
}
|
|
356
|
+
const rows = db.prepare(sql).all(...params);
|
|
357
|
+
return rows.filter((row) => path.dirname(row.file_path) === resolvedDir);
|
|
358
|
+
}
|
|
359
|
+
function rowsInDirectory(db, dirPath, bundleId) {
|
|
360
|
+
return selectRowsInDirectory(db, dirPath, "id, item_ref, file_path", bundleId);
|
|
345
361
|
}
|
|
346
362
|
function deleteEntryRows(db, rows, options = {}) {
|
|
347
363
|
if (rows.length === 0)
|
|
@@ -629,9 +645,7 @@ export function getEntryById(db, id) {
|
|
|
629
645
|
};
|
|
630
646
|
}
|
|
631
647
|
export function getEntriesByDir(db, dirPath) {
|
|
632
|
-
|
|
633
|
-
const rows = db.prepare(`SELECT ${ENTRY_COLUMNS} FROM entries`).all().filter((row) => ids.has(row.id));
|
|
634
|
-
return parseEntryRows(rows, "getEntriesByDir");
|
|
648
|
+
return parseEntryRows(selectRowsInDirectory(db, dirPath, ENTRY_COLUMNS), "getEntriesByDir");
|
|
635
649
|
}
|
|
636
650
|
/** Return every directory previously indexed for one canonical bundle. */
|
|
637
651
|
export function getIndexedDirPathsByBundleId(db, bundleId) {
|
|
@@ -640,9 +654,8 @@ export function getIndexedDirPathsByBundleId(db, bundleId) {
|
|
|
640
654
|
}
|
|
641
655
|
/** Return every persisted bundle owner for one physical directory. */
|
|
642
656
|
export function getIndexedBundleIdsByDir(db, dirPath) {
|
|
643
|
-
const
|
|
644
|
-
|
|
645
|
-
return [...new Set(rows.filter((row) => ids.has(row.id)).map((row) => row.bundle_id))];
|
|
657
|
+
const rows = selectRowsInDirectory(db, dirPath, "bundle_id, file_path");
|
|
658
|
+
return [...new Set(rows.map((row) => row.bundle_id))];
|
|
646
659
|
}
|
|
647
660
|
/**
|
|
648
661
|
* Resolve a single `entries.id` by exact `file_path` (the canonical on-disk
|
|
@@ -23,7 +23,7 @@ export function deleteMeta(db, key) {
|
|
|
23
23
|
// ── Per-directory index state ───────────────────────────────────────────────
|
|
24
24
|
export function getIndexDirState(db, dirPath) {
|
|
25
25
|
const row = db
|
|
26
|
-
.prepare("SELECT dir_path, file_set_hash, file_mtime_max_ms, reason, updated_at FROM index_dir_state WHERE dir_path = ?")
|
|
26
|
+
.prepare("SELECT dir_path, file_set_hash, file_mtime_max_ms, reason, updated_at, row_count FROM index_dir_state WHERE dir_path = ?")
|
|
27
27
|
.get(dirPath);
|
|
28
28
|
if (!row)
|
|
29
29
|
return undefined;
|
|
@@ -33,16 +33,18 @@ export function getIndexDirState(db, dirPath) {
|
|
|
33
33
|
fileMtimeMaxMs: row.file_mtime_max_ms,
|
|
34
34
|
reason: row.reason,
|
|
35
35
|
updatedAt: row.updated_at,
|
|
36
|
+
rowCount: row.row_count ?? undefined,
|
|
36
37
|
};
|
|
37
38
|
}
|
|
38
39
|
export function upsertIndexDirState(db, state) {
|
|
39
|
-
db.prepare(`INSERT INTO index_dir_state (dir_path, file_set_hash, file_mtime_max_ms, reason, updated_at)
|
|
40
|
-
VALUES (?, ?, ?, ?, ?)
|
|
40
|
+
db.prepare(`INSERT INTO index_dir_state (dir_path, file_set_hash, file_mtime_max_ms, reason, updated_at, row_count)
|
|
41
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
41
42
|
ON CONFLICT(dir_path) DO UPDATE SET
|
|
42
43
|
file_set_hash = excluded.file_set_hash,
|
|
43
44
|
file_mtime_max_ms = excluded.file_mtime_max_ms,
|
|
44
45
|
reason = excluded.reason,
|
|
45
|
-
updated_at = excluded.updated_at
|
|
46
|
+
updated_at = excluded.updated_at,
|
|
47
|
+
row_count = excluded.row_count`).run(state.dirPath, state.fileSetHash, state.fileMtimeMaxMs, state.reason, new Date().toISOString(), state.rowCount ?? null);
|
|
46
48
|
}
|
|
47
49
|
export function deleteIndexDirState(db, dirPath) {
|
|
48
50
|
db.prepare("DELETE FROM index_dir_state WHERE dir_path = ?").run(dirPath);
|
|
@@ -268,9 +268,11 @@ export function ensureSchema(db, embeddingDim) {
|
|
|
268
268
|
file_set_hash TEXT NOT NULL,
|
|
269
269
|
file_mtime_max_ms REAL NOT NULL,
|
|
270
270
|
reason TEXT NOT NULL,
|
|
271
|
-
updated_at TEXT NOT NULL
|
|
271
|
+
updated_at TEXT NOT NULL,
|
|
272
|
+
row_count INTEGER
|
|
272
273
|
);
|
|
273
274
|
`);
|
|
275
|
+
ensureIndexDirStateRowCountColumn(db);
|
|
274
276
|
// LLM enrichment result cache. Stores a SHA-256 body hash and the JSON
|
|
275
277
|
// result for each asset so that subsequent `akm index --enrich` runs can
|
|
276
278
|
// skip the LLM call when the body hasn't changed. The cache is keyed by
|
|
@@ -379,3 +381,16 @@ function tableExists(db, name) {
|
|
|
379
381
|
const row = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=? LIMIT 1").get(name);
|
|
380
382
|
return row !== undefined && row !== null;
|
|
381
383
|
}
|
|
384
|
+
/**
|
|
385
|
+
* #900: `row_count` was added after the table's first release, so a database
|
|
386
|
+
* created before it needs an `ALTER TABLE` (`CREATE TABLE IF NOT EXISTS` only
|
|
387
|
+
* shapes a fresh table). Idempotent. Pre-existing rows keep NULL until their
|
|
388
|
+
* directory is next drained; index.db is a regenerable cache, so nothing is
|
|
389
|
+
* backfilled.
|
|
390
|
+
*/
|
|
391
|
+
function ensureIndexDirStateRowCountColumn(db) {
|
|
392
|
+
const columns = db.prepare("PRAGMA table_info(index_dir_state)").all();
|
|
393
|
+
if (!columns.some((column) => column.name === "row_count")) {
|
|
394
|
+
db.exec("ALTER TABLE index_dir_state ADD COLUMN row_count INTEGER");
|
|
395
|
+
}
|
|
396
|
+
}
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { stashDirFor } from "../../core/asset/asset-placement.js";
|
|
14
14
|
import { bundleRefToString, isBundleSlug, parseBundleRef } from "../../core/asset/asset-ref.js";
|
|
15
|
+
import { warnOnce } from "../../core/warn.js";
|
|
15
16
|
/** Serialize `Proposal.changes` for `metadata_json` (see {@link StoredFileChange}). */
|
|
16
17
|
function changesToStored(changes) {
|
|
17
18
|
return changes.map((c, i) => ({
|
|
@@ -361,8 +362,10 @@ export function listStateProposals(db, options = {}) {
|
|
|
361
362
|
proposals.push(proposalRowToProposal(row));
|
|
362
363
|
}
|
|
363
364
|
catch (error) {
|
|
365
|
+
// Once per row per process (#898): health alone reads this table several
|
|
366
|
+
// times per invocation.
|
|
364
367
|
const message = error instanceof Error ? error.message : String(error);
|
|
365
|
-
|
|
368
|
+
warnOnce(`unparseable-proposal-row:${row.id}`, `[akm] Skipping unparseable proposal row (id=${row.id}, ref=${row.ref}): ${message}`);
|
|
366
369
|
}
|
|
367
370
|
}
|
|
368
371
|
return proposals;
|
|
@@ -152,7 +152,22 @@ function samePath(left, right) {
|
|
|
152
152
|
};
|
|
153
153
|
return normalize(left) === normalize(right);
|
|
154
154
|
}
|
|
155
|
+
// #901: the global root can't change within a process lifetime, and every
|
|
156
|
+
// `resolveAkmInvocation()` call in a scheduler-sync process re-derives the
|
|
157
|
+
// same `nodePath` — so this spawns `npm root --global` at most once per
|
|
158
|
+
// process instead of once per call. Keyed by nodePath (rather than a bare
|
|
159
|
+
// once-only flag) so a differently-invoked probe later in the same process
|
|
160
|
+
// still gets its own answer instead of a stale one.
|
|
161
|
+
let cachedNpmGlobalRoot;
|
|
155
162
|
function resolveNpmGlobalRoot(nodePath, env) {
|
|
163
|
+
if (cachedNpmGlobalRoot && cachedNpmGlobalRoot.nodePath === nodePath) {
|
|
164
|
+
return cachedNpmGlobalRoot.value;
|
|
165
|
+
}
|
|
166
|
+
const value = resolveNpmGlobalRootUncached(nodePath, env);
|
|
167
|
+
cachedNpmGlobalRoot = { nodePath, value };
|
|
168
|
+
return value;
|
|
169
|
+
}
|
|
170
|
+
function resolveNpmGlobalRootUncached(nodePath, env) {
|
|
156
171
|
const npmCli = resolveAssociatedNpmCli(nodePath);
|
|
157
172
|
if (!npmCli)
|
|
158
173
|
return undefined;
|