akm-cli 0.9.15-beta.1 → 0.9.15-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +234 -13
- package/dist/akm +54 -1
- package/dist/akm-migrate +34 -1
- package/dist/cli.js +40 -4
- package/dist/commands/health/checks.js +11 -2
- package/dist/commands/health/scheduler-binary.js +120 -0
- package/dist/commands/health.js +9 -0
- package/dist/commands/improve/locks.js +3 -2
- package/dist/commands/sources/installed-stashes.js +58 -16
- package/dist/commands/sources/stash-cli.js +17 -0
- package/dist/core/config/schema/embedding.js +41 -1
- package/dist/core/errors.js +1 -0
- package/dist/core/file-lock.js +49 -15
- package/dist/core/parent-watchdog.js +64 -0
- package/dist/core/run-lock.js +13 -2
- package/dist/indexer/index-rebuild-lock.js +4 -4
- package/dist/indexer/index-written-assets.js +9 -1
- package/dist/indexer/indexer.js +123 -19
- package/dist/indexer/materialize-embeddings.js +345 -37
- package/dist/indexer/search/search-source.js +23 -1
- package/dist/llm/embedders/remote.js +443 -47
- package/dist/scripts/akm-migrate-node.js +454 -85
- package/dist/scripts/akm-migrate.js +454 -85
- package/dist/storage/repositories/embedding-salvage-repository.js +184 -0
- package/dist/storage/repositories/index-schema.js +16 -0
- package/dist/tasks/run/run-native-task.js +23 -1
- package/docs/migration/release-notes/0.9.15.md +103 -4
- package/docs/migration/release-notes/README.md +3 -2
- package/docs/reference/cli.md +55 -8
- package/docs/reference/configuration.md +83 -15
- package/package.json +1 -1
- package/schemas/akm-config.json +36 -9
package/dist/commands/health.js
CHANGED
|
@@ -27,6 +27,7 @@ import { buildImproveSkipSummary, computeWallTimeStats, countAgentFailureReasons
|
|
|
27
27
|
import { emptyLlmUsageAggregate, readLlmUsageAggregate } from "./health/llm-usage.js";
|
|
28
28
|
import { computeDegradationMetrics, computeDenominatorFixedCoverage, computeEnrichmentMintingRollup, probeStateDbRoundTrip, } from "./health/metrics.js";
|
|
29
29
|
import { collectPluginStalenessAdvisories } from "./health/plugin-staleness.js";
|
|
30
|
+
import { collectSchedulerBinaryAdvisory } from "./health/scheduler-binary.js";
|
|
30
31
|
import { collectStashExposureAdvisory } from "./health/stash-exposure.js";
|
|
31
32
|
import { collectSurfacesAdvisories } from "./health/surfaces.js";
|
|
32
33
|
import { buildPerRunSummaries } from "./health/task-runs.js";
|
|
@@ -542,6 +543,12 @@ export async function akmHealth(options = {}) {
|
|
|
542
543
|
// above — started here, alongside it, and awaited later.
|
|
543
544
|
const versionDriftPromise = collectVersionDriftAdvisory(Boolean(options.probe), { cliVersion: pkgVersion });
|
|
544
545
|
versionDriftPromise.catch(() => undefined);
|
|
546
|
+
// #953: same --probe-gated, best-effort discipline as versionDriftPromise
|
|
547
|
+
// above.
|
|
548
|
+
const schedulerBinaryDriftPromise = collectSchedulerBinaryAdvisory(Boolean(options.probe), {
|
|
549
|
+
cliVersion: pkgVersion,
|
|
550
|
+
});
|
|
551
|
+
schedulerBinaryDriftPromise.catch(() => undefined);
|
|
545
552
|
const taskHistory = gatherTaskHistoryPhase(db, logsDb, since, stateDbPath, now);
|
|
546
553
|
const { tableNames, missingTables, probe } = taskHistory;
|
|
547
554
|
const { egressConfigView, thinkingOffEngines } = gatherEgressConfigPhase();
|
|
@@ -559,6 +566,7 @@ export async function akmHealth(options = {}) {
|
|
|
559
566
|
const improveRunsInLookbackWindow = countImproveRunsSince(db, engineLastUsedSinceIso);
|
|
560
567
|
const engineProbes = await engineProbesPromise;
|
|
561
568
|
const versionDrift = await versionDriftPromise;
|
|
569
|
+
const schedulerBinaryDrift = await schedulerBinaryDriftPromise;
|
|
562
570
|
// Read once, shared by the `thinking-control` check (#949) and the
|
|
563
571
|
// `metrics.llmUsage` report field below — same window, same aggregate.
|
|
564
572
|
const llmUsage = readLlmUsageAggregate(stateDbPath, since);
|
|
@@ -592,6 +600,7 @@ export async function akmHealth(options = {}) {
|
|
|
592
600
|
activeImproveStrategyEngines,
|
|
593
601
|
engineLastUsed,
|
|
594
602
|
improveRunsInLookbackWindow,
|
|
603
|
+
schedulerBinaryDrift,
|
|
595
604
|
};
|
|
596
605
|
for (const check of HEALTH_CHECKS) {
|
|
597
606
|
const result = check.run(checkContext);
|
|
@@ -6,7 +6,7 @@ import { ConfigError } from "../../core/errors.js";
|
|
|
6
6
|
import { appendEvent } from "../../core/events.js";
|
|
7
7
|
import { releaseLock } from "../../core/file-lock.js";
|
|
8
8
|
import { tryWithMaintenanceStartBarrier, withMaintenanceStartBarrier } from "../../core/maintenance-barrier.js";
|
|
9
|
-
import { tryAcquireRunLock } from "../../core/run-lock.js";
|
|
9
|
+
import { formatLockHolderPid, tryAcquireRunLock } from "../../core/run-lock.js";
|
|
10
10
|
import { warn } from "../../core/warn.js";
|
|
11
11
|
export function improveLockPath(lockBaseDir) {
|
|
12
12
|
return path.join(lockBaseDir, "improve.lock");
|
|
@@ -59,7 +59,8 @@ function tryAcquireImproveLockUnlocked(lockPath, skipIfLocked, onRecovered) {
|
|
|
59
59
|
if (result.state === "acquired") {
|
|
60
60
|
return { state: "acquired", ownership: result.ownership };
|
|
61
61
|
}
|
|
62
|
-
const {
|
|
62
|
+
const { startedAt } = result.holder;
|
|
63
|
+
const pid = formatLockHolderPid(result.holder);
|
|
63
64
|
if (skipIfLocked) {
|
|
64
65
|
warn(`[improve] another improve run holds the lock (PID ${pid}, started ${startedAt}); skipping (--skip-if-locked)`);
|
|
65
66
|
return { state: "skipped" };
|
|
@@ -26,7 +26,7 @@ import { beginImmediateTransaction, getStateDbPath, openStateDatabase } from "..
|
|
|
26
26
|
import { warn } from "../../core/warn.js";
|
|
27
27
|
import { resolveGitContentRoot } from "../../core/write-source.js";
|
|
28
28
|
import { withAssetMutationLease } from "../../indexer/index-writer-lock.js";
|
|
29
|
-
import { akmIndex } from "../../indexer/indexer.js";
|
|
29
|
+
import { akmIndex, runEmbeddingPass } from "../../indexer/indexer.js";
|
|
30
30
|
import { compareAndSwapLockfileSnapshot, publishLockfileUpdate, readLockfile, readLockfileForUpdate, } from "../../integrations/lockfile.js";
|
|
31
31
|
import { parseRegistryRef } from "../../registry/resolve.js";
|
|
32
32
|
import { sha256Hex } from "../../runtime.js";
|
|
@@ -350,11 +350,19 @@ export async function akmRemove(input) {
|
|
|
350
350
|
},
|
|
351
351
|
};
|
|
352
352
|
}
|
|
353
|
-
/**
|
|
353
|
+
/**
|
|
354
|
+
* Read the current index generation without creating or hydrating anything.
|
|
355
|
+
* This path never ran an embedding pass (#954, field-report follow-up), so it has no
|
|
356
|
+
* `verification` to report — {@link buildUpdateResponse} falls back to the
|
|
357
|
+
* two facts it can actually know (whether semantic search is configured on
|
|
358
|
+
* at all) rather than fabricating verification numbers (`entryCount: 0`,
|
|
359
|
+
* `ok: true`) for a run that never verified anything.
|
|
360
|
+
*/
|
|
354
361
|
function readCurrentIndexSummary() {
|
|
355
362
|
const db = openReadonlyExistingDatabase(getDbPath());
|
|
356
|
-
if (!db)
|
|
363
|
+
if (!db) {
|
|
357
364
|
return { mode: "incremental", totalEntries: 0, directoriesScanned: 0, directoriesSkipped: 0 };
|
|
365
|
+
}
|
|
358
366
|
try {
|
|
359
367
|
return {
|
|
360
368
|
mode: "incremental",
|
|
@@ -388,6 +396,12 @@ function buildUpdateResponse(stashDir, target, all, processed, opts) {
|
|
|
388
396
|
directoriesScanned: index.directoriesScanned,
|
|
389
397
|
directoriesSkipped: index.directoriesSkipped,
|
|
390
398
|
...(index.scanComplete !== undefined ? { scanComplete: index.scanComplete } : {}),
|
|
399
|
+
// A real embedding pass (`akmIndex`/`runEmbeddingPass`) reports its own
|
|
400
|
+
// verified `semanticStatus`. When no pass ran this update (the
|
|
401
|
+
// no-op/nothing-configured fallback) the only two facts known without
|
|
402
|
+
// fabricating a verification are whether semantic search is off at all
|
|
403
|
+
// or, if not, that its state is simply unverified this run.
|
|
404
|
+
semanticStatus: index.verification?.semanticStatus ?? (finalConfig.semanticSearchMode === "off" ? "disabled" : "pending"),
|
|
391
405
|
},
|
|
392
406
|
};
|
|
393
407
|
}
|
|
@@ -499,6 +513,45 @@ function closeUnifiedUpdateTransaction(transaction, committed) {
|
|
|
499
513
|
: `[akm bundle update] rolled back, but closing its database handles failed: ${String(closeError)}`);
|
|
500
514
|
}
|
|
501
515
|
}
|
|
516
|
+
/**
|
|
517
|
+
* Run the embedding phase AFTER the coordinator's atomic commit, on its own
|
|
518
|
+
* fresh connection (#954). Before this, `akmUpdate` called
|
|
519
|
+
* `akmIndex()` for its embedding phase too, INSIDE this same unified
|
|
520
|
+
* `BEGIN IMMEDIATE` — so every per-batch commit the materializer opened
|
|
521
|
+
* nested as an unobservable SAVEPOINT, and a SIGKILL mid-run lost every
|
|
522
|
+
* embedding of the run rather than just the one in flight. The
|
|
523
|
+
* ambient-transaction drift guard (#954) now rejects that outright, so
|
|
524
|
+
* `akmIndex`'s own embedding phase is skipped for a deferred update
|
|
525
|
+
* transaction and this runs instead, once content/lock/index/state are
|
|
526
|
+
* already durably committed.
|
|
527
|
+
*
|
|
528
|
+
* A failing pass (provider down, timeout) does NOT fail the update: the
|
|
529
|
+
* bundle content and index are already committed successfully, exactly like
|
|
530
|
+
* a plain `akm index` whose embedding phase fails — only the reported
|
|
531
|
+
* `verification` reflects the shortfall (`semanticStatus: "blocked"`).
|
|
532
|
+
*/
|
|
533
|
+
async function runPostCommitEmbeddingPass(index) {
|
|
534
|
+
const config = loadConfig();
|
|
535
|
+
let db;
|
|
536
|
+
try {
|
|
537
|
+
const embeddingDim = config.embedding?.dimension;
|
|
538
|
+
db = openIndexDatabase(getDbPath(), embeddingDim ? { embeddingDim } : undefined);
|
|
539
|
+
const { verification } = await runEmbeddingPass({ db, config, onProgress: () => { } });
|
|
540
|
+
return { ...index, verification };
|
|
541
|
+
}
|
|
542
|
+
catch (error) {
|
|
543
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
544
|
+
warn(`[akm bundle update] post-commit embedding pass failed: ${message}`);
|
|
545
|
+
return {
|
|
546
|
+
...index,
|
|
547
|
+
verification: { ...index.verification, ok: false, semanticStatus: "blocked", message },
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
finally {
|
|
551
|
+
if (db)
|
|
552
|
+
closeDatabase(db);
|
|
553
|
+
}
|
|
554
|
+
}
|
|
502
555
|
function pathAtOrBelow(candidate, root) {
|
|
503
556
|
const relative = path.relative(path.resolve(root), path.resolve(candidate));
|
|
504
557
|
return relative === "" || (relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
|
|
@@ -779,14 +832,8 @@ async function publishPreparedPlainUpdate(id, ref, prepared, stashDir, allowInse
|
|
|
779
832
|
updateTransactionHook("before-commit", id, { db: transaction.db });
|
|
780
833
|
commitUnifiedUpdateTransaction(transaction);
|
|
781
834
|
committed = true;
|
|
782
|
-
try {
|
|
783
|
-
transaction.deferred.afterCommit?.();
|
|
784
|
-
}
|
|
785
|
-
catch (error) {
|
|
786
|
-
warn(`[akm bundle update] committed, but semantic status refresh failed: ${String(error)}`);
|
|
787
|
-
}
|
|
788
835
|
prepared.publication?.commit();
|
|
789
|
-
return index;
|
|
836
|
+
return await runPostCommitEmbeddingPass(index);
|
|
790
837
|
}
|
|
791
838
|
catch (error) {
|
|
792
839
|
let recoveryError = transaction ? rollbackUnifiedUpdateTransaction(transaction) : undefined;
|
|
@@ -1093,12 +1140,6 @@ async function updateManagedInstall(managed, force, yes, stashDir, allowInsecure
|
|
|
1093
1140
|
updateTransactionHook("before-commit", managed.installId, { db: transaction.db });
|
|
1094
1141
|
commitUnifiedUpdateTransaction(transaction);
|
|
1095
1142
|
committed = true;
|
|
1096
|
-
try {
|
|
1097
|
-
transaction.deferred.afterCommit?.();
|
|
1098
|
-
}
|
|
1099
|
-
catch (error) {
|
|
1100
|
-
warn(`[akm bundle update] committed, but semantic status refresh failed: ${String(error)}`);
|
|
1101
|
-
}
|
|
1102
1143
|
}
|
|
1103
1144
|
catch (error) {
|
|
1104
1145
|
let recoveryError = transaction ? rollbackUnifiedUpdateTransaction(transaction) : undefined;
|
|
@@ -1146,6 +1187,7 @@ async function updateManagedInstall(managed, force, yes, stashDir, allowInsecure
|
|
|
1146
1187
|
}
|
|
1147
1188
|
}
|
|
1148
1189
|
prepared.publication?.commit();
|
|
1190
|
+
index = await runPostCommitEmbeddingPass(index);
|
|
1149
1191
|
if (movedRoot) {
|
|
1150
1192
|
const currentConfig = loadConfig();
|
|
1151
1193
|
const currentLocks = readLockfile();
|
|
@@ -47,6 +47,8 @@ import { akmIndex } from "../../indexer/indexer.js";
|
|
|
47
47
|
import { getHyphenatedBoolean, getOutputMode } from "../../output/context.js";
|
|
48
48
|
import { inferAssetName, mergeXrefsIntoContent, readKnowledgeInput, resolveSupersedesForWrite, resolveSupersedesWriteTarget, resolveXrefsForWrite, writeMarkdownAsset, } from "../read/knowledge.js";
|
|
49
49
|
import { assembleInfo } from "./info.js";
|
|
50
|
+
/** Matches the high-frequency per-committed-batch progress line (#954), excluded from non-verbose JSON-mode stderr. */
|
|
51
|
+
const EMBEDDED_BATCH_PROGRESS_PATTERN = /^Embedded \d+\/\d+ entries\.$/;
|
|
50
52
|
export const indexCommand = defineCommand({
|
|
51
53
|
meta: { name: "index", description: "Build search index (incremental by default; --full forces full reindex)" },
|
|
52
54
|
args: {
|
|
@@ -98,6 +100,10 @@ export const indexCommand = defineCommand({
|
|
|
98
100
|
skipped: {
|
|
99
101
|
reason: "lock-held",
|
|
100
102
|
pid: lockAcquisition.holder.pid,
|
|
103
|
+
// #956: the launcher pid (when known) alongside the pid that
|
|
104
|
+
// actually holds the lock — every process listing and task log
|
|
105
|
+
// shows the launcher pid, not the bun/node child's.
|
|
106
|
+
launcherPid: lockAcquisition.holder.launcherPid,
|
|
101
107
|
startedAt: lockAcquisition.holder.startedAt,
|
|
102
108
|
},
|
|
103
109
|
});
|
|
@@ -137,6 +143,17 @@ export const indexCommand = defineCommand({
|
|
|
137
143
|
spin.stop(`${progressPrefix}${message}`);
|
|
138
144
|
spin.start(`${progressPrefix}${message}`);
|
|
139
145
|
}
|
|
146
|
+
else if (!EMBEDDED_BATCH_PROGRESS_PATTERN.test(message)) {
|
|
147
|
+
// Non-verbose, non-text (JSON/yaml/etc) mode: silence used to be
|
|
148
|
+
// total until the run finished (#954) — a stalled
|
|
149
|
+
// run looked identical to "nothing written". Phase-start
|
|
150
|
+
// messages and the embedding heartbeat now reach stderr here
|
|
151
|
+
// too; the high-frequency per-batch `Embedded N/M entries.`
|
|
152
|
+
// line (emitted after every committed batch)
|
|
153
|
+
// is deliberately excluded — that would be spam, not a
|
|
154
|
+
// heartbeat.
|
|
155
|
+
info(`[index:${phase}] ${progressPrefix}${message}`);
|
|
156
|
+
}
|
|
140
157
|
},
|
|
141
158
|
signal: controller.signal,
|
|
142
159
|
});
|
|
@@ -33,10 +33,50 @@ export const EmbeddingConnectionConfigSchema = z
|
|
|
33
33
|
// `akm index` when ensureSchema rejects it (§24.2 "Semantic" gate).
|
|
34
34
|
dimension: positiveInt.max(4096).optional(),
|
|
35
35
|
localModel: z.string().min(1).optional(),
|
|
36
|
+
/**
|
|
37
|
+
* Per-document token cap applied BEFORE batching (default 512,
|
|
38
|
+
* `DEFAULT_MAX_INPUT_TOKENS` in `src/llm/embedders/remote.ts`, #956).
|
|
39
|
+
* The materializer truncates a document's embedded text to
|
|
40
|
+
* this cap (head only, unicode-safe) instead of skipping it outright, so
|
|
41
|
+
* one oversized entry can no longer fail a whole batch. Distinct from
|
|
42
|
+
* `maxTokens` below, which bounds a whole HTTP REQUEST (many documents);
|
|
43
|
+
* this bounds one DOCUMENT.
|
|
44
|
+
*/
|
|
45
|
+
maxInputTokens: positiveInt.optional(),
|
|
46
|
+
/**
|
|
47
|
+
* Client-side per-request token budget — how many documents' estimated
|
|
48
|
+
* tokens fit in one HTTP request (default `DEFAULT_TOKEN_BUDGET` = 6000
|
|
49
|
+
* in `src/llm/embedders/remote.ts`). With the 512-token `maxInputTokens`
|
|
50
|
+
* cap above, a request carries about 11 documents by default.
|
|
51
|
+
*/
|
|
36
52
|
maxTokens: positiveInt.optional(),
|
|
37
53
|
batchSize: positiveInt.optional(),
|
|
38
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Ollama's `num_ctx` ONLY (#956) — sent verbatim as
|
|
56
|
+
* `options.num_ctx` on the native `/api/embed` request. It no longer also
|
|
57
|
+
* feeds the client-side request token budget (`maxTokens` above): the two
|
|
58
|
+
* used to share this one field, so setting it for the server's context
|
|
59
|
+
* window silently changed request batching too.
|
|
60
|
+
*/
|
|
39
61
|
contextLength: positiveInt.optional(),
|
|
40
62
|
ollamaOptions: EmbeddingOllamaOptionsSchema.optional(),
|
|
63
|
+
/**
|
|
64
|
+
* Per-request timeout in milliseconds for a remote embedding request
|
|
65
|
+
* (default 120_000, `DEFAULT_EMBEDDING_TIMEOUT_MS` in
|
|
66
|
+
* `src/llm/embedders/remote.ts`). The prior fixed 30s cut off a slow
|
|
67
|
+
* local model server on a large token-bounded batch mid-response, with
|
|
68
|
+
* no retry — every batch that hit it was silently dropped (#954).
|
|
69
|
+
*/
|
|
70
|
+
timeoutMs: positiveInt.optional(),
|
|
71
|
+
/**
|
|
72
|
+
* Overrides the fixed in-flight request window (#954, added after field
|
|
73
|
+
* evidence from multi-slot local servers). Bounded 1-16. Unset keeps
|
|
74
|
+
* today's default: 1 for a loopback endpoint, 2 for a remote one
|
|
75
|
+
* (`resolveEmbeddingConcurrency`, `src/llm/embedders/remote.ts`). Set it
|
|
76
|
+
* only for an endpoint that genuinely serves parallel requests (llama.cpp
|
|
77
|
+
* `--parallel N`, vLLM) — request SIZE (`batchSize`, `maxTokens`/
|
|
78
|
+
* `contextLength`) remains the first throughput lever.
|
|
79
|
+
*/
|
|
80
|
+
concurrency: positiveInt.max(16).optional(),
|
|
41
81
|
})
|
|
42
82
|
.passthrough();
|
package/dist/core/errors.js
CHANGED
|
@@ -82,6 +82,7 @@ const USAGE_HINTS = {
|
|
|
82
82
|
const TRANSIENT_HINTS = {
|
|
83
83
|
RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
|
|
84
84
|
STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing.",
|
|
85
|
+
INDEX_DB_CONTENDED: "Another akm process is writing index.db; retry shortly, or pass --skip-if-locked on scheduled runs.",
|
|
85
86
|
};
|
|
86
87
|
/** Default hint for each NotFoundError code. */
|
|
87
88
|
const NOT_FOUND_HINTS = {
|
package/dist/core/file-lock.js
CHANGED
|
@@ -120,9 +120,36 @@ function releaseLockRaw(lockPath) {
|
|
|
120
120
|
export function tryAcquireLockSync(lockPath, payload) {
|
|
121
121
|
return withLockOperationMutex(lockPath, () => tryAcquireLockRaw(lockPath, payload));
|
|
122
122
|
}
|
|
123
|
-
/**
|
|
123
|
+
/**
|
|
124
|
+
* Best-effort launcher pid from `AKM_LAUNCHER_PID` (set by
|
|
125
|
+
* `scripts/node-runtime/akm`/`akm-migrate`, #956) — undefined when unset,
|
|
126
|
+
* empty, or not a positive integer (never trust an ambient env var blindly
|
|
127
|
+
* into a lock message). Also the gate the parent-death watchdog
|
|
128
|
+
* (`core/parent-watchdog.ts`) uses to stay inert outside a launcher-managed
|
|
129
|
+
* run.
|
|
130
|
+
*/
|
|
131
|
+
export function launcherPidFromEnv() {
|
|
132
|
+
const raw = process.env.AKM_LAUNCHER_PID;
|
|
133
|
+
if (!raw)
|
|
134
|
+
return undefined;
|
|
135
|
+
const pid = Number.parseInt(raw, 10);
|
|
136
|
+
return Number.isInteger(pid) && pid > 0 ? pid : undefined;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Build a PID-bearing payload with a unique token for one acquisition attempt.
|
|
140
|
+
* Adds `launcherPid` (#956) whenever this process is running under the
|
|
141
|
+
* published launcher, so a lock's holder can be identified by the pid every
|
|
142
|
+
* process listing and task log actually shows (the launcher's) as well as
|
|
143
|
+
* the pid that holds the lock (the bun/node child).
|
|
144
|
+
*/
|
|
124
145
|
export function createLockPayload(metadata = {}) {
|
|
125
|
-
|
|
146
|
+
const launcherPid = launcherPidFromEnv();
|
|
147
|
+
return JSON.stringify({
|
|
148
|
+
...metadata,
|
|
149
|
+
pid: process.pid,
|
|
150
|
+
...(launcherPid !== undefined ? { launcherPid } : {}),
|
|
151
|
+
lockId: randomUUID(),
|
|
152
|
+
});
|
|
126
153
|
}
|
|
127
154
|
/**
|
|
128
155
|
* Inspect an existing sentinel at `lockPath` without modifying it.
|
|
@@ -153,17 +180,17 @@ export function probeLock(lockPath, opts) {
|
|
|
153
180
|
return { state: "absent" };
|
|
154
181
|
const { rawContent, identity } = snapshot;
|
|
155
182
|
const ageMs = Date.now() - identity.mtimeMs;
|
|
156
|
-
const holderPid =
|
|
183
|
+
const { holderPid, launcherPid } = extractLockIdentity(rawContent);
|
|
157
184
|
if (holderPid === undefined) {
|
|
158
185
|
return { state: "stale", reason: "invalid_pid", ageMs, rawContent, identity };
|
|
159
186
|
}
|
|
160
187
|
if (!isProcessAlive(holderPid)) {
|
|
161
|
-
return { state: "stale", reason: "pid_dead", holderPid, ageMs, rawContent, identity };
|
|
188
|
+
return { state: "stale", reason: "pid_dead", holderPid, launcherPid, ageMs, rawContent, identity };
|
|
162
189
|
}
|
|
163
190
|
if (opts?.staleAfterMs !== undefined && ageMs > opts.staleAfterMs) {
|
|
164
|
-
return { state: "stale", reason: "age_exceeded", holderPid, ageMs, rawContent, identity };
|
|
191
|
+
return { state: "stale", reason: "age_exceeded", holderPid, launcherPid, ageMs, rawContent, identity };
|
|
165
192
|
}
|
|
166
|
-
return { state: "held", holderPid, ageMs, rawContent, identity };
|
|
193
|
+
return { state: "held", holderPid, launcherPid, ageMs, rawContent, identity };
|
|
167
194
|
}
|
|
168
195
|
/**
|
|
169
196
|
* Revalidate and quarantine the probed sentinel while holding the same operation
|
|
@@ -254,25 +281,32 @@ export function releaseLock(ownership) {
|
|
|
254
281
|
});
|
|
255
282
|
}
|
|
256
283
|
/**
|
|
257
|
-
* Extract a
|
|
258
|
-
*
|
|
259
|
-
* a
|
|
260
|
-
*
|
|
284
|
+
* Extract a holder pid, and (#956) a launcher pid when the payload recorded
|
|
285
|
+
* one, from a sentinel body. Accepts the two shapes used across the
|
|
286
|
+
* codebase: a bare numeric string (config-io, vault, lockfile — never
|
|
287
|
+
* carries a `launcherPid`) and a JSON object with `pid`/`launcherPid` fields
|
|
288
|
+
* (`createLockPayload`). `holderPid` is undefined when the body is
|
|
289
|
+
* unparseable or yields a non-positive integer; `launcherPid` is undefined
|
|
290
|
+
* whenever the payload has none, independent of whether `holderPid` parsed.
|
|
261
291
|
*/
|
|
262
|
-
function
|
|
292
|
+
function extractLockIdentity(content) {
|
|
263
293
|
const trimmed = content.trim();
|
|
264
294
|
if (!trimmed)
|
|
265
|
-
return
|
|
295
|
+
return {};
|
|
266
296
|
if (trimmed.startsWith("{")) {
|
|
267
297
|
try {
|
|
268
298
|
const parsed = JSON.parse(trimmed);
|
|
269
299
|
const pid = typeof parsed.pid === "number" ? parsed.pid : Number.NaN;
|
|
270
|
-
|
|
300
|
+
const rawLauncherPid = typeof parsed.launcherPid === "number" ? parsed.launcherPid : Number.NaN;
|
|
301
|
+
return {
|
|
302
|
+
holderPid: Number.isInteger(pid) && pid > 0 ? pid : undefined,
|
|
303
|
+
launcherPid: Number.isInteger(rawLauncherPid) && rawLauncherPid > 0 ? rawLauncherPid : undefined,
|
|
304
|
+
};
|
|
271
305
|
}
|
|
272
306
|
catch {
|
|
273
|
-
return
|
|
307
|
+
return {};
|
|
274
308
|
}
|
|
275
309
|
}
|
|
276
310
|
const pid = Number.parseInt(trimmed, 10);
|
|
277
|
-
return Number.isInteger(pid) && pid > 0 ? pid : undefined;
|
|
311
|
+
return { holderPid: Number.isInteger(pid) && pid > 0 ? pid : undefined };
|
|
278
312
|
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Parent-death watchdog (#956).
|
|
6
|
+
*
|
|
7
|
+
* The published launcher (`scripts/node-runtime/akm`) now forwards SIGTERM to
|
|
8
|
+
* its child, so a `kill <launcher-pid>` no longer orphans it — but a launcher
|
|
9
|
+
* that dies WITHOUT delivering a signal (SIGKILL, an out-of-memory kill, a
|
|
10
|
+
* supervisor that force-removes the process) still reparents the child to
|
|
11
|
+
* init with nothing to catch. Field evidence: 40 orphaned
|
|
12
|
+
* `bun …/dist/cli.js` processes over one day, up to 10h old, some created by
|
|
13
|
+
* a curate hook killing its own launcher on timeout — every akm invocation
|
|
14
|
+
* under the launcher is exposed to this, not only `akm index`, so this
|
|
15
|
+
* watchdog runs for every command (wired at the CLI entry point in
|
|
16
|
+
* `src/cli.ts`), not just index.
|
|
17
|
+
*
|
|
18
|
+
* `process.ppid` on POSIX changes the instant the parent exits (the child is
|
|
19
|
+
* reparented, usually to pid 1) — polling it is the standard orphan-detection
|
|
20
|
+
* trick when no direct death notification exists. This module owns only the
|
|
21
|
+
* poll/compare mechanics; the caller's `onOrphaned` decides what "stop"
|
|
22
|
+
* means. `src/cli.ts` wires it to `process.kill(process.pid, "SIGTERM")`,
|
|
23
|
+
* reusing the same self-signal every command already has rather than a
|
|
24
|
+
* second, parallel abort path. Only `akm index`/`improve` register their own
|
|
25
|
+
* SIGTERM listener (the index command's AbortController in
|
|
26
|
+
* `commands/sources/stash-cli.ts`) and get a graceful, in-process shutdown
|
|
27
|
+
* from it. Every other command has no listener of its own, so the runtime's
|
|
28
|
+
* default disposition terminates it directly — `exit` handlers (lock
|
|
29
|
+
* release included) do NOT run — and any lock it held is cleared later by
|
|
30
|
+
* the next acquirer's dead-pid stale-reclaim (`file-lock.ts`), not by
|
|
31
|
+
* in-process release. Either way the orphaned process stops, which is this
|
|
32
|
+
* watchdog's actual job.
|
|
33
|
+
*/
|
|
34
|
+
/** Pure comparison seam: true once the observed ppid differs from the one seen at startup. */
|
|
35
|
+
export function isReparented(initialPpid, currentPpid) {
|
|
36
|
+
return currentPpid !== initialPpid;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Start polling `getPpid()` every `intervalMs` and invoke `onOrphaned` once,
|
|
40
|
+
* the first time the observed ppid no longer matches `initialPpid`. The timer
|
|
41
|
+
* is unref'ed so it never keeps the process alive on its own — a command that
|
|
42
|
+
* finishes normally still exits promptly.
|
|
43
|
+
*/
|
|
44
|
+
export function startParentDeathWatchdog(options) {
|
|
45
|
+
const { initialPpid, onOrphaned } = options;
|
|
46
|
+
const intervalMs = options.intervalMs ?? 2000;
|
|
47
|
+
const getPpid = options.getPpid ?? (() => process.ppid);
|
|
48
|
+
const setIntervalFn = options.setIntervalFn ?? setInterval;
|
|
49
|
+
const clearIntervalFn = options.clearIntervalFn ?? clearInterval;
|
|
50
|
+
let fired = false;
|
|
51
|
+
const timer = setIntervalFn(() => {
|
|
52
|
+
if (fired)
|
|
53
|
+
return;
|
|
54
|
+
if (isReparented(initialPpid, getPpid())) {
|
|
55
|
+
fired = true;
|
|
56
|
+
onOrphaned();
|
|
57
|
+
}
|
|
58
|
+
}, intervalMs);
|
|
59
|
+
if (typeof timer !== "number")
|
|
60
|
+
timer.unref?.();
|
|
61
|
+
return {
|
|
62
|
+
stop: () => clearIntervalFn(timer),
|
|
63
|
+
};
|
|
64
|
+
}
|
package/dist/core/run-lock.js
CHANGED
|
@@ -31,6 +31,17 @@ import path from "node:path";
|
|
|
31
31
|
import { ConfigError } from "./errors.js";
|
|
32
32
|
import { createLockPayload, probeLock, reclaimStaleLock, tryAcquireLockSync } from "./file-lock.js";
|
|
33
33
|
import { describeInaccessiblePath } from "./path-access.js";
|
|
34
|
+
/**
|
|
35
|
+
* Render a lock holder's pid for a message: `"4242"`, or `"4242 (launcher
|
|
36
|
+
* 4240)"` when the holder's launcher pid is known (#956) — every process
|
|
37
|
+
* listing and task log shows the launcher pid, not the bun/node child's, so
|
|
38
|
+
* naming only the holder pid left an operator unable to connect the two.
|
|
39
|
+
*/
|
|
40
|
+
export function formatLockHolderPid(holder) {
|
|
41
|
+
if (holder.pid === null)
|
|
42
|
+
return "unknown";
|
|
43
|
+
return holder.launcherPid !== null ? `${holder.pid} (launcher ${holder.launcherPid})` : String(holder.pid);
|
|
44
|
+
}
|
|
34
45
|
function parseLockPayload(rawContent) {
|
|
35
46
|
if (!rawContent)
|
|
36
47
|
return null;
|
|
@@ -42,7 +53,7 @@ function parseLockPayload(rawContent) {
|
|
|
42
53
|
}
|
|
43
54
|
}
|
|
44
55
|
function holderOf(lock) {
|
|
45
|
-
return { pid: lock?.pid ?? null, startedAt: lock?.startedAt ?? null };
|
|
56
|
+
return { pid: lock?.pid ?? null, startedAt: lock?.startedAt ?? null, launcherPid: lock?.launcherPid ?? null };
|
|
46
57
|
}
|
|
47
58
|
/**
|
|
48
59
|
* Attempt to acquire `lockPath`. Returns `{ state: "acquired" }` with an
|
|
@@ -69,7 +80,7 @@ export function tryAcquireRunLock(lockPath, options) {
|
|
|
69
80
|
return { state: "acquired", ownership };
|
|
70
81
|
// Re-grabbed by another racer in this exact window — no holder detail
|
|
71
82
|
// available without re-probing (which could itself race again).
|
|
72
|
-
return { state: "held", holder: { pid: null, startedAt: null } };
|
|
83
|
+
return { state: "held", holder: { pid: null, startedAt: null, launcherPid: null } };
|
|
73
84
|
}
|
|
74
85
|
if (probe.state === "inaccessible") {
|
|
75
86
|
throw new ConfigError(`${options.label} lock exists but is not readable: ${describeInaccessiblePath(lockPath, probe.code)}.`, "DATA_DIR_UNREADABLE");
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
import { releaseLock } from "../core/file-lock.js";
|
|
23
23
|
import { tryWithMaintenanceStartBarrier, withMaintenanceStartBarrier } from "../core/maintenance-barrier.js";
|
|
24
24
|
import { getIndexRebuildLockPath } from "../core/paths.js";
|
|
25
|
-
import { tryAcquireRunLock } from "../core/run-lock.js";
|
|
25
|
+
import { formatLockHolderPid, tryAcquireRunLock } from "../core/run-lock.js";
|
|
26
26
|
import { warn, warnVerbose } from "../core/warn.js";
|
|
27
27
|
export function indexRebuildLockPath() {
|
|
28
28
|
return getIndexRebuildLockPath();
|
|
@@ -53,18 +53,18 @@ export function tryAcquireIndexRebuildLock(skipIfLocked) {
|
|
|
53
53
|
const result = tryWithMaintenanceStartBarrier(acquire);
|
|
54
54
|
if (!result) {
|
|
55
55
|
warn("[index] maintenance barrier held; skipping (--skip-if-locked)");
|
|
56
|
-
return { state: "skipped", holder: { pid: null, startedAt: null } };
|
|
56
|
+
return { state: "skipped", holder: { pid: null, startedAt: null, launcherPid: null } };
|
|
57
57
|
}
|
|
58
58
|
if (result.state === "acquired")
|
|
59
59
|
return result;
|
|
60
|
-
warn(`[index] another index run holds the lock (PID ${result.holder
|
|
60
|
+
warn(`[index] another index run holds the lock (PID ${formatLockHolderPid(result.holder)}, started ${result.holder.startedAt}); ` +
|
|
61
61
|
"skipping (--skip-if-locked)");
|
|
62
62
|
return { state: "skipped", holder: result.holder };
|
|
63
63
|
}
|
|
64
64
|
const result = withMaintenanceStartBarrier(acquire);
|
|
65
65
|
if (result.state === "acquired")
|
|
66
66
|
return result;
|
|
67
|
-
warn(`[index] another index run is active (pid ${result.holder
|
|
67
|
+
warn(`[index] another index run is active (pid ${formatLockHolderPid(result.holder)}, started ${result.holder.startedAt}); ` +
|
|
68
68
|
"this run will contend with it — pass --skip-if-locked for scheduled runs");
|
|
69
69
|
return { state: "contended", holder: result.holder };
|
|
70
70
|
}
|
|
@@ -27,6 +27,7 @@ import { isDataDirUnreadableError } from "../core/errors.js";
|
|
|
27
27
|
import { probeLock } from "../core/file-lock.js";
|
|
28
28
|
import { isPathAbsent } from "../core/path-access.js";
|
|
29
29
|
import { getDbPath, getIndexRebuildLockPath } from "../core/paths.js";
|
|
30
|
+
import { formatLockHolderPid } from "../core/run-lock.js";
|
|
30
31
|
import { warn, warnVerbose } from "../core/warn.js";
|
|
31
32
|
import { closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
32
33
|
import { deleteEntriesByIds, getEntryCount, upsertEntry } from "../storage/repositories/index-entries-repository.js";
|
|
@@ -75,7 +76,14 @@ export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
|
75
76
|
// rebuild in progress will pick up the change on its own.
|
|
76
77
|
const rebuildProbe = probeLock(getIndexRebuildLockPath());
|
|
77
78
|
if (rebuildProbe.state === "held") {
|
|
78
|
-
|
|
79
|
+
// #956: name the launcher pid alongside the holder pid when known —
|
|
80
|
+
// every process listing and task log shows the launcher's pid, not
|
|
81
|
+
// the bun/node child's.
|
|
82
|
+
const holderLabel = formatLockHolderPid({
|
|
83
|
+
pid: rebuildProbe.holderPid,
|
|
84
|
+
launcherPid: rebuildProbe.launcherPid ?? null,
|
|
85
|
+
});
|
|
86
|
+
warn(`index rebuild in progress (pid ${holderLabel}); the next index pass will index ${filePaths.join(", ")}`);
|
|
79
87
|
return true;
|
|
80
88
|
}
|
|
81
89
|
const dbPath = getDbPath();
|