akm-cli 0.9.14 → 0.9.15-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +397 -0
- package/STABILITY.md +6 -3
- package/dist/assets/prompts/reflect-feedback-framing.md +1 -0
- package/dist/assets/prompts/reflect-llm-framed-contract.md +2 -0
- package/dist/assets/prompts/reflect-llm-schema-contract.md +2 -0
- package/dist/assets/tasks/core/improve.yml +1 -1
- package/dist/assets/tasks/core/index-refresh.yml +1 -1
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-consolidate.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-frequent.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-nightly.yml +1 -1
- package/dist/cli/retired-commands.js +0 -1
- package/dist/cli/shared.js +9 -0
- package/dist/cli/unknown-flags.js +1 -0
- package/dist/cli.js +3 -2
- package/dist/commands/config-cli.js +85 -3
- package/dist/commands/env/env-cli.js +1 -42
- package/dist/commands/env/env.js +1 -1
- package/dist/commands/env/secret-cli.js +1 -2
- package/dist/commands/health/checks.js +357 -63
- package/dist/commands/health/engine-usage.js +45 -0
- package/dist/commands/health/improve-metrics.js +18 -0
- package/dist/commands/health/llm-usage.js +41 -1
- package/dist/commands/health/plugin-staleness.js +7 -3
- package/dist/commands/health/version-drift.js +93 -0
- package/dist/commands/health/windows.js +3 -1
- package/dist/commands/health.js +44 -9
- package/dist/commands/improve/consolidate/chunking.js +4 -2
- package/dist/commands/improve/improve-cli.js +99 -5
- package/dist/commands/improve/improve-report.js +154 -0
- package/dist/commands/improve/improve-result-file.js +45 -33
- package/dist/commands/improve/improve-strategies.js +133 -3
- package/dist/commands/improve/improve-usage-report.js +182 -0
- package/dist/commands/improve/improve.js +40 -3
- package/dist/commands/improve/locks.js +27 -78
- package/dist/commands/improve/planner.js +1 -0
- package/dist/commands/improve/preparation.js +9 -1
- package/dist/commands/improve/reflect.js +44 -4
- package/dist/commands/models-cli.js +50 -1
- package/dist/commands/proposal/repository.js +8 -3
- package/dist/commands/proposal/validators/proposal-quality-validators.js +41 -6
- package/dist/commands/proposal/validators/proposal-validators.js +24 -0
- package/dist/commands/read/search-cli.js +38 -2
- package/dist/commands/read/show.js +103 -4
- package/dist/commands/sources/info.js +5 -1
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/stash-cli.js +31 -0
- package/dist/commands/tasks/tasks-cli.js +49 -2
- package/dist/commands/workflow-cli.js +86 -12
- package/dist/core/asset/markdown-fragments.js +35 -0
- package/dist/core/config/config-schema.js +14 -0
- package/dist/core/config/config.js +302 -24
- package/dist/core/env-secret-ref.js +58 -5
- package/dist/core/errors.js +30 -0
- package/dist/core/improve-result.js +51 -0
- package/dist/core/loopback.js +17 -0
- package/dist/core/paths.js +11 -0
- package/dist/core/run-lock.js +96 -0
- package/dist/core/sensitive-marker-path.js +19 -0
- package/dist/core/state-db.js +74 -14
- package/dist/indexer/index-rebuild-lock.js +73 -0
- package/dist/indexer/index-writer-lock.js +40 -1
- package/dist/indexer/index-written-assets.js +21 -1
- package/dist/indexer/indexer.js +18 -17
- package/dist/indexer/materialize-embeddings.js +282 -32
- package/dist/indexer/search/db-search.js +49 -2
- package/dist/integrations/agent/engine-resolution.js +96 -6
- package/dist/integrations/agent/execution-definitions.js +6 -15
- package/dist/integrations/agent/execution-lowering.js +6 -1
- package/dist/integrations/agent/execution-preparation.js +1 -1
- package/dist/integrations/agent/model-map.js +123 -20
- package/dist/integrations/agent/prompts.js +40 -8
- package/dist/integrations/agent/runner-dispatch.js +9 -3
- package/dist/integrations/agent/runner.js +2 -0
- package/dist/llm/client.js +8 -3
- package/dist/llm/embedder.js +20 -8
- package/dist/llm/embedders/local.js +10 -2
- package/dist/llm/embedders/remote.js +188 -21
- package/dist/output/shapes/helpers.js +38 -2
- package/dist/output/shapes/models-list.js +16 -0
- package/dist/output/shapes/passthrough.js +2 -0
- package/dist/output/shapes.js +4 -0
- package/dist/output/text/command-format.js +29 -0
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/improve-report.js +27 -0
- package/dist/{commands/env/marker-path.js → output/text/models.js} +4 -3
- package/dist/output/text/show-format.js +4 -0
- package/dist/output/text.js +4 -0
- package/dist/scripts/akm-migrate-node.js +24798 -21732
- package/dist/scripts/akm-migrate.js +23408 -20343
- package/dist/storage/repositories/improve-runs-repository.js +34 -0
- package/dist/storage/repositories/index-fts-repository.js +49 -6
- package/dist/storage/repositories/index-vec-repository.js +30 -0
- package/dist/storage/repositories/workflow-runs-repository.js +55 -18
- package/dist/tasks/backends/cron.js +14 -7
- package/dist/tasks/run/run-workflow-task.js +16 -0
- package/dist/workflows/exec/child-workflow.js +2 -2
- package/dist/workflows/exec/dispatch-redaction.js +21 -9
- package/dist/workflows/exec/run-workflow.js +6 -5
- package/dist/workflows/runtime/runs.js +33 -5
- package/docs/migration/release-notes/0.9.15.md +52 -0
- package/docs/migration/release-notes/README.md +4 -0
- package/docs/reference/cli.md +245 -29
- package/docs/reference/configuration.md +180 -19
- package/docs/reference/data-and-telemetry.md +8 -0
- package/docs/reference/tasks.md +16 -1
- package/docs/reference/workflow-schema.md +5 -1
- package/package.json +1 -1
- package/schemas/akm-config.json +8 -0
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import { cloneExecutionJsonObject } from "../execution/json.js";
|
|
5
5
|
import { isRecord } from "./common.js";
|
|
6
|
+
import { IMPROVE_PROCESS_ENGINE_CAPABILITIES } from "./config/engine-semantics.js";
|
|
6
7
|
const COMMON_FIELDS = [
|
|
7
8
|
"schemaVersion",
|
|
8
9
|
"ok",
|
|
@@ -17,6 +18,7 @@ const COMMON_FIELDS = [
|
|
|
17
18
|
"plannedRefs",
|
|
18
19
|
"plan",
|
|
19
20
|
"actions",
|
|
21
|
+
"skippedProcesses",
|
|
20
22
|
"distillSkipped",
|
|
21
23
|
"validationFailures",
|
|
22
24
|
"schemaRepairs",
|
|
@@ -47,6 +49,7 @@ const COMMON_FIELDS = [
|
|
|
47
49
|
"sync",
|
|
48
50
|
"writtenPaths",
|
|
49
51
|
"terminated",
|
|
52
|
+
"usageReport",
|
|
50
53
|
];
|
|
51
54
|
const V2_FIELDS = new Set([...COMMON_FIELDS, "strategy", "strategyFilteredRefs"]);
|
|
52
55
|
function fail(message) {
|
|
@@ -179,6 +182,50 @@ function validateProactivePlan(value) {
|
|
|
179
182
|
fail("plan.proactive.selected must equal plan.proactive.selectedRefs.length");
|
|
180
183
|
}
|
|
181
184
|
}
|
|
185
|
+
/** #947 — plan.processes: one row per IMPROVE_PROCESS_ENGINE_CAPABILITIES name, plus an optional "triage.judgment" row. */
|
|
186
|
+
function validateProcessRoutingRows(value) {
|
|
187
|
+
if (!Array.isArray(value))
|
|
188
|
+
fail("plan.processes must be an array");
|
|
189
|
+
const canonicalNames = Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES);
|
|
190
|
+
const engineKinds = new Set(["llm", "agent", "sdk"]);
|
|
191
|
+
const seen = new Set();
|
|
192
|
+
for (const row of value) {
|
|
193
|
+
if (!isRecord(row))
|
|
194
|
+
fail("plan.processes entries must be objects");
|
|
195
|
+
requireExactFields(row, new Set(["process", "enabled", "engine", "model", "engineKind", "notices", "unavailable", "eligibleRefs"]));
|
|
196
|
+
if (typeof row.process !== "string" ||
|
|
197
|
+
!(canonicalNames.includes(row.process) || row.process === "triage.judgment")) {
|
|
198
|
+
fail("plan.processes.process is invalid");
|
|
199
|
+
}
|
|
200
|
+
if (seen.has(row.process))
|
|
201
|
+
fail(`plan.processes must not repeat "${row.process}"`);
|
|
202
|
+
seen.add(row.process);
|
|
203
|
+
if (typeof row.enabled !== "boolean")
|
|
204
|
+
fail("plan.processes.enabled must be a boolean");
|
|
205
|
+
if (row.engine !== undefined && typeof row.engine !== "string")
|
|
206
|
+
fail("plan.processes.engine must be a string");
|
|
207
|
+
if (row.model !== undefined && typeof row.model !== "string")
|
|
208
|
+
fail("plan.processes.model must be a string");
|
|
209
|
+
if (row.engineKind !== undefined && (typeof row.engineKind !== "string" || !engineKinds.has(row.engineKind))) {
|
|
210
|
+
fail("plan.processes.engineKind is invalid");
|
|
211
|
+
}
|
|
212
|
+
validateLoweringNotices(row.notices);
|
|
213
|
+
if (row.unavailable !== undefined) {
|
|
214
|
+
if (!isRecord(row.unavailable))
|
|
215
|
+
fail("plan.processes.unavailable must be an object");
|
|
216
|
+
requireExactFields(row.unavailable, new Set(["configKey", "reason"]));
|
|
217
|
+
if (typeof row.unavailable.configKey !== "string" || typeof row.unavailable.reason !== "string") {
|
|
218
|
+
fail("plan.processes.unavailable must contain string configKey and reason");
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
if (row.eligibleRefs !== undefined)
|
|
222
|
+
requireCount(row, "eligibleRefs", "plan.processes entry");
|
|
223
|
+
}
|
|
224
|
+
for (const name of canonicalNames) {
|
|
225
|
+
if (!seen.has(name))
|
|
226
|
+
fail(`plan.processes must contain exactly one "${name}" row`);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
182
229
|
function validateImprovePlan(value, dryRun, plannedRefNames) {
|
|
183
230
|
if (!isRecord(value))
|
|
184
231
|
fail("plan must be an object");
|
|
@@ -190,6 +237,7 @@ function validateImprovePlan(value, dryRun, plannedRefNames) {
|
|
|
190
237
|
"limits",
|
|
191
238
|
"gates",
|
|
192
239
|
"effectiveRefs",
|
|
240
|
+
"processes",
|
|
193
241
|
"proactive",
|
|
194
242
|
"consolidation",
|
|
195
243
|
"stages",
|
|
@@ -328,6 +376,7 @@ function validateImprovePlan(value, dryRun, plannedRefNames) {
|
|
|
328
376
|
if (value.limits.totalCeiling !== undefined && value.effectiveRefs.length > value.limits.totalCeiling) {
|
|
329
377
|
fail("plan.effectiveRefs cannot exceed plan.limits.totalCeiling");
|
|
330
378
|
}
|
|
379
|
+
validateProcessRoutingRows(value.processes);
|
|
331
380
|
if (value.proactive !== undefined)
|
|
332
381
|
validateProactivePlan(value.proactive);
|
|
333
382
|
validateConsolidationPlan(value.consolidation);
|
|
@@ -392,6 +441,7 @@ function validateCommon(value) {
|
|
|
392
441
|
}
|
|
393
442
|
for (const field of [
|
|
394
443
|
"actions",
|
|
444
|
+
"skippedProcesses",
|
|
395
445
|
"validationFailures",
|
|
396
446
|
"schemaRepairs",
|
|
397
447
|
"extract",
|
|
@@ -442,6 +492,7 @@ function validateCommon(value) {
|
|
|
442
492
|
"terminated",
|
|
443
493
|
"plan",
|
|
444
494
|
"deadUrlCoverage",
|
|
495
|
+
"usageReport",
|
|
445
496
|
]) {
|
|
446
497
|
if (value[field] !== undefined && !isRecord(value[field]))
|
|
447
498
|
fail(`${field} must be an object`);
|
package/dist/core/loopback.js
CHANGED
|
@@ -87,3 +87,20 @@ export function isLoopbackEndpoint(endpoint) {
|
|
|
87
87
|
return true;
|
|
88
88
|
}
|
|
89
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* The lowest-common-denominator concurrency default for an LLM/embedding
|
|
92
|
+
* endpoint: 1 for a loopback endpoint (a local model server serves one
|
|
93
|
+
* inference at a time; concurrent requests thrash it — reload thrash, HTTP
|
|
94
|
+
* 500 "Model reloaded"), 2 for a remote one (enough to overlap request
|
|
95
|
+
* latency without hammering a rate-limited API). A leaf helper so both
|
|
96
|
+
* callers — the indexer's LLM enrichment pool (`getDefaultLlmConcurrency`,
|
|
97
|
+
* `src/indexer/indexer.ts`) and the embedding pool
|
|
98
|
+
* (`resolveEmbeddingConcurrency`, `src/llm/embedders/remote.ts`) — share one
|
|
99
|
+
* definition instead of mirroring it; `src/llm/embedders/remote.ts` cannot
|
|
100
|
+
* import `getDefaultLlmConcurrency` directly (`src/indexer/indexer.ts`
|
|
101
|
+
* already depends on this module transitively through
|
|
102
|
+
* materialize-embeddings.ts).
|
|
103
|
+
*/
|
|
104
|
+
export function defaultConcurrencyForEndpoint(endpoint) {
|
|
105
|
+
return isLoopbackEndpoint(endpoint) ? 1 : 2;
|
|
106
|
+
}
|
package/dist/core/paths.js
CHANGED
|
@@ -234,6 +234,17 @@ export function getDbPath(env = process.env) {
|
|
|
234
234
|
export function getIndexWriterLockPath() {
|
|
235
235
|
return path.join(getDataDir(), "index.db.write.lock");
|
|
236
236
|
}
|
|
237
|
+
/**
|
|
238
|
+
* Path to the opt-in, PID-liveness-only rebuild lock an explicit `akm index`
|
|
239
|
+
* run acquires (#956). Distinct from {@link getIndexWriterLockPath}'s
|
|
240
|
+
* `index.db.write.lock`, which is the asset-mutation lease and unrelated to
|
|
241
|
+
* indexing since #872 — this lock never blocks and is never required, it
|
|
242
|
+
* only lets a scheduled/opportunistic `akm index --skip-if-locked` step
|
|
243
|
+
* aside instead of contending with a run already in progress.
|
|
244
|
+
*/
|
|
245
|
+
export function getIndexRebuildLockPath() {
|
|
246
|
+
return path.join(getDataDir(), "index.rebuild.lock");
|
|
247
|
+
}
|
|
237
248
|
export function getMaintenanceBarrierPath() {
|
|
238
249
|
return path.join(getDataDir(), "maintenance.barrier.lock");
|
|
239
250
|
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* PID-liveness-only run lock — the shared mechanics behind `akm improve`'s
|
|
6
|
+
* whole-run lock (`commands/improve/locks.ts`) and the index rebuild lock
|
|
7
|
+
* (`indexer/index-rebuild-lock.ts`, #956).
|
|
8
|
+
*
|
|
9
|
+
* No `staleAfterMs`: only a verifiably dead holder is ever reclaimed. This is
|
|
10
|
+
* the #872 lesson encoded at the mechanics layer so every future caller gets
|
|
11
|
+
* it for free — an age-based stale window let a live-but-wedged holder's
|
|
12
|
+
* lease survive forever from the holder's own point of view while stranding
|
|
13
|
+
* every OTHER invocation once the clock passed, which cost one real install
|
|
14
|
+
* a half-day indexing outage. SQLite's own WAL + busy_timeout + BEGIN
|
|
15
|
+
* IMMEDIATE already serialize concurrent writers at the correctness layer —
|
|
16
|
+
* this lock only ever avoids duplicate LOGICAL work or lets a caller choose
|
|
17
|
+
* to skip instead of contend.
|
|
18
|
+
*
|
|
19
|
+
* This module performs exactly one create-or-probe attempt (with the same
|
|
20
|
+
* absent-race and stale-reclaim retries `improve` established) and reports
|
|
21
|
+
* whether the caller now owns the lock or, if not, who currently holds it.
|
|
22
|
+
* It does not decide what "held" means — skip, throw, or warn-and-proceed is
|
|
23
|
+
* entirely up to the caller — and it does not serialize the attempt itself:
|
|
24
|
+
* every caller must wrap the call in
|
|
25
|
+
* `withMaintenanceStartBarrier`/`tryWithMaintenanceStartBarrier`
|
|
26
|
+
* (`core/maintenance-barrier.ts`) so two racing processes never both create
|
|
27
|
+
* the sentinel in the same window.
|
|
28
|
+
*/
|
|
29
|
+
import fs from "node:fs";
|
|
30
|
+
import path from "node:path";
|
|
31
|
+
import { ConfigError } from "./errors.js";
|
|
32
|
+
import { createLockPayload, probeLock, reclaimStaleLock, tryAcquireLockSync } from "./file-lock.js";
|
|
33
|
+
import { describeInaccessiblePath } from "./path-access.js";
|
|
34
|
+
function parseLockPayload(rawContent) {
|
|
35
|
+
if (!rawContent)
|
|
36
|
+
return null;
|
|
37
|
+
try {
|
|
38
|
+
return JSON.parse(rawContent);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
function holderOf(lock) {
|
|
45
|
+
return { pid: lock?.pid ?? null, startedAt: lock?.startedAt ?? null };
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Attempt to acquire `lockPath`. Returns `{ state: "acquired" }` with an
|
|
49
|
+
* ownership handle for {@link releaseLock}, or `{ state: "held", holder }`
|
|
50
|
+
* naming the current holder (best-effort; `pid`/`startedAt` are `null` when
|
|
51
|
+
* the holder identity could not be determined, e.g. a release-then-reacquire
|
|
52
|
+
* race). Throws {@link ConfigError} if the sentinel exists but cannot be read
|
|
53
|
+
* (#791: a lock we cannot see may be genuinely held — never a reclaim
|
|
54
|
+
* candidate).
|
|
55
|
+
*/
|
|
56
|
+
export function tryAcquireRunLock(lockPath, options) {
|
|
57
|
+
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
58
|
+
const lockPayload = () => createLockPayload({ startedAt: new Date().toISOString(), ...options.payloadMetadata });
|
|
59
|
+
let ownership = tryAcquireLockSync(lockPath, lockPayload());
|
|
60
|
+
if (ownership)
|
|
61
|
+
return { state: "acquired", ownership };
|
|
62
|
+
const probe = probeLock(lockPath);
|
|
63
|
+
// Race: the holder released between the failed acquire above and this
|
|
64
|
+
// probe. Retry once rather than falling through with a null-PID "held"
|
|
65
|
+
// report for a lock nobody actually holds.
|
|
66
|
+
if (probe.state === "absent") {
|
|
67
|
+
ownership = tryAcquireLockSync(lockPath, lockPayload());
|
|
68
|
+
if (ownership)
|
|
69
|
+
return { state: "acquired", ownership };
|
|
70
|
+
// Re-grabbed by another racer in this exact window — no holder detail
|
|
71
|
+
// available without re-probing (which could itself race again).
|
|
72
|
+
return { state: "held", holder: { pid: null, startedAt: null } };
|
|
73
|
+
}
|
|
74
|
+
if (probe.state === "inaccessible") {
|
|
75
|
+
throw new ConfigError(`${options.label} lock exists but is not readable: ${describeInaccessiblePath(lockPath, probe.code)}.`, "DATA_DIR_UNREADABLE");
|
|
76
|
+
}
|
|
77
|
+
const lock = parseLockPayload(probe.rawContent);
|
|
78
|
+
if (probe.state === "stale") {
|
|
79
|
+
if (!reclaimStaleLock(lockPath, probe)) {
|
|
80
|
+
return { state: "held", holder: holderOf(lock) };
|
|
81
|
+
}
|
|
82
|
+
options.onReclaimed?.({
|
|
83
|
+
holderPid: lock?.pid ?? probe.holderPid ?? null,
|
|
84
|
+
lockedAt: lock?.startedAt ?? null,
|
|
85
|
+
ageMs: probe.ageMs ?? null,
|
|
86
|
+
reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
|
|
87
|
+
});
|
|
88
|
+
ownership = tryAcquireLockSync(lockPath, lockPayload());
|
|
89
|
+
if (ownership)
|
|
90
|
+
return { state: "acquired", ownership };
|
|
91
|
+
// Acquired by another racer during stale recovery.
|
|
92
|
+
return { state: "held", holder: holderOf(lock) };
|
|
93
|
+
}
|
|
94
|
+
// probe.state === "held"
|
|
95
|
+
return { state: "held", holder: holderOf(lock) };
|
|
96
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Path to the sibling marker file that suppresses listing for a sensitive
|
|
6
|
+
* env/secret asset. Deliberately its own leaf module (no imports) rather than
|
|
7
|
+
* living on `core/env-secret-ref.ts` (#950 originally put it there): that
|
|
8
|
+
* module transitively imports the source providers via
|
|
9
|
+
* `indexer/search/search-source`, and `commands/env/env.ts` is imported back
|
|
10
|
+
* from `core/adapter/adapters/akm-metadata.ts` (for `scanEnvKeyNames`) — so an
|
|
11
|
+
* `env.ts` import of the heavier module closes a real cycle through the
|
|
12
|
+
* adapter registry (`core/adapter/adapters/index.ts`'s `BUILTIN_ADAPTERS`
|
|
13
|
+
* construction sees `akmAdapter` still TDZ). Keeping this helper leaf-only
|
|
14
|
+
* lets `commands/env/env.ts` and `core/env-secret-ref.ts` both depend on it
|
|
15
|
+
* without either pulling the other's heavier graph.
|
|
16
|
+
*/
|
|
17
|
+
export function sensitiveMarkerPath(assetPath, type) {
|
|
18
|
+
return type === "env" ? assetPath.replace(/\.env$/, ".sensitive") : `${assetPath}.sensitive`;
|
|
19
|
+
}
|
package/dist/core/state-db.js
CHANGED
|
@@ -60,6 +60,22 @@
|
|
|
60
60
|
* of events.jsonl is replaced by WAL-mode serialised writes — acceptable because
|
|
61
61
|
* CLI commands are almost always single-writer.
|
|
62
62
|
*
|
|
63
|
+
* ## Writer contention (#948)
|
|
64
|
+
*
|
|
65
|
+
* Every open already carries a 30s `busy_timeout` (`sqlite-pragmas.ts`), so a
|
|
66
|
+
* single blocked statement waits before failing. `withImmediateTransaction`
|
|
67
|
+
* additionally retries `BEGIN IMMEDIATE` itself
|
|
68
|
+
* ({@link WITH_IMMEDIATE_TX_MAX_ATTEMPTS} attempts) for the rarer case of two
|
|
69
|
+
* writers racing the BEGIN statement back-to-back. If every attempt is still
|
|
70
|
+
* contention-shaped ({@link isSqliteContentionError} — SQLITE_BUSY/LOCKED, the
|
|
71
|
+
* matching message text, or the phantom-BEGIN marker), the exhaustion throw is
|
|
72
|
+
* reclassified into `TransientError("STATE_DB_CONTENDED")` (exit 75, #948
|
|
73
|
+
* addendum) instead of surfacing the raw driver text: another akm process (an
|
|
74
|
+
* unrelated `improve`, `workflow run`, or task run — not necessarily
|
|
75
|
+
* contending for the same row) is writing state.db right now. A genuinely
|
|
76
|
+
* unrelated error (real corruption, a body-thrown failure) is never
|
|
77
|
+
* reclassified and rethrows exactly as raised.
|
|
78
|
+
*
|
|
63
79
|
* @module state-db
|
|
64
80
|
*/
|
|
65
81
|
import { randomUUID } from "node:crypto";
|
|
@@ -70,6 +86,7 @@ import { openDatabase } from "../storage/database.js";
|
|
|
70
86
|
import { assertMigrationLedger } from "../storage/engines/sqlite-migrations.js";
|
|
71
87
|
import { openManagedDatabase, withManagedDb } from "../storage/managed-db.js";
|
|
72
88
|
import { pkgVersion } from "../version.js";
|
|
89
|
+
import { TransientError } from "./errors.js";
|
|
73
90
|
import { acquireMaintenanceActivitySync } from "./maintenance-barrier.js";
|
|
74
91
|
import { getDataDir } from "./paths.js";
|
|
75
92
|
import { runMigrations, STATE_MIGRATIONS } from "./state/migrations.js";
|
|
@@ -621,20 +638,30 @@ export function withStateDbTelemetry(fn, busyTimeoutMs = 250) {
|
|
|
621
638
|
* the live queue state rather than clobbering each other.
|
|
622
639
|
*/
|
|
623
640
|
/**
|
|
624
|
-
*
|
|
625
|
-
* transient
|
|
626
|
-
* retry
|
|
627
|
-
*
|
|
628
|
-
*
|
|
629
|
-
*
|
|
641
|
+
* Whether `err` is one of the SQLite conditions a concurrent-writer race can
|
|
642
|
+
* throw that are transient — the statement did NOT corrupt anything, another
|
|
643
|
+
* writer just holds the lock right now — and therefore safe to retry or
|
|
644
|
+
* reclassify as ordinary contention rather than a genuine failure:
|
|
645
|
+
* - `SQLITE_BUSY` / `SQLITE_LOCKED` (either driver's `.code`).
|
|
646
|
+
* - "database is locked" / "database table is locked" message text.
|
|
647
|
+
* - the phantom-BEGIN marker synthesized below when `BEGIN IMMEDIATE`
|
|
648
|
+
* returns without actually opening a transaction.
|
|
630
649
|
*
|
|
631
|
-
*
|
|
632
|
-
*
|
|
633
|
-
*
|
|
634
|
-
*
|
|
635
|
-
*
|
|
650
|
+
* This is the single shared classifier for "is this ordinary state.db
|
|
651
|
+
* contention" (#948) — `isRetryableBeginError` below delegates to it, and so
|
|
652
|
+
* does {@link WorkflowRunsRepository}'s lease-contention classifier (which
|
|
653
|
+
* additionally matches a couple of corruption-shaped texts specific to its
|
|
654
|
+
* own narrower cross-process race and keeps its own live-lease confirmation
|
|
655
|
+
* before reclassifying). Matching this set alone is never sufficient to
|
|
656
|
+
* declare something DEFINITELY contention when a caller needs corroborating
|
|
657
|
+
* evidence (see the lease path); for `beginImmediateTransaction`'s own
|
|
658
|
+
* exhaustion case there is no such evidence available, so the five 30s
|
|
659
|
+
* `busy_timeout` waits already spent stand as the evidence instead.
|
|
636
660
|
*/
|
|
637
|
-
function
|
|
661
|
+
export function isSqliteContentionError(err) {
|
|
662
|
+
const code = err?.code;
|
|
663
|
+
if (code === "SQLITE_BUSY" || code === "SQLITE_LOCKED")
|
|
664
|
+
return true;
|
|
638
665
|
const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
639
666
|
return (msg.includes("database is locked") ||
|
|
640
667
|
msg.includes("database table is locked") ||
|
|
@@ -642,6 +669,20 @@ function isRetryableBeginError(err) {
|
|
|
642
669
|
// without opening a transaction. Safe to retry: fn() has not run.
|
|
643
670
|
msg.includes("did not open a transaction"));
|
|
644
671
|
}
|
|
672
|
+
/**
|
|
673
|
+
* Errors `BEGIN IMMEDIATE` can throw under concurrent-writer contention that
|
|
674
|
+
* are transient (the statement did NOT start a usable transaction) and safe
|
|
675
|
+
* to retry. An error thrown by `fn` is a real failure and is NEVER retried.
|
|
676
|
+
*
|
|
677
|
+
* "cannot start a transaction within a transaction" is deliberately NOT
|
|
678
|
+
* retryable: it means a transaction is already open on this connection (a
|
|
679
|
+
* re-entrant call — handled by the entry guard in withImmediateTransaction),
|
|
680
|
+
* and "retrying" it with a ROLLBACK would destroy the caller's transaction
|
|
681
|
+
* (issue #686).
|
|
682
|
+
*/
|
|
683
|
+
function isRetryableBeginError(err) {
|
|
684
|
+
return isSqliteContentionError(err);
|
|
685
|
+
}
|
|
645
686
|
const WITH_IMMEDIATE_TX_MAX_ATTEMPTS = 5;
|
|
646
687
|
/** Portable synchronous sleep (works under both Bun and Node). Delegates to the runtime boundary's `sleepSync`. */
|
|
647
688
|
function sleepSyncMs(ms) {
|
|
@@ -658,6 +699,25 @@ function sleepSyncMs(ms) {
|
|
|
658
699
|
* publication have all succeeded. The caller that asked for this split phase
|
|
659
700
|
* owns the matching COMMIT/ROLLBACK.
|
|
660
701
|
*/
|
|
702
|
+
/**
|
|
703
|
+
* Reclassify an exhausted-retry BEGIN failure that is still contention-shaped
|
|
704
|
+
* (#948) into a `TransientError("STATE_DB_CONTENDED")`, mirroring the
|
|
705
|
+
* RUN_LEASE_HELD precedent (`WorkflowRunsRepository.acquireEngineLease`): the
|
|
706
|
+
* driver text is accurate but unhelpful (`{"ok":false,"error":"database is
|
|
707
|
+
* locked"}`, exit 70/INTERNAL) — this instead reads as a retryable-shortly
|
|
708
|
+
* signal (exit 75, sysexits EX_TEMPFAIL — #948 addendum) with the original
|
|
709
|
+
* error preserved as `cause` for `--verbose`/debugging. A genuinely unrelated
|
|
710
|
+
* error (not contention-shaped) is rethrown exactly as raised, never
|
|
711
|
+
* reclassified.
|
|
712
|
+
*/
|
|
713
|
+
function throwBeginFailure(err) {
|
|
714
|
+
if (isSqliteContentionError(err)) {
|
|
715
|
+
const contended = new TransientError("akm's state database is busy (another akm process is writing it); retry shortly.", "STATE_DB_CONTENDED");
|
|
716
|
+
contended.cause = err;
|
|
717
|
+
throw contended;
|
|
718
|
+
}
|
|
719
|
+
throw err;
|
|
720
|
+
}
|
|
661
721
|
export function beginImmediateTransaction(db) {
|
|
662
722
|
if (db.inTransaction) {
|
|
663
723
|
throw new Error("beginImmediateTransaction requires a connection with no active transaction");
|
|
@@ -685,10 +745,10 @@ export function beginImmediateTransaction(db) {
|
|
|
685
745
|
sleepSyncMs(2 ** (attempt - 1));
|
|
686
746
|
continue;
|
|
687
747
|
}
|
|
688
|
-
|
|
748
|
+
throwBeginFailure(err);
|
|
689
749
|
}
|
|
690
750
|
}
|
|
691
|
-
|
|
751
|
+
throwBeginFailure(lastBeginErr);
|
|
692
752
|
}
|
|
693
753
|
export function withImmediateTransaction(db, fn) {
|
|
694
754
|
// Re-entrancy guard (issue #686): if a transaction is already open on this
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Opt-in, non-blocking rebuild lock for `akm index` (#956).
|
|
6
|
+
*
|
|
7
|
+
* #872 removed the blocking index-rebuild lease: the index is a regenerable
|
|
8
|
+
* cache, so a concurrent rebuild only wastes work rather than corrupts
|
|
9
|
+
* anything, and a live-but-wedged holder passed a PID-liveness check forever
|
|
10
|
+
* — only an age-based clock could ever free it, which is exactly the hazard
|
|
11
|
+
* #872 deleted. This module does not reinstate that lock. It adds a
|
|
12
|
+
* PID-liveness-only sentinel an explicit `akm index` run acquires and
|
|
13
|
+
* releases on exit purely so a *scheduled or opportunistic* run
|
|
14
|
+
* (`--skip-if-locked`) can step aside instead of piling up behind a rebuild
|
|
15
|
+
* already in progress. A human-typed `akm index` with no flag is never
|
|
16
|
+
* gated: it warns and proceeds exactly as it did before this lock existed.
|
|
17
|
+
*
|
|
18
|
+
* Built on the shared PID-liveness mechanics in `core/run-lock.ts` (the same
|
|
19
|
+
* ones `akm improve`'s whole-run lock uses) — see that module's doc for the
|
|
20
|
+
* no-stale-age-window rationale.
|
|
21
|
+
*/
|
|
22
|
+
import { releaseLock } from "../core/file-lock.js";
|
|
23
|
+
import { tryWithMaintenanceStartBarrier, withMaintenanceStartBarrier } from "../core/maintenance-barrier.js";
|
|
24
|
+
import { getIndexRebuildLockPath } from "../core/paths.js";
|
|
25
|
+
import { tryAcquireRunLock } from "../core/run-lock.js";
|
|
26
|
+
import { warn, warnVerbose } from "../core/warn.js";
|
|
27
|
+
export function indexRebuildLockPath() {
|
|
28
|
+
return getIndexRebuildLockPath();
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Acquire the rebuild lock for the duration of one `akm index` run.
|
|
32
|
+
*
|
|
33
|
+
* - Free: always returns `"acquired"`.
|
|
34
|
+
* - Held, `skipIfLocked`: warns once (naming the holder) and returns
|
|
35
|
+
* `"skipped"` — the caller must not run `akmIndex()` at all.
|
|
36
|
+
* - Held, no flag: warns once and returns `"contended"` — the caller runs
|
|
37
|
+
* `akmIndex()` unlocked, exactly as every `akm index` did before #956.
|
|
38
|
+
*
|
|
39
|
+
* A dead holder's lease is reclaimed silently (verbose-only log line, never
|
|
40
|
+
* a user-facing warning) — the operator did nothing wrong and nothing here
|
|
41
|
+
* requires their attention.
|
|
42
|
+
*/
|
|
43
|
+
export function tryAcquireIndexRebuildLock(skipIfLocked) {
|
|
44
|
+
const lockPath = indexRebuildLockPath();
|
|
45
|
+
const acquire = () => tryAcquireRunLock(lockPath, {
|
|
46
|
+
label: "index rebuild",
|
|
47
|
+
onReclaimed: (info) => {
|
|
48
|
+
warnVerbose(`[index] reclaimed a rebuild lock left by pid ${info.holderPid ?? "unknown"} ` +
|
|
49
|
+
`(${info.reason}); that process is no longer running.`);
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
if (skipIfLocked) {
|
|
53
|
+
const result = tryWithMaintenanceStartBarrier(acquire);
|
|
54
|
+
if (!result) {
|
|
55
|
+
warn("[index] maintenance barrier held; skipping (--skip-if-locked)");
|
|
56
|
+
return { state: "skipped", holder: { pid: null, startedAt: null } };
|
|
57
|
+
}
|
|
58
|
+
if (result.state === "acquired")
|
|
59
|
+
return result;
|
|
60
|
+
warn(`[index] another index run holds the lock (PID ${result.holder.pid}, started ${result.holder.startedAt}); ` +
|
|
61
|
+
"skipping (--skip-if-locked)");
|
|
62
|
+
return { state: "skipped", holder: result.holder };
|
|
63
|
+
}
|
|
64
|
+
const result = withMaintenanceStartBarrier(acquire);
|
|
65
|
+
if (result.state === "acquired")
|
|
66
|
+
return result;
|
|
67
|
+
warn(`[index] another index run is active (pid ${result.holder.pid}, started ${result.holder.startedAt}); ` +
|
|
68
|
+
"this run will contend with it — pass --skip-if-locked for scheduled runs");
|
|
69
|
+
return { state: "contended", holder: result.holder };
|
|
70
|
+
}
|
|
71
|
+
export function releaseIndexRebuildLock(ownership) {
|
|
72
|
+
releaseLock(ownership);
|
|
73
|
+
}
|
|
@@ -28,9 +28,17 @@ import path from "node:path";
|
|
|
28
28
|
import { createLockPayload, probeLock, reclaimStaleLock, releaseLock, tryAcquireLockSync, } from "../core/file-lock.js";
|
|
29
29
|
import { tryAcquireMaintenanceBarrier } from "../core/maintenance-barrier.js";
|
|
30
30
|
import { getDbPath, getIndexWriterLockPath } from "../core/paths.js";
|
|
31
|
+
import { warn } from "../core/warn.js";
|
|
31
32
|
import { sleepSync } from "../runtime.js";
|
|
32
33
|
const ASSET_MUTATION_WAIT_MS = 100;
|
|
33
34
|
const DEFAULT_ASSET_MUTATION_MAX_WAIT_MS = 10 * 60 * 1000;
|
|
35
|
+
/** How often a blocked sync waiter re-announces that it is still waiting. Mirrors the async path's cadence. */
|
|
36
|
+
const ASSET_MUTATION_WAIT_NOTICE_INTERVAL_MS = 15_000;
|
|
37
|
+
/** TEST-ONLY. Shrinks the sync path's max-wait/notice-cadence so a test doesn't block for real minutes/seconds. */
|
|
38
|
+
let syncTimingOverridesForTests;
|
|
39
|
+
export function _setAssetMutationLeaseSyncTimingForTests(overrides) {
|
|
40
|
+
syncTimingOverridesForTests = overrides;
|
|
41
|
+
}
|
|
34
42
|
const leaseContext = new AsyncLocalStorage();
|
|
35
43
|
function buildPayload(purpose) {
|
|
36
44
|
return createLockPayload({
|
|
@@ -42,6 +50,26 @@ function buildPayload(purpose) {
|
|
|
42
50
|
function delay(ms) {
|
|
43
51
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
44
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Describe who currently holds `lockPath`, from `probeLock`'s own payload
|
|
55
|
+
* (the same `purpose`/`pid`/`startedAt` fields `buildPayload` writes) —
|
|
56
|
+
* never the waiting caller's own `purpose`, which names what SELF is trying
|
|
57
|
+
* to do, not who is in the way.
|
|
58
|
+
*/
|
|
59
|
+
function describeAssetMutationHolder(lockPath) {
|
|
60
|
+
const probe = probeLock(lockPath);
|
|
61
|
+
const rawContent = probe.state === "held" || probe.state === "stale" ? probe.rawContent : undefined;
|
|
62
|
+
if (!rawContent)
|
|
63
|
+
return "an asset write";
|
|
64
|
+
try {
|
|
65
|
+
const payload = JSON.parse(rawContent);
|
|
66
|
+
const who = payload.purpose ?? "an asset write";
|
|
67
|
+
return `${who} (pid ${payload.pid ?? "?"}, started ${payload.startedAt ?? "?"})`;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return "an asset write";
|
|
71
|
+
}
|
|
72
|
+
}
|
|
45
73
|
function throwIfAborted(signal) {
|
|
46
74
|
if (!signal?.aborted)
|
|
47
75
|
return;
|
|
@@ -142,14 +170,25 @@ export function withAssetMutationLeaseSync(purpose, run) {
|
|
|
142
170
|
const context = inherited ?? new Set();
|
|
143
171
|
const execute = () => {
|
|
144
172
|
const startedAt = Date.now();
|
|
173
|
+
const maxWaitMs = syncTimingOverridesForTests?.maxWaitMs ?? DEFAULT_ASSET_MUTATION_MAX_WAIT_MS;
|
|
174
|
+
const waitNoticeIntervalMs = syncTimingOverridesForTests?.waitNoticeIntervalMs ?? ASSET_MUTATION_WAIT_NOTICE_INTERVAL_MS;
|
|
145
175
|
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
146
176
|
let lease;
|
|
177
|
+
let lastWaitNoticeMs = 0;
|
|
147
178
|
while (!lease) {
|
|
148
179
|
lease = tryAcquireAssetMutationLease(lockPath, purpose);
|
|
149
180
|
if (!lease) {
|
|
150
|
-
|
|
181
|
+
const waitedMs = Date.now() - startedAt;
|
|
182
|
+
if (waitedMs >= maxWaitMs) {
|
|
151
183
|
throw new Error(`timed out waiting for asset mutation lease for ${purpose}`);
|
|
152
184
|
}
|
|
185
|
+
// #956: the sync path used to wait up to 10 minutes with zero
|
|
186
|
+
// progress feedback. Mirror the async path's 15s onWait cadence, but
|
|
187
|
+
// actually warn by default — no caller wires onWait on this path.
|
|
188
|
+
if (waitedMs - lastWaitNoticeMs >= waitNoticeIntervalMs) {
|
|
189
|
+
warn(`waiting for ${describeAssetMutationHolder(lockPath)} — ${Math.round(waitedMs / 1000)}s`);
|
|
190
|
+
lastWaitNoticeMs = waitedMs;
|
|
191
|
+
}
|
|
153
192
|
sleepSync(ASSET_MUTATION_WAIT_MS);
|
|
154
193
|
}
|
|
155
194
|
}
|
|
@@ -24,8 +24,9 @@ import path from "node:path";
|
|
|
24
24
|
import { akmAdapter } from "../core/adapter/adapters/akm-adapter.js";
|
|
25
25
|
import { loadConfig } from "../core/config/config.js";
|
|
26
26
|
import { isDataDirUnreadableError } from "../core/errors.js";
|
|
27
|
+
import { probeLock } from "../core/file-lock.js";
|
|
27
28
|
import { isPathAbsent } from "../core/path-access.js";
|
|
28
|
-
import { getDbPath } from "../core/paths.js";
|
|
29
|
+
import { getDbPath, getIndexRebuildLockPath } from "../core/paths.js";
|
|
29
30
|
import { warn, warnVerbose } from "../core/warn.js";
|
|
30
31
|
import { closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
31
32
|
import { deleteEntriesByIds, getEntryCount, upsertEntry } from "../storage/repositories/index-entries-repository.js";
|
|
@@ -54,10 +55,29 @@ export const WRITE_PATH_INDEX_BUSY_TIMEOUT_MS = 5_000;
|
|
|
54
55
|
* An absent or empty index is skipped on purpose — bootstrap belongs to the
|
|
55
56
|
* first read (`ensureIndex`) or an explicit `akm index`, which also cover
|
|
56
57
|
* embeddings and the other passes this fast path skips.
|
|
58
|
+
*
|
|
59
|
+
* A live rebuild holding the rebuild lock is the SAME kind of skip, not a
|
|
60
|
+
* failure (#956 fix): a live `akm index` run owns bringing the index to the
|
|
61
|
+
* expected state on its own, so this call returns `true` on that skip
|
|
62
|
+
* exactly like the absent-index and empty-index cases above. Callers that
|
|
63
|
+
* gate their own success on this boolean (`acceptProposal`, `source clone`)
|
|
64
|
+
* must never fail or warn just because a concurrent rebuild is in progress.
|
|
57
65
|
*/
|
|
58
66
|
export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
59
67
|
try {
|
|
60
68
|
return await (async () => {
|
|
69
|
+
// #956: a live `akm index` rebuild holds index.db under one long
|
|
70
|
+
// transaction (persistDirRecords), which this fast path's own 5s
|
|
71
|
+
// busy_timeout would just contend with pointlessly. Interactive
|
|
72
|
+
// commands (remember, import, extract session assets) must never wait
|
|
73
|
+
// on it — skip the inline upsert/embedding entirely; the write itself
|
|
74
|
+
// (file + commit) has already succeeded by the time this runs, and the
|
|
75
|
+
// rebuild in progress will pick up the change on its own.
|
|
76
|
+
const rebuildProbe = probeLock(getIndexRebuildLockPath());
|
|
77
|
+
if (rebuildProbe.state === "held") {
|
|
78
|
+
warn(`index rebuild in progress (pid ${rebuildProbe.holderPid}); the next index pass will index ${filePaths.join(", ")}`);
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
61
81
|
const dbPath = getDbPath();
|
|
62
82
|
// `true` here means "the index is in the state the caller expects" — and
|
|
63
83
|
// `acceptProposal` advances its journal to `index-finalized` on the
|