akm-cli 0.9.0 → 0.9.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +724 -0
- package/README.md +28 -63
- package/STABILITY.md +4 -2
- package/dist/cli/parse-args.js +7 -1
- package/dist/commands/agent/contribute-cli.js +1 -1
- package/dist/commands/env/child-env.js +14 -0
- package/dist/commands/feedback-cli.js +7 -1
- package/dist/commands/health/llm-usage.js +2 -1
- package/dist/commands/health/surfaces.js +4 -77
- package/dist/commands/health.js +65 -11
- package/dist/commands/improve/distill/quality-gate.js +6 -1
- package/dist/commands/improve/eligibility.js +7 -1
- package/dist/commands/improve/eval-cases.js +2 -0
- package/dist/commands/improve/improve.js +126 -10
- package/dist/commands/improve/locks.js +7 -0
- package/dist/commands/improve/memory/memory-improve.js +9 -0
- package/dist/commands/improve/run-context.js +5 -0
- package/dist/commands/improve/session-asset.js +4 -0
- package/dist/commands/lint/base-linter.js +31 -7
- package/dist/commands/lint/index.js +205 -51
- package/dist/commands/lint/types.js +22 -1
- package/dist/commands/proposal/repository.js +17 -1
- package/dist/commands/sources/add-cli.js +8 -2
- package/dist/commands/sources/info.js +12 -2
- package/dist/commands/sources/installed-stashes.js +6 -1
- package/dist/commands/sources/migration-help.js +12 -3
- package/dist/commands/sources/self-update.js +9 -1
- package/dist/commands/tasks/tasks.js +8 -2
- package/dist/commands/workflow-cli.js +17 -11
- package/dist/core/abort-deadline.js +28 -0
- package/dist/core/adapter/adapters/agent-skills-adapter.js +83 -5
- package/dist/core/adapter/adapters/akm-adapter.js +13 -10
- package/dist/core/adapter/adapters/akm-lint.js +78 -22
- package/dist/core/adapter/adapters/akm-task-adapter.js +43 -20
- package/dist/core/adapter/adapters/dotenv-adapter.js +21 -0
- package/dist/core/adapter/adapters/tool-dir-shared.js +5 -3
- package/dist/core/asset/frontmatter.js +10 -1
- package/dist/core/common.js +147 -9
- package/dist/core/concurrent.js +32 -0
- package/dist/core/config/config-io.js +5 -45
- package/dist/core/config/schema/engines.js +14 -3
- package/dist/core/config/schema/workflow.js +11 -0
- package/dist/core/errors.js +25 -0
- package/dist/core/events.js +30 -24
- package/dist/core/extra-params.js +11 -0
- package/dist/core/file-lock.js +7 -1
- package/dist/core/fs-txn.js +15 -2
- package/dist/core/improve-result.js +5 -0
- package/dist/core/json-schema.js +344 -9
- package/dist/core/loopback.js +89 -0
- package/dist/core/migration-operation.js +17 -2
- package/dist/core/path-access.js +107 -0
- package/dist/core/paths.js +16 -2
- package/dist/core/redaction.js +86 -18
- package/dist/core/spawn-env.js +234 -0
- package/dist/core/state-db-scope.js +134 -0
- package/dist/core/state-db.js +1 -0
- package/dist/core/subprocess.js +181 -37
- package/dist/core/write-provenance.js +85 -0
- package/dist/core/write-source.js +33 -2
- package/dist/indexer/db/graph-db.js +17 -6
- package/dist/indexer/ensure-index.js +10 -3
- package/dist/indexer/index-written-assets.js +17 -2
- package/dist/indexer/indexer.js +86 -21
- package/dist/indexer/passes/memory-inference.js +4 -0
- package/dist/indexer/search/db-search.js +25 -17
- package/dist/indexer/walk/walker.js +6 -1
- package/dist/integrations/agent/detect.js +13 -1
- package/dist/integrations/agent/engine-resolution.js +24 -11
- package/dist/integrations/agent/model-aliases.js +1 -1
- package/dist/integrations/agent/profiles.js +9 -1
- package/dist/integrations/agent/spawn.js +15 -87
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +21 -0
- package/dist/integrations/lockfile.js +55 -2
- package/dist/llm/client.js +14 -19
- package/dist/llm/embedder.js +23 -3
- package/dist/llm/embedders/remote.js +27 -2
- package/dist/output/html-render.js +40 -1
- package/dist/output/text/lint-format.js +17 -4
- package/dist/runtime.js +23 -1
- package/dist/scripts/akm-migrate-node.js +1714 -836
- package/dist/scripts/akm-migrate.js +1682 -804
- package/dist/setup/setup.js +22 -7
- package/dist/sources/providers/git-install.js +25 -2
- package/dist/sources/providers/git-stash.js +19 -0
- package/dist/sources/providers/git.js +1 -1
- package/dist/sources/snapshot-fetchers/content-extract.js +63 -1
- package/dist/sources/snapshot-fetchers/website-ingest.js +126 -20
- package/dist/storage/database.js +71 -7
- package/dist/storage/engines/sqlite-migrations.js +61 -2
- package/dist/storage/managed-db.js +19 -0
- package/dist/storage/repositories/index-connection.js +39 -4
- package/dist/storage/repositories/index-entries-repository.js +6 -1
- package/dist/storage/repositories/index-meta-repository.js +11 -0
- package/dist/storage/repositories/index-schema.js +17 -2
- package/dist/storage/repositories/index-vec-repository.js +43 -5
- package/dist/storage/repositories/workflow-runs-repository.js +66 -13
- package/dist/storage/sqlite-pragmas.js +12 -1
- package/dist/tasks/log-redaction.js +156 -0
- package/dist/tasks/parser.js +82 -5
- package/dist/tasks/runner.js +222 -17
- package/dist/tasks/scheduler-invocation.js +19 -0
- package/dist/tasks/schema.js +86 -1
- package/dist/text-import-hook.mjs +1 -1
- package/dist/workflows/concurrency-policy.js +95 -1
- package/dist/workflows/exec/dispatch-redaction.js +114 -0
- package/dist/workflows/exec/exec-unit.js +542 -0
- package/dist/workflows/exec/frozen-judge.js +114 -42
- package/dist/workflows/exec/native-executor.js +465 -238
- package/dist/workflows/exec/param-secrets.js +4 -3
- package/dist/workflows/exec/run-workflow.js +424 -219
- package/dist/workflows/exec/step-work.js +506 -167
- package/dist/workflows/exec/unit-dispatch.js +31 -1
- package/dist/workflows/exec/unit-writer.js +53 -13
- package/dist/workflows/exec/worktree.js +454 -41
- package/dist/workflows/ir/compile.js +26 -2
- package/dist/workflows/ir/freeze.js +82 -15
- package/dist/workflows/ir/schema.js +105 -20
- package/dist/workflows/parser.js +242 -19
- package/dist/workflows/program/schema.js +24 -0
- package/dist/workflows/renderer.js +32 -4
- package/dist/workflows/resource-limits.js +182 -0
- package/dist/workflows/runtime/runs.js +146 -6
- package/dist/workflows/validate-summary.js +17 -2
- package/docs/README.md +74 -32
- package/docs/migration/release-notes/0.9.0.md +2 -1
- package/docs/migration/v0.7-to-v0.8.md +2 -1
- package/docs/migration/v0.8-to-v0.9.md +3 -1
- package/docs/reference/README.md +11 -4
- package/docs/reference/bundle-types.md +19 -0
- package/docs/reference/cli.md +105 -16
- package/docs/reference/configuration.md +15 -2
- package/docs/reference/data-and-telemetry.md +30 -10
- package/docs/reference/supported-formats.md +50 -0
- package/docs/reference/workflow-schema.md +1014 -0
- package/docs/reference/workflows.md +37 -633
- package/package.json +13 -6
- package/schemas/akm-config.json +18 -5
- package/schemas/akm-task.json +27 -5
- package/schemas/akm-workflow.json +92 -13
package/dist/core/subprocess.js
CHANGED
|
@@ -12,8 +12,10 @@
|
|
|
12
12
|
* kill reaps the whole descendant tree — no orphaned children.
|
|
13
13
|
* • A SIGTERM→SIGKILL kill ladder on timeout/abort — a child that ignores
|
|
14
14
|
* SIGTERM is force-killed after a grace period instead of wedging forever.
|
|
15
|
-
* •
|
|
16
|
-
* wall budget even when the child leaves a pipe endpoint open
|
|
15
|
+
* • Time-bounded output capture ({@link readStream}) that cannot block past
|
|
16
|
+
* the wall budget even when the child leaves a pipe endpoint open, plus an
|
|
17
|
+
* OPT-IN RETENTION cap (`maxOutputBytes`); see {@link readStream} for what
|
|
18
|
+
* the cap does and does not bound.
|
|
17
19
|
* • Injectable `spawnFn`/`setTimeoutFn`/`clearTimeoutFn` seams so callers
|
|
18
20
|
* can drive the machinery deterministically in tests.
|
|
19
21
|
*
|
|
@@ -70,33 +72,115 @@ export function scheduleKillLadder(proc, opts) {
|
|
|
70
72
|
if (typeof sigkillTimer !== "number")
|
|
71
73
|
sigkillTimer.unref?.();
|
|
72
74
|
}
|
|
75
|
+
/**
|
|
76
|
+
* The joined, human-readable reason ONE managed run's capture is incomplete, or
|
|
77
|
+
* `undefined` when both pipes drained cleanly.
|
|
78
|
+
*
|
|
79
|
+
* Shared so every caller that promotes captured output treats an incomplete
|
|
80
|
+
* capture the same way. A pipe that errored, or that hit the stream-drain
|
|
81
|
+
* timeout because a background descendant kept the fd open after the leader
|
|
82
|
+
* exited 0, yields a PARTIAL string — and a caller that reads only `exitCode`
|
|
83
|
+
* would promote that partial as if it were the command's whole output.
|
|
84
|
+
* `maxBytes` overflow is deliberately NOT reported here: it is a distinct,
|
|
85
|
+
* caller-classified condition (see {@link StreamReadResult.overflowed}), not a
|
|
86
|
+
* drain malfunction.
|
|
87
|
+
*/
|
|
88
|
+
export function streamCaptureFailure(stdout, stderr) {
|
|
89
|
+
const failures = [];
|
|
90
|
+
if (stdout.error)
|
|
91
|
+
failures.push(`stdout read failed: ${errorText(stdout.error)}`);
|
|
92
|
+
if (stderr.error)
|
|
93
|
+
failures.push(`stderr read failed: ${errorText(stderr.error)}`);
|
|
94
|
+
if (stdout.timedOut)
|
|
95
|
+
failures.push("stdout drain timed out");
|
|
96
|
+
if (stderr.timedOut)
|
|
97
|
+
failures.push("stderr drain timed out");
|
|
98
|
+
return failures.length === 0 ? undefined : failures.join("; ");
|
|
99
|
+
}
|
|
100
|
+
function errorText(error) {
|
|
101
|
+
return error instanceof Error ? error.message : String(error);
|
|
102
|
+
}
|
|
73
103
|
const STREAM_READ_TIMEOUT = Symbol("stream-read-timeout");
|
|
104
|
+
/**
|
|
105
|
+
* Index at or before `limit` where a UTF-8 CHARACTER starts, so a retention cut
|
|
106
|
+
* never lands inside a multi-byte sequence. Continuation bytes are `10xxxxxx`;
|
|
107
|
+
* a well-formed sequence has at most three of them, so this walks back at most
|
|
108
|
+
* three positions.
|
|
109
|
+
*/
|
|
110
|
+
function utf8BoundaryAtOrBefore(value, limit) {
|
|
111
|
+
let cut = limit;
|
|
112
|
+
while (cut > 0 && (value[cut] & 0xc0) === 0x80)
|
|
113
|
+
cut--;
|
|
114
|
+
return cut;
|
|
115
|
+
}
|
|
74
116
|
/**
|
|
75
117
|
* Drain a readable stream to text, optionally racing each read against a
|
|
76
118
|
* timeout so a process that is killed via SIGTERM/SIGKILL but whose pipe
|
|
77
119
|
* endpoints stay open (e.g. background threads still holding the fd) cannot
|
|
78
120
|
* block the caller indefinitely. On timeout we return whatever was decoded
|
|
79
121
|
* before the pipe stopped draining.
|
|
122
|
+
*
|
|
123
|
+
* ## The `maxBytes` cap bounds MEMORY, not the child
|
|
124
|
+
*
|
|
125
|
+
* The drain ALWAYS runs to the end of the stream. `maxBytes` caps only what is
|
|
126
|
+
* RETAINED: once the cap is reached the loop keeps calling `reader.read()` and
|
|
127
|
+
* throws the bytes away. That is what makes the cap safe to impose on a
|
|
128
|
+
* process that is still running — a reader that stopped pulling would fill the
|
|
129
|
+
* pipe buffer and BLOCK the child on its next write, turning "the output got
|
|
130
|
+
* long" into "the command hangs until the wall timeout". Draining and
|
|
131
|
+
* discarding costs nothing but the reads, and the child finishes normally.
|
|
80
132
|
*/
|
|
81
133
|
export async function readStream(stream, opts) {
|
|
82
134
|
if (!stream)
|
|
83
|
-
return { text: "", timedOut: false };
|
|
135
|
+
return { text: "", timedOut: false, overflowed: false, bytesRead: 0, retainedBytes: 0 };
|
|
84
136
|
const reader = stream.getReader();
|
|
85
137
|
const decoder = new TextDecoder();
|
|
138
|
+
const maxBytes = opts?.maxBytes;
|
|
86
139
|
let text = "";
|
|
140
|
+
let bytesRead = 0;
|
|
141
|
+
let retainedBytes = 0;
|
|
142
|
+
let overflowed = false;
|
|
143
|
+
/**
|
|
144
|
+
* Common per-chunk accumulate. Uncapped it appends everything. Capped, it
|
|
145
|
+
* appends until the cap and then only COUNTS — the caller's loop keeps reading
|
|
146
|
+
* either way, which is the whole point.
|
|
147
|
+
*/
|
|
148
|
+
const absorb = (value) => {
|
|
149
|
+
bytesRead += value.byteLength;
|
|
150
|
+
if (maxBytes === undefined) {
|
|
151
|
+
retainedBytes += value.byteLength;
|
|
152
|
+
text += decoder.decode(value, { stream: true });
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
// Already past the cap: discard. Never resume retaining — a gap in the
|
|
156
|
+
// middle would splice two disjoint regions into one string.
|
|
157
|
+
if (overflowed)
|
|
158
|
+
return;
|
|
159
|
+
if (retainedBytes + value.byteLength <= maxBytes) {
|
|
160
|
+
retainedBytes += value.byteLength;
|
|
161
|
+
text += decoder.decode(value, { stream: true });
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
overflowed = true;
|
|
165
|
+
const keep = utf8BoundaryAtOrBefore(value, maxBytes - retainedBytes);
|
|
166
|
+
if (keep > 0) {
|
|
167
|
+
retainedBytes += keep;
|
|
168
|
+
text += decoder.decode(value.subarray(0, keep), { stream: true });
|
|
169
|
+
}
|
|
170
|
+
};
|
|
87
171
|
if (!opts?.timeoutMs) {
|
|
88
172
|
try {
|
|
89
173
|
while (true) {
|
|
90
174
|
const chunk = await reader.read();
|
|
91
175
|
if (chunk.done)
|
|
92
176
|
break;
|
|
93
|
-
|
|
177
|
+
absorb(chunk.value);
|
|
94
178
|
}
|
|
95
179
|
text += decoder.decode();
|
|
96
|
-
return { text, timedOut: false };
|
|
180
|
+
return { text, timedOut: false, overflowed, bytesRead, retainedBytes };
|
|
97
181
|
}
|
|
98
182
|
catch (error) {
|
|
99
|
-
return { text, timedOut: false, error };
|
|
183
|
+
return { text, timedOut: false, error, overflowed, bytesRead, retainedBytes };
|
|
100
184
|
}
|
|
101
185
|
finally {
|
|
102
186
|
try {
|
|
@@ -109,33 +193,46 @@ export async function readStream(stream, opts) {
|
|
|
109
193
|
}
|
|
110
194
|
const setTimeoutImpl = opts.setTimeoutFn ?? setTimeout;
|
|
111
195
|
const clearTimeoutImpl = opts.clearTimeoutFn ?? clearTimeout;
|
|
196
|
+
const timeoutMs = opts.timeoutMs;
|
|
112
197
|
let timer;
|
|
198
|
+
let drained = false;
|
|
113
199
|
const timeoutPromise = new Promise((resolve) => {
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
timer
|
|
200
|
+
const arm = () => {
|
|
201
|
+
// The drain already finished while the deadline was still waiting to be
|
|
202
|
+
// armed; a timer started now would belong to nobody.
|
|
203
|
+
if (drained)
|
|
204
|
+
return;
|
|
205
|
+
timer = setTimeoutImpl(() => {
|
|
206
|
+
timer = undefined;
|
|
207
|
+
resolve(STREAM_READ_TIMEOUT);
|
|
208
|
+
}, timeoutMs);
|
|
209
|
+
if (typeof timer !== "number")
|
|
210
|
+
timer.unref?.();
|
|
211
|
+
};
|
|
212
|
+
if (opts.armTimeoutAfter)
|
|
213
|
+
void opts.armTimeoutAfter.then(arm, arm);
|
|
214
|
+
else
|
|
215
|
+
arm();
|
|
120
216
|
});
|
|
121
217
|
try {
|
|
122
218
|
while (true) {
|
|
123
219
|
const chunk = await Promise.race([reader.read(), timeoutPromise]);
|
|
124
220
|
if (chunk === STREAM_READ_TIMEOUT) {
|
|
125
221
|
void reader.cancel().catch(() => { });
|
|
126
|
-
return { text, timedOut: true };
|
|
222
|
+
return { text, timedOut: true, overflowed, bytesRead, retainedBytes };
|
|
127
223
|
}
|
|
128
224
|
if (chunk.done)
|
|
129
225
|
break;
|
|
130
|
-
|
|
226
|
+
absorb(chunk.value);
|
|
131
227
|
}
|
|
132
228
|
text += decoder.decode();
|
|
133
|
-
return { text, timedOut: false };
|
|
229
|
+
return { text, timedOut: false, overflowed, bytesRead, retainedBytes };
|
|
134
230
|
}
|
|
135
231
|
catch (error) {
|
|
136
|
-
return { text, timedOut: false, error };
|
|
232
|
+
return { text, timedOut: false, error, overflowed, bytesRead, retainedBytes };
|
|
137
233
|
}
|
|
138
234
|
finally {
|
|
235
|
+
drained = true;
|
|
139
236
|
if (timer !== undefined) {
|
|
140
237
|
clearTimeoutImpl(timer);
|
|
141
238
|
}
|
|
@@ -147,8 +244,29 @@ export async function readStream(stream, opts) {
|
|
|
147
244
|
}
|
|
148
245
|
}
|
|
149
246
|
}
|
|
150
|
-
const EMPTY_READ = {
|
|
247
|
+
const EMPTY_READ = {
|
|
248
|
+
text: "",
|
|
249
|
+
timedOut: false,
|
|
250
|
+
overflowed: false,
|
|
251
|
+
bytesRead: 0,
|
|
252
|
+
retainedBytes: 0,
|
|
253
|
+
};
|
|
254
|
+
/**
|
|
255
|
+
* Drain deadline for a run with no wall budget, counted from the child's EXIT
|
|
256
|
+
* (never from capture — see {@link runManagedSubprocess}). It bounds only the
|
|
257
|
+
* window in which a descendant can keep an inherited pipe open after the leader
|
|
258
|
+
* is gone, so it can be generous without ever capping the run itself.
|
|
259
|
+
*/
|
|
151
260
|
const UNBOUNDED_STREAM_READ_SAFETY_MS = 60 * 60 * 1000;
|
|
261
|
+
/**
|
|
262
|
+
* Drain deadline for a run WITH a wall budget, counted from the moment the
|
|
263
|
+
* pipe's owner left — the child exited, or (a child that survived its own kill
|
|
264
|
+
* ladder) the budget expired. Anything still holding the fd open then is a
|
|
265
|
+
* background descendant, not the command, so this is the same 2 s the old
|
|
266
|
+
* capture-anchored deadline allowed past the budget; only its starting point
|
|
267
|
+
* moved earlier.
|
|
268
|
+
*/
|
|
269
|
+
const POST_EXIT_STREAM_DRAIN_GRACE_MS = 2_000;
|
|
152
270
|
function toError(err) {
|
|
153
271
|
return err instanceof Error ? err : new Error(String(err));
|
|
154
272
|
}
|
|
@@ -207,8 +325,18 @@ export async function runManagedSubprocess(cmd, opts) {
|
|
|
207
325
|
// Skipped entirely when timeoutMs is null.
|
|
208
326
|
let timedOut = false;
|
|
209
327
|
let timer;
|
|
328
|
+
// Settles when a bounded run's wall budget expires — the second way (after
|
|
329
|
+
// the child's own exit) a captured pipe can stop belonging to a live command.
|
|
330
|
+
// See the drain-deadline note below.
|
|
331
|
+
let onBudgetExpiry;
|
|
332
|
+
const budgetExpired = capture && timeoutMs !== null
|
|
333
|
+
? new Promise((resolve) => {
|
|
334
|
+
onBudgetExpiry = resolve;
|
|
335
|
+
})
|
|
336
|
+
: undefined;
|
|
210
337
|
if (timeoutMs !== null) {
|
|
211
338
|
timer = setTimeoutImpl(() => {
|
|
339
|
+
onBudgetExpiry?.();
|
|
212
340
|
scheduleKillLadder(proc, {
|
|
213
341
|
onKill: () => {
|
|
214
342
|
timedOut = true;
|
|
@@ -239,25 +367,33 @@ export async function runManagedSubprocess(cmd, opts) {
|
|
|
239
367
|
else
|
|
240
368
|
abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
241
369
|
}
|
|
242
|
-
// Stream-drain timeout:
|
|
243
|
-
// safety bound when execution itself is unbounded.
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
370
|
+
// Stream-drain timeout: a short grace once nothing living owns the pipe any
|
|
371
|
+
// more, or a one-hour safety bound when execution itself is unbounded.
|
|
372
|
+
//
|
|
373
|
+
// Neither is armed while the command is still running: every byte up to then
|
|
374
|
+
// is a live process's, and cancelling the reader would discard the output of
|
|
375
|
+
// work that is still going. What ends that ownership is where the two cases
|
|
376
|
+
// differ. Unbounded, the caller asked for NO cap on the run, so only the
|
|
377
|
+
// child's own exit can end it. Bounded, the wall budget ends it too — a child
|
|
378
|
+
// that outlived its own kill ladder is not something akm can keep waiting on
|
|
379
|
+
// — which leaves the SAME budget + grace ceiling as before for a command that
|
|
380
|
+
// really did spend its whole budget, and cuts the common case that used to
|
|
381
|
+
// stall: a leader exiting in milliseconds no longer holds the drain (and with
|
|
382
|
+
// it the unit, and with it a fan-out slot) open for a budget it never came
|
|
383
|
+
// close to spending.
|
|
384
|
+
const streamDrainTimeoutMs = timeoutMs !== null ? POST_EXIT_STREAM_DRAIN_GRACE_MS : UNBOUNDED_STREAM_READ_SAFETY_MS;
|
|
385
|
+
// Settles on a rejected `proc.exited` too: either way the child is gone.
|
|
386
|
+
const childSettled = capture ? proc.exited.catch(() => undefined) : undefined;
|
|
387
|
+
const pipeOwnerGone = childSettled && budgetExpired ? Promise.race([childSettled, budgetExpired]) : childSettled;
|
|
388
|
+
const readOpts = {
|
|
389
|
+
timeoutMs: streamDrainTimeoutMs,
|
|
390
|
+
setTimeoutFn: setTimeoutImpl,
|
|
391
|
+
clearTimeoutFn: clearTimeoutImpl,
|
|
392
|
+
...(pipeOwnerGone ? { armTimeoutAfter: pipeOwnerGone } : {}),
|
|
393
|
+
...(opts.maxOutputBytes !== undefined ? { maxBytes: opts.maxOutputBytes } : {}),
|
|
394
|
+
};
|
|
395
|
+
const stdoutPromise = capture ? readStream(proc.stdout ?? null, readOpts) : Promise.resolve(EMPTY_READ);
|
|
396
|
+
const stderrPromise = capture ? readStream(proc.stderr ?? null, readOpts) : Promise.resolve(EMPTY_READ);
|
|
261
397
|
// Optional stdin payload (captured mode only). Race the write/close against
|
|
262
398
|
// proc.exited so a child that never drains stdin cannot pin us past the
|
|
263
399
|
// timeout.
|
|
@@ -301,5 +437,13 @@ export async function runManagedSubprocess(cmd, opts) {
|
|
|
301
437
|
clearTimeoutImpl(timer);
|
|
302
438
|
abortSignal?.removeEventListener("abort", onAbort);
|
|
303
439
|
const [stdoutRead, stderrRead] = await Promise.all([stdoutPromise, stderrPromise]);
|
|
304
|
-
return {
|
|
440
|
+
return {
|
|
441
|
+
exitCode,
|
|
442
|
+
stdout: stdoutRead.text,
|
|
443
|
+
stderr: stderrRead.text,
|
|
444
|
+
timedOut,
|
|
445
|
+
aborted,
|
|
446
|
+
stdoutRead,
|
|
447
|
+
stderrRead,
|
|
448
|
+
};
|
|
305
449
|
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Run-scoped write provenance (#652).
|
|
6
|
+
*
|
|
7
|
+
* # Why this exists
|
|
8
|
+
*
|
|
9
|
+
* `akm improve`'s end-of-run auto-sync used to infer "what did this run write?"
|
|
10
|
+
* by diffing two Git dirty-path snapshots: everything dirty at end-of-run minus
|
|
11
|
+
* everything dirty when the run acquired its lock. That inference is wrong in
|
|
12
|
+
* both directions:
|
|
13
|
+
*
|
|
14
|
+
* - an unrelated managed-dir edit made by a human DURING a long run appears in
|
|
15
|
+
* the end-of-run diff and gets swept into akm's commit;
|
|
16
|
+
* - a path that was ALREADY dirty when the run started and is then rewritten by
|
|
17
|
+
* the run is subtracted out and never committed.
|
|
18
|
+
*
|
|
19
|
+
* The journal below replaces the inference with a record. Every akm write path
|
|
20
|
+
* that mutates a file on disk calls {@link recordWrittenPath}; a run opens a
|
|
21
|
+
* journal for its duration and reads back the exact set of paths it touched.
|
|
22
|
+
*
|
|
23
|
+
* # Contract
|
|
24
|
+
*
|
|
25
|
+
* - A *write* and a *removal* are recorded identically — the journal records
|
|
26
|
+
* "this run mutated this path", not what the mutation was. The final on-disk
|
|
27
|
+
* state is what gets staged (`git add -A -- <path>` stages a deletion just as
|
|
28
|
+
* happily as a modification), so a path written and then reverted, purged, or
|
|
29
|
+
* rolled back needs no special handling: it is journaled once and the stager
|
|
30
|
+
* sees whatever survived.
|
|
31
|
+
* - Recording is a no-op when no journal is open, so non-improve command paths
|
|
32
|
+
* pay nothing but a `Set.size` check.
|
|
33
|
+
* - Journals nest: every open journal observes every recorded path. Concurrent
|
|
34
|
+
* in-process runs (tests) therefore over-report rather than cross-attribute
|
|
35
|
+
* writes to the wrong run, which is the safe direction — over-reporting stages
|
|
36
|
+
* a path that has no diff, under-reporting loses a write.
|
|
37
|
+
* - Nothing here throws. A provenance failure must never break a write.
|
|
38
|
+
*/
|
|
39
|
+
import path from "node:path";
|
|
40
|
+
const activeJournals = new Set();
|
|
41
|
+
/** Open a journal. The caller MUST close it (`end()`) in a `finally`. */
|
|
42
|
+
export function beginWriteProvenance() {
|
|
43
|
+
const state = { touched: new Set() };
|
|
44
|
+
activeJournals.add(state);
|
|
45
|
+
const snapshot = () => [...state.touched].sort();
|
|
46
|
+
return {
|
|
47
|
+
writtenPaths: snapshot,
|
|
48
|
+
end: () => {
|
|
49
|
+
activeJournals.delete(state);
|
|
50
|
+
return snapshot();
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
/** True while at least one journal is open. */
|
|
55
|
+
export function isWriteProvenanceActive() {
|
|
56
|
+
return activeJournals.size > 0;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Record that the current run mutated `filePath` (write, create, rename, or
|
|
60
|
+
* delete). No-op when no journal is open. Never throws.
|
|
61
|
+
*/
|
|
62
|
+
export function recordWrittenPath(filePath) {
|
|
63
|
+
if (activeJournals.size === 0 || !filePath)
|
|
64
|
+
return;
|
|
65
|
+
let absolute;
|
|
66
|
+
try {
|
|
67
|
+
absolute = path.resolve(filePath);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
for (const journal of activeJournals)
|
|
73
|
+
journal.touched.add(absolute);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Normalize an absolute journaled path against `root`: POSIX-relative when the
|
|
77
|
+
* path is inside `root`, `undefined` otherwise (the caller decides whether an
|
|
78
|
+
* out-of-root write is reportable or, for staging, simply out of scope).
|
|
79
|
+
*/
|
|
80
|
+
export function relativeWrittenPath(root, absolutePath) {
|
|
81
|
+
const relative = path.relative(root, absolutePath).replaceAll(path.sep, "/");
|
|
82
|
+
if (!relative || relative === ".." || relative.startsWith("../") || path.isAbsolute(relative))
|
|
83
|
+
return undefined;
|
|
84
|
+
return relative;
|
|
85
|
+
}
|
|
@@ -33,11 +33,12 @@ import { ensureAkmMarkdownType } from "./asset/akm-markdown.js";
|
|
|
33
33
|
import { assetPathForName, stashDirFor } from "./asset/asset-placement.js";
|
|
34
34
|
import { conceptIdFromTypeName, displayRef } from "./asset/resolve-ref.js";
|
|
35
35
|
import { deriveBundleId } from "./bundle-id.js";
|
|
36
|
-
import { isWithin, resolveStashDir } from "./common.js";
|
|
36
|
+
import { existingFileMode, isWithin, resolveStashDir, writeFileAtomic } from "./common.js";
|
|
37
37
|
import { resolveConfiguredSources } from "./config/config.js";
|
|
38
38
|
import { ConfigError, UsageError } from "./errors.js";
|
|
39
39
|
import { sanitizeCommitMessage } from "./git-message.js";
|
|
40
40
|
import { warn } from "./warn.js";
|
|
41
|
+
import { recordWrittenPath } from "./write-provenance.js";
|
|
41
42
|
/**
|
|
42
43
|
* Source kinds that the loader is allowed to mark `writable: true`. Anything
|
|
43
44
|
* else is rejected at config load (per locked decision 4) — see
|
|
@@ -332,8 +333,15 @@ export async function writeAssetToSource(source, config, ref, content) {
|
|
|
332
333
|
const preflight = preflightGitPathMutation(source, filePath);
|
|
333
334
|
try {
|
|
334
335
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
335
|
-
|
|
336
|
+
// Atomic: truncate-and-rewrite left a window in which a crash, a full disk,
|
|
337
|
+
// or a concurrent reader saw a half-written or empty asset — destroying user
|
|
338
|
+
// content that was fine a moment earlier. writeFileAtomic writes a sibling
|
|
339
|
+
// temp file, fdatasyncs it, and renames over the target.
|
|
340
|
+
writeFileAtomic(filePath, normalized, existingFileMode(filePath));
|
|
336
341
|
recordWriteTargetPath(source, filePath);
|
|
342
|
+
// #652: run-scoped write provenance — the canonical asset write is the
|
|
343
|
+
// single largest contributor to an improve run's written-path set.
|
|
344
|
+
recordWrittenPath(filePath);
|
|
337
345
|
}
|
|
338
346
|
catch (error) {
|
|
339
347
|
discardEmptyGitPreflight(preflight);
|
|
@@ -367,6 +375,9 @@ export async function deleteAssetFromSource(source, config, ref) {
|
|
|
367
375
|
try {
|
|
368
376
|
fs.unlinkSync(filePath);
|
|
369
377
|
recordWriteTargetPath(source, filePath);
|
|
378
|
+
// #652: a removal is journaled exactly like a write — the stager stages the
|
|
379
|
+
// final on-disk state, so a deleted path lands as a staged deletion.
|
|
380
|
+
recordWrittenPath(filePath);
|
|
370
381
|
}
|
|
371
382
|
catch (error) {
|
|
372
383
|
discardEmptyGitPreflight(preflight);
|
|
@@ -996,11 +1007,31 @@ function ensureWritable(source, config) {
|
|
|
996
1007
|
throw new UsageError(`Source "${source.name}" is not writable. Set \`writable: true\` on the source config entry to enable writes.`, "INVALID_FLAG_VALUE");
|
|
997
1008
|
}
|
|
998
1009
|
}
|
|
1010
|
+
/**
|
|
1011
|
+
* MS-DOS device names Windows still reserves in every directory, with or
|
|
1012
|
+
* without an extension (CON, PRN, AUX, NUL, COM1-9, LPT1-9).
|
|
1013
|
+
*/
|
|
1014
|
+
const WINDOWS_RESERVED_DEVICE_NAMES = new Set([
|
|
1015
|
+
"con",
|
|
1016
|
+
"prn",
|
|
1017
|
+
"aux",
|
|
1018
|
+
"nul",
|
|
1019
|
+
...Array.from({ length: 9 }, (_, i) => `com${i + 1}`),
|
|
1020
|
+
...Array.from({ length: 9 }, (_, i) => `lpt${i + 1}`),
|
|
1021
|
+
]);
|
|
999
1022
|
function resolveAssetFilePath(source, ref) {
|
|
1000
1023
|
const basename = path.posix.basename(ref.name.replaceAll("\\", "/")).replace(/\.md$/i, "").toLowerCase();
|
|
1001
1024
|
if (basename === "index" || basename === "log") {
|
|
1002
1025
|
throw new UsageError(`Reserved concept name "${basename}" cannot be written.`, "INVALID_FLAG_VALUE");
|
|
1003
1026
|
}
|
|
1027
|
+
// Windows resolves these names as DEVICES no matter the directory or the
|
|
1028
|
+
// extension, so `CON.md` is not a file — a write goes to the console and a
|
|
1029
|
+
// read blocks on console input. Rejected on every platform so a stash stays
|
|
1030
|
+
// portable: an asset authored on Linux must not become unopenable when the
|
|
1031
|
+
// same bundle is used on Windows.
|
|
1032
|
+
if (WINDOWS_RESERVED_DEVICE_NAMES.has(basename)) {
|
|
1033
|
+
throw new UsageError(`Asset name "${basename}" is a reserved Windows device name and cannot be written.`, "INVALID_FLAG_VALUE");
|
|
1034
|
+
}
|
|
1004
1035
|
const typeDir = stashDirFor(ref.type);
|
|
1005
1036
|
if (!typeDir) {
|
|
1006
1037
|
throw new UsageError(`Unknown asset type "${ref.type}". Cannot resolve a write path.`, "INVALID_FLAG_VALUE");
|
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
import
|
|
5
|
-
import {
|
|
4
|
+
import { rethrowIfDataDirUnreadable, rethrowIfTestIsolationError } from "../../core/errors.js";
|
|
5
|
+
import { isPathAbsent } from "../../core/path-access.js";
|
|
6
6
|
import { getDbPath } from "../../core/paths.js";
|
|
7
7
|
import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
|
|
8
8
|
function withReadableGraphDb(db, fn) {
|
|
9
9
|
if (db)
|
|
10
10
|
return fn(db);
|
|
11
11
|
const dbPath = getDbPath();
|
|
12
|
-
|
|
12
|
+
// `GRAPH_DB_MISSING` is the loaders' "nothing extracted yet" sentinel — every
|
|
13
|
+
// caller below turns it into `null`/`[]`. Reserve it for a genuinely ABSENT
|
|
14
|
+
// index: an index that exists and cannot be read must reach the caller as the
|
|
15
|
+
// ConfigError `openExistingDatabase` raises, not as "no graph data" (#791).
|
|
16
|
+
if (isPathAbsent(dbPath))
|
|
13
17
|
throw new Error("GRAPH_DB_MISSING");
|
|
14
18
|
const opened = openExistingDatabase(dbPath);
|
|
15
19
|
try {
|
|
@@ -244,8 +248,11 @@ export function loadGraphFilesOnly(stashPath, db) {
|
|
|
244
248
|
});
|
|
245
249
|
}
|
|
246
250
|
catch (err) {
|
|
247
|
-
// Never mask the bun-test isolation guard as "no stored graph files"
|
|
251
|
+
// Never mask the bun-test isolation guard as "no stored graph files",
|
|
252
|
+
// and never mask an index we are not allowed to read as one with no
|
|
253
|
+
// graph in it (#791) — `GRAPH_DB_MISSING` above is the only "absent".
|
|
248
254
|
rethrowIfTestIsolationError(err);
|
|
255
|
+
rethrowIfDataDirUnreadable(err);
|
|
249
256
|
return [];
|
|
250
257
|
}
|
|
251
258
|
}
|
|
@@ -315,8 +322,10 @@ export function loadStoredGraphMeta(stashPath, db) {
|
|
|
315
322
|
});
|
|
316
323
|
}
|
|
317
324
|
catch (err) {
|
|
318
|
-
// Never mask the bun-test isolation guard as "no stored graph meta"
|
|
325
|
+
// Never mask the bun-test isolation guard as "no stored graph meta",
|
|
326
|
+
// nor an unreadable index as one that simply has no graph (#791).
|
|
319
327
|
rethrowIfTestIsolationError(err);
|
|
328
|
+
rethrowIfDataDirUnreadable(err);
|
|
320
329
|
return null;
|
|
321
330
|
}
|
|
322
331
|
}
|
|
@@ -408,8 +417,10 @@ export function loadStoredGraphSnapshot(stashPath, db) {
|
|
|
408
417
|
});
|
|
409
418
|
}
|
|
410
419
|
catch (err) {
|
|
411
|
-
// Never mask the bun-test isolation guard as "no stored graph snapshot"
|
|
420
|
+
// Never mask the bun-test isolation guard as "no stored graph snapshot",
|
|
421
|
+
// nor an unreadable index as one that simply has no graph (#791).
|
|
412
422
|
rethrowIfTestIsolationError(err);
|
|
423
|
+
rethrowIfDataDirUnreadable(err);
|
|
413
424
|
return null;
|
|
414
425
|
}
|
|
415
426
|
}
|
|
@@ -22,9 +22,10 @@
|
|
|
22
22
|
import fs from "node:fs";
|
|
23
23
|
import path from "node:path";
|
|
24
24
|
import { placementSpecList } from "../core/asset/asset-placement.js";
|
|
25
|
+
import { classifyPathAccess } from "../core/path-access.js";
|
|
25
26
|
import { getDbPath } from "../core/paths.js";
|
|
26
27
|
import { warn } from "../core/warn.js";
|
|
27
|
-
import { closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
28
|
+
import { assertIndexPathReadable, closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
28
29
|
import { getEntryCount, getIndexedFilePaths } from "../storage/repositories/index-entries-repository.js";
|
|
29
30
|
import { getMeta } from "../storage/repositories/index-meta-repository.js";
|
|
30
31
|
import { warnOnBundleRenameDrift } from "./bundle-identity-guard.js";
|
|
@@ -109,7 +110,11 @@ function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths) {
|
|
|
109
110
|
*/
|
|
110
111
|
export function isIndexStale(stashDir) {
|
|
111
112
|
const dbPath = getDbPath();
|
|
112
|
-
|
|
113
|
+
// Raises on an index we cannot READ rather than calling it stale — "stale"
|
|
114
|
+
// sends us into an inline reindex that will fail anyway, and whose failure is
|
|
115
|
+
// reported as the misleading "proceeding with existing index" (#791).
|
|
116
|
+
assertIndexPathReadable(dbPath);
|
|
117
|
+
if (classifyPathAccess(dbPath).access === "absent")
|
|
113
118
|
return true;
|
|
114
119
|
let db;
|
|
115
120
|
try {
|
|
@@ -152,8 +157,10 @@ export function isIndexStale(stashDir) {
|
|
|
152
157
|
* or built for a different stash), so those cases must rebuild inline.
|
|
153
158
|
*/
|
|
154
159
|
function indexCanServeStash(stashDir) {
|
|
160
|
+
// Same rule as isIndexStale: unreadable is an error, not "cannot serve" (#791).
|
|
155
161
|
const dbPath = getDbPath();
|
|
156
|
-
|
|
162
|
+
assertIndexPathReadable(dbPath);
|
|
163
|
+
if (classifyPathAccess(dbPath).access === "absent")
|
|
157
164
|
return false;
|
|
158
165
|
let db;
|
|
159
166
|
try {
|
|
@@ -22,8 +22,10 @@
|
|
|
22
22
|
import fs from "node:fs";
|
|
23
23
|
import path from "node:path";
|
|
24
24
|
import { akmAdapter } from "../core/adapter/adapters/akm-adapter.js";
|
|
25
|
+
import { isDataDirUnreadableError } from "../core/errors.js";
|
|
26
|
+
import { isPathAbsent } from "../core/path-access.js";
|
|
25
27
|
import { getDbPath } from "../core/paths.js";
|
|
26
|
-
import { warnVerbose } from "../core/warn.js";
|
|
28
|
+
import { warn, warnVerbose } from "../core/warn.js";
|
|
27
29
|
import { closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
28
30
|
import { deleteEntriesByIds, getEntryCount, upsertEntry, upsertWorkflowDocument, } from "../storage/repositories/index-entries-repository.js";
|
|
29
31
|
import { rebuildFts } from "../storage/repositories/index-fts-repository.js";
|
|
@@ -57,7 +59,12 @@ export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
|
57
59
|
try {
|
|
58
60
|
return await withIndexWriterLease({ purpose: "index-written-assets" }, async () => {
|
|
59
61
|
const dbPath = getDbPath();
|
|
60
|
-
|
|
62
|
+
// `true` here means "the index is in the state the caller expects" — and
|
|
63
|
+
// `acceptProposal` advances its journal to `index-finalized` on the
|
|
64
|
+
// strength of it. Only a genuinely ABSENT index earns that answer: an
|
|
65
|
+
// index we cannot read has NOT been updated, so it falls through to
|
|
66
|
+
// `openExistingDatabase` and surfaces as the honest `false` (#791).
|
|
67
|
+
if (isPathAbsent(dbPath))
|
|
61
68
|
return true;
|
|
62
69
|
// The full walk never descends into dot-directories (for example `.meta/`)
|
|
63
70
|
// — mirror that dot-segment skip here so this fast path indexes exactly
|
|
@@ -146,6 +153,14 @@ export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
|
146
153
|
});
|
|
147
154
|
}
|
|
148
155
|
catch (error) {
|
|
156
|
+
// A permission fault is the one failure the next full index will NOT heal,
|
|
157
|
+
// so it does not get the verbose-only treatment the other skips do: fail
|
|
158
|
+
// open (the caller's write still stands) but say so where an operator can
|
|
159
|
+
// see it (#791).
|
|
160
|
+
if (isDataDirUnreadableError(error)) {
|
|
161
|
+
warn(`Write-path index update skipped — ${error.message} The asset will not appear in search until that is fixed.`);
|
|
162
|
+
return false;
|
|
163
|
+
}
|
|
149
164
|
warnVerbose("Write-path index update skipped (asset appears after the next full index):", error instanceof Error ? error.message : String(error));
|
|
150
165
|
return false;
|
|
151
166
|
}
|