akm-cli 0.9.0 → 0.9.1-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +707 -0
- package/README.md +28 -63
- package/STABILITY.md +4 -2
- package/dist/commands/agent/contribute-cli.js +1 -1
- package/dist/commands/feedback-cli.js +7 -1
- package/dist/commands/health/llm-usage.js +2 -1
- package/dist/commands/health/surfaces.js +4 -77
- package/dist/commands/health.js +65 -11
- package/dist/commands/improve/distill/quality-gate.js +6 -1
- package/dist/commands/improve/eligibility.js +7 -1
- package/dist/commands/improve/improve.js +126 -10
- package/dist/commands/improve/locks.js +7 -0
- package/dist/commands/improve/memory/memory-improve.js +8 -0
- package/dist/commands/improve/run-context.js +5 -0
- package/dist/commands/improve/session-asset.js +4 -0
- package/dist/commands/lint/base-linter.js +31 -7
- package/dist/commands/lint/index.js +200 -50
- package/dist/commands/lint/types.js +22 -1
- package/dist/commands/proposal/repository.js +17 -1
- package/dist/commands/sources/info.js +12 -2
- package/dist/commands/sources/installed-stashes.js +6 -1
- package/dist/commands/tasks/tasks.js +8 -2
- package/dist/commands/workflow-cli.js +17 -11
- package/dist/core/abort-deadline.js +28 -0
- package/dist/core/adapter/adapters/agent-skills-adapter.js +67 -5
- package/dist/core/adapter/adapters/akm-adapter.js +13 -10
- package/dist/core/adapter/adapters/akm-lint.js +72 -20
- package/dist/core/adapter/adapters/akm-task-adapter.js +40 -19
- package/dist/core/adapter/adapters/tool-dir-shared.js +5 -3
- package/dist/core/asset/frontmatter.js +4 -0
- package/dist/core/common.js +66 -6
- package/dist/core/concurrent.js +32 -0
- package/dist/core/config/schema/workflow.js +11 -0
- package/dist/core/errors.js +25 -0
- package/dist/core/events.js +30 -24
- package/dist/core/file-lock.js +7 -1
- package/dist/core/improve-result.js +5 -0
- package/dist/core/json-schema.js +327 -9
- package/dist/core/loopback.js +89 -0
- package/dist/core/migration-operation.js +17 -2
- package/dist/core/path-access.js +107 -0
- package/dist/core/redaction.js +64 -17
- package/dist/core/spawn-env.js +234 -0
- package/dist/core/state-db-scope.js +134 -0
- package/dist/core/subprocess.js +181 -37
- package/dist/core/write-provenance.js +85 -0
- package/dist/core/write-source.js +7 -0
- package/dist/indexer/db/graph-db.js +17 -6
- package/dist/indexer/ensure-index.js +10 -3
- package/dist/indexer/index-written-assets.js +17 -2
- package/dist/indexer/indexer.js +55 -15
- package/dist/indexer/passes/memory-inference.js +4 -0
- package/dist/indexer/search/db-search.js +8 -15
- package/dist/integrations/agent/engine-resolution.js +24 -11
- package/dist/integrations/agent/model-aliases.js +1 -1
- package/dist/integrations/agent/profiles.js +9 -1
- package/dist/integrations/agent/spawn.js +15 -87
- package/dist/integrations/lockfile.js +45 -2
- package/dist/output/text/lint-format.js +17 -4
- package/dist/scripts/akm-migrate-node.js +1435 -753
- package/dist/scripts/akm-migrate.js +1403 -721
- package/dist/sources/providers/git-stash.js +19 -0
- package/dist/sources/providers/git.js +1 -1
- package/dist/sources/snapshot-fetchers/website-ingest.js +126 -20
- package/dist/storage/database.js +6 -1
- package/dist/storage/managed-db.js +19 -0
- package/dist/storage/repositories/index-connection.js +28 -3
- package/dist/storage/repositories/index-entries-repository.js +6 -1
- package/dist/storage/repositories/workflow-runs-repository.js +66 -13
- package/dist/tasks/log-redaction.js +156 -0
- package/dist/tasks/parser.js +82 -5
- package/dist/tasks/runner.js +139 -11
- package/dist/tasks/schema.js +65 -0
- package/dist/workflows/concurrency-policy.js +95 -1
- package/dist/workflows/exec/dispatch-redaction.js +114 -0
- package/dist/workflows/exec/exec-unit.js +542 -0
- package/dist/workflows/exec/frozen-judge.js +114 -42
- package/dist/workflows/exec/native-executor.js +457 -238
- package/dist/workflows/exec/param-secrets.js +4 -3
- package/dist/workflows/exec/run-workflow.js +424 -219
- package/dist/workflows/exec/step-work.js +496 -165
- package/dist/workflows/exec/unit-dispatch.js +31 -1
- package/dist/workflows/exec/unit-writer.js +53 -13
- package/dist/workflows/exec/worktree.js +454 -41
- package/dist/workflows/ir/compile.js +26 -2
- package/dist/workflows/ir/freeze.js +82 -15
- package/dist/workflows/ir/schema.js +105 -20
- package/dist/workflows/parser.js +216 -18
- package/dist/workflows/program/schema.js +24 -0
- package/dist/workflows/renderer.js +32 -4
- package/dist/workflows/resource-limits.js +182 -0
- package/dist/workflows/runtime/runs.js +146 -6
- package/dist/workflows/validate-summary.js +17 -2
- package/docs/README.md +74 -32
- package/docs/migration/release-notes/0.9.0.md +2 -1
- package/docs/migration/v0.7-to-v0.8.md +2 -1
- package/docs/migration/v0.8-to-v0.9.md +3 -1
- package/docs/reference/README.md +11 -4
- package/docs/reference/bundle-types.md +19 -0
- package/docs/reference/cli.md +105 -16
- package/docs/reference/configuration.md +15 -2
- package/docs/reference/data-and-telemetry.md +30 -10
- package/docs/reference/supported-formats.md +50 -0
- package/docs/reference/workflow-schema.md +1014 -0
- package/docs/reference/workflows.md +37 -633
- package/package.json +13 -6
- package/schemas/akm-config.json +8 -0
- package/schemas/akm-task.json +27 -5
- package/schemas/akm-workflow.json +85 -10
|
@@ -56,6 +56,21 @@ export class GitStashPushError extends Error {
|
|
|
56
56
|
}
|
|
57
57
|
const GIT_PUSH_TIMEOUT_MS = 120_000;
|
|
58
58
|
const ZERO_OID = "0000000000000000000000000000000000000000";
|
|
59
|
+
let exactCommitHookForTests;
|
|
60
|
+
/**
|
|
61
|
+
* TEST-ONLY. Interleave work into the exact-path commit sequence; `undefined`
|
|
62
|
+
* restores. Exists because the CAS window is a few microseconds wide between
|
|
63
|
+
* two synchronous `git` invocations — a wall-clock race would be
|
|
64
|
+
* non-deterministic, and every earlier pre-check would swallow a commit that
|
|
65
|
+
* landed before the window opened. Inert in production.
|
|
66
|
+
*/
|
|
67
|
+
export function _setGitExactCommitHookForTests(hook) {
|
|
68
|
+
exactCommitHookForTests = hook;
|
|
69
|
+
}
|
|
70
|
+
/** Fire a named exact-path-commit race point (no-op outside tests). */
|
|
71
|
+
function gitExactCommitHook(point) {
|
|
72
|
+
exactCommitHookForTests?.(point);
|
|
73
|
+
}
|
|
59
74
|
/**
|
|
60
75
|
* Resolve the writable flag for an end-of-run / `akm sync` commit from the
|
|
61
76
|
* configured default bundle.
|
|
@@ -523,6 +538,10 @@ function createExactPathCommit(repoDir, options) {
|
|
|
523
538
|
throw new Error(`Git target changed before its exact commit could be attached.`);
|
|
524
539
|
}
|
|
525
540
|
assertWorktreeMatchesExpected(repoDir, options.paths, expected);
|
|
541
|
+
// Race window: everything below this line is defended only by the
|
|
542
|
+
// update-ref compare-and-swap. See
|
|
543
|
+
// tests/integration/sync-exact-commit-cas.test.ts.
|
|
544
|
+
gitExactCommitHook("before-update-ref");
|
|
526
545
|
const update = runGit(["-C", repoDir, "update-ref", branchRef, commitOid, options.baseHead ?? ZERO_OID]);
|
|
527
546
|
if (update.status !== 0) {
|
|
528
547
|
throw new Error(`Git target advanced before its exact commit could be attached.`);
|
|
@@ -10,4 +10,4 @@
|
|
|
10
10
|
// keeps importing from a single module namespace.
|
|
11
11
|
export { classifyCloneFailure, cloneRepo, inspectGitUpstream, runGit, syncExistingWritableCheckout, } from "./git-install.js";
|
|
12
12
|
export { ensureGitMirror, GitSourceProvider, getCachePaths, parseGitRepoUrl, syncMirroredRepo, } from "./git-provider.js";
|
|
13
|
-
export { GitStashPushError, isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "./git-stash.js";
|
|
13
|
+
export { _setGitExactCommitHookForTests, GitStashPushError, isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "./git-stash.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
import { createHash } from "node:crypto";
|
|
4
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import fs from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { fetchWithRetry, isWithin, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir, todayIso, } from "../../core/common.js";
|
|
@@ -201,19 +201,111 @@ async function fetchSnapshotViaRegistry(startUrl, stashDir, allowPrivateHosts, r
|
|
|
201
201
|
};
|
|
202
202
|
return dispatchSnapshotFetchers(parsed, context, stashDir);
|
|
203
203
|
}
|
|
204
|
+
let snapshotWriteHookForTests;
|
|
205
|
+
/**
|
|
206
|
+
* TEST-ONLY. Interrupt a refresh partway through its page-write loop;
|
|
207
|
+
* `undefined` restores. Exists because "a process killed mid-refresh" cannot
|
|
208
|
+
* be staged from outside the module — the whole loop is a single synchronous
|
|
209
|
+
* burst between two awaits. Inert in production (one `undefined?.()` per page).
|
|
210
|
+
*/
|
|
211
|
+
export function _setWebsiteSnapshotWriteHookForTests(hook) {
|
|
212
|
+
snapshotWriteHookForTests = hook;
|
|
213
|
+
}
|
|
214
|
+
function snapshotWriteHook(event) {
|
|
215
|
+
snapshotWriteHookForTests?.(event);
|
|
216
|
+
}
|
|
217
|
+
function snapshotSiblingPrefix(stashDir, kind) {
|
|
218
|
+
return `.${path.basename(stashDir)}.${kind}-`;
|
|
219
|
+
}
|
|
220
|
+
function snapshotSiblingPath(stashDir, kind) {
|
|
221
|
+
const unique = `${process.pid}-${randomBytes(6).toString("hex")}`;
|
|
222
|
+
return path.join(path.dirname(stashDir), `${snapshotSiblingPrefix(stashDir, kind)}${unique}`);
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Age gate for the staging sweep. Nothing enforces one refresh at a time for a
|
|
226
|
+
* given website source, so a sibling directory may belong to a refresh that is
|
|
227
|
+
* still running in another process; deleting it would break a healthy run
|
|
228
|
+
* instead of cleaning up after a dead one. Only clearly-abandoned directories
|
|
229
|
+
* (untouched for an hour — far longer than the 10-minute crawl wall-clock cap)
|
|
230
|
+
* are swept. Leftovers are inert until then: they are dot-prefixed, so the
|
|
231
|
+
* indexer's walk skips them.
|
|
232
|
+
*/
|
|
233
|
+
const SNAPSHOT_STAGING_SWEEP_AGE_MS = 60 * 60 * 1000;
|
|
234
|
+
/** Remove staging/retired directories abandoned by an earlier interrupted run. */
|
|
235
|
+
function sweepSnapshotStaging(stashDir) {
|
|
236
|
+
const parent = path.dirname(stashDir);
|
|
237
|
+
let entries;
|
|
238
|
+
try {
|
|
239
|
+
entries = fs.readdirSync(parent);
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
const prefixes = [snapshotSiblingPrefix(stashDir, "staging"), snapshotSiblingPrefix(stashDir, "retired")];
|
|
245
|
+
const cutoff = Date.now() - SNAPSHOT_STAGING_SWEEP_AGE_MS;
|
|
246
|
+
for (const entry of entries) {
|
|
247
|
+
if (!prefixes.some((prefix) => entry.startsWith(prefix)))
|
|
248
|
+
continue;
|
|
249
|
+
const abandoned = path.join(parent, entry);
|
|
250
|
+
try {
|
|
251
|
+
if (fs.statSync(abandoned).mtimeMs > cutoff)
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
fs.rmSync(abandoned, { recursive: true, force: true });
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
function beginSnapshotStaging(stashDir) {
|
|
261
|
+
fs.mkdirSync(path.dirname(stashDir), { recursive: true });
|
|
262
|
+
sweepSnapshotStaging(stashDir);
|
|
263
|
+
const dir = snapshotSiblingPath(stashDir, "staging");
|
|
264
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
265
|
+
return { dir, target: stashDir };
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Swap the staged snapshot in. POSIX cannot atomically exchange two non-empty
|
|
269
|
+
* directories, so the previous snapshot is renamed ASIDE first and deleted
|
|
270
|
+
* afterwards: the window in which the target does not exist is one syscall
|
|
271
|
+
* wide instead of an entire write loop.
|
|
272
|
+
*/
|
|
273
|
+
function publishSnapshotStaging(staging) {
|
|
274
|
+
let retired;
|
|
275
|
+
if (fs.existsSync(staging.target)) {
|
|
276
|
+
retired = snapshotSiblingPath(staging.target, "retired");
|
|
277
|
+
fs.renameSync(staging.target, retired);
|
|
278
|
+
}
|
|
279
|
+
fs.renameSync(staging.dir, staging.target);
|
|
280
|
+
if (retired)
|
|
281
|
+
fs.rmSync(retired, { recursive: true, force: true });
|
|
282
|
+
}
|
|
283
|
+
/** Drop an unpublished staging directory (no-op once it has been renamed). */
|
|
284
|
+
function discardSnapshotStaging(staging) {
|
|
285
|
+
fs.rmSync(staging.dir, { recursive: true, force: true });
|
|
286
|
+
}
|
|
204
287
|
/** Materialize a single fetcher snapshot as the source's whole stash. */
|
|
205
288
|
function writeSnapshotToStash(stashDir, snapshot) {
|
|
206
289
|
const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
|
|
207
290
|
const relPath = avoidReservedBasename(preferredName);
|
|
291
|
+
// Validate against the FINAL location so the guarantee (and the error text)
|
|
292
|
+
// is independent of where the file is staged.
|
|
208
293
|
const knowledgeDir = path.join(stashDir, "knowledge");
|
|
209
|
-
|
|
210
|
-
if (!isWithin(filePath, knowledgeDir)) {
|
|
294
|
+
if (!isWithin(path.resolve(knowledgeDir, `${relPath}.md`), knowledgeDir)) {
|
|
211
295
|
throw new UsageError(`Snapshot fetcher returned an unsafe preferred name: ${JSON.stringify(preferredName)}`);
|
|
212
296
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
297
|
+
const staging = beginSnapshotStaging(stashDir);
|
|
298
|
+
try {
|
|
299
|
+
const filePath = path.resolve(path.join(staging.dir, "knowledge"), `${relPath}.md`);
|
|
300
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
301
|
+
const slug = relPath.split("/").pop() ?? "index";
|
|
302
|
+
fs.writeFileSync(filePath, buildMarkdownSnapshot({ url: snapshot.url, title: snapshot.title, markdown: snapshot.markdown }, slug, snapshot.tags), "utf8");
|
|
303
|
+
snapshotWriteHook({ point: "page-written", index: 1, total: 1, relPath: `knowledge/${relPath}.md` });
|
|
304
|
+
publishSnapshotStaging(staging);
|
|
305
|
+
}
|
|
306
|
+
finally {
|
|
307
|
+
discardSnapshotStaging(staging);
|
|
308
|
+
}
|
|
217
309
|
}
|
|
218
310
|
async function scrapeWebsiteToStash(startUrl, stashDir, options) {
|
|
219
311
|
// Offer the URL to the specialized fetchers before falling back to a crawl.
|
|
@@ -229,19 +321,33 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
|
|
|
229
321
|
if (pages.length === 0) {
|
|
230
322
|
throw new Error(`No content could be scraped from ${startUrl}`);
|
|
231
323
|
}
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
const
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
324
|
+
const staging = beginSnapshotStaging(stashDir);
|
|
325
|
+
try {
|
|
326
|
+
const knowledgeDir = path.join(staging.dir, "knowledge");
|
|
327
|
+
fs.mkdirSync(knowledgeDir, { recursive: true });
|
|
328
|
+
const usedPaths = new Set();
|
|
329
|
+
let written = 0;
|
|
330
|
+
for (const page of pages) {
|
|
331
|
+
const relPath = avoidReservedBasename(urlToRelativePath(page.url));
|
|
332
|
+
const uniquePath = uniqueSlug(relPath, usedPaths);
|
|
333
|
+
const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
|
|
334
|
+
const dir = path.dirname(filePath);
|
|
335
|
+
if (dir !== knowledgeDir)
|
|
336
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
337
|
+
const slug = uniquePath.split("/").pop() ?? "index";
|
|
338
|
+
fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
|
|
339
|
+
written++;
|
|
340
|
+
snapshotWriteHook({
|
|
341
|
+
point: "page-written",
|
|
342
|
+
index: written,
|
|
343
|
+
total: pages.length,
|
|
344
|
+
relPath: `knowledge/${uniquePath}.md`,
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
publishSnapshotStaging(staging);
|
|
348
|
+
}
|
|
349
|
+
finally {
|
|
350
|
+
discardSnapshotStaging(staging);
|
|
245
351
|
}
|
|
246
352
|
}
|
|
247
353
|
export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
|
package/dist/storage/database.js
CHANGED
|
@@ -162,7 +162,12 @@ function loadBetterSqlite3() {
|
|
|
162
162
|
}
|
|
163
163
|
catch (err) {
|
|
164
164
|
throw new Error("akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
|
|
165
|
-
" •
|
|
165
|
+
" • If the error below says the module was compiled against a DIFFERENT Node.js\n" +
|
|
166
|
+
" version, you upgraded Node after installing akm. Reinstall akm (or run\n" +
|
|
167
|
+
" `npm rebuild better-sqlite3` in its install directory) so the binding is\n" +
|
|
168
|
+
" rebuilt for the Node you are now running. This is the common case after a\n" +
|
|
169
|
+
" Node major upgrade, and it is NOT a broken install.\n" +
|
|
170
|
+
" • Otherwise, reinstall akm with a working C/C++ build toolchain so its optional\n" +
|
|
166
171
|
" 'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
|
|
167
172
|
" will NOT be resolved — Node loads it from akm's own node_modules).\n" +
|
|
168
173
|
" • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
|
|
@@ -26,6 +26,25 @@ import { applyStandardPragmas } from "./sqlite-pragmas.js";
|
|
|
26
26
|
* Open a managed SQLite database: ensure the parent dir exists, open the handle,
|
|
27
27
|
* apply standard pragmas, then run the schema initializer. The single home for
|
|
28
28
|
* the open→pragmas→migrate recipe.
|
|
29
|
+
*
|
|
30
|
+
* ── On file permissions (reverted, issue #791) ──
|
|
31
|
+
*
|
|
32
|
+
* This function briefly chmodded the database, its `-wal`/`-shm` sidecars, and
|
|
33
|
+
* THE CONTAINING DIRECTORY to owner-only on every open (#756). That was a
|
|
34
|
+
* mistake and is deliberately not coming back:
|
|
35
|
+
*
|
|
36
|
+
* - It mutated state akm did not create. The data directory belongs to the
|
|
37
|
+
* operator; a read of the index is not consent to re-permission their disk.
|
|
38
|
+
* - It ran on the most-traveled path in the CLI, including `create: false`
|
|
39
|
+
* (read-only) opens, so any command at all silently converted a legacy
|
|
40
|
+
* `0755` directory to `0700` with no prompt, warning, or migration note.
|
|
41
|
+
* - It therefore broke installs that share `$XDG_DATA_HOME` across uids —
|
|
42
|
+
* agent sandboxes, containers, service accounts — which worked in 0.9.0.
|
|
43
|
+
* Worse, the read path answers an unreadable index with a false
|
|
44
|
+
* "No search index available" at exit 0 rather than an error (#791).
|
|
45
|
+
*
|
|
46
|
+
* Files akm creates here get the process umask, which is the operator's lever
|
|
47
|
+
* for this and always was. akm neither sets these modes nor reports on them.
|
|
29
48
|
*/
|
|
30
49
|
export function openManagedDatabase(spec) {
|
|
31
50
|
const dir = path.dirname(spec.path);
|
|
@@ -10,8 +10,9 @@
|
|
|
10
10
|
* import their opener from a sibling here instead of reaching up into the
|
|
11
11
|
* indexer — inverting the old storage→indexer arrow.
|
|
12
12
|
*/
|
|
13
|
-
import fs from "node:fs";
|
|
14
13
|
import { createRequire } from "node:module";
|
|
14
|
+
import { ConfigError } from "../../core/errors.js";
|
|
15
|
+
import { classifyPathAccess, describeInaccessiblePath } from "../../core/path-access.js";
|
|
15
16
|
import { getDbPath } from "../../core/paths.js";
|
|
16
17
|
import { openDatabase } from "../database.js";
|
|
17
18
|
import { openManagedDatabase } from "../managed-db.js";
|
|
@@ -71,18 +72,42 @@ export function openExistingDatabase(dbPath) {
|
|
|
71
72
|
// tests/storage/open-existing-database-no-create.test.ts. `create: false`
|
|
72
73
|
// below is the race-free backstop for this pre-check.
|
|
73
74
|
const resolvedPath = dbPath ?? getDbPath();
|
|
74
|
-
|
|
75
|
+
assertIndexPathReadable(resolvedPath);
|
|
76
|
+
if (classifyPathAccess(resolvedPath).access === "absent") {
|
|
75
77
|
throw new Error(`Index database not found at ${resolvedPath}. Run 'akm index' to build it.`);
|
|
76
78
|
}
|
|
77
79
|
return openManagedDatabase({ path: resolvedPath, init: loadVecExtension, create: false });
|
|
78
80
|
}
|
|
81
|
+
/**
|
|
82
|
+
* Refuse to treat an UNREADABLE index as a missing one (#791).
|
|
83
|
+
*
|
|
84
|
+
* `fs.existsSync()` — which every one of these gates used to call — returns
|
|
85
|
+
* `false` for `EACCES` exactly as for `ENOENT`, so an index this process cannot
|
|
86
|
+
* read looked identical to one that had never been built. Callers then took
|
|
87
|
+
* their "no index yet" branch: `search`/`curate` returned no hits at exit 0 and
|
|
88
|
+
* told the user to run `akm index`, which would not have helped and which they
|
|
89
|
+
* may not have permission to do either.
|
|
90
|
+
*
|
|
91
|
+
* A `ConfigError` here exits 78 through the standard `{ok:false, error, code}`
|
|
92
|
+
* envelope, so both a human and a machine caller can tell "nothing indexed"
|
|
93
|
+
* from "I cannot see the index".
|
|
94
|
+
*/
|
|
95
|
+
export function assertIndexPathReadable(resolvedPath) {
|
|
96
|
+
const { access, code } = classifyPathAccess(resolvedPath);
|
|
97
|
+
if (access !== "inaccessible")
|
|
98
|
+
return;
|
|
99
|
+
throw new ConfigError(`Index database exists but is not readable: ${describeInaccessiblePath(resolvedPath, code)}.`, "DATA_DIR_UNREADABLE");
|
|
100
|
+
}
|
|
79
101
|
/**
|
|
80
102
|
* Open an existing index for queries without creating directories, a database
|
|
81
103
|
* file, journals, or running write-capable pragmas/schema initialization.
|
|
82
104
|
*/
|
|
83
105
|
export function openReadonlyExistingDatabase(dbPath) {
|
|
84
106
|
const resolvedPath = dbPath ?? getDbPath();
|
|
85
|
-
|
|
107
|
+
// `undefined` means "no index" — reserve it for a genuinely absent one, and
|
|
108
|
+
// let an unreadable index raise instead of masquerading as absent (#791).
|
|
109
|
+
assertIndexPathReadable(resolvedPath);
|
|
110
|
+
if (classifyPathAccess(resolvedPath).access === "absent")
|
|
86
111
|
return undefined;
|
|
87
112
|
return openDatabase(resolvedPath, { readonly: true, create: false });
|
|
88
113
|
}
|
|
@@ -13,6 +13,7 @@ import path from "node:path";
|
|
|
13
13
|
import { parseBundleRef } from "../../core/asset/asset-ref.js";
|
|
14
14
|
import { conceptIdFromTypeName } from "../../core/asset/resolve-ref.js";
|
|
15
15
|
import { bestEffort } from "../../core/best-effort.js";
|
|
16
|
+
import { isPathAbsent } from "../../core/path-access.js";
|
|
16
17
|
import { getStateDbPath, withStateDb } from "../../core/state-db.js";
|
|
17
18
|
import { warn } from "../../core/warn.js";
|
|
18
19
|
import { buildSearchText } from "../../indexer/search/search-fields.js";
|
|
@@ -278,7 +279,11 @@ export function rekeyEntryInPlace(db, opts) {
|
|
|
278
279
|
* history (live-asset-wins). Best-effort + guarded on state.db's existence.
|
|
279
280
|
*/
|
|
280
281
|
function rewriteUsageEventRefForMove(opts) {
|
|
281
|
-
|
|
282
|
+
// Every other failure in here throws (see the catch below) precisely because
|
|
283
|
+
// a move that quietly drops its usage history is a wrong answer wearing a
|
|
284
|
+
// success. An unreadable state.db must not be the one silent exception —
|
|
285
|
+
// only a state.db that was never created skips (#791).
|
|
286
|
+
if (isPathAbsent(getStateDbPath()))
|
|
282
287
|
return;
|
|
283
288
|
// `usage_events.entry_ref` is the fully-qualified item_ref
|
|
284
289
|
// (`<bundle>//<conceptId>`).
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import { openStateDatabase, withImmediateTransaction } from "../../core/state-db.js";
|
|
5
|
+
import { borrowScopedStateDb, withStateDbScope } from "../../core/state-db-scope.js";
|
|
5
6
|
import { resolveStorageLocations } from "../locations.js";
|
|
6
7
|
/**
|
|
7
8
|
* Repository owning every raw SQL statement against `workflow_runs` and
|
|
@@ -214,8 +215,10 @@ export class WorkflowRunsRepository {
|
|
|
214
215
|
//
|
|
215
216
|
// Writes to `workflow_run_units` should go through the serialized writer
|
|
216
217
|
// queue (`src/workflows/exec/unit-writer.ts`) when N units may complete
|
|
217
|
-
// concurrently — SQLite has a single writer and
|
|
218
|
-
//
|
|
218
|
+
// concurrently — SQLite has a single writer per database FILE, and outside a
|
|
219
|
+
// {@link withWorkflowRunsConnection} scope `withWorkflowRunsRepo` opens a
|
|
220
|
+
// fresh connection per call (so N concurrent writers would contend against
|
|
221
|
+
// each other for the write lock).
|
|
219
222
|
getUnitsForRun(runId) {
|
|
220
223
|
return this.db
|
|
221
224
|
.prepare("SELECT * FROM workflow_run_units WHERE run_id = ? ORDER BY started_at ASC, unit_id ASC")
|
|
@@ -323,22 +326,55 @@ export class WorkflowRunsRepository {
|
|
|
323
326
|
`recoverable state.`);
|
|
324
327
|
}
|
|
325
328
|
}
|
|
329
|
+
/**
|
|
330
|
+
* Finish a unit row ONLY while it is still the exact row a specific dispatch
|
|
331
|
+
* inserted: `running`, with that dispatch's `started_at`. The native
|
|
332
|
+
* executor's guarded finish (single-driver invariant): a run stolen by
|
|
333
|
+
* another engine re-dispatches the unit through {@link insertUnit}, which
|
|
334
|
+
* REPLACES the row (fresh `started_at`, bumped `attempts`) — the stale
|
|
335
|
+
* driver's finish then matches NOTHING instead of clobbering the new
|
|
336
|
+
* driver's live dispatch. Returns whether the row was finished; a zero-row
|
|
337
|
+
* match is a caller-classified outcome here (the executor distinguishes
|
|
338
|
+
* "replaced by another driver" from "row vanished"), unlike
|
|
339
|
+
* {@link finishUnit}'s loud throw, whose callers guarantee their row exists.
|
|
340
|
+
*/
|
|
341
|
+
finishUnitFromDispatch(input) {
|
|
342
|
+
const result = this.db
|
|
343
|
+
.prepare(`UPDATE workflow_run_units
|
|
344
|
+
SET status = ?, result_json = ?, tokens = ?, failure_reason = ?, session_id = ?, finished_at = ?
|
|
345
|
+
WHERE run_id = ? AND unit_id = ? AND status = 'running' AND started_at = ?`)
|
|
346
|
+
.run(input.status, input.resultJson, input.tokens, input.failureReason, input.sessionId ?? null, input.finishedAt, input.runId, input.unitId, input.dispatchStartedAt);
|
|
347
|
+
return Number(result.changes) === 1;
|
|
348
|
+
}
|
|
326
349
|
}
|
|
327
350
|
/**
|
|
328
|
-
*
|
|
329
|
-
*
|
|
330
|
-
*
|
|
331
|
-
* connection exactly once when `fn` settles.
|
|
351
|
+
* Run `fn` against a {@link WorkflowRunsRepository} bound to state.db
|
|
352
|
+
* ({@link StorageLocations.stateDb}, the post-cutover home of the
|
|
353
|
+
* `workflow_runs` / `workflow_run_steps` / `workflow_run_units` tables).
|
|
332
354
|
*
|
|
333
|
-
*
|
|
334
|
-
* `
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
*
|
|
338
|
-
*
|
|
355
|
+
* Connection lifetime — BORROW-OR-OWN (mirrors `withStateDb`'s `borrowed`
|
|
356
|
+
* option and `appendEvent`'s `ctx.db` seam):
|
|
357
|
+
*
|
|
358
|
+
* - Inside a {@link withWorkflowRunsConnection} scope, the ambient handle is
|
|
359
|
+
* BORROWED and left open for the rest of the scope. A wide `map` fan-out
|
|
360
|
+
* therefore opens ONE connection for the whole step instead of two per unit
|
|
361
|
+
* (insert + finish) — `openStateDatabase` registers a maintenance activity
|
|
362
|
+
* lockfile and opens a read-only ledger-preflight handle on every call, so
|
|
363
|
+
* the per-call cost is milliseconds, not microseconds.
|
|
364
|
+
* - Outside a scope the behaviour is unchanged: open a fresh connection, run
|
|
365
|
+
* `fn`, close it in a `finally`.
|
|
366
|
+
*
|
|
367
|
+
* Repository read methods fully materialise their results, so closing an owned
|
|
368
|
+
* handle here never truncates lazy iteration (WS5 connection-lifetime rule).
|
|
369
|
+
* The signature and semantics are identical in both modes — reuse is purely an
|
|
370
|
+
* internal optimisation and no caller needs to know which mode it is in.
|
|
339
371
|
*/
|
|
340
372
|
export async function withWorkflowRunsRepo(fn) {
|
|
341
|
-
const
|
|
373
|
+
const stateDb = resolveStorageLocations().stateDb;
|
|
374
|
+
const borrowed = borrowScopedStateDb(stateDb);
|
|
375
|
+
if (borrowed)
|
|
376
|
+
return await Promise.resolve(fn(new WorkflowRunsRepository(borrowed)));
|
|
377
|
+
const db = openStateDatabase(stateDb);
|
|
342
378
|
try {
|
|
343
379
|
return await Promise.resolve(fn(new WorkflowRunsRepository(db)));
|
|
344
380
|
}
|
|
@@ -346,3 +382,20 @@ export async function withWorkflowRunsRepo(fn) {
|
|
|
346
382
|
db.close();
|
|
347
383
|
}
|
|
348
384
|
}
|
|
385
|
+
/**
|
|
386
|
+
* Run `fn` with ONE state.db connection shared by every `withWorkflowRunsRepo`
|
|
387
|
+
* call (and every {@link import("../../core/events.js").appendEvent}) inside its
|
|
388
|
+
* async extent. The handle opens on first use and closes when `fn` settles;
|
|
389
|
+
* nesting joins the outer scope.
|
|
390
|
+
*
|
|
391
|
+
* Correctness under concurrency: `bun:sqlite` statements and
|
|
392
|
+
* `withImmediateTransaction` bodies run synchronously to completion, so
|
|
393
|
+
* logically concurrent units cannot interleave statements on the shared handle
|
|
394
|
+
* in a single-threaded event loop — sharing REMOVES in-process writer
|
|
395
|
+
* contention instead of creating it. Cross-process arbitration (WAL,
|
|
396
|
+
* `busy_timeout`, the run lease) is untouched. See `core/state-db-scope.ts` for
|
|
397
|
+
* the escaped-async-work guard.
|
|
398
|
+
*/
|
|
399
|
+
export function withWorkflowRunsConnection(fn) {
|
|
400
|
+
return withStateDbScope(fn, { path: resolveStorageLocations().stateDb });
|
|
401
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Decide which exact values must be scrubbed from a task's persisted run log
|
|
6
|
+
* (issue #755).
|
|
7
|
+
*
|
|
8
|
+
* # The gap
|
|
9
|
+
*
|
|
10
|
+
* All three task target kinds funnel through `persistRunLog`, which applied
|
|
11
|
+
* only `redactCredentialPatterns` — credential *shapes* (`Bearer …`, `sk-…`,
|
|
12
|
+
* webhook URLs). Prompt- and workflow-target runs additionally redact the exact
|
|
13
|
+
* secret values reachable by the run before their output ever reaches the log.
|
|
14
|
+
* Command-target runs did not: a scheduled command that echoed a configured
|
|
15
|
+
* secret whose value is not credential-shaped persisted it verbatim, into both
|
|
16
|
+
* the `.log` file and `logs.db`, for the whole retention window.
|
|
17
|
+
*
|
|
18
|
+
* # Why the obvious fix is wrong
|
|
19
|
+
*
|
|
20
|
+
* The issue proposed reusing {@link isEnvPassthroughValueSafeToExpose} — the
|
|
21
|
+
* filter the prompt path uses — over the env handed to the child. That filter
|
|
22
|
+
* fails CLOSED for any name outside a 22-entry allowlist, which is right where
|
|
23
|
+
* it currently runs: the prompt path filters `envPassthrough`, a short list the
|
|
24
|
+
* operator explicitly declared. A command task inherits the WHOLE ambient
|
|
25
|
+
* environment, so the same rule classifies essentially everything as secret.
|
|
26
|
+
* Measured on a developer machine: 127 of 132 variables, 25 of them with
|
|
27
|
+
* one-character values (`SHLVL=1`, `OLDPWD=/`, `GIT_TERMINAL_PROMPT=0`).
|
|
28
|
+
* Redaction is substring replacement, so those become live needles:
|
|
29
|
+
*
|
|
30
|
+
* Build finished in 12.4s -> Build finished in [REDACTED]2.[REDACTED]s
|
|
31
|
+
* 3 tests passed, 0 failed -> [REDACTED] tests passed, [REDACTED] failed
|
|
32
|
+
* wrote dist/index.js (48 KB) -> wrote dist[REDACTED]index.js ([REDACTED]8 KB)
|
|
33
|
+
*
|
|
34
|
+
* A fix that destroys every command log is not a fix.
|
|
35
|
+
*
|
|
36
|
+
* # What this does instead
|
|
37
|
+
*
|
|
38
|
+
* Three sources, and the distinction between them is the whole design:
|
|
39
|
+
*
|
|
40
|
+
* 1. **Declared by config** — the config names which variables hold
|
|
41
|
+
* credentials (`engines.<n>.apiKey: ${VAR}`, `embedding.apiKey`, and the
|
|
42
|
+
* implicit `AKM_ENGINE_<NAME>_API_KEY` / `AKM_LLM_API_KEY` /
|
|
43
|
+
* `AKM_EMBED_API_KEY` recipes). akm KNOWS these are secret.
|
|
44
|
+
* 2. **Declared by the task** — the `redact:` list, names only.
|
|
45
|
+
* 3. **Inferred** — a name-shape heuristic over the remaining environment, for
|
|
46
|
+
* the ambient credential akm was never told about.
|
|
47
|
+
*
|
|
48
|
+
* Only (3) is a guess, so only (3) carries {@link MIN_INFERRED_SECRET_LENGTH}.
|
|
49
|
+
* A declared secret is redacted at ANY length, because the operator told us
|
|
50
|
+
* what it is; applying a floor to declared values would silently stop redacting
|
|
51
|
+
* short secrets that are scrubbed today. The floor exists solely to stop a
|
|
52
|
+
* *guess* from mangling a log, and 8 clears every real credential format (AWS
|
|
53
|
+
* key id 20, GitHub PAT 40, `sk-…` 40+) while excluding the flags and counters
|
|
54
|
+
* that a name heuristic occasionally catches.
|
|
55
|
+
*
|
|
56
|
+
* Note what is deliberately NOT collected: the akm secret store on disk. A
|
|
57
|
+
* spawned command sees `process.env`, not akm's stores — a stored secret can
|
|
58
|
+
* only be echoed if it is already in the environment, where the rules above
|
|
59
|
+
* catch it by name. Walking every bundle's `secrets/` on each task firing would
|
|
60
|
+
* cost a recursive readdir plus an unbounded read for values the child cannot
|
|
61
|
+
* reach anyway. `redact:` is the escape hatch for a secret injected under a
|
|
62
|
+
* name none of the rules recognise.
|
|
63
|
+
*/
|
|
64
|
+
import { collectSensitiveValues, isEnvPassthroughValueSafeToExpose } from "../core/redaction.js";
|
|
65
|
+
import { collectEngineCredentialValues } from "../integrations/agent/engine-resolution.js";
|
|
66
|
+
/**
|
|
67
|
+
* Shortest value an INFERRED (name-heuristic) match may contribute as a
|
|
68
|
+
* redaction needle. Declared secrets bypass this entirely.
|
|
69
|
+
*
|
|
70
|
+
* Redaction replaces substrings, so a short needle is not merely useless — it
|
|
71
|
+
* corrupts unrelated output. Below 8 the noise tier is fully intact (`1`, `0`,
|
|
72
|
+
* `/`, `80`, `true`, `xhigh`, `31999`); at 8 a chance collision with ordinary
|
|
73
|
+
* log vocabulary is negligible, and every credential format in real use is far
|
|
74
|
+
* longer. It also matches the conventional minimum password length, so a
|
|
75
|
+
* secret shorter than this is already outside normal policy.
|
|
76
|
+
*/
|
|
77
|
+
export const MIN_INFERRED_SECRET_LENGTH = 8;
|
|
78
|
+
/**
|
|
79
|
+
* Environment names whose VALUE is treated as a credential on shape alone.
|
|
80
|
+
*
|
|
81
|
+
* The keyword must be a whole `_`-delimited word. Anchoring on only one side
|
|
82
|
+
* would drag in ordinary configuration from whichever side is left open:
|
|
83
|
+
* a leading anchor alone matches `KEYBOARD_LAYOUT` and `AUTHOR`, a trailing one
|
|
84
|
+
* matches `MONKEY` and `BYPASS`. Whole-word matching gets `GH_TOKEN`,
|
|
85
|
+
* `NPM_AUTH_TOKEN`, `MY_API_KEY`, `DB_PASS` and `AWS_SECRET_ACCESS_KEY` right.
|
|
86
|
+
*/
|
|
87
|
+
const INFERRED_SECRET_NAME = /(?:^|_)(?:API_?KEY|KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASS|CREDENTIALS?|AUTH)(?:_|$)/i;
|
|
88
|
+
/**
|
|
89
|
+
* Credential variables whose name is a single glued word, which no
|
|
90
|
+
* word-boundary rule can see. Enumerated rather than matched: loosening the
|
|
91
|
+
* pattern enough to catch `PGPASSWORD` also catches `MONKEY`.
|
|
92
|
+
*
|
|
93
|
+
* `PWD` cannot be a keyword above for the same reason it appears here as part
|
|
94
|
+
* of `MYSQL_PWD` — on its own it is the working directory.
|
|
95
|
+
*/
|
|
96
|
+
const KNOWN_SECRET_NAMES = new Set(["PGPASSWORD", "MYSQL_PWD"]);
|
|
97
|
+
/** True when the NAME alone marks this variable as holding a credential. */
|
|
98
|
+
export function isInferredSecretName(name) {
|
|
99
|
+
return KNOWN_SECRET_NAMES.has(name.toUpperCase()) || INFERRED_SECRET_NAME.test(name);
|
|
100
|
+
}
|
|
101
|
+
/** Resolve `${VAR}` / `$VAR` to the variable NAME, or undefined for anything else. */
|
|
102
|
+
function envRefName(spec) {
|
|
103
|
+
if (!spec)
|
|
104
|
+
return undefined;
|
|
105
|
+
const match = /^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$/.exec(spec.trim());
|
|
106
|
+
return match?.[1];
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Every value that must be scrubbed from one task's persisted output.
|
|
110
|
+
*
|
|
111
|
+
* `declaredNames` comes from the task's `redact:` list. A name that is unset in
|
|
112
|
+
* `env` simply contributes nothing — naming a variable you do not currently
|
|
113
|
+
* export is not an error.
|
|
114
|
+
*/
|
|
115
|
+
export function collectTaskLogSensitiveValues(input) {
|
|
116
|
+
const { env, config, declaredNames } = input;
|
|
117
|
+
const values = new Set();
|
|
118
|
+
const addDeclared = (value) => {
|
|
119
|
+
if (value === undefined)
|
|
120
|
+
return;
|
|
121
|
+
const trimmed = value.trim();
|
|
122
|
+
// Both spellings: `resolveSecret` does not trim but `resolveCredentialFromEnv`
|
|
123
|
+
// does, so a variable with trailing whitespace reaches an output boundary in
|
|
124
|
+
// either form depending on which path materialized it.
|
|
125
|
+
if (value.length > 0)
|
|
126
|
+
values.add(value);
|
|
127
|
+
if (trimmed.length > 0)
|
|
128
|
+
values.add(trimmed);
|
|
129
|
+
};
|
|
130
|
+
// (1) Declared by config — engine credentials, via the collector the prompt
|
|
131
|
+
// path already uses, plus the embedding key it does not cover.
|
|
132
|
+
if (config) {
|
|
133
|
+
for (const value of collectEngineCredentialValues(config, env))
|
|
134
|
+
values.add(value);
|
|
135
|
+
addDeclared(env[envRefName(config.embedding?.apiKey) ?? "AKM_EMBED_API_KEY"]);
|
|
136
|
+
}
|
|
137
|
+
// (2) Declared by the task's `redact:` list — names only, never values.
|
|
138
|
+
for (const name of declaredNames ?? [])
|
|
139
|
+
addDeclared(env[name]);
|
|
140
|
+
// (3) Inferred from the name shape. The only guessing tier, so the only one
|
|
141
|
+
// with a length floor — and still subject to the value-level check that keeps
|
|
142
|
+
// an allowlisted name from being treated as secret.
|
|
143
|
+
for (const [name, value] of Object.entries(env)) {
|
|
144
|
+
if (value === undefined || value.length < MIN_INFERRED_SECRET_LENGTH)
|
|
145
|
+
continue;
|
|
146
|
+
if (!isInferredSecretName(name))
|
|
147
|
+
continue;
|
|
148
|
+
if (isEnvPassthroughValueSafeToExpose(name, value))
|
|
149
|
+
continue;
|
|
150
|
+
values.add(value);
|
|
151
|
+
}
|
|
152
|
+
// Expands credential-bearing URLs into their embedded components. Can yield
|
|
153
|
+
// needles shorter than the floor (a URL's password), which is correct: the
|
|
154
|
+
// operator's own value implied them.
|
|
155
|
+
return collectSensitiveValues(values);
|
|
156
|
+
}
|