akm-cli 0.9.0 → 0.9.1-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +707 -0
  2. package/README.md +28 -63
  3. package/STABILITY.md +4 -2
  4. package/dist/commands/agent/contribute-cli.js +1 -1
  5. package/dist/commands/feedback-cli.js +7 -1
  6. package/dist/commands/health/llm-usage.js +2 -1
  7. package/dist/commands/health/surfaces.js +4 -77
  8. package/dist/commands/health.js +65 -11
  9. package/dist/commands/improve/distill/quality-gate.js +6 -1
  10. package/dist/commands/improve/eligibility.js +7 -1
  11. package/dist/commands/improve/improve.js +126 -10
  12. package/dist/commands/improve/locks.js +7 -0
  13. package/dist/commands/improve/memory/memory-improve.js +8 -0
  14. package/dist/commands/improve/run-context.js +5 -0
  15. package/dist/commands/improve/session-asset.js +4 -0
  16. package/dist/commands/lint/base-linter.js +31 -7
  17. package/dist/commands/lint/index.js +200 -50
  18. package/dist/commands/lint/types.js +22 -1
  19. package/dist/commands/proposal/repository.js +17 -1
  20. package/dist/commands/sources/info.js +12 -2
  21. package/dist/commands/sources/installed-stashes.js +6 -1
  22. package/dist/commands/tasks/tasks.js +8 -2
  23. package/dist/commands/workflow-cli.js +17 -11
  24. package/dist/core/abort-deadline.js +28 -0
  25. package/dist/core/adapter/adapters/agent-skills-adapter.js +67 -5
  26. package/dist/core/adapter/adapters/akm-adapter.js +13 -10
  27. package/dist/core/adapter/adapters/akm-lint.js +72 -20
  28. package/dist/core/adapter/adapters/akm-task-adapter.js +40 -19
  29. package/dist/core/adapter/adapters/tool-dir-shared.js +5 -3
  30. package/dist/core/asset/frontmatter.js +4 -0
  31. package/dist/core/common.js +66 -6
  32. package/dist/core/concurrent.js +32 -0
  33. package/dist/core/config/schema/workflow.js +11 -0
  34. package/dist/core/errors.js +25 -0
  35. package/dist/core/events.js +30 -24
  36. package/dist/core/file-lock.js +7 -1
  37. package/dist/core/improve-result.js +5 -0
  38. package/dist/core/json-schema.js +327 -9
  39. package/dist/core/loopback.js +89 -0
  40. package/dist/core/migration-operation.js +17 -2
  41. package/dist/core/path-access.js +107 -0
  42. package/dist/core/redaction.js +64 -17
  43. package/dist/core/spawn-env.js +234 -0
  44. package/dist/core/state-db-scope.js +134 -0
  45. package/dist/core/subprocess.js +181 -37
  46. package/dist/core/write-provenance.js +85 -0
  47. package/dist/core/write-source.js +7 -0
  48. package/dist/indexer/db/graph-db.js +17 -6
  49. package/dist/indexer/ensure-index.js +10 -3
  50. package/dist/indexer/index-written-assets.js +17 -2
  51. package/dist/indexer/indexer.js +55 -15
  52. package/dist/indexer/passes/memory-inference.js +4 -0
  53. package/dist/indexer/search/db-search.js +8 -15
  54. package/dist/integrations/agent/engine-resolution.js +24 -11
  55. package/dist/integrations/agent/model-aliases.js +1 -1
  56. package/dist/integrations/agent/profiles.js +9 -1
  57. package/dist/integrations/agent/spawn.js +15 -87
  58. package/dist/integrations/lockfile.js +45 -2
  59. package/dist/output/text/lint-format.js +17 -4
  60. package/dist/scripts/akm-migrate-node.js +1435 -753
  61. package/dist/scripts/akm-migrate.js +1403 -721
  62. package/dist/sources/providers/git-stash.js +19 -0
  63. package/dist/sources/providers/git.js +1 -1
  64. package/dist/sources/snapshot-fetchers/website-ingest.js +126 -20
  65. package/dist/storage/database.js +6 -1
  66. package/dist/storage/managed-db.js +19 -0
  67. package/dist/storage/repositories/index-connection.js +28 -3
  68. package/dist/storage/repositories/index-entries-repository.js +6 -1
  69. package/dist/storage/repositories/workflow-runs-repository.js +66 -13
  70. package/dist/tasks/log-redaction.js +156 -0
  71. package/dist/tasks/parser.js +82 -5
  72. package/dist/tasks/runner.js +139 -11
  73. package/dist/tasks/schema.js +65 -0
  74. package/dist/workflows/concurrency-policy.js +95 -1
  75. package/dist/workflows/exec/dispatch-redaction.js +114 -0
  76. package/dist/workflows/exec/exec-unit.js +542 -0
  77. package/dist/workflows/exec/frozen-judge.js +114 -42
  78. package/dist/workflows/exec/native-executor.js +457 -238
  79. package/dist/workflows/exec/param-secrets.js +4 -3
  80. package/dist/workflows/exec/run-workflow.js +424 -219
  81. package/dist/workflows/exec/step-work.js +496 -165
  82. package/dist/workflows/exec/unit-dispatch.js +31 -1
  83. package/dist/workflows/exec/unit-writer.js +53 -13
  84. package/dist/workflows/exec/worktree.js +454 -41
  85. package/dist/workflows/ir/compile.js +26 -2
  86. package/dist/workflows/ir/freeze.js +82 -15
  87. package/dist/workflows/ir/schema.js +105 -20
  88. package/dist/workflows/parser.js +216 -18
  89. package/dist/workflows/program/schema.js +24 -0
  90. package/dist/workflows/renderer.js +32 -4
  91. package/dist/workflows/resource-limits.js +182 -0
  92. package/dist/workflows/runtime/runs.js +146 -6
  93. package/dist/workflows/validate-summary.js +17 -2
  94. package/docs/README.md +74 -32
  95. package/docs/migration/release-notes/0.9.0.md +2 -1
  96. package/docs/migration/v0.7-to-v0.8.md +2 -1
  97. package/docs/migration/v0.8-to-v0.9.md +3 -1
  98. package/docs/reference/README.md +11 -4
  99. package/docs/reference/bundle-types.md +19 -0
  100. package/docs/reference/cli.md +105 -16
  101. package/docs/reference/configuration.md +15 -2
  102. package/docs/reference/data-and-telemetry.md +30 -10
  103. package/docs/reference/supported-formats.md +50 -0
  104. package/docs/reference/workflow-schema.md +1014 -0
  105. package/docs/reference/workflows.md +37 -633
  106. package/package.json +13 -6
  107. package/schemas/akm-config.json +8 -0
  108. package/schemas/akm-task.json +27 -5
  109. package/schemas/akm-workflow.json +85 -10
@@ -56,6 +56,21 @@ export class GitStashPushError extends Error {
56
56
  }
57
57
  const GIT_PUSH_TIMEOUT_MS = 120_000;
58
58
  const ZERO_OID = "0000000000000000000000000000000000000000";
59
+ let exactCommitHookForTests;
60
+ /**
61
+ * TEST-ONLY. Interleave work into the exact-path commit sequence; `undefined`
62
+ * restores. Exists because the CAS window is a few microseconds wide between
63
+ * two synchronous `git` invocations — a wall-clock race would be
64
+ * non-deterministic, and every earlier pre-check would swallow a commit that
65
+ * landed before the window opened. Inert in production.
66
+ */
67
+ export function _setGitExactCommitHookForTests(hook) {
68
+ exactCommitHookForTests = hook;
69
+ }
70
+ /** Fire a named exact-path-commit race point (no-op outside tests). */
71
+ function gitExactCommitHook(point) {
72
+ exactCommitHookForTests?.(point);
73
+ }
59
74
  /**
60
75
  * Resolve the writable flag for an end-of-run / `akm sync` commit from the
61
76
  * configured default bundle.
@@ -523,6 +538,10 @@ function createExactPathCommit(repoDir, options) {
523
538
  throw new Error(`Git target changed before its exact commit could be attached.`);
524
539
  }
525
540
  assertWorktreeMatchesExpected(repoDir, options.paths, expected);
541
+ // Race window: everything below this line is defended only by the
542
+ // update-ref compare-and-swap. See
543
+ // tests/integration/sync-exact-commit-cas.test.ts.
544
+ gitExactCommitHook("before-update-ref");
526
545
  const update = runGit(["-C", repoDir, "update-ref", branchRef, commitOid, options.baseHead ?? ZERO_OID]);
527
546
  if (update.status !== 0) {
528
547
  throw new Error(`Git target advanced before its exact commit could be attached.`);
@@ -10,4 +10,4 @@
10
10
  // keeps importing from a single module namespace.
11
11
  export { classifyCloneFailure, cloneRepo, inspectGitUpstream, runGit, syncExistingWritableCheckout, } from "./git-install.js";
12
12
  export { ensureGitMirror, GitSourceProvider, getCachePaths, parseGitRepoUrl, syncMirroredRepo, } from "./git-provider.js";
13
- export { GitStashPushError, isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "./git-stash.js";
13
+ export { _setGitExactCommitHookForTests, GitStashPushError, isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "./git-stash.js";
@@ -1,7 +1,7 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
- import { createHash } from "node:crypto";
4
+ import { createHash, randomBytes } from "node:crypto";
5
5
  import fs from "node:fs";
6
6
  import path from "node:path";
7
7
  import { fetchWithRetry, isWithin, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir, todayIso, } from "../../core/common.js";
@@ -201,19 +201,111 @@ async function fetchSnapshotViaRegistry(startUrl, stashDir, allowPrivateHosts, r
201
201
  };
202
202
  return dispatchSnapshotFetchers(parsed, context, stashDir);
203
203
  }
204
+ let snapshotWriteHookForTests;
205
+ /**
206
+ * TEST-ONLY. Interrupt a refresh partway through its page-write loop;
207
+ * `undefined` restores. Exists because "a process killed mid-refresh" cannot
208
+ * be staged from outside the module — the whole loop is a single synchronous
209
+ * burst between two awaits. Inert in production (one `undefined?.()` per page).
210
+ */
211
+ export function _setWebsiteSnapshotWriteHookForTests(hook) {
212
+ snapshotWriteHookForTests = hook;
213
+ }
214
+ function snapshotWriteHook(event) {
215
+ snapshotWriteHookForTests?.(event);
216
+ }
217
+ function snapshotSiblingPrefix(stashDir, kind) {
218
+ return `.${path.basename(stashDir)}.${kind}-`;
219
+ }
220
+ function snapshotSiblingPath(stashDir, kind) {
221
+ const unique = `${process.pid}-${randomBytes(6).toString("hex")}`;
222
+ return path.join(path.dirname(stashDir), `${snapshotSiblingPrefix(stashDir, kind)}${unique}`);
223
+ }
224
+ /**
225
+ * Age gate for the staging sweep. Nothing enforces one refresh at a time for a
226
+ * given website source, so a sibling directory may belong to a refresh that is
227
+ * still running in another process; deleting it would break a healthy run
228
+ * instead of cleaning up after a dead one. Only clearly-abandoned directories
229
+ * (untouched for an hour — far longer than the 10-minute crawl wall-clock cap)
230
+ * are swept. Leftovers are inert until then: they are dot-prefixed, so the
231
+ * indexer's walk skips them.
232
+ */
233
+ const SNAPSHOT_STAGING_SWEEP_AGE_MS = 60 * 60 * 1000;
234
+ /** Remove staging/retired directories abandoned by an earlier interrupted run. */
235
+ function sweepSnapshotStaging(stashDir) {
236
+ const parent = path.dirname(stashDir);
237
+ let entries;
238
+ try {
239
+ entries = fs.readdirSync(parent);
240
+ }
241
+ catch {
242
+ return;
243
+ }
244
+ const prefixes = [snapshotSiblingPrefix(stashDir, "staging"), snapshotSiblingPrefix(stashDir, "retired")];
245
+ const cutoff = Date.now() - SNAPSHOT_STAGING_SWEEP_AGE_MS;
246
+ for (const entry of entries) {
247
+ if (!prefixes.some((prefix) => entry.startsWith(prefix)))
248
+ continue;
249
+ const abandoned = path.join(parent, entry);
250
+ try {
251
+ if (fs.statSync(abandoned).mtimeMs > cutoff)
252
+ continue;
253
+ }
254
+ catch {
255
+ continue;
256
+ }
257
+ fs.rmSync(abandoned, { recursive: true, force: true });
258
+ }
259
+ }
260
+ function beginSnapshotStaging(stashDir) {
261
+ fs.mkdirSync(path.dirname(stashDir), { recursive: true });
262
+ sweepSnapshotStaging(stashDir);
263
+ const dir = snapshotSiblingPath(stashDir, "staging");
264
+ fs.mkdirSync(dir, { recursive: true });
265
+ return { dir, target: stashDir };
266
+ }
267
+ /**
268
+ * Swap the staged snapshot in. POSIX cannot atomically exchange two non-empty
269
+ * directories, so the previous snapshot is renamed ASIDE first and deleted
270
+ * afterwards: the window in which the target does not exist is one syscall
271
+ * wide instead of an entire write loop.
272
+ */
273
+ function publishSnapshotStaging(staging) {
274
+ let retired;
275
+ if (fs.existsSync(staging.target)) {
276
+ retired = snapshotSiblingPath(staging.target, "retired");
277
+ fs.renameSync(staging.target, retired);
278
+ }
279
+ fs.renameSync(staging.dir, staging.target);
280
+ if (retired)
281
+ fs.rmSync(retired, { recursive: true, force: true });
282
+ }
283
+ /** Drop an unpublished staging directory (no-op once it has been renamed). */
284
+ function discardSnapshotStaging(staging) {
285
+ fs.rmSync(staging.dir, { recursive: true, force: true });
286
+ }
204
287
  /** Materialize a single fetcher snapshot as the source's whole stash. */
205
288
  function writeSnapshotToStash(stashDir, snapshot) {
206
289
  const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
207
290
  const relPath = avoidReservedBasename(preferredName);
291
+ // Validate against the FINAL location so the guarantee (and the error text)
292
+ // is independent of where the file is staged.
208
293
  const knowledgeDir = path.join(stashDir, "knowledge");
209
- const filePath = path.resolve(knowledgeDir, `${relPath}.md`);
210
- if (!isWithin(filePath, knowledgeDir)) {
294
+ if (!isWithin(path.resolve(knowledgeDir, `${relPath}.md`), knowledgeDir)) {
211
295
  throw new UsageError(`Snapshot fetcher returned an unsafe preferred name: ${JSON.stringify(preferredName)}`);
212
296
  }
213
- fs.rmSync(stashDir, { recursive: true, force: true });
214
- fs.mkdirSync(path.dirname(filePath), { recursive: true });
215
- const slug = relPath.split("/").pop() ?? "index";
216
- fs.writeFileSync(filePath, buildMarkdownSnapshot({ url: snapshot.url, title: snapshot.title, markdown: snapshot.markdown }, slug, snapshot.tags), "utf8");
297
+ const staging = beginSnapshotStaging(stashDir);
298
+ try {
299
+ const filePath = path.resolve(path.join(staging.dir, "knowledge"), `${relPath}.md`);
300
+ fs.mkdirSync(path.dirname(filePath), { recursive: true });
301
+ const slug = relPath.split("/").pop() ?? "index";
302
+ fs.writeFileSync(filePath, buildMarkdownSnapshot({ url: snapshot.url, title: snapshot.title, markdown: snapshot.markdown }, slug, snapshot.tags), "utf8");
303
+ snapshotWriteHook({ point: "page-written", index: 1, total: 1, relPath: `knowledge/${relPath}.md` });
304
+ publishSnapshotStaging(staging);
305
+ }
306
+ finally {
307
+ discardSnapshotStaging(staging);
308
+ }
217
309
  }
218
310
  async function scrapeWebsiteToStash(startUrl, stashDir, options) {
219
311
  // Offer the URL to the specialized fetchers before falling back to a crawl.
@@ -229,19 +321,33 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
229
321
  if (pages.length === 0) {
230
322
  throw new Error(`No content could be scraped from ${startUrl}`);
231
323
  }
232
- fs.rmSync(stashDir, { recursive: true, force: true });
233
- const knowledgeDir = path.join(stashDir, "knowledge");
234
- fs.mkdirSync(knowledgeDir, { recursive: true });
235
- const usedPaths = new Set();
236
- for (const page of pages) {
237
- const relPath = avoidReservedBasename(urlToRelativePath(page.url));
238
- const uniquePath = uniqueSlug(relPath, usedPaths);
239
- const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
240
- const dir = path.dirname(filePath);
241
- if (dir !== knowledgeDir)
242
- fs.mkdirSync(dir, { recursive: true });
243
- const slug = uniquePath.split("/").pop() ?? "index";
244
- fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
324
+ const staging = beginSnapshotStaging(stashDir);
325
+ try {
326
+ const knowledgeDir = path.join(staging.dir, "knowledge");
327
+ fs.mkdirSync(knowledgeDir, { recursive: true });
328
+ const usedPaths = new Set();
329
+ let written = 0;
330
+ for (const page of pages) {
331
+ const relPath = avoidReservedBasename(urlToRelativePath(page.url));
332
+ const uniquePath = uniqueSlug(relPath, usedPaths);
333
+ const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
334
+ const dir = path.dirname(filePath);
335
+ if (dir !== knowledgeDir)
336
+ fs.mkdirSync(dir, { recursive: true });
337
+ const slug = uniquePath.split("/").pop() ?? "index";
338
+ fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
339
+ written++;
340
+ snapshotWriteHook({
341
+ point: "page-written",
342
+ index: written,
343
+ total: pages.length,
344
+ relPath: `knowledge/${uniquePath}.md`,
345
+ });
346
+ }
347
+ publishSnapshotStaging(staging);
348
+ }
349
+ finally {
350
+ discardSnapshotStaging(staging);
245
351
  }
246
352
  }
247
353
  export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
@@ -162,7 +162,12 @@ function loadBetterSqlite3() {
162
162
  }
163
163
  catch (err) {
164
164
  throw new Error("akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
165
- " • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
165
+ " • If the error below says the module was compiled against a DIFFERENT Node.js\n" +
166
+ " version, you upgraded Node after installing akm. Reinstall akm (or run\n" +
167
+ " `npm rebuild better-sqlite3` in its install directory) so the binding is\n" +
168
+ " rebuilt for the Node you are now running. This is the common case after a\n" +
169
+ " Node major upgrade, and it is NOT a broken install.\n" +
170
+ " • Otherwise, reinstall akm with a working C/C++ build toolchain so its optional\n" +
166
171
  " 'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
167
172
  " will NOT be resolved — Node loads it from akm's own node_modules).\n" +
168
173
  " • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
@@ -26,6 +26,25 @@ import { applyStandardPragmas } from "./sqlite-pragmas.js";
26
26
  * Open a managed SQLite database: ensure the parent dir exists, open the handle,
27
27
  * apply standard pragmas, then run the schema initializer. The single home for
28
28
  * the open→pragmas→migrate recipe.
29
+ *
30
+ * ── On file permissions (reverted, issue #791) ──
31
+ *
32
+ * This function briefly chmodded the database, its `-wal`/`-shm` sidecars, and
33
+ * THE CONTAINING DIRECTORY to owner-only on every open (#756). That was a
34
+ * mistake and is deliberately not coming back:
35
+ *
36
+ * - It mutated state akm did not create. The data directory belongs to the
37
+ * operator; a read of the index is not consent to re-permission their disk.
38
+ * - It ran on the most-traveled path in the CLI, including `create: false`
39
+ * (read-only) opens, so any command at all silently converted a legacy
40
+ * `0755` directory to `0700` with no prompt, warning, or migration note.
41
+ * - It therefore broke installs that share `$XDG_DATA_HOME` across uids —
42
+ * agent sandboxes, containers, service accounts — which worked in 0.9.0.
43
+ * Worse, the read path answers an unreadable index with a false
44
+ * "No search index available" at exit 0 rather than an error (#791).
45
+ *
46
+ * Files akm creates here get the process umask, which is the operator's lever
47
+ * for this and always was. akm neither sets these modes nor reports on them.
29
48
  */
30
49
  export function openManagedDatabase(spec) {
31
50
  const dir = path.dirname(spec.path);
@@ -10,8 +10,9 @@
10
10
  * import their opener from a sibling here instead of reaching up into the
11
11
  * indexer — inverting the old storage→indexer arrow.
12
12
  */
13
- import fs from "node:fs";
14
13
  import { createRequire } from "node:module";
14
+ import { ConfigError } from "../../core/errors.js";
15
+ import { classifyPathAccess, describeInaccessiblePath } from "../../core/path-access.js";
15
16
  import { getDbPath } from "../../core/paths.js";
16
17
  import { openDatabase } from "../database.js";
17
18
  import { openManagedDatabase } from "../managed-db.js";
@@ -71,18 +72,42 @@ export function openExistingDatabase(dbPath) {
71
72
  // tests/storage/open-existing-database-no-create.test.ts. `create: false`
72
73
  // below is the race-free backstop for this pre-check.
73
74
  const resolvedPath = dbPath ?? getDbPath();
74
- if (!fs.existsSync(resolvedPath)) {
75
+ assertIndexPathReadable(resolvedPath);
76
+ if (classifyPathAccess(resolvedPath).access === "absent") {
75
77
  throw new Error(`Index database not found at ${resolvedPath}. Run 'akm index' to build it.`);
76
78
  }
77
79
  return openManagedDatabase({ path: resolvedPath, init: loadVecExtension, create: false });
78
80
  }
81
+ /**
82
+ * Refuse to treat an UNREADABLE index as a missing one (#791).
83
+ *
84
+ * `fs.existsSync()` — which every one of these gates used to call — returns
85
+ * `false` for `EACCES` exactly as for `ENOENT`, so an index this process cannot
86
+ * read looked identical to one that had never been built. Callers then took
87
+ * their "no index yet" branch: `search`/`curate` returned no hits at exit 0 and
88
+ * told the user to run `akm index`, which would not have helped and which they
89
+ * may not have permission to do either.
90
+ *
91
+ * A `ConfigError` here exits 78 through the standard `{ok:false, error, code}`
92
+ * envelope, so both a human and a machine caller can tell "nothing indexed"
93
+ * from "I cannot see the index".
94
+ */
95
+ export function assertIndexPathReadable(resolvedPath) {
96
+ const { access, code } = classifyPathAccess(resolvedPath);
97
+ if (access !== "inaccessible")
98
+ return;
99
+ throw new ConfigError(`Index database exists but is not readable: ${describeInaccessiblePath(resolvedPath, code)}.`, "DATA_DIR_UNREADABLE");
100
+ }
79
101
  /**
80
102
  * Open an existing index for queries without creating directories, a database
81
103
  * file, journals, or running write-capable pragmas/schema initialization.
82
104
  */
83
105
  export function openReadonlyExistingDatabase(dbPath) {
84
106
  const resolvedPath = dbPath ?? getDbPath();
85
- if (!fs.existsSync(resolvedPath))
107
+ // `undefined` means "no index" — reserve it for a genuinely absent one, and
108
+ // let an unreadable index raise instead of masquerading as absent (#791).
109
+ assertIndexPathReadable(resolvedPath);
110
+ if (classifyPathAccess(resolvedPath).access === "absent")
86
111
  return undefined;
87
112
  return openDatabase(resolvedPath, { readonly: true, create: false });
88
113
  }
@@ -13,6 +13,7 @@ import path from "node:path";
13
13
  import { parseBundleRef } from "../../core/asset/asset-ref.js";
14
14
  import { conceptIdFromTypeName } from "../../core/asset/resolve-ref.js";
15
15
  import { bestEffort } from "../../core/best-effort.js";
16
+ import { isPathAbsent } from "../../core/path-access.js";
16
17
  import { getStateDbPath, withStateDb } from "../../core/state-db.js";
17
18
  import { warn } from "../../core/warn.js";
18
19
  import { buildSearchText } from "../../indexer/search/search-fields.js";
@@ -278,7 +279,11 @@ export function rekeyEntryInPlace(db, opts) {
278
279
  * history (live-asset-wins). Best-effort + guarded on state.db's existence.
279
280
  */
280
281
  function rewriteUsageEventRefForMove(opts) {
281
- if (!fs.existsSync(getStateDbPath()))
282
+ // Every other failure in here throws (see the catch below) precisely because
283
+ // a move that quietly drops its usage history is a wrong answer wearing a
284
+ // success. An unreadable state.db must not be the one silent exception —
285
+ // only a state.db that was never created skips (#791).
286
+ if (isPathAbsent(getStateDbPath()))
282
287
  return;
283
288
  // `usage_events.entry_ref` is the fully-qualified item_ref
284
289
  // (`<bundle>//<conceptId>`).
@@ -2,6 +2,7 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { openStateDatabase, withImmediateTransaction } from "../../core/state-db.js";
5
+ import { borrowScopedStateDb, withStateDbScope } from "../../core/state-db-scope.js";
5
6
  import { resolveStorageLocations } from "../locations.js";
6
7
  /**
7
8
  * Repository owning every raw SQL statement against `workflow_runs` and
@@ -214,8 +215,10 @@ export class WorkflowRunsRepository {
214
215
  //
215
216
  // Writes to `workflow_run_units` should go through the serialized writer
216
217
  // queue (`src/workflows/exec/unit-writer.ts`) when N units may complete
217
- // concurrently — SQLite has a single writer and `withWorkflowRunsRepo`
218
- // opens a fresh connection per call.
218
+ // concurrently — SQLite has a single writer per database FILE, and outside a
219
+ // {@link withWorkflowRunsConnection} scope `withWorkflowRunsRepo` opens a
220
+ // fresh connection per call (so N concurrent writers would contend against
221
+ // each other for the write lock).
219
222
  getUnitsForRun(runId) {
220
223
  return this.db
221
224
  .prepare("SELECT * FROM workflow_run_units WHERE run_id = ? ORDER BY started_at ASC, unit_id ASC")
@@ -323,22 +326,55 @@ export class WorkflowRunsRepository {
323
326
  `recoverable state.`);
324
327
  }
325
328
  }
329
+ /**
330
+ * Finish a unit row ONLY while it is still the exact row a specific dispatch
331
+ * inserted: `running`, with that dispatch's `started_at`. The native
332
+ * executor's guarded finish (single-driver invariant): a run stolen by
333
+ * another engine re-dispatches the unit through {@link insertUnit}, which
334
+ * REPLACES the row (fresh `started_at`, bumped `attempts`) — the stale
335
+ * driver's finish then matches NOTHING instead of clobbering the new
336
+ * driver's live dispatch. Returns whether the row was finished; a zero-row
337
+ * match is a caller-classified outcome here (the executor distinguishes
338
+ * "replaced by another driver" from "row vanished"), unlike
339
+ * {@link finishUnit}'s loud throw, whose callers guarantee their row exists.
340
+ */
341
+ finishUnitFromDispatch(input) {
342
+ const result = this.db
343
+ .prepare(`UPDATE workflow_run_units
344
+ SET status = ?, result_json = ?, tokens = ?, failure_reason = ?, session_id = ?, finished_at = ?
345
+ WHERE run_id = ? AND unit_id = ? AND status = 'running' AND started_at = ?`)
346
+ .run(input.status, input.resultJson, input.tokens, input.failureReason, input.sessionId ?? null, input.finishedAt, input.runId, input.unitId, input.dispatchStartedAt);
347
+ return Number(result.changes) === 1;
348
+ }
326
349
  }
327
350
  /**
328
- * Open state.db (bound to {@link StorageLocations.stateDb}, the post-cutover
329
- * home of the `workflow_runs` / `workflow_run_steps` / `workflow_run_units`
330
- * tables), run `fn` against a {@link WorkflowRunsRepository}, and close the
331
- * connection exactly once when `fn` settles.
351
+ * Run `fn` against a {@link WorkflowRunsRepository} bound to state.db
352
+ * ({@link StorageLocations.stateDb}, the post-cutover home of the
353
+ * `workflow_runs` / `workflow_run_steps` / `workflow_run_units` tables).
332
354
  *
333
- * Fresh-connection-per-call, mirroring the former workflow.db loan pattern:
334
- * `openStateDatabase` acquires its own maintenance activity + asserts the
335
- * current ledger, and the repository owns all table-scoped SQL, so the merge
336
- * into state.db is a zero-SQL-rewrite repoint. Repository read methods fully
337
- * materialise their results, so closing here never truncates lazy iteration
338
- * (WS5 connection-lifetime rule).
355
+ * Connection lifetime BORROW-OR-OWN (mirrors `withStateDb`'s `borrowed`
356
+ * option and `appendEvent`'s `ctx.db` seam):
357
+ *
358
+ * - Inside a {@link withWorkflowRunsConnection} scope, the ambient handle is
359
+ * BORROWED and left open for the rest of the scope. A wide `map` fan-out
360
+ * therefore opens ONE connection for the whole step instead of two per unit
361
+ * (insert + finish) — `openStateDatabase` registers a maintenance activity
362
+ * lockfile and opens a read-only ledger-preflight handle on every call, so
363
+ * the per-call cost is milliseconds, not microseconds.
364
+ * - Outside a scope the behaviour is unchanged: open a fresh connection, run
365
+ * `fn`, close it in a `finally`.
366
+ *
367
+ * Repository read methods fully materialise their results, so closing an owned
368
+ * handle here never truncates lazy iteration (WS5 connection-lifetime rule).
369
+ * The signature and semantics are identical in both modes — reuse is purely an
370
+ * internal optimisation and no caller needs to know which mode it is in.
339
371
  */
340
372
  export async function withWorkflowRunsRepo(fn) {
341
- const db = openStateDatabase(resolveStorageLocations().stateDb);
373
+ const stateDb = resolveStorageLocations().stateDb;
374
+ const borrowed = borrowScopedStateDb(stateDb);
375
+ if (borrowed)
376
+ return await Promise.resolve(fn(new WorkflowRunsRepository(borrowed)));
377
+ const db = openStateDatabase(stateDb);
342
378
  try {
343
379
  return await Promise.resolve(fn(new WorkflowRunsRepository(db)));
344
380
  }
@@ -346,3 +382,20 @@ export async function withWorkflowRunsRepo(fn) {
346
382
  db.close();
347
383
  }
348
384
  }
385
+ /**
386
+ * Run `fn` with ONE state.db connection shared by every `withWorkflowRunsRepo`
387
+ * call (and every {@link import("../../core/events.js").appendEvent}) inside its
388
+ * async extent. The handle opens on first use and closes when `fn` settles;
389
+ * nesting joins the outer scope.
390
+ *
391
+ * Correctness under concurrency: `bun:sqlite` statements and
392
+ * `withImmediateTransaction` bodies run synchronously to completion, so
393
+ * logically concurrent units cannot interleave statements on the shared handle
394
+ * in a single-threaded event loop — sharing REMOVES in-process writer
395
+ * contention instead of creating it. Cross-process arbitration (WAL,
396
+ * `busy_timeout`, the run lease) is untouched. See `core/state-db-scope.ts` for
397
+ * the escaped-async-work guard.
398
+ */
399
+ export function withWorkflowRunsConnection(fn) {
400
+ return withStateDbScope(fn, { path: resolveStorageLocations().stateDb });
401
+ }
@@ -0,0 +1,156 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Decide which exact values must be scrubbed from a task's persisted run log
6
+ * (issue #755).
7
+ *
8
+ * # The gap
9
+ *
10
+ * All three task target kinds funnel through `persistRunLog`, which applied
11
+ * only `redactCredentialPatterns` — credential *shapes* (`Bearer …`, `sk-…`,
12
+ * webhook URLs). Prompt- and workflow-target runs additionally redact the exact
13
+ * secret values reachable by the run before their output ever reaches the log.
14
+ * Command-target runs did not: a scheduled command that echoed a configured
15
+ * secret whose value is not credential-shaped persisted it verbatim, into both
16
+ * the `.log` file and `logs.db`, for the whole retention window.
17
+ *
18
+ * # Why the obvious fix is wrong
19
+ *
20
+ * The issue proposed reusing {@link isEnvPassthroughValueSafeToExpose} — the
21
+ * filter the prompt path uses — over the env handed to the child. That filter
22
+ * fails CLOSED for any name outside a 22-entry allowlist, which is right where
23
+ * it currently runs: the prompt path filters `envPassthrough`, a short list the
24
+ * operator explicitly declared. A command task inherits the WHOLE ambient
25
+ * environment, so the same rule classifies essentially everything as secret.
26
+ * Measured on a developer machine: 127 of 132 variables, 25 of them with
27
+ * one-character values (`SHLVL=1`, `OLDPWD=/`, `GIT_TERMINAL_PROMPT=0`).
28
+ * Redaction is substring replacement, so those become live needles:
29
+ *
30
+ * Build finished in 12.4s -> Build finished in [REDACTED]2.[REDACTED]s
31
+ * 3 tests passed, 0 failed -> [REDACTED] tests passed, [REDACTED] failed
32
+ * wrote dist/index.js (48 KB) -> wrote dist[REDACTED]index.js ([REDACTED]8 KB)
33
+ *
34
+ * A fix that destroys every command log is not a fix.
35
+ *
36
+ * # What this does instead
37
+ *
38
+ * Three sources, and the distinction between them is the whole design:
39
+ *
40
+ * 1. **Declared by config** — the config names which variables hold
41
+ * credentials (`engines.<n>.apiKey: ${VAR}`, `embedding.apiKey`, and the
42
+ * implicit `AKM_ENGINE_<NAME>_API_KEY` / `AKM_LLM_API_KEY` /
43
+ * `AKM_EMBED_API_KEY` recipes). akm KNOWS these are secret.
44
+ * 2. **Declared by the task** — the `redact:` list, names only.
45
+ * 3. **Inferred** — a name-shape heuristic over the remaining environment, for
46
+ * the ambient credential akm was never told about.
47
+ *
48
+ * Only (3) is a guess, so only (3) carries {@link MIN_INFERRED_SECRET_LENGTH}.
49
+ * A declared secret is redacted at ANY length, because the operator told us
50
+ * what it is; applying a floor to declared values would silently stop redacting
51
+ * short secrets that are scrubbed today. The floor exists solely to stop a
52
+ * *guess* from mangling a log, and 8 clears every real credential format (AWS
53
+ * key id 20, GitHub PAT 40, `sk-…` 40+) while excluding the flags and counters
54
+ * that a name heuristic occasionally catches.
55
+ *
56
+ * Note what is deliberately NOT collected: the akm secret store on disk. A
57
+ * spawned command sees `process.env`, not akm's stores — a stored secret can
58
+ * only be echoed if it is already in the environment, where the rules above
59
+ * catch it by name. Walking every bundle's `secrets/` on each task firing would
60
+ * cost a recursive readdir plus an unbounded read for values the child cannot
61
+ * reach anyway. `redact:` is the escape hatch for a secret injected under a
62
+ * name none of the rules recognise.
63
+ */
64
+ import { collectSensitiveValues, isEnvPassthroughValueSafeToExpose } from "../core/redaction.js";
65
+ import { collectEngineCredentialValues } from "../integrations/agent/engine-resolution.js";
66
+ /**
67
+ * Shortest value an INFERRED (name-heuristic) match may contribute as a
68
+ * redaction needle. Declared secrets bypass this entirely.
69
+ *
70
+ * Redaction replaces substrings, so a short needle is not merely useless — it
71
+ * corrupts unrelated output. Below 8 the noise tier is fully intact (`1`, `0`,
72
+ * `/`, `80`, `true`, `xhigh`, `31999`); at 8 a chance collision with ordinary
73
+ * log vocabulary is negligible, and every credential format in real use is far
74
+ * longer. It also matches the conventional minimum password length, so a
75
+ * secret shorter than this is already outside normal policy.
76
+ */
77
+ export const MIN_INFERRED_SECRET_LENGTH = 8;
78
+ /**
79
+ * Environment names whose VALUE is treated as a credential on shape alone.
80
+ *
81
+ * The keyword must be a whole `_`-delimited word. Anchoring on only one side
82
+ * would drag in ordinary configuration from whichever side is left open:
83
+ * a leading anchor alone matches `KEYBOARD_LAYOUT` and `AUTHOR`, a trailing one
84
+ * matches `MONKEY` and `BYPASS`. Whole-word matching gets `GH_TOKEN`,
85
+ * `NPM_AUTH_TOKEN`, `MY_API_KEY`, `DB_PASS` and `AWS_SECRET_ACCESS_KEY` right.
86
+ */
87
+ const INFERRED_SECRET_NAME = /(?:^|_)(?:API_?KEY|KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASS|CREDENTIALS?|AUTH)(?:_|$)/i;
88
+ /**
89
+ * Credential variables whose name is a single glued word, which no
90
+ * word-boundary rule can see. Enumerated rather than matched: loosening the
91
+ * pattern enough to catch `PGPASSWORD` also catches `MONKEY`.
92
+ *
93
+ * `PWD` cannot be a keyword above for the same reason it appears here as part
94
+ * of `MYSQL_PWD` — on its own it is the working directory.
95
+ */
96
+ const KNOWN_SECRET_NAMES = new Set(["PGPASSWORD", "MYSQL_PWD"]);
97
+ /** True when the NAME alone marks this variable as holding a credential. */
98
+ export function isInferredSecretName(name) {
99
+ return KNOWN_SECRET_NAMES.has(name.toUpperCase()) || INFERRED_SECRET_NAME.test(name);
100
+ }
101
+ /** Resolve `${VAR}` / `$VAR` to the variable NAME, or undefined for anything else. */
102
+ function envRefName(spec) {
103
+ if (!spec)
104
+ return undefined;
105
+ const match = /^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$/.exec(spec.trim());
106
+ return match?.[1];
107
+ }
108
+ /**
109
+ * Every value that must be scrubbed from one task's persisted output.
110
+ *
111
+ * `declaredNames` comes from the task's `redact:` list. A name that is unset in
112
+ * `env` simply contributes nothing — naming a variable you do not currently
113
+ * export is not an error.
114
+ */
115
+ export function collectTaskLogSensitiveValues(input) {
116
+ const { env, config, declaredNames } = input;
117
+ const values = new Set();
118
+ const addDeclared = (value) => {
119
+ if (value === undefined)
120
+ return;
121
+ const trimmed = value.trim();
122
+ // Both spellings: `resolveSecret` does not trim but `resolveCredentialFromEnv`
123
+ // does, so a variable with trailing whitespace reaches an output boundary in
124
+ // either form depending on which path materialized it.
125
+ if (value.length > 0)
126
+ values.add(value);
127
+ if (trimmed.length > 0)
128
+ values.add(trimmed);
129
+ };
130
+ // (1) Declared by config — engine credentials, via the collector the prompt
131
+ // path already uses, plus the embedding key it does not cover.
132
+ if (config) {
133
+ for (const value of collectEngineCredentialValues(config, env))
134
+ values.add(value);
135
+ addDeclared(env[envRefName(config.embedding?.apiKey) ?? "AKM_EMBED_API_KEY"]);
136
+ }
137
+ // (2) Declared by the task's `redact:` list — names only, never values.
138
+ for (const name of declaredNames ?? [])
139
+ addDeclared(env[name]);
140
+ // (3) Inferred from the name shape. The only guessing tier, so the only one
141
+ // with a length floor — and still subject to the value-level check that keeps
142
+ // an allowlisted name from being treated as secret.
143
+ for (const [name, value] of Object.entries(env)) {
144
+ if (value === undefined || value.length < MIN_INFERRED_SECRET_LENGTH)
145
+ continue;
146
+ if (!isInferredSecretName(name))
147
+ continue;
148
+ if (isEnvPassthroughValueSafeToExpose(name, value))
149
+ continue;
150
+ values.add(value);
151
+ }
152
+ // Expands credential-bearing URLs into their embedded components. Can yield
153
+ // needles shorter than the floor (a URL's password), which is correct: the
154
+ // operator's own value implied them.
155
+ return collectSensitiveValues(values);
156
+ }