akm-cli 0.9.0 → 0.9.1-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/CHANGELOG.md +724 -0
  2. package/README.md +28 -63
  3. package/STABILITY.md +4 -2
  4. package/dist/cli/parse-args.js +7 -1
  5. package/dist/commands/agent/contribute-cli.js +1 -1
  6. package/dist/commands/env/child-env.js +14 -0
  7. package/dist/commands/feedback-cli.js +7 -1
  8. package/dist/commands/health/llm-usage.js +2 -1
  9. package/dist/commands/health/surfaces.js +4 -77
  10. package/dist/commands/health.js +65 -11
  11. package/dist/commands/improve/distill/quality-gate.js +6 -1
  12. package/dist/commands/improve/eligibility.js +7 -1
  13. package/dist/commands/improve/eval-cases.js +2 -0
  14. package/dist/commands/improve/improve.js +126 -10
  15. package/dist/commands/improve/locks.js +7 -0
  16. package/dist/commands/improve/memory/memory-improve.js +9 -0
  17. package/dist/commands/improve/run-context.js +5 -0
  18. package/dist/commands/improve/session-asset.js +4 -0
  19. package/dist/commands/lint/base-linter.js +31 -7
  20. package/dist/commands/lint/index.js +205 -51
  21. package/dist/commands/lint/types.js +22 -1
  22. package/dist/commands/proposal/repository.js +17 -1
  23. package/dist/commands/sources/add-cli.js +8 -2
  24. package/dist/commands/sources/info.js +12 -2
  25. package/dist/commands/sources/installed-stashes.js +6 -1
  26. package/dist/commands/sources/migration-help.js +12 -3
  27. package/dist/commands/sources/self-update.js +9 -1
  28. package/dist/commands/tasks/tasks.js +8 -2
  29. package/dist/commands/workflow-cli.js +17 -11
  30. package/dist/core/abort-deadline.js +28 -0
  31. package/dist/core/adapter/adapters/agent-skills-adapter.js +83 -5
  32. package/dist/core/adapter/adapters/akm-adapter.js +13 -10
  33. package/dist/core/adapter/adapters/akm-lint.js +78 -22
  34. package/dist/core/adapter/adapters/akm-task-adapter.js +43 -20
  35. package/dist/core/adapter/adapters/dotenv-adapter.js +21 -0
  36. package/dist/core/adapter/adapters/tool-dir-shared.js +5 -3
  37. package/dist/core/asset/frontmatter.js +10 -1
  38. package/dist/core/common.js +147 -9
  39. package/dist/core/concurrent.js +32 -0
  40. package/dist/core/config/config-io.js +5 -45
  41. package/dist/core/config/schema/engines.js +14 -3
  42. package/dist/core/config/schema/workflow.js +11 -0
  43. package/dist/core/errors.js +25 -0
  44. package/dist/core/events.js +30 -24
  45. package/dist/core/extra-params.js +11 -0
  46. package/dist/core/file-lock.js +7 -1
  47. package/dist/core/fs-txn.js +15 -2
  48. package/dist/core/improve-result.js +5 -0
  49. package/dist/core/json-schema.js +344 -9
  50. package/dist/core/loopback.js +89 -0
  51. package/dist/core/migration-operation.js +17 -2
  52. package/dist/core/path-access.js +107 -0
  53. package/dist/core/paths.js +16 -2
  54. package/dist/core/redaction.js +86 -18
  55. package/dist/core/spawn-env.js +234 -0
  56. package/dist/core/state-db-scope.js +134 -0
  57. package/dist/core/state-db.js +1 -0
  58. package/dist/core/subprocess.js +181 -37
  59. package/dist/core/write-provenance.js +85 -0
  60. package/dist/core/write-source.js +33 -2
  61. package/dist/indexer/db/graph-db.js +17 -6
  62. package/dist/indexer/ensure-index.js +10 -3
  63. package/dist/indexer/index-written-assets.js +17 -2
  64. package/dist/indexer/indexer.js +86 -21
  65. package/dist/indexer/passes/memory-inference.js +4 -0
  66. package/dist/indexer/search/db-search.js +25 -17
  67. package/dist/indexer/walk/walker.js +6 -1
  68. package/dist/integrations/agent/detect.js +13 -1
  69. package/dist/integrations/agent/engine-resolution.js +24 -11
  70. package/dist/integrations/agent/model-aliases.js +1 -1
  71. package/dist/integrations/agent/profiles.js +9 -1
  72. package/dist/integrations/agent/spawn.js +15 -87
  73. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +21 -0
  74. package/dist/integrations/lockfile.js +55 -2
  75. package/dist/llm/client.js +14 -19
  76. package/dist/llm/embedder.js +23 -3
  77. package/dist/llm/embedders/remote.js +27 -2
  78. package/dist/output/html-render.js +40 -1
  79. package/dist/output/text/lint-format.js +17 -4
  80. package/dist/runtime.js +23 -1
  81. package/dist/scripts/akm-migrate-node.js +1714 -836
  82. package/dist/scripts/akm-migrate.js +1682 -804
  83. package/dist/setup/setup.js +22 -7
  84. package/dist/sources/providers/git-install.js +25 -2
  85. package/dist/sources/providers/git-stash.js +19 -0
  86. package/dist/sources/providers/git.js +1 -1
  87. package/dist/sources/snapshot-fetchers/content-extract.js +63 -1
  88. package/dist/sources/snapshot-fetchers/website-ingest.js +126 -20
  89. package/dist/storage/database.js +71 -7
  90. package/dist/storage/engines/sqlite-migrations.js +61 -2
  91. package/dist/storage/managed-db.js +19 -0
  92. package/dist/storage/repositories/index-connection.js +39 -4
  93. package/dist/storage/repositories/index-entries-repository.js +6 -1
  94. package/dist/storage/repositories/index-meta-repository.js +11 -0
  95. package/dist/storage/repositories/index-schema.js +17 -2
  96. package/dist/storage/repositories/index-vec-repository.js +43 -5
  97. package/dist/storage/repositories/workflow-runs-repository.js +66 -13
  98. package/dist/storage/sqlite-pragmas.js +12 -1
  99. package/dist/tasks/log-redaction.js +156 -0
  100. package/dist/tasks/parser.js +82 -5
  101. package/dist/tasks/runner.js +222 -17
  102. package/dist/tasks/scheduler-invocation.js +19 -0
  103. package/dist/tasks/schema.js +86 -1
  104. package/dist/text-import-hook.mjs +1 -1
  105. package/dist/workflows/concurrency-policy.js +95 -1
  106. package/dist/workflows/exec/dispatch-redaction.js +114 -0
  107. package/dist/workflows/exec/exec-unit.js +542 -0
  108. package/dist/workflows/exec/frozen-judge.js +114 -42
  109. package/dist/workflows/exec/native-executor.js +465 -238
  110. package/dist/workflows/exec/param-secrets.js +4 -3
  111. package/dist/workflows/exec/run-workflow.js +424 -219
  112. package/dist/workflows/exec/step-work.js +506 -167
  113. package/dist/workflows/exec/unit-dispatch.js +31 -1
  114. package/dist/workflows/exec/unit-writer.js +53 -13
  115. package/dist/workflows/exec/worktree.js +454 -41
  116. package/dist/workflows/ir/compile.js +26 -2
  117. package/dist/workflows/ir/freeze.js +82 -15
  118. package/dist/workflows/ir/schema.js +105 -20
  119. package/dist/workflows/parser.js +242 -19
  120. package/dist/workflows/program/schema.js +24 -0
  121. package/dist/workflows/renderer.js +32 -4
  122. package/dist/workflows/resource-limits.js +182 -0
  123. package/dist/workflows/runtime/runs.js +146 -6
  124. package/dist/workflows/validate-summary.js +17 -2
  125. package/docs/README.md +74 -32
  126. package/docs/migration/release-notes/0.9.0.md +2 -1
  127. package/docs/migration/v0.7-to-v0.8.md +2 -1
  128. package/docs/migration/v0.8-to-v0.9.md +3 -1
  129. package/docs/reference/README.md +11 -4
  130. package/docs/reference/bundle-types.md +19 -0
  131. package/docs/reference/cli.md +105 -16
  132. package/docs/reference/configuration.md +15 -2
  133. package/docs/reference/data-and-telemetry.md +30 -10
  134. package/docs/reference/supported-formats.md +50 -0
  135. package/docs/reference/workflow-schema.md +1014 -0
  136. package/docs/reference/workflows.md +37 -633
  137. package/package.json +13 -6
  138. package/schemas/akm-config.json +18 -5
  139. package/schemas/akm-task.json +27 -5
  140. package/schemas/akm-workflow.json +92 -13
@@ -496,17 +496,32 @@ export function buildSetupSteps(options) {
496
496
  ];
497
497
  return { steps, outcome };
498
498
  }
499
- export async function runSetupWizard(opts) {
500
- assertSetupConfigPreflight();
501
- p.intro("akm setup");
502
- const current = loadUserConfig();
503
- const configPath = getConfigPath();
504
- // Resolve stash directory early so akmInit can run before any prompts
505
- const resolvedStashDir = opts?.dir ? path.resolve(opts.dir) : (primaryBundlePath(current) ?? getDefaultStashDir());
499
+ /**
500
+ * Resolve the stash directory, apply stash isolation, and THEN read the config.
501
+ *
502
+ * Order is the whole point. The pre-isolation read exists only to discover
503
+ * where the stash is; `applyStashIsolationToEnv` can repoint AKM_CONFIG_DIR,
504
+ * after which both the config contents and `getConfigPath()` differ. Reading
505
+ * the merge base before isolation meant an isolated run merged the wizard's
506
+ * answers onto the HOST config and reported the host path as the save target
507
+ * while writing somewhere else.
508
+ */
509
+ function resolveIsolatedSetupConfig(opts) {
510
+ const preIsolationConfig = loadUserConfig();
511
+ // Resolve stash directory early so akmInit can run before any prompts.
512
+ const resolvedStashDir = opts?.dir
513
+ ? path.resolve(opts.dir)
514
+ : (primaryBundlePath(preIsolationConfig) ?? getDefaultStashDir());
506
515
  // Refuse explicit --dir /tmp/... before doing any work — protects the host
507
516
  // config from being clobbered with a stashDir that the OS may reap.
508
517
  assertSetupSandbox(resolvedStashDir, opts?.dir != null);
509
518
  applyStashIsolationToEnv(resolvedStashDir, opts?.dir != null);
519
+ return { current: loadUserConfig(), configPath: getConfigPath(), resolvedStashDir };
520
+ }
521
+ export async function runSetupWizard(opts) {
522
+ assertSetupConfigPreflight();
523
+ p.intro("akm setup");
524
+ const { current, configPath, resolvedStashDir } = resolveIsolatedSetupConfig(opts);
510
525
  // Quick connectivity check — skip network-dependent steps when offline
511
526
  const online = await isOnline();
512
527
  if (!online) {
@@ -90,6 +90,16 @@ export function assertNoIgnoredPathOverwrite(repoDir, targetRevision) {
90
90
  throw new UsageError(`Git update would overwrite ignored local path ${path.join(repoDir, conflict)}; move or remove it before update.`);
91
91
  }
92
92
  }
93
+ /**
94
+ * Whether a requested ref is a full commit hash rather than a branch or tag.
95
+ *
96
+ * Only the unambiguous 40-hex (SHA-1) and 64-hex (SHA-256) forms count. An
97
+ * abbreviated hash is indistinguishable from a legal branch name, so it stays
98
+ * on the `--branch` path where git itself reports the mismatch.
99
+ */
100
+ function isCommitSha(ref) {
101
+ return /^[0-9a-f]{40}$/i.test(ref) || /^[0-9a-f]{64}$/i.test(ref);
102
+ }
93
103
  function normalizeRemoteUrl(value) {
94
104
  return value
95
105
  .trim()
@@ -199,10 +209,17 @@ async function doSyncGit(parsed, options) {
199
209
  let installRoot;
200
210
  let stashRoot;
201
211
  try {
212
+ // `git clone --branch` accepts only a branch or tag name, never a raw commit
213
+ // hash — so pinning an install to a commit (`#<40-hex-sha>`, which
214
+ // parseGithubShorthand/parseGitUrl accept and validateGitRef allows) made the
215
+ // clone fail outright. A commit pin needs a full clone followed by a
216
+ // checkout of that revision; `--depth 1` cannot fetch an arbitrary commit
217
+ // either, so the read-only shallow optimization is skipped in that case.
218
+ const pinnedCommit = parsed.requestedRef !== undefined && isCommitSha(parsed.requestedRef);
202
219
  const cloneArgs = ["clone"];
203
- if (!options?.writable)
220
+ if (!options?.writable && !pinnedCommit)
204
221
  cloneArgs.push("--depth", "1");
205
- if (parsed.requestedRef) {
222
+ if (parsed.requestedRef && !pinnedCommit) {
206
223
  cloneArgs.push("--branch", parsed.requestedRef);
207
224
  }
208
225
  cloneArgs.push(parsed.url, cloneDir);
@@ -210,6 +227,12 @@ async function doSyncGit(parsed, options) {
210
227
  if (cloneResult.status !== 0) {
211
228
  throw new Error(classifyCloneFailure(parsed.url, cloneResult.stderr, cloneResult.error));
212
229
  }
230
+ if (pinnedCommit) {
231
+ const checkout = runGit(["-C", cloneDir, "checkout", "--detach", parsed.requestedRef], { timeout: 120_000 });
232
+ if (checkout.status !== 0) {
233
+ throw new Error(`Could not check out commit ${parsed.requestedRef} from ${parsed.url}: ${checkout.stderr.trim() || "unknown git error"}`);
234
+ }
235
+ }
213
236
  // R-011: `resolved.resolvedRevision` was resolved via a SEPARATE
214
237
  // `git ls-remote` round-trip before this clone ran (resolveGitArtifact /
215
238
  // resolveGithubArtifact in registry/resolve.ts) and was never checked
@@ -56,6 +56,21 @@ export class GitStashPushError extends Error {
56
56
  }
57
57
  const GIT_PUSH_TIMEOUT_MS = 120_000;
58
58
  const ZERO_OID = "0000000000000000000000000000000000000000";
59
+ let exactCommitHookForTests;
60
+ /**
61
+ * TEST-ONLY. Interleave work into the exact-path commit sequence; `undefined`
62
+ * restores. Exists because the CAS window is a few microseconds wide between
63
+ * two synchronous `git` invocations — a wall-clock race would be
64
+ * non-deterministic, and every earlier pre-check would swallow a commit that
65
+ * landed before the window opened. Inert in production.
66
+ */
67
+ export function _setGitExactCommitHookForTests(hook) {
68
+ exactCommitHookForTests = hook;
69
+ }
70
+ /** Fire a named exact-path-commit race point (no-op outside tests). */
71
+ function gitExactCommitHook(point) {
72
+ exactCommitHookForTests?.(point);
73
+ }
59
74
  /**
60
75
  * Resolve the writable flag for an end-of-run / `akm sync` commit from the
61
76
  * configured default bundle.
@@ -523,6 +538,10 @@ function createExactPathCommit(repoDir, options) {
523
538
  throw new Error(`Git target changed before its exact commit could be attached.`);
524
539
  }
525
540
  assertWorktreeMatchesExpected(repoDir, options.paths, expected);
541
+ // Race window: everything below this line is defended only by the
542
+ // update-ref compare-and-swap. See
543
+ // tests/integration/sync-exact-commit-cas.test.ts.
544
+ gitExactCommitHook("before-update-ref");
526
545
  const update = runGit(["-C", repoDir, "update-ref", branchRef, commitOid, options.baseHead ?? ZERO_OID]);
527
546
  if (update.status !== 0) {
528
547
  throw new Error(`Git target advanced before its exact commit could be attached.`);
@@ -10,4 +10,4 @@
10
10
  // keeps importing from a single module namespace.
11
11
  export { classifyCloneFailure, cloneRepo, inspectGitUpstream, runGit, syncExistingWritableCheckout, } from "./git-install.js";
12
12
  export { ensureGitMirror, GitSourceProvider, getCachePaths, parseGitRepoUrl, syncMirroredRepo, } from "./git-provider.js";
13
- export { GitStashPushError, isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "./git-stash.js";
13
+ export { _setGitExactCommitHookForTests, GitStashPushError, isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "./git-stash.js";
@@ -217,6 +217,30 @@ function markdownDestination(url) {
217
217
  * the stack — which would otherwise abort an entire crawl.
218
218
  */
219
219
  const MAX_NESTING_DEPTH = 2_000;
220
+ /**
221
+ * HTML5 void elements. They have no closing tag and are conventionally written
222
+ * WITHOUT a trailing slash, so a depth counter that only decrements on `</x>`
223
+ * or `<x/>` treats each one as a permanent +1. The counter then measured total
224
+ * void-element COUNT rather than nesting depth, and an ordinary page with more
225
+ * than MAX_NESTING_DEPTH images or line breaks was misjudged as pathologically
226
+ * nested and degraded to plain text.
227
+ */
228
+ const VOID_ELEMENTS = new Set([
229
+ "area",
230
+ "base",
231
+ "br",
232
+ "col",
233
+ "embed",
234
+ "hr",
235
+ "img",
236
+ "input",
237
+ "link",
238
+ "meta",
239
+ "param",
240
+ "source",
241
+ "track",
242
+ "wbr",
243
+ ]);
220
244
  function exceedsNestingBudget(html) {
221
245
  let depth = 0;
222
246
  let max = 0;
@@ -226,6 +250,8 @@ function exceedsNestingBudget(html) {
226
250
  const selfClosing = match[3] === "/";
227
251
  if (selfClosing)
228
252
  continue;
253
+ if (VOID_ELEMENTS.has(match[2].toLowerCase()))
254
+ continue;
229
255
  if (closing)
230
256
  depth = Math.max(0, depth - 1);
231
257
  else {
@@ -463,9 +489,45 @@ function escapeResidualMarkup(markdown) {
463
489
  // Only `<` that begins a tag-like construct; a bare `a < b` stays readable.
464
490
  return markdown.replace(/<(?=[a-zA-Z/!?])/g, "&lt;");
465
491
  }
492
+ /**
493
+ * Apply {@link escapeResidualMarkup} everywhere EXCEPT inside fenced code
494
+ * blocks.
495
+ *
496
+ * Markup inside a fence is inert — a renderer shows it as text, it cannot
497
+ * execute — so the escape buys no safety there and actively corrupts content.
498
+ * Turndown emits code with entities already decoded, so a documentation page
499
+ * showing `&lt;div&gt;` in a `<pre><code>` block became a fence containing
500
+ * `<div>`, which this rewrote to `&lt;div>`: a half-escaped, wrong-on-both-ends
501
+ * rendering of the example the page exists to show. Every HTML, XML and JSX
502
+ * snippet in a snapshot was affected.
503
+ */
504
+ function escapeOutsideCodeFences(markdown) {
505
+ const lines = markdown.split("\n");
506
+ let inFence = false;
507
+ let fenceMarker = "";
508
+ for (let i = 0; i < lines.length; i++) {
509
+ const line = lines[i];
510
+ const fence = /^\s*(`{3,}|~{3,})/.exec(line);
511
+ if (fence) {
512
+ const marker = fence[1];
513
+ if (!inFence) {
514
+ inFence = true;
515
+ fenceMarker = marker[0];
516
+ }
517
+ else if (marker[0] === fenceMarker) {
518
+ inFence = false;
519
+ fenceMarker = "";
520
+ }
521
+ continue;
522
+ }
523
+ if (!inFence)
524
+ lines[i] = escapeResidualMarkup(line);
525
+ }
526
+ return lines.join("\n");
527
+ }
466
528
  /** Escape residual markup, then normalize whitespace, into the final snapshot. */
467
529
  function finalizeMarkdown(markdown) {
468
- return escapeResidualMarkup(markdown)
530
+ return escapeOutsideCodeFences(markdown)
469
531
  .replace(/\r/g, "")
470
532
  .replace(/[ \t]+\n/g, "\n")
471
533
  .replace(/\n{3,}/g, "\n\n")
@@ -1,7 +1,7 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
- import { createHash } from "node:crypto";
4
+ import { createHash, randomBytes } from "node:crypto";
5
5
  import fs from "node:fs";
6
6
  import path from "node:path";
7
7
  import { fetchWithRetry, isWithin, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir, todayIso, } from "../../core/common.js";
@@ -201,19 +201,111 @@ async function fetchSnapshotViaRegistry(startUrl, stashDir, allowPrivateHosts, r
201
201
  };
202
202
  return dispatchSnapshotFetchers(parsed, context, stashDir);
203
203
  }
204
+ let snapshotWriteHookForTests;
205
+ /**
206
+ * TEST-ONLY. Interrupt a refresh partway through its page-write loop;
207
+ * `undefined` restores. Exists because "a process killed mid-refresh" cannot
208
+ * be staged from outside the module — the whole loop is a single synchronous
209
+ * burst between two awaits. Inert in production (one `undefined?.()` per page).
210
+ */
211
+ export function _setWebsiteSnapshotWriteHookForTests(hook) {
212
+ snapshotWriteHookForTests = hook;
213
+ }
214
+ function snapshotWriteHook(event) {
215
+ snapshotWriteHookForTests?.(event);
216
+ }
217
+ function snapshotSiblingPrefix(stashDir, kind) {
218
+ return `.${path.basename(stashDir)}.${kind}-`;
219
+ }
220
+ function snapshotSiblingPath(stashDir, kind) {
221
+ const unique = `${process.pid}-${randomBytes(6).toString("hex")}`;
222
+ return path.join(path.dirname(stashDir), `${snapshotSiblingPrefix(stashDir, kind)}${unique}`);
223
+ }
224
+ /**
225
+ * Age gate for the staging sweep. Nothing enforces one refresh at a time for a
226
+ * given website source, so a sibling directory may belong to a refresh that is
227
+ * still running in another process; deleting it would break a healthy run
228
+ * instead of cleaning up after a dead one. Only clearly-abandoned directories
229
+ * (untouched for an hour — far longer than the 10-minute crawl wall-clock cap)
230
+ * are swept. Leftovers are inert until then: they are dot-prefixed, so the
231
+ * indexer's walk skips them.
232
+ */
233
+ const SNAPSHOT_STAGING_SWEEP_AGE_MS = 60 * 60 * 1000;
234
+ /** Remove staging/retired directories abandoned by an earlier interrupted run. */
235
+ function sweepSnapshotStaging(stashDir) {
236
+ const parent = path.dirname(stashDir);
237
+ let entries;
238
+ try {
239
+ entries = fs.readdirSync(parent);
240
+ }
241
+ catch {
242
+ return;
243
+ }
244
+ const prefixes = [snapshotSiblingPrefix(stashDir, "staging"), snapshotSiblingPrefix(stashDir, "retired")];
245
+ const cutoff = Date.now() - SNAPSHOT_STAGING_SWEEP_AGE_MS;
246
+ for (const entry of entries) {
247
+ if (!prefixes.some((prefix) => entry.startsWith(prefix)))
248
+ continue;
249
+ const abandoned = path.join(parent, entry);
250
+ try {
251
+ if (fs.statSync(abandoned).mtimeMs > cutoff)
252
+ continue;
253
+ }
254
+ catch {
255
+ continue;
256
+ }
257
+ fs.rmSync(abandoned, { recursive: true, force: true });
258
+ }
259
+ }
260
+ function beginSnapshotStaging(stashDir) {
261
+ fs.mkdirSync(path.dirname(stashDir), { recursive: true });
262
+ sweepSnapshotStaging(stashDir);
263
+ const dir = snapshotSiblingPath(stashDir, "staging");
264
+ fs.mkdirSync(dir, { recursive: true });
265
+ return { dir, target: stashDir };
266
+ }
267
+ /**
268
+ * Swap the staged snapshot in. POSIX cannot atomically exchange two non-empty
269
+ * directories, so the previous snapshot is renamed ASIDE first and deleted
270
+ * afterwards: the window in which the target does not exist is one syscall
271
+ * wide instead of an entire write loop.
272
+ */
273
+ function publishSnapshotStaging(staging) {
274
+ let retired;
275
+ if (fs.existsSync(staging.target)) {
276
+ retired = snapshotSiblingPath(staging.target, "retired");
277
+ fs.renameSync(staging.target, retired);
278
+ }
279
+ fs.renameSync(staging.dir, staging.target);
280
+ if (retired)
281
+ fs.rmSync(retired, { recursive: true, force: true });
282
+ }
283
+ /** Drop an unpublished staging directory (no-op once it has been renamed). */
284
+ function discardSnapshotStaging(staging) {
285
+ fs.rmSync(staging.dir, { recursive: true, force: true });
286
+ }
204
287
  /** Materialize a single fetcher snapshot as the source's whole stash. */
205
288
  function writeSnapshotToStash(stashDir, snapshot) {
206
289
  const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
207
290
  const relPath = avoidReservedBasename(preferredName);
291
+ // Validate against the FINAL location so the guarantee (and the error text)
292
+ // is independent of where the file is staged.
208
293
  const knowledgeDir = path.join(stashDir, "knowledge");
209
- const filePath = path.resolve(knowledgeDir, `${relPath}.md`);
210
- if (!isWithin(filePath, knowledgeDir)) {
294
+ if (!isWithin(path.resolve(knowledgeDir, `${relPath}.md`), knowledgeDir)) {
211
295
  throw new UsageError(`Snapshot fetcher returned an unsafe preferred name: ${JSON.stringify(preferredName)}`);
212
296
  }
213
- fs.rmSync(stashDir, { recursive: true, force: true });
214
- fs.mkdirSync(path.dirname(filePath), { recursive: true });
215
- const slug = relPath.split("/").pop() ?? "index";
216
- fs.writeFileSync(filePath, buildMarkdownSnapshot({ url: snapshot.url, title: snapshot.title, markdown: snapshot.markdown }, slug, snapshot.tags), "utf8");
297
+ const staging = beginSnapshotStaging(stashDir);
298
+ try {
299
+ const filePath = path.resolve(path.join(staging.dir, "knowledge"), `${relPath}.md`);
300
+ fs.mkdirSync(path.dirname(filePath), { recursive: true });
301
+ const slug = relPath.split("/").pop() ?? "index";
302
+ fs.writeFileSync(filePath, buildMarkdownSnapshot({ url: snapshot.url, title: snapshot.title, markdown: snapshot.markdown }, slug, snapshot.tags), "utf8");
303
+ snapshotWriteHook({ point: "page-written", index: 1, total: 1, relPath: `knowledge/${relPath}.md` });
304
+ publishSnapshotStaging(staging);
305
+ }
306
+ finally {
307
+ discardSnapshotStaging(staging);
308
+ }
217
309
  }
218
310
  async function scrapeWebsiteToStash(startUrl, stashDir, options) {
219
311
  // Offer the URL to the specialized fetchers before falling back to a crawl.
@@ -229,19 +321,33 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
229
321
  if (pages.length === 0) {
230
322
  throw new Error(`No content could be scraped from ${startUrl}`);
231
323
  }
232
- fs.rmSync(stashDir, { recursive: true, force: true });
233
- const knowledgeDir = path.join(stashDir, "knowledge");
234
- fs.mkdirSync(knowledgeDir, { recursive: true });
235
- const usedPaths = new Set();
236
- for (const page of pages) {
237
- const relPath = avoidReservedBasename(urlToRelativePath(page.url));
238
- const uniquePath = uniqueSlug(relPath, usedPaths);
239
- const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
240
- const dir = path.dirname(filePath);
241
- if (dir !== knowledgeDir)
242
- fs.mkdirSync(dir, { recursive: true });
243
- const slug = uniquePath.split("/").pop() ?? "index";
244
- fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
324
+ const staging = beginSnapshotStaging(stashDir);
325
+ try {
326
+ const knowledgeDir = path.join(staging.dir, "knowledge");
327
+ fs.mkdirSync(knowledgeDir, { recursive: true });
328
+ const usedPaths = new Set();
329
+ let written = 0;
330
+ for (const page of pages) {
331
+ const relPath = avoidReservedBasename(urlToRelativePath(page.url));
332
+ const uniquePath = uniqueSlug(relPath, usedPaths);
333
+ const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
334
+ const dir = path.dirname(filePath);
335
+ if (dir !== knowledgeDir)
336
+ fs.mkdirSync(dir, { recursive: true });
337
+ const slug = uniquePath.split("/").pop() ?? "index";
338
+ fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
339
+ written++;
340
+ snapshotWriteHook({
341
+ point: "page-written",
342
+ index: written,
343
+ total: pages.length,
344
+ relPath: `knowledge/${uniquePath}.md`,
345
+ });
346
+ }
347
+ publishSnapshotStaging(staging);
348
+ }
349
+ finally {
350
+ discardSnapshotStaging(staging);
245
351
  }
246
352
  }
247
353
  export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
@@ -149,6 +149,48 @@ function loadBunSqlite() {
149
149
  return bunSqliteModule;
150
150
  }
151
151
  let betterSqlite3Ctor;
152
+ /**
153
+ * The binding is absent or unbuildable — a toolchain/install problem.
154
+ */
155
+ const MISSING_BINDING_REMEDY = "akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
156
+ " • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
157
+ " 'better-sqlite3' native binding builds (a global `npm i -g better-sqlite3`\n" +
158
+ " will NOT be resolved — Node loads it from akm's own node_modules).\n" +
159
+ " • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.";
160
+ /**
161
+ * Recognize a binding built for a DIFFERENT Node ABI than the one now running,
162
+ * and answer with the one command that fixes it.
163
+ *
164
+ * This is the most likely failure a real user hits, and it is not a broken
165
+ * install: a native addon is compiled (or a prebuilt binary is selected) for
166
+ * the Node major present at `npm install` time. Upgrade Node afterwards and the
167
+ * same file no longer loads.
168
+ *
169
+ * It is matched rather than described because the symptom text varies and the
170
+ * previous wording only named ONE of them. The prebuilt-binary path — which is
171
+ * now the normal path, since better-sqlite3 is pinned to a version publishing a
172
+ * prebuild for every supported Node (see package.json → pinNotes) — reports
173
+ * `Module did not self-register`, saying nothing about versions at all. A
174
+ * from-source build reports the explicit `NODE_MODULE_VERSION` mismatch. Asking
175
+ * the user to decide which bullet applies is the step worth deleting.
176
+ */
177
+ export function abiMismatchRemedy(message) {
178
+ const ABI_MISMATCH_SHAPES = [
179
+ "did not self-register", // prebuilt binary for another ABI
180
+ "NODE_MODULE_VERSION", // explicit mismatch, from-source build
181
+ "was compiled against a different", // same, older phrasing
182
+ "invalid ELF header", // binary for another platform/arch entirely
183
+ ];
184
+ if (!ABI_MISMATCH_SHAPES.some((shape) => message.includes(shape)))
185
+ return undefined;
186
+ return ("akm could not load 'better-sqlite3': its native binding was built for a different\n" +
187
+ `Node.js version than the one now running (this Node is ABI ${process.versions.modules}).\n` +
188
+ "This is what happens when Node is upgraded after akm is installed. It is NOT a\n" +
189
+ "broken install, and reinstalling akm is not required.\n" +
190
+ " Fix: npm rebuild better-sqlite3 # in akm's install directory\n" +
191
+ " Or: npm install -g akm-cli # reinstall, rebuilding against this Node\n" +
192
+ " Or: run akm under Bun, whose built-in SQLite driver needs no native binding.");
193
+ }
152
194
  function loadBetterSqlite3() {
153
195
  if (!betterSqlite3Ctor) {
154
196
  // Runtime-gated dynamic require: only reached when NOT on Bun, so Bun never
@@ -161,12 +203,13 @@ function loadBetterSqlite3() {
161
203
  mod = nodeRequire("better-sqlite3");
162
204
  }
163
205
  catch (err) {
164
- throw new Error("akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
165
- " • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
166
- " 'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
167
- " will NOT be resolved Node loads it from akm's own node_modules).\n" +
168
- " • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
169
- ` Underlying load error: ${err instanceof Error ? err.message : String(err)}`);
206
+ // An ABI mismatch does NOT arrive here `require` succeeds and the
207
+ // failure lands at construction (see openNodeDatabase). This path is a
208
+ // genuinely absent or unresolvable module. `abiMismatchRemedy` is still
209
+ // consulted because a from-source build CAN fail at load with the
210
+ // explicit NODE_MODULE_VERSION message.
211
+ const raw = err instanceof Error ? err.message : String(err);
212
+ throw new Error(`${abiMismatchRemedy(raw) ?? MISSING_BINDING_REMEDY}\n Underlying load error: ${raw}`);
170
213
  }
171
214
  betterSqlite3Ctor = mod.default ?? mod;
172
215
  }
@@ -183,7 +226,24 @@ function openNodeDatabase(path, opts) {
183
226
  options.readonly = opts.readonly;
184
227
  if (opts?.create === false)
185
228
  options.fileMustExist = true;
186
- const db = opts ? new BetterSqlite3(path, options) : new BetterSqlite3(path);
229
+ // Construction, not `require`, is where an ABI mismatch surfaces.
230
+ // `require("better-sqlite3")` SUCCEEDS against a binding built for another
231
+ // Node ABI — the package resolves its `.node` file lazily — so the loader's
232
+ // catch never sees this error and cannot explain it. Verified against a real
233
+ // ABI-127 binding under Node 24 (ABI 137): `require()` returned a function
234
+ // and `new Database(...)` threw. Wrapping the require alone left the most
235
+ // likely real-world failure reported as a bare Node internals message.
236
+ let db;
237
+ try {
238
+ db = opts ? new BetterSqlite3(path, options) : new BetterSqlite3(path);
239
+ }
240
+ catch (err) {
241
+ const raw = err instanceof Error ? err.message : String(err);
242
+ const remedy = abiMismatchRemedy(raw);
243
+ if (!remedy)
244
+ throw err;
245
+ throw new Error(`${remedy}\n Underlying error: ${raw}`);
246
+ }
187
247
  return {
188
248
  prepare: db.prepare.bind(db),
189
249
  exec: db.exec.bind(db),
@@ -191,6 +251,10 @@ function openNodeDatabase(path, opts) {
191
251
  // bun:sqlite also provides db.run(). Normalize the latter at the provider
192
252
  // boundary so callers and maintenance wrappers can rely on one contract.
193
253
  run: (sql, ...params) => db.prepare(sql).run(...params),
254
+ // sqlite-vec's load(db) calls db.loadExtension(). Without forwarding it the
255
+ // extension could never load on Node, so the vector fast path was dead
256
+ // across the entire npm distribution even when sqlite-vec was installed.
257
+ loadExtension: db.loadExtension.bind(db),
194
258
  transaction: db.transaction.bind(db),
195
259
  get inTransaction() {
196
260
  return db.inTransaction;
@@ -100,9 +100,68 @@ export function runMigrations(db, migrations, opts) {
100
100
  if (applied.has(migration.id))
101
101
  continue;
102
102
  opts?.beforeMigration?.(migration);
103
- db.transaction(() => {
103
+ withImmediateWriteLock(db, () => {
104
+ // Re-check under the write lock. `applied` is a snapshot taken before the
105
+ // loop, so two processes bootstrapping the same fresh DB concurrently
106
+ // (both see existed=false, both run with applyPending) could each decide
107
+ // to apply migration N. The first commits; the second must not re-run the
108
+ // DDL and must not hit a UNIQUE violation on the ledger insert.
109
+ const already = db.prepare("SELECT 1 FROM schema_migrations WHERE id = ?").get(migration.id);
110
+ if (already)
111
+ return;
104
112
  db.exec(migration.up);
105
113
  db.prepare("INSERT INTO schema_migrations (id) VALUES (?)").run(migration.id);
106
- })();
114
+ });
115
+ applied.add(migration.id);
107
116
  }
108
117
  }
118
+ /** Attempts to acquire the write lock before giving up to the caller. */
119
+ const IMMEDIATE_LOCK_MAX_ATTEMPTS = 5;
120
+ /**
121
+ * Run `fn` inside a `BEGIN IMMEDIATE` transaction.
122
+ *
123
+ * The write lock is taken up front rather than upgraded from a read lock, so a
124
+ * second process bootstrapping the same database WAITS for the first to commit
125
+ * instead of racing it. `db.transaction()` opens a DEFERRED transaction, which
126
+ * only takes the write lock on first write — leaving the read-then-write gap
127
+ * this guards.
128
+ *
129
+ * Deliberately local rather than reusing `withImmediateTransaction` from
130
+ * core/state-db: that module imports this one, so the dependency cannot be
131
+ * pointed the other way.
132
+ */
133
+ function withImmediateWriteLock(db, fn) {
134
+ if (db.inTransaction) {
135
+ fn();
136
+ return;
137
+ }
138
+ let lastBeginErr;
139
+ for (let attempt = 1; attempt <= IMMEDIATE_LOCK_MAX_ATTEMPTS; attempt++) {
140
+ try {
141
+ db.exec("BEGIN IMMEDIATE");
142
+ }
143
+ catch (err) {
144
+ // Busy despite busy_timeout (another writer holding it across the whole
145
+ // window). Retry a bounded number of times before surfacing.
146
+ lastBeginErr = err;
147
+ continue;
148
+ }
149
+ try {
150
+ fn();
151
+ db.exec("COMMIT");
152
+ return;
153
+ }
154
+ catch (err) {
155
+ try {
156
+ db.exec("ROLLBACK");
157
+ }
158
+ catch {
159
+ // Already rolled back by SQLite (e.g. the statement aborted the txn).
160
+ }
161
+ throw err;
162
+ }
163
+ }
164
+ throw lastBeginErr instanceof Error
165
+ ? lastBeginErr
166
+ : new Error(`could not acquire the migration write lock after ${IMMEDIATE_LOCK_MAX_ATTEMPTS} attempts`);
167
+ }
@@ -26,6 +26,25 @@ import { applyStandardPragmas } from "./sqlite-pragmas.js";
26
26
  * Open a managed SQLite database: ensure the parent dir exists, open the handle,
27
27
  * apply standard pragmas, then run the schema initializer. The single home for
28
28
  * the open→pragmas→migrate recipe.
29
+ *
30
+ * ── On file permissions (reverted, issue #791) ──
31
+ *
32
+ * This function briefly chmodded the database, its `-wal`/`-shm` sidecars, and
33
+ * THE CONTAINING DIRECTORY to owner-only on every open (#756). That was a
34
+ * mistake and is deliberately not coming back:
35
+ *
36
+ * - It mutated state akm did not create. The data directory belongs to the
37
+ * operator; a read of the index is not consent to re-permission their disk.
38
+ * - It ran on the most-traveled path in the CLI, including `create: false`
39
+ * (read-only) opens, so any command at all silently converted a legacy
40
+ * `0755` directory to `0700` with no prompt, warning, or migration note.
41
+ * - It therefore broke installs that share `$XDG_DATA_HOME` across uids —
42
+ * agent sandboxes, containers, service accounts — which worked in 0.9.0.
43
+ * Worse, the read path answers an unreadable index with a false
44
+ * "No search index available" at exit 0 rather than an error (#791).
45
+ *
46
+ * Files akm creates here get the process umask, which is the operator's lever
47
+ * for this and always was. akm neither sets these modes nor reports on them.
29
48
  */
30
49
  export function openManagedDatabase(spec) {
31
50
  const dir = path.dirname(spec.path);