akm-cli 0.9.0 → 0.9.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +724 -0
- package/README.md +28 -63
- package/STABILITY.md +4 -2
- package/dist/cli/parse-args.js +7 -1
- package/dist/commands/agent/contribute-cli.js +1 -1
- package/dist/commands/env/child-env.js +14 -0
- package/dist/commands/feedback-cli.js +7 -1
- package/dist/commands/health/llm-usage.js +2 -1
- package/dist/commands/health/surfaces.js +4 -77
- package/dist/commands/health.js +65 -11
- package/dist/commands/improve/distill/quality-gate.js +6 -1
- package/dist/commands/improve/eligibility.js +7 -1
- package/dist/commands/improve/eval-cases.js +2 -0
- package/dist/commands/improve/improve.js +126 -10
- package/dist/commands/improve/locks.js +7 -0
- package/dist/commands/improve/memory/memory-improve.js +9 -0
- package/dist/commands/improve/run-context.js +5 -0
- package/dist/commands/improve/session-asset.js +4 -0
- package/dist/commands/lint/base-linter.js +31 -7
- package/dist/commands/lint/index.js +205 -51
- package/dist/commands/lint/types.js +22 -1
- package/dist/commands/proposal/repository.js +17 -1
- package/dist/commands/sources/add-cli.js +8 -2
- package/dist/commands/sources/info.js +12 -2
- package/dist/commands/sources/installed-stashes.js +6 -1
- package/dist/commands/sources/migration-help.js +12 -3
- package/dist/commands/sources/self-update.js +9 -1
- package/dist/commands/tasks/tasks.js +8 -2
- package/dist/commands/workflow-cli.js +17 -11
- package/dist/core/abort-deadline.js +28 -0
- package/dist/core/adapter/adapters/agent-skills-adapter.js +83 -5
- package/dist/core/adapter/adapters/akm-adapter.js +13 -10
- package/dist/core/adapter/adapters/akm-lint.js +78 -22
- package/dist/core/adapter/adapters/akm-task-adapter.js +43 -20
- package/dist/core/adapter/adapters/dotenv-adapter.js +21 -0
- package/dist/core/adapter/adapters/tool-dir-shared.js +5 -3
- package/dist/core/asset/frontmatter.js +10 -1
- package/dist/core/common.js +147 -9
- package/dist/core/concurrent.js +32 -0
- package/dist/core/config/config-io.js +5 -45
- package/dist/core/config/schema/engines.js +14 -3
- package/dist/core/config/schema/workflow.js +11 -0
- package/dist/core/errors.js +25 -0
- package/dist/core/events.js +30 -24
- package/dist/core/extra-params.js +11 -0
- package/dist/core/file-lock.js +7 -1
- package/dist/core/fs-txn.js +15 -2
- package/dist/core/improve-result.js +5 -0
- package/dist/core/json-schema.js +344 -9
- package/dist/core/loopback.js +89 -0
- package/dist/core/migration-operation.js +17 -2
- package/dist/core/path-access.js +107 -0
- package/dist/core/paths.js +16 -2
- package/dist/core/redaction.js +86 -18
- package/dist/core/spawn-env.js +234 -0
- package/dist/core/state-db-scope.js +134 -0
- package/dist/core/state-db.js +1 -0
- package/dist/core/subprocess.js +181 -37
- package/dist/core/write-provenance.js +85 -0
- package/dist/core/write-source.js +33 -2
- package/dist/indexer/db/graph-db.js +17 -6
- package/dist/indexer/ensure-index.js +10 -3
- package/dist/indexer/index-written-assets.js +17 -2
- package/dist/indexer/indexer.js +86 -21
- package/dist/indexer/passes/memory-inference.js +4 -0
- package/dist/indexer/search/db-search.js +25 -17
- package/dist/indexer/walk/walker.js +6 -1
- package/dist/integrations/agent/detect.js +13 -1
- package/dist/integrations/agent/engine-resolution.js +24 -11
- package/dist/integrations/agent/model-aliases.js +1 -1
- package/dist/integrations/agent/profiles.js +9 -1
- package/dist/integrations/agent/spawn.js +15 -87
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +21 -0
- package/dist/integrations/lockfile.js +55 -2
- package/dist/llm/client.js +14 -19
- package/dist/llm/embedder.js +23 -3
- package/dist/llm/embedders/remote.js +27 -2
- package/dist/output/html-render.js +40 -1
- package/dist/output/text/lint-format.js +17 -4
- package/dist/runtime.js +23 -1
- package/dist/scripts/akm-migrate-node.js +1714 -836
- package/dist/scripts/akm-migrate.js +1682 -804
- package/dist/setup/setup.js +22 -7
- package/dist/sources/providers/git-install.js +25 -2
- package/dist/sources/providers/git-stash.js +19 -0
- package/dist/sources/providers/git.js +1 -1
- package/dist/sources/snapshot-fetchers/content-extract.js +63 -1
- package/dist/sources/snapshot-fetchers/website-ingest.js +126 -20
- package/dist/storage/database.js +71 -7
- package/dist/storage/engines/sqlite-migrations.js +61 -2
- package/dist/storage/managed-db.js +19 -0
- package/dist/storage/repositories/index-connection.js +39 -4
- package/dist/storage/repositories/index-entries-repository.js +6 -1
- package/dist/storage/repositories/index-meta-repository.js +11 -0
- package/dist/storage/repositories/index-schema.js +17 -2
- package/dist/storage/repositories/index-vec-repository.js +43 -5
- package/dist/storage/repositories/workflow-runs-repository.js +66 -13
- package/dist/storage/sqlite-pragmas.js +12 -1
- package/dist/tasks/log-redaction.js +156 -0
- package/dist/tasks/parser.js +82 -5
- package/dist/tasks/runner.js +222 -17
- package/dist/tasks/scheduler-invocation.js +19 -0
- package/dist/tasks/schema.js +86 -1
- package/dist/text-import-hook.mjs +1 -1
- package/dist/workflows/concurrency-policy.js +95 -1
- package/dist/workflows/exec/dispatch-redaction.js +114 -0
- package/dist/workflows/exec/exec-unit.js +542 -0
- package/dist/workflows/exec/frozen-judge.js +114 -42
- package/dist/workflows/exec/native-executor.js +465 -238
- package/dist/workflows/exec/param-secrets.js +4 -3
- package/dist/workflows/exec/run-workflow.js +424 -219
- package/dist/workflows/exec/step-work.js +506 -167
- package/dist/workflows/exec/unit-dispatch.js +31 -1
- package/dist/workflows/exec/unit-writer.js +53 -13
- package/dist/workflows/exec/worktree.js +454 -41
- package/dist/workflows/ir/compile.js +26 -2
- package/dist/workflows/ir/freeze.js +82 -15
- package/dist/workflows/ir/schema.js +105 -20
- package/dist/workflows/parser.js +242 -19
- package/dist/workflows/program/schema.js +24 -0
- package/dist/workflows/renderer.js +32 -4
- package/dist/workflows/resource-limits.js +182 -0
- package/dist/workflows/runtime/runs.js +146 -6
- package/dist/workflows/validate-summary.js +17 -2
- package/docs/README.md +74 -32
- package/docs/migration/release-notes/0.9.0.md +2 -1
- package/docs/migration/v0.7-to-v0.8.md +2 -1
- package/docs/migration/v0.8-to-v0.9.md +3 -1
- package/docs/reference/README.md +11 -4
- package/docs/reference/bundle-types.md +19 -0
- package/docs/reference/cli.md +105 -16
- package/docs/reference/configuration.md +15 -2
- package/docs/reference/data-and-telemetry.md +30 -10
- package/docs/reference/supported-formats.md +50 -0
- package/docs/reference/workflow-schema.md +1014 -0
- package/docs/reference/workflows.md +37 -633
- package/package.json +13 -6
- package/schemas/akm-config.json +18 -5
- package/schemas/akm-task.json +27 -5
- package/schemas/akm-workflow.json +92 -13
package/dist/setup/setup.js
CHANGED
|
@@ -496,17 +496,32 @@ export function buildSetupSteps(options) {
|
|
|
496
496
|
];
|
|
497
497
|
return { steps, outcome };
|
|
498
498
|
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
499
|
+
/**
|
|
500
|
+
* Resolve the stash directory, apply stash isolation, and THEN read the config.
|
|
501
|
+
*
|
|
502
|
+
* Order is the whole point. The pre-isolation read exists only to discover
|
|
503
|
+
* where the stash is; `applyStashIsolationToEnv` can repoint AKM_CONFIG_DIR,
|
|
504
|
+
* after which both the config contents and `getConfigPath()` differ. Reading
|
|
505
|
+
* the merge base before isolation meant an isolated run merged the wizard's
|
|
506
|
+
* answers onto the HOST config and reported the host path as the save target
|
|
507
|
+
* while writing somewhere else.
|
|
508
|
+
*/
|
|
509
|
+
function resolveIsolatedSetupConfig(opts) {
|
|
510
|
+
const preIsolationConfig = loadUserConfig();
|
|
511
|
+
// Resolve stash directory early so akmInit can run before any prompts.
|
|
512
|
+
const resolvedStashDir = opts?.dir
|
|
513
|
+
? path.resolve(opts.dir)
|
|
514
|
+
: (primaryBundlePath(preIsolationConfig) ?? getDefaultStashDir());
|
|
506
515
|
// Refuse explicit --dir /tmp/... before doing any work — protects the host
|
|
507
516
|
// config from being clobbered with a stashDir that the OS may reap.
|
|
508
517
|
assertSetupSandbox(resolvedStashDir, opts?.dir != null);
|
|
509
518
|
applyStashIsolationToEnv(resolvedStashDir, opts?.dir != null);
|
|
519
|
+
return { current: loadUserConfig(), configPath: getConfigPath(), resolvedStashDir };
|
|
520
|
+
}
|
|
521
|
+
export async function runSetupWizard(opts) {
|
|
522
|
+
assertSetupConfigPreflight();
|
|
523
|
+
p.intro("akm setup");
|
|
524
|
+
const { current, configPath, resolvedStashDir } = resolveIsolatedSetupConfig(opts);
|
|
510
525
|
// Quick connectivity check — skip network-dependent steps when offline
|
|
511
526
|
const online = await isOnline();
|
|
512
527
|
if (!online) {
|
|
@@ -90,6 +90,16 @@ export function assertNoIgnoredPathOverwrite(repoDir, targetRevision) {
|
|
|
90
90
|
throw new UsageError(`Git update would overwrite ignored local path ${path.join(repoDir, conflict)}; move or remove it before update.`);
|
|
91
91
|
}
|
|
92
92
|
}
|
|
93
|
+
/**
|
|
94
|
+
* Whether a requested ref is a full commit hash rather than a branch or tag.
|
|
95
|
+
*
|
|
96
|
+
* Only the unambiguous 40-hex (SHA-1) and 64-hex (SHA-256) forms count. An
|
|
97
|
+
* abbreviated hash is indistinguishable from a legal branch name, so it stays
|
|
98
|
+
* on the `--branch` path where git itself reports the mismatch.
|
|
99
|
+
*/
|
|
100
|
+
function isCommitSha(ref) {
|
|
101
|
+
return /^[0-9a-f]{40}$/i.test(ref) || /^[0-9a-f]{64}$/i.test(ref);
|
|
102
|
+
}
|
|
93
103
|
function normalizeRemoteUrl(value) {
|
|
94
104
|
return value
|
|
95
105
|
.trim()
|
|
@@ -199,10 +209,17 @@ async function doSyncGit(parsed, options) {
|
|
|
199
209
|
let installRoot;
|
|
200
210
|
let stashRoot;
|
|
201
211
|
try {
|
|
212
|
+
// `git clone --branch` accepts only a branch or tag name, never a raw commit
|
|
213
|
+
// hash — so pinning an install to a commit (`#<40-hex-sha>`, which
|
|
214
|
+
// parseGithubShorthand/parseGitUrl accept and validateGitRef allows) made the
|
|
215
|
+
// clone fail outright. A commit pin needs a full clone followed by a
|
|
216
|
+
// checkout of that revision; `--depth 1` cannot fetch an arbitrary commit
|
|
217
|
+
// either, so the read-only shallow optimization is skipped in that case.
|
|
218
|
+
const pinnedCommit = parsed.requestedRef !== undefined && isCommitSha(parsed.requestedRef);
|
|
202
219
|
const cloneArgs = ["clone"];
|
|
203
|
-
if (!options?.writable)
|
|
220
|
+
if (!options?.writable && !pinnedCommit)
|
|
204
221
|
cloneArgs.push("--depth", "1");
|
|
205
|
-
if (parsed.requestedRef) {
|
|
222
|
+
if (parsed.requestedRef && !pinnedCommit) {
|
|
206
223
|
cloneArgs.push("--branch", parsed.requestedRef);
|
|
207
224
|
}
|
|
208
225
|
cloneArgs.push(parsed.url, cloneDir);
|
|
@@ -210,6 +227,12 @@ async function doSyncGit(parsed, options) {
|
|
|
210
227
|
if (cloneResult.status !== 0) {
|
|
211
228
|
throw new Error(classifyCloneFailure(parsed.url, cloneResult.stderr, cloneResult.error));
|
|
212
229
|
}
|
|
230
|
+
if (pinnedCommit) {
|
|
231
|
+
const checkout = runGit(["-C", cloneDir, "checkout", "--detach", parsed.requestedRef], { timeout: 120_000 });
|
|
232
|
+
if (checkout.status !== 0) {
|
|
233
|
+
throw new Error(`Could not check out commit ${parsed.requestedRef} from ${parsed.url}: ${checkout.stderr.trim() || "unknown git error"}`);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
213
236
|
// R-011: `resolved.resolvedRevision` was resolved via a SEPARATE
|
|
214
237
|
// `git ls-remote` round-trip before this clone ran (resolveGitArtifact /
|
|
215
238
|
// resolveGithubArtifact in registry/resolve.ts) and was never checked
|
|
@@ -56,6 +56,21 @@ export class GitStashPushError extends Error {
|
|
|
56
56
|
}
|
|
57
57
|
const GIT_PUSH_TIMEOUT_MS = 120_000;
|
|
58
58
|
const ZERO_OID = "0000000000000000000000000000000000000000";
|
|
59
|
+
let exactCommitHookForTests;
|
|
60
|
+
/**
|
|
61
|
+
* TEST-ONLY. Interleave work into the exact-path commit sequence; `undefined`
|
|
62
|
+
* restores. Exists because the CAS window is a few microseconds wide between
|
|
63
|
+
* two synchronous `git` invocations — a wall-clock race would be
|
|
64
|
+
* non-deterministic, and every earlier pre-check would swallow a commit that
|
|
65
|
+
* landed before the window opened. Inert in production.
|
|
66
|
+
*/
|
|
67
|
+
export function _setGitExactCommitHookForTests(hook) {
|
|
68
|
+
exactCommitHookForTests = hook;
|
|
69
|
+
}
|
|
70
|
+
/** Fire a named exact-path-commit race point (no-op outside tests). */
|
|
71
|
+
function gitExactCommitHook(point) {
|
|
72
|
+
exactCommitHookForTests?.(point);
|
|
73
|
+
}
|
|
59
74
|
/**
|
|
60
75
|
* Resolve the writable flag for an end-of-run / `akm sync` commit from the
|
|
61
76
|
* configured default bundle.
|
|
@@ -523,6 +538,10 @@ function createExactPathCommit(repoDir, options) {
|
|
|
523
538
|
throw new Error(`Git target changed before its exact commit could be attached.`);
|
|
524
539
|
}
|
|
525
540
|
assertWorktreeMatchesExpected(repoDir, options.paths, expected);
|
|
541
|
+
// Race window: everything below this line is defended only by the
|
|
542
|
+
// update-ref compare-and-swap. See
|
|
543
|
+
// tests/integration/sync-exact-commit-cas.test.ts.
|
|
544
|
+
gitExactCommitHook("before-update-ref");
|
|
526
545
|
const update = runGit(["-C", repoDir, "update-ref", branchRef, commitOid, options.baseHead ?? ZERO_OID]);
|
|
527
546
|
if (update.status !== 0) {
|
|
528
547
|
throw new Error(`Git target advanced before its exact commit could be attached.`);
|
|
@@ -10,4 +10,4 @@
|
|
|
10
10
|
// keeps importing from a single module namespace.
|
|
11
11
|
export { classifyCloneFailure, cloneRepo, inspectGitUpstream, runGit, syncExistingWritableCheckout, } from "./git-install.js";
|
|
12
12
|
export { ensureGitMirror, GitSourceProvider, getCachePaths, parseGitRepoUrl, syncMirroredRepo, } from "./git-provider.js";
|
|
13
|
-
export { GitStashPushError, isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "./git-stash.js";
|
|
13
|
+
export { _setGitExactCommitHookForTests, GitStashPushError, isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "./git-stash.js";
|
|
@@ -217,6 +217,30 @@ function markdownDestination(url) {
|
|
|
217
217
|
* the stack — which would otherwise abort an entire crawl.
|
|
218
218
|
*/
|
|
219
219
|
const MAX_NESTING_DEPTH = 2_000;
|
|
220
|
+
/**
|
|
221
|
+
* HTML5 void elements. They have no closing tag and are conventionally written
|
|
222
|
+
* WITHOUT a trailing slash, so a depth counter that only decrements on `</x>`
|
|
223
|
+
* or `<x/>` treats each one as a permanent +1. The counter then measured total
|
|
224
|
+
* void-element COUNT rather than nesting depth, and an ordinary page with more
|
|
225
|
+
* than MAX_NESTING_DEPTH images or line breaks was misjudged as pathologically
|
|
226
|
+
* nested and degraded to plain text.
|
|
227
|
+
*/
|
|
228
|
+
const VOID_ELEMENTS = new Set([
|
|
229
|
+
"area",
|
|
230
|
+
"base",
|
|
231
|
+
"br",
|
|
232
|
+
"col",
|
|
233
|
+
"embed",
|
|
234
|
+
"hr",
|
|
235
|
+
"img",
|
|
236
|
+
"input",
|
|
237
|
+
"link",
|
|
238
|
+
"meta",
|
|
239
|
+
"param",
|
|
240
|
+
"source",
|
|
241
|
+
"track",
|
|
242
|
+
"wbr",
|
|
243
|
+
]);
|
|
220
244
|
function exceedsNestingBudget(html) {
|
|
221
245
|
let depth = 0;
|
|
222
246
|
let max = 0;
|
|
@@ -226,6 +250,8 @@ function exceedsNestingBudget(html) {
|
|
|
226
250
|
const selfClosing = match[3] === "/";
|
|
227
251
|
if (selfClosing)
|
|
228
252
|
continue;
|
|
253
|
+
if (VOID_ELEMENTS.has(match[2].toLowerCase()))
|
|
254
|
+
continue;
|
|
229
255
|
if (closing)
|
|
230
256
|
depth = Math.max(0, depth - 1);
|
|
231
257
|
else {
|
|
@@ -463,9 +489,45 @@ function escapeResidualMarkup(markdown) {
|
|
|
463
489
|
// Only `<` that begins a tag-like construct; a bare `a < b` stays readable.
|
|
464
490
|
return markdown.replace(/<(?=[a-zA-Z/!?])/g, "<");
|
|
465
491
|
}
|
|
492
|
+
/**
|
|
493
|
+
* Apply {@link escapeResidualMarkup} everywhere EXCEPT inside fenced code
|
|
494
|
+
* blocks.
|
|
495
|
+
*
|
|
496
|
+
* Markup inside a fence is inert — a renderer shows it as text, it cannot
|
|
497
|
+
* execute — so the escape buys no safety there and actively corrupts content.
|
|
498
|
+
* Turndown emits code with entities already decoded, so a documentation page
|
|
499
|
+
* showing `<div>` in a `<pre><code>` block became a fence containing
|
|
500
|
+
* `<div>`, which this rewrote to `<div>`: a half-escaped, wrong-on-both-ends
|
|
501
|
+
* rendering of the example the page exists to show. Every HTML, XML and JSX
|
|
502
|
+
* snippet in a snapshot was affected.
|
|
503
|
+
*/
|
|
504
|
+
function escapeOutsideCodeFences(markdown) {
|
|
505
|
+
const lines = markdown.split("\n");
|
|
506
|
+
let inFence = false;
|
|
507
|
+
let fenceMarker = "";
|
|
508
|
+
for (let i = 0; i < lines.length; i++) {
|
|
509
|
+
const line = lines[i];
|
|
510
|
+
const fence = /^\s*(`{3,}|~{3,})/.exec(line);
|
|
511
|
+
if (fence) {
|
|
512
|
+
const marker = fence[1];
|
|
513
|
+
if (!inFence) {
|
|
514
|
+
inFence = true;
|
|
515
|
+
fenceMarker = marker[0];
|
|
516
|
+
}
|
|
517
|
+
else if (marker[0] === fenceMarker) {
|
|
518
|
+
inFence = false;
|
|
519
|
+
fenceMarker = "";
|
|
520
|
+
}
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
if (!inFence)
|
|
524
|
+
lines[i] = escapeResidualMarkup(line);
|
|
525
|
+
}
|
|
526
|
+
return lines.join("\n");
|
|
527
|
+
}
|
|
466
528
|
/** Escape residual markup, then normalize whitespace, into the final snapshot. */
|
|
467
529
|
function finalizeMarkdown(markdown) {
|
|
468
|
-
return
|
|
530
|
+
return escapeOutsideCodeFences(markdown)
|
|
469
531
|
.replace(/\r/g, "")
|
|
470
532
|
.replace(/[ \t]+\n/g, "\n")
|
|
471
533
|
.replace(/\n{3,}/g, "\n\n")
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
import { createHash } from "node:crypto";
|
|
4
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import fs from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { fetchWithRetry, isWithin, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir, todayIso, } from "../../core/common.js";
|
|
@@ -201,19 +201,111 @@ async function fetchSnapshotViaRegistry(startUrl, stashDir, allowPrivateHosts, r
|
|
|
201
201
|
};
|
|
202
202
|
return dispatchSnapshotFetchers(parsed, context, stashDir);
|
|
203
203
|
}
|
|
204
|
+
let snapshotWriteHookForTests;
|
|
205
|
+
/**
|
|
206
|
+
* TEST-ONLY. Interrupt a refresh partway through its page-write loop;
|
|
207
|
+
* `undefined` restores. Exists because "a process killed mid-refresh" cannot
|
|
208
|
+
* be staged from outside the module — the whole loop is a single synchronous
|
|
209
|
+
* burst between two awaits. Inert in production (one `undefined?.()` per page).
|
|
210
|
+
*/
|
|
211
|
+
export function _setWebsiteSnapshotWriteHookForTests(hook) {
|
|
212
|
+
snapshotWriteHookForTests = hook;
|
|
213
|
+
}
|
|
214
|
+
function snapshotWriteHook(event) {
|
|
215
|
+
snapshotWriteHookForTests?.(event);
|
|
216
|
+
}
|
|
217
|
+
function snapshotSiblingPrefix(stashDir, kind) {
|
|
218
|
+
return `.${path.basename(stashDir)}.${kind}-`;
|
|
219
|
+
}
|
|
220
|
+
function snapshotSiblingPath(stashDir, kind) {
|
|
221
|
+
const unique = `${process.pid}-${randomBytes(6).toString("hex")}`;
|
|
222
|
+
return path.join(path.dirname(stashDir), `${snapshotSiblingPrefix(stashDir, kind)}${unique}`);
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Age gate for the staging sweep. Nothing enforces one refresh at a time for a
|
|
226
|
+
* given website source, so a sibling directory may belong to a refresh that is
|
|
227
|
+
* still running in another process; deleting it would break a healthy run
|
|
228
|
+
* instead of cleaning up after a dead one. Only clearly-abandoned directories
|
|
229
|
+
* (untouched for an hour — far longer than the 10-minute crawl wall-clock cap)
|
|
230
|
+
* are swept. Leftovers are inert until then: they are dot-prefixed, so the
|
|
231
|
+
* indexer's walk skips them.
|
|
232
|
+
*/
|
|
233
|
+
const SNAPSHOT_STAGING_SWEEP_AGE_MS = 60 * 60 * 1000;
|
|
234
|
+
/** Remove staging/retired directories abandoned by an earlier interrupted run. */
|
|
235
|
+
function sweepSnapshotStaging(stashDir) {
|
|
236
|
+
const parent = path.dirname(stashDir);
|
|
237
|
+
let entries;
|
|
238
|
+
try {
|
|
239
|
+
entries = fs.readdirSync(parent);
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
const prefixes = [snapshotSiblingPrefix(stashDir, "staging"), snapshotSiblingPrefix(stashDir, "retired")];
|
|
245
|
+
const cutoff = Date.now() - SNAPSHOT_STAGING_SWEEP_AGE_MS;
|
|
246
|
+
for (const entry of entries) {
|
|
247
|
+
if (!prefixes.some((prefix) => entry.startsWith(prefix)))
|
|
248
|
+
continue;
|
|
249
|
+
const abandoned = path.join(parent, entry);
|
|
250
|
+
try {
|
|
251
|
+
if (fs.statSync(abandoned).mtimeMs > cutoff)
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
fs.rmSync(abandoned, { recursive: true, force: true });
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
function beginSnapshotStaging(stashDir) {
|
|
261
|
+
fs.mkdirSync(path.dirname(stashDir), { recursive: true });
|
|
262
|
+
sweepSnapshotStaging(stashDir);
|
|
263
|
+
const dir = snapshotSiblingPath(stashDir, "staging");
|
|
264
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
265
|
+
return { dir, target: stashDir };
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Swap the staged snapshot in. POSIX cannot atomically exchange two non-empty
|
|
269
|
+
* directories, so the previous snapshot is renamed ASIDE first and deleted
|
|
270
|
+
* afterwards: the window in which the target does not exist is one syscall
|
|
271
|
+
* wide instead of an entire write loop.
|
|
272
|
+
*/
|
|
273
|
+
function publishSnapshotStaging(staging) {
|
|
274
|
+
let retired;
|
|
275
|
+
if (fs.existsSync(staging.target)) {
|
|
276
|
+
retired = snapshotSiblingPath(staging.target, "retired");
|
|
277
|
+
fs.renameSync(staging.target, retired);
|
|
278
|
+
}
|
|
279
|
+
fs.renameSync(staging.dir, staging.target);
|
|
280
|
+
if (retired)
|
|
281
|
+
fs.rmSync(retired, { recursive: true, force: true });
|
|
282
|
+
}
|
|
283
|
+
/** Drop an unpublished staging directory (no-op once it has been renamed). */
|
|
284
|
+
function discardSnapshotStaging(staging) {
|
|
285
|
+
fs.rmSync(staging.dir, { recursive: true, force: true });
|
|
286
|
+
}
|
|
204
287
|
/** Materialize a single fetcher snapshot as the source's whole stash. */
|
|
205
288
|
function writeSnapshotToStash(stashDir, snapshot) {
|
|
206
289
|
const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
|
|
207
290
|
const relPath = avoidReservedBasename(preferredName);
|
|
291
|
+
// Validate against the FINAL location so the guarantee (and the error text)
|
|
292
|
+
// is independent of where the file is staged.
|
|
208
293
|
const knowledgeDir = path.join(stashDir, "knowledge");
|
|
209
|
-
|
|
210
|
-
if (!isWithin(filePath, knowledgeDir)) {
|
|
294
|
+
if (!isWithin(path.resolve(knowledgeDir, `${relPath}.md`), knowledgeDir)) {
|
|
211
295
|
throw new UsageError(`Snapshot fetcher returned an unsafe preferred name: ${JSON.stringify(preferredName)}`);
|
|
212
296
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
297
|
+
const staging = beginSnapshotStaging(stashDir);
|
|
298
|
+
try {
|
|
299
|
+
const filePath = path.resolve(path.join(staging.dir, "knowledge"), `${relPath}.md`);
|
|
300
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
301
|
+
const slug = relPath.split("/").pop() ?? "index";
|
|
302
|
+
fs.writeFileSync(filePath, buildMarkdownSnapshot({ url: snapshot.url, title: snapshot.title, markdown: snapshot.markdown }, slug, snapshot.tags), "utf8");
|
|
303
|
+
snapshotWriteHook({ point: "page-written", index: 1, total: 1, relPath: `knowledge/${relPath}.md` });
|
|
304
|
+
publishSnapshotStaging(staging);
|
|
305
|
+
}
|
|
306
|
+
finally {
|
|
307
|
+
discardSnapshotStaging(staging);
|
|
308
|
+
}
|
|
217
309
|
}
|
|
218
310
|
async function scrapeWebsiteToStash(startUrl, stashDir, options) {
|
|
219
311
|
// Offer the URL to the specialized fetchers before falling back to a crawl.
|
|
@@ -229,19 +321,33 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
|
|
|
229
321
|
if (pages.length === 0) {
|
|
230
322
|
throw new Error(`No content could be scraped from ${startUrl}`);
|
|
231
323
|
}
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
const
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
324
|
+
const staging = beginSnapshotStaging(stashDir);
|
|
325
|
+
try {
|
|
326
|
+
const knowledgeDir = path.join(staging.dir, "knowledge");
|
|
327
|
+
fs.mkdirSync(knowledgeDir, { recursive: true });
|
|
328
|
+
const usedPaths = new Set();
|
|
329
|
+
let written = 0;
|
|
330
|
+
for (const page of pages) {
|
|
331
|
+
const relPath = avoidReservedBasename(urlToRelativePath(page.url));
|
|
332
|
+
const uniquePath = uniqueSlug(relPath, usedPaths);
|
|
333
|
+
const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
|
|
334
|
+
const dir = path.dirname(filePath);
|
|
335
|
+
if (dir !== knowledgeDir)
|
|
336
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
337
|
+
const slug = uniquePath.split("/").pop() ?? "index";
|
|
338
|
+
fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
|
|
339
|
+
written++;
|
|
340
|
+
snapshotWriteHook({
|
|
341
|
+
point: "page-written",
|
|
342
|
+
index: written,
|
|
343
|
+
total: pages.length,
|
|
344
|
+
relPath: `knowledge/${uniquePath}.md`,
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
publishSnapshotStaging(staging);
|
|
348
|
+
}
|
|
349
|
+
finally {
|
|
350
|
+
discardSnapshotStaging(staging);
|
|
245
351
|
}
|
|
246
352
|
}
|
|
247
353
|
export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
|
package/dist/storage/database.js
CHANGED
|
@@ -149,6 +149,48 @@ function loadBunSqlite() {
|
|
|
149
149
|
return bunSqliteModule;
|
|
150
150
|
}
|
|
151
151
|
let betterSqlite3Ctor;
|
|
152
|
+
/**
|
|
153
|
+
* The binding is absent or unbuildable — a toolchain/install problem.
|
|
154
|
+
*/
|
|
155
|
+
const MISSING_BINDING_REMEDY = "akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
|
|
156
|
+
" • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
|
|
157
|
+
" 'better-sqlite3' native binding builds (a global `npm i -g better-sqlite3`\n" +
|
|
158
|
+
" will NOT be resolved — Node loads it from akm's own node_modules).\n" +
|
|
159
|
+
" • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.";
|
|
160
|
+
/**
|
|
161
|
+
* Recognize a binding built for a DIFFERENT Node ABI than the one now running,
|
|
162
|
+
* and answer with the one command that fixes it.
|
|
163
|
+
*
|
|
164
|
+
* This is the most likely failure a real user hits, and it is not a broken
|
|
165
|
+
* install: a native addon is compiled (or a prebuilt binary is selected) for
|
|
166
|
+
* the Node major present at `npm install` time. Upgrade Node afterwards and the
|
|
167
|
+
* same file no longer loads.
|
|
168
|
+
*
|
|
169
|
+
* It is matched rather than described because the symptom text varies and the
|
|
170
|
+
* previous wording only named ONE of them. The prebuilt-binary path — which is
|
|
171
|
+
* now the normal path, since better-sqlite3 is pinned to a version publishing a
|
|
172
|
+
* prebuild for every supported Node (see package.json → pinNotes) — reports
|
|
173
|
+
* `Module did not self-register`, saying nothing about versions at all. A
|
|
174
|
+
* from-source build reports the explicit `NODE_MODULE_VERSION` mismatch. Asking
|
|
175
|
+
* the user to decide which bullet applies is the step worth deleting.
|
|
176
|
+
*/
|
|
177
|
+
export function abiMismatchRemedy(message) {
|
|
178
|
+
const ABI_MISMATCH_SHAPES = [
|
|
179
|
+
"did not self-register", // prebuilt binary for another ABI
|
|
180
|
+
"NODE_MODULE_VERSION", // explicit mismatch, from-source build
|
|
181
|
+
"was compiled against a different", // same, older phrasing
|
|
182
|
+
"invalid ELF header", // binary for another platform/arch entirely
|
|
183
|
+
];
|
|
184
|
+
if (!ABI_MISMATCH_SHAPES.some((shape) => message.includes(shape)))
|
|
185
|
+
return undefined;
|
|
186
|
+
return ("akm could not load 'better-sqlite3': its native binding was built for a different\n" +
|
|
187
|
+
`Node.js version than the one now running (this Node is ABI ${process.versions.modules}).\n` +
|
|
188
|
+
"This is what happens when Node is upgraded after akm is installed. It is NOT a\n" +
|
|
189
|
+
"broken install, and reinstalling akm is not required.\n" +
|
|
190
|
+
" Fix: npm rebuild better-sqlite3 # in akm's install directory\n" +
|
|
191
|
+
" Or: npm install -g akm-cli # reinstall, rebuilding against this Node\n" +
|
|
192
|
+
" Or: run akm under Bun, whose built-in SQLite driver needs no native binding.");
|
|
193
|
+
}
|
|
152
194
|
function loadBetterSqlite3() {
|
|
153
195
|
if (!betterSqlite3Ctor) {
|
|
154
196
|
// Runtime-gated dynamic require: only reached when NOT on Bun, so Bun never
|
|
@@ -161,12 +203,13 @@ function loadBetterSqlite3() {
|
|
|
161
203
|
mod = nodeRequire("better-sqlite3");
|
|
162
204
|
}
|
|
163
205
|
catch (err) {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
206
|
+
// An ABI mismatch does NOT arrive here — `require` succeeds and the
|
|
207
|
+
// failure lands at construction (see openNodeDatabase). This path is a
|
|
208
|
+
// genuinely absent or unresolvable module. `abiMismatchRemedy` is still
|
|
209
|
+
// consulted because a from-source build CAN fail at load with the
|
|
210
|
+
// explicit NODE_MODULE_VERSION message.
|
|
211
|
+
const raw = err instanceof Error ? err.message : String(err);
|
|
212
|
+
throw new Error(`${abiMismatchRemedy(raw) ?? MISSING_BINDING_REMEDY}\n Underlying load error: ${raw}`);
|
|
170
213
|
}
|
|
171
214
|
betterSqlite3Ctor = mod.default ?? mod;
|
|
172
215
|
}
|
|
@@ -183,7 +226,24 @@ function openNodeDatabase(path, opts) {
|
|
|
183
226
|
options.readonly = opts.readonly;
|
|
184
227
|
if (opts?.create === false)
|
|
185
228
|
options.fileMustExist = true;
|
|
186
|
-
|
|
229
|
+
// Construction, not `require`, is where an ABI mismatch surfaces.
|
|
230
|
+
// `require("better-sqlite3")` SUCCEEDS against a binding built for another
|
|
231
|
+
// Node ABI — the package resolves its `.node` file lazily — so the loader's
|
|
232
|
+
// catch never sees this error and cannot explain it. Verified against a real
|
|
233
|
+
// ABI-127 binding under Node 24 (ABI 137): `require()` returned a function
|
|
234
|
+
// and `new Database(...)` threw. Wrapping the require alone left the most
|
|
235
|
+
// likely real-world failure reported as a bare Node internals message.
|
|
236
|
+
let db;
|
|
237
|
+
try {
|
|
238
|
+
db = opts ? new BetterSqlite3(path, options) : new BetterSqlite3(path);
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
const raw = err instanceof Error ? err.message : String(err);
|
|
242
|
+
const remedy = abiMismatchRemedy(raw);
|
|
243
|
+
if (!remedy)
|
|
244
|
+
throw err;
|
|
245
|
+
throw new Error(`${remedy}\n Underlying error: ${raw}`);
|
|
246
|
+
}
|
|
187
247
|
return {
|
|
188
248
|
prepare: db.prepare.bind(db),
|
|
189
249
|
exec: db.exec.bind(db),
|
|
@@ -191,6 +251,10 @@ function openNodeDatabase(path, opts) {
|
|
|
191
251
|
// bun:sqlite also provides db.run(). Normalize the latter at the provider
|
|
192
252
|
// boundary so callers and maintenance wrappers can rely on one contract.
|
|
193
253
|
run: (sql, ...params) => db.prepare(sql).run(...params),
|
|
254
|
+
// sqlite-vec's load(db) calls db.loadExtension(). Without forwarding it the
|
|
255
|
+
// extension could never load on Node, so the vector fast path was dead
|
|
256
|
+
// across the entire npm distribution even when sqlite-vec was installed.
|
|
257
|
+
loadExtension: db.loadExtension.bind(db),
|
|
194
258
|
transaction: db.transaction.bind(db),
|
|
195
259
|
get inTransaction() {
|
|
196
260
|
return db.inTransaction;
|
|
@@ -100,9 +100,68 @@ export function runMigrations(db, migrations, opts) {
|
|
|
100
100
|
if (applied.has(migration.id))
|
|
101
101
|
continue;
|
|
102
102
|
opts?.beforeMigration?.(migration);
|
|
103
|
-
db
|
|
103
|
+
withImmediateWriteLock(db, () => {
|
|
104
|
+
// Re-check under the write lock. `applied` is a snapshot taken before the
|
|
105
|
+
// loop, so two processes bootstrapping the same fresh DB concurrently
|
|
106
|
+
// (both see existed=false, both run with applyPending) could each decide
|
|
107
|
+
// to apply migration N. The first commits; the second must not re-run the
|
|
108
|
+
// DDL and must not hit a UNIQUE violation on the ledger insert.
|
|
109
|
+
const already = db.prepare("SELECT 1 FROM schema_migrations WHERE id = ?").get(migration.id);
|
|
110
|
+
if (already)
|
|
111
|
+
return;
|
|
104
112
|
db.exec(migration.up);
|
|
105
113
|
db.prepare("INSERT INTO schema_migrations (id) VALUES (?)").run(migration.id);
|
|
106
|
-
})
|
|
114
|
+
});
|
|
115
|
+
applied.add(migration.id);
|
|
107
116
|
}
|
|
108
117
|
}
|
|
118
|
+
/** Attempts to acquire the write lock before giving up to the caller. */
|
|
119
|
+
const IMMEDIATE_LOCK_MAX_ATTEMPTS = 5;
|
|
120
|
+
/**
|
|
121
|
+
* Run `fn` inside a `BEGIN IMMEDIATE` transaction.
|
|
122
|
+
*
|
|
123
|
+
* The write lock is taken up front rather than upgraded from a read lock, so a
|
|
124
|
+
* second process bootstrapping the same database WAITS for the first to commit
|
|
125
|
+
* instead of racing it. `db.transaction()` opens a DEFERRED transaction, which
|
|
126
|
+
* only takes the write lock on first write — leaving the read-then-write gap
|
|
127
|
+
* this guards.
|
|
128
|
+
*
|
|
129
|
+
* Deliberately local rather than reusing `withImmediateTransaction` from
|
|
130
|
+
* core/state-db: that module imports this one, so the dependency cannot be
|
|
131
|
+
* pointed the other way.
|
|
132
|
+
*/
|
|
133
|
+
function withImmediateWriteLock(db, fn) {
|
|
134
|
+
if (db.inTransaction) {
|
|
135
|
+
fn();
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
let lastBeginErr;
|
|
139
|
+
for (let attempt = 1; attempt <= IMMEDIATE_LOCK_MAX_ATTEMPTS; attempt++) {
|
|
140
|
+
try {
|
|
141
|
+
db.exec("BEGIN IMMEDIATE");
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
// Busy despite busy_timeout (another writer holding it across the whole
|
|
145
|
+
// window). Retry a bounded number of times before surfacing.
|
|
146
|
+
lastBeginErr = err;
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
try {
|
|
150
|
+
fn();
|
|
151
|
+
db.exec("COMMIT");
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
catch (err) {
|
|
155
|
+
try {
|
|
156
|
+
db.exec("ROLLBACK");
|
|
157
|
+
}
|
|
158
|
+
catch {
|
|
159
|
+
// Already rolled back by SQLite (e.g. the statement aborted the txn).
|
|
160
|
+
}
|
|
161
|
+
throw err;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
throw lastBeginErr instanceof Error
|
|
165
|
+
? lastBeginErr
|
|
166
|
+
: new Error(`could not acquire the migration write lock after ${IMMEDIATE_LOCK_MAX_ATTEMPTS} attempts`);
|
|
167
|
+
}
|
|
@@ -26,6 +26,25 @@ import { applyStandardPragmas } from "./sqlite-pragmas.js";
|
|
|
26
26
|
* Open a managed SQLite database: ensure the parent dir exists, open the handle,
|
|
27
27
|
* apply standard pragmas, then run the schema initializer. The single home for
|
|
28
28
|
* the open→pragmas→migrate recipe.
|
|
29
|
+
*
|
|
30
|
+
* ── On file permissions (reverted, issue #791) ──
|
|
31
|
+
*
|
|
32
|
+
* This function briefly chmodded the database, its `-wal`/`-shm` sidecars, and
|
|
33
|
+
* THE CONTAINING DIRECTORY to owner-only on every open (#756). That was a
|
|
34
|
+
* mistake and is deliberately not coming back:
|
|
35
|
+
*
|
|
36
|
+
* - It mutated state akm did not create. The data directory belongs to the
|
|
37
|
+
* operator; a read of the index is not consent to re-permission their disk.
|
|
38
|
+
* - It ran on the most-traveled path in the CLI, including `create: false`
|
|
39
|
+
* (read-only) opens, so any command at all silently converted a legacy
|
|
40
|
+
* `0755` directory to `0700` with no prompt, warning, or migration note.
|
|
41
|
+
* - It therefore broke installs that share `$XDG_DATA_HOME` across uids —
|
|
42
|
+
* agent sandboxes, containers, service accounts — which worked in 0.9.0.
|
|
43
|
+
* Worse, the read path answers an unreadable index with a false
|
|
44
|
+
* "No search index available" at exit 0 rather than an error (#791).
|
|
45
|
+
*
|
|
46
|
+
* Files akm creates here get the process umask, which is the operator's lever
|
|
47
|
+
* for this and always was. akm neither sets these modes nor reports on them.
|
|
29
48
|
*/
|
|
30
49
|
export function openManagedDatabase(spec) {
|
|
31
50
|
const dir = path.dirname(spec.path);
|