akm-cli 0.9.1 → 0.9.2-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +103 -28
- package/README.md +3 -1
- package/SECURITY.md +1 -1
- package/STABILITY.md +1 -1
- package/dist/akm +2 -2
- package/dist/akm-migrate +2 -2
- package/dist/assets/hints/cli-hints-full.md +14 -9
- package/dist/assets/improve-strategies/proactive-maintenance.json +1 -1
- package/dist/assets/improve-strategies/reflect-distill.json +1 -1
- package/dist/assets/models.json +35 -0
- package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +3 -4
- package/dist/assets/stash-skeleton/facts/conventions/organization.md +1 -3
- package/dist/assets/tasks/core/extract.yml +6 -5
- package/dist/assets/tasks/core/improve.yml +6 -5
- package/dist/assets/tasks/core/index-refresh.yml +6 -5
- package/dist/assets/tasks/core/sync.yml +6 -5
- package/dist/assets/tasks/core/version-check.yml +6 -5
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-consolidate.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-frequent.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-nightly.yml +6 -5
- package/dist/cli/confirm.js +2 -2
- package/dist/cli/parse-args.js +3 -24
- package/dist/cli/retired-commands.js +1 -1
- package/dist/cli/shared.js +2 -2
- package/dist/cli.js +11 -9
- package/dist/commands/agent/agent-dispatch.js +55 -89
- package/dist/commands/agent/contribute-cli.js +12 -45
- package/dist/commands/command/builtin-action.js +32 -0
- package/dist/commands/command/command-cli.js +99 -0
- package/dist/commands/command/command-execution.js +308 -0
- package/dist/commands/command/execution-source-loader.js +176 -0
- package/dist/commands/command/portable-template.js +60 -0
- package/dist/commands/config-cli.js +10 -4
- package/dist/commands/env/env.js +4 -2
- package/dist/commands/feedback-cli.js +1 -1
- package/dist/commands/health/checks.js +241 -29
- package/dist/commands/health/html-report.js +0 -14
- package/dist/commands/health/report-view-model.js +0 -1
- package/dist/commands/health/surfaces.js +6 -7
- package/dist/commands/health/types.js +0 -2
- package/dist/commands/health.js +63 -18
- package/dist/commands/improve/collapse-detector.js +5 -6
- package/dist/commands/improve/consolidate.js +251 -214
- package/dist/commands/improve/distill/promote-memory.js +71 -34
- package/dist/commands/improve/distill/quality-gate.js +17 -5
- package/dist/commands/improve/distill.js +232 -155
- package/dist/commands/improve/eligibility.js +112 -79
- package/dist/commands/improve/execution.js +57 -0
- package/dist/commands/improve/extract-cli.js +5 -5
- package/dist/commands/improve/extract-prompt.js +64 -22
- package/dist/commands/improve/extract.js +608 -360
- package/dist/commands/improve/improve-strategies.js +43 -14
- package/dist/commands/improve/improve.js +249 -29
- package/dist/commands/improve/loop-stages.js +11 -17
- package/dist/commands/improve/memory/memory-contradiction-detect.js +90 -66
- package/dist/commands/improve/outcome-loop.js +22 -38
- package/dist/commands/improve/planner.js +134 -0
- package/dist/commands/improve/preparation.js +730 -409
- package/dist/commands/improve/reflect.js +386 -223
- package/dist/commands/improve/run-context.js +3 -4
- package/dist/commands/improve/salience.js +6 -58
- package/dist/commands/improve/session-asset.js +12 -12
- package/dist/commands/lint/index.js +101 -29
- package/dist/commands/migrate-cli.js +11 -69
- package/dist/commands/migration-tool.js +6 -9
- package/dist/commands/models-cli.js +27 -0
- package/dist/commands/proposal/drain.js +258 -186
- package/dist/commands/proposal/proposal-cli.js +32 -10
- package/dist/commands/proposal/proposal.js +2 -5
- package/dist/commands/proposal/propose.js +192 -172
- package/dist/commands/proposal/repository.js +54 -91
- package/dist/commands/proposal/validators/proposal-validators.js +9 -7
- package/dist/commands/read/curate.js +53 -22
- package/dist/commands/read/registry-search.js +25 -9
- package/dist/commands/read/remember-cli.js +14 -2
- package/dist/commands/read/search.js +10 -4
- package/dist/commands/read/show.js +139 -153
- package/dist/commands/registry-cli.js +16 -7
- package/dist/commands/remember.js +33 -18
- package/dist/commands/sources/add-cli.js +19 -178
- package/dist/commands/sources/bundle-cli.js +15 -3
- package/dist/commands/sources/dangerous-env-audit.js +135 -0
- package/dist/commands/sources/info.js +2 -1
- package/dist/commands/sources/installed-stashes.js +901 -177
- package/dist/commands/sources/schema-repair.js +174 -95
- package/dist/commands/sources/self-update.js +30 -74
- package/dist/commands/sources/source-add.js +3 -5
- package/dist/commands/sources/sources-cli.js +2 -15
- package/dist/commands/sources/update-transaction.js +220 -0
- package/dist/commands/tasks/tasks-cli.js +3 -3
- package/dist/commands/tasks/tasks.js +736 -317
- package/dist/commands/workflow-cli.js +2 -2
- package/dist/core/adapter/adapters/agent-skills-adapter.js +3 -0
- package/dist/core/adapter/adapters/akm-adapter.js +85 -35
- package/dist/core/adapter/adapters/akm-lint.js +54 -39
- package/dist/core/adapter/adapters/akm-metadata.js +45 -45
- package/dist/core/adapter/adapters/akm-task-adapter.js +32 -49
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +38 -23
- package/dist/core/adapter/adapters/dotenv-adapter.js +30 -1
- package/dist/core/adapter/adapters/generic-files-adapter.js +11 -0
- package/dist/core/adapter/adapters/index.js +0 -9
- package/dist/core/adapter/adapters/llm-wiki-adapter.js +4 -0
- package/dist/core/adapter/adapters/okf-adapter.js +4 -0
- package/dist/core/adapter/adapters/opencode-adapter.js +5 -8
- package/dist/core/adapter/adapters/tool-dir-shared.js +63 -6
- package/dist/core/adapter/adapters/website-snapshot-adapter.js +4 -0
- package/dist/core/adapter/execution-source.js +308 -0
- package/dist/core/adapter/recognize-match.js +36 -13
- package/dist/core/adapter/registry.js +0 -9
- package/dist/core/asset/stash-meta.js +94 -4
- package/dist/core/common.js +6 -11
- package/dist/core/config/config-io.js +3 -3
- package/dist/core/config/config-schema.js +18 -40
- package/dist/core/config/config-sources.js +11 -21
- package/dist/core/config/config-walker.js +31 -13
- package/dist/core/config/config.js +23 -26
- package/dist/core/config/schema/engines.js +8 -7
- package/dist/core/config/schema/improve-processes.js +29 -5
- package/dist/core/config/schema/index-config.js +0 -27
- package/dist/core/config/schema/primitives.js +1 -23
- package/dist/core/config/schema/sources-bundles.js +13 -16
- package/dist/core/errors.js +2 -0
- package/dist/core/events.js +68 -32
- package/dist/core/extra-params.js +1 -0
- package/dist/core/improve-result.js +315 -0
- package/dist/core/lesson-lint.js +0 -6
- package/dist/core/maintenance-barrier.js +4 -4
- package/dist/core/network-policy.js +152 -0
- package/dist/core/paths.js +1 -1
- package/dist/core/recognition-util.js +4 -4
- package/dist/core/registry-url.js +456 -0
- package/dist/core/state/migrations.js +161 -47
- package/dist/core/state-db.js +453 -80
- package/dist/core/system-error.js +32 -0
- package/dist/core/time.js +2 -12
- package/dist/core/write-source.js +0 -18
- package/dist/execution/directory-identity.js +52 -0
- package/dist/execution/executable-identity.js +107 -0
- package/dist/execution/guarded-source.js +398 -0
- package/dist/execution/json.js +95 -0
- package/dist/{commands/health/types-session-log.js → execution/limits.js} +2 -1
- package/dist/execution/record.js +55 -0
- package/dist/execution/resolved-request.js +730 -0
- package/dist/execution/source.js +320 -0
- package/dist/indexer/bundle-identity-guard.js +5 -4
- package/dist/indexer/db/graph-db.js +33 -0
- package/dist/indexer/graph/graph-boost.js +3 -4
- package/dist/indexer/graph/graph-extraction.js +562 -373
- package/dist/indexer/index-written-assets.js +78 -39
- package/dist/indexer/indexer.js +471 -432
- package/dist/indexer/installations.js +6 -0
- package/dist/indexer/lookup/adapter-concept-owner.js +283 -0
- package/dist/indexer/materialize-embeddings.js +155 -0
- package/dist/indexer/passes/memory-inference.js +227 -174
- package/dist/indexer/passes/metadata.js +263 -118
- package/dist/indexer/scan/doc-to-entry.js +7 -10
- package/dist/indexer/scan/drain-dir.js +51 -23
- package/dist/indexer/search/db-search.js +156 -50
- package/dist/indexer/search/fts-query.js +40 -40
- package/dist/indexer/search/ranking.js +36 -1
- package/dist/indexer/search/search-attribution.js +3 -1
- package/dist/indexer/search/search-fields.js +23 -14
- package/dist/indexer/search/search-hit-enrichers.js +1 -1
- package/dist/indexer/search/search-source.js +7 -16
- package/dist/indexer/search/semantic-status.js +10 -1
- package/dist/indexer/usage/show-usage.js +105 -0
- package/dist/indexer/usage/usage-events.js +7 -2
- package/dist/indexer/walk/matchers.js +40 -10
- package/dist/indexer/walk/path-resolver.js +5 -2
- package/dist/indexer/walk/walker.js +20 -2
- package/dist/integrations/agent/builder-shared.js +3 -6
- package/dist/integrations/agent/conversation-fallback.js +16 -0
- package/dist/integrations/agent/engine-resolution.js +87 -87
- package/dist/integrations/agent/execution-cascade.js +566 -0
- package/dist/integrations/agent/execution-definitions.js +211 -0
- package/dist/integrations/agent/execution-lowering.js +811 -0
- package/dist/integrations/agent/execution-preparation.js +67 -0
- package/dist/integrations/agent/index.js +0 -2
- package/dist/integrations/agent/inline-execution.js +74 -0
- package/dist/integrations/agent/model-map.js +515 -0
- package/dist/integrations/agent/persona-fallback.js +30 -0
- package/dist/integrations/agent/request-lowering.js +186 -0
- package/dist/integrations/agent/runner-dispatch.js +230 -37
- package/dist/integrations/agent/runner.js +12 -83
- package/dist/integrations/harnesses/aider/agent-builder.js +8 -0
- package/dist/integrations/harnesses/aider/index.js +0 -1
- package/dist/integrations/harnesses/amazonq/agent-builder.js +8 -0
- package/dist/integrations/harnesses/amazonq/index.js +0 -1
- package/dist/integrations/harnesses/claude/agent-builder.js +14 -1
- package/dist/integrations/harnesses/claude/index.js +1 -5
- package/dist/integrations/harnesses/claude/session-log.js +3 -33
- package/dist/integrations/harnesses/codex/agent-builder.js +8 -0
- package/dist/integrations/harnesses/codex/index.js +0 -1
- package/dist/integrations/harnesses/copilot/agent-builder.js +8 -0
- package/dist/integrations/harnesses/copilot/index.js +0 -1
- package/dist/integrations/harnesses/gemini/agent-builder.js +8 -0
- package/dist/integrations/harnesses/gemini/index.js +0 -1
- package/dist/integrations/harnesses/index.js +4 -44
- package/dist/integrations/harnesses/opencode/agent-builder.js +16 -9
- package/dist/integrations/harnesses/opencode/index.js +0 -2
- package/dist/integrations/harnesses/opencode/session-log.js +14 -204
- package/dist/integrations/harnesses/opencode-sdk/harness.js +12 -1
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +40 -42
- package/dist/integrations/harnesses/openhands/agent-builder.js +8 -0
- package/dist/integrations/harnesses/openhands/index.js +0 -1
- package/dist/integrations/harnesses/pi/agent-builder.js +8 -0
- package/dist/integrations/harnesses/pi/index.js +0 -1
- package/dist/integrations/harnesses/shared.js +0 -1
- package/dist/integrations/harnesses/types.js +1 -3
- package/dist/integrations/lockfile.js +82 -79
- package/dist/integrations/session-logs/index.js +6 -17
- package/dist/integrations/session-logs/provider-base.js +1 -29
- package/dist/llm/client.js +10 -5
- package/dist/llm/embedder.js +6 -7
- package/dist/llm/embedders/local.js +37 -88
- package/dist/llm/embedders/types.js +1 -1
- package/dist/llm/graph-extract.js +75 -50
- package/dist/llm/index-passes.js +43 -5
- package/dist/llm/memory-infer.js +8 -6
- package/dist/llm/metadata-enhance.js +5 -3
- package/dist/llm/structured-call.js +122 -25
- package/dist/output/format-exempt.js +1 -1
- package/dist/output/render-registry.js +0 -16
- package/dist/output/renderers.js +12 -7
- package/dist/output/shapes/curate.js +1 -0
- package/dist/output/shapes/helpers.js +10 -2
- package/dist/output/shapes/passthrough.js +2 -0
- package/dist/output/text/command-format.js +31 -33
- package/dist/output/text/health-format.js +1 -29
- package/dist/output/text/migrate.js +6 -56
- package/dist/output/text/proposal-format.js +16 -1
- package/dist/output/text/workflow-format.js +16 -0
- package/dist/registry/network.js +279 -0
- package/dist/registry/pinned-request-helper.js +247 -0
- package/dist/registry/pinned-transport.js +717 -0
- package/dist/registry/providers/skills-sh.js +18 -6
- package/dist/registry/providers/static-index.js +20 -7
- package/dist/registry/resolve.js +53 -28
- package/dist/scripts/akm-migrate-node.js +19334 -52269
- package/dist/scripts/akm-migrate.js +19270 -51612
- package/dist/setup/registry-stash-loader.js +64 -20
- package/dist/setup/semantic-assets.js +9 -34
- package/dist/setup/setup.js +12 -30
- package/dist/setup/source-identity.js +17 -0
- package/dist/setup/steps/sources.js +36 -15
- package/dist/setup/steps/tasks.js +39 -11
- package/dist/sources/providers/git-provider.js +3 -3
- package/dist/sources/providers/npm.js +2 -2
- package/dist/sources/providers/provider-utils.js +4 -3
- package/dist/sources/providers/website.js +11 -7
- package/dist/sources/snapshot-fetchers/host-guard.js +9 -136
- package/dist/sources/snapshot-fetchers/website-ingest.js +25 -109
- package/dist/sources/website-url.js +73 -0
- package/dist/storage/engines/sqlite-migrations.js +81 -26
- package/dist/storage/managed-db.js +27 -24
- package/dist/storage/repositories/events-repository.js +3 -0
- package/dist/storage/repositories/index-connection.js +42 -10
- package/dist/storage/repositories/index-entries-repository.js +203 -229
- package/dist/storage/repositories/index-entry-mapper.js +8 -12
- package/dist/storage/repositories/index-entry-schema.js +255 -0
- package/dist/storage/repositories/index-fts-repository.js +64 -71
- package/dist/storage/repositories/index-llm-cache-repository.js +8 -13
- package/dist/storage/repositories/index-meta-repository.js +0 -11
- package/dist/storage/repositories/index-schema.js +74 -350
- package/dist/storage/repositories/index-utility-repository.js +12 -17
- package/dist/storage/repositories/index-vec-repository.js +56 -7
- package/dist/storage/repositories/proposals-repository.js +4 -127
- package/dist/storage/repositories/registry-cache.js +2 -1
- package/dist/storage/repositories/task-history-repository.js +20 -40
- package/dist/storage/repositories/workflow-runs-repository.js +228 -129
- package/dist/storage/sqlite-read-snapshot.js +148 -0
- package/dist/tasks/backends/cron.js +170 -42
- package/dist/tasks/backends/index.js +1 -1
- package/dist/tasks/backends/launchd.js +787 -202
- package/dist/tasks/backends/schtasks.js +282 -83
- package/dist/tasks/embedded.js +7 -7
- package/dist/tasks/frozen-script.js +50 -0
- package/dist/tasks/resolve-akm-bin.js +5 -1
- package/dist/tasks/runner.js +239 -251
- package/dist/tasks/runtime-v3.js +281 -0
- package/dist/tasks/scheduler-binding.js +272 -0
- package/dist/tasks/scheduler-invocation.js +57 -43
- package/dist/tasks/scheduler-sync.js +654 -0
- package/dist/tasks/source-v3.js +752 -0
- package/dist/tasks/standalone-script-entry.js +5 -0
- package/dist/tasks/task-id.js +29 -0
- package/dist/workflows/authoring/authoring.js +15 -32
- package/dist/workflows/exec/dispatch-redaction.js +14 -8
- package/dist/workflows/exec/exec-unit.js +7 -28
- package/dist/workflows/exec/frozen-judge.js +57 -89
- package/dist/workflows/exec/lowering-notices.js +23 -0
- package/dist/workflows/exec/native-executor.js +301 -458
- package/dist/workflows/exec/param-secrets.js +4 -3
- package/dist/workflows/exec/run-workflow.js +26 -32
- package/dist/workflows/exec/step-work.js +105 -109
- package/dist/workflows/exec/unit-dispatch.js +103 -27
- package/dist/workflows/exec/unit-writer.js +3 -3
- package/dist/workflows/exec/worktree.js +2 -2
- package/dist/workflows/ir/compile.js +86 -72
- package/dist/workflows/ir/environment-v4.js +328 -0
- package/dist/workflows/ir/freeze-v4.js +122 -0
- package/dist/workflows/ir/plan-hash.js +13 -7
- package/dist/workflows/ir/schema-v4.js +525 -0
- package/dist/workflows/ir/schema.js +25 -284
- package/dist/workflows/ir/source-freeze-v4.js +506 -0
- package/dist/workflows/parser.js +27 -24
- package/dist/workflows/program/schema.js +1 -2
- package/dist/workflows/renderer.js +42 -29
- package/dist/workflows/resource-limits.js +4 -5
- package/dist/workflows/runtime/agent-identity.js +11 -13
- package/dist/workflows/runtime/plan-classifier.js +8 -8
- package/dist/workflows/runtime/runs.js +27 -43
- package/dist/workflows/runtime/workflow-asset-loader.js +45 -205
- package/dist/workflows/source-files.js +373 -0
- package/dist/workflows/source-ir/compile.js +196 -0
- package/dist/workflows/source-ir/github-yaml.js +577 -0
- package/dist/workflows/source-ir/ordering.js +38 -0
- package/dist/workflows/source-ir/program.js +50 -0
- package/dist/workflows/source-ir/result.js +26 -0
- package/dist/workflows/source-ir/schema.js +772 -0
- package/dist/workflows/source-ir/semantics.js +242 -0
- package/dist/workflows/source-ir/uses.js +14 -0
- package/docs/README.md +2 -0
- package/docs/migration/README.md +3 -1
- package/docs/migration/release-notes/0.9.2.md +55 -0
- package/docs/migration/release-notes/README.md +5 -0
- package/docs/migration/v0.8-to-v0.9.md +76 -1077
- package/docs/migration/v0.9.0-troubleshooting.md +104 -516
- package/docs/migration/v0.9.1-to-v0.9.2.md +150 -0
- package/docs/reference/README.md +1 -0
- package/docs/reference/cli.md +230 -98
- package/docs/reference/configuration.md +159 -36
- package/docs/reference/data-and-telemetry.md +19 -1
- package/docs/reference/supported-formats.md +23 -3
- package/docs/reference/tasks.md +182 -0
- package/docs/reference/workflow-schema.md +91 -40
- package/docs/reference/workflows.md +33 -6
- package/package.json +10 -6
- package/schemas/akm-config.json +372 -224
- package/schemas/akm-task.json +324 -80
- package/schemas/akm-workflow.json +6 -9
- package/dist/core/migration-operation.js +0 -75
- package/dist/integrations/agent/model-aliases.js +0 -74
- package/dist/tasks/parser.js +0 -380
- package/dist/tasks/schema.js +0 -123
- package/dist/tasks/validator.js +0 -80
- package/dist/workflows/ir/freeze.js +0 -320
- package/dist/workflows/runtime/document-cache.js +0 -13
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
import { lookup as dnsLookup } from "node:dns/promises";
|
|
5
|
-
import { isIP } from "node:net";
|
|
6
4
|
import { fetchWithRetry, readBodyWithByteCap } from "../../core/common.js";
|
|
5
|
+
import { bareHostname, classifyNetworkAddress, classifyNetworkHostname, resolveHostnameAddresses, } from "../../core/network-policy.js";
|
|
7
6
|
/** Redirect budget for a single guarded request chain. */
|
|
8
7
|
const MAX_GUARDED_REDIRECTS = 8;
|
|
9
8
|
export function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
|
|
@@ -22,10 +21,6 @@ export function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
|
|
|
22
21
|
throw new ErrorType(`Refusing to fetch non-public website host: ${parsedUrl.hostname}`);
|
|
23
22
|
}
|
|
24
23
|
}
|
|
25
|
-
async function defaultResolveHostname(hostname) {
|
|
26
|
-
const records = await dnsLookup(hostname, { all: true });
|
|
27
|
-
return records.map((record) => record.address);
|
|
28
|
-
}
|
|
29
24
|
/**
|
|
30
25
|
* Resolve-then-validate SSRF guard against DNS rebinding / private-range
|
|
31
26
|
* bypasses. {@link assertWebsiteRequestUrl} only rejects IP-literal and
|
|
@@ -46,18 +41,16 @@ async function defaultResolveHostname(hostname) {
|
|
|
46
41
|
export async function assertResolvedHostAllowed(hostname, options) {
|
|
47
42
|
if (options?.allowPrivateHosts === true)
|
|
48
43
|
return;
|
|
49
|
-
const bare =
|
|
50
|
-
const
|
|
51
|
-
if (
|
|
52
|
-
|
|
53
|
-
if (forbidden)
|
|
44
|
+
const bare = bareHostname(hostname.toLowerCase());
|
|
45
|
+
const literalClass = classifyNetworkAddress(bare);
|
|
46
|
+
if (literalClass !== "invalid") {
|
|
47
|
+
if (literalClass !== "public")
|
|
54
48
|
throw new Error(`Refusing to fetch non-public website host: ${hostname}`);
|
|
55
49
|
return;
|
|
56
50
|
}
|
|
57
|
-
const resolve = options?.resolveHostname ?? defaultResolveHostname;
|
|
58
51
|
let addresses;
|
|
59
52
|
try {
|
|
60
|
-
addresses = await
|
|
53
|
+
addresses = await resolveHostnameAddresses(bare, options?.resolveHostname);
|
|
61
54
|
}
|
|
62
55
|
catch {
|
|
63
56
|
throw new Error(`Refusing to fetch ${hostname}: DNS resolution failed`);
|
|
@@ -66,138 +59,18 @@ export async function assertResolvedHostAllowed(hostname, options) {
|
|
|
66
59
|
throw new Error(`Refusing to fetch ${hostname}: hostname resolved to no addresses`);
|
|
67
60
|
}
|
|
68
61
|
for (const address of addresses) {
|
|
69
|
-
|
|
70
|
-
const forbidden = version === 4 ? isForbiddenIpv4(address) : version === 6 ? isForbiddenIpv6(stripIpv6Brackets(address)) : true;
|
|
71
|
-
if (forbidden) {
|
|
62
|
+
if (classifyNetworkAddress(address) !== "public") {
|
|
72
63
|
throw new Error(`Refusing to fetch ${hostname}: resolves to non-public or unparseable address ${address}`);
|
|
73
64
|
}
|
|
74
65
|
}
|
|
75
66
|
}
|
|
76
|
-
// WHATWG URL.hostname wraps IPv6 literals in brackets (e.g. "[::1]"), but
|
|
77
|
-
// node:net's isIP() only recognizes the bare address form and returns 0 for
|
|
78
|
-
// anything bracketed — silently skipping all IPv6 forbidden-host checks
|
|
79
|
-
// below for every hostname parsed off a URL. Strip the brackets before any
|
|
80
|
-
// isIP()/isForbiddenIpv6() call so those checks actually run.
|
|
81
|
-
function stripIpv6Brackets(hostname) {
|
|
82
|
-
return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
|
|
83
|
-
}
|
|
84
67
|
function isForbiddenWebsiteHostname(hostname, options) {
|
|
85
68
|
if (options?.allowPrivateHosts === true)
|
|
86
69
|
return false;
|
|
87
|
-
|
|
88
|
-
return true;
|
|
89
|
-
}
|
|
90
|
-
const bareHostname = stripIpv6Brackets(hostname);
|
|
91
|
-
const ipVersion = isIP(bareHostname);
|
|
92
|
-
if (ipVersion === 4)
|
|
93
|
-
return isForbiddenIpv4(bareHostname);
|
|
94
|
-
if (ipVersion === 6)
|
|
95
|
-
return isForbiddenIpv6(bareHostname);
|
|
96
|
-
return false;
|
|
70
|
+
return classifyNetworkHostname(hostname) !== "public";
|
|
97
71
|
}
|
|
98
72
|
export function isLoopbackWebsiteHostname(hostname) {
|
|
99
|
-
|
|
100
|
-
return true;
|
|
101
|
-
const bareHostname = stripIpv6Brackets(hostname);
|
|
102
|
-
const ipVersion = isIP(bareHostname);
|
|
103
|
-
if (ipVersion === 4)
|
|
104
|
-
return bareHostname.startsWith("127.");
|
|
105
|
-
if (ipVersion === 6)
|
|
106
|
-
return bareHostname === "::1";
|
|
107
|
-
return false;
|
|
108
|
-
}
|
|
109
|
-
function isForbiddenIpv4(hostname) {
|
|
110
|
-
const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
|
|
111
|
-
const [a = -1, b = -1, c = -1, d = -1] = parts;
|
|
112
|
-
if (parts.length !== 4 || [a, b, c, d].some((part) => !Number.isInteger(part) || part < 0 || part > 255)) {
|
|
113
|
-
return true;
|
|
114
|
-
}
|
|
115
|
-
return (a === 0 ||
|
|
116
|
-
a === 10 ||
|
|
117
|
-
(a === 100 && b >= 64 && b <= 127) ||
|
|
118
|
-
a === 127 ||
|
|
119
|
-
(a === 169 && b === 254) ||
|
|
120
|
-
(a === 172 && b >= 16 && b <= 31) ||
|
|
121
|
-
(a === 192 && b === 0 && c === 0 && d !== 9 && d !== 10) ||
|
|
122
|
-
(a === 192 && b === 0 && c === 2) ||
|
|
123
|
-
(a === 192 && b === 168) ||
|
|
124
|
-
(a === 192 && b === 88 && c === 99) ||
|
|
125
|
-
(a === 198 && (b === 18 || b === 19)) ||
|
|
126
|
-
(a === 198 && b === 51 && c === 100) ||
|
|
127
|
-
(a === 203 && b === 0 && c === 113) ||
|
|
128
|
-
a >= 224);
|
|
129
|
-
}
|
|
130
|
-
function parseIpv6Words(hostname) {
|
|
131
|
-
let normalized;
|
|
132
|
-
try {
|
|
133
|
-
normalized = stripIpv6Brackets(new URL(`http://[${stripIpv6Brackets(hostname)}]/`).hostname).toLowerCase();
|
|
134
|
-
}
|
|
135
|
-
catch {
|
|
136
|
-
return null;
|
|
137
|
-
}
|
|
138
|
-
const halves = normalized.split("::");
|
|
139
|
-
if (halves.length > 2)
|
|
140
|
-
return null;
|
|
141
|
-
const left = halves[0] ? halves[0].split(":") : [];
|
|
142
|
-
const right = halves[1] ? halves[1].split(":") : [];
|
|
143
|
-
const missing = 8 - left.length - right.length;
|
|
144
|
-
if ((halves.length === 1 && missing !== 0) || (halves.length === 2 && missing < 1))
|
|
145
|
-
return null;
|
|
146
|
-
const parts = halves.length === 2 ? [...left, ...Array(missing).fill("0"), ...right] : left;
|
|
147
|
-
if (parts.length !== 8 || parts.some((part) => !/^[0-9a-f]{1,4}$/.test(part)))
|
|
148
|
-
return null;
|
|
149
|
-
return parts.map((part) => Number.parseInt(part, 16));
|
|
150
|
-
}
|
|
151
|
-
function ipv6Value(hostname) {
|
|
152
|
-
const words = parseIpv6Words(hostname);
|
|
153
|
-
if (!words)
|
|
154
|
-
return null;
|
|
155
|
-
return words.reduce((value, word) => (value << 16n) | BigInt(word), 0n);
|
|
156
|
-
}
|
|
157
|
-
const FORBIDDEN_IPV6_RANGES = [
|
|
158
|
-
["64:ff9b:1::", 48],
|
|
159
|
-
["100::", 64],
|
|
160
|
-
["2001::", 32],
|
|
161
|
-
["2001:2::", 48],
|
|
162
|
-
["2001:10::", 28],
|
|
163
|
-
["2001:20::", 28],
|
|
164
|
-
["2001:db8::", 32],
|
|
165
|
-
["3fff::", 20],
|
|
166
|
-
["5f00::", 16],
|
|
167
|
-
["fc00::", 7],
|
|
168
|
-
["fe80::", 10],
|
|
169
|
-
["fec0::", 10],
|
|
170
|
-
["ff00::", 8],
|
|
171
|
-
];
|
|
172
|
-
function embeddedIpv4Address(words) {
|
|
173
|
-
const firstSixZero = words.slice(0, 6).every((word) => word === 0);
|
|
174
|
-
const mapped = words.slice(0, 5).every((word) => word === 0) && words[5] === 0xffff;
|
|
175
|
-
const translated = words.slice(0, 4).every((word) => word === 0) && words[4] === 0xffff && words[5] === 0;
|
|
176
|
-
const wellKnownNat64 = words[0] === 0x64 && words[1] === 0xff9b && words.slice(2, 6).every((word) => word === 0);
|
|
177
|
-
if (words[0] === 0x2002)
|
|
178
|
-
return ipv4FromHextets(words[1] ?? 0, words[2] ?? 0);
|
|
179
|
-
if (!firstSixZero && !mapped && !translated && !wellKnownNat64)
|
|
180
|
-
return null;
|
|
181
|
-
return ipv4FromHextets(words[6] ?? 0, words[7] ?? 0);
|
|
182
|
-
}
|
|
183
|
-
function ipv4FromHextets(high, low) {
|
|
184
|
-
return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
|
|
185
|
-
}
|
|
186
|
-
function isForbiddenIpv6(hostname) {
|
|
187
|
-
const words = parseIpv6Words(hostname);
|
|
188
|
-
const value = ipv6Value(hostname);
|
|
189
|
-
if (!words || value === null)
|
|
190
|
-
return true;
|
|
191
|
-
const embeddedIpv4 = embeddedIpv4Address(words);
|
|
192
|
-
if (embeddedIpv4)
|
|
193
|
-
return isForbiddenIpv4(embeddedIpv4);
|
|
194
|
-
return FORBIDDEN_IPV6_RANGES.some(([prefix, bits]) => {
|
|
195
|
-
const prefixValue = ipv6Value(prefix);
|
|
196
|
-
if (prefixValue === null)
|
|
197
|
-
return true;
|
|
198
|
-
const shift = BigInt(128 - bits);
|
|
199
|
-
return value >> shift === prefixValue >> shift;
|
|
200
|
-
});
|
|
73
|
+
return classifyNetworkHostname(hostname) === "loopback";
|
|
201
74
|
}
|
|
202
75
|
function redirectInit(init, currentUrl, nextUrl) {
|
|
203
76
|
if (new URL(currentUrl).origin === new URL(nextUrl).origin)
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
import {
|
|
4
|
+
import { randomBytes } from "node:crypto";
|
|
5
5
|
import fs from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { fetchWithRetry, isWithin, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir, todayIso, } from "../../core/common.js";
|
|
8
8
|
import { ConfigError, UsageError } from "../../core/errors.js";
|
|
9
|
-
import { getRegistryIndexCacheDir } from "../../core/paths.js";
|
|
10
9
|
import { warn, warnVerbose } from "../../core/warn.js";
|
|
11
10
|
import { withFreshnessCache } from "../freshness.js";
|
|
12
11
|
import { sanitizeString } from "../providers/provider-utils.js";
|
|
12
|
+
import { getWebsiteCachePaths, normalizeSiteUrl, validateWebsiteInputUrl, validateWebsiteUrl, } from "../website-url.js";
|
|
13
13
|
import { htmlToMarkdownAndLinks } from "./content-extract.js";
|
|
14
14
|
import { escapeMarkdownStructure } from "./fetcher-util.js";
|
|
15
|
-
import { assertResolvedHostAllowed, assertWebsiteRequestUrl
|
|
15
|
+
import { assertResolvedHostAllowed, assertWebsiteRequestUrl } from "./host-guard.js";
|
|
16
16
|
import { loadWikiSnapshotFetchers } from "./registry.js";
|
|
17
17
|
import { createAllowAllRobotsPolicy, createRobotsPolicy, isPathAllowedByRobots, ROBOTS_BODY_TIMEOUT_MS, ROBOTS_BYTE_CAP, } from "./robots.js";
|
|
18
18
|
/** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
|
|
@@ -37,25 +37,15 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
|
|
|
37
37
|
*/
|
|
38
38
|
const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
39
39
|
const WEBSITE_MAX_REDIRECTS = 8;
|
|
40
|
-
/**
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
* Returns `null` for an explicit opt-out (`false`, or `0`), the configured
|
|
44
|
-
* number of milliseconds when positive, and `undefined` to mean "unset, use
|
|
45
|
-
* the default". Anything else is ignored rather than failing a crawl over a
|
|
46
|
-
* malformed knob.
|
|
47
|
-
*/
|
|
48
|
-
function coerceCrawlTimeoutMs(value) {
|
|
49
|
-
if (value === false || value === 0)
|
|
50
|
-
return null;
|
|
51
|
-
if (value === true || value === undefined || value === null)
|
|
52
|
-
return undefined;
|
|
53
|
-
const parsed = typeof value === "number" ? value : typeof value === "string" ? Number.parseInt(value, 10) : Number.NaN;
|
|
54
|
-
if (!Number.isFinite(parsed))
|
|
40
|
+
/** Resolve the exact persisted crawl timeout; zero explicitly disables it. */
|
|
41
|
+
export function resolveCrawlTimeoutMs(value) {
|
|
42
|
+
if (value === undefined)
|
|
55
43
|
return undefined;
|
|
56
|
-
if (
|
|
44
|
+
if (value === 0)
|
|
57
45
|
return null;
|
|
58
|
-
|
|
46
|
+
if (typeof value === "number" && Number.isInteger(value) && value > 0)
|
|
47
|
+
return value;
|
|
48
|
+
throw new ConfigError(`Invalid value for crawlTimeoutMs: expected a non-negative integer, got ${JSON.stringify(value)}.`);
|
|
59
49
|
}
|
|
60
50
|
/**
|
|
61
51
|
* How many times a URL may be pushed back for not fitting its origin's
|
|
@@ -70,19 +60,6 @@ const MAX_CRAWL_DEFERRALS = 3;
|
|
|
70
60
|
* wall-clock cap elapses.
|
|
71
61
|
*/
|
|
72
62
|
const WEBSITE_PAGE_BODY_TIMEOUT_MS = 30_000;
|
|
73
|
-
export function shouldAllowPrivateWebsiteHostsForTests() {
|
|
74
|
-
return process.env.BUN_TEST === "1" || process.env.NODE_ENV === "test";
|
|
75
|
-
}
|
|
76
|
-
export function shouldAllowPrivateWebsiteUrlForTests(rawUrl) {
|
|
77
|
-
if (!shouldAllowPrivateWebsiteHostsForTests())
|
|
78
|
-
return false;
|
|
79
|
-
try {
|
|
80
|
-
return isLoopbackWebsiteHostname(new URL(rawUrl).hostname.toLowerCase());
|
|
81
|
-
}
|
|
82
|
-
catch {
|
|
83
|
-
return false;
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
63
|
function resolveFetcherStashDir(explicitStashDir) {
|
|
87
64
|
if (explicitStashDir)
|
|
88
65
|
return explicitStashDir;
|
|
@@ -93,19 +70,10 @@ function resolveFetcherStashDir(explicitStashDir) {
|
|
|
93
70
|
return null;
|
|
94
71
|
}
|
|
95
72
|
}
|
|
96
|
-
export function getWebsiteCachePaths(siteUrl) {
|
|
97
|
-
const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
|
|
98
|
-
const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
|
|
99
|
-
return {
|
|
100
|
-
rootDir,
|
|
101
|
-
stashDir: path.join(rootDir, "stash"),
|
|
102
|
-
manifestPath: path.join(rootDir, "manifest.json"),
|
|
103
|
-
};
|
|
104
|
-
}
|
|
105
73
|
export async function ensureWebsiteMirror(config, options) {
|
|
106
74
|
const rawUrl = config.url ?? "";
|
|
107
75
|
const normalizedUrl = validateWebsiteUrl(rawUrl, { allowPrivateHosts: options?.allowPrivateHosts });
|
|
108
|
-
const cachePaths = getWebsiteCachePaths(normalizedUrl);
|
|
76
|
+
const cachePaths = getWebsiteCachePaths(normalizedUrl, options?.cacheRootDir);
|
|
109
77
|
const requireStashDir = options?.requireStashDir === true;
|
|
110
78
|
await withFreshnessCache({
|
|
111
79
|
markerPath: cachePaths.manifestPath,
|
|
@@ -117,12 +85,12 @@ export async function ensureWebsiteMirror(config, options) {
|
|
|
117
85
|
fs.mkdirSync(cachePaths.rootDir, { recursive: true });
|
|
118
86
|
await scrapeWebsiteToStash(normalizedUrl, cachePaths.stashDir, {
|
|
119
87
|
fetcherStashDir: resolveFetcherStashDir(),
|
|
120
|
-
maxPages:
|
|
121
|
-
maxDepth:
|
|
122
|
-
respectRobots:
|
|
88
|
+
maxPages: resolvePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT, "maxPages"),
|
|
89
|
+
maxDepth: resolvePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT, "maxDepth"),
|
|
90
|
+
respectRobots: resolveRespectRobots(config.options?.respectRobots),
|
|
123
91
|
allowPrivateHosts: options?.allowPrivateHosts,
|
|
124
92
|
wallClockCapMs: options?.wallClockCapMs,
|
|
125
|
-
crawlTimeoutMs:
|
|
93
|
+
crawlTimeoutMs: resolveCrawlTimeoutMs(config.options?.crawlTimeoutMs),
|
|
126
94
|
resolveSecret: options?.resolveSecret,
|
|
127
95
|
// As-supplied, pre-normalization start URL (see crawlWebsite's
|
|
128
96
|
// `rawStartUrl` doc comment): threaded through purely for the C-02
|
|
@@ -544,7 +512,7 @@ async function crawlWebsite(startUrl, options) {
|
|
|
544
512
|
const visited = new Set();
|
|
545
513
|
const pages = [];
|
|
546
514
|
// Precedence: the test-only seam, then the user's `crawlTimeoutMs`, then the
|
|
547
|
-
// default. `crawlTimeoutMs: 0`
|
|
515
|
+
// default. `crawlTimeoutMs: 0` disables the cap outright, for a
|
|
548
516
|
// deliberately long-running crawl the user is willing to babysit.
|
|
549
517
|
const configuredCapMs = options.crawlTimeoutMs === null ? null : (options.crawlTimeoutMs ?? WEBSITE_CRAWL_WALL_CLOCK_MS);
|
|
550
518
|
const wallClockCapMs = options.wallClockCapMs ?? configuredCapMs;
|
|
@@ -866,28 +834,13 @@ export async function loadRobotsTxt(robotsUrl, options) {
|
|
|
866
834
|
return { kind: "unavailable" };
|
|
867
835
|
}
|
|
868
836
|
}
|
|
869
|
-
/**
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
* `sources[].options` bag is `z.record(z.unknown())` and accepts anything, so
|
|
873
|
-
* the runtime read still validates. A misspelled non-boolean opt-out fails
|
|
874
|
-
* loudly (`ConfigError`) rather than silently defaulting either way — the
|
|
875
|
-
* user would otherwise think robots.txt handling is something other than
|
|
876
|
-
* what akm is actually doing (spec §4.7).
|
|
877
|
-
*/
|
|
878
|
-
export function coerceRespectRobots(value) {
|
|
879
|
-
if (value === undefined || value === null)
|
|
837
|
+
/** Resolve the exact persisted robots policy. */
|
|
838
|
+
export function resolveRespectRobots(value) {
|
|
839
|
+
if (value === undefined)
|
|
880
840
|
return true;
|
|
881
841
|
if (typeof value === "boolean")
|
|
882
842
|
return value;
|
|
883
|
-
|
|
884
|
-
const normalized = value.trim().toLowerCase();
|
|
885
|
-
if (normalized === "true")
|
|
886
|
-
return true;
|
|
887
|
-
if (normalized === "false")
|
|
888
|
-
return false;
|
|
889
|
-
}
|
|
890
|
-
throw new ConfigError(`Invalid value for respectRobots: expected a boolean (or "true"/"false"), got ${JSON.stringify(value)}.`);
|
|
843
|
+
throw new ConfigError(`Invalid value for respectRobots: expected a boolean, got ${JSON.stringify(value)}.`);
|
|
891
844
|
}
|
|
892
845
|
function buildMarkdownSnapshot(page, slug, tags) {
|
|
893
846
|
const title = sanitizeString(page.title, 200) || slug;
|
|
@@ -917,41 +870,6 @@ function buildMarkdownSnapshot(page, slug, tags) {
|
|
|
917
870
|
"",
|
|
918
871
|
].join("\n");
|
|
919
872
|
}
|
|
920
|
-
export function validateWebsiteUrl(rawUrl, options) {
|
|
921
|
-
return validateWebsiteUrlWithError(rawUrl, ConfigError, options);
|
|
922
|
-
}
|
|
923
|
-
export function validateWebsiteInputUrl(rawUrl, options) {
|
|
924
|
-
return validateWebsiteUrlWithError(rawUrl, UsageError, options);
|
|
925
|
-
}
|
|
926
|
-
function validateWebsiteUrlWithError(rawUrl, ErrorType, options) {
|
|
927
|
-
if (!rawUrl) {
|
|
928
|
-
throw new ErrorType("Website provider requires a URL");
|
|
929
|
-
}
|
|
930
|
-
let parsed;
|
|
931
|
-
try {
|
|
932
|
-
parsed = new URL(rawUrl);
|
|
933
|
-
}
|
|
934
|
-
catch {
|
|
935
|
-
throw new ErrorType(`Website URL is not valid: "${rawUrl}"`);
|
|
936
|
-
}
|
|
937
|
-
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
938
|
-
throw new ErrorType(`Website URL must use http:// or https://, got "${parsed.protocol}" in "${rawUrl}"`);
|
|
939
|
-
}
|
|
940
|
-
if (parsed.username || parsed.password) {
|
|
941
|
-
throw new ErrorType("Website URL must not contain embedded credentials");
|
|
942
|
-
}
|
|
943
|
-
assertWebsiteRequestUrl(parsed.toString(), ErrorType, options);
|
|
944
|
-
parsed.hash = "";
|
|
945
|
-
return normalizeSiteUrl(parsed.toString());
|
|
946
|
-
}
|
|
947
|
-
function normalizeSiteUrl(rawUrl) {
|
|
948
|
-
const parsed = new URL(rawUrl);
|
|
949
|
-
parsed.hash = "";
|
|
950
|
-
if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
|
|
951
|
-
parsed.pathname = parsed.pathname.replace(/\/+$/, "");
|
|
952
|
-
}
|
|
953
|
-
return parsed.toString();
|
|
954
|
-
}
|
|
955
873
|
function normalizeCrawlUrl(rawUrl) {
|
|
956
874
|
try {
|
|
957
875
|
const parsed = new URL(rawUrl);
|
|
@@ -1024,15 +942,12 @@ function uniqueSlug(base, used) {
|
|
|
1024
942
|
used.add(candidate);
|
|
1025
943
|
return candidate;
|
|
1026
944
|
}
|
|
1027
|
-
function
|
|
945
|
+
export function resolvePositiveInt(value, fallback, optionName) {
|
|
946
|
+
if (value === undefined)
|
|
947
|
+
return fallback;
|
|
1028
948
|
if (typeof value === "number" && Number.isInteger(value) && value > 0)
|
|
1029
949
|
return value;
|
|
1030
|
-
|
|
1031
|
-
const parsed = Number.parseInt(value, 10);
|
|
1032
|
-
if (Number.isInteger(parsed) && parsed > 0)
|
|
1033
|
-
return parsed;
|
|
1034
|
-
}
|
|
1035
|
-
return fallback;
|
|
950
|
+
throw new ConfigError(`Invalid value for ${optionName}: expected a positive integer, got ${JSON.stringify(value)}.`);
|
|
1036
951
|
}
|
|
1037
952
|
function looksLikeMarkup(body) {
|
|
1038
953
|
return /<html[\s>]|<body[\s>]|<\/[a-z][\w:-]*>/i.test(body);
|
|
@@ -1101,5 +1016,6 @@ function safeCodePointToString(value) {
|
|
|
1101
1016
|
return undefined;
|
|
1102
1017
|
}
|
|
1103
1018
|
}
|
|
1019
|
+
export { getWebsiteCachePaths, normalizeSiteUrl, shouldAllowPrivateWebsiteHostsForTests, shouldAllowPrivateWebsiteUrlForTests, validateWebsiteInputUrl, validateWebsiteUrl, } from "../website-url.js";
|
|
1104
1020
|
// Re-exported for existing importers (the SSRF suite pins these entry points).
|
|
1105
1021
|
export { assertResolvedHostAllowed } from "./host-guard.js";
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Leaf URL and cache-path derivations shared by website source providers and
|
|
6
|
+
* the website ingest pipeline.
|
|
7
|
+
*
|
|
8
|
+
* Keep this module independent of website-ingest and the snapshot-fetcher
|
|
9
|
+
* registry. Provider registration must be safe to import from low-level source
|
|
10
|
+
* resolution code without pulling the fetcher subgraph into that leaf.
|
|
11
|
+
*/
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
13
|
+
import path from "node:path";
|
|
14
|
+
import { ConfigError, UsageError } from "../core/errors.js";
|
|
15
|
+
import { getRegistryIndexCacheDir } from "../core/paths.js";
|
|
16
|
+
import { assertWebsiteRequestUrl, isLoopbackWebsiteHostname, } from "./snapshot-fetchers/host-guard.js";
|
|
17
|
+
export function shouldAllowPrivateWebsiteHostsForTests() {
|
|
18
|
+
return process.env.BUN_TEST === "1" || process.env.NODE_ENV === "test";
|
|
19
|
+
}
|
|
20
|
+
export function shouldAllowPrivateWebsiteUrlForTests(rawUrl) {
|
|
21
|
+
if (!shouldAllowPrivateWebsiteHostsForTests())
|
|
22
|
+
return false;
|
|
23
|
+
try {
|
|
24
|
+
return isLoopbackWebsiteHostname(new URL(rawUrl).hostname.toLowerCase());
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export function getWebsiteCachePaths(siteUrl, cacheRootOverride) {
|
|
31
|
+
const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
|
|
32
|
+
const rootDir = path.join(cacheRootOverride ?? getRegistryIndexCacheDir(), `website-${key}`);
|
|
33
|
+
return {
|
|
34
|
+
rootDir,
|
|
35
|
+
stashDir: path.join(rootDir, "stash"),
|
|
36
|
+
manifestPath: path.join(rootDir, "manifest.json"),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
export function validateWebsiteUrl(rawUrl, options) {
|
|
40
|
+
return validateWebsiteUrlWithError(rawUrl, ConfigError, options);
|
|
41
|
+
}
|
|
42
|
+
export function validateWebsiteInputUrl(rawUrl, options) {
|
|
43
|
+
return validateWebsiteUrlWithError(rawUrl, UsageError, options);
|
|
44
|
+
}
|
|
45
|
+
function validateWebsiteUrlWithError(rawUrl, ErrorType, options) {
|
|
46
|
+
if (!rawUrl) {
|
|
47
|
+
throw new ErrorType("Website provider requires a URL");
|
|
48
|
+
}
|
|
49
|
+
let parsed;
|
|
50
|
+
try {
|
|
51
|
+
parsed = new URL(rawUrl);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
throw new ErrorType(`Website URL is not valid: "${rawUrl}"`);
|
|
55
|
+
}
|
|
56
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
57
|
+
throw new ErrorType(`Website URL must use http:// or https://, got "${parsed.protocol}" in "${rawUrl}"`);
|
|
58
|
+
}
|
|
59
|
+
if (parsed.username || parsed.password) {
|
|
60
|
+
throw new ErrorType("Website URL must not contain embedded credentials");
|
|
61
|
+
}
|
|
62
|
+
assertWebsiteRequestUrl(parsed.toString(), ErrorType, options);
|
|
63
|
+
parsed.hash = "";
|
|
64
|
+
return normalizeSiteUrl(parsed.toString());
|
|
65
|
+
}
|
|
66
|
+
export function normalizeSiteUrl(rawUrl) {
|
|
67
|
+
const parsed = new URL(rawUrl);
|
|
68
|
+
parsed.hash = "";
|
|
69
|
+
if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
|
|
70
|
+
parsed.pathname = parsed.pathname.replace(/\/+$/, "");
|
|
71
|
+
}
|
|
72
|
+
return parsed.toString();
|
|
73
|
+
}
|
|
@@ -9,12 +9,12 @@ export function assertMigrationRegistry(migrations) {
|
|
|
9
9
|
seen.add(migration.id);
|
|
10
10
|
}
|
|
11
11
|
}
|
|
12
|
-
function
|
|
12
|
+
export function migrationLedgerExists(db) {
|
|
13
13
|
return !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_migrations'").get();
|
|
14
14
|
}
|
|
15
15
|
/** Inspect the database's applied IDs against the exact ordered registry prefix. */
|
|
16
16
|
function inspectLedgerAgainst(db, registryIds) {
|
|
17
|
-
if (!
|
|
17
|
+
if (!migrationLedgerExists(db))
|
|
18
18
|
return { status: registryIds.length === 0 ? "current" : "old", migrationIds: [] };
|
|
19
19
|
const rows = db.prepare("SELECT id FROM schema_migrations ORDER BY rowid").all();
|
|
20
20
|
const migrationIds = rows.map((row) => row.id);
|
|
@@ -53,13 +53,6 @@ export function assertMigrationLedger(db, migrations) {
|
|
|
53
53
|
}
|
|
54
54
|
return state;
|
|
55
55
|
}
|
|
56
|
-
export function assertCurrentMigrationLedger(db, migrations) {
|
|
57
|
-
const state = assertMigrationLedger(db, migrations);
|
|
58
|
-
if (state.status !== "current") {
|
|
59
|
-
throw new Error(`Refusing to open an obsolete writable schema; run \`akm migrate apply\`: ${state.detail ?? "pending migrations"}.`);
|
|
60
|
-
}
|
|
61
|
-
return state;
|
|
62
|
-
}
|
|
63
56
|
/**
|
|
64
57
|
* Create the migrations ledger table if it does not exist. Must be called
|
|
65
58
|
* unconditionally on every open so a fresh database bootstraps correctly.
|
|
@@ -73,13 +66,14 @@ export function ensureMigrationsTable(db) {
|
|
|
73
66
|
`);
|
|
74
67
|
}
|
|
75
68
|
/**
|
|
76
|
-
* Apply every pending migration, one transaction per migration.
|
|
69
|
+
* Apply every pending migration, normally one transaction per migration.
|
|
77
70
|
*
|
|
78
71
|
* Each migration is applied in its own transaction so a failure in migration N
|
|
79
72
|
* does not roll back already-applied migrations 1..N-1. The migration row is
|
|
80
|
-
* inserted
|
|
81
|
-
*
|
|
82
|
-
* the
|
|
73
|
+
* inserted after the DDL succeeds in the same transaction, so a crash rolls
|
|
74
|
+
* back both that migration's SQL and its ledger row. A caller may explicitly
|
|
75
|
+
* group the initial prefix when ledger initialization and dependent early
|
|
76
|
+
* migrations form one safety boundary.
|
|
83
77
|
*
|
|
84
78
|
* @param db The open SQLite database.
|
|
85
79
|
* @param migrations The module's ordered, append-only migration list.
|
|
@@ -87,13 +81,44 @@ export function ensureMigrationsTable(db) {
|
|
|
87
81
|
*/
|
|
88
82
|
export function runMigrations(db, migrations, opts) {
|
|
89
83
|
assertMigrationRegistry(migrations);
|
|
90
|
-
|
|
84
|
+
const lockedPrefixThrough = opts?.lockInitialMigrationPrefixThrough;
|
|
85
|
+
const lockedPrefixEnd = lockedPrefixThrough
|
|
86
|
+
? migrations.findIndex((migration) => migration.id === lockedPrefixThrough)
|
|
87
|
+
: -1;
|
|
88
|
+
if (lockedPrefixThrough && lockedPrefixEnd < 0) {
|
|
89
|
+
throw new Error(`Initial locked migration prefix ends at unknown migration ID ${lockedPrefixThrough}.`);
|
|
90
|
+
}
|
|
91
|
+
if (lockedPrefixEnd >= 0) {
|
|
92
|
+
withImmediateWriteLock(db, () => {
|
|
93
|
+
if (!migrationLedgerExists(db)) {
|
|
94
|
+
opts?.beforeLedgerInitializationLocked?.(db);
|
|
95
|
+
ensureMigrationsTable(db);
|
|
96
|
+
}
|
|
97
|
+
assertMigrationLedger(db, migrations);
|
|
98
|
+
for (const migration of migrations.slice(0, lockedPrefixEnd + 1)) {
|
|
99
|
+
const already = db.prepare("SELECT 1 FROM schema_migrations WHERE id = ?").get(migration.id);
|
|
100
|
+
if (already)
|
|
101
|
+
continue;
|
|
102
|
+
opts?.beforeMigration?.(migration);
|
|
103
|
+
opts?.beforeMigrationLocked?.(migration, db);
|
|
104
|
+
db.exec(migration.up);
|
|
105
|
+
db.prepare("INSERT INTO schema_migrations (id) VALUES (?)").run(migration.id);
|
|
106
|
+
}
|
|
107
|
+
});
|
|
91
108
|
assertMigrationLedger(db, migrations);
|
|
92
|
-
return;
|
|
93
109
|
}
|
|
94
|
-
if (
|
|
110
|
+
else if (migrationLedgerExists(db)) {
|
|
111
|
+
assertMigrationLedger(db, migrations);
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
withImmediateWriteLock(db, () => {
|
|
115
|
+
if (migrationLedgerExists(db))
|
|
116
|
+
return;
|
|
117
|
+
opts?.beforeLedgerInitializationLocked?.(db);
|
|
118
|
+
ensureMigrationsTable(db);
|
|
119
|
+
});
|
|
95
120
|
assertMigrationLedger(db, migrations);
|
|
96
|
-
|
|
121
|
+
}
|
|
97
122
|
const appliedRows = db.prepare("SELECT id FROM schema_migrations ORDER BY rowid").all();
|
|
98
123
|
const applied = new Set(appliedRows.map((r) => r.id));
|
|
99
124
|
for (const migration of migrations) {
|
|
@@ -103,12 +128,13 @@ export function runMigrations(db, migrations, opts) {
|
|
|
103
128
|
withImmediateWriteLock(db, () => {
|
|
104
129
|
// Re-check under the write lock. `applied` is a snapshot taken before the
|
|
105
130
|
// loop, so two processes bootstrapping the same fresh DB concurrently
|
|
106
|
-
// (both see existed=false
|
|
131
|
+
// (both see existed=false) could each decide
|
|
107
132
|
// to apply migration N. The first commits; the second must not re-run the
|
|
108
133
|
// DDL and must not hit a UNIQUE violation on the ledger insert.
|
|
109
134
|
const already = db.prepare("SELECT 1 FROM schema_migrations WHERE id = ?").get(migration.id);
|
|
110
135
|
if (already)
|
|
111
136
|
return;
|
|
137
|
+
opts?.beforeMigrationLocked?.(migration, db);
|
|
112
138
|
db.exec(migration.up);
|
|
113
139
|
db.prepare("INSERT INTO schema_migrations (id) VALUES (?)").run(migration.id);
|
|
114
140
|
});
|
|
@@ -117,6 +143,17 @@ export function runMigrations(db, migrations, opts) {
|
|
|
117
143
|
}
|
|
118
144
|
/** Attempts to acquire the write lock before giving up to the caller. */
|
|
119
145
|
const IMMEDIATE_LOCK_MAX_ATTEMPTS = 5;
|
|
146
|
+
function isRetryableImmediateBeginError(error) {
|
|
147
|
+
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
148
|
+
return (message.includes("database is locked") ||
|
|
149
|
+
message.includes("database table is locked") ||
|
|
150
|
+
message.includes("did not open a transaction"));
|
|
151
|
+
}
|
|
152
|
+
function sleepImmediateRetry(ms) {
|
|
153
|
+
if (ms <= 0)
|
|
154
|
+
return;
|
|
155
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
156
|
+
}
|
|
120
157
|
/**
|
|
121
158
|
* Run `fn` inside a `BEGIN IMMEDIATE` transaction.
|
|
122
159
|
*
|
|
@@ -139,24 +176,42 @@ function withImmediateWriteLock(db, fn) {
|
|
|
139
176
|
for (let attempt = 1; attempt <= IMMEDIATE_LOCK_MAX_ATTEMPTS; attempt++) {
|
|
140
177
|
try {
|
|
141
178
|
db.exec("BEGIN IMMEDIATE");
|
|
179
|
+
if (!db.inTransaction) {
|
|
180
|
+
throw new Error("BEGIN IMMEDIATE did not open a transaction (phantom contention state)");
|
|
181
|
+
}
|
|
142
182
|
}
|
|
143
183
|
catch (err) {
|
|
144
|
-
// Busy despite busy_timeout (another writer holding it across the whole
|
|
145
|
-
// window). Retry a bounded number of times before surfacing.
|
|
146
184
|
lastBeginErr = err;
|
|
147
|
-
|
|
185
|
+
if (isRetryableImmediateBeginError(err) && attempt < IMMEDIATE_LOCK_MAX_ATTEMPTS) {
|
|
186
|
+
if (db.inTransaction) {
|
|
187
|
+
try {
|
|
188
|
+
db.exec("ROLLBACK");
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
// Transaction already gone; retrying is safe because fn has not run.
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
sleepImmediateRetry(2 ** (attempt - 1));
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
throw err;
|
|
148
198
|
}
|
|
149
199
|
try {
|
|
150
200
|
fn();
|
|
201
|
+
if (!db.inTransaction) {
|
|
202
|
+
throw new Error("Migration write lock invariant violated: transaction opened by BEGIN IMMEDIATE was no longer active after the migration body ran; refusing to COMMIT (writes may have escaped serialization)");
|
|
203
|
+
}
|
|
151
204
|
db.exec("COMMIT");
|
|
152
205
|
return;
|
|
153
206
|
}
|
|
154
207
|
catch (err) {
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
208
|
+
if (db.inTransaction) {
|
|
209
|
+
try {
|
|
210
|
+
db.exec("ROLLBACK");
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
// Already rolled back by SQLite (e.g. the statement aborted the txn).
|
|
214
|
+
}
|
|
160
215
|
}
|
|
161
216
|
throw err;
|
|
162
217
|
}
|