akm-cli 0.9.0-rc.13 → 0.9.0-rc.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +117 -23
- package/dist/akm-migrate +3 -1
- package/dist/assets/hints/cli-hints-full.md +3 -4
- package/dist/assets/hints/cli-hints-short.md +5 -5
- package/dist/assets/workflows/workflow-template.md +4 -3
- package/dist/cli/invocation.js +3 -2
- package/dist/cli/retired-commands.js +3 -0
- package/dist/cli/unknown-flags.js +226 -0
- package/dist/cli.js +19 -35
- package/dist/commands/agent/contribute-cli.js +15 -3
- package/dist/commands/feedback-cli.js +6 -3
- package/dist/commands/improve/collapse-detector.js +2 -3
- package/dist/commands/lint/base-linter.js +4 -16
- package/dist/commands/lint/index.js +13 -13
- package/dist/commands/log.js +6 -1
- package/dist/commands/migration-tool.js +4 -5
- package/dist/commands/observability-cli.js +1 -1
- package/dist/commands/proposal/repository.js +5 -5
- package/dist/commands/read/knowledge.js +2 -0
- package/dist/commands/registry-cli.js +5 -3
- package/dist/commands/sources/add-cli.js +6 -6
- package/dist/commands/sources/self-update.js +30 -7
- package/dist/commands/sources/source-add.js +17 -2
- package/dist/commands/tasks/tasks.js +8 -3
- package/dist/commands/workflow-cli.js +142 -119
- package/dist/core/adapter/adapters/akm-lint.js +17 -13
- package/dist/core/adapter/adapters/akm-task-adapter.js +14 -11
- package/dist/core/asset/akm-markdown.js +41 -8
- package/dist/core/asset/frontmatter.js +22 -0
- package/dist/core/asset/resolve-ref.js +23 -3
- package/dist/core/common.js +45 -2
- package/dist/core/config/config-schema.js +8 -0
- package/dist/core/config/schema/experimental.js +5 -13
- package/dist/core/config/schema/sources-bundles.js +11 -0
- package/dist/core/config/schema/workflow.js +3 -1
- package/dist/core/errors.js +5 -0
- package/dist/core/logs-db.js +2 -1
- package/dist/core/parse.js +4 -1
- package/dist/core/state/migrations.js +11 -14
- package/dist/core/state-db.js +4 -6
- package/dist/core/subprocess.js +6 -4
- package/dist/core/type-presentation.js +1 -1
- package/dist/indexer/indexer.js +16 -1
- package/dist/indexer/search/search-source.js +1 -1
- package/dist/indexer/walk/matchers.js +3 -1
- package/dist/integrations/agent/config.js +2 -2
- package/dist/integrations/agent/detect.js +49 -19
- package/dist/integrations/agent/profiles.js +10 -0
- package/dist/integrations/agent/spawn.js +1 -2
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +7 -3
- package/dist/integrations/lockfile.js +11 -56
- package/dist/output/shapes/passthrough.js +0 -4
- package/dist/output/text/command-format.js +3 -3
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/show-directives.js +4 -6
- package/dist/output/text/workflow-format.js +7 -62
- package/dist/output/text/workflow.js +1 -5
- package/dist/scripts/akm-migrate-node.js +58773 -0
- package/dist/scripts/akm-migrate.js +33976 -11391
- package/dist/setup/detect.js +40 -15
- package/dist/setup/setup.js +1 -1
- package/dist/sources/providers/git-stash.js +4 -2
- package/dist/sources/providers/website.js +5 -0
- package/dist/sources/snapshot-fetchers/bluesky.js +146 -0
- package/dist/sources/snapshot-fetchers/content-extract.js +370 -0
- package/dist/sources/snapshot-fetchers/fetcher-util.js +40 -0
- package/dist/sources/snapshot-fetchers/host-guard.js +199 -0
- package/dist/sources/snapshot-fetchers/registry.js +10 -1
- package/dist/sources/snapshot-fetchers/robots.js +348 -0
- package/dist/sources/snapshot-fetchers/rss.js +279 -0
- package/dist/sources/snapshot-fetchers/secret-seam.js +42 -0
- package/dist/sources/snapshot-fetchers/website-ingest.js +488 -257
- package/dist/sources/snapshot-fetchers/x.js +193 -0
- package/dist/storage/engines/sqlite-migrations.js +22 -107
- package/dist/tasks/runner.js +19 -16
- package/dist/tasks/schema.js +24 -1
- package/dist/workflows/exec/brief.js +1 -1
- package/dist/workflows/exec/frozen-judge.js +28 -2
- package/dist/workflows/exec/native-executor.js +18 -1
- package/dist/workflows/exec/report.js +11 -4
- package/dist/workflows/exec/run-workflow.js +103 -44
- package/dist/workflows/exec/step-work.js +19 -18
- package/dist/workflows/exec/unit-dispatch.js +4 -0
- package/dist/workflows/exec/workflow-engine-gate.js +11 -13
- package/dist/workflows/ir/compile.js +2 -2
- package/dist/workflows/ir/freeze.js +16 -8
- package/dist/workflows/ir/params.js +134 -10
- package/dist/workflows/ir/plan-hash.js +1 -1
- package/dist/workflows/ir/schema.js +6 -2
- package/dist/workflows/renderer.js +2 -2
- package/dist/workflows/runtime/checkin.js +3 -3
- package/dist/workflows/runtime/runs.js +50 -29
- package/dist/workflows/validate-summary.js +30 -14
- package/docs/migration/release-notes/0.9.0.md +31 -4
- package/docs/migration/v0.8-to-v0.9.md +69 -100
- package/docs/reference/data-and-telemetry.md +5 -5
- package/package.json +6 -2
- package/schemas/akm-config.json +24 -0
- package/schemas/akm-workflow.json +1 -1
- package/dist/workflows/cli.js +0 -33
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import { lookup as dnsLookup } from "node:dns/promises";
|
|
5
|
+
import { isIP } from "node:net";
|
|
6
|
+
import { fetchWithRetry, readBodyWithByteCap } from "../../core/common.js";
|
|
7
|
+
/** Redirect budget for a single guarded request chain. */
|
|
8
|
+
const MAX_GUARDED_REDIRECTS = 8;
|
|
9
|
+
export function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
|
|
10
|
+
const parsedUrl = new URL(rawUrl);
|
|
11
|
+
const hostname = parsedUrl.hostname.toLowerCase();
|
|
12
|
+
if (hostname.endsWith(".invalid")) {
|
|
13
|
+
throw new ErrorType(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
|
|
14
|
+
}
|
|
15
|
+
if (isForbiddenWebsiteHostname(hostname, options)) {
|
|
16
|
+
throw new ErrorType(`Refusing to fetch non-public website host: ${parsedUrl.hostname}`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
async function defaultResolveHostname(hostname) {
|
|
20
|
+
const records = await dnsLookup(hostname, { all: true });
|
|
21
|
+
return records.map((record) => record.address);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Resolve-then-validate SSRF guard against DNS rebinding / private-range
|
|
25
|
+
* bypasses. {@link assertWebsiteRequestUrl} only rejects IP-literal and
|
|
26
|
+
* well-known-name hosts; a hostname like `private-host.example.com` that
|
|
27
|
+
* resolves to `10.0.0.1` passes those checks and then fetch connects to the
|
|
28
|
+
* private address. Here we resolve EVERY A/AAAA record and validate each
|
|
29
|
+
* against the same forbidden-range rules, failing CLOSED on an empty answer or
|
|
30
|
+
* resolver error.
|
|
31
|
+
*
|
|
32
|
+
* TOCTOU residual (documented, not fully closable here): Bun's `fetch` exposes
|
|
33
|
+
* no custom `lookup`/agent hook, so we cannot pin the socket to the exact IP we
|
|
34
|
+
* validated — a hostile resolver could return a public IP to this lookup and a
|
|
35
|
+
* private IP microseconds later at connect time (classic rebinding). This still
|
|
36
|
+
* removes the TRIVIAL `hostname A 10.0.0.1` bypass, which is the strongest
|
|
37
|
+
* guarantee available without a pinned-connection fetch API. Re-run on every
|
|
38
|
+
* redirect hop (the crawler recurses through `fetchWebsiteResponse`).
|
|
39
|
+
*/
|
|
40
|
+
export async function assertResolvedHostAllowed(hostname, options) {
|
|
41
|
+
if (options?.allowPrivateHosts === true)
|
|
42
|
+
return;
|
|
43
|
+
const bare = stripIpv6Brackets(hostname.toLowerCase());
|
|
44
|
+
// IP-literal hosts are already fully validated by assertWebsiteRequestUrl's
|
|
45
|
+
// range checks; resolving them is a no-op (and dnsLookup would just echo it).
|
|
46
|
+
if (isIP(bare) !== 0)
|
|
47
|
+
return;
|
|
48
|
+
const resolve = options?.resolveHostname ?? defaultResolveHostname;
|
|
49
|
+
let addresses;
|
|
50
|
+
try {
|
|
51
|
+
addresses = await resolve(bare);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
throw new Error(`Refusing to fetch ${hostname}: DNS resolution failed`);
|
|
55
|
+
}
|
|
56
|
+
if (addresses.length === 0) {
|
|
57
|
+
throw new Error(`Refusing to fetch ${hostname}: hostname resolved to no addresses`);
|
|
58
|
+
}
|
|
59
|
+
for (const address of addresses) {
|
|
60
|
+
const version = isIP(address);
|
|
61
|
+
const forbidden = version === 4 ? isForbiddenIpv4(address) : version === 6 ? isForbiddenIpv6(stripIpv6Brackets(address)) : true;
|
|
62
|
+
if (forbidden) {
|
|
63
|
+
throw new Error(`Refusing to fetch ${hostname}: resolves to non-public or unparseable address ${address}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
// WHATWG URL.hostname wraps IPv6 literals in brackets (e.g. "[::1]"), but
|
|
68
|
+
// node:net's isIP() only recognizes the bare address form and returns 0 for
|
|
69
|
+
// anything bracketed — silently skipping all IPv6 forbidden-host checks
|
|
70
|
+
// below for every hostname parsed off a URL. Strip the brackets before any
|
|
71
|
+
// isIP()/isForbiddenIpv6() call so those checks actually run.
|
|
72
|
+
function stripIpv6Brackets(hostname) {
|
|
73
|
+
return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
|
|
74
|
+
}
|
|
75
|
+
function isForbiddenWebsiteHostname(hostname, options) {
|
|
76
|
+
if (options?.allowPrivateHosts === true)
|
|
77
|
+
return false;
|
|
78
|
+
if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal") {
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
const bareHostname = stripIpv6Brackets(hostname);
|
|
82
|
+
const ipVersion = isIP(bareHostname);
|
|
83
|
+
if (ipVersion === 4)
|
|
84
|
+
return isForbiddenIpv4(bareHostname);
|
|
85
|
+
if (ipVersion === 6)
|
|
86
|
+
return isForbiddenIpv6(bareHostname);
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
export function isLoopbackWebsiteHostname(hostname) {
|
|
90
|
+
if (hostname === "localhost" || hostname.endsWith(".localhost"))
|
|
91
|
+
return true;
|
|
92
|
+
const bareHostname = stripIpv6Brackets(hostname);
|
|
93
|
+
const ipVersion = isIP(bareHostname);
|
|
94
|
+
if (ipVersion === 4)
|
|
95
|
+
return bareHostname.startsWith("127.");
|
|
96
|
+
if (ipVersion === 6)
|
|
97
|
+
return bareHostname === "::1";
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
function isForbiddenIpv4(hostname) {
|
|
101
|
+
const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
|
|
102
|
+
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
|
|
103
|
+
return true;
|
|
104
|
+
const a = parts[0];
|
|
105
|
+
const b = parts[1];
|
|
106
|
+
return (a === 0 ||
|
|
107
|
+
a === 10 ||
|
|
108
|
+
a === 127 ||
|
|
109
|
+
(a === 169 && b === 254) ||
|
|
110
|
+
(a === 172 && b >= 16 && b <= 31) ||
|
|
111
|
+
(a === 192 && b === 168));
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Extracts the embedded IPv4 address from an IPv4-mapped IPv6 literal
|
|
115
|
+
* (`::ffff:a.b.c.d` or its canonical hex form `::ffff:xxxx:yyyy`), or
|
|
116
|
+
* returns null if `hostname` isn't one.
|
|
117
|
+
*/
|
|
118
|
+
function extractIpv4MappedAddress(normalizedHostname) {
|
|
119
|
+
const match = normalizedHostname.match(/^::ffff:(?:(\d{1,3}(?:\.\d{1,3}){3})|([0-9a-f]{1,4}):([0-9a-f]{1,4}))$/);
|
|
120
|
+
if (!match)
|
|
121
|
+
return null;
|
|
122
|
+
if (match[1])
|
|
123
|
+
return match[1];
|
|
124
|
+
const high = Number.parseInt(match[2], 16);
|
|
125
|
+
const low = Number.parseInt(match[3], 16);
|
|
126
|
+
return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
|
|
127
|
+
}
|
|
128
|
+
function isForbiddenIpv6(hostname) {
|
|
129
|
+
const normalized = hostname.toLowerCase();
|
|
130
|
+
const mappedIpv4 = extractIpv4MappedAddress(normalized);
|
|
131
|
+
if (mappedIpv4)
|
|
132
|
+
return isForbiddenIpv4(mappedIpv4);
|
|
133
|
+
return (normalized === "::" ||
|
|
134
|
+
normalized === "::1" ||
|
|
135
|
+
normalized.startsWith("fc") ||
|
|
136
|
+
normalized.startsWith("fd") ||
|
|
137
|
+
normalized.startsWith("fe8") ||
|
|
138
|
+
normalized.startsWith("fe9") ||
|
|
139
|
+
normalized.startsWith("fea") ||
|
|
140
|
+
normalized.startsWith("feb"));
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Fetch with the full SSRF guard chain applied to the initial URL and to every
|
|
144
|
+
* redirect hop.
|
|
145
|
+
*
|
|
146
|
+
* `redirect: "manual"` is essential: with the default `follow`, the platform
|
|
147
|
+
* resolves and connects to redirect targets that were never validated, so a
|
|
148
|
+
* public host can bounce a request into the private network.
|
|
149
|
+
*/
|
|
150
|
+
export async function fetchGuardedResponse(url, init, opts, redirectCount = 0) {
|
|
151
|
+
assertWebsiteRequestUrl(url, Error, opts);
|
|
152
|
+
await assertResolvedHostAllowed(new URL(url).hostname, opts);
|
|
153
|
+
const response = await fetchWithRetry(url, { ...init, redirect: "manual" }, {
|
|
154
|
+
timeout: opts.timeoutMs,
|
|
155
|
+
retries: opts.retries ?? 1,
|
|
156
|
+
});
|
|
157
|
+
if (response.status >= 300 && response.status < 400) {
|
|
158
|
+
if (redirectCount >= MAX_GUARDED_REDIRECTS) {
|
|
159
|
+
throw new Error(`Too many redirects while fetching ${url}`);
|
|
160
|
+
}
|
|
161
|
+
const location = response.headers.get("location");
|
|
162
|
+
if (!location) {
|
|
163
|
+
throw new Error(`Redirect response from ${url} did not include a Location header`);
|
|
164
|
+
}
|
|
165
|
+
const nextUrl = new URL(location, url).toString();
|
|
166
|
+
return fetchGuardedResponse(nextUrl, init, opts, redirectCount + 1);
|
|
167
|
+
}
|
|
168
|
+
return response;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* GET a JSON document from a PINNED, constant API host (X, Bluesky XRPC), with
|
|
172
|
+
* a byte-capped read. Returns the parsed value, or null on any 3xx/non-2xx/
|
|
173
|
+
* parse failure.
|
|
174
|
+
*
|
|
175
|
+
* `redirect: "manual"` + rejecting 3xx is load-bearing: these endpoints have no
|
|
176
|
+
* legitimate reason to redirect, and following one would send the next request
|
|
177
|
+
* (bearer token included) to an unvalidated host with none of the SSRF guards
|
|
178
|
+
* the crawl path applies. The full resolve-then-validate DNS guard is
|
|
179
|
+
* intentionally NOT applied here because the host is a compile-time constant,
|
|
180
|
+
* not caller-supplied — the guarded chokepoint is {@link fetchGuardedResponse},
|
|
181
|
+
* which the caller-supplied crawl/feed paths use instead.
|
|
182
|
+
*/
|
|
183
|
+
export async function fetchPinnedJson(url, options) {
|
|
184
|
+
const response = await fetchWithRetry(url, { headers: options.headers, signal: options.signal, redirect: "manual" }, { timeout: options.timeoutMs, retries: options.retries ?? 1 });
|
|
185
|
+
if (response.status >= 300 && response.status < 400)
|
|
186
|
+
return null;
|
|
187
|
+
if (!response.ok)
|
|
188
|
+
return null;
|
|
189
|
+
try {
|
|
190
|
+
const body = await readBodyWithByteCap(response, options.byteCap, {
|
|
191
|
+
bodyTimeoutMs: options.bodyTimeoutMs,
|
|
192
|
+
signal: options.signal,
|
|
193
|
+
});
|
|
194
|
+
return JSON.parse(body);
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
return null;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
@@ -5,10 +5,19 @@ import fs from "node:fs";
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { pathToFileURL } from "node:url";
|
|
7
7
|
import { warn } from "../../core/warn.js";
|
|
8
|
+
import blueskyFetcher from "./bluesky.js";
|
|
9
|
+
import rssFetcher from "./rss.js";
|
|
10
|
+
import xFetcher from "./x.js";
|
|
8
11
|
import youtubeFetcher from "./youtube.js";
|
|
9
12
|
const FETCHER_DIR = path.join("scripts", "wiki-fetchers");
|
|
10
13
|
const FETCHER_FILE_PATTERN = /\.(?:ts|js|mjs)$/i;
|
|
11
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Order matters: the first fetcher whose `matches` returns true and whose
|
|
16
|
+
* `fetch` returns non-null wins. Host-specific fetchers therefore precede the
|
|
17
|
+
* generic RSS fetcher, whose `matches` is path-shaped and deliberately loose.
|
|
18
|
+
* Any fetcher returning null falls through to the generic website crawler.
|
|
19
|
+
*/
|
|
20
|
+
const BUILTIN_FETCHERS = [youtubeFetcher, blueskyFetcher, xFetcher, rssFetcher];
|
|
12
21
|
function isWikiSnapshotFetcher(value) {
|
|
13
22
|
if (!value || typeof value !== "object")
|
|
14
23
|
return false;
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* robots.txt parsing, matching, and a crawl-scoped fetch/cache policy.
|
|
6
|
+
*
|
|
7
|
+
* Behavioral reference: docs/plans/specs/p1-robots.md (authoritative). This
|
|
8
|
+
* module is a clean rewrite against akm's contracts, not a port of any
|
|
9
|
+
* upstream source file.
|
|
10
|
+
*
|
|
11
|
+
* Pure by design: no fetching happens here (see `RobotsTxtLoader`/
|
|
12
|
+
* `loadRobotsTxt` in `website-ingest.ts`). There is no crawl-semantic
|
|
13
|
+
* module-level state: the per-origin HTTP cache lives on the object
|
|
14
|
+
* `createRobotsPolicy` returns so it dies with the crawl that created it —
|
|
15
|
+
* a module-level `origin -> RobotsRuleSet` map would leak `disallowAll`
|
|
16
|
+
* results across crawls and across the test suite's process-wide `bun test`
|
|
17
|
+
* run (see spec §6.1). The module-level `WeakMap` below is a different
|
|
18
|
+
* category: it memoizes a pure function of an individual rule object's own
|
|
19
|
+
* `pattern` field, keyed by that object's identity. It cannot leak meaning
|
|
20
|
+
* between origins or tests — distinct `parseRobotsTxt` calls (even for
|
|
21
|
+
* identical robots.txt text) allocate distinct rule objects with distinct
|
|
22
|
+
* cache slots, and slots are reclaimed by the GC once a rule set is
|
|
23
|
+
* unreachable. It is memoization scoped to a single parsed rule set, not
|
|
24
|
+
* shared cross-crawl state.
|
|
25
|
+
*/
|
|
26
|
+
import { warnVerbose } from "../../core/warn.js";
|
|
27
|
+
// ── §2 Constants (pinned by spec) ───────────────────────────────────────────
|
|
28
|
+
/** Group-matching tokens for `User-agent`, compared case-insensitively. */
|
|
29
|
+
export const ROBOTS_PRODUCT_TOKENS = ["akm", "akm-cli"];
|
|
30
|
+
/** 512 KiB. RFC 9309 §2.5 requires parsing at least 500 KiB. */
|
|
31
|
+
export const ROBOTS_BYTE_CAP = 512 * 1024;
|
|
32
|
+
/** Matches the existing website page-fetch timeout. */
|
|
33
|
+
export const ROBOTS_FETCH_TIMEOUT_MS = 15_000;
|
|
34
|
+
/** Body-read deadline; robots.txt is tiny. */
|
|
35
|
+
export const ROBOTS_BODY_TIMEOUT_MS = 15_000;
|
|
36
|
+
/** Ceiling applied to any parsed `Crawl-delay`, so a hostile robots.txt cannot stall the crawl. */
|
|
37
|
+
export const MAX_CRAWL_DELAY_MS = 10_000;
|
|
38
|
+
/** Identical to the existing website-fetch `User-Agent` header. */
|
|
39
|
+
export const ROBOTS_USER_AGENT_HEADER = "akm-cli website provider";
|
|
40
|
+
/**
|
|
41
|
+
* Frozen so a stray mutation (e.g. an accidental `.rules.push(...)` somewhere
|
|
42
|
+
* that borrows one of these as a starting point) can never poison every other
|
|
43
|
+
* crawl sharing this instance (spec §10, Review log 2026-08-01 finding 1).
|
|
44
|
+
*/
|
|
45
|
+
const EMPTY_ROBOTS_RULES = Object.freeze([]);
|
|
46
|
+
export const ALLOW_ALL_RULES = Object.freeze({
|
|
47
|
+
disallowAll: false,
|
|
48
|
+
rules: EMPTY_ROBOTS_RULES,
|
|
49
|
+
crawlDelayMs: null,
|
|
50
|
+
});
|
|
51
|
+
export const DISALLOW_ALL_RULES = Object.freeze({
|
|
52
|
+
disallowAll: true,
|
|
53
|
+
rules: EMPTY_ROBOTS_RULES,
|
|
54
|
+
crawlDelayMs: null,
|
|
55
|
+
});
|
|
56
|
+
/** Strips a trailing `# comment` (RFC 9309 comments run to end of line). */
|
|
57
|
+
function stripRobotsComment(line) {
|
|
58
|
+
const hashIndex = line.indexOf("#");
|
|
59
|
+
return hashIndex === -1 ? line : line.slice(0, hashIndex);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Parses a `Crawl-delay` value (seconds, possibly fractional) into ms.
|
|
63
|
+
* Non-positive, unparseable, and empty values are ignored (`null`).
|
|
64
|
+
* `Infinity` is preserved so the caller's clamp reduces it to the ceiling
|
|
65
|
+
* rather than the value silently vanishing.
|
|
66
|
+
*/
|
|
67
|
+
function parseCrawlDelayMs(raw) {
|
|
68
|
+
if (!raw)
|
|
69
|
+
return null;
|
|
70
|
+
const seconds = Number(raw);
|
|
71
|
+
if (Number.isNaN(seconds) || seconds <= 0)
|
|
72
|
+
return null;
|
|
73
|
+
return Math.round(seconds * 1000);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Pure. Never throws, never fetches.
|
|
77
|
+
*
|
|
78
|
+
* Grouping follows RFC 9309 §2.2.1: consecutive `User-agent` lines (no
|
|
79
|
+
* intervening directive) form one group; a matching *specific* product-token
|
|
80
|
+
* group suppresses the wildcard (`*`) group entirely; rules from all matching
|
|
81
|
+
* groups of the winning kind are unioned; `Crawl-delay` is the maximum across
|
|
82
|
+
* those groups, clamped once at the end.
|
|
83
|
+
*/
|
|
84
|
+
export function parseRobotsTxt(text) {
|
|
85
|
+
const stripped = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
|
86
|
+
const lines = stripped.split(/\r\n|\r|\n/);
|
|
87
|
+
const groups = [];
|
|
88
|
+
let current = null;
|
|
89
|
+
// True once any non-user-agent directive has been seen for `current`, so
|
|
90
|
+
// the next `User-agent` line starts a NEW group instead of extending it.
|
|
91
|
+
let sawDirectiveSinceLastUserAgent = false;
|
|
92
|
+
for (const rawLine of lines) {
|
|
93
|
+
const line = stripRobotsComment(rawLine).trim();
|
|
94
|
+
if (!line)
|
|
95
|
+
continue;
|
|
96
|
+
const colonIndex = line.indexOf(":");
|
|
97
|
+
if (colonIndex === -1)
|
|
98
|
+
continue; // P-21: no colon => ignored
|
|
99
|
+
const directive = line.slice(0, colonIndex).trim().toLowerCase();
|
|
100
|
+
const value = line.slice(colonIndex + 1).trim();
|
|
101
|
+
if (directive === "user-agent") {
|
|
102
|
+
if (!value)
|
|
103
|
+
continue;
|
|
104
|
+
const token = value.toLowerCase();
|
|
105
|
+
if (current && !sawDirectiveSinceLastUserAgent) {
|
|
106
|
+
current.agents.push(token);
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
current = { agents: [token], rules: [], crawlDelayMsRaw: null };
|
|
110
|
+
groups.push(current);
|
|
111
|
+
sawDirectiveSinceLastUserAgent = false;
|
|
112
|
+
}
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
if (!current)
|
|
116
|
+
continue; // P-13: directive with no preceding User-agent => ignored
|
|
117
|
+
sawDirectiveSinceLastUserAgent = true;
|
|
118
|
+
if (directive === "disallow" || directive === "allow") {
|
|
119
|
+
if (!value)
|
|
120
|
+
continue; // P-16: empty value matches nothing
|
|
121
|
+
if (!(value.startsWith("/") || value.startsWith("*"))) {
|
|
122
|
+
// P-18: neither "/" nor "*"-rooted => ignored, diagnosed quietly.
|
|
123
|
+
warnVerbose("[akm] robots.txt: ignoring %s value %s (patterns must start with '/' or '*')", directive, value);
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
const rule = {
|
|
127
|
+
kind: directive === "allow" ? "allow" : "disallow",
|
|
128
|
+
pattern: value,
|
|
129
|
+
specificity: value.length,
|
|
130
|
+
};
|
|
131
|
+
// Compile once, here, at parse time — not on every URL check (spec
|
|
132
|
+
// §6.4, review finding robots.ts:238). See `compiledPatternCache`.
|
|
133
|
+
compiledPatternCache.set(rule, compilePatternSegments(value));
|
|
134
|
+
current.rules.push(rule);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
if (directive === "crawl-delay") {
|
|
138
|
+
const parsed = parseCrawlDelayMs(value);
|
|
139
|
+
if (parsed !== null)
|
|
140
|
+
current.crawlDelayMsRaw = parsed;
|
|
141
|
+
}
|
|
142
|
+
// Sitemap / unknown directives: ignored without diagnostic (P-19, P-20).
|
|
143
|
+
}
|
|
144
|
+
const productTokens = ROBOTS_PRODUCT_TOKENS;
|
|
145
|
+
const specificGroups = groups.filter((group) => group.agents.some((agent) => productTokens.includes(agent)));
|
|
146
|
+
const wildcardGroups = groups.filter((group) => group.agents.includes("*"));
|
|
147
|
+
const selected = specificGroups.length > 0 ? specificGroups : wildcardGroups;
|
|
148
|
+
const rules = selected.flatMap((group) => group.rules);
|
|
149
|
+
let maxDelayMs = null;
|
|
150
|
+
for (const group of selected) {
|
|
151
|
+
if (group.crawlDelayMsRaw !== null) {
|
|
152
|
+
maxDelayMs = maxDelayMs === null ? group.crawlDelayMsRaw : Math.max(maxDelayMs, group.crawlDelayMsRaw);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
const crawlDelayMs = maxDelayMs === null ? null : Math.min(maxDelayMs, MAX_CRAWL_DELAY_MS);
|
|
156
|
+
return { disallowAll: false, rules, crawlDelayMs };
|
|
157
|
+
}
|
|
158
|
+
const COLLAPSE_STARS = /\*+/g;
|
|
159
|
+
const PERCENT_ENCODED_OCTET = /%[0-9a-fA-F]{2}/g;
|
|
160
|
+
/**
|
|
161
|
+
* RFC 3986 §2.3 "unreserved" set: ALPHA / DIGIT / "-" / "." / "_" / "~".
|
|
162
|
+
* A percent-encoded octet in this set is byte-identical to its literal
|
|
163
|
+
* character (`%73` and `s` are the same byte), so it carries no meaning a
|
|
164
|
+
* URL consumer can rely on. Every other octet — `%2F` ('/'), `%3F` ('?'),
|
|
165
|
+
* `%23` ('#'), `%2A` ('*'), non-ASCII bytes like `%C3` — is "reserved" or
|
|
166
|
+
* otherwise structurally significant and must be left percent-encoded:
|
|
167
|
+
* decoding e.g. `%2F` would silently turn one path segment into two.
|
|
168
|
+
*/
|
|
169
|
+
function isUnreservedByte(byte) {
|
|
170
|
+
return ((byte >= 0x41 && byte <= 0x5a) || // A-Z
|
|
171
|
+
(byte >= 0x61 && byte <= 0x7a) || // a-z
|
|
172
|
+
(byte >= 0x30 && byte <= 0x39) || // 0-9
|
|
173
|
+
byte === 0x2d || // -
|
|
174
|
+
byte === 0x2e || // .
|
|
175
|
+
byte === 0x5f || // _
|
|
176
|
+
byte === 0x7e // ~
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Normalizes percent-encoding for robots.txt matching, mirroring the RFC
|
|
181
|
+
* 9309 reference matcher: percent-encoded UNRESERVED octets are decoded to
|
|
182
|
+
* their literal character; every other `%XX` escape is left exactly as
|
|
183
|
+
* written (including hex-digit case). Applied identically to rule patterns
|
|
184
|
+
* (via `compilePatternSegments`, at parse time and in the defensive lazy
|
|
185
|
+
* fallback) and match targets (`isPathAllowedByRobots`), so a percent-encoded
|
|
186
|
+
* alias of a disallowed path — e.g. a link written as `/%73ecret/` — cannot
|
|
187
|
+
* bypass `Disallow: /secret/` simply because `URL.pathname` preserves
|
|
188
|
+
* %-escapes verbatim. Reserved-octet patterns like spec §4.3 M-31's
|
|
189
|
+
* `/caf%C3%A9` are untouched on both sides (0xC3/0xA9 are not unreserved),
|
|
190
|
+
* so they keep matching themselves exactly as before.
|
|
191
|
+
*/
|
|
192
|
+
function normalizePercentEncoding(value) {
|
|
193
|
+
if (!value.includes("%"))
|
|
194
|
+
return value;
|
|
195
|
+
return value.replace(PERCENT_ENCODED_OCTET, (octetEscape) => {
|
|
196
|
+
const byte = Number.parseInt(octetEscape.slice(1), 16);
|
|
197
|
+
if (isUnreservedByte(byte))
|
|
198
|
+
return String.fromCharCode(byte);
|
|
199
|
+
// A reserved octet stays encoded, but `%2F` and `%2f` denote the same
|
|
200
|
+
// byte — RFC 3986 §6.2.2.1 makes the hex digits case-insensitive. Without
|
|
201
|
+
// canonicalizing the case, `Disallow: /a%2Fb` would fail to match a link
|
|
202
|
+
// spelled `/a%2fb` and the page would be crawled anyway.
|
|
203
|
+
return octetEscape.toUpperCase();
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
function compilePatternSegments(pattern) {
|
|
207
|
+
const anchored = pattern.endsWith("$");
|
|
208
|
+
const body = anchored ? pattern.slice(0, -1) : pattern;
|
|
209
|
+
const collapsed = normalizePercentEncoding(body).replace(COLLAPSE_STARS, "*");
|
|
210
|
+
return { anchored, segments: collapsed.split("*") };
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Per-rule-object cache of the compiled pattern (see the file-level doc
|
|
214
|
+
* comment for why this is not "module-level state" in the sense spec §6.1
|
|
215
|
+
* forbids). `parseRobotsTxt` populates this eagerly, once, when it creates
|
|
216
|
+
* each `RobotsPathRule` — the compile-once requirement from spec §6.4 /
|
|
217
|
+
* review finding robots.ts:238. `compiledPatternFor` below is a defensive
|
|
218
|
+
* fallback for `RobotsRuleSet`s assembled by hand (e.g. test fixtures)
|
|
219
|
+
* rather than via `parseRobotsTxt`; it still compiles at most once per rule
|
|
220
|
+
* object, just lazily on first use instead of at parse time.
|
|
221
|
+
*/
|
|
222
|
+
const compiledPatternCache = new WeakMap();
|
|
223
|
+
function compiledPatternFor(rule) {
|
|
224
|
+
const cached = compiledPatternCache.get(rule);
|
|
225
|
+
if (cached)
|
|
226
|
+
return cached;
|
|
227
|
+
const compiled = compilePatternSegments(rule.pattern);
|
|
228
|
+
compiledPatternCache.set(rule, compiled);
|
|
229
|
+
return compiled;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Matches a precompiled pattern against `target` (`pathname + search`).
|
|
233
|
+
* Greedy left-to-right, no backtracking — see the section doc comment above.
|
|
234
|
+
*/
|
|
235
|
+
function matchesCompiledPattern(compiled, target) {
|
|
236
|
+
const { anchored, segments } = compiled;
|
|
237
|
+
const first = segments[0] ?? "";
|
|
238
|
+
if (!target.startsWith(first))
|
|
239
|
+
return false;
|
|
240
|
+
const pos = first.length;
|
|
241
|
+
const lastIndex = segments.length - 1;
|
|
242
|
+
if (lastIndex === 0) {
|
|
243
|
+
// No wildcard at all: already a prefix match; anchored additionally
|
|
244
|
+
// requires the pattern to consume the whole target.
|
|
245
|
+
return anchored ? target.length === first.length : true;
|
|
246
|
+
}
|
|
247
|
+
let cursor = pos;
|
|
248
|
+
for (let i = 1; i < lastIndex; i++) {
|
|
249
|
+
const segment = segments[i] ?? "";
|
|
250
|
+
const found = target.indexOf(segment, cursor);
|
|
251
|
+
if (found === -1)
|
|
252
|
+
return false;
|
|
253
|
+
// Greedy-leftmost is safe here: an earlier match of segment i can only
|
|
254
|
+
// give segment i+1 MORE room to be found later, never less, since the
|
|
255
|
+
// `*` between them absorbs any extra characters. No backtracking needed.
|
|
256
|
+
cursor = found + segment.length;
|
|
257
|
+
}
|
|
258
|
+
const last = segments[lastIndex] ?? "";
|
|
259
|
+
if (anchored) {
|
|
260
|
+
if (last.length > target.length - cursor)
|
|
261
|
+
return false;
|
|
262
|
+
return target.endsWith(last);
|
|
263
|
+
}
|
|
264
|
+
return last === "" || target.indexOf(last, cursor) !== -1;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Pure. `url` is an absolute http(s) URL string. Matching uses
|
|
268
|
+
* `pathname + search` of the parsed URL. Returns true when the URL cannot be
|
|
269
|
+
* parsed (caller has already validated it; do not fail closed on a parse bug).
|
|
270
|
+
*/
|
|
271
|
+
export function isPathAllowedByRobots(rules, url) {
|
|
272
|
+
if (rules.disallowAll)
|
|
273
|
+
return false;
|
|
274
|
+
let parsed;
|
|
275
|
+
try {
|
|
276
|
+
parsed = new URL(url);
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
return true; // M-32: fail open, never closed, on an unparseable URL.
|
|
280
|
+
}
|
|
281
|
+
// Normalized the same way `compilePatternSegments` normalizes the rule
|
|
282
|
+
// pattern (see `normalizePercentEncoding`'s doc comment) — otherwise a
|
|
283
|
+
// percent-encoded alias of a disallowed path (e.g. `/%73ecret/` for a
|
|
284
|
+
// `Disallow: /secret/` rule) would never match because `URL.pathname`
|
|
285
|
+
// preserves %-escapes verbatim.
|
|
286
|
+
const target = normalizePercentEncoding(`${parsed.pathname}${parsed.search}`);
|
|
287
|
+
let best = null;
|
|
288
|
+
for (const rule of rules.rules) {
|
|
289
|
+
if (!matchesCompiledPattern(compiledPatternFor(rule), target))
|
|
290
|
+
continue;
|
|
291
|
+
const isMoreSpecific = !best || rule.specificity > best.specificity;
|
|
292
|
+
// Tie goes to Allow (least restrictive) — order-independent: this only
|
|
293
|
+
// upgrades a disallow to an allow at equal specificity, never the reverse.
|
|
294
|
+
const isTieBrokenByAllow = best !== null && rule.specificity === best.specificity && rule.kind === "allow" && best.kind === "disallow";
|
|
295
|
+
if (isMoreSpecific || isTieBrokenByAllow)
|
|
296
|
+
best = rule;
|
|
297
|
+
}
|
|
298
|
+
return best ? best.kind === "allow" : true;
|
|
299
|
+
}
|
|
300
|
+
// ── §4.4 createRobotsPolicy / createAllowAllRobotsPolicy ────────────────────
|
|
301
|
+
function outcomeToRuleSet(outcome) {
|
|
302
|
+
switch (outcome.kind) {
|
|
303
|
+
case "body":
|
|
304
|
+
return parseRobotsTxt(outcome.text);
|
|
305
|
+
case "unavailable":
|
|
306
|
+
return ALLOW_ALL_RULES;
|
|
307
|
+
case "unreachable":
|
|
308
|
+
return DISALLOW_ALL_RULES;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Crawl-scoped. Caches per origin (including in-flight promises) so the
|
|
313
|
+
* `load` callback runs at most once per origin regardless of how many pages
|
|
314
|
+
* or concurrent lookups reference it. Deliberately NOT a module-level cache
|
|
315
|
+
* (see the file-level doc comment) — callers construct one instance per crawl.
|
|
316
|
+
*/
|
|
317
|
+
export function createRobotsPolicy(load) {
|
|
318
|
+
const cache = new Map();
|
|
319
|
+
function rulesForOrigin(origin) {
|
|
320
|
+
const cached = cache.get(origin);
|
|
321
|
+
if (cached)
|
|
322
|
+
return cached;
|
|
323
|
+
const robotsUrl = new URL("/robots.txt", origin).toString();
|
|
324
|
+
const pending = load(robotsUrl).then(outcomeToRuleSet);
|
|
325
|
+
// A rejected fetch must not poison the cache: the next lookup for this
|
|
326
|
+
// origin should retry the loader rather than replaying the same failure
|
|
327
|
+
// forever (spec §4.4 L-04).
|
|
328
|
+
pending.catch(() => {
|
|
329
|
+
if (cache.get(origin) === pending)
|
|
330
|
+
cache.delete(origin);
|
|
331
|
+
});
|
|
332
|
+
cache.set(origin, pending);
|
|
333
|
+
return pending;
|
|
334
|
+
}
|
|
335
|
+
return {
|
|
336
|
+
rulesFor: (url) => rulesForOrigin(new URL(url).origin),
|
|
337
|
+
isAllowed: async (url) => isPathAllowedByRobots(await rulesForOrigin(new URL(url).origin), url),
|
|
338
|
+
crawlDelayMs: async (url) => (await rulesForOrigin(new URL(url).origin)).crawlDelayMs ?? 0,
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
/** Never fetches. isAllowed => true, crawlDelayMs => 0. Used when respectRobots is false. */
|
|
342
|
+
export function createAllowAllRobotsPolicy() {
|
|
343
|
+
return {
|
|
344
|
+
rulesFor: async () => ALLOW_ALL_RULES,
|
|
345
|
+
isAllowed: async () => true,
|
|
346
|
+
crawlDelayMs: async () => 0,
|
|
347
|
+
};
|
|
348
|
+
}
|