akm-cli 0.9.0-rc.13 → 0.9.0-rc.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/CHANGELOG.md +117 -23
  2. package/dist/akm-migrate +3 -1
  3. package/dist/assets/hints/cli-hints-full.md +3 -4
  4. package/dist/assets/hints/cli-hints-short.md +5 -5
  5. package/dist/assets/workflows/workflow-template.md +4 -3
  6. package/dist/cli/invocation.js +3 -2
  7. package/dist/cli/retired-commands.js +3 -0
  8. package/dist/cli/unknown-flags.js +226 -0
  9. package/dist/cli.js +19 -35
  10. package/dist/commands/agent/contribute-cli.js +15 -3
  11. package/dist/commands/feedback-cli.js +6 -3
  12. package/dist/commands/improve/collapse-detector.js +2 -3
  13. package/dist/commands/lint/base-linter.js +4 -16
  14. package/dist/commands/lint/index.js +13 -13
  15. package/dist/commands/log.js +6 -1
  16. package/dist/commands/migration-tool.js +4 -5
  17. package/dist/commands/observability-cli.js +1 -1
  18. package/dist/commands/proposal/repository.js +5 -5
  19. package/dist/commands/read/knowledge.js +2 -0
  20. package/dist/commands/registry-cli.js +5 -3
  21. package/dist/commands/sources/add-cli.js +6 -6
  22. package/dist/commands/sources/self-update.js +30 -7
  23. package/dist/commands/sources/source-add.js +17 -2
  24. package/dist/commands/tasks/tasks.js +8 -3
  25. package/dist/commands/workflow-cli.js +142 -119
  26. package/dist/core/adapter/adapters/akm-lint.js +17 -13
  27. package/dist/core/adapter/adapters/akm-task-adapter.js +14 -11
  28. package/dist/core/asset/akm-markdown.js +41 -8
  29. package/dist/core/asset/frontmatter.js +22 -0
  30. package/dist/core/asset/resolve-ref.js +23 -3
  31. package/dist/core/common.js +45 -2
  32. package/dist/core/config/config-schema.js +8 -0
  33. package/dist/core/config/schema/experimental.js +5 -13
  34. package/dist/core/config/schema/sources-bundles.js +11 -0
  35. package/dist/core/config/schema/workflow.js +3 -1
  36. package/dist/core/errors.js +5 -0
  37. package/dist/core/logs-db.js +2 -1
  38. package/dist/core/parse.js +4 -1
  39. package/dist/core/state/migrations.js +11 -14
  40. package/dist/core/state-db.js +4 -6
  41. package/dist/core/subprocess.js +6 -4
  42. package/dist/core/type-presentation.js +1 -1
  43. package/dist/indexer/indexer.js +16 -1
  44. package/dist/indexer/search/search-source.js +1 -1
  45. package/dist/indexer/walk/matchers.js +3 -1
  46. package/dist/integrations/agent/config.js +2 -2
  47. package/dist/integrations/agent/detect.js +49 -19
  48. package/dist/integrations/agent/profiles.js +10 -0
  49. package/dist/integrations/agent/spawn.js +1 -2
  50. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +7 -3
  51. package/dist/integrations/lockfile.js +11 -56
  52. package/dist/output/shapes/passthrough.js +0 -4
  53. package/dist/output/text/command-format.js +3 -3
  54. package/dist/output/text/helpers.js +1 -1
  55. package/dist/output/text/show-directives.js +4 -6
  56. package/dist/output/text/workflow-format.js +7 -62
  57. package/dist/output/text/workflow.js +1 -5
  58. package/dist/scripts/akm-migrate-node.js +58773 -0
  59. package/dist/scripts/akm-migrate.js +33976 -11391
  60. package/dist/setup/detect.js +40 -15
  61. package/dist/setup/setup.js +1 -1
  62. package/dist/sources/providers/git-stash.js +4 -2
  63. package/dist/sources/providers/website.js +5 -0
  64. package/dist/sources/snapshot-fetchers/bluesky.js +146 -0
  65. package/dist/sources/snapshot-fetchers/content-extract.js +370 -0
  66. package/dist/sources/snapshot-fetchers/fetcher-util.js +40 -0
  67. package/dist/sources/snapshot-fetchers/host-guard.js +199 -0
  68. package/dist/sources/snapshot-fetchers/registry.js +10 -1
  69. package/dist/sources/snapshot-fetchers/robots.js +348 -0
  70. package/dist/sources/snapshot-fetchers/rss.js +279 -0
  71. package/dist/sources/snapshot-fetchers/secret-seam.js +42 -0
  72. package/dist/sources/snapshot-fetchers/website-ingest.js +488 -257
  73. package/dist/sources/snapshot-fetchers/x.js +193 -0
  74. package/dist/storage/engines/sqlite-migrations.js +22 -107
  75. package/dist/tasks/runner.js +19 -16
  76. package/dist/tasks/schema.js +24 -1
  77. package/dist/workflows/exec/brief.js +1 -1
  78. package/dist/workflows/exec/frozen-judge.js +28 -2
  79. package/dist/workflows/exec/native-executor.js +18 -1
  80. package/dist/workflows/exec/report.js +11 -4
  81. package/dist/workflows/exec/run-workflow.js +103 -44
  82. package/dist/workflows/exec/step-work.js +19 -18
  83. package/dist/workflows/exec/unit-dispatch.js +4 -0
  84. package/dist/workflows/exec/workflow-engine-gate.js +11 -13
  85. package/dist/workflows/ir/compile.js +2 -2
  86. package/dist/workflows/ir/freeze.js +16 -8
  87. package/dist/workflows/ir/params.js +134 -10
  88. package/dist/workflows/ir/plan-hash.js +1 -1
  89. package/dist/workflows/ir/schema.js +6 -2
  90. package/dist/workflows/renderer.js +2 -2
  91. package/dist/workflows/runtime/checkin.js +3 -3
  92. package/dist/workflows/runtime/runs.js +50 -29
  93. package/dist/workflows/validate-summary.js +30 -14
  94. package/docs/migration/release-notes/0.9.0.md +31 -4
  95. package/docs/migration/v0.8-to-v0.9.md +69 -100
  96. package/docs/reference/data-and-telemetry.md +5 -5
  97. package/package.json +6 -2
  98. package/schemas/akm-config.json +24 -0
  99. package/schemas/akm-workflow.json +1 -1
  100. package/dist/workflows/cli.js +0 -33
@@ -0,0 +1,199 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { lookup as dnsLookup } from "node:dns/promises";
5
+ import { isIP } from "node:net";
6
+ import { fetchWithRetry, readBodyWithByteCap } from "../../core/common.js";
7
+ /** Redirect budget for a single guarded request chain. */
8
+ const MAX_GUARDED_REDIRECTS = 8;
9
+ export function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
10
+ const parsedUrl = new URL(rawUrl);
11
+ const hostname = parsedUrl.hostname.toLowerCase();
12
+ if (hostname.endsWith(".invalid")) {
13
+ throw new ErrorType(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
14
+ }
15
+ if (isForbiddenWebsiteHostname(hostname, options)) {
16
+ throw new ErrorType(`Refusing to fetch non-public website host: ${parsedUrl.hostname}`);
17
+ }
18
+ }
19
+ async function defaultResolveHostname(hostname) {
20
+ const records = await dnsLookup(hostname, { all: true });
21
+ return records.map((record) => record.address);
22
+ }
23
+ /**
24
+ * Resolve-then-validate SSRF guard against DNS rebinding / private-range
25
+ * bypasses. {@link assertWebsiteRequestUrl} only rejects IP-literal and
26
+ * well-known-name hosts; a hostname like `private-host.example.com` that
27
+ * resolves to `10.0.0.1` passes those checks and then fetch connects to the
28
+ * private address. Here we resolve EVERY A/AAAA record and validate each
29
+ * against the same forbidden-range rules, failing CLOSED on an empty answer or
30
+ * resolver error.
31
+ *
32
+ * TOCTOU residual (documented, not fully closable here): Bun's `fetch` exposes
33
+ * no custom `lookup`/agent hook, so we cannot pin the socket to the exact IP we
34
+ * validated — a hostile resolver could return a public IP to this lookup and a
35
+ * private IP microseconds later at connect time (classic rebinding). This still
36
+ * removes the TRIVIAL `hostname A 10.0.0.1` bypass, which is the strongest
37
+ * guarantee available without a pinned-connection fetch API. Re-run on every
38
+ * redirect hop (the crawler recurses through `fetchWebsiteResponse`).
39
+ */
40
+ export async function assertResolvedHostAllowed(hostname, options) {
41
+ if (options?.allowPrivateHosts === true)
42
+ return;
43
+ const bare = stripIpv6Brackets(hostname.toLowerCase());
44
+ // IP-literal hosts are already fully validated by assertWebsiteRequestUrl's
45
+ // range checks; resolving them is a no-op (and dnsLookup would just echo it).
46
+ if (isIP(bare) !== 0)
47
+ return;
48
+ const resolve = options?.resolveHostname ?? defaultResolveHostname;
49
+ let addresses;
50
+ try {
51
+ addresses = await resolve(bare);
52
+ }
53
+ catch {
54
+ throw new Error(`Refusing to fetch ${hostname}: DNS resolution failed`);
55
+ }
56
+ if (addresses.length === 0) {
57
+ throw new Error(`Refusing to fetch ${hostname}: hostname resolved to no addresses`);
58
+ }
59
+ for (const address of addresses) {
60
+ const version = isIP(address);
61
+ const forbidden = version === 4 ? isForbiddenIpv4(address) : version === 6 ? isForbiddenIpv6(stripIpv6Brackets(address)) : true;
62
+ if (forbidden) {
63
+ throw new Error(`Refusing to fetch ${hostname}: resolves to non-public or unparseable address ${address}`);
64
+ }
65
+ }
66
+ }
67
+ // WHATWG URL.hostname wraps IPv6 literals in brackets (e.g. "[::1]"), but
68
+ // node:net's isIP() only recognizes the bare address form and returns 0 for
69
+ // anything bracketed — silently skipping all IPv6 forbidden-host checks
70
+ // below for every hostname parsed off a URL. Strip the brackets before any
71
+ // isIP()/isForbiddenIpv6() call so those checks actually run.
72
+ function stripIpv6Brackets(hostname) {
73
+ return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
74
+ }
75
+ function isForbiddenWebsiteHostname(hostname, options) {
76
+ if (options?.allowPrivateHosts === true)
77
+ return false;
78
+ if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal") {
79
+ return true;
80
+ }
81
+ const bareHostname = stripIpv6Brackets(hostname);
82
+ const ipVersion = isIP(bareHostname);
83
+ if (ipVersion === 4)
84
+ return isForbiddenIpv4(bareHostname);
85
+ if (ipVersion === 6)
86
+ return isForbiddenIpv6(bareHostname);
87
+ return false;
88
+ }
89
+ export function isLoopbackWebsiteHostname(hostname) {
90
+ if (hostname === "localhost" || hostname.endsWith(".localhost"))
91
+ return true;
92
+ const bareHostname = stripIpv6Brackets(hostname);
93
+ const ipVersion = isIP(bareHostname);
94
+ if (ipVersion === 4)
95
+ return bareHostname.startsWith("127.");
96
+ if (ipVersion === 6)
97
+ return bareHostname === "::1";
98
+ return false;
99
+ }
100
+ function isForbiddenIpv4(hostname) {
101
+ const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
102
+ if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
103
+ return true;
104
+ const a = parts[0];
105
+ const b = parts[1];
106
+ return (a === 0 ||
107
+ a === 10 ||
108
+ a === 127 ||
109
+ (a === 169 && b === 254) ||
110
+ (a === 172 && b >= 16 && b <= 31) ||
111
+ (a === 192 && b === 168));
112
+ }
113
+ /**
114
+ * Extracts the embedded IPv4 address from an IPv4-mapped IPv6 literal
115
+ * (`::ffff:a.b.c.d` or its canonical hex form `::ffff:xxxx:yyyy`), or
116
+ * returns null if `hostname` isn't one.
117
+ */
118
+ function extractIpv4MappedAddress(normalizedHostname) {
119
+ const match = normalizedHostname.match(/^::ffff:(?:(\d{1,3}(?:\.\d{1,3}){3})|([0-9a-f]{1,4}):([0-9a-f]{1,4}))$/);
120
+ if (!match)
121
+ return null;
122
+ if (match[1])
123
+ return match[1];
124
+ const high = Number.parseInt(match[2], 16);
125
+ const low = Number.parseInt(match[3], 16);
126
+ return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
127
+ }
128
+ function isForbiddenIpv6(hostname) {
129
+ const normalized = hostname.toLowerCase();
130
+ const mappedIpv4 = extractIpv4MappedAddress(normalized);
131
+ if (mappedIpv4)
132
+ return isForbiddenIpv4(mappedIpv4);
133
+ return (normalized === "::" ||
134
+ normalized === "::1" ||
135
+ normalized.startsWith("fc") ||
136
+ normalized.startsWith("fd") ||
137
+ normalized.startsWith("fe8") ||
138
+ normalized.startsWith("fe9") ||
139
+ normalized.startsWith("fea") ||
140
+ normalized.startsWith("feb"));
141
+ }
142
+ /**
143
+ * Fetch with the full SSRF guard chain applied to the initial URL and to every
144
+ * redirect hop.
145
+ *
146
+ * `redirect: "manual"` is essential: with the default `follow`, the platform
147
+ * resolves and connects to redirect targets that were never validated, so a
148
+ * public host can bounce a request into the private network.
149
+ */
150
+ export async function fetchGuardedResponse(url, init, opts, redirectCount = 0) {
151
+ assertWebsiteRequestUrl(url, Error, opts);
152
+ await assertResolvedHostAllowed(new URL(url).hostname, opts);
153
+ const response = await fetchWithRetry(url, { ...init, redirect: "manual" }, {
154
+ timeout: opts.timeoutMs,
155
+ retries: opts.retries ?? 1,
156
+ });
157
+ if (response.status >= 300 && response.status < 400) {
158
+ if (redirectCount >= MAX_GUARDED_REDIRECTS) {
159
+ throw new Error(`Too many redirects while fetching ${url}`);
160
+ }
161
+ const location = response.headers.get("location");
162
+ if (!location) {
163
+ throw new Error(`Redirect response from ${url} did not include a Location header`);
164
+ }
165
+ const nextUrl = new URL(location, url).toString();
166
+ return fetchGuardedResponse(nextUrl, init, opts, redirectCount + 1);
167
+ }
168
+ return response;
169
+ }
170
+ /**
171
+ * GET a JSON document from a PINNED, constant API host (X, Bluesky XRPC), with
172
+ * a byte-capped read. Returns the parsed value, or null on any 3xx/non-2xx/
173
+ * parse failure.
174
+ *
175
+ * `redirect: "manual"` + rejecting 3xx is load-bearing: these endpoints have no
176
+ * legitimate reason to redirect, and following one would send the next request
177
+ * (bearer token included) to an unvalidated host with none of the SSRF guards
178
+ * the crawl path applies. The full resolve-then-validate DNS guard is
179
+ * intentionally NOT applied here because the host is a compile-time constant,
180
+ * not caller-supplied — the guarded chokepoint is {@link fetchGuardedResponse},
181
+ * which the caller-supplied crawl/feed paths use instead.
182
+ */
183
+ export async function fetchPinnedJson(url, options) {
184
+ const response = await fetchWithRetry(url, { headers: options.headers, signal: options.signal, redirect: "manual" }, { timeout: options.timeoutMs, retries: options.retries ?? 1 });
185
+ if (response.status >= 300 && response.status < 400)
186
+ return null;
187
+ if (!response.ok)
188
+ return null;
189
+ try {
190
+ const body = await readBodyWithByteCap(response, options.byteCap, {
191
+ bodyTimeoutMs: options.bodyTimeoutMs,
192
+ signal: options.signal,
193
+ });
194
+ return JSON.parse(body);
195
+ }
196
+ catch {
197
+ return null;
198
+ }
199
+ }
@@ -5,10 +5,19 @@ import fs from "node:fs";
5
5
  import path from "node:path";
6
6
  import { pathToFileURL } from "node:url";
7
7
  import { warn } from "../../core/warn.js";
8
+ import blueskyFetcher from "./bluesky.js";
9
+ import rssFetcher from "./rss.js";
10
+ import xFetcher from "./x.js";
8
11
  import youtubeFetcher from "./youtube.js";
9
12
  const FETCHER_DIR = path.join("scripts", "wiki-fetchers");
10
13
  const FETCHER_FILE_PATTERN = /\.(?:ts|js|mjs)$/i;
11
- const BUILTIN_FETCHERS = [youtubeFetcher];
14
+ /**
15
+ * Order matters: the first fetcher whose `matches` returns true and whose
16
+ * `fetch` returns non-null wins. Host-specific fetchers therefore precede the
17
+ * generic RSS fetcher, whose `matches` is path-shaped and deliberately loose.
18
+ * Any fetcher returning null falls through to the generic website crawler.
19
+ */
20
+ const BUILTIN_FETCHERS = [youtubeFetcher, blueskyFetcher, xFetcher, rssFetcher];
12
21
  function isWikiSnapshotFetcher(value) {
13
22
  if (!value || typeof value !== "object")
14
23
  return false;
@@ -0,0 +1,348 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * robots.txt parsing, matching, and a crawl-scoped fetch/cache policy.
6
+ *
7
+ * Behavioral reference: docs/plans/specs/p1-robots.md (authoritative). This
8
+ * module is a clean rewrite against akm's contracts, not a port of any
9
+ * upstream source file.
10
+ *
11
+ * Pure by design: no fetching happens here (see `RobotsTxtLoader`/
12
+ * `loadRobotsTxt` in `website-ingest.ts`). There is no crawl-semantic
13
+ * module-level state: the per-origin HTTP cache lives on the object
14
+ * `createRobotsPolicy` returns so it dies with the crawl that created it —
15
+ * a module-level `origin -> RobotsRuleSet` map would leak `disallowAll`
16
+ * results across crawls and across the test suite's process-wide `bun test`
17
+ * run (see spec §6.1). The module-level `WeakMap` below is a different
18
+ * category: it memoizes a pure function of an individual rule object's own
19
+ * `pattern` field, keyed by that object's identity. It cannot leak meaning
20
+ * between origins or tests — distinct `parseRobotsTxt` calls (even for
21
+ * identical robots.txt text) allocate distinct rule objects with distinct
22
+ * cache slots, and slots are reclaimed by the GC once a rule set is
23
+ * unreachable. It is memoization scoped to a single parsed rule set, not
24
+ * shared cross-crawl state.
25
+ */
26
+ import { warnVerbose } from "../../core/warn.js";
27
+ // ── §2 Constants (pinned by spec) ───────────────────────────────────────────
28
+ /** Group-matching tokens for `User-agent`, compared case-insensitively. */
29
+ export const ROBOTS_PRODUCT_TOKENS = ["akm", "akm-cli"];
30
+ /** 512 KiB. RFC 9309 §2.5 requires parsing at least 500 KiB. */
31
+ export const ROBOTS_BYTE_CAP = 512 * 1024;
32
+ /** Matches the existing website page-fetch timeout. */
33
+ export const ROBOTS_FETCH_TIMEOUT_MS = 15_000;
34
+ /** Body-read deadline; robots.txt is tiny. */
35
+ export const ROBOTS_BODY_TIMEOUT_MS = 15_000;
36
+ /** Ceiling applied to any parsed `Crawl-delay`, so a hostile robots.txt cannot stall the crawl. */
37
+ export const MAX_CRAWL_DELAY_MS = 10_000;
38
+ /** Identical to the existing website-fetch `User-Agent` header. */
39
+ export const ROBOTS_USER_AGENT_HEADER = "akm-cli website provider";
40
+ /**
41
+ * Frozen so a stray mutation (e.g. an accidental `.rules.push(...)` somewhere
42
+ * that borrows one of these as a starting point) can never poison every other
43
+ * crawl sharing this instance (spec §10, Review log 2026-08-01 finding 1).
44
+ */
45
+ const EMPTY_ROBOTS_RULES = Object.freeze([]);
46
+ export const ALLOW_ALL_RULES = Object.freeze({
47
+ disallowAll: false,
48
+ rules: EMPTY_ROBOTS_RULES,
49
+ crawlDelayMs: null,
50
+ });
51
+ export const DISALLOW_ALL_RULES = Object.freeze({
52
+ disallowAll: true,
53
+ rules: EMPTY_ROBOTS_RULES,
54
+ crawlDelayMs: null,
55
+ });
56
+ /** Strips a trailing `# comment` (RFC 9309 comments run to end of line). */
57
+ function stripRobotsComment(line) {
58
+ const hashIndex = line.indexOf("#");
59
+ return hashIndex === -1 ? line : line.slice(0, hashIndex);
60
+ }
61
+ /**
62
+ * Parses a `Crawl-delay` value (seconds, possibly fractional) into ms.
63
+ * Non-positive, unparseable, and empty values are ignored (`null`).
64
+ * `Infinity` is preserved so the caller's clamp reduces it to the ceiling
65
+ * rather than the value silently vanishing.
66
+ */
67
+ function parseCrawlDelayMs(raw) {
68
+ if (!raw)
69
+ return null;
70
+ const seconds = Number(raw);
71
+ if (Number.isNaN(seconds) || seconds <= 0)
72
+ return null;
73
+ return Math.round(seconds * 1000);
74
+ }
75
+ /**
76
+ * Pure. Never throws, never fetches.
77
+ *
78
+ * Grouping follows RFC 9309 §2.2.1: consecutive `User-agent` lines (no
79
+ * intervening directive) form one group; a matching *specific* product-token
80
+ * group suppresses the wildcard (`*`) group entirely; rules from all matching
81
+ * groups of the winning kind are unioned; `Crawl-delay` is the maximum across
82
+ * those groups, clamped once at the end.
83
+ */
84
+ export function parseRobotsTxt(text) {
85
+ const stripped = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
86
+ const lines = stripped.split(/\r\n|\r|\n/);
87
+ const groups = [];
88
+ let current = null;
89
+ // True once any non-user-agent directive has been seen for `current`, so
90
+ // the next `User-agent` line starts a NEW group instead of extending it.
91
+ let sawDirectiveSinceLastUserAgent = false;
92
+ for (const rawLine of lines) {
93
+ const line = stripRobotsComment(rawLine).trim();
94
+ if (!line)
95
+ continue;
96
+ const colonIndex = line.indexOf(":");
97
+ if (colonIndex === -1)
98
+ continue; // P-21: no colon => ignored
99
+ const directive = line.slice(0, colonIndex).trim().toLowerCase();
100
+ const value = line.slice(colonIndex + 1).trim();
101
+ if (directive === "user-agent") {
102
+ if (!value)
103
+ continue;
104
+ const token = value.toLowerCase();
105
+ if (current && !sawDirectiveSinceLastUserAgent) {
106
+ current.agents.push(token);
107
+ }
108
+ else {
109
+ current = { agents: [token], rules: [], crawlDelayMsRaw: null };
110
+ groups.push(current);
111
+ sawDirectiveSinceLastUserAgent = false;
112
+ }
113
+ continue;
114
+ }
115
+ if (!current)
116
+ continue; // P-13: directive with no preceding User-agent => ignored
117
+ sawDirectiveSinceLastUserAgent = true;
118
+ if (directive === "disallow" || directive === "allow") {
119
+ if (!value)
120
+ continue; // P-16: empty value matches nothing
121
+ if (!(value.startsWith("/") || value.startsWith("*"))) {
122
+ // P-18: neither "/" nor "*"-rooted => ignored, diagnosed quietly.
123
+ warnVerbose("[akm] robots.txt: ignoring %s value %s (patterns must start with '/' or '*')", directive, value);
124
+ continue;
125
+ }
126
+ const rule = {
127
+ kind: directive === "allow" ? "allow" : "disallow",
128
+ pattern: value,
129
+ specificity: value.length,
130
+ };
131
+ // Compile once, here, at parse time — not on every URL check (spec
132
+ // §6.4, review finding robots.ts:238). See `compiledPatternCache`.
133
+ compiledPatternCache.set(rule, compilePatternSegments(value));
134
+ current.rules.push(rule);
135
+ continue;
136
+ }
137
+ if (directive === "crawl-delay") {
138
+ const parsed = parseCrawlDelayMs(value);
139
+ if (parsed !== null)
140
+ current.crawlDelayMsRaw = parsed;
141
+ }
142
+ // Sitemap / unknown directives: ignored without diagnostic (P-19, P-20).
143
+ }
144
+ const productTokens = ROBOTS_PRODUCT_TOKENS;
145
+ const specificGroups = groups.filter((group) => group.agents.some((agent) => productTokens.includes(agent)));
146
+ const wildcardGroups = groups.filter((group) => group.agents.includes("*"));
147
+ const selected = specificGroups.length > 0 ? specificGroups : wildcardGroups;
148
+ const rules = selected.flatMap((group) => group.rules);
149
+ let maxDelayMs = null;
150
+ for (const group of selected) {
151
+ if (group.crawlDelayMsRaw !== null) {
152
+ maxDelayMs = maxDelayMs === null ? group.crawlDelayMsRaw : Math.max(maxDelayMs, group.crawlDelayMsRaw);
153
+ }
154
+ }
155
+ const crawlDelayMs = maxDelayMs === null ? null : Math.min(maxDelayMs, MAX_CRAWL_DELAY_MS);
156
+ return { disallowAll: false, rules, crawlDelayMs };
157
+ }
158
+ const COLLAPSE_STARS = /\*+/g;
159
+ const PERCENT_ENCODED_OCTET = /%[0-9a-fA-F]{2}/g;
160
+ /**
161
+ * RFC 3986 §2.3 "unreserved" set: ALPHA / DIGIT / "-" / "." / "_" / "~".
162
+ * A percent-encoded octet in this set is byte-identical to its literal
163
+ * character (`%73` and `s` are the same byte), so it carries no meaning a
164
+ * URL consumer can rely on. Every other octet — `%2F` ('/'), `%3F` ('?'),
165
+ * `%23` ('#'), `%2A` ('*'), non-ASCII bytes like `%C3` — is "reserved" or
166
+ * otherwise structurally significant and must be left percent-encoded:
167
+ * decoding e.g. `%2F` would silently turn one path segment into two.
168
+ */
169
+ function isUnreservedByte(byte) {
170
+ return ((byte >= 0x41 && byte <= 0x5a) || // A-Z
171
+ (byte >= 0x61 && byte <= 0x7a) || // a-z
172
+ (byte >= 0x30 && byte <= 0x39) || // 0-9
173
+ byte === 0x2d || // -
174
+ byte === 0x2e || // .
175
+ byte === 0x5f || // _
176
+ byte === 0x7e // ~
177
+ );
178
+ }
179
+ /**
180
+ * Normalizes percent-encoding for robots.txt matching, mirroring the RFC
181
+ * 9309 reference matcher: percent-encoded UNRESERVED octets are decoded to
182
+ * their literal character; every other `%XX` escape is left exactly as
183
+ * written (including hex-digit case). Applied identically to rule patterns
184
+ * (via `compilePatternSegments`, at parse time and in the defensive lazy
185
+ * fallback) and match targets (`isPathAllowedByRobots`), so a percent-encoded
186
+ * alias of a disallowed path — e.g. a link written as `/%73ecret/` — cannot
187
+ * bypass `Disallow: /secret/` simply because `URL.pathname` preserves
188
+ * %-escapes verbatim. Reserved-octet patterns like spec §4.3 M-31's
189
+ * `/caf%C3%A9` are untouched on both sides (0xC3/0xA9 are not unreserved),
190
+ * so they keep matching themselves exactly as before.
191
+ */
192
+ function normalizePercentEncoding(value) {
193
+ if (!value.includes("%"))
194
+ return value;
195
+ return value.replace(PERCENT_ENCODED_OCTET, (octetEscape) => {
196
+ const byte = Number.parseInt(octetEscape.slice(1), 16);
197
+ if (isUnreservedByte(byte))
198
+ return String.fromCharCode(byte);
199
+ // A reserved octet stays encoded, but `%2F` and `%2f` denote the same
200
+ // byte — RFC 3986 §6.2.2.1 makes the hex digits case-insensitive. Without
201
+ // canonicalizing the case, `Disallow: /a%2Fb` would fail to match a link
202
+ // spelled `/a%2fb` and the page would be crawled anyway.
203
+ return octetEscape.toUpperCase();
204
+ });
205
+ }
206
+ function compilePatternSegments(pattern) {
207
+ const anchored = pattern.endsWith("$");
208
+ const body = anchored ? pattern.slice(0, -1) : pattern;
209
+ const collapsed = normalizePercentEncoding(body).replace(COLLAPSE_STARS, "*");
210
+ return { anchored, segments: collapsed.split("*") };
211
+ }
212
+ /**
213
+ * Per-rule-object cache of the compiled pattern (see the file-level doc
214
+ * comment for why this is not "module-level state" in the sense spec §6.1
215
+ * forbids). `parseRobotsTxt` populates this eagerly, once, when it creates
216
+ * each `RobotsPathRule` — the compile-once requirement from spec §6.4 /
217
+ * review finding robots.ts:238. `compiledPatternFor` below is a defensive
218
+ * fallback for `RobotsRuleSet`s assembled by hand (e.g. test fixtures)
219
+ * rather than via `parseRobotsTxt`; it still compiles at most once per rule
220
+ * object, just lazily on first use instead of at parse time.
221
+ */
222
+ const compiledPatternCache = new WeakMap();
223
+ function compiledPatternFor(rule) {
224
+ const cached = compiledPatternCache.get(rule);
225
+ if (cached)
226
+ return cached;
227
+ const compiled = compilePatternSegments(rule.pattern);
228
+ compiledPatternCache.set(rule, compiled);
229
+ return compiled;
230
+ }
231
+ /**
232
+ * Matches a precompiled pattern against `target` (`pathname + search`).
233
+ * Greedy left-to-right, no backtracking — see the section doc comment above.
234
+ */
235
+ function matchesCompiledPattern(compiled, target) {
236
+ const { anchored, segments } = compiled;
237
+ const first = segments[0] ?? "";
238
+ if (!target.startsWith(first))
239
+ return false;
240
+ const pos = first.length;
241
+ const lastIndex = segments.length - 1;
242
+ if (lastIndex === 0) {
243
+ // No wildcard at all: already a prefix match; anchored additionally
244
+ // requires the pattern to consume the whole target.
245
+ return anchored ? target.length === first.length : true;
246
+ }
247
+ let cursor = pos;
248
+ for (let i = 1; i < lastIndex; i++) {
249
+ const segment = segments[i] ?? "";
250
+ const found = target.indexOf(segment, cursor);
251
+ if (found === -1)
252
+ return false;
253
+ // Greedy-leftmost is safe here: an earlier match of segment i can only
254
+ // give segment i+1 MORE room to be found later, never less, since the
255
+ // `*` between them absorbs any extra characters. No backtracking needed.
256
+ cursor = found + segment.length;
257
+ }
258
+ const last = segments[lastIndex] ?? "";
259
+ if (anchored) {
260
+ if (last.length > target.length - cursor)
261
+ return false;
262
+ return target.endsWith(last);
263
+ }
264
+ return last === "" || target.indexOf(last, cursor) !== -1;
265
+ }
266
+ /**
267
+ * Pure. `url` is an absolute http(s) URL string. Matching uses
268
+ * `pathname + search` of the parsed URL. Returns true when the URL cannot be
269
+ * parsed (caller has already validated it; do not fail closed on a parse bug).
270
+ */
271
+ export function isPathAllowedByRobots(rules, url) {
272
+ if (rules.disallowAll)
273
+ return false;
274
+ let parsed;
275
+ try {
276
+ parsed = new URL(url);
277
+ }
278
+ catch {
279
+ return true; // M-32: fail open, never closed, on an unparseable URL.
280
+ }
281
+ // Normalized the same way `compilePatternSegments` normalizes the rule
282
+ // pattern (see `normalizePercentEncoding`'s doc comment) — otherwise a
283
+ // percent-encoded alias of a disallowed path (e.g. `/%73ecret/` for a
284
+ // `Disallow: /secret/` rule) would never match because `URL.pathname`
285
+ // preserves %-escapes verbatim.
286
+ const target = normalizePercentEncoding(`${parsed.pathname}${parsed.search}`);
287
+ let best = null;
288
+ for (const rule of rules.rules) {
289
+ if (!matchesCompiledPattern(compiledPatternFor(rule), target))
290
+ continue;
291
+ const isMoreSpecific = !best || rule.specificity > best.specificity;
292
+ // Tie goes to Allow (least restrictive) — order-independent: this only
293
+ // upgrades a disallow to an allow at equal specificity, never the reverse.
294
+ const isTieBrokenByAllow = best !== null && rule.specificity === best.specificity && rule.kind === "allow" && best.kind === "disallow";
295
+ if (isMoreSpecific || isTieBrokenByAllow)
296
+ best = rule;
297
+ }
298
+ return best ? best.kind === "allow" : true;
299
+ }
300
+ // ── §4.4 createRobotsPolicy / createAllowAllRobotsPolicy ────────────────────
301
+ function outcomeToRuleSet(outcome) {
302
+ switch (outcome.kind) {
303
+ case "body":
304
+ return parseRobotsTxt(outcome.text);
305
+ case "unavailable":
306
+ return ALLOW_ALL_RULES;
307
+ case "unreachable":
308
+ return DISALLOW_ALL_RULES;
309
+ }
310
+ }
311
+ /**
312
+ * Crawl-scoped. Caches per origin (including in-flight promises) so the
313
+ * `load` callback runs at most once per origin regardless of how many pages
314
+ * or concurrent lookups reference it. Deliberately NOT a module-level cache
315
+ * (see the file-level doc comment) — callers construct one instance per crawl.
316
+ */
317
+ export function createRobotsPolicy(load) {
318
+ const cache = new Map();
319
+ function rulesForOrigin(origin) {
320
+ const cached = cache.get(origin);
321
+ if (cached)
322
+ return cached;
323
+ const robotsUrl = new URL("/robots.txt", origin).toString();
324
+ const pending = load(robotsUrl).then(outcomeToRuleSet);
325
+ // A rejected fetch must not poison the cache: the next lookup for this
326
+ // origin should retry the loader rather than replaying the same failure
327
+ // forever (spec §4.4 L-04).
328
+ pending.catch(() => {
329
+ if (cache.get(origin) === pending)
330
+ cache.delete(origin);
331
+ });
332
+ cache.set(origin, pending);
333
+ return pending;
334
+ }
335
+ return {
336
+ rulesFor: (url) => rulesForOrigin(new URL(url).origin),
337
+ isAllowed: async (url) => isPathAllowedByRobots(await rulesForOrigin(new URL(url).origin), url),
338
+ crawlDelayMs: async (url) => (await rulesForOrigin(new URL(url).origin)).crawlDelayMs ?? 0,
339
+ };
340
+ }
341
+ /** Never fetches. isAllowed => true, crawlDelayMs => 0. Used when respectRobots is false. */
342
+ export function createAllowAllRobotsPolicy() {
343
+ return {
344
+ rulesFor: async () => ALLOW_ALL_RULES,
345
+ isAllowed: async () => true,
346
+ crawlDelayMs: async () => 0,
347
+ };
348
+ }