akm-cli 0.9.0-rc.13 → 0.9.0-rc.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/CHANGELOG.md +117 -23
  2. package/dist/akm-migrate +3 -1
  3. package/dist/assets/hints/cli-hints-full.md +3 -4
  4. package/dist/assets/hints/cli-hints-short.md +5 -5
  5. package/dist/assets/workflows/workflow-template.md +4 -3
  6. package/dist/cli/invocation.js +3 -2
  7. package/dist/cli/retired-commands.js +3 -0
  8. package/dist/cli/unknown-flags.js +226 -0
  9. package/dist/cli.js +19 -35
  10. package/dist/commands/agent/contribute-cli.js +15 -3
  11. package/dist/commands/feedback-cli.js +6 -3
  12. package/dist/commands/improve/collapse-detector.js +2 -3
  13. package/dist/commands/lint/base-linter.js +4 -16
  14. package/dist/commands/lint/index.js +13 -13
  15. package/dist/commands/log.js +6 -1
  16. package/dist/commands/migration-tool.js +4 -5
  17. package/dist/commands/observability-cli.js +1 -1
  18. package/dist/commands/proposal/repository.js +5 -5
  19. package/dist/commands/read/knowledge.js +2 -0
  20. package/dist/commands/registry-cli.js +5 -3
  21. package/dist/commands/sources/add-cli.js +6 -6
  22. package/dist/commands/sources/self-update.js +30 -7
  23. package/dist/commands/sources/source-add.js +17 -2
  24. package/dist/commands/tasks/tasks.js +8 -3
  25. package/dist/commands/workflow-cli.js +142 -119
  26. package/dist/core/adapter/adapters/akm-lint.js +17 -13
  27. package/dist/core/adapter/adapters/akm-task-adapter.js +14 -11
  28. package/dist/core/asset/akm-markdown.js +41 -8
  29. package/dist/core/asset/frontmatter.js +22 -0
  30. package/dist/core/asset/resolve-ref.js +23 -3
  31. package/dist/core/common.js +45 -2
  32. package/dist/core/config/config-schema.js +8 -0
  33. package/dist/core/config/schema/experimental.js +5 -13
  34. package/dist/core/config/schema/sources-bundles.js +11 -0
  35. package/dist/core/config/schema/workflow.js +3 -1
  36. package/dist/core/errors.js +5 -0
  37. package/dist/core/logs-db.js +2 -1
  38. package/dist/core/parse.js +4 -1
  39. package/dist/core/state/migrations.js +11 -14
  40. package/dist/core/state-db.js +4 -6
  41. package/dist/core/subprocess.js +6 -4
  42. package/dist/core/type-presentation.js +1 -1
  43. package/dist/indexer/indexer.js +16 -1
  44. package/dist/indexer/search/search-source.js +1 -1
  45. package/dist/indexer/walk/matchers.js +3 -1
  46. package/dist/integrations/agent/config.js +2 -2
  47. package/dist/integrations/agent/detect.js +49 -19
  48. package/dist/integrations/agent/profiles.js +10 -0
  49. package/dist/integrations/agent/spawn.js +1 -2
  50. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +7 -3
  51. package/dist/integrations/lockfile.js +11 -56
  52. package/dist/output/shapes/passthrough.js +0 -4
  53. package/dist/output/text/command-format.js +3 -3
  54. package/dist/output/text/helpers.js +1 -1
  55. package/dist/output/text/show-directives.js +4 -6
  56. package/dist/output/text/workflow-format.js +7 -62
  57. package/dist/output/text/workflow.js +1 -5
  58. package/dist/scripts/akm-migrate-node.js +58773 -0
  59. package/dist/scripts/akm-migrate.js +33976 -11391
  60. package/dist/setup/detect.js +40 -15
  61. package/dist/setup/setup.js +1 -1
  62. package/dist/sources/providers/git-stash.js +4 -2
  63. package/dist/sources/providers/website.js +5 -0
  64. package/dist/sources/snapshot-fetchers/bluesky.js +146 -0
  65. package/dist/sources/snapshot-fetchers/content-extract.js +370 -0
  66. package/dist/sources/snapshot-fetchers/fetcher-util.js +40 -0
  67. package/dist/sources/snapshot-fetchers/host-guard.js +199 -0
  68. package/dist/sources/snapshot-fetchers/registry.js +10 -1
  69. package/dist/sources/snapshot-fetchers/robots.js +348 -0
  70. package/dist/sources/snapshot-fetchers/rss.js +279 -0
  71. package/dist/sources/snapshot-fetchers/secret-seam.js +42 -0
  72. package/dist/sources/snapshot-fetchers/website-ingest.js +488 -257
  73. package/dist/sources/snapshot-fetchers/x.js +193 -0
  74. package/dist/storage/engines/sqlite-migrations.js +22 -107
  75. package/dist/tasks/runner.js +19 -16
  76. package/dist/tasks/schema.js +24 -1
  77. package/dist/workflows/exec/brief.js +1 -1
  78. package/dist/workflows/exec/frozen-judge.js +28 -2
  79. package/dist/workflows/exec/native-executor.js +18 -1
  80. package/dist/workflows/exec/report.js +11 -4
  81. package/dist/workflows/exec/run-workflow.js +103 -44
  82. package/dist/workflows/exec/step-work.js +19 -18
  83. package/dist/workflows/exec/unit-dispatch.js +4 -0
  84. package/dist/workflows/exec/workflow-engine-gate.js +11 -13
  85. package/dist/workflows/ir/compile.js +2 -2
  86. package/dist/workflows/ir/freeze.js +16 -8
  87. package/dist/workflows/ir/params.js +134 -10
  88. package/dist/workflows/ir/plan-hash.js +1 -1
  89. package/dist/workflows/ir/schema.js +6 -2
  90. package/dist/workflows/renderer.js +2 -2
  91. package/dist/workflows/runtime/checkin.js +3 -3
  92. package/dist/workflows/runtime/runs.js +50 -29
  93. package/dist/workflows/validate-summary.js +30 -14
  94. package/docs/migration/release-notes/0.9.0.md +31 -4
  95. package/docs/migration/v0.8-to-v0.9.md +69 -100
  96. package/docs/reference/data-and-telemetry.md +5 -5
  97. package/package.json +6 -2
  98. package/schemas/akm-config.json +24 -0
  99. package/schemas/akm-workflow.json +1 -1
  100. package/dist/workflows/cli.js +0 -33
@@ -2,17 +2,18 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { createHash } from "node:crypto";
5
- import { lookup as dnsLookup } from "node:dns/promises";
6
5
  import fs from "node:fs";
7
- import { isIP } from "node:net";
8
6
  import path from "node:path";
9
7
  import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir, todayIso, } from "../../core/common.js";
10
8
  import { ConfigError, UsageError } from "../../core/errors.js";
11
9
  import { getRegistryIndexCacheDir } from "../../core/paths.js";
12
- import { warn } from "../../core/warn.js";
10
+ import { warn, warnVerbose } from "../../core/warn.js";
13
11
  import { withFreshnessCache } from "../freshness.js";
14
12
  import { sanitizeString } from "../providers/provider-utils.js";
13
+ import { htmlToMarkdownAndLinks } from "./content-extract.js";
14
+ import { assertResolvedHostAllowed, assertWebsiteRequestUrl, isLoopbackWebsiteHostname, } from "./host-guard.js";
15
15
  import { loadWikiSnapshotFetchers } from "./registry.js";
16
+ import { createAllowAllRobotsPolicy, createRobotsPolicy, isPathAllowedByRobots, ROBOTS_BODY_TIMEOUT_MS, ROBOTS_BYTE_CAP, } from "./robots.js";
16
17
  /** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
17
18
  const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
18
19
  /** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
@@ -35,6 +36,32 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
35
36
  */
36
37
  const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
37
38
  const WEBSITE_MAX_REDIRECTS = 8;
39
+ /**
40
+ * Coerces the user-facing `crawlTimeoutMs` option.
41
+ *
42
+ * Returns `null` for an explicit opt-out (`false`, or `0`), the configured
43
+ * number of milliseconds when positive, and `undefined` to mean "unset, use
44
+ * the default". Anything else is ignored rather than failing a crawl over a
45
+ * malformed knob.
46
+ */
47
+ function coerceCrawlTimeoutMs(value) {
48
+ if (value === false || value === 0)
49
+ return null;
50
+ if (value === true || value === undefined || value === null)
51
+ return undefined;
52
+ const parsed = typeof value === "number" ? value : typeof value === "string" ? Number.parseInt(value, 10) : Number.NaN;
53
+ if (!Number.isFinite(parsed))
54
+ return undefined;
55
+ if (parsed <= 0)
56
+ return null;
57
+ return parsed;
58
+ }
59
+ /**
60
+ * How many times a URL may be pushed back for not fitting its origin's
61
+ * `Crawl-delay` in the remaining budget before it is reported unfetched.
62
+ * Bounds the requeue loop when every remaining URL is rate-limited.
63
+ */
64
+ const MAX_CRAWL_DEFERRALS = 3;
38
65
  /**
39
66
  * Body-read deadline for a single page (30s). The per-request fetch timeout
40
67
  * (15s) bounds only the connection/header phase; without this a server that
@@ -90,7 +117,16 @@ export async function ensureWebsiteMirror(config, options) {
90
117
  await scrapeWebsiteToStash(normalizedUrl, cachePaths.stashDir, {
91
118
  maxPages: coercePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT),
92
119
  maxDepth: coercePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT),
120
+ respectRobots: coerceRespectRobots(config.options?.respectRobots),
93
121
  allowPrivateHosts: options?.allowPrivateHosts,
122
+ wallClockCapMs: options?.wallClockCapMs,
123
+ crawlTimeoutMs: coerceCrawlTimeoutMs(config.options?.crawlTimeoutMs),
124
+ resolveSecret: options?.resolveSecret,
125
+ // As-supplied, pre-normalization start URL (see crawlWebsite's
126
+ // `rawStartUrl` doc comment): threaded through purely for the C-02
127
+ // robots.txt check, which must see the trailing slash the user
128
+ // actually typed before `normalizeSiteUrl` strips it.
129
+ rawStartUrl: rawUrl,
94
130
  });
95
131
  fs.writeFileSync(cachePaths.manifestPath, `${JSON.stringify({ url: normalizedUrl, fetchedAt: new Date().toISOString() }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
96
132
  },
@@ -117,7 +153,68 @@ function hasExtractedSite(stashDir) {
117
153
  return false;
118
154
  }
119
155
  }
156
+ /**
157
+ * Iterate the snapshot-fetcher registry against a parsed URL, returning the
158
+ * first fetcher that produces content, or null when none match. A fetcher that
159
+ * throws is logged and treated as a non-match — one broken fetcher must not
160
+ * fail the whole source.
161
+ */
162
+ async function dispatchSnapshotFetchers(parsed, context, stashDir) {
163
+ for (const fetcher of await loadWikiSnapshotFetchers(stashDir)) {
164
+ try {
165
+ if (!fetcher.matches(parsed, context))
166
+ continue;
167
+ const snapshot = await fetcher.fetch(parsed, context);
168
+ if (snapshot)
169
+ return snapshot;
170
+ }
171
+ catch (error) {
172
+ warn("[akm] snapshot fetcher %s threw on %s: %s", fetcher.name, parsed.toString(), error instanceof Error ? error.message : String(error));
173
+ }
174
+ }
175
+ return null;
176
+ }
177
+ /**
178
+ * Run the snapshot-fetcher registry against a URL. Returns null when no
179
+ * fetcher matches or produces content, so the caller falls back to a crawl.
180
+ */
181
+ async function fetchSnapshotViaRegistry(startUrl, stashDir, allowPrivateHosts, resolveSecret) {
182
+ let parsed;
183
+ try {
184
+ parsed = new URL(startUrl);
185
+ }
186
+ catch {
187
+ return null;
188
+ }
189
+ const context = {
190
+ stashDir,
191
+ timeoutMs: 15_000,
192
+ ...(resolveSecret ? { resolveSecret } : {}),
193
+ ...(allowPrivateHosts ? { allowPrivateHosts: true } : {}),
194
+ };
195
+ return dispatchSnapshotFetchers(parsed, context, stashDir);
196
+ }
197
+ /** Materialize a single fetcher snapshot as the source's whole stash. */
198
+ function writeSnapshotToStash(stashDir, snapshot) {
199
+ const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
200
+ const relPath = avoidReservedBasename(preferredName);
201
+ const knowledgeDir = path.join(stashDir, "knowledge");
202
+ fs.rmSync(stashDir, { recursive: true, force: true });
203
+ const filePath = path.join(knowledgeDir, `${relPath}.md`);
204
+ fs.mkdirSync(path.dirname(filePath), { recursive: true });
205
+ const slug = relPath.split("/").pop() ?? "index";
206
+ fs.writeFileSync(filePath, buildMarkdownSnapshot({ url: snapshot.url, title: snapshot.title, markdown: snapshot.markdown }, slug, snapshot.tags), "utf8");
207
+ }
120
208
  async function scrapeWebsiteToStash(startUrl, stashDir, options) {
209
+ // Offer the URL to the specialized fetchers before falling back to a crawl.
210
+ // Without this, `akm bundle add <feed|profile URL>` reaches only the generic
211
+ // crawler and the RSS/Bluesky/X/YouTube fetchers are unreachable outside the
212
+ // `akm import` path. A fetcher returning null falls through to the crawl.
213
+ const fetched = await fetchSnapshotViaRegistry(startUrl, stashDir, options.allowPrivateHosts, options.resolveSecret);
214
+ if (fetched) {
215
+ writeSnapshotToStash(stashDir, fetched);
216
+ return;
217
+ }
121
218
  const pages = await crawlWebsite(startUrl, options);
122
219
  if (pages.length === 0) {
123
220
  throw new Error(`No content could be scraped from ${startUrl}`);
@@ -145,20 +242,12 @@ export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
145
242
  stashDir: stashDir ?? "",
146
243
  timeoutMs: options?.timeoutMs ?? 15_000,
147
244
  signal: options?.signal,
245
+ ...(options?.resolveSecret ? { resolveSecret: options.resolveSecret } : {}),
246
+ ...(options?.allowPrivateHosts ? { allowPrivateHosts: true } : {}),
148
247
  };
149
- for (const fetcher of await loadWikiSnapshotFetchers(stashDir)) {
150
- try {
151
- if (!fetcher.matches(parsedUrl, context))
152
- continue;
153
- const snapshot = await fetcher.fetch(parsedUrl, context);
154
- if (!snapshot)
155
- continue;
156
- return websiteMarkdownSnapshotFromResult(snapshot);
157
- }
158
- catch (error) {
159
- warn("[akm] wiki-fetcher %s threw on %s: %s", fetcher.name, normalizedUrl, error instanceof Error ? error.message : String(error));
160
- }
161
- }
248
+ const snapshot = await dispatchSnapshotFetchers(parsedUrl, context, stashDir);
249
+ if (snapshot)
250
+ return websiteMarkdownSnapshotFromResult(snapshot);
162
251
  const fetched = await fetchWebsitePage(normalizedUrl, { allowPrivateHosts: options?.allowPrivateHosts });
163
252
  if (!fetched) {
164
253
  throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
@@ -184,13 +273,168 @@ function websiteMarkdownSnapshotFromResult(snapshot) {
184
273
  }, slug || "website", snapshot.tags),
185
274
  };
186
275
  }
276
+ function sleep(ms) {
277
+ return new Promise((resolve) => setTimeout(resolve, ms));
278
+ }
279
+ /**
280
+ * Evaluates a URL against robots.txt, checking both `normalizedUrl` — the
281
+ * form akm treats as canonical for dedup/storage, with any bare trailing
282
+ * slash already stripped by `normalizeCrawlUrl`/`normalizeSiteUrl` — and,
283
+ * when it differs, `rawUrl`: the URL exactly as discovered (a link's literal
284
+ * `href`), as-supplied (the user's typed start URL), or redirected-to (a
285
+ * `Location` header), before any such stripping. Delegates the actual
286
+ * allow/disallow decision to {@link decideRobotsAllowance}'s asymmetric
287
+ * matrix — see its doc comment — rather than a plain AND of both forms: a
288
+ * `Disallow: /dir/`-shaped rule needs the un-stripped `rawUrl` to ever match
289
+ * (closing that gap), while an `Allow: /docs/`-shaped rule needs it too, in
290
+ * the *other* direction (a normalized-disallowed-but-raw-allowed URL must
291
+ * still be treated as allowed here, not just at the point where akm chooses
292
+ * which literal URL to fetch — a URL reaching this function has already been
293
+ * fetched, under whichever form `crawlWebsite`/`resolveCrawlRobotsDecision`
294
+ * selected, so only the `allowed` verdict matters here, never `fetchUrl`).
295
+ *
296
+ * `normalizeCrawlUrl`/`normalizeSiteUrl` strip a bare trailing slash before
297
+ * any robots check ever runs, so a `Disallow: /dir/`-shaped rule — which
298
+ * requires a literal trailing `/` in the target, see
299
+ * `matchesCompiledPattern`'s prefix check — can never match the stripped
300
+ * alias. Checking the un-stripped `rawUrl` closes that gap without changing
301
+ * what akm treats as the canonical URL for storage/dedup, and (crucially)
302
+ * without over-blocking a URL that never had a trailing slash to begin with
303
+ * — e.g. a `/secret` link that happens to redirect to `/secret/`: the
304
+ * pre-redirect request itself is unaffected by a `Disallow: /secret/` rule,
305
+ * only the redirect target is.
306
+ */
307
+ async function isCrawlUrlAllowedByRobots(robots, normalizedUrl, rawUrl) {
308
+ return (await resolveCrawlRobotsDecision(robots, normalizedUrl, rawUrl)).allowed;
309
+ }
310
+ /**
311
+ * Decides whether `normalizedUrl` (akm's canonical form — dedup/cache key,
312
+ * with any bare trailing slash already stripped by `normalizeCrawlUrl`/
313
+ * `normalizeSiteUrl`) may be crawled, and which literal URL to actually
314
+ * request.
315
+ *
316
+ * A `Disallow: /dir/`-shaped rule requires a literal trailing `/` in the
317
+ * target (see `matchesCompiledPattern`'s prefix check), so it can never match
318
+ * the slash-stripped normalized alias — checking the un-stripped `rawUrl` (a
319
+ * link's literal `href`, the user's as-typed start URL, or a redirect
320
+ * `Location`) closes that gap. Symmetrically, an `Allow: /docs/`-shaped rule
321
+ * requires that same trailing `/` to match, so a start URL or link typed as
322
+ * `.../docs/` under `Disallow: / \n Allow: /docs/` is allowed in its raw form
323
+ * but disallowed once normalized — over-blocking a site the owner explicitly
324
+ * opened to crawlers.
325
+ *
326
+ * Resolution matrix (raw form only consulted when it differs from normalized):
327
+ * - normalized allowed, raw allowed (or no distinct raw) => allowed, fetch normalized
328
+ * - normalized allowed, raw disallowed => BLOCKED (raw wins: closes the Disallow: /dir/ gap)
329
+ * - normalized disallowed, raw allowed => allowed, fetch the RAW url (closes the Allow: /docs/ gap)
330
+ * - normalized disallowed, raw disallowed => BLOCKED
331
+ *
332
+ * Only the third row switches the fetch target; every other row fetches the
333
+ * normalized form akm already treats as canonical. Do not collapse this to a
334
+ * plain OR of the two checks — that would also flip the second row to
335
+ * "allowed" and reopen the `Disallow: /dir/` bypass this matrix exists to
336
+ * close.
337
+ */
338
+ function decideRobotsAllowance(rules, normalizedUrl, rawUrl) {
339
+ const normalizedAllowed = isPathAllowedByRobots(rules, normalizedUrl);
340
+ if (!rawUrl || rawUrl === normalizedUrl) {
341
+ return { allowed: normalizedAllowed, fetchUrl: normalizedUrl };
342
+ }
343
+ const rawAllowed = isPathAllowedByRobots(rules, rawUrl);
344
+ if (!normalizedAllowed && rawAllowed) {
345
+ return { allowed: true, fetchUrl: rawUrl };
346
+ }
347
+ return { allowed: normalizedAllowed && rawAllowed, fetchUrl: normalizedUrl };
348
+ }
349
+ /**
350
+ * Async wrapper of {@link decideRobotsAllowance} for call sites holding a
351
+ * `RobotsPolicy` (which resolves/caches rules per origin) rather than an
352
+ * already-fetched `RobotsRuleSet`.
353
+ */
354
+ async function resolveCrawlRobotsDecision(robots, normalizedUrl, rawUrl) {
355
+ const rules = await robots.rulesFor(normalizedUrl);
356
+ return decideRobotsAllowance(rules, normalizedUrl, rawUrl);
357
+ }
358
+ /**
359
+ * C-02/C-03: fail fast, before any page fetch, when the start URL itself is
360
+ * off-limits. A 5xx robots.txt (RobotsPolicy caches `DISALLOW_ALL_RULES` for
361
+ * that case) gets a distinct message calling out the server error, per spec
362
+ * §4.6.
363
+ *
364
+ * Checks both `start`'s (normalized) URL and `rawStartUrl` — the URL exactly
365
+ * as the user supplied it in config, before `validateWebsiteUrl` ->
366
+ * `normalizeSiteUrl` stripped any trailing slash — via
367
+ * `decideRobotsAllowance`. Without this, a start URL typed as `.../secret/`
368
+ * under `Disallow: /secret/` would never match that rule and would be
369
+ * crawled instead of rejected with the spec §4.6 C-02 UsageError; conversely,
370
+ * a start URL typed as `.../docs/` under `Disallow: / \n Allow: /docs/`
371
+ * would be normalized to `.../docs`, fail to match `Allow: /docs/`, and be
372
+ * rejected even though the site owner explicitly opened `/docs/` to
373
+ * crawlers. `crawlWebsite`'s queue gate applies the same decision (and, in
374
+ * the Allow case, actually fetches the raw URL this function only validates
375
+ * against) — see its call to `resolveCrawlRobotsDecision`.
376
+ */
377
+ async function assertStartUrlAllowedByRobots(robots, start, rawStartUrl) {
378
+ const startUrl = start.toString();
379
+ const robotsUrl = new URL("/robots.txt", start.origin).toString();
380
+ const rules = await robots.rulesFor(startUrl);
381
+ if (rules.disallowAll) {
382
+ throw new UsageError(`Refusing to crawl ${startUrl}: ${robotsUrl} returned a server error, which robots.txt conventions ` +
383
+ `treat as a full disallow until it recovers. Set respectRobots: false on this website source to bypass ` +
384
+ `robots.txt.`);
385
+ }
386
+ let rawStartUrlNormalized;
387
+ if (rawStartUrl) {
388
+ try {
389
+ rawStartUrlNormalized = new URL(rawStartUrl).toString();
390
+ }
391
+ catch {
392
+ rawStartUrlNormalized = undefined;
393
+ }
394
+ }
395
+ const { allowed } = decideRobotsAllowance(rules, startUrl, rawStartUrlNormalized);
396
+ if (!allowed) {
397
+ throw new UsageError(`Refusing to crawl ${startUrl}: disallowed by ${robotsUrl}. Set respectRobots: false on this website ` +
398
+ `source to bypass robots.txt.`);
399
+ }
400
+ }
187
401
  async function crawlWebsite(startUrl, options) {
188
402
  const start = new URL(normalizeSiteUrl(startUrl));
189
403
  const allowedOrigin = start.origin;
190
- const queue = [{ url: start.toString(), depth: 0 }];
404
+ const queue = [
405
+ { url: start.toString(), rawUrl: options.rawStartUrl ?? start.toString(), depth: 0, deferrals: 0 },
406
+ ];
191
407
  const visited = new Set();
192
408
  const pages = [];
193
- const deadline = Date.now() + WEBSITE_CRAWL_WALL_CLOCK_MS;
409
+ // Precedence: the test-only seam, then the user's `crawlTimeoutMs`, then the
410
+ // default. `crawlTimeoutMs: 0` / `false` disables the cap outright, for a
411
+ // deliberately long-running crawl the user is willing to babysit.
412
+ const configuredCapMs = options.crawlTimeoutMs === null ? null : (options.crawlTimeoutMs ?? WEBSITE_CRAWL_WALL_CLOCK_MS);
413
+ const wallClockCapMs = options.wallClockCapMs ?? configuredCapMs;
414
+ const capDisabled = wallClockCapMs === null;
415
+ const deadline = capDisabled ? Number.POSITIVE_INFINITY : Date.now() + wallClockCapMs;
416
+ // Between-iteration deadline checks cannot interrupt work already in
417
+ // flight: a single request's `Retry-After` sleep, or a slow body read, can
418
+ // run far past the cap on its own. This signal makes the cap a HARD limit —
419
+ // it aborts the in-flight fetch and the retry sleep alike.
420
+ const abortController = new AbortController();
421
+ const capTimer = capDisabled
422
+ ? undefined
423
+ : setTimeout(() => abortController.abort(new Error(`Website crawl exceeded its ${wallClockCapMs / 1000}s limit`)), wallClockCapMs);
424
+ const crawlSignal = abortController.signal;
425
+ const robots = options.respectRobots === false
426
+ ? createAllowAllRobotsPolicy()
427
+ : createRobotsPolicy((robotsUrl) => loadRobotsTxt(robotsUrl, { allowPrivateHosts: options.allowPrivateHosts, signal: crawlSignal }));
428
+ await assertStartUrlAllowedByRobots(robots, start, options.rawStartUrl);
429
+ // Counts actual `fetchWebsitePage` invocations (regardless of outcome) so
430
+ // Crawl-delay pacing skips the first fetch and never charges a delay slot
431
+ // to a URL that robots.txt skipped without ever being fetched (C-11).
432
+ let fetchAttempts = 0;
433
+ // URLs pushed back because their Crawl-delay would not fit in the remaining
434
+ // budget, and URLs that ran out of retries entirely. Reported at the end so
435
+ // a rate-limited origin is visible rather than silently missing.
436
+ const deferred = new Set();
437
+ const unfetched = new Set();
194
438
  while (queue.length > 0 && pages.length < options.maxPages) {
195
439
  if (Date.now() > deadline)
196
440
  break;
@@ -200,8 +444,49 @@ async function crawlWebsite(startUrl, options) {
200
444
  const normalized = normalizeCrawlUrl(next.url);
201
445
  if (!normalized || visited.has(normalized))
202
446
  continue;
447
+ const decision = await resolveCrawlRobotsDecision(robots, normalized, next.rawUrl);
448
+ if (!decision.allowed) {
449
+ // Deliberately NOT marked visited. `/docs/` and `/docs` share a
450
+ // normalized key but get different robots verdicts (a `Disallow: /docs/`
451
+ // rule matches only the trailing-slash form). Marking the key visited
452
+ // here would let whichever alias happened to be discovered first — and
453
+ // was then rejected — permanently suppress the allowed alias, making
454
+ // crawl coverage depend on link order. Robots rules are cached per
455
+ // origin, so re-evaluating a repeated disallowed alias is cheap.
456
+ warnVerbose("[akm] website crawl: skipping %s (disallowed by robots.txt)", normalized);
457
+ continue;
458
+ }
459
+ if (fetchAttempts > 0) {
460
+ const delayMs = await robots.crawlDelayMs(normalized);
461
+ if (delayMs > 0) {
462
+ if (Date.now() + delayMs >= deadline) {
463
+ // Sleeping this one out would blow the wall-clock cap. Defer it to
464
+ // the back of the queue instead of ending the crawl here: other
465
+ // origins may have no Crawl-delay and can still be fetched with the
466
+ // time that remains. Anything still deferred when the deadline
467
+ // arrives is reported as unfetched below rather than silently
468
+ // dropped. `deferred` is bounded so a queue of delayed URLs cannot
469
+ // spin forever re-appending to itself.
470
+ if (next.deferrals < MAX_CRAWL_DEFERRALS) {
471
+ queue.push({ ...next, deferrals: next.deferrals + 1 });
472
+ deferred.add(normalized);
473
+ }
474
+ else {
475
+ unfetched.add(normalized);
476
+ }
477
+ continue;
478
+ }
479
+ await sleep(delayMs);
480
+ }
481
+ }
203
482
  visited.add(normalized);
204
- const fetched = await fetchWebsitePage(normalized, { allowPrivateHosts: options.allowPrivateHosts });
483
+ deferred.delete(normalized);
484
+ fetchAttempts++;
485
+ const fetched = await fetchWebsitePage(decision.fetchUrl, {
486
+ allowPrivateHosts: options.allowPrivateHosts,
487
+ robots,
488
+ signal: crawlSignal,
489
+ });
205
490
  if (!fetched)
206
491
  continue;
207
492
  pages.push(fetched.page);
@@ -212,19 +497,53 @@ async function crawlWebsite(startUrl, options) {
212
497
  break;
213
498
  if (link.origin !== allowedOrigin)
214
499
  continue;
215
- const candidate = normalizeCrawlUrl(link.toString());
500
+ const rawLinkUrl = link.toString();
501
+ const candidate = normalizeCrawlUrl(rawLinkUrl);
216
502
  if (!candidate || visited.has(candidate) || isAssetLikePath(link.pathname))
217
503
  continue;
218
- queue.push({ url: candidate, depth: next.depth + 1 });
504
+ queue.push({ url: candidate, rawUrl: rawLinkUrl, depth: next.depth + 1, deferrals: 0 });
219
505
  }
220
506
  }
221
- if (Date.now() > deadline) {
222
- warn("[akm] website crawl stopped at the %ds wall-clock cap with %d/%d pages collected from %s.", WEBSITE_CRAWL_WALL_CLOCK_MS / 1000, pages.length, options.maxPages, startUrl);
507
+ if (!capDisabled && Date.now() > deadline) {
508
+ warn("[akm] website crawl stopped at the %ds wall-clock cap with %d/%d pages collected from %s. " +
509
+ "Raise crawlTimeoutMs on this website source, or set it to 0 to disable the cap.", wallClockCapMs / 1000, pages.length, options.maxPages, startUrl);
510
+ }
511
+ if (capTimer)
512
+ clearTimeout(capTimer);
513
+ // A URL still deferred when the loop ends never got fetched — report it
514
+ // rather than letting a rate-limited origin go missing without a trace.
515
+ for (const url of deferred)
516
+ unfetched.add(url);
517
+ if (unfetched.size > 0) {
518
+ warn("[akm] website crawl: %d URL(s) were not fetched — their origin's Crawl-delay did not fit in the " +
519
+ "remaining time budget. First: %s", unfetched.size, [...unfetched].slice(0, 3).join(", "));
223
520
  }
224
521
  return pages;
225
522
  }
523
+ /**
524
+ * Sentinel thrown by `fetchWebsiteResponse` when a redirect hop's target is
525
+ * disallowed by robots.txt (see the doc comment above the check in
526
+ * `fetchWebsiteResponse`). Never escapes `fetchWebsitePage`, which maps it to
527
+ * `null` — the same "page skipped, no error" outcome as any other
528
+ * robots-disallowed URL. Not exported; purely an internal control-flow
529
+ * signal between the two functions.
530
+ */
531
+ class RobotsDisallowedRedirectError extends Error {
532
+ constructor(url) {
533
+ super(`robots.txt disallows redirect target ${url}`);
534
+ this.name = "RobotsDisallowedRedirectError";
535
+ }
536
+ }
226
537
  async function fetchWebsitePage(pageUrl, options) {
227
- const response = await fetchWebsiteResponse(pageUrl, 0, options);
538
+ let response;
539
+ try {
540
+ response = await fetchWebsiteResponse(pageUrl, 0, options);
541
+ }
542
+ catch (err) {
543
+ if (err instanceof RobotsDisallowedRedirectError)
544
+ return null;
545
+ throw err;
546
+ }
228
547
  if (!response.ok) {
229
548
  if (response.status === 404)
230
549
  return null;
@@ -242,17 +561,31 @@ async function fetchWebsitePage(pageUrl, options) {
242
561
  return null;
243
562
  throw err;
244
563
  }
245
- const finalUrl = normalizeCrawlUrl(response.url || pageUrl) ?? pageUrl;
564
+ const rawFinalUrl = response.url || pageUrl;
565
+ const finalUrl = normalizeCrawlUrl(rawFinalUrl) ?? pageUrl;
246
566
  assertWebsiteRequestUrl(finalUrl, Error, options);
567
+ // Re-check robots.txt against the FINAL (post-redirect) URL, not just the
568
+ // pre-redirect URL crawlWebsite already gated. normalizeCrawlUrl strips
569
+ // trailing slashes before the initial gate, so a rule shaped like
570
+ // `Disallow: /secret/` correctly lets `/secret` through that gate; if the
571
+ // server then redirects to `/secret/` (a common trailing-slash
572
+ // canonicalization), the disallowed page would otherwise be fetched and
573
+ // stored without ever being weighed against robots.txt. See spec §4.6 C-04.
574
+ // Checks `rawFinalUrl` (the un-normalized `response.url`, slash intact) as
575
+ // well as `finalUrl`, per `isCrawlUrlAllowedByRobots` — `normalizeCrawlUrl`
576
+ // would otherwise strip the very trailing slash a `Disallow: /secret/`
577
+ // rule needs to match.
578
+ if (options?.robots && !(await isCrawlUrlAllowedByRobots(options.robots, finalUrl, rawFinalUrl))) {
579
+ warnVerbose("[akm] website crawl: skipping %s (disallowed by robots.txt after redirect)", finalUrl);
580
+ return null;
581
+ }
247
582
  if (contentType.includes("text/html") || contentType.includes("application/xhtml+xml") || looksLikeMarkup(body)) {
248
583
  const title = extractHtmlTitle(body) || new URL(finalUrl).hostname;
584
+ // One parse yields both the content Markdown and the whole-document links.
585
+ const { markdown, links } = htmlToMarkdownAndLinks(body, finalUrl);
249
586
  return {
250
- page: {
251
- url: finalUrl,
252
- title,
253
- markdown: htmlToMarkdown(body, finalUrl),
254
- },
255
- links: extractSameDocumentLinks(body, finalUrl),
587
+ page: { url: finalUrl, title, markdown },
588
+ links,
256
589
  };
257
590
  }
258
591
  return {
@@ -277,6 +610,7 @@ async function fetchWebsiteResponse(pageUrl, redirectCount = 0, options) {
277
610
  "User-Agent": "akm-cli website provider",
278
611
  },
279
612
  redirect: "manual",
613
+ ...(options?.signal ? { signal: options.signal } : {}),
280
614
  }, { timeout: 15_000, retries: 1 });
281
615
  if (response.status >= 300 && response.status < 400) {
282
616
  if (redirectCount >= WEBSITE_MAX_REDIRECTS) {
@@ -288,10 +622,111 @@ async function fetchWebsiteResponse(pageUrl, redirectCount = 0, options) {
288
622
  }
289
623
  const nextUrl = new URL(location, pageUrl).toString();
290
624
  assertWebsiteRequestUrl(nextUrl, Error, options);
625
+ // Robots-check every intermediate redirect hop, not just the pre-redirect
626
+ // queue URL (`crawlWebsite`'s gate) and the FINAL URL (the post-redirect
627
+ // recheck below in `fetchWebsitePage`). Without this, a chain like
628
+ // `/go` -> 302 `/secret/` -> 302 `/public` issues a live GET to
629
+ // `/secret/` even when robots.txt disallows it, because that hop is
630
+ // never the queue URL and never the final URL. Only gated when a
631
+ // `RobotsPolicy` was actually threaded in (`crawlWebsite`); single-URL
632
+ // `fetchWebsiteMarkdownSnapshot` fetches stay deliberately ungated per
633
+ // spec §1.
634
+ if (options?.robots) {
635
+ const normalizedNext = normalizeCrawlUrl(nextUrl);
636
+ if (!normalizedNext) {
637
+ // `normalizeCrawlUrl` returns null for anything that isn't http(s) —
638
+ // a redirect `Location` can legally point at `mailto:`, `tel:`, a
639
+ // bare relative path that resolves to an opaque scheme, etc.
640
+ // `RobotsPolicy.rulesFor` computes `new URL(url).origin` and then
641
+ // resolves `/robots.txt` against it; for a non-http(s) URL that
642
+ // origin is the literal string "null", and re-resolving against it
643
+ // throws an unhandled TypeError that aborts the whole crawl. Refuse
644
+ // the hop outright instead of ever handing such a URL to the policy —
645
+ // `fetchWebsitePage` maps this to a graceful skip. Regression
646
+ // introduced by a67412c, which fell back to the raw `nextUrl` here.
647
+ warnVerbose("[akm] website crawl: skipping redirect to %s (not an http(s) URL)", nextUrl);
648
+ throw new RobotsDisallowedRedirectError(nextUrl);
649
+ }
650
+ if (!(await isCrawlUrlAllowedByRobots(options.robots, normalizedNext, nextUrl))) {
651
+ warnVerbose("[akm] website crawl: skipping redirect to %s (disallowed by robots.txt)", nextUrl);
652
+ throw new RobotsDisallowedRedirectError(nextUrl);
653
+ }
654
+ }
291
655
  return fetchWebsiteResponse(nextUrl, redirectCount + 1, options);
292
656
  }
293
657
  return response;
294
658
  }
659
+ /**
660
+ * Fetches and classifies `<origin>/robots.txt`. Reuses `fetchWebsiteResponse`
661
+ * (spec §6.2: no second fetch path) so robots.txt gets the exact same SSRF
662
+ * guards, retry, and redirect handling as a page fetch.
663
+ *
664
+ * Steps 1–2 (the guard on the INITIAL URL) run OUTSIDE the try/catch on
665
+ * purpose: a guard rejection there must propagate as-is, never be downgraded
666
+ * to "unavailable" (spec §4.5 F-12, §6.2). Guard rejections on a LATER
667
+ * redirect hop happen inside `fetchWebsiteResponse`, which the try/catch
668
+ * below does cover — the guard has already refused to fetch that host, so
669
+ * only the error *reporting* is downgraded (F-11).
670
+ */
671
+ export async function loadRobotsTxt(robotsUrl, options) {
672
+ assertWebsiteRequestUrl(robotsUrl, UsageError, options);
673
+ await assertResolvedHostAllowed(new URL(robotsUrl).hostname, options);
674
+ try {
675
+ const response = await fetchWebsiteResponse(robotsUrl, 0, options);
676
+ if (response.status >= 200 && response.status < 300) {
677
+ try {
678
+ const text = await readBodyWithByteCap(response, ROBOTS_BYTE_CAP, {
679
+ bodyTimeoutMs: ROBOTS_BODY_TIMEOUT_MS,
680
+ });
681
+ return { kind: "body", text };
682
+ }
683
+ catch (err) {
684
+ if (err instanceof ResponseTooLargeError) {
685
+ warn("[akm] robots.txt at %s exceeded the %d-byte cap; treating it as unavailable.", robotsUrl, ROBOTS_BYTE_CAP);
686
+ return { kind: "unavailable" };
687
+ }
688
+ throw err;
689
+ }
690
+ }
691
+ if (response.status >= 500 && response.status < 600) {
692
+ // RFC 9309 §2.3.1.4: an unreachable robots.txt is a full disallow, not
693
+ // an allow-all. `fetchWithRetry` already retried this once, so a
694
+ // transient blip does not trip it.
695
+ warn("[akm] robots.txt at %s returned %d; treating the crawl as fully disallowed until it recovers. " +
696
+ "Set respectRobots: false on this website source to bypass robots.txt.", robotsUrl, response.status);
697
+ return { kind: "unreachable" };
698
+ }
699
+ // 4xx (404 is the common, silent case), and any other non-2xx/5xx status.
700
+ return { kind: "unavailable" };
701
+ }
702
+ catch (err) {
703
+ warnVerbose("[akm] failed to fetch robots.txt at %s: %s", robotsUrl, err instanceof Error ? err.message : String(err));
704
+ return { kind: "unavailable" };
705
+ }
706
+ }
707
+ /**
708
+ * Coerces `SourceConfigEntry.options.respectRobots` to a boolean. The bundle
709
+ * descriptor is boolean-validated at config load (schema), but the legacy
710
+ * `sources[].options` bag is `z.record(z.unknown())` and accepts anything, so
711
+ * the runtime read still validates. A misspelled non-boolean opt-out fails
712
+ * loudly (`ConfigError`) rather than silently defaulting either way — the
713
+ * user would otherwise think robots.txt handling is something other than
714
+ * what akm is actually doing (spec §4.7).
715
+ */
716
+ export function coerceRespectRobots(value) {
717
+ if (value === undefined || value === null)
718
+ return true;
719
+ if (typeof value === "boolean")
720
+ return value;
721
+ if (typeof value === "string") {
722
+ const normalized = value.trim().toLowerCase();
723
+ if (normalized === "true")
724
+ return true;
725
+ if (normalized === "false")
726
+ return false;
727
+ }
728
+ throw new ConfigError(`Invalid value for respectRobots: expected a boolean (or "true"/"false"), got ${JSON.stringify(value)}.`);
729
+ }
295
730
  function buildMarkdownSnapshot(page, slug, tags) {
296
731
  const title = sanitizeString(page.title, 200) || slug;
297
732
  const host = sanitizeString(new URL(page.url).hostname, 120);
@@ -439,92 +874,9 @@ function coercePositiveInt(value, fallback) {
439
874
  function looksLikeMarkup(body) {
440
875
  return /<html[\s>]|<body[\s>]|<\/[a-z][\w:-]*>/i.test(body);
441
876
  }
442
- function extractHtmlTitle(html) {
443
- const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
444
- if (title)
445
- return decodeHtmlEntities(stripTags(title)).trim();
446
- const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1];
447
- if (h1)
448
- return decodeHtmlEntities(stripTags(h1)).trim();
449
- return undefined;
450
- }
451
- function extractTextTitle(text) {
452
- for (const line of text.split(/\r?\n/)) {
453
- const trimmed = line.trim();
454
- if (!trimmed)
455
- continue;
456
- if (trimmed.startsWith("#"))
457
- return trimmed.replace(/^#+\s*/, "");
458
- return trimmed.slice(0, 120);
459
- }
460
- return undefined;
461
- }
462
- function extractSameDocumentLinks(html, pageUrl) {
463
- const links = [];
464
- const hrefPattern = /<a\b[^>]*href\s*=\s*(['"])(.*?)\1[^>]*>/gi;
465
- for (const match of html.matchAll(hrefPattern)) {
466
- const href = match[2]?.trim();
467
- if (!href || href.startsWith("#"))
468
- continue;
469
- try {
470
- const resolved = new URL(href, pageUrl);
471
- if (!isSafeLinkUrl(resolved))
472
- continue;
473
- links.push(resolved);
474
- }
475
- catch {
476
- /* ignore malformed links */
477
- }
478
- }
479
- return links;
480
- }
481
- function htmlToMarkdown(html, pageUrl) {
482
- let text = html;
483
- text = stripDangerousBlockTag(text, "script");
484
- text = stripDangerousBlockTag(text, "style");
485
- text = stripDangerousBlockTag(text, "noscript");
486
- text = stripDangerousBlockTag(text, "template");
487
- text = text.replace(/<pre\b[^>]*><code\b[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_match, code) => {
488
- const decoded = decodeHtmlEntities(stripTags(code)).trim();
489
- return decoded ? `\n\n\`\`\`\n${decoded}\n\`\`\`\n\n` : "\n\n";
490
- });
491
- text = text.replace(/<code\b[^>]*>([\s\S]*?)<\/code>/gi, (_match, code) => {
492
- const decoded = decodeHtmlEntities(stripTags(code)).trim();
493
- return decoded ? `\`${decoded}\`` : "";
494
- });
495
- text = text.replace(/<a\b[^>]*href\s*=\s*(['"])(.*?)\1[^>]*>([\s\S]*?)<\/a>/gi, (_match, _q, href, body) => {
496
- const label = decodeHtmlEntities(stripTags(body)).trim();
497
- if (!label)
498
- return "";
499
- try {
500
- const resolved = new URL(href, pageUrl);
501
- if (!isSafeLinkUrl(resolved))
502
- return label;
503
- return `[${label}](${resolved})`;
504
- }
505
- catch {
506
- return label;
507
- }
508
- });
509
- text = text.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_match, level, body) => {
510
- const heading = decodeHtmlEntities(stripTags(body)).trim();
511
- return heading ? `\n\n${"#".repeat(Number(level))} ${heading}\n\n` : "\n\n";
512
- });
513
- text = text.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_match, body) => {
514
- const item = decodeHtmlEntities(stripTags(body)).trim();
515
- return item ? `\n- ${item}` : "";
516
- });
517
- text = text.replace(/<(p|div|section|article|main|header|footer|blockquote|table|tr)\b[^>]*>/gi, "\n\n");
518
- text = text.replace(/<\/(p|div|section|article|main|header|footer|blockquote|table|tr)>/gi, "\n\n");
519
- text = text.replace(/<br\s*\/?>/gi, "\n");
520
- text = text.replace(/<\/?(ul|ol)\b[^>]*>/gi, "\n");
521
- text = decodeHtmlEntities(stripTags(text));
522
- text = text
523
- .replace(/\r/g, "")
524
- .replace(/[ \t]+\n/g, "\n")
525
- .replace(/\n{3,}/g, "\n\n")
526
- .trim();
527
- return text;
877
+ /** True for URL paths that are plainly binary assets, never crawlable pages. */
878
+ function isAssetLikePath(pathname) {
879
+ return /\.(css|js|json|png|jpe?g|gif|svg|ico|webp|pdf|zip|tar|gz|mp4|mp3|woff2?)$/i.test(pathname);
528
880
  }
529
881
  function stripTags(value) {
530
882
  return value.replace(/<[^>]+>/g, " ");
@@ -549,148 +901,25 @@ function decodeHtmlEntities(value) {
549
901
  return namedEntities[normalized] ?? match;
550
902
  });
551
903
  }
552
- function isAssetLikePath(pathname) {
553
- return /\.(css|js|json|png|jpe?g|gif|svg|ico|webp|pdf|zip|tar|gz|mp4|mp3|woff2?)$/i.test(pathname);
554
- }
555
- function isSafeLinkUrl(url) {
556
- return url.protocol === "http:" || url.protocol === "https:";
557
- }
558
- function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
559
- const parsedUrl = new URL(rawUrl);
560
- const hostname = parsedUrl.hostname.toLowerCase();
561
- if (hostname.endsWith(".invalid")) {
562
- throw new ErrorType(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
563
- }
564
- if (isForbiddenWebsiteHostname(hostname, options)) {
565
- throw new ErrorType(`Refusing to fetch non-public website host: ${parsedUrl.hostname}`);
566
- }
567
- }
568
- async function defaultResolveHostname(hostname) {
569
- const records = await dnsLookup(hostname, { all: true });
570
- return records.map((record) => record.address);
571
- }
572
- /**
573
- * Resolve-then-validate SSRF guard against DNS rebinding / private-range
574
- * bypasses. {@link assertWebsiteRequestUrl} only rejects IP-literal and
575
- * well-known-name hosts; a hostname like `private-host.example.com` that
576
- * resolves to `10.0.0.1` passes those checks and then fetch connects to the
577
- * private address. Here we resolve EVERY A/AAAA record and validate each
578
- * against the same forbidden-range rules, failing CLOSED on an empty answer or
579
- * resolver error.
580
- *
581
- * TOCTOU residual (documented, not fully closable here): Bun's `fetch` exposes
582
- * no custom `lookup`/agent hook, so we cannot pin the socket to the exact IP we
583
- * validated — a hostile resolver could return a public IP to this lookup and a
584
- * private IP microseconds later at connect time (classic rebinding). This still
585
- * removes the TRIVIAL `hostname A 10.0.0.1` bypass, which is the strongest
586
- * guarantee available without a pinned-connection fetch API. Re-run on every
587
- * redirect hop (the crawler recurses through `fetchWebsiteResponse`).
588
- */
589
- export async function assertResolvedHostAllowed(hostname, options) {
590
- if (options?.allowPrivateHosts === true)
591
- return;
592
- const bare = stripIpv6Brackets(hostname.toLowerCase());
593
- // IP-literal hosts are already fully validated by assertWebsiteRequestUrl's
594
- // range checks; resolving them is a no-op (and dnsLookup would just echo it).
595
- if (isIP(bare) !== 0)
596
- return;
597
- const resolve = options?.resolveHostname ?? defaultResolveHostname;
598
- let addresses;
599
- try {
600
- addresses = await resolve(bare);
601
- }
602
- catch {
603
- throw new Error(`Refusing to fetch ${hostname}: DNS resolution failed`);
604
- }
605
- if (addresses.length === 0) {
606
- throw new Error(`Refusing to fetch ${hostname}: hostname resolved to no addresses`);
607
- }
608
- for (const address of addresses) {
609
- const version = isIP(address);
610
- const forbidden = version === 4 ? isForbiddenIpv4(address) : version === 6 ? isForbiddenIpv6(stripIpv6Brackets(address)) : true;
611
- if (forbidden) {
612
- throw new Error(`Refusing to fetch ${hostname}: resolves to non-public or unparseable address ${address}`);
613
- }
614
- }
615
- }
616
- // WHATWG URL.hostname wraps IPv6 literals in brackets (e.g. "[::1]"), but
617
- // node:net's isIP() only recognizes the bare address form and returns 0 for
618
- // anything bracketed — silently skipping all IPv6 forbidden-host checks
619
- // below for every hostname parsed off a URL. Strip the brackets before any
620
- // isIP()/isForbiddenIpv6() call so those checks actually run.
621
- function stripIpv6Brackets(hostname) {
622
- return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
904
+ function extractHtmlTitle(html) {
905
+ const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
906
+ if (title)
907
+ return decodeHtmlEntities(stripTags(title)).trim();
908
+ const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1];
909
+ if (h1)
910
+ return decodeHtmlEntities(stripTags(h1)).trim();
911
+ return undefined;
623
912
  }
624
- function isForbiddenWebsiteHostname(hostname, options) {
625
- if (options?.allowPrivateHosts === true)
626
- return false;
627
- if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal") {
628
- return true;
913
+ function extractTextTitle(text) {
914
+ for (const line of text.split(/\r?\n/)) {
915
+ const trimmed = line.trim();
916
+ if (!trimmed)
917
+ continue;
918
+ if (trimmed.startsWith("#"))
919
+ return trimmed.replace(/^#+\s*/, "");
920
+ return trimmed.slice(0, 120);
629
921
  }
630
- const bareHostname = stripIpv6Brackets(hostname);
631
- const ipVersion = isIP(bareHostname);
632
- if (ipVersion === 4)
633
- return isForbiddenIpv4(bareHostname);
634
- if (ipVersion === 6)
635
- return isForbiddenIpv6(bareHostname);
636
- return false;
637
- }
638
- function isLoopbackWebsiteHostname(hostname) {
639
- if (hostname === "localhost" || hostname.endsWith(".localhost"))
640
- return true;
641
- const bareHostname = stripIpv6Brackets(hostname);
642
- const ipVersion = isIP(bareHostname);
643
- if (ipVersion === 4)
644
- return bareHostname.startsWith("127.");
645
- if (ipVersion === 6)
646
- return bareHostname === "::1";
647
- return false;
648
- }
649
- function isForbiddenIpv4(hostname) {
650
- const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
651
- if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
652
- return true;
653
- const a = parts[0];
654
- const b = parts[1];
655
- return (a === 0 ||
656
- a === 10 ||
657
- a === 127 ||
658
- (a === 169 && b === 254) ||
659
- (a === 172 && b >= 16 && b <= 31) ||
660
- (a === 192 && b === 168));
661
- }
662
- /**
663
- * Extracts the embedded IPv4 address from an IPv4-mapped IPv6 literal
664
- * (`::ffff:a.b.c.d` or its canonical hex form `::ffff:xxxx:yyyy`), or
665
- * returns null if `hostname` isn't one.
666
- */
667
- function extractIpv4MappedAddress(normalizedHostname) {
668
- const match = normalizedHostname.match(/^::ffff:(?:(\d{1,3}(?:\.\d{1,3}){3})|([0-9a-f]{1,4}):([0-9a-f]{1,4}))$/);
669
- if (!match)
670
- return null;
671
- if (match[1])
672
- return match[1];
673
- const high = Number.parseInt(match[2], 16);
674
- const low = Number.parseInt(match[3], 16);
675
- return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
676
- }
677
- function isForbiddenIpv6(hostname) {
678
- const normalized = hostname.toLowerCase();
679
- const mappedIpv4 = extractIpv4MappedAddress(normalized);
680
- if (mappedIpv4)
681
- return isForbiddenIpv4(mappedIpv4);
682
- return (normalized === "::" ||
683
- normalized === "::1" ||
684
- normalized.startsWith("fc") ||
685
- normalized.startsWith("fd") ||
686
- normalized.startsWith("fe8") ||
687
- normalized.startsWith("fe9") ||
688
- normalized.startsWith("fea") ||
689
- normalized.startsWith("feb"));
690
- }
691
- function stripDangerousBlockTag(value, tagName) {
692
- const pattern = new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}\\s*>`, "gi");
693
- return value.replace(pattern, "");
922
+ return undefined;
694
923
  }
695
924
  function safeCodePointToString(value) {
696
925
  if (!Number.isFinite(value) || value < 0 || value > 0x10ffff)
@@ -702,3 +931,5 @@ function safeCodePointToString(value) {
702
931
  return undefined;
703
932
  }
704
933
  }
934
+ // Re-exported for existing importers (the SSRF suite pins these entry points).
935
+ export { assertResolvedHostAllowed } from "./host-guard.js";