pagetrace 0.11.0 → 0.12.0

package/CHANGELOG.md

@@ -6,6 +6,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 While the version is below 1.0.0, breaking changes ship in a minor release.
 
+## [0.12.0] - 2026-09-08
+
+### Added
+
+- `--external` checks links that leave the site, on `links`, `audit`, `check`
+  and `snapshot`. Only 404 and 410 count as dead: a 403 from a bot wall, a 429,
+  a timeout and a TLS failure all describe the request rather than the page.
+  `HEAD` first with a `GET` fallback, one request per unique URL across the
+  site, capped at 200. Off by default — it fans out to hosts you do not control,
+  and recording their availability in a lockfile turns a third party's bad
+  afternoon into a diff in your repository.
+- `link.external.dead` (warn) and `link.external.dead.added`. A warning rather
+  than an error: the page belongs to someone who can delete it without asking.
+
+### Fixed
+
+- Absolute internal links were invisible. `extractLinks` dropped every href with
+  a scheme, so a site writing `https://example.com/about` rather than `/about`
+  had its links checked not at all. They are now resolved against the site's own
+  origin like any other link.
+
 ## [0.11.0] - 2026-09-08
 
 ### Added
@@ -338,6 +359,7 @@ Initial release. `snapshot`, `check` and `audit` commands; filesystem and HTTP
 crawling; diff classified by transition; absolute, cross-page and hreflang audit
 rules; pretty, JSON, markdown, GitHub and HTML reporters.
 
+[0.12.0]: https://github.com/shyamexe/pagetrace/compare/v0.11.0...v0.12.0
 [0.11.0]: https://github.com/shyamexe/pagetrace/compare/v0.10.0...v0.11.0
 [0.10.0]: https://github.com/shyamexe/pagetrace/compare/v0.9.1...v0.10.0
 [0.9.1]: https://github.com/shyamexe/pagetrace/compare/v0.9.0...v0.9.1

package/README.md

@@ -160,6 +160,7 @@ pagetrace update              # install it globally
 | `redirect.changed` | warn | A route redirects somewhere new |
 | `canonical.redirects` | warn | A canonical points at a URL that redirects |
 | `link.broken.added` | error | A page started linking to a URL that does not exist |
+| `link.external.dead` | warn | An outbound link answers 404 (needs `--external`) |
 | `sitemap.dead` | error | The sitemap lists a URL that answers 404 |
 | `sitemap.redirect` | warn | The sitemap lists a URL that redirects |
 | `og.removed` / `hreflang.removed` | warn | Social or i18n tags dropped |
@@ -174,7 +175,17 @@ npx pagetrace links --dir ./out --format json
 
 It crawls once, runs the same rules as `audit`, and prints only the link findings — exit 1 if any, or `No broken links found — 42 pages checked.`
 
-Internal links are checked too. Only the broken ones are stored, so a site's navigation never lands in the lockfile: `link.broken` for a link that is already dead, `link.broken.added` for one this build broke. A `--dir` crawl is authoritative — the build directory is the whole site — while a crawl confirms each candidate with a real request first, because a sitemap routinely omits pages that are live. External links are deliberately not checked: a Cloudflare 403 and a rate limit both look like a dead page, and that is where link checkers earn their reputation for false positives.
+Internal links are checked always. Outbound links are checked on request:
+
+```bash
+npx pagetrace links --url https://example.com --external
+```
+
+Only `404` and `410` count as dead. A `403` from a bot wall, a `429`, a timeout or a TLS failure are all reported as nothing at all, because they describe the request rather than the page — which is how link checkers turn into noise nobody reads. `HEAD` first, falling back to `GET` for servers that refuse it, one request per unique URL across the whole site, capped at 200.
+
+Think twice before putting `--external` in `check`. A third party's bad afternoon becomes a diff in your repository and a red build you cannot fix.
+
+Internal links are checked too. Only the broken ones are stored, so a site's navigation never lands in the lockfile: `link.broken` for a link that is already dead, `link.broken.added` for one this build broke. A `--dir` crawl is authoritative — the build directory is the whole site — while a crawl confirms each candidate with a real request first, because a sitemap routinely omits pages that are live. External links are checked only with `--external`, and only a 404 or 410 counts.
 
 Redirects are recorded from the response itself, so they cost no extra requests. A redirect that only adds or drops a trailing slash is server configuration rather than drift and is not reported. `canonical.redirects` is only raised when the canonical's target was actually crawled, so a `--limit` run cannot invent it.
 

package/dist/cli.cjs

@@ -254,13 +254,24 @@ function auditPage(page, config = {}) {
   return findings;
 }
 function auditLinks(page) {
-  return (page.brokenLinks ?? []).map((target) => ({
-    code: "link.broken",
-    severity: "error",
-    route: page.route,
-    message: `Links to ${target}, which does not exist.`,
-    after: target
-  }));
+  return [
+    ...(page.brokenLinks ?? []).map((target) => ({
+      code: "link.broken",
+      severity: "error",
+      route: page.route,
+      message: `Links to ${target}, which does not exist.`,
+      after: target
+    })),
+    // A warning, not an error: the page it points at belongs to somebody else,
+    // who can delete it on a Tuesday without asking you.
+    ...(page.deadExternal ?? []).map((target) => ({
+      code: "link.external.dead",
+      severity: "warn",
+      route: page.route,
+      message: `Links out to ${target}, which answers 404.`,
+      after: target
+    }))
+  ];
 }
 function auditSite(snapshot) {
   const findings = [];
@@ -631,6 +642,12 @@ function diffPage(before, after) {
     push("link.broken.removed", "info", `Link to ${target} was fixed or removed.`, {
       before: target
     });
+  const deadBefore = before.deadExternal ?? [];
+  const deadAfter = after.deadExternal ?? [];
+  for (const target of deadAfter.filter((t) => !deadBefore.includes(t)))
+    push("link.external.dead.added", "warn", `Links out to ${target}, which answers 404.`, {
+      after: target
+    });
   const droppedHreflang = Object.keys(before.hreflang).filter((k) => !(k in after.hreflang));
   if (droppedHreflang.length > 0)
     push("hreflang.removed", "warn", "hreflang alternates were removed.", {
@@ -739,6 +756,10 @@ var GUIDANCE = {
       nextjs: "A `<Link href>` pointing at a route that no longer exists. TypeScript will not catch it \u2014 typed routes are opt-in via `experimental.typedRoutes`."
     }
   },
+  "link.external.dead": {
+    why: "An outbound link to a page that is gone sends readers to a 404 and spends the trust the link was passing on nothing. Unlike an internal link, the page is not yours to restore.",
+    fix: "Point the link at the current URL, at an archived copy, or remove it. A link that has been dead a while is usually a citation worth replacing rather than deleting."
+  },
   "sitemap.dead": {
     why: "A sitemap is a list of URLs you are asking to have crawled. Entries that 404 spend crawl budget on nothing and lower the trust placed in the rest of the file.",
     fix: "Remove the URL from the sitemap, or restore the page. If it moved, redirect it and list the destination instead.",
@@ -1520,8 +1541,8 @@ function extractLinks(html) {
   const hrefs = /* @__PURE__ */ new Set();
   for (const anchor of root.querySelectorAll("a[href]")) {
     const href = anchor.getAttribute("href")?.trim();
-    if (!href) continue;
-    if (href.startsWith("#") || href.startsWith("//") || /^[a-z][a-z0-9+.-]*:/i.test(href)) continue;
+    if (!href || href.startsWith("#")) continue;
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href) && !/^https?:/i.test(href)) continue;
     hrefs.add(href);
   }
   return [...hrefs];
@@ -1615,6 +1636,7 @@ async function walkHtml(dir, acc = []) {
   }
   return acc;
 }
+var USER_AGENT = "pagetrace (+https://npmjs.com/package/pagetrace)";
 var DEFAULT_TIMEOUT_MS = 15e3;
 var MAX_ATTEMPTS = 3;
 var RETRY_BASE_MS = 300;
@@ -1630,7 +1652,7 @@ async function fetchDoc(url, timeoutMs = DEFAULT_TIMEOUT_MS) {
     try {
       response = await fetch(url, {
         signal: AbortSignal.timeout(timeoutMs),
-        headers: { "user-agent": "pagetrace (+https://npmjs.com/package/pagetrace)" }
+        headers: { "user-agent": USER_AGENT }
       });
     } catch (cause) {
       lastError = new Error(`Could not reach ${url}: ${cause.message}`, { cause });
@@ -1675,7 +1697,9 @@ async function snapshotFromDir(dir, config = {}) {
   if (robots !== null) site.robotsTxt = extractRobotsTxt(robots, agents);
   const llms = await (0, import_promises.readFile)((0, import_node_path.join)(dir, "llms.txt"), "utf8").catch(() => null);
   if (llms !== null) site.llmsTxt = extractLlmsTxt(llms);
-  await resolveBrokenLinks(pages, links, site.origin ?? "https://pagetrace.invalid");
+  const origin = site.origin ?? "https://pagetrace.invalid";
+  await resolveBrokenLinks(pages, links, origin);
+  if (config.checkExternal) await resolveDeadExternal(pages, links, origin, 5);
   return { schemaVersion: 1, createdAt: (/* @__PURE__ */ new Date()).toISOString(), site, pages };
 }
 var ASSET_PATH = /\.(?!html?$)[a-z0-9]+$/i;
@@ -1714,6 +1738,63 @@ async function resolveBrokenLinks(pages, links, origin, verify) {
     if (confirmed.length > 0) pages[route].brokenLinks = confirmed.sort();
   }
 }
+function externalUrl(href, from, origin) {
+  try {
+    const url = new URL(href, `${origin}${from === "/" ? "" : from}/`);
+    if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+    if (url.origin === origin) return null;
+    url.hash = "";
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+var MAX_EXTERNAL_CHECKS = 200;
+async function findDeadExternal(urls, concurrency, timeoutMs = DEFAULT_TIMEOUT_MS) {
+  const dead = /* @__PURE__ */ new Set();
+  const queue = urls.slice(0, MAX_EXTERNAL_CHECKS);
+  const request = (url, method) => fetch(url, {
+    method,
+    redirect: "follow",
+    signal: AbortSignal.timeout(timeoutMs),
+    headers: { "user-agent": USER_AGENT }
+  });
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+      while (queue.length > 0) {
+        const url = queue.shift();
+        try {
+          let response = await request(url, "HEAD");
+          if (response.status === 405 || response.status === 501) {
+            response = await request(url, "GET");
+          }
+          if (response.status === 404 || response.status === 410) dead.add(url);
+        } catch {
+        }
+      }
+    })
+  );
+  return dead;
+}
+async function resolveDeadExternal(pages, links, origin, concurrency, timeoutMs) {
+  const perPage = /* @__PURE__ */ new Map();
+  const unique = /* @__PURE__ */ new Set();
+  for (const [route, hrefs] of Object.entries(links)) {
+    const external = [
+      ...new Set(
+        hrefs.map((href) => externalUrl(href, route, origin)).filter((url) => url !== null)
+      )
+    ];
+    if (external.length === 0) continue;
+    perPage.set(route, external);
+    for (const url of external) unique.add(url);
+  }
+  const dead = await findDeadExternal([...unique], concurrency, timeoutMs);
+  for (const [route, external] of perPage) {
+    const found = external.filter((url) => dead.has(url)).sort();
+    if (found.length > 0) pages[route].deadExternal = found;
+  }
+}
 var MAX_NESTED_SITEMAPS = 50;
 async function snapshotFromOrigin(origin, options = {}) {
   const base = new URL(origin);
@@ -1802,6 +1883,9 @@ async function snapshotFromOrigin(origin, options = {}) {
       return false;
     }
   });
+  if (options.checkExternal) {
+    await resolveDeadExternal(pages, links, base.origin, concurrency, timeout);
+  }
   return { schemaVersion: 1, createdAt: (/* @__PURE__ */ new Date()).toISOString(), site, pages };
 }
 
@@ -1854,13 +1938,15 @@ async function loadConfig(path = DEFAULT_CONFIG) {
   }
 }
 async function build(flags, config) {
-  if (flags.dir) return snapshotFromDir(flags.dir, config);
+  const checkExternal = flags.external ?? config.checkExternal;
+  if (flags.dir) return snapshotFromDir(flags.dir, { ...config, checkExternal });
   if (flags.url)
     return snapshotFromOrigin(flags.url, {
       ...config,
       limit: flags.limit,
       concurrency: flags.concurrency,
-      ignoreRobots: flags.ignoreRobots ?? config.ignoreRobots
+      ignoreRobots: flags.ignoreRobots ?? config.ignoreRobots,
+      checkExternal
     });
   throw new Error("Provide a source: --dir <build directory> or --url <origin>.");
 }
@@ -1879,7 +1965,7 @@ function render(findings, format) {
   }
 }
 var cli = (0, import_cac.cac)("pagetrace");
-cli.command("snapshot", "Record the current SEO/AEO surface to a lockfile").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--out <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).action(async (flags) => {
+cli.command("snapshot", "Record the current SEO/AEO surface to a lockfile").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--out <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).action(async (flags) => {
   const config = await loadConfig(flags.config);
   const snapshot = await build(flags, config);
   const written = await writeLockfile(flags.out, snapshot);
@@ -1888,7 +1974,7 @@ cli.command("snapshot", "Record the current SEO/AEO surface to a lockfile").opti
     written ? import_picocolors2.default.green(`Wrote ${flags.out} \u2014 ${count} page${count === 1 ? "" : "s"}.`) : import_picocolors2.default.dim(`${flags.out} is already up to date \u2014 ${count} page${count === 1 ? "" : "s"}.`)
   );
 });
-cli.command("check", "Compare the current surface against the lockfile").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--lockfile <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown | github | sarif", { default: "pretty" }).option("--fail-on <severity>", "error | warn | info", { default: "error" }).option("--audit", "Also run absolute rules, not just the diff", { default: true }).option("--update", "Write the new state to the lockfile after reporting").option("--baseline-branch <ref>", "Read the baseline lockfile from a git ref instead of disk").action(async (flags) => {
+cli.command("check", "Compare the current surface against the lockfile").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--lockfile <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown | github | sarif", { default: "pretty" }).option("--fail-on <severity>", "error | warn | info", { default: "error" }).option("--audit", "Also run absolute rules, not just the diff", { default: true }).option("--update", "Write the new state to the lockfile after reporting").option("--baseline-branch <ref>", "Read the baseline lockfile from a git ref instead of disk").action(async (flags) => {
   const failOn = parseFailOn(flags.failOn, false);
   const config = await loadConfig(flags.config);
   const next = await build(flags, config);
@@ -1918,7 +2004,7 @@ Failing: ${summary.error} error, ${summary.warn} warning (--fail-on ${failOn}).`
     process.exitCode = EXIT_FINDINGS;
   }
 });
-cli.command("audit", "Audit a site as it stands, with explanations and fixes").option("--url <origin>", "Live origin to crawl").option("--dir <dir>", "Directory of built HTML").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown | html", { default: "pretty" }).option("--out <file>", "Write the report to a file instead of stdout").option("--fail-on <severity>", "error | warn | info | never", { default: "never" }).action(async (flags) => {
+cli.command("audit", "Audit a site as it stands, with explanations and fixes").option("--url <origin>", "Live origin to crawl").option("--dir <dir>", "Directory of built HTML").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown | html", { default: "pretty" }).option("--out <file>", "Write the report to a file instead of stdout").option("--fail-on <severity>", "error | warn | info | never", { default: "never" }).action(async (flags) => {
   const failOn = parseFailOn(flags.failOn, true);
   const config = await loadConfig(flags.config);
   const snapshot = await build(flags, config);
@@ -1958,7 +2044,7 @@ cli.command("audit", "Audit a site as it stands, with explanations and fixes").o
     process.exitCode = EXIT_FINDINGS;
   }
 });
-cli.command("links", "Find internal links that point at no page").option("--url <origin>", "Live origin to crawl").option("--dir <dir>", "Directory of built HTML").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown", { default: "pretty" }).option("--out <file>", "Write the report to a file instead of stdout").option("--fail-on <severity>", "error | warn | info | never", { default: "error" }).action(async (flags) => {
+cli.command("links", "Find internal links that point at no page").option("--url <origin>", "Live origin to crawl").option("--dir <dir>", "Directory of built HTML").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown", { default: "pretty" }).option("--out <file>", "Write the report to a file instead of stdout").option("--fail-on <severity>", "error | warn | info | never", { default: "error" }).action(async (flags) => {
   const failOn = parseFailOn(flags.failOn, true);
   const config = await loadConfig(flags.config);
   const snapshot = await build(flags, config);
@@ -2005,7 +2091,7 @@ cli.command("links", "Find internal links that point at no page").option("--url
     process.exitCode = EXIT_FINDINGS;
   }
 });
-cli.command("init", "Write a config file and take the first snapshot").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--out <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).action(async (flags) => {
+cli.command("init", "Write a config file and take the first snapshot").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--out <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).action(async (flags) => {
   const existing = await (0, import_promises2.readFile)(flags.config, "utf8").catch(() => null);
   if (existing === null) {
     const config = flags.url ? { siteUrl: new URL(flags.url).origin } : {};
@@ -2028,11 +2114,11 @@ Commit both files, then run \`pagetrace check ${flags.dir ? `--dir ${flags.dir}`
 });
 cli.command("update", "Check npm for a newer pagetrace and install it").option("--check", "Only report whether an update exists").action(async (flags) => {
   const latest = await latestVersion("pagetrace");
-  if (!isNewer(latest, "0.11.0")) {
-    console.log(import_picocolors2.default.green(`pagetrace ${"0.11.0"} is the latest version.`));
+  if (!isNewer(latest, "0.12.0")) {
+    console.log(import_picocolors2.default.green(`pagetrace ${"0.12.0"} is the latest version.`));
     return;
   }
-  console.log(import_picocolors2.default.yellow(`Update available: ${"0.11.0"} \u2192 ${latest}`));
+  console.log(import_picocolors2.default.yellow(`Update available: ${"0.12.0"} \u2192 ${latest}`));
   if (flags.check) return;
   if (process.argv[1]?.startsWith(process.cwd())) {
     console.log(
@@ -2050,7 +2136,7 @@ cli.command("update", "Check npm for a newer pagetrace and install it").option("
   console.log(import_picocolors2.default.green(`Updated to pagetrace ${latest}.`));
 });
 cli.help();
-cli.version("0.11.0");
+cli.version("0.12.0");
 async function main() {
   try {
     cli.parse(process.argv, { run: false });

package/dist/cli.js

@@ -231,13 +231,24 @@ function auditPage(page, config = {}) {
   return findings;
 }
 function auditLinks(page) {
-  return (page.brokenLinks ?? []).map((target) => ({
-    code: "link.broken",
-    severity: "error",
-    route: page.route,
-    message: `Links to ${target}, which does not exist.`,
-    after: target
-  }));
+  return [
+    ...(page.brokenLinks ?? []).map((target) => ({
+      code: "link.broken",
+      severity: "error",
+      route: page.route,
+      message: `Links to ${target}, which does not exist.`,
+      after: target
+    })),
+    // A warning, not an error: the page it points at belongs to somebody else,
+    // who can delete it on a Tuesday without asking you.
+    ...(page.deadExternal ?? []).map((target) => ({
+      code: "link.external.dead",
+      severity: "warn",
+      route: page.route,
+      message: `Links out to ${target}, which answers 404.`,
+      after: target
+    }))
+  ];
 }
 function auditSite(snapshot) {
   const findings = [];
@@ -608,6 +619,12 @@ function diffPage(before, after) {
     push("link.broken.removed", "info", `Link to ${target} was fixed or removed.`, {
       before: target
     });
+  const deadBefore = before.deadExternal ?? [];
+  const deadAfter = after.deadExternal ?? [];
+  for (const target of deadAfter.filter((t) => !deadBefore.includes(t)))
+    push("link.external.dead.added", "warn", `Links out to ${target}, which answers 404.`, {
+      after: target
+    });
   const droppedHreflang = Object.keys(before.hreflang).filter((k) => !(k in after.hreflang));
   if (droppedHreflang.length > 0)
     push("hreflang.removed", "warn", "hreflang alternates were removed.", {
@@ -716,6 +733,10 @@ var GUIDANCE = {
       nextjs: "A `<Link href>` pointing at a route that no longer exists. TypeScript will not catch it \u2014 typed routes are opt-in via `experimental.typedRoutes`."
     }
   },
+  "link.external.dead": {
+    why: "An outbound link to a page that is gone sends readers to a 404 and spends the trust the link was passing on nothing. Unlike an internal link, the page is not yours to restore.",
+    fix: "Point the link at the current URL, at an archived copy, or remove it. A link that has been dead a while is usually a citation worth replacing rather than deleting."
+  },
   "sitemap.dead": {
     why: "A sitemap is a list of URLs you are asking to have crawled. Entries that 404 spend crawl budget on nothing and lower the trust placed in the rest of the file.",
     fix: "Remove the URL from the sitemap, or restore the page. If it moved, redirect it and list the destination instead.",
@@ -1497,8 +1518,8 @@ function extractLinks(html) {
   const hrefs = /* @__PURE__ */ new Set();
   for (const anchor of root.querySelectorAll("a[href]")) {
     const href = anchor.getAttribute("href")?.trim();
-    if (!href) continue;
-    if (href.startsWith("#") || href.startsWith("//") || /^[a-z][a-z0-9+.-]*:/i.test(href)) continue;
+    if (!href || href.startsWith("#")) continue;
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href) && !/^https?:/i.test(href)) continue;
     hrefs.add(href);
   }
   return [...hrefs];
@@ -1592,6 +1613,7 @@ async function walkHtml(dir, acc = []) {
   }
   return acc;
 }
+var USER_AGENT = "pagetrace (+https://npmjs.com/package/pagetrace)";
 var DEFAULT_TIMEOUT_MS = 15e3;
 var MAX_ATTEMPTS = 3;
 var RETRY_BASE_MS = 300;
@@ -1607,7 +1629,7 @@ async function fetchDoc(url, timeoutMs = DEFAULT_TIMEOUT_MS) {
     try {
       response = await fetch(url, {
         signal: AbortSignal.timeout(timeoutMs),
-        headers: { "user-agent": "pagetrace (+https://npmjs.com/package/pagetrace)" }
+        headers: { "user-agent": USER_AGENT }
       });
     } catch (cause) {
       lastError = new Error(`Could not reach ${url}: ${cause.message}`, { cause });
@@ -1652,7 +1674,9 @@ async function snapshotFromDir(dir, config = {}) {
   if (robots !== null) site.robotsTxt = extractRobotsTxt(robots, agents);
   const llms = await readFile(join(dir, "llms.txt"), "utf8").catch(() => null);
   if (llms !== null) site.llmsTxt = extractLlmsTxt(llms);
-  await resolveBrokenLinks(pages, links, site.origin ?? "https://pagetrace.invalid");
+  const origin = site.origin ?? "https://pagetrace.invalid";
+  await resolveBrokenLinks(pages, links, origin);
+  if (config.checkExternal) await resolveDeadExternal(pages, links, origin, 5);
   return { schemaVersion: 1, createdAt: (/* @__PURE__ */ new Date()).toISOString(), site, pages };
 }
 var ASSET_PATH = /\.(?!html?$)[a-z0-9]+$/i;
@@ -1691,6 +1715,63 @@ async function resolveBrokenLinks(pages, links, origin, verify) {
     if (confirmed.length > 0) pages[route].brokenLinks = confirmed.sort();
   }
 }
+function externalUrl(href, from, origin) {
+  try {
+    const url = new URL(href, `${origin}${from === "/" ? "" : from}/`);
+    if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+    if (url.origin === origin) return null;
+    url.hash = "";
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+var MAX_EXTERNAL_CHECKS = 200;
+async function findDeadExternal(urls, concurrency, timeoutMs = DEFAULT_TIMEOUT_MS) {
+  const dead = /* @__PURE__ */ new Set();
+  const queue = urls.slice(0, MAX_EXTERNAL_CHECKS);
+  const request = (url, method) => fetch(url, {
+    method,
+    redirect: "follow",
+    signal: AbortSignal.timeout(timeoutMs),
+    headers: { "user-agent": USER_AGENT }
+  });
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+      while (queue.length > 0) {
+        const url = queue.shift();
+        try {
+          let response = await request(url, "HEAD");
+          if (response.status === 405 || response.status === 501) {
+            response = await request(url, "GET");
+          }
+          if (response.status === 404 || response.status === 410) dead.add(url);
+        } catch {
+        }
+      }
+    })
+  );
+  return dead;
+}
+async function resolveDeadExternal(pages, links, origin, concurrency, timeoutMs) {
+  const perPage = /* @__PURE__ */ new Map();
+  const unique = /* @__PURE__ */ new Set();
+  for (const [route, hrefs] of Object.entries(links)) {
+    const external = [
+      ...new Set(
+        hrefs.map((href) => externalUrl(href, route, origin)).filter((url) => url !== null)
+      )
+    ];
+    if (external.length === 0) continue;
+    perPage.set(route, external);
+    for (const url of external) unique.add(url);
+  }
+  const dead = await findDeadExternal([...unique], concurrency, timeoutMs);
+  for (const [route, external] of perPage) {
+    const found = external.filter((url) => dead.has(url)).sort();
+    if (found.length > 0) pages[route].deadExternal = found;
+  }
+}
 var MAX_NESTED_SITEMAPS = 50;
 async function snapshotFromOrigin(origin, options = {}) {
   const base = new URL(origin);
@@ -1779,6 +1860,9 @@ async function snapshotFromOrigin(origin, options = {}) {
       return false;
     }
   });
+  if (options.checkExternal) {
+    await resolveDeadExternal(pages, links, base.origin, concurrency, timeout);
+  }
   return { schemaVersion: 1, createdAt: (/* @__PURE__ */ new Date()).toISOString(), site, pages };
 }
 
@@ -1831,13 +1915,15 @@ async function loadConfig(path = DEFAULT_CONFIG) {
   }
 }
 async function build(flags, config) {
-  if (flags.dir) return snapshotFromDir(flags.dir, config);
+  const checkExternal = flags.external ?? config.checkExternal;
+  if (flags.dir) return snapshotFromDir(flags.dir, { ...config, checkExternal });
   if (flags.url)
     return snapshotFromOrigin(flags.url, {
       ...config,
       limit: flags.limit,
       concurrency: flags.concurrency,
-      ignoreRobots: flags.ignoreRobots ?? config.ignoreRobots
+      ignoreRobots: flags.ignoreRobots ?? config.ignoreRobots,
+      checkExternal
     });
   throw new Error("Provide a source: --dir <build directory> or --url <origin>.");
 }
@@ -1856,7 +1942,7 @@ function render(findings, format) {
   }
 }
 var cli = cac("pagetrace");
-cli.command("snapshot", "Record the current SEO/AEO surface to a lockfile").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--out <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).action(async (flags) => {
+cli.command("snapshot", "Record the current SEO/AEO surface to a lockfile").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--out <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).action(async (flags) => {
   const config = await loadConfig(flags.config);
   const snapshot = await build(flags, config);
   const written = await writeLockfile(flags.out, snapshot);
@@ -1865,7 +1951,7 @@ cli.command("snapshot", "Record the current SEO/AEO surface to a lockfile").opti
     written ? pc2.green(`Wrote ${flags.out} \u2014 ${count} page${count === 1 ? "" : "s"}.`) : pc2.dim(`${flags.out} is already up to date \u2014 ${count} page${count === 1 ? "" : "s"}.`)
   );
 });
-cli.command("check", "Compare the current surface against the lockfile").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--lockfile <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown | github | sarif", { default: "pretty" }).option("--fail-on <severity>", "error | warn | info", { default: "error" }).option("--audit", "Also run absolute rules, not just the diff", { default: true }).option("--update", "Write the new state to the lockfile after reporting").option("--baseline-branch <ref>", "Read the baseline lockfile from a git ref instead of disk").action(async (flags) => {
+cli.command("check", "Compare the current surface against the lockfile").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--lockfile <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown | github | sarif", { default: "pretty" }).option("--fail-on <severity>", "error | warn | info", { default: "error" }).option("--audit", "Also run absolute rules, not just the diff", { default: true }).option("--update", "Write the new state to the lockfile after reporting").option("--baseline-branch <ref>", "Read the baseline lockfile from a git ref instead of disk").action(async (flags) => {
   const failOn = parseFailOn(flags.failOn, false);
   const config = await loadConfig(flags.config);
   const next = await build(flags, config);
@@ -1895,7 +1981,7 @@ Failing: ${summary.error} error, ${summary.warn} warning (--fail-on ${failOn}).`
     process.exitCode = EXIT_FINDINGS;
   }
 });
-cli.command("audit", "Audit a site as it stands, with explanations and fixes").option("--url <origin>", "Live origin to crawl").option("--dir <dir>", "Directory of built HTML").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown | html", { default: "pretty" }).option("--out <file>", "Write the report to a file instead of stdout").option("--fail-on <severity>", "error | warn | info | never", { default: "never" }).action(async (flags) => {
+cli.command("audit", "Audit a site as it stands, with explanations and fixes").option("--url <origin>", "Live origin to crawl").option("--dir <dir>", "Directory of built HTML").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown | html", { default: "pretty" }).option("--out <file>", "Write the report to a file instead of stdout").option("--fail-on <severity>", "error | warn | info | never", { default: "never" }).action(async (flags) => {
   const failOn = parseFailOn(flags.failOn, true);
   const config = await loadConfig(flags.config);
   const snapshot = await build(flags, config);
@@ -1935,7 +2021,7 @@ cli.command("audit", "Audit a site as it stands, with explanations and fixes").o
     process.exitCode = EXIT_FINDINGS;
   }
 });
-cli.command("links", "Find internal links that point at no page").option("--url <origin>", "Live origin to crawl").option("--dir <dir>", "Directory of built HTML").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown", { default: "pretty" }).option("--out <file>", "Write the report to a file instead of stdout").option("--fail-on <severity>", "error | warn | info | never", { default: "error" }).action(async (flags) => {
+cli.command("links", "Find internal links that point at no page").option("--url <origin>", "Live origin to crawl").option("--dir <dir>", "Directory of built HTML").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).option("--format <format>", "pretty | json | markdown", { default: "pretty" }).option("--out <file>", "Write the report to a file instead of stdout").option("--fail-on <severity>", "error | warn | info | never", { default: "error" }).action(async (flags) => {
   const failOn = parseFailOn(flags.failOn, true);
   const config = await loadConfig(flags.config);
   const snapshot = await build(flags, config);
@@ -1982,7 +2068,7 @@ cli.command("links", "Find internal links that point at no page").option("--url
     process.exitCode = EXIT_FINDINGS;
   }
 });
-cli.command("init", "Write a config file and take the first snapshot").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--out <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).action(async (flags) => {
+cli.command("init", "Write a config file and take the first snapshot").option("--dir <dir>", "Directory of built HTML").option("--url <origin>", "Live origin to crawl").option("--limit <n>", "Max pages to crawl", { default: 200 }).option("--concurrency <n>", "Parallel requests", { default: 5 }).option("--ignore-robots", "Crawl paths that robots.txt disallows").option("--external", "Also check links that leave the site").option("--out <file>", "Lockfile path", { default: DEFAULT_LOCKFILE }).option("--config <file>", "Config file", { default: DEFAULT_CONFIG }).action(async (flags) => {
   const existing = await readFile2(flags.config, "utf8").catch(() => null);
   if (existing === null) {
     const config = flags.url ? { siteUrl: new URL(flags.url).origin } : {};
@@ -2005,11 +2091,11 @@ Commit both files, then run \`pagetrace check ${flags.dir ? `--dir ${flags.dir}`
 });
 cli.command("update", "Check npm for a newer pagetrace and install it").option("--check", "Only report whether an update exists").action(async (flags) => {
   const latest = await latestVersion("pagetrace");
-  if (!isNewer(latest, "0.11.0")) {
-    console.log(pc2.green(`pagetrace ${"0.11.0"} is the latest version.`));
+  if (!isNewer(latest, "0.12.0")) {
+    console.log(pc2.green(`pagetrace ${"0.12.0"} is the latest version.`));
     return;
   }
-  console.log(pc2.yellow(`Update available: ${"0.11.0"} \u2192 ${latest}`));
+  console.log(pc2.yellow(`Update available: ${"0.12.0"} \u2192 ${latest}`));
   if (flags.check) return;
   if (process.argv[1]?.startsWith(process.cwd())) {
     console.log(
@@ -2027,7 +2113,7 @@ cli.command("update", "Check npm for a newer pagetrace and install it").option("
   console.log(pc2.green(`Updated to pagetrace ${latest}.`));
 });
 cli.help();
-cli.version("0.11.0");
+cli.version("0.12.0");
 async function main() {
   try {
     cli.parse(process.argv, { run: false });

package/dist/index.cjs

@@ -297,13 +297,24 @@ function auditPage(page, config = {}) {
   return findings;
 }
 function auditLinks(page) {
-  return (page.brokenLinks ?? []).map((target) => ({
-    code: "link.broken",
-    severity: "error",
-    route: page.route,
-    message: `Links to ${target}, which does not exist.`,
-    after: target
-  }));
+  return [
+    ...(page.brokenLinks ?? []).map((target) => ({
+      code: "link.broken",
+      severity: "error",
+      route: page.route,
+      message: `Links to ${target}, which does not exist.`,
+      after: target
+    })),
+    // A warning, not an error: the page it points at belongs to somebody else,
+    // who can delete it on a Tuesday without asking you.
+    ...(page.deadExternal ?? []).map((target) => ({
+      code: "link.external.dead",
+      severity: "warn",
+      route: page.route,
+      message: `Links out to ${target}, which answers 404.`,
+      after: target
+    }))
+  ];
 }
 function auditSite(snapshot) {
   const findings = [];
@@ -674,6 +685,12 @@ function diffPage(before, after) {
     push("link.broken.removed", "info", `Link to ${target} was fixed or removed.`, {
       before: target
     });
+  const deadBefore = before.deadExternal ?? [];
+  const deadAfter = after.deadExternal ?? [];
+  for (const target of deadAfter.filter((t) => !deadBefore.includes(t)))
+    push("link.external.dead.added", "warn", `Links out to ${target}, which answers 404.`, {
+      after: target
+    });
   const droppedHreflang = Object.keys(before.hreflang).filter((k) => !(k in after.hreflang));
   if (droppedHreflang.length > 0)
     push("hreflang.removed", "warn", "hreflang alternates were removed.", {
@@ -962,8 +979,8 @@ function extractLinks(html) {
   const hrefs = /* @__PURE__ */ new Set();
   for (const anchor of root.querySelectorAll("a[href]")) {
     const href = anchor.getAttribute("href")?.trim();
-    if (!href) continue;
-    if (href.startsWith("#") || href.startsWith("//") || /^[a-z][a-z0-9+.-]*:/i.test(href)) continue;
+    if (!href || href.startsWith("#")) continue;
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href) && !/^https?:/i.test(href)) continue;
     hrefs.add(href);
   }
   return [...hrefs];
@@ -1021,6 +1038,10 @@ var GUIDANCE = {
       nextjs: "A `<Link href>` pointing at a route that no longer exists. TypeScript will not catch it \u2014 typed routes are opt-in via `experimental.typedRoutes`."
     }
   },
+  "link.external.dead": {
+    why: "An outbound link to a page that is gone sends readers to a 404 and spends the trust the link was passing on nothing. Unlike an internal link, the page is not yours to restore.",
+    fix: "Point the link at the current URL, at an archived copy, or remove it. A link that has been dead a while is usually a citation worth replacing rather than deleting."
+  },
   "sitemap.dead": {
     why: "A sitemap is a list of URLs you are asking to have crawled. Entries that 404 spend crawl budget on nothing and lower the trust placed in the rest of the file.",
     fix: "Remove the URL from the sitemap, or restore the page. If it moved, redirect it and list the destination instead.",
@@ -1656,6 +1677,7 @@ async function walkHtml(dir, acc = []) {
   }
   return acc;
 }
+var USER_AGENT = "pagetrace (+https://npmjs.com/package/pagetrace)";
 var DEFAULT_TIMEOUT_MS = 15e3;
 var MAX_ATTEMPTS = 3;
 var RETRY_BASE_MS = 300;
@@ -1671,7 +1693,7 @@ async function fetchDoc(url, timeoutMs = DEFAULT_TIMEOUT_MS) {
     try {
       response = await fetch(url, {
         signal: AbortSignal.timeout(timeoutMs),
-        headers: { "user-agent": "pagetrace (+https://npmjs.com/package/pagetrace)" }
+        headers: { "user-agent": USER_AGENT }
       });
     } catch (cause) {
       lastError = new Error(`Could not reach ${url}: ${cause.message}`, { cause });
@@ -1716,7 +1738,9 @@ async function snapshotFromDir(dir, config = {}) {
   if (robots !== null) site.robotsTxt = extractRobotsTxt(robots, agents);
   const llms = await (0, import_promises.readFile)((0, import_node_path.join)(dir, "llms.txt"), "utf8").catch(() => null);
   if (llms !== null) site.llmsTxt = extractLlmsTxt(llms);
-  await resolveBrokenLinks(pages, links, site.origin ?? "https://pagetrace.invalid");
+  const origin = site.origin ?? "https://pagetrace.invalid";
+  await resolveBrokenLinks(pages, links, origin);
+  if (config.checkExternal) await resolveDeadExternal(pages, links, origin, 5);
   return { schemaVersion: 1, createdAt: (/* @__PURE__ */ new Date()).toISOString(), site, pages };
 }
 var ASSET_PATH = /\.(?!html?$)[a-z0-9]+$/i;
@@ -1755,6 +1779,63 @@ async function resolveBrokenLinks(pages, links, origin, verify) {
     if (confirmed.length > 0) pages[route].brokenLinks = confirmed.sort();
   }
 }
+function externalUrl(href, from, origin) {
+  try {
+    const url = new URL(href, `${origin}${from === "/" ? "" : from}/`);
+    if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+    if (url.origin === origin) return null;
+    url.hash = "";
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+var MAX_EXTERNAL_CHECKS = 200;
+async function findDeadExternal(urls, concurrency, timeoutMs = DEFAULT_TIMEOUT_MS) {
+  const dead = /* @__PURE__ */ new Set();
+  const queue = urls.slice(0, MAX_EXTERNAL_CHECKS);
+  const request = (url, method) => fetch(url, {
+    method,
+    redirect: "follow",
+    signal: AbortSignal.timeout(timeoutMs),
+    headers: { "user-agent": USER_AGENT }
+  });
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+      while (queue.length > 0) {
+        const url = queue.shift();
+        try {
+          let response = await request(url, "HEAD");
+          if (response.status === 405 || response.status === 501) {
+            response = await request(url, "GET");
+          }
+          if (response.status === 404 || response.status === 410) dead.add(url);
+        } catch {
+        }
+      }
+    })
+  );
+  return dead;
+}
+async function resolveDeadExternal(pages, links, origin, concurrency, timeoutMs) {
+  const perPage = /* @__PURE__ */ new Map();
+  const unique = /* @__PURE__ */ new Set();
+  for (const [route, hrefs] of Object.entries(links)) {
+    const external = [
+      ...new Set(
+        hrefs.map((href) => externalUrl(href, route, origin)).filter((url) => url !== null)
+      )
+    ];
+    if (external.length === 0) continue;
+    perPage.set(route, external);
+    for (const url of external) unique.add(url);
+  }
+  const dead = await findDeadExternal([...unique], concurrency, timeoutMs);
+  for (const [route, external] of perPage) {
+    const found = external.filter((url) => dead.has(url)).sort();
+    if (found.length > 0) pages[route].deadExternal = found;
+  }
+}
 var MAX_NESTED_SITEMAPS = 50;
 async function snapshotFromOrigin(origin, options = {}) {
   const base = new URL(origin);
@@ -1843,6 +1924,9 @@ async function snapshotFromOrigin(origin, options = {}) {
       return false;
     }
   });
+  if (options.checkExternal) {
+    await resolveDeadExternal(pages, links, base.origin, concurrency, timeout);
+  }
   return { schemaVersion: 1, createdAt: (/* @__PURE__ */ new Date()).toISOString(), site, pages };
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/index.d.cts

@@ -50,6 +50,12 @@ interface PageFingerprint {
      * property the lockfile has to have.
      */
     brokenLinks?: string[];
+    /**
+     * External links that answered 404 or 410. Only present when the crawl was
+     * asked to check them: it means a request per unique external URL, to hosts
+     * you do not control.
+     */
+    deadExternal?: string[];
 }
 /** Site-wide signals that live outside any single page. */
 interface SiteFingerprint {
@@ -139,6 +145,12 @@ interface Config {
      * worse than crawling a site you already own.
      */
     ignoreRobots?: boolean;
+    /**
+     * Also check links that leave the site. Off by default: it fans out to hosts
+     * you do not control, and a lockfile that records their availability turns a
+     * third party's bad afternoon into a diff in your repository.
+     */
+    checkExternal?: boolean;
 }
 
 /**
@@ -186,12 +198,12 @@ declare function isCrawlable(path: string, rules: {
     allow?: string[];
 } | null | undefined): boolean;
 /**
- * Internal-looking hrefs on the page, deduplicated, in document order.
+ * Every href on the page worth resolving, deduplicated, in document order.
+ * Both internal and external: snapshot.ts is the only layer that knows the
+ * page's own URL, so classification happens there.
  *
- * Resolution happens in snapshot.ts, which is the only layer that knows the
- * page's own URL and the full set of routes. Anything with a scheme is somebody
- * else's problem: an external link checker fans out to hosts you do not
- * control, where a Cloudflare 403 and a rate limit both look like a dead page.
+ * Dropped here because they can never be dead: fragments, mailto:, tel:,
+ * javascript: and other non-http schemes.
  */
 declare function extractLinks(html: string): string[];
 /** Parse llms.txt, capturing section headings so truncation is detectable. */

package/dist/index.d.ts

@@ -50,6 +50,12 @@ interface PageFingerprint {
      * property the lockfile has to have.
      */
     brokenLinks?: string[];
+    /**
+     * External links that answered 404 or 410. Only present when the crawl was
+     * asked to check them: it means a request per unique external URL, to hosts
+     * you do not control.
+     */
+    deadExternal?: string[];
 }
 /** Site-wide signals that live outside any single page. */
 interface SiteFingerprint {
@@ -139,6 +145,12 @@ interface Config {
      * worse than crawling a site you already own.
      */
     ignoreRobots?: boolean;
+    /**
+     * Also check links that leave the site. Off by default: it fans out to hosts
+     * you do not control, and a lockfile that records their availability turns a
+     * third party's bad afternoon into a diff in your repository.
+     */
+    checkExternal?: boolean;
 }
 
 /**
@@ -186,12 +198,12 @@ declare function isCrawlable(path: string, rules: {
     allow?: string[];
 } | null | undefined): boolean;
 /**
- * Internal-looking hrefs on the page, deduplicated, in document order.
+ * Every href on the page worth resolving, deduplicated, in document order.
+ * Both internal and external: snapshot.ts is the only layer that knows the
+ * page's own URL, so classification happens there.
  *
- * Resolution happens in snapshot.ts, which is the only layer that knows the
- * page's own URL and the full set of routes. Anything with a scheme is somebody
- * else's problem: an external link checker fans out to hosts you do not
- * control, where a Cloudflare 403 and a rate limit both look like a dead page.
+ * Dropped here because they can never be dead: fragments, mailto:, tel:,
+ * javascript: and other non-http schemes.
  */
 declare function extractLinks(html: string): string[];
 /** Parse llms.txt, capturing section headings so truncation is detectable. */

package/dist/index.js

@@ -223,13 +223,24 @@ function auditPage(page, config = {}) {
   return findings;
 }
 function auditLinks(page) {
-  return (page.brokenLinks ?? []).map((target) => ({
-    code: "link.broken",
-    severity: "error",
-    route: page.route,
-    message: `Links to ${target}, which does not exist.`,
-    after: target
-  }));
+  return [
+    ...(page.brokenLinks ?? []).map((target) => ({
+      code: "link.broken",
+      severity: "error",
+      route: page.route,
+      message: `Links to ${target}, which does not exist.`,
+      after: target
+    })),
+    // A warning, not an error: the page it points at belongs to somebody else,
+    // who can delete it on a Tuesday without asking you.
+    ...(page.deadExternal ?? []).map((target) => ({
+      code: "link.external.dead",
+      severity: "warn",
+      route: page.route,
+      message: `Links out to ${target}, which answers 404.`,
+      after: target
+    }))
+  ];
 }
 function auditSite(snapshot) {
   const findings = [];
@@ -600,6 +611,12 @@ function diffPage(before, after) {
     push("link.broken.removed", "info", `Link to ${target} was fixed or removed.`, {
       before: target
     });
+  const deadBefore = before.deadExternal ?? [];
+  const deadAfter = after.deadExternal ?? [];
+  for (const target of deadAfter.filter((t) => !deadBefore.includes(t)))
+    push("link.external.dead.added", "warn", `Links out to ${target}, which answers 404.`, {
+      after: target
+    });
   const droppedHreflang = Object.keys(before.hreflang).filter((k) => !(k in after.hreflang));
   if (droppedHreflang.length > 0)
     push("hreflang.removed", "warn", "hreflang alternates were removed.", {
@@ -888,8 +905,8 @@ function extractLinks(html) {
   const hrefs = /* @__PURE__ */ new Set();
   for (const anchor of root.querySelectorAll("a[href]")) {
     const href = anchor.getAttribute("href")?.trim();
-    if (!href) continue;
-    if (href.startsWith("#") || href.startsWith("//") || /^[a-z][a-z0-9+.-]*:/i.test(href)) continue;
+    if (!href || href.startsWith("#")) continue;
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href) && !/^https?:/i.test(href)) continue;
     hrefs.add(href);
   }
   return [...hrefs];
@@ -947,6 +964,10 @@ var GUIDANCE = {
       nextjs: "A `<Link href>` pointing at a route that no longer exists. TypeScript will not catch it \u2014 typed routes are opt-in via `experimental.typedRoutes`."
     }
   },
+  "link.external.dead": {
+    why: "An outbound link to a page that is gone sends readers to a 404 and spends the trust the link was passing on nothing. Unlike an internal link, the page is not yours to restore.",
+    fix: "Point the link at the current URL, at an archived copy, or remove it. A link that has been dead a while is usually a citation worth replacing rather than deleting."
+  },
   "sitemap.dead": {
     why: "A sitemap is a list of URLs you are asking to have crawled. Entries that 404 spend crawl budget on nothing and lower the trust placed in the rest of the file.",
     fix: "Remove the URL from the sitemap, or restore the page. If it moved, redirect it and list the destination instead.",
@@ -1582,6 +1603,7 @@ async function walkHtml(dir, acc = []) {
   }
   return acc;
 }
+var USER_AGENT = "pagetrace (+https://npmjs.com/package/pagetrace)";
 var DEFAULT_TIMEOUT_MS = 15e3;
 var MAX_ATTEMPTS = 3;
 var RETRY_BASE_MS = 300;
@@ -1597,7 +1619,7 @@ async function fetchDoc(url, timeoutMs = DEFAULT_TIMEOUT_MS) {
     try {
       response = await fetch(url, {
         signal: AbortSignal.timeout(timeoutMs),
-        headers: { "user-agent": "pagetrace (+https://npmjs.com/package/pagetrace)" }
+        headers: { "user-agent": USER_AGENT }
       });
     } catch (cause) {
       lastError = new Error(`Could not reach ${url}: ${cause.message}`, { cause });
@@ -1642,7 +1664,9 @@ async function snapshotFromDir(dir, config = {}) {
   if (robots !== null) site.robotsTxt = extractRobotsTxt(robots, agents);
   const llms = await readFile(join(dir, "llms.txt"), "utf8").catch(() => null);
   if (llms !== null) site.llmsTxt = extractLlmsTxt(llms);
-  await resolveBrokenLinks(pages, links, site.origin ?? "https://pagetrace.invalid");
+  const origin = site.origin ?? "https://pagetrace.invalid";
+  await resolveBrokenLinks(pages, links, origin);
+  if (config.checkExternal) await resolveDeadExternal(pages, links, origin, 5);
   return { schemaVersion: 1, createdAt: (/* @__PURE__ */ new Date()).toISOString(), site, pages };
 }
 var ASSET_PATH = /\.(?!html?$)[a-z0-9]+$/i;
@@ -1681,6 +1705,63 @@ async function resolveBrokenLinks(pages, links, origin, verify) {
     if (confirmed.length > 0) pages[route].brokenLinks = confirmed.sort();
   }
 }
+function externalUrl(href, from, origin) {
+  try {
+    const url = new URL(href, `${origin}${from === "/" ? "" : from}/`);
+    if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+    if (url.origin === origin) return null;
+    url.hash = "";
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+var MAX_EXTERNAL_CHECKS = 200;
+async function findDeadExternal(urls, concurrency, timeoutMs = DEFAULT_TIMEOUT_MS) {
+  const dead = /* @__PURE__ */ new Set();
+  const queue = urls.slice(0, MAX_EXTERNAL_CHECKS);
+  const request = (url, method) => fetch(url, {
+    method,
+    redirect: "follow",
+    signal: AbortSignal.timeout(timeoutMs),
+    headers: { "user-agent": USER_AGENT }
+  });
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+      while (queue.length > 0) {
+        const url = queue.shift();
+        try {
+          let response = await request(url, "HEAD");
+          if (response.status === 405 || response.status === 501) {
+            response = await request(url, "GET");
+          }
+          if (response.status === 404 || response.status === 410) dead.add(url);
+        } catch {
+        }
+      }
+    })
+  );
+  return dead;
+}
+async function resolveDeadExternal(pages, links, origin, concurrency, timeoutMs) {
+  const perPage = /* @__PURE__ */ new Map();
+  const unique = /* @__PURE__ */ new Set();
+  for (const [route, hrefs] of Object.entries(links)) {
+    const external = [
+      ...new Set(
+        hrefs.map((href) => externalUrl(href, route, origin)).filter((url) => url !== null)
+      )
+    ];
+    if (external.length === 0) continue;
+    perPage.set(route, external);
+    for (const url of external) unique.add(url);
+  }
+  const dead = await findDeadExternal([...unique], concurrency, timeoutMs);
+  for (const [route, external] of perPage) {
+    const found = external.filter((url) => dead.has(url)).sort();
+    if (found.length > 0) pages[route].deadExternal = found;
+  }
+}
 var MAX_NESTED_SITEMAPS = 50;
 async function snapshotFromOrigin(origin, options = {}) {
   const base = new URL(origin);
@@ -1769,6 +1850,9 @@ async function snapshotFromOrigin(origin, options = {}) {
       return false;
     }
   });
+  if (options.checkExternal) {
+    await resolveDeadExternal(pages, links, base.origin, concurrency, timeout);
+  }
   return { schemaVersion: 1, createdAt: (/* @__PURE__ */ new Date()).toISOString(), site, pages };
 }
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pagetrace",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Baseline your site's SEO and AEO surface, diff every build against it, and fail CI on regressions.",
   "main": "./dist/index.cjs",
   "scripts": {