meodp 0.1.0 → 0.2.0

package/dist/shared/meodp.93e06e00.d.ts

@@ -0,0 +1,64 @@
+interface LinkTarget {
+    url: string;
+    name?: string;
+}
+type Availability = 'reachable' | 'restricted' | 'unavailable';
+type FailureReason = 'http' | 'dns' | 'tls' | 'timeout' | 'network' | 'redirect';
+interface LinkObservation extends LinkTarget {
+    status: Availability;
+    checkedAt: string;
+    durationMs: number;
+    attempts: number;
+    finalUrl: string;
+    httpStatus?: number;
+    redirects: {
+        url: string;
+        status: number;
+        location: string;
+    }[];
+    reason?: FailureReason;
+    detail?: string;
+    consecutiveFailures: number;
+    firstFailureAt?: string;
+    lastSuccessAt?: string;
+    changed: boolean;
+    recovered: boolean;
+}
+interface CheckReport {
+    schemaVersion: 1;
+    observer: string;
+    startedAt: string;
+    completedAt: string;
+    summary: Record<Availability, number> & {
+        total: number;
+        redirected: number;
+        recovered: number;
+    };
+    results: LinkObservation[];
+}
+interface CheckOptions {
+    /** Maximum simultaneous site checks. Default: 5. */
+    concurrency?: number;
+    /** Timeout for each HTTP request, including its body. Default: 10000 ms. */
+    timeoutMs?: number;
+    /** Retries for transport errors and 5xx responses. Default: 1; 429 is not retried. */
+    retries?: number;
+    /** Maximum followed redirects per attempt. Default: 5. */
+    maxRedirects?: number;
+    /** Identifies the network/environment. Must match previousReport.observer. */
+    observer?: string;
+    previousReport?: CheckReport;
+    onResult?: (result: LinkObservation) => void | Promise<void>;
+}
+interface SitemapOptions extends CheckOptions {
+    /** Treat the input as a site URL: read robots.txt, falling back to /sitemap.xml. */
+    discover?: boolean;
+    /** Reject before page checks if discovery exceeds this many unique pages. Default: 10000. */
+    maxUrls?: number;
+    /** Maximum number of unique sitemap documents to read. Default: 100. */
+    maxSitemaps?: number;
+    /** Maximum downloaded and decompressed size per discovery document. Default: 10 MiB. */
+    maxSitemapBytes?: number;
+}
+
+export type { Availability as A, CheckReport as C, FailureReason as F, LinkTarget as L, SitemapOptions as S, CheckOptions as a, LinkObservation as b };

package/dist/shared/{meodp.d0916bc5.mjs → meodp.9c706389.mjs}

@@ -1,185 +1,6 @@
-import { setTimeout } from 'node:timers/promises';
-import { LinkChecker } from 'linkinator';
-import { hostname } from 'node:os';
 import { randomUUID } from 'node:crypto';
 import { mkdir, writeFile, readFile, rename } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
-import { Buffer } from 'node:buffer';
-import { promisify } from 'node:util';
-import { gunzip } from 'node:zlib';
-import { XMLValidator, XMLParser } from 'fast-xml-parser';
-
-function normalizeUrl(value) {
-  const url = new URL(value);
-  if (!["http:", "https:"].includes(url.protocol) || url.username || url.password)
-    throw new Error("Expected an HTTP(S) URL without embedded credentials");
-  url.hash = "";
-  return url.href;
-}
-function normalizeTargets(input) {
-  if (!Array.isArray(input))
-    throw new TypeError("Input must be an array of URLs or objects with a url field");
-  const seen = /* @__PURE__ */ new Set();
-  return input.map((item, index) => {
-    const value = typeof item === "string" ? { url: item } : item;
-    if (!value || typeof value !== "object" || !("url" in value) || typeof value.url !== "string")
-      throw new TypeError(`Invalid link at index ${index}: expected a url string`);
-    let url;
-    try {
-      url = normalizeUrl(value.url.trim());
-    } catch {
-      throw new TypeError(`Invalid HTTP(S) URL at index ${index}`);
-    }
-    if ("name" in value && value.name !== void 0 && typeof value.name !== "string")
-      throw new TypeError(`Invalid name at index ${index}: expected a string`);
-    return { url, ..."name" in value && typeof value.name === "string" ? { name: value.name } : {} };
-  }).filter((item) => {
-    if (seen.has(item.url))
-      return false;
-    seen.add(item.url);
-    return true;
-  });
-}
-
-function integerOption(value, fallback, name, minimum, maximum) {
-  const result = value ?? fallback;
-  if (!Number.isInteger(result) || result < minimum || result > maximum)
-    throw new RangeError(`${name} must be an integer between ${minimum} and ${maximum}`);
-  return result;
-}
-function normalizeCheckOptions(options) {
-  const concurrency = integerOption(options.concurrency, 5, "concurrency", 1, 100);
-  const timeoutMs = integerOption(options.timeoutMs, 1e4, "timeoutMs", 1, 3e5);
-  const retries = integerOption(options.retries, 1, "retries", 0, 5);
-  const maxRedirects = integerOption(options.maxRedirects, 5, "maxRedirects", 0, 20);
-  const observer = options.observer ?? hostname();
-  if (!observer.trim())
-    throw new TypeError("observer must not be empty");
-  if (options.previousReport && options.previousReport.observer !== observer)
-    throw new Error("Previous report belongs to another observer; use a separate history file");
-  return { concurrency, timeoutMs, retries, maxRedirects, observer };
-}
-
-function failureDetail(result) {
-  const error = result.failureDetails?.find((item) => item instanceof Error);
-  const parts = [];
-  let current = error;
-  for (let depth = 0; current && typeof current === "object" && depth < 5; depth++) {
-    for (const key of ["name", "code", "message"]) {
-      const value = Reflect.get(current, key);
-      if (typeof value === "string")
-        parts.push(value);
-    }
-    current = "cause" in current ? current.cause : void 0;
-  }
-  const detail = parts.join(": ") || `HTTP ${result.status ?? 0}`;
-  if (/ENOTFOUND|EAI_AGAIN|EAI_FAIL/.test(detail))
-    return { reason: "dns", detail };
-  if (/CERT|TLS|SSL/i.test(detail))
-    return { reason: "tls", detail };
-  if (/timeout|timed out|ETIMEDOUT|UND_ERR_.*TIMEOUT/i.test(detail))
-    return { reason: "timeout", detail };
-  return { reason: result.status ? "http" : "network", detail };
-}
-async function probe(url, timeoutMs, maxRedirects) {
-  const redirects = [];
-  const visited = /* @__PURE__ */ new Set();
-  let current = url;
-  for (; ; ) {
-    visited.add(current);
-    const checker = new LinkChecker();
-    const { links } = await checker.check({
-      path: current,
-      recurse: false,
-      concurrency: 1,
-      timeout: timeoutMs,
-      retry: false,
-      retryErrors: false,
-      redirects: "error",
-      linksToSkip: async (candidate) => candidate !== current
-    });
-    const result = links.find((item) => item.url === current && !item.parent);
-    if (!result)
-      throw new Error("The HTTP checker returned no result for the requested URL");
-    const httpStatus = result.status || void 0;
-    const base = { finalUrl: current, httpStatus, redirects };
-    if (httpStatus && [301, 302, 303, 307, 308].includes(httpStatus)) {
-      const response = result.failureDetails?.find((item) => "headers" in item);
-      const location = response && "headers" in response ? response.headers.location : void 0;
-      let next;
-      try {
-        if (!location)
-          throw new Error("Missing redirect location");
-        next = normalizeUrl(new URL(location, current).href);
-      } catch {
-        return { ...base, status: "unavailable", reason: "redirect", detail: "Missing or invalid HTTP(S) redirect destination" };
-      }
-      redirects.push({ url: current, status: httpStatus, location: next });
-      if (visited.has(next) || redirects.length > maxRedirects) {
-        return {
-          ...base,
-          status: "unavailable",
-          reason: "redirect",
-          detail: visited.has(next) ? "Redirect loop" : "Redirect limit exceeded"
-        };
-      }
-      current = next;
-      continue;
-    }
-    if (httpStatus && httpStatus >= 200 && httpStatus < 300)
-      return { ...base, status: "reachable" };
-    if (httpStatus && [401, 403, 429, 451, 999].includes(httpStatus))
-      return { ...base, status: "restricted", reason: "http", detail: `HTTP ${httpStatus}: access requires review` };
-    return { ...base, status: "unavailable", ...failureDetail(result) };
-  }
-}
-async function checkLinks(input, options = {}) {
-  const targets = normalizeTargets(input);
-  const { concurrency, timeoutMs, retries, maxRedirects, observer } = normalizeCheckOptions(options);
-  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
-  const previous = new Map(options.previousReport?.results.map((item) => [item.url, item]));
-  const results = [];
-  let cursor = 0;
-  await Promise.all(Array.from({ length: Math.min(concurrency, targets.length) }, async () => {
-    while (cursor < targets.length) {
-      const index = cursor++;
-      const target = targets[index];
-      const start = Date.now();
-      let attempts = 0;
-      let observation;
-      do {
-        if (attempts > 0)
-          await setTimeout(500 * 2 ** (attempts - 1));
-        attempts++;
-        observation = await probe(target.url, timeoutMs, maxRedirects);
-      } while (attempts <= retries && observation.status === "unavailable" && observation.reason !== "redirect" && (!observation.httpStatus || observation.httpStatus >= 500));
-      const checkedAt = (/* @__PURE__ */ new Date()).toISOString();
-      const old = previous.get(target.url);
-      const failed = observation.status === "unavailable";
-      const result = {
-        ...target,
-        ...observation,
-        checkedAt,
-        durationMs: Date.now() - start,
-        attempts,
-        consecutiveFailures: failed ? (old?.status === "unavailable" ? old.consecutiveFailures : 0) + 1 : 0,
-        firstFailureAt: failed ? old?.status === "unavailable" ? old.firstFailureAt ?? checkedAt : checkedAt : void 0,
-        lastSuccessAt: observation.status === "reachable" ? checkedAt : old?.lastSuccessAt,
-        changed: Boolean(old && (old.status !== observation.status || old.finalUrl !== observation.finalUrl)),
-        recovered: observation.status === "reachable" && old?.status === "unavailable"
-      };
-      results[index] = result;
-      await options.onResult?.(result);
-    }
-  }));
-  const summary = { total: results.length, reachable: 0, restricted: 0, unavailable: 0, redirected: 0, recovered: 0 };
-  for (const result of results) {
-    summary[result.status]++;
-    summary.redirected += Number(result.redirects.length > 0);
-    summary.recovered += Number(result.recovered);
-  }
-  return { schemaVersion: 1, observer, startedAt, completedAt: (/* @__PURE__ */ new Date()).toISOString(), summary, results };
-}
 
 function parseReport(data) {
   const invalid = (field) => {
@@ -396,174 +217,4 @@ async function saveReport(report, path) {
   await rename(temporary, path);
 }
 
-const unzip = promisify(gunzip);
-class DocumentError extends Error {
-  constructor(message, retryable = false) {
-    super(message);
-    this.retryable = retryable;
-  }
-}
-function discoveryOptions(options) {
-  const normalized = normalizeCheckOptions(options);
-  if (options.discover !== void 0 && typeof options.discover !== "boolean")
-    throw new TypeError("discover must be a boolean");
-  return {
-    ...normalized,
-    maxUrls: integerOption(options.maxUrls, 1e4, "maxUrls", 1, 5e4),
-    maxSitemaps: integerOption(options.maxSitemaps, 100, "maxSitemaps", 1, 1e4),
-    maxSitemapBytes: integerOption(options.maxSitemapBytes, 10 * 1024 * 1024, "maxSitemapBytes", 1, 100 * 1024 * 1024)
-  };
-}
-async function readDocument(source, options, allowMissing = false) {
-  for (let attempt = 0; ; attempt++) {
-    try {
-      let url = source;
-      const visited = /* @__PURE__ */ new Set();
-      for (; ; ) {
-        visited.add(url);
-        const response = await fetch(url, {
-          redirect: "manual",
-          signal: AbortSignal.timeout(options.timeoutMs)
-        });
-        if ([301, 302, 303, 307, 308].includes(response.status)) {
-          await response.body?.cancel();
-          const location = response.headers.get("location");
-          let next;
-          try {
-            if (!location)
-              throw new Error("Missing location");
-            next = normalizeUrl(new URL(location, url).href);
-          } catch {
-            throw new DocumentError(`Invalid sitemap/robots redirect from ${url}`);
-          }
-          if (visited.has(next) || visited.size > options.maxRedirects)
-            throw new DocumentError(`Sitemap/robots redirect loop or limit exceeded: ${url}`);
-          url = next;
-          continue;
-        }
-        if (!response.ok) {
-          await response.body?.cancel();
-          if (allowMissing && [404, 410].includes(response.status))
-            return void 0;
-          throw new DocumentError(`Unable to read ${url}: HTTP ${response.status}`, response.status >= 500);
-        }
-        const chunks = [];
-        let size = 0;
-        if (response.body) {
-          const reader = response.body.getReader();
-          try {
-            for (; ; ) {
-              const { done, value } = await reader.read();
-              if (done)
-                break;
-              size += value.byteLength;
-              if (size > options.maxSitemapBytes)
-                throw new DocumentError(`Discovery document exceeds maxSitemapBytes: ${url}`);
-              chunks.push(value);
-            }
-          } finally {
-            await reader.cancel().catch(() => {
-            });
-            reader.releaseLock();
-          }
-        }
-        let body = Buffer.concat(chunks);
-        if (body[0] === 31 && body[1] === 139) {
-          try {
-            body = await unzip(body, { maxOutputLength: options.maxSitemapBytes });
-          } catch {
-            throw new DocumentError(`Invalid gzip or decompressed document exceeds maxSitemapBytes: ${url}`);
-          }
-        }
-        return { url, text: body.toString("utf8") };
-      }
-    } catch (error) {
-      if (attempt >= options.retries || error instanceof DocumentError && !error.retryable)
-        throw new Error(`Sitemap discovery failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
-      await setTimeout(500 * 2 ** attempt);
-    }
-  }
-}
-function parseLocations(text, source) {
-  if (/<!DOCTYPE|<!ENTITY/i.test(text) || XMLValidator.validate(text) !== true)
-    throw new Error(`Invalid sitemap XML: ${source}`);
-  const parser = new XMLParser({
-    ignoreAttributes: true,
-    removeNSPrefix: true,
-    parseTagValue: false,
-    isArray: (name) => name === "url" || name === "sitemap"
-  });
-  const document = parser.parse(text);
-  const roots = Object.keys(document).filter((key) => !key.startsWith("?"));
-  if (roots.length !== 1 || !["urlset", "sitemapindex"].includes(roots[0]))
-    throw new Error(`Expected a sitemap urlset or sitemapindex: ${source}`);
-  const isIndex = roots[0] === "sitemapindex";
-  const root = document[roots[0]];
-  if (root?.[isIndex ? "url" : "sitemap"])
-    throw new Error(`Mixed sitemap index and page entries: ${source}`);
-  const entries = root?.[isIndex ? "sitemap" : "url"] ?? [];
-  const urls = entries.map((entry) => {
-    if (!entry || typeof entry !== "object" || !("loc" in entry) || typeof entry.loc !== "string" || !entry.loc.trim())
-      throw new Error(`Missing or invalid sitemap loc: ${source}`);
-    try {
-      return normalizeUrl(new URL(entry.loc.trim(), source).href);
-    } catch {
-      throw new Error(`Invalid HTTP(S) URL in sitemap: ${source}`);
-    }
-  });
-  return { isIndex, urls };
-}
-async function readSitemapUrls(source, options = {}) {
-  const input = normalizeUrl(source.trim());
-  const config = discoveryOptions(options);
-  let pending = [input];
-  if (options.discover) {
-    const robots = await readDocument(new URL("/robots.txt", input).href, config, true);
-    const declared = robots?.text.split(/\r?\n/).flatMap((line) => {
-      const match = line.match(/^\s*sitemap\s*:\s*(\S+)/i);
-      return match ? [normalizeUrl(new URL(match[1], robots.url).href)] : [];
-    }) ?? [];
-    pending = declared.length ? declared : [new URL("/sitemap.xml", input).href];
-  }
-  const queued = new Set(pending);
-  pending = [...queued];
-  const visited = /* @__PURE__ */ new Set();
-  const pages = /* @__PURE__ */ new Set();
-  let documentCount = 0;
-  for (let cursor = 0; cursor < pending.length; cursor++) {
-    const url = pending[cursor];
-    if (visited.has(url))
-      continue;
-    if (++documentCount > config.maxSitemaps)
-      throw new Error("Sitemap discovery exceeds maxSitemaps; no pages were checked");
-    visited.add(url);
-    const document = await readDocument(url, config);
-    if (!document)
-      throw new Error(`Unable to read sitemap: ${url}`);
-    visited.add(document.url);
-    const { isIndex, urls } = parseLocations(document.text, document.url);
-    for (const location of urls) {
-      if (isIndex) {
-        if (!queued.has(location) && !visited.has(location)) {
-          queued.add(location);
-          pending.push(location);
-        }
-      } else {
-        pages.add(location);
-        if (pages.size > config.maxUrls)
-          throw new Error("Sitemap discovery exceeds maxUrls; no pages were checked");
-      }
-    }
-  }
-  if (!pages.size)
-    throw new Error("Sitemap discovery found no page URLs; no pages were checked");
-  return [...pages];
-}
-async function checkSitemap(source, options = {}) {
-  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
-  const urls = await readSitemapUrls(source, options);
-  const report = await checkLinks(urls, options);
-  return { ...report, startedAt };
-}
-
-export { checkLinks as a, writeReportSite as b, checkSitemap as c, readSitemapUrls as d, formatReport as f, normalizeTargets as n, parseReport as p, readReport as r, saveReport as s, writeReports as w };
+export { writeReportSite as a, createReportHtml as c, formatReport as f, parseReport as p, readReport as r, saveReport as s, writeReports as w };

package/dist/shared/meodp.a704f8db.d.mts

@@ -0,0 +1,41 @@
+import { C as CheckReport } from './meodp.93e06e00.mjs';
+
+interface ReportSiteOptions {
+    /** Automatically load this HTTP(S) or relative JSON URL when hosted. */
+    dataUrl?: string;
+}
+
+type ReporterName = 'json' | 'markdown' | 'html';
+type HtmlReporterOptions = ReportSiteOptions & ({
+    outputFolder?: string;
+    outputFile?: never;
+} | {
+    outputFile: string;
+    outputFolder?: never;
+});
+type ReporterDescription = ReporterName | readonly ['json' | 'markdown', {
+    outputFile?: string;
+}?] | readonly ['html', HtmlReporterOptions?];
+/** A name, a list of names, or a list of [name, options] tuples. */
+type ReporterConfig = ReporterName | readonly ReporterDescription[];
+interface ReporterOutput {
+    reporter: ReporterName;
+    files: string[];
+}
+interface ReporterOptions extends ReportSiteOptions {
+    /** Default directory for reporters without explicit output paths. */
+    outputDir?: string;
+    /** Resolve relative output paths from this directory (default: working directory). */
+    cwd?: string;
+}
+/** Run built-in reporters on saved data; no network requests or browser launch. */
+declare function writeReporters(report: CheckReport | undefined, reporter: ReporterConfig, options?: ReporterOptions): Promise<ReporterOutput[]>;
+
+interface ReportVerificationOptions {
+    url: string;
+    attempts?: number;
+    delayMs?: number;
+}
+declare function waitForReport(expected: CheckReport, { url, attempts, delayMs }: ReportVerificationOptions): Promise<void>;
+
+export { type HtmlReporterOptions as H, type ReporterConfig as R, type ReportVerificationOptions as a, type ReportSiteOptions as b, type ReporterDescription as c, type ReporterName as d, type ReporterOptions as e, type ReporterOutput as f, waitForReport as g, writeReporters as w };