@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/archive/types.d.ts

@@ -67,6 +67,11 @@ export interface Config extends Required<Pick<ParseURLOptions, 'disableQueries'>
     userAgent: string;
     /** Whether robots.txt restrictions were ignored during crawling. */
     ignoreRobots: boolean;
+    /**
+     * CSS selector overriding beholder's automatic main-content-region
+     * detection, or `null`/omitted to use the automatic heuristic.
+     */
+    mainContentSelector?: string | null;
 }
 /**
  * Provenance of a page or resource row — which crawler channel originally
@@ -123,9 +128,9 @@ export interface InventoryRunMeta {
     ran_at: string;
     /** Human-readable identifier (e.g. `'prod-2026-06-21'`). `null` when the caller did not supply one. */
     list_label?: string | null;
-    /** SHA-256 hex digest of the source file. `null` if hashing failed (e.g. file vanished mid-run). */
+    /** SHA-256 hex digest of the source file. `null` for programmatic callers that built the URL list in-memory (no source file to hash). */
     source_file_sha256?: string | null;
-    /** Number of non-empty lines in the input list (= URL count before scope filtering). */
+    /** Number of valid URLs in the input list, after the CLI has warned-and-dropped unparseable-URL lines but before scope filtering. */
     total_lines?: number | null;
     /** Number of new HTML seeds inserted as `content_items` rows by this run. */
     new_pages?: number | null;
@@ -133,9 +138,54 @@ export interface InventoryRunMeta {
     new_resources?: number | null;
     /** Number of input URLs dropped because they fell outside the archived scope. */
     scope_skipped?: number | null;
+    /** Number of source-file lines dropped by the CLI for failing URL validation, before this row's `total_lines` was counted. `null` for programmatic callers that built the URL list in-memory (no source file to have invalid lines). */
+    invalid_skipped?: number | null;
     /** Free-form text for backfill annotations or operator notes. */
     notes?: string | null;
 }
+/**
+ * A row in `network_outages` — one detected operator-network outage.
+ *
+ * Append-only except `ended_at`: it is written once, `NULL`, when the
+ * outage is first detected, and updated exactly once when a recovery probe
+ * succeeds. A row can also be left `ended_at = NULL` forever if the crawl
+ * process is killed mid-outage — readers must resolve this via a clamp
+ * (see `is-within-outage-window.ts` and the `db-ops/outages/` writer that
+ * closes stale-open rows on the next writer session), never by treating
+ * `NULL` as an unbounded window.
+ */
+export interface NetworkOutageRow {
+    id: number;
+    /** Epoch ms, backdated to the earliest error still inside the detector's sliding window at trigger time. */
+    started_at: number;
+    /** Epoch ms the sliding window actually crossed both thresholds. */
+    detected_at: number;
+    /** Epoch ms the recovery probe first succeeded, or `null` while still open / if the session crashed before recovery. */
+    ended_at: number | null;
+    /** Hostname the recovery probe targeted, or `null` if none was available (see `choose-probe-host.ts`). */
+    probe_host: string | null;
+    /** Error count in the detector's window at trigger time. */
+    trigger_error_count: number;
+    /** Distinct host count in the detector's window at trigger time. */
+    trigger_host_count: number;
+}
+/**
+ * Fields required to record a newly-detected outage via
+ * `Database.insertNetworkOutage`. camelCase (unlike {@link NetworkOutageRow}
+ * / {@link InventoryRunMeta}) because callers build this directly from
+ * `NetworkOutageDetector`'s camelCase `OutageSuspect` plus a probe host —
+ * the db-op does the camelCase → snake_case column mapping on write.
+ */
+export interface InsertNetworkOutageParams {
+    /** Backdated to the earliest error still inside the detector's window at trigger time. */
+    startedAt: number;
+    /** When the sliding window actually crossed both thresholds. */
+    detectedAt: number;
+    /** Hostname the recovery probe will target, or `null` if none was available. */
+    probeHost: string | null;
+    triggerErrorCount: number;
+    triggerHostCount: number;
+}
 /**
  * Filter type for querying pages from the database.
  *
@@ -292,6 +342,40 @@ export interface DB_Page {
     jsonld_count: number | null;
     /** Sorted unique provider names, comma-separated (empty string when none). */
     tags_providers_csv: string | null;
+    /** Detected main-content element's `nodeName` (e.g. `'MAIN'`), or null. */
+    main_content_node_name: string | null;
+    /** Detected main-content element's `id`, or null. */
+    main_content_id: string | null;
+    /** Detected main-content element's `role` attribute, or null. */
+    main_content_role: string | null;
+    /** Diagnostic tag+id+class selector for the detected element, or null. */
+    main_content_selector: string | null;
+    /** JSON-encoded array of the detected element's CSS classes, or null. */
+    main_content_class_list: string | null;
+    /** Character count of the main region's text content, or null. */
+    main_content_word_count: number | null;
+    /** Character count of `document.body`'s text content, or null. */
+    main_content_body_word_count: number | null;
+    /** Number of headings within the main region, or null. */
+    main_content_heading_count: number | null;
+    /** Number of images within the main region, or null. */
+    main_content_image_count: number | null;
+    /** Number of tables within the main region, or null. */
+    main_content_table_count: number | null;
+    /** Number of button-like elements within the main region, or null. */
+    main_content_button_count: number | null;
+    /** Number of iframes within the main region, or null. */
+    main_content_iframe_count: number | null;
+    /** Number of videos within the main region, or null. */
+    main_content_video_count: number | null;
+    /** Number of audios within the main region, or null. */
+    main_content_audio_count: number | null;
+    /** Number of canvases within the main region, or null. */
+    main_content_canvas_count: number | null;
+    /** `document.body.scrollHeight` at the desktop-compact preset, or null. */
+    scroll_height_desktop: number | null;
+    /** `document.body.scrollHeight` at the mobile-small preset, or null. */
+    scroll_height_mobile: number | null;
     /** JSON-serialised nested Meta sub-objects not flattened above. */
     meta_extras: string | null;
     /** JSON-serialized network logs captured during scraping, or null if not collected. */

package/lib/archive/url-alias/compute-tier-a-alias-key.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Computes the Tier A alias key for `url`: two URLs share a Tier A key when
+ * they differ only by scheme (`http` vs `https` are folded together),
+ * host letter-casing, or a `/index.{ext}` path suffix — differences that
+ * are safe to treat as "the same resource" from the URL string alone, with
+ * no need to compare rendered content.
+ *
+ * This is a canonical-key function, not a pairwise comparator: grouping
+ * rows by exact string equality of this function's output is definitionally
+ * an equivalence relation (reflexive, symmetric, transitive), unlike
+ * `compareUrlSortKeys`'s pairwise "roughly equal" comparison (see
+ * ARCHITECTURE.md's "URL natural-sort comparator は推移律を保証しない" —
+ * that non-transitivity problem is inherent to pairwise comparators, and
+ * does not apply to a canonical-key partition like this one).
+ *
+ * Deliberately does not touch the query string or fragment: a query-string
+ * difference is not addressed by this feature (out of scope — see the
+ * `body_hash` masking, a separate content-hash-based mechanism, for
+ * detecting meaningless-parameter duplicates), and fragments are
+ * client-side only and never affect server-rendered content.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier A key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierAAliasKey('http://Example.com/about/index.html');
+ * // 'example.com/about/'
+ * computeTierAAliasKey('https://example.com/about/');
+ * // 'example.com/about/' -- same key: scheme and /index.html are folded
+ * ```
+ */
+export declare function computeTierAAliasKey(url: string): string | null;

package/lib/archive/url-alias/compute-tier-a-alias-key.js

@@ -0,0 +1,36 @@
+import { formatAliasKey } from './format-alias-key.js';
+import { parseAliasKeyParts } from './parse-alias-key-parts.js';
+/**
+ * Computes the Tier A alias key for `url`: two URLs share a Tier A key when
+ * they differ only by scheme (`http` vs `https` are folded together),
+ * host letter-casing, or a `/index.{ext}` path suffix — differences that
+ * are safe to treat as "the same resource" from the URL string alone, with
+ * no need to compare rendered content.
+ *
+ * This is a canonical-key function, not a pairwise comparator: grouping
+ * rows by exact string equality of this function's output is definitionally
+ * an equivalence relation (reflexive, symmetric, transitive), unlike
+ * `compareUrlSortKeys`'s pairwise "roughly equal" comparison (see
+ * ARCHITECTURE.md's "URL natural-sort comparator は推移律を保証しない" —
+ * that non-transitivity problem is inherent to pairwise comparators, and
+ * does not apply to a canonical-key partition like this one).
+ *
+ * Deliberately does not touch the query string or fragment: a query-string
+ * difference is not addressed by this feature (out of scope — see the
+ * `body_hash` masking, a separate content-hash-based mechanism, for
+ * detecting meaningless-parameter duplicates), and fragments are
+ * client-side only and never affect server-rendered content.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier A key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierAAliasKey('http://Example.com/about/index.html');
+ * // 'example.com/about/'
+ * computeTierAAliasKey('https://example.com/about/');
+ * // 'example.com/about/' -- same key: scheme and /index.html are folded
+ * ```
+ */
+export function computeTierAAliasKey(url) {
+    const parts = parseAliasKeyParts(url);
+    return parts ? formatAliasKey(parts) : null;
+}

package/lib/archive/url-alias/compute-tier-b-alias-key.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Computes the Tier B alias key for `url`: like `computeTierAAliasKey`,
+ * but additionally strips one trailing `/` from the path (the bare root
+ * path `/` is left alone). Two URLs sharing a Tier B key differ only by
+ * trailing-slash presence — a resource-identity signal considerably weaker
+ * than Tier A's (a web server can legitimately serve different content at
+ * `/foo` and `/foo/`), so callers must additionally require a matching
+ * `page_meta.body_hash` before treating a Tier B match as the same page.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier B key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierBAliasKey('https://example.com/foo');
+ * computeTierBAliasKey('https://example.com/foo/');
+ * // both: 'example.com/foo' -- same key, but the caller must still confirm
+ * // a matching body_hash before merging these as the same page.
+ * ```
+ */
+export declare function computeTierBAliasKey(url: string): string | null;

package/lib/archive/url-alias/compute-tier-b-alias-key.js

@@ -0,0 +1,30 @@
+import { formatAliasKey } from './format-alias-key.js';
+import { parseAliasKeyParts } from './parse-alias-key-parts.js';
+/**
+ * Computes the Tier B alias key for `url`: like `computeTierAAliasKey`,
+ * but additionally strips one trailing `/` from the path (the bare root
+ * path `/` is left alone). Two URLs sharing a Tier B key differ only by
+ * trailing-slash presence — a resource-identity signal considerably weaker
+ * than Tier A's (a web server can legitimately serve different content at
+ * `/foo` and `/foo/`), so callers must additionally require a matching
+ * `page_meta.body_hash` before treating a Tier B match as the same page.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier B key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierBAliasKey('https://example.com/foo');
+ * computeTierBAliasKey('https://example.com/foo/');
+ * // both: 'example.com/foo' -- same key, but the caller must still confirm
+ * // a matching body_hash before merging these as the same page.
+ * ```
+ */
+export function computeTierBAliasKey(url) {
+    const parts = parseAliasKeyParts(url);
+    if (!parts) {
+        return null;
+    }
+    const path = parts.path.length > 1 && parts.path.endsWith('/')
+        ? parts.path.slice(0, -1)
+        : parts.path;
+    return formatAliasKey({ ...parts, path });
+}

package/lib/archive/url-alias/format-alias-key.d.ts

@@ -0,0 +1,13 @@
+import type { AliasKeyParts } from './types.js';
+/**
+ * Formats parsed alias-key parts into a single string key. `host`/`port`
+ * never contain `/` or `?`, and `path`/`search` always start with those
+ * characters respectively (or `search` is empty), so concatenation without
+ * an explicit delimiter is unambiguous.
+ * @param parts - The parts to format.
+ * @returns The formatted key string.
+ * @example
+ * formatAliasKey({ host: 'example.com', port: '', path: '/about/', search: '' });
+ * // 'example.com/about/'
+ */
+export declare function formatAliasKey(parts: AliasKeyParts): string;

package/lib/archive/url-alias/format-alias-key.js

@@ -0,0 +1,14 @@
+/**
+ * Formats parsed alias-key parts into a single string key. `host`/`port`
+ * never contain `/` or `?`, and `path`/`search` always start with those
+ * characters respectively (or `search` is empty), so concatenation without
+ * an explicit delimiter is unambiguous.
+ * @param parts - The parts to format.
+ * @returns The formatted key string.
+ * @example
+ * formatAliasKey({ host: 'example.com', port: '', path: '/about/', search: '' });
+ * // 'example.com/about/'
+ */
+export function formatAliasKey(parts) {
+    return `${parts.host}${parts.port ? `:${parts.port}` : ''}${parts.path}${parts.search}`;
+}

package/lib/archive/url-alias/parse-alias-key-parts.d.ts

@@ -0,0 +1,15 @@
+import type { AliasKeyParts } from './types.js';
+/**
+ * Parses `url` and extracts the components Tier A / Tier B keys are built
+ * from. Returns `null` for anything that isn't a parseable `http`/`https`
+ * URL — such a row is defensively excluded from alias candidacy entirely
+ * (a non-http(s) URL should not occur for anything the crawler itself
+ * stored, since every `url_refs.url` was already parsed once during
+ * crawling).
+ * @param url - The URL string to parse.
+ * @returns The extracted parts, or `null` if unparseable / not http(s).
+ * @example
+ * parseAliasKeyParts('https://Example.com/about/index.html');
+ * // { host: 'example.com', port: '', path: '/about/', search: '' }
+ */
+export declare function parseAliasKeyParts(url: string): AliasKeyParts | null;

package/lib/archive/url-alias/parse-alias-key-parts.js

@@ -0,0 +1,40 @@
+const INDEX_SUFFIX_PATTERN = /\/index\.\w+$/i;
+/**
+ * Parses `url` and extracts the components Tier A / Tier B keys are built
+ * from. Returns `null` for anything that isn't a parseable `http`/`https`
+ * URL — such a row is defensively excluded from alias candidacy entirely
+ * (a non-http(s) URL should not occur for anything the crawler itself
+ * stored, since every `url_refs.url` was already parsed once during
+ * crawling).
+ * @param url - The URL string to parse.
+ * @returns The extracted parts, or `null` if unparseable / not http(s).
+ * @example
+ * parseAliasKeyParts('https://Example.com/about/index.html');
+ * // { host: 'example.com', port: '', path: '/about/', search: '' }
+ */
+export function parseAliasKeyParts(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+        return null;
+    }
+    return {
+        // WHATWG URL already lowercases `hostname` during parsing; the
+        // explicit `.toLowerCase()` documents that intent rather than
+        // relying on it silently.
+        host: parsed.hostname.toLowerCase(),
+        // Empty string when the port is the scheme's default (or
+        // unspecified) — this is why `http://example.com/` and
+        // `https://example.com/` fold to the same key below (both have
+        // `port === ''`) while an explicit non-default port on either
+        // scheme is preserved and kept distinct.
+        port: parsed.port,
+        path: parsed.pathname.replace(INDEX_SUFFIX_PATTERN, '/'),
+        search: parsed.search,
+    };
+}

package/lib/archive/url-alias/types.d.ts

@@ -0,0 +1,7 @@
+/** Components a Tier A / Tier B alias key is built from. */
+export interface AliasKeyParts {
+    host: string;
+    port: string;
+    path: string;
+    search: string;
+}

package/lib/archive/url-alias/types.js

@@ -0,0 +1 @@
+export {};

package/lib/crawler/choose-probe-host.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Pick the hostname a network-outage probe should target.
+ *
+ * Prefers a host that has already answered successfully during this crawl
+ * session (`Crawler.#successfulHosts` — the same "proven alive" evidence
+ * `shouldBurnHost` uses) over a root URL's hostname, and never falls back to
+ * a hardcoded external address (e.g. `1.1.1.1`): the probe should depend
+ * only on infrastructure the crawl is already touching. `ReadonlySet`
+ * iteration order in JS is insertion order, so this deterministically picks
+ * the first host to have succeeded this session.
+ * @param successfulHosts - Hostnames observed to respond in this session.
+ * @param roots - The crawl's configured root URLs (`CrawlerOptions.roots`),
+ *   used as a fallback before any host has succeeded yet.
+ * @returns A probe target hostname, or `null` if neither source yields one
+ *   (e.g. a fresh session with a malformed/empty roots list and no
+ *   successes yet — the caller should treat this as "cannot probe").
+ * @example
+ * ```ts
+ * chooseProbeHost(new Set(['a.example']), ['https://b.example/']); // 'a.example'
+ * chooseProbeHost(new Set(), ['https://b.example/']); // 'b.example'
+ * chooseProbeHost(new Set(), []); // null
+ * ```
+ */
+export declare function chooseProbeHost(successfulHosts: ReadonlySet<string>, roots: readonly string[]): string | null;

package/lib/crawler/choose-probe-host.js

@@ -0,0 +1,38 @@
+/**
+ * Pick the hostname a network-outage probe should target.
+ *
+ * Prefers a host that has already answered successfully during this crawl
+ * session (`Crawler.#successfulHosts` — the same "proven alive" evidence
+ * `shouldBurnHost` uses) over a root URL's hostname, and never falls back to
+ * a hardcoded external address (e.g. `1.1.1.1`): the probe should depend
+ * only on infrastructure the crawl is already touching. `ReadonlySet`
+ * iteration order in JS is insertion order, so this deterministically picks
+ * the first host to have succeeded this session.
+ * @param successfulHosts - Hostnames observed to respond in this session.
+ * @param roots - The crawl's configured root URLs (`CrawlerOptions.roots`),
+ *   used as a fallback before any host has succeeded yet.
+ * @returns A probe target hostname, or `null` if neither source yields one
+ *   (e.g. a fresh session with a malformed/empty roots list and no
+ *   successes yet — the caller should treat this as "cannot probe").
+ * @example
+ * ```ts
+ * chooseProbeHost(new Set(['a.example']), ['https://b.example/']); // 'a.example'
+ * chooseProbeHost(new Set(), ['https://b.example/']); // 'b.example'
+ * chooseProbeHost(new Set(), []); // null
+ * ```
+ */
+export function chooseProbeHost(successfulHosts, roots) {
+    const [firstSuccessfulHost] = successfulHosts;
+    if (firstSuccessfulHost !== undefined) {
+        return firstSuccessfulHost;
+    }
+    for (const root of roots) {
+        try {
+            return new URL(root).hostname;
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}

package/lib/crawler/clear-dns-burned-host-cache.js

@@ -1,3 +1,4 @@
+import { dnsBurnedHostBurnTimestamps } from './dns-burned-host-burn-timestamps.js';
 import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
 /**
@@ -8,4 +9,5 @@ import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circui
 export function clearDnsBurnedHostCache() {
     dnsBurnedHostCache.clear();
     dnsBurnedHostShortCircuitCounter.count = 0;
+    dnsBurnedHostBurnTimestamps.clear();
 }