@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/archive/migrate-content-items-alias-of-id.js

@@ -0,0 +1,58 @@
+/**
+ * Adds the `content_items.alias_of_id` column to archives created before
+ * this feature, then ensures its index exists.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit — the same
+ * self-referencing shape as `redirect_dest_id` (see
+ * `create-entity-tables.ts`'s JSDoc on why that FK must be deferred: a
+ * lower-id row can reference a higher-id row within the same write).
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `alias_of_id` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `alias_of_id` at that point; an unconditional
+ * `CREATE INDEX ... alias_of_id` there would fail with `no such column`
+ * before this migration ever runs. This function is the one place
+ * guaranteed to run only after the column is confirmed present, for both
+ * archive kinds.
+ *
+ * Only adds the column and its index — it does not compute values for
+ * existing rows (they stay `NULL`). That computation runs separately, from
+ * `backfillAliasOfId` during a viewer-read-model build, since it requires
+ * comparing every page's title and (for the trailing-slash tier) its
+ * `body_hash` against every other page.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `content_items`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateContentItemsAliasOfId(instance) {
+    const hasContentItems = await instance.schema.hasTable('content_items');
+    if (!hasContentItems) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('content_items', 'alias_of_id');
+    if (!hasColumn) {
+        await instance.raw('ALTER TABLE content_items ADD COLUMN alias_of_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED');
+        // eslint-disable-next-line no-console
+        console.error('[migrate] content_items.alias_of_id column added');
+    }
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_alias_of_id ON content_items(alias_of_id)');
+}

package/lib/archive/migrate-inventory-runs-invalid-skipped.d.ts

@@ -0,0 +1,11 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `inventory_runs.invalid_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInventoryRunsInvalidSkipped(instance: Knex): Promise<void>;

package/lib/archive/migrate-inventory-runs-invalid-skipped.js

@@ -0,0 +1,24 @@
+/**
+ * Adds the `inventory_runs.invalid_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInventoryRunsInvalidSkipped(instance) {
+    const hasTable = await instance.schema.hasTable('inventory_runs');
+    if (!hasTable) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('inventory_runs', 'invalid_skipped');
+    if (hasColumn) {
+        return;
+    }
+    await instance.schema.table('inventory_runs', (t) => {
+        t.integer('invalid_skipped');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] inventory_runs.invalid_skipped column added');
+}

package/lib/archive/migrate-page-meta-body-hash.d.ts

@@ -0,0 +1,39 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `page_meta.body_hash` column to archives created before this
+ * feature, then ensures its index exists.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-main-contents-columns.js').migrateMainContentsColumns}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `body_hash` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `body_hash` at that point; an unconditional
+ * `CREATE INDEX ... body_hash` there would fail with `no such column` before
+ * this migration ever runs. This function is the one place guaranteed to
+ * run only after the column is confirmed present, for both archive kinds.
+ *
+ * Only adds the column and its index — it does not backfill values for
+ * existing rows (they stay `NULL`). That backfill runs separately, from
+ * `backfillBodyHashFromHtmlBlobs` during a viewer-read-model build, since it
+ * requires decompressing every page's stored HTML and is too heavy to run on
+ * every archive open.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `page_meta`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migratePageMetaBodyHash(instance: Knex): Promise<void>;

package/lib/archive/migrate-page-meta-body-hash.js

@@ -0,0 +1,52 @@
+/**
+ * Adds the `page_meta.body_hash` column to archives created before this
+ * feature, then ensures its index exists.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-main-contents-columns.js').migrateMainContentsColumns}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `body_hash` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `body_hash` at that point; an unconditional
+ * `CREATE INDEX ... body_hash` there would fail with `no such column` before
+ * this migration ever runs. This function is the one place guaranteed to
+ * run only after the column is confirmed present, for both archive kinds.
+ *
+ * Only adds the column and its index — it does not backfill values for
+ * existing rows (they stay `NULL`). That backfill runs separately, from
+ * `backfillBodyHashFromHtmlBlobs` during a viewer-read-model build, since it
+ * requires decompressing every page's stored HTML and is too heavy to run on
+ * every archive open.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `page_meta`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migratePageMetaBodyHash(instance) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'body_hash');
+    if (!hasColumn) {
+        await instance.schema.table('page_meta', (t) => {
+            t.binary('body_hash');
+        });
+        // eslint-disable-next-line no-console
+        console.error('[migrate] page_meta.body_hash column added');
+    }
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_body_hash ON page_meta(body_hash)');
+}

package/lib/archive/migrate-page-meta-console-error-count.d.ts

@@ -0,0 +1,22 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `page_meta.console_error_count` column to archives created
+ * before this feature (issue #228).
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same
+ * catch-up shape as {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}.
+ *
+ * Unlike `body_hash`, there is no separate backfill step: a page that has
+ * never been re-scraped since this feature shipped has no
+ * `page_console_logs` rows either, so `0` (the value `replaceConsoleLogs`
+ * writes for a page with no error/pageerror entries) is already the
+ * correct value, not a placeholder pending backfill.
+ *
+ * Idempotent: adding the column is a no-op once it exists.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migratePageMetaConsoleErrorCount(instance: Knex): Promise<void>;

package/lib/archive/migrate-page-meta-console-error-count.js

@@ -0,0 +1,34 @@
+/**
+ * Adds the `page_meta.console_error_count` column to archives created
+ * before this feature (issue #228).
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same
+ * catch-up shape as {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}.
+ *
+ * Unlike `body_hash`, there is no separate backfill step: a page that has
+ * never been re-scraped since this feature shipped has no
+ * `page_console_logs` rows either, so `0` (the value `replaceConsoleLogs`
+ * writes for a page with no error/pageerror entries) is already the
+ * correct value, not a placeholder pending backfill.
+ *
+ * Idempotent: adding the column is a no-op once it exists.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migratePageMetaConsoleErrorCount(instance) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'console_error_count');
+    if (!hasColumn) {
+        await instance.schema.table('page_meta', (t) => {
+            t.integer('console_error_count');
+        });
+        // eslint-disable-next-line no-console
+        console.error('[migrate] page_meta.console_error_count column added');
+    }
+}

package/lib/archive/page.d.ts

@@ -406,6 +406,7 @@ export default class Page {
         isInternalPage: boolean;
         getAnchors: Anchor[];
         getReferrers: Referrer[];
+        dir: string | number | null;
         title: string | number | null;
         main_content_word_count: string | number | null;
         main_content_node_name: string | number | null;
@@ -425,7 +426,6 @@ export default class Page {
         scroll_height_desktop: string | number | null;
         scroll_height_mobile: string | number | null;
         lang: string | number | null;
-        dir: string | number | null;
         charset: string | number | null;
         baseHref: string | number | null;
         viewport_raw: string | number | null;

package/lib/archive/types.d.ts

@@ -128,9 +128,9 @@ export interface InventoryRunMeta {
     ran_at: string;
     /** Human-readable identifier (e.g. `'prod-2026-06-21'`). `null` when the caller did not supply one. */
     list_label?: string | null;
-    /** SHA-256 hex digest of the source file. `null` if hashing failed (e.g. file vanished mid-run). */
+    /** SHA-256 hex digest of the source file. `null` for programmatic callers that built the URL list in-memory (no source file to hash). */
     source_file_sha256?: string | null;
-    /** Number of non-empty lines in the input list (= URL count before scope filtering). */
+    /** Number of valid URLs in the input list, after the CLI has warned-and-dropped unparseable-URL lines but before scope filtering. */
     total_lines?: number | null;
     /** Number of new HTML seeds inserted as `content_items` rows by this run. */
     new_pages?: number | null;
@@ -138,9 +138,67 @@ export interface InventoryRunMeta {
     new_resources?: number | null;
     /** Number of input URLs dropped because they fell outside the archived scope. */
     scope_skipped?: number | null;
+    /** Number of source-file lines dropped by the CLI for failing URL validation, before this row's `total_lines` was counted. `null` for programmatic callers that built the URL list in-memory (no source file to have invalid lines). */
+    invalid_skipped?: number | null;
     /** Free-form text for backfill annotations or operator notes. */
     notes?: string | null;
 }
+/**
+ * A row in `network_outages` — one detected operator-network outage.
+ *
+ * Append-only except `ended_at`: it is written once, `NULL`, when the
+ * outage is first detected, and updated exactly once when a recovery probe
+ * succeeds. A row can also be left `ended_at = NULL` forever if the crawl
+ * process is killed mid-outage — readers must resolve this via a clamp
+ * (see `is-within-outage-window.ts` and the `db-ops/outages/` writer that
+ * closes stale-open rows on the next writer session), never by treating
+ * `NULL` as an unbounded window.
+ */
+export interface NetworkOutageRow {
+    id: number;
+    /** Epoch ms, backdated to the earliest error still inside the detector's sliding window at trigger time. */
+    started_at: number;
+    /** Epoch ms the sliding window actually crossed both thresholds. */
+    detected_at: number;
+    /** Epoch ms the recovery probe first succeeded, or `null` while still open / if the session crashed before recovery. */
+    ended_at: number | null;
+    /** Hostname the recovery probe targeted, or `null` if none was available (see `choose-probe-host.ts`). */
+    probe_host: string | null;
+    /** Error count in the detector's window at trigger time. */
+    trigger_error_count: number;
+    /** Distinct host count in the detector's window at trigger time. */
+    trigger_host_count: number;
+}
+/**
+ * Fields required to record a newly-detected outage via
+ * `Database.insertNetworkOutage`. camelCase (unlike {@link NetworkOutageRow}
+ * / {@link InventoryRunMeta}) because callers build this directly from
+ * `NetworkOutageDetector`'s camelCase `OutageSuspect` plus a probe host —
+ * the db-op does the camelCase → snake_case column mapping on write.
+ */
+export interface InsertNetworkOutageParams {
+    /** Backdated to the earliest error still inside the detector's window at trigger time. */
+    startedAt: number;
+    /** When the sliding window actually crossed both thresholds. */
+    detectedAt: number;
+    /** Hostname the recovery probe will target, or `null` if none was available. */
+    probeHost: string | null;
+    triggerErrorCount: number;
+    triggerHostCount: number;
+}
+/**
+ * Fields required to record a newly-capped URL shape via
+ * `Database.insertDedupeCapEvent`. camelCase, mapped to snake_case columns
+ * on write — same convention as {@link InsertNetworkOutageParams}.
+ */
+export interface InsertDedupeCapEventParams {
+    shapeKey: string;
+    sampleUrl: string;
+    bodyHash: Buffer;
+    effectiveThreshold: number;
+    observedCount: number;
+    detectedAt: number;
+}
 /**
  * Filter type for querying pages from the database.
  *

package/lib/archive/url-alias/compute-tier-a-alias-key.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Computes the Tier A alias key for `url`: two URLs share a Tier A key when
+ * they differ only by scheme (`http` vs `https` are folded together),
+ * host letter-casing, or a `/index.{ext}` path suffix — differences that
+ * are safe to treat as "the same resource" from the URL string alone, with
+ * no need to compare rendered content.
+ *
+ * This is a canonical-key function, not a pairwise comparator: grouping
+ * rows by exact string equality of this function's output is definitionally
+ * an equivalence relation (reflexive, symmetric, transitive), unlike
+ * `compareUrlSortKeys`'s pairwise "roughly equal" comparison (see
+ * ARCHITECTURE.md's "URL natural-sort comparator は推移律を保証しない" —
+ * that non-transitivity problem is inherent to pairwise comparators, and
+ * does not apply to a canonical-key partition like this one).
+ *
+ * Deliberately does not touch the query string or fragment: a query-string
+ * difference is not addressed by this feature (out of scope — see the
+ * `body_hash` masking, a separate content-hash-based mechanism, for
+ * detecting meaningless-parameter duplicates), and fragments are
+ * client-side only and never affect server-rendered content.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier A key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierAAliasKey('http://Example.com/about/index.html');
+ * // 'example.com/about/'
+ * computeTierAAliasKey('https://example.com/about/');
+ * // 'example.com/about/' -- same key: scheme and /index.html are folded
+ * ```
+ */
+export declare function computeTierAAliasKey(url: string): string | null;

package/lib/archive/url-alias/compute-tier-a-alias-key.js

@@ -0,0 +1,36 @@
+import { formatAliasKey } from './format-alias-key.js';
+import { parseAliasKeyParts } from './parse-alias-key-parts.js';
+/**
+ * Computes the Tier A alias key for `url`: two URLs share a Tier A key when
+ * they differ only by scheme (`http` vs `https` are folded together),
+ * host letter-casing, or a `/index.{ext}` path suffix — differences that
+ * are safe to treat as "the same resource" from the URL string alone, with
+ * no need to compare rendered content.
+ *
+ * This is a canonical-key function, not a pairwise comparator: grouping
+ * rows by exact string equality of this function's output is definitionally
+ * an equivalence relation (reflexive, symmetric, transitive), unlike
+ * `compareUrlSortKeys`'s pairwise "roughly equal" comparison (see
+ * ARCHITECTURE.md's "URL natural-sort comparator は推移律を保証しない" —
+ * that non-transitivity problem is inherent to pairwise comparators, and
+ * does not apply to a canonical-key partition like this one).
+ *
+ * Deliberately does not touch the query string or fragment: a query-string
+ * difference is not addressed by this feature (out of scope — see the
+ * `body_hash` masking, a separate content-hash-based mechanism, for
+ * detecting meaningless-parameter duplicates), and fragments are
+ * client-side only and never affect server-rendered content.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier A key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierAAliasKey('http://Example.com/about/index.html');
+ * // 'example.com/about/'
+ * computeTierAAliasKey('https://example.com/about/');
+ * // 'example.com/about/' -- same key: scheme and /index.html are folded
+ * ```
+ */
+export function computeTierAAliasKey(url) {
+    const parts = parseAliasKeyParts(url);
+    return parts ? formatAliasKey(parts) : null;
+}

package/lib/archive/url-alias/compute-tier-b-alias-key.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Computes the Tier B alias key for `url`: like `computeTierAAliasKey`,
+ * but additionally strips one trailing `/` from the path (the bare root
+ * path `/` is left alone). Two URLs sharing a Tier B key differ only by
+ * trailing-slash presence — a resource-identity signal considerably weaker
+ * than Tier A's (a web server can legitimately serve different content at
+ * `/foo` and `/foo/`), so callers must additionally require a matching
+ * `page_meta.body_hash` before treating a Tier B match as the same page.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier B key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierBAliasKey('https://example.com/foo');
+ * computeTierBAliasKey('https://example.com/foo/');
+ * // both: 'example.com/foo' -- same key, but the caller must still confirm
+ * // a matching body_hash before merging these as the same page.
+ * ```
+ */
+export declare function computeTierBAliasKey(url: string): string | null;

package/lib/archive/url-alias/compute-tier-b-alias-key.js

@@ -0,0 +1,30 @@
+import { formatAliasKey } from './format-alias-key.js';
+import { parseAliasKeyParts } from './parse-alias-key-parts.js';
+/**
+ * Computes the Tier B alias key for `url`: like `computeTierAAliasKey`,
+ * but additionally strips one trailing `/` from the path (the bare root
+ * path `/` is left alone). Two URLs sharing a Tier B key differ only by
+ * trailing-slash presence — a resource-identity signal considerably weaker
+ * than Tier A's (a web server can legitimately serve different content at
+ * `/foo` and `/foo/`), so callers must additionally require a matching
+ * `page_meta.body_hash` before treating a Tier B match as the same page.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier B key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierBAliasKey('https://example.com/foo');
+ * computeTierBAliasKey('https://example.com/foo/');
+ * // both: 'example.com/foo' -- same key, but the caller must still confirm
+ * // a matching body_hash before merging these as the same page.
+ * ```
+ */
+export function computeTierBAliasKey(url) {
+    const parts = parseAliasKeyParts(url);
+    if (!parts) {
+        return null;
+    }
+    const path = parts.path.length > 1 && parts.path.endsWith('/')
+        ? parts.path.slice(0, -1)
+        : parts.path;
+    return formatAliasKey({ ...parts, path });
+}

package/lib/archive/url-alias/format-alias-key.d.ts

@@ -0,0 +1,13 @@
+import type { AliasKeyParts } from './types.js';
+/**
+ * Formats parsed alias-key parts into a single string key. `host`/`port`
+ * never contain `/` or `?`, and `path`/`search` always start with those
+ * characters respectively (or `search` is empty), so concatenation without
+ * an explicit delimiter is unambiguous.
+ * @param parts - The parts to format.
+ * @returns The formatted key string.
+ * @example
+ * formatAliasKey({ host: 'example.com', port: '', path: '/about/', search: '' });
+ * // 'example.com/about/'
+ */
+export declare function formatAliasKey(parts: AliasKeyParts): string;

package/lib/archive/url-alias/format-alias-key.js

@@ -0,0 +1,14 @@
+/**
+ * Formats parsed alias-key parts into a single string key. `host`/`port`
+ * never contain `/` or `?`, and `path`/`search` always start with those
+ * characters respectively (or `search` is empty), so concatenation without
+ * an explicit delimiter is unambiguous.
+ * @param parts - The parts to format.
+ * @returns The formatted key string.
+ * @example
+ * formatAliasKey({ host: 'example.com', port: '', path: '/about/', search: '' });
+ * // 'example.com/about/'
+ */
+export function formatAliasKey(parts) {
+    return `${parts.host}${parts.port ? `:${parts.port}` : ''}${parts.path}${parts.search}`;
+}

package/lib/archive/url-alias/parse-alias-key-parts.d.ts

@@ -0,0 +1,15 @@
+import type { AliasKeyParts } from './types.js';
+/**
+ * Parses `url` and extracts the components Tier A / Tier B keys are built
+ * from. Returns `null` for anything that isn't a parseable `http`/`https`
+ * URL — such a row is defensively excluded from alias candidacy entirely
+ * (a non-http(s) URL should not occur for anything the crawler itself
+ * stored, since every `url_refs.url` was already parsed once during
+ * crawling).
+ * @param url - The URL string to parse.
+ * @returns The extracted parts, or `null` if unparseable / not http(s).
+ * @example
+ * parseAliasKeyParts('https://Example.com/about/index.html');
+ * // { host: 'example.com', port: '', path: '/about/', search: '' }
+ */
+export declare function parseAliasKeyParts(url: string): AliasKeyParts | null;

package/lib/archive/url-alias/parse-alias-key-parts.js

@@ -0,0 +1,40 @@
+const INDEX_SUFFIX_PATTERN = /\/index\.\w+$/i;
+/**
+ * Parses `url` and extracts the components Tier A / Tier B keys are built
+ * from. Returns `null` for anything that isn't a parseable `http`/`https`
+ * URL — such a row is defensively excluded from alias candidacy entirely
+ * (a non-http(s) URL should not occur for anything the crawler itself
+ * stored, since every `url_refs.url` was already parsed once during
+ * crawling).
+ * @param url - The URL string to parse.
+ * @returns The extracted parts, or `null` if unparseable / not http(s).
+ * @example
+ * parseAliasKeyParts('https://Example.com/about/index.html');
+ * // { host: 'example.com', port: '', path: '/about/', search: '' }
+ */
+export function parseAliasKeyParts(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+        return null;
+    }
+    return {
+        // WHATWG URL already lowercases `hostname` during parsing; the
+        // explicit `.toLowerCase()` documents that intent rather than
+        // relying on it silently.
+        host: parsed.hostname.toLowerCase(),
+        // Empty string when the port is the scheme's default (or
+        // unspecified) — this is why `http://example.com/` and
+        // `https://example.com/` fold to the same key below (both have
+        // `port === ''`) while an explicit non-default port on either
+        // scheme is preserved and kept distinct.
+        port: parsed.port,
+        path: parsed.pathname.replace(INDEX_SUFFIX_PATTERN, '/'),
+        search: parsed.search,
+    };
+}

package/lib/archive/url-alias/types.d.ts

@@ -0,0 +1,7 @@
+/** Components a Tier A / Tier B alias key is built from. */
+export interface AliasKeyParts {
+    host: string;
+    port: string;
+    path: string;
+    search: string;
+}

package/lib/archive/url-alias/types.js

@@ -0,0 +1 @@
+export {};

package/lib/classify-error-kind.d.ts

@@ -14,6 +14,7 @@ import type { ErrorKind } from './types.js';
  * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
  * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
  * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * classifyErrorKind('Maximum number of redirects exceeded'); // 'redirect-loop'
  * ```
  */
 export declare function classifyErrorKind(message: string): ErrorKind;

package/lib/classify-error-kind.js

@@ -53,6 +53,19 @@ const MATCHERS = [
         kind: 'parse-error',
         pattern: /Parse Error|Expected HTTP\/|Unexpected end of stream/i,
     },
+    // `follow-redirects` (pinned at 1.16.0, `fetch-destination.ts`'s HEAD/GET
+    // pre-flight) throws exactly "Maximum number of redirects exceeded" —
+    // no cause token, no URL — when a chain never terminates within its
+    // `maxRedirects` budget. `ERR_TOO_MANY_REDIRECTS` is the equivalent
+    // Chromium net-error code, covering the same symptom surfaced through a
+    // puppeteer navigation instead of the Node HTTP client. Both mean the
+    // SAME thing: the site's own redirect configuration loops and will loop
+    // again on any future fetch, which is why this is deterministic (not a
+    // transient network condition) — see `PERMANENT_ERROR_KINDS`.
+    {
+        kind: 'redirect-loop',
+        pattern: /Maximum number of redirects exceeded|ERR_TOO_MANY_REDIRECTS/i,
+    },
     // `client-blocked` covers Chromium's ERR_BLOCKED_* family — the browser
     // actively decided to reject the request (ad/tracker heuristics, CSP,
     // CORB / ORB, administrator block list, fingerprinting protection,
@@ -110,6 +123,7 @@ const MATCHERS = [
  * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
  * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
  * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * classifyErrorKind('Maximum number of redirects exceeded'); // 'redirect-loop'
  * ```
  */
 export function classifyErrorKind(message) {

package/lib/crawler/assert-chrome-installed.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Verifies that Puppeteer can resolve an installed Chrome/Chromium executable
+ * before a crawl starts.
+ *
+ * A crawl otherwise only discovers a missing browser deep inside the
+ * per-URL scrape loop (`Crawler#_launchBrowserAndScrape`), where it surfaces
+ * as one more scrape error among many — the CLI still prints "Crawl
+ * completed" and writes an archive, so a missing Chrome (a fatal
+ * precondition, not a per-page failure) is easy to miss. Calling this once,
+ * before any archive I/O begins, turns it into an immediate, actionable
+ * failure instead.
+ * @param executablePath - Explicit override, matching
+ *   {@link CrawlerOptions.executablePath}. Pass `null` (or omit) to check
+ *   Puppeteer's own pinned Chrome resolution instead.
+ * @throws {Error} When the resolved executable path does not exist on disk.
+ * @example
+ * ```ts
+ * import { assertChromeIsInstalled } from '@nitpicker/crawler';
+ *
+ * // Throws with install instructions before any crawl work starts.
+ * await assertChromeIsInstalled();
+ * ```
+ */
+export declare function assertChromeIsInstalled(executablePath?: string | null): Promise<void>;