@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/populate-ref-tables/compute-header-flags.js

@@ -0,0 +1,70 @@
+/**
+ * Computes the {@link HeaderFlagsRow} for one decomposed header set.
+ *
+ * Each `has_*` boolean is a straight "is this name present" check on the
+ * decomposed entries — semantically equivalent to `checkHeaders` /
+ * `headerPresenceExpression` in `@nitpicker/query`, but expressed in JS
+ * over the already-normalized `HeaderEntry.name` (lower-cased) so no
+ * `LIKE` pattern is needed. This drops the JSON-key-shape false-positive
+ * risk that `headerPresenceExpression`'s SQL guard against (a value that
+ * happens to mention another header's name won't fire here because names
+ * come from the parsed JSON's KEYS, not from a substring scan).
+ * @param entries - Every decomposed entry for one `header_sets` row.
+ * @returns The row-shaped flags to insert into `header_flags`.
+ */
+export function computeHeaderFlags(entries) {
+    const cachePolicies = [];
+    let hasCsp = false;
+    let hasXFrameOptions = false;
+    let hasXContentTypeOptions = false;
+    let hasHsts = false;
+    let hasReferrerPolicy = false;
+    let hasPermissionsPolicy = false;
+    let hasSetCookie = false;
+    for (const entry of entries) {
+        switch (entry.name) {
+            case 'content-security-policy': {
+                hasCsp = true;
+                break;
+            }
+            case 'x-frame-options': {
+                hasXFrameOptions = true;
+                break;
+            }
+            case 'x-content-type-options': {
+                hasXContentTypeOptions = true;
+                break;
+            }
+            case 'strict-transport-security': {
+                hasHsts = true;
+                break;
+            }
+            case 'referrer-policy': {
+                hasReferrerPolicy = true;
+                break;
+            }
+            case 'permissions-policy': {
+                hasPermissionsPolicy = true;
+                break;
+            }
+            case 'set-cookie': {
+                hasSetCookie = true;
+                break;
+            }
+            case 'cache-control': {
+                cachePolicies.push(entry.value);
+                break;
+            }
+        }
+    }
+    return {
+        has_csp: hasCsp ? 1 : 0,
+        has_x_frame_options: hasXFrameOptions ? 1 : 0,
+        has_x_content_type_options: hasXContentTypeOptions ? 1 : 0,
+        has_hsts: hasHsts ? 1 : 0,
+        has_referrer_policy: hasReferrerPolicy ? 1 : 0,
+        has_permissions_policy: hasPermissionsPolicy ? 1 : 0,
+        has_set_cookie: hasSetCookie ? 1 : 0,
+        cache_policy: cachePolicies.length === 0 ? null : cachePolicies.join(', '),
+    };
+}

package/lib/archive/populate-ref-tables/content-type-rules.d.ts

@@ -0,0 +1,38 @@
+import type { ContentTypeCategory } from './types.js';
+/**
+ * Single matcher slot for a Content-Type rule. See `@nitpicker/query`'s
+ * `content-type-rules.ts` for the wider design discussion — this file only
+ * carries the subset required for JS-side classification (no SQL builders,
+ * which live in `@nitpicker/query`).
+ */
+type Matcher = {
+    readonly kind: 'exact';
+    readonly value: string;
+} | {
+    readonly kind: 'prefix';
+    readonly value: string;
+} | {
+    readonly kind: 'suffix';
+    readonly value: string;
+};
+/**
+ * One row of the classification table: attach a category to a set of
+ * matchers, all of which OR together.
+ */
+interface Rule {
+    readonly category: Exclude<ContentTypeCategory, 'other' | 'unknown'>;
+    readonly matchers: readonly Matcher[];
+}
+/**
+ * Content-Type classification rules, evaluated in array order — first match
+ * wins. Kept in lock-step with the source-of-truth table in
+ * `packages/@nitpicker/query/src/content-type-rules.ts`. Do not reorder
+ * without also editing the query-side copy (a drift would misroute
+ * `image/svg+xml` between `image` and `xml`, etc.).
+ *
+ * A future cleanup can lift the rule table into `@d-zero/shared` so both
+ * `@nitpicker/crawler` and `@nitpicker/query` import a single source of
+ * truth; that migration is out of 0.13's scope.
+ */
+export declare const CONTENT_TYPE_RULES: readonly Rule[];
+export {};

package/lib/archive/populate-ref-tables/content-type-rules.js

@@ -0,0 +1,133 @@
+/**
+ * Content-Type classification rules, evaluated in array order — first match
+ * wins. Kept in lock-step with the source-of-truth table in
+ * `packages/@nitpicker/query/src/content-type-rules.ts`. Do not reorder
+ * without also editing the query-side copy (a drift would misroute
+ * `image/svg+xml` between `image` and `xml`, etc.).
+ *
+ * A future cleanup can lift the rule table into `@d-zero/shared` so both
+ * `@nitpicker/crawler` and `@nitpicker/query` import a single source of
+ * truth; that migration is out of 0.13's scope.
+ */
+export const CONTENT_TYPE_RULES = [
+    {
+        category: 'html',
+        matchers: [
+            { kind: 'exact', value: 'text/html' },
+            { kind: 'exact', value: 'application/xhtml+xml' },
+        ],
+    },
+    {
+        category: 'pdf',
+        matchers: [{ kind: 'exact', value: 'application/pdf' }],
+    },
+    {
+        category: 'csv',
+        matchers: [
+            { kind: 'exact', value: 'text/csv' },
+            { kind: 'exact', value: 'application/csv' },
+            { kind: 'exact', value: 'application/x-csv' },
+            { kind: 'exact', value: 'text/tab-separated-values' },
+            { kind: 'exact', value: 'text/tsv' },
+        ],
+    },
+    {
+        category: 'word',
+        matchers: [
+            { kind: 'exact', value: 'application/msword' },
+            {
+                kind: 'exact',
+                value: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+            },
+        ],
+    },
+    {
+        category: 'excel',
+        matchers: [
+            { kind: 'exact', value: 'application/vnd.ms-excel' },
+            {
+                kind: 'exact',
+                value: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            },
+        ],
+    },
+    {
+        category: 'powerpoint',
+        matchers: [
+            { kind: 'exact', value: 'application/vnd.ms-powerpoint' },
+            {
+                kind: 'exact',
+                value: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+            },
+        ],
+    },
+    {
+        category: 'image',
+        matchers: [{ kind: 'prefix', value: 'image/' }],
+    },
+    {
+        category: 'audio',
+        matchers: [{ kind: 'prefix', value: 'audio/' }],
+    },
+    {
+        category: 'video',
+        matchers: [{ kind: 'prefix', value: 'video/' }],
+    },
+    {
+        category: 'font',
+        matchers: [
+            { kind: 'prefix', value: 'font/' },
+            { kind: 'prefix', value: 'application/font-' },
+            { kind: 'exact', value: 'application/vnd.ms-fontobject' },
+        ],
+    },
+    {
+        category: 'css',
+        matchers: [{ kind: 'exact', value: 'text/css' }],
+    },
+    {
+        category: 'javascript',
+        matchers: [
+            { kind: 'exact', value: 'text/javascript' },
+            { kind: 'exact', value: 'application/javascript' },
+            { kind: 'exact', value: 'application/x-javascript' },
+            { kind: 'exact', value: 'application/ecmascript' },
+        ],
+    },
+    {
+        category: 'json',
+        matchers: [
+            { kind: 'exact', value: 'application/json' },
+            { kind: 'exact', value: 'application/yaml' },
+            { kind: 'exact', value: 'application/x-yaml' },
+            { kind: 'exact', value: 'text/yaml' },
+            { kind: 'exact', value: 'text/x-yaml' },
+            { kind: 'suffix', value: '+json' },
+            { kind: 'suffix', value: '+yaml' },
+        ],
+    },
+    {
+        category: 'xml',
+        matchers: [
+            { kind: 'exact', value: 'application/xml' },
+            { kind: 'exact', value: 'text/xml' },
+            { kind: 'suffix', value: '+xml' },
+        ],
+    },
+    {
+        category: 'archive',
+        matchers: [
+            { kind: 'exact', value: 'application/zip' },
+            { kind: 'exact', value: 'application/gzip' },
+            { kind: 'exact', value: 'application/x-gzip' },
+            { kind: 'exact', value: 'application/x-tar' },
+            { kind: 'exact', value: 'application/x-7z-compressed' },
+            { kind: 'exact', value: 'application/x-rar-compressed' },
+            { kind: 'exact', value: 'application/octet-stream' },
+        ],
+    },
+    {
+        category: 'text',
+        matchers: [{ kind: 'prefix', value: 'text/' }],
+    },
+];

package/lib/archive/populate-ref-tables/create-header-table-caches.d.ts

@@ -0,0 +1,25 @@
+import type { HeaderTableCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Creates a {@link HeaderTableCaches} bundle warmed from the DB's current
+ * `header_name_refs` / `header_value_refs` / `header_sets` rows.
+ *
+ * Warming lets idempotent re-runs (and resumed / appended crawls against
+ * an archive that already holds header rows) reuse existing ids instead
+ * of re-issuing an upsert per entry. On a fresh archive all three tables
+ * are empty and the warm-up is three cheap SELECTs.
+ *
+ * The bundle is only valid for the connection (or transaction lineage)
+ * it was warmed against — ids observed here are stable because the
+ * header dictionaries are append-only, but a cache warmed on connection
+ * A does not see rows a concurrent writer B added. The crawler holds a
+ * single writer connection per archive, so this cannot occur in
+ * practice.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @returns Warmed cache bundle ready for
+ *   {@link ./upsert-one-header-set.ts}.
+ * @example
+ * const caches = await createHeaderTableCaches(knex);
+ * const setId = await upsertOneHeaderSet(knex, decomposed, caches);
+ */
+export declare function createHeaderTableCaches(trx: Knex): Promise<HeaderTableCaches>;

package/lib/archive/populate-ref-tables/create-header-table-caches.js

@@ -0,0 +1,49 @@
+import { headerValueCacheKey } from './header-value-cache-key.js';
+/**
+ * Creates a {@link HeaderTableCaches} bundle warmed from the DB's current
+ * `header_name_refs` / `header_value_refs` / `header_sets` rows.
+ *
+ * Warming lets idempotent re-runs (and resumed / appended crawls against
+ * an archive that already holds header rows) reuse existing ids instead
+ * of re-issuing an upsert per entry. On a fresh archive all three tables
+ * are empty and the warm-up is three cheap SELECTs.
+ *
+ * The bundle is only valid for the connection (or transaction lineage)
+ * it was warmed against — ids observed here are stable because the
+ * header dictionaries are append-only, but a cache warmed on connection
+ * A does not see rows a concurrent writer B added. The crawler holds a
+ * single writer connection per archive, so this cannot occur in
+ * practice.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @returns Warmed cache bundle ready for
+ *   {@link ./upsert-one-header-set.ts}.
+ * @example
+ * const caches = await createHeaderTableCaches(knex);
+ * const setId = await upsertOneHeaderSet(knex, decomposed, caches);
+ */
+export async function createHeaderTableCaches(trx) {
+    const nameIdCache = new Map();
+    const nameRows = await trx('header_name_refs').select();
+    for (const row of nameRows) {
+        nameIdCache.set(row.name, row.id);
+    }
+    const valueIdCache = new Map();
+    const valueRows = await trx('header_value_refs').select();
+    for (const row of valueRows) {
+        valueIdCache.set(headerValueCacheKey(Buffer.from(row.hash), row.value), row.id);
+    }
+    const setIdByRawJsonHash = new Map();
+    const setIdByRawHash = new Map();
+    const setRows = await trx('header_sets').select('id', 'raw_json_hash', 'raw_hash');
+    for (const row of setRows) {
+        setIdByRawJsonHash.set(Buffer.from(row.raw_json_hash).toString('hex'), row.id);
+        setIdByRawHash.set(Buffer.from(row.raw_hash).toString('hex'), row.id);
+    }
+    return {
+        nameIdCache,
+        valueIdCache,
+        setIdByRawJsonHash,
+        setIdByRawHash,
+        setIdsProcessedThisRun: new Set(),
+    };
+}

package/lib/archive/populate-ref-tables/data-uri-url-refs-limit.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Threshold above which a `data:` URI is treated as a `blob_refs` payload
+ * rather than a `url_refs` URL. Shared between {@link ./populate-url-refs.ts}
+ * (which excludes `> LIMIT` data URIs from `url_refs`) and
+ * {@link ./populate-blob-refs.ts} (which routes them to `blob_refs`).
+ *
+ * The two populators MUST agree on this constant — a mismatch creates a
+ * routing hole where URIs of the disputed length land in neither table.
+ * Keeping the value here as a single-file export makes drift impossible.
+ *
+ * 512 was chosen because well-formed http(s) URLs virtually never exceed
+ * 512 characters in practice, while the smallest observed data URIs are
+ * ~600 bytes.
+ */
+export declare const DATA_URI_URL_REFS_LIMIT = 512;

package/lib/archive/populate-ref-tables/data-uri-url-refs-limit.js

@@ -0,0 +1,15 @@
+/**
+ * Threshold above which a `data:` URI is treated as a `blob_refs` payload
+ * rather than a `url_refs` URL. Shared between {@link ./populate-url-refs.ts}
+ * (which excludes `> LIMIT` data URIs from `url_refs`) and
+ * {@link ./populate-blob-refs.ts} (which routes them to `blob_refs`).
+ *
+ * The two populators MUST agree on this constant — a mismatch creates a
+ * routing hole where URIs of the disputed length land in neither table.
+ * Keeping the value here as a single-file export makes drift impossible.
+ *
+ * 512 was chosen because well-formed http(s) URLs virtually never exceed
+ * 512 characters in practice, while the smallest observed data URIs are
+ * ~600 bytes.
+ */
+export const DATA_URI_URL_REFS_LIMIT = 512;

package/lib/archive/populate-ref-tables/decode-data-uri.d.ts

@@ -0,0 +1,21 @@
+import type { DecodedDataUri } from './types.js';
+/**
+ * Attempts to decode a `data:...` URI into its raw payload bytes.
+ *
+ * Supports the two encoding forms in RFC 2397:
+ *
+ * - `data:<mime>;base64,<base64-bytes>` — the base64 tail is decoded via
+ *   `Buffer.from(..., 'base64')`.
+ * - `data:<mime>,<percent-encoded-bytes>` — the tail is percent-decoded
+ *   via `decodeURIComponent(...)`, then re-encoded as UTF-8. This form is
+ *   commonly used for SVG data URIs.
+ *
+ * Returns `null` on any parse / decode failure so the caller can skip the
+ * value without aborting the whole `blob_refs` population step. A single
+ * malformed data URI in a 470 K-row archive should not halt the migration
+ * — the raw `url` string is still preserved in the source `images` row.
+ * @param value - Raw column value; may or may not start with `data:`.
+ * @returns Decoded bytes, or `null` when the value isn't a data URI or
+ *   can't be decoded.
+ */
+export declare function decodeDataUri(value: string): DecodedDataUri | null;

package/lib/archive/populate-ref-tables/decode-data-uri.js

@@ -0,0 +1,126 @@
+/**
+ * Attempts to decode a `data:...` URI into its raw payload bytes.
+ *
+ * Supports the two encoding forms in RFC 2397:
+ *
+ * - `data:<mime>;base64,<base64-bytes>` — the base64 tail is decoded via
+ *   `Buffer.from(..., 'base64')`.
+ * - `data:<mime>,<percent-encoded-bytes>` — the tail is percent-decoded
+ *   via `decodeURIComponent(...)`, then re-encoded as UTF-8. This form is
+ *   commonly used for SVG data URIs.
+ *
+ * Returns `null` on any parse / decode failure so the caller can skip the
+ * value without aborting the whole `blob_refs` population step. A single
+ * malformed data URI in a 470 K-row archive should not halt the migration
+ * — the raw `url` string is still preserved in the source `images` row.
+ * @param value - Raw column value; may or may not start with `data:`.
+ * @returns Decoded bytes, or `null` when the value isn't a data URI or
+ *   can't be decoded.
+ */
+export function decodeDataUri(value) {
+    if (!value.startsWith('data:')) {
+        return null;
+    }
+    const commaIndex = value.indexOf(',');
+    if (commaIndex === -1) {
+        return null;
+    }
+    const header = value.slice(5, commaIndex);
+    const payload = value.slice(commaIndex + 1);
+    try {
+        if (/;\s*base64$/i.test(header)) {
+            // `Buffer.from(..., 'base64')` silently strips characters outside
+            // the base64 alphabet — a malformed payload decodes to garbage
+            // instead of throwing. Validate the alphabet + padding shape
+            // first so callers can distinguish "successfully decoded" from
+            // "silently corrupted" and get the promised `null` on malformed
+            // input.
+            if (!isValidBase64(payload)) {
+                return null;
+            }
+            const bytes = Buffer.from(payload, 'base64');
+            return { bytes };
+        }
+        return { bytes: percentDecodeToBytes(payload) };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Standard base64 alphabet + padding, plus one common URL-safe variant
+ * that occasionally appears in `data:` URIs even though RFC 2397 says
+ * only the standard alphabet. `-` / `_` are the URL-safe substitutes
+ * for `+` / `/`.
+ */
+const BASE64_ALPHABET_RE = /^[\w+/-]*={0,2}$/;
+/**
+ * Validates whether a base64 payload is well-formed enough that
+ * `Buffer.from(payload, 'base64')` won't need to silently strip
+ * characters. Whitespace-tolerant (RFC 2045 permits internal
+ * whitespace in base64 line-wrapped output).
+ * @param payload - Base64 payload (everything after `,` in the data URI).
+ * @returns `true` when the payload contains only valid base64 characters
+ *   + optional trailing padding.
+ */
+function isValidBase64(payload) {
+    const stripped = payload.replaceAll(/\s+/g, '');
+    if (stripped === '') {
+        return true;
+    }
+    if (!BASE64_ALPHABET_RE.test(stripped)) {
+        return false;
+    }
+    // base64 chunks group by 4 characters — a well-formed payload has
+    // length ≡ 0 (mod 4) once padding is applied. Padding-less trailing
+    // chunks of length 2 or 3 are also accepted (Node's decoder tolerates
+    // missing `=`), but length 1 or a padding-only chunk is malformed.
+    const remainder = stripped.length % 4;
+    if (remainder === 1) {
+        return false;
+    }
+    return true;
+}
+/**
+ * Percent-decodes a data-URI payload byte-by-byte, avoiding the round
+ * through `decodeURIComponent` → UTF-8 re-encode that mangles arbitrary
+ * binary payloads (e.g. a `%FF` byte becomes a JS string of code
+ * point U+00FF, then re-encodes to `[0xC3, 0xBF]` — the byte value
+ * changes). RFC 2397's non-base64 form treats the payload as octets,
+ * not as a UTF-8 string, so an octet-preserving decode is the correct
+ * inverse.
+ * @param payload - Raw payload text (everything after the `,`).
+ * @returns Payload bytes.
+ * @throws {URIError} If a `%` escape is truncated or not two hex
+ *   digits; the caller's try/catch maps this to a `null` return.
+ */
+function percentDecodeToBytes(payload) {
+    const chunks = [];
+    let i = 0;
+    while (i < payload.length) {
+        const ch = payload.codePointAt(i);
+        if (ch === 0x25 /* % */) {
+            if (i + 2 >= payload.length) {
+                throw new URIError('truncated % escape');
+            }
+            const hex = payload.slice(i + 1, i + 3);
+            if (!/^[\da-f]{2}$/i.test(hex)) {
+                throw new URIError('invalid % escape');
+            }
+            chunks.push(Buffer.from(hex, 'hex'));
+            i += 3;
+            continue;
+        }
+        if (ch > 127 /* 0x7F, ASCII high-bit boundary */) {
+            // A non-ASCII code point in a percent-encoded payload is
+            // unusual (browsers percent-encode outbound), but WHATWG
+            // tolerates it — encode as UTF-8 to preserve the character.
+            chunks.push(Buffer.from(payload[i], 'utf8'));
+            i += 1;
+            continue;
+        }
+        chunks.push(Buffer.from([ch]));
+        i += 1;
+    }
+    return Buffer.concat(chunks);
+}

package/lib/archive/populate-ref-tables/decompose-header-set.d.ts

@@ -0,0 +1,29 @@
+import type { DecomposedHeaderSet } from './types.js';
+/**
+ * Parses one raw `responseHeaders` JSON string and produces the derived
+ * data shape required to insert into `header_sets`, `header_set_entries`,
+ * and `header_flags`.
+ *
+ * Behaviour:
+ *
+ * - `null` / `""` / `"null"` / `"{}"` / non-object JSON → returns `null`.
+ *   The caller sets `header_set_id = null` on the referring row.
+ * - Multiple values per header name (JSON arrays, e.g.
+ *   `{ "set-cookie": ["a=1", "b=2"] }`) become multiple
+ *   {@link HeaderEntry} rows with `occurrence` ordinals 1, 2, ... The
+ *   composite PK on `header_set_entries` `(header_set_id, name_id,
+ *   occurrence)` guarantees no truncation.
+ * - Names are lower-cased. Values are trimmed of whitespace only in the
+ *   sorted-hash construction; the raw value is preserved verbatim in
+ *   the emitted `HeaderEntry.value` (and hence in `header_value_refs`).
+ *   Trimming inside hash construction lets `content-type: text/html`
+ *   and `content-type:  text/html ` dedup to one stable set.
+ * - Sort order for hashing is `(name, occurrence)` in binary form —
+ *   deterministic regardless of the input JSON's insertion order or
+ *   how the JSON serializer arranged keys.
+ * @param rawJson - The raw JSON string exactly as stored in
+ *   `pages.responseHeaders` / `resources.responseHeaders`.
+ * @returns Decomposed shape, or `null` when the row has no meaningful
+ *   header set.
+ */
+export declare function decomposeHeaderSet(rawJson: string | null): DecomposedHeaderSet | null;