@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/db-ops/pages/write/insert-page.d.ts

@@ -0,0 +1,36 @@
+import type { PageData } from '../../../../utils/types/types.js';
+import type { PageSource } from '../../../types.js';
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Upserts page data into `content_items` + `page_meta` (inserts the
+ * identity row if new, updates if existing).
+ *
+ * `source` is intentionally NOT in the UPDATE clause — provenance is set
+ * once at INSERT time inside `resolveContentItemId`, and existing rows
+ * keep whatever label they were first inserted with, except for the
+ * priority lattice applied below: `'crawled'` > `'inventory-seed'` >
+ * `'inventory-discovered'`. The inventory feature exists to surface
+ * orphans (= URLs NOT reachable from the original crawl roots), so
+ * anything reachable via the crawled chain must be labelled `'crawled'`
+ * even if previously labelled `'inventory-*'`.
+ *
+ * The page's response headers are decomposed and written into the
+ * header dictionary tables here — per response, not deferred to
+ * crawl-end — and the resulting `header_set_id` lands on the same
+ * `content_items` UPDATE. Meta text / URL / JSON values are interned
+ * into `text_refs` / `url_refs` / `json_refs` and `page_meta` stores
+ * only the FK ids; on a re-scrape the `page_meta` row is fully replaced
+ * (`ON CONFLICT(page_id) DO UPDATE` over every column) exactly as the
+ * legacy flat-column UPDATE overwrote every column.
+ * @param knex - Knex query builder connected to the archive DB. Used as the
+ *   fallback when `trx` is not provided.
+ * @param caches - The connection's write-side id caches.
+ * @param page - The scraped page data.
+ * @param isTarget - Whether this page is a crawl target.
+ * @param trx - Optional transaction all statements run through.
+ * @param source - Inventory provenance for the INSERT path. Ignored on UPDATE
+ *   except through the priority lattice described above.
+ * @returns The `content_items.id` of the inserted or updated row.
+ */
+export declare function insertPage(knex: Knex, caches: WriteRefCaches, page: PageData, isTarget: boolean, trx?: Knex.Transaction, source?: PageSource): Promise<number>;

package/lib/archive/db-ops/pages/write/insert-page.js

@@ -0,0 +1,208 @@
+import { normalizeContentType } from '../../../../crawler/normalize-content-type.js';
+import { computePageDenormalized } from '../../../meta/compute-page-denormalized.js';
+import { deriveFlatFromMeta } from '../../../meta/derive-flat-from-meta.js';
+import { deriveMetaExtras } from '../../../meta/derive-meta-extras.js';
+import { PAGE_META_COLUMN_MAPS } from '../../../page-meta-column-maps.js';
+import { upsertTextRefs } from '../../../populate-entity-tables/upsert-text-refs.js';
+import { DATA_URI_URL_REFS_LIMIT } from '../../../populate-ref-tables/data-uri-url-refs-limit.js';
+import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
+import { upsertContentTypeRef } from '../../_shared/upsert-content-type-ref.js';
+import { upsertJsonRef } from '../../_shared/upsert-json-ref.js';
+import { upsertResponseHeaders } from '../../_shared/upsert-response-headers.js';
+import { upsertUrlRef } from '../../_shared/upsert-url-ref.js';
+/**
+ * Upserts page data into `content_items` + `page_meta` (inserts the
+ * identity row if new, updates if existing).
+ *
+ * `source` is intentionally NOT in the UPDATE clause — provenance is set
+ * once at INSERT time inside `resolveContentItemId`, and existing rows
+ * keep whatever label they were first inserted with, except for the
+ * priority lattice applied below: `'crawled'` > `'inventory-seed'` >
+ * `'inventory-discovered'`. The inventory feature exists to surface
+ * orphans (= URLs NOT reachable from the original crawl roots), so
+ * anything reachable via the crawled chain must be labelled `'crawled'`
+ * even if previously labelled `'inventory-*'`.
+ *
+ * The page's response headers are decomposed and written into the
+ * header dictionary tables here — per response, not deferred to
+ * crawl-end — and the resulting `header_set_id` lands on the same
+ * `content_items` UPDATE. Meta text / URL / JSON values are interned
+ * into `text_refs` / `url_refs` / `json_refs` and `page_meta` stores
+ * only the FK ids; on a re-scrape the `page_meta` row is fully replaced
+ * (`ON CONFLICT(page_id) DO UPDATE` over every column) exactly as the
+ * legacy flat-column UPDATE overwrote every column.
+ * @param knex - Knex query builder connected to the archive DB. Used as the
+ *   fallback when `trx` is not provided.
+ * @param caches - The connection's write-side id caches.
+ * @param page - The scraped page data.
+ * @param isTarget - Whether this page is a crawl target.
+ * @param trx - Optional transaction all statements run through.
+ * @param source - Inventory provenance for the INSERT path. Ignored on UPDATE
+ *   except through the priority lattice described above.
+ * @returns The `content_items.id` of the inserted or updated row.
+ */
+export async function insertPage(knex, caches, page, isTarget, trx, source) {
+    const qb = trx ?? knex;
+    const pageId = await resolveContentItemId(qb, caches, page.url.withoutHashAndAuth, undefined, source);
+    const flat = deriveFlatFromMeta(page.meta, page.url.href);
+    const denorm = computePageDenormalized(page.meta);
+    const extras = deriveMetaExtras(page.meta);
+    const now = Date.now();
+    // Canonicalize so the stored dictionary value matches the exact-string
+    // page-ness predicate (`content_type_refs.raw = 'text/html'`) used by
+    // the read layer and the case-insensitive `isHtmlContentType` used in
+    // code. Responses are recorded verbatim upstream, so `Text/HTML` /
+    // `text/html ` can otherwise fork dictionary rows and silently
+    // misclassify.
+    const contentType = normalizeContentType(page.contentType);
+    const contentTypeId = contentType == null || contentType === ''
+        ? null
+        : await upsertContentTypeRef(qb, caches, contentType);
+    const headerSetId = await upsertResponseHeaders(qb, caches, page.responseHeaders);
+    // Source priority on UPDATE: 'crawled' > 'inventory-seed' >
+    // 'inventory-discovered'. In current callers, `source` only arrives as
+    // `'inventory-seed'` / `'inventory-discovered'` / `undefined`
+    // (`derivePageSource` never emits `'crawled'`, and outside inventory
+    // mode `source` is `undefined` so this CASE never runs). The
+    // `? = 'crawled'` branch is therefore reachable only via a future
+    // call site that wants to explicitly assert a crawled lineage —
+    // today the actual crawled-wins downgrade fires in
+    // `resolveContentItemId` when an anchor lineage `'crawled'` lands on
+    // an existing `'inventory-*'` row. The branch is kept so the CASE
+    // completely describes the priority lattice in one place.
+    const sourceUpdate = source === undefined
+        ? {}
+        : {
+            source: qb.raw(`CASE
+								WHEN source = 'crawled' OR ? = 'crawled' THEN 'crawled'
+								WHEN source = 'inventory-seed' OR ? = 'inventory-seed' THEN 'inventory-seed'
+								WHEN source = 'inventory-discovered' OR ? = 'inventory-discovered' THEN 'inventory-discovered'
+								ELSE source
+							END`, [source, source, source]),
+        };
+    await qb('content_items')
+        .where('id', pageId)
+        .update({
+        scraped: 1,
+        is_target: isTarget ? 1 : 0,
+        is_external: page.isExternal ? 1 : 0,
+        status: page.status,
+        status_text: page.statusText,
+        content_type_id: contentTypeId,
+        content_length: page.contentLength,
+        header_set_id: headerSetId,
+        // Timestamps: `first_crawled_at` is set only on first scrape —
+        // `COALESCE` preserves the existing value so a re-scrape
+        // (`--append`, `--retry-failed`) does not erase the discovery time.
+        // `last_crawled_at` is updated every successful scrape.
+        first_crawled_at: qb.raw('COALESCE(first_crawled_at, ?)', [now]),
+        last_crawled_at: now,
+        is_skipped: page.isSkipped ? 1 : 0,
+        ...sourceUpdate,
+    });
+    // The cache's `source` mirror only tracks the resolveContentItemId
+    // lattice; when the CASE above rewrites the column, refresh the cache
+    // from the caller-supplied label so a later crawled-wins check sees
+    // the current value. (`'crawled'` never arrives here today — see the
+    // CASE comment — so the only observable effect is seed-over-discovered
+    // promotion, which the lattice below reproduces.)
+    if (source !== undefined) {
+        const entry = caches.contentItems.get(page.url.withoutHashAndAuth);
+        if (entry !== undefined && entry.source === 'inventory-discovered') {
+            entry.source = source;
+        }
+    }
+    await upsertPageMeta(qb, caches, pageId, flat, denorm, extras);
+    return pageId;
+}
+/**
+ * Builds and upserts the `page_meta` row for one scraped page: interns
+ * text / URL / JSON values into their ref tables, then fully replaces
+ * the row (every column participates in the `DO UPDATE`) so a re-scrape
+ * behaves exactly like the legacy per-column UPDATE.
+ * @param qb - Knex instance or transaction.
+ * @param caches - The connection's write-side id caches.
+ * @param pageId - The owning `content_items.id`.
+ * @param flat - Flat meta columns from `deriveFlatFromMeta`.
+ * @param denorm - Denormalised aggregates from `computePageDenormalized`.
+ * @param extras - Nested-Meta catch-all from `deriveMetaExtras`.
+ */
+async function upsertPageMeta(qb, caches, pageId, flat, denorm, extras) {
+    const texts = new Set();
+    for (const { source } of PAGE_META_COLUMN_MAPS.text) {
+        const value = flat[source];
+        if (typeof value === 'string' && value !== '') {
+            texts.add(value);
+        }
+    }
+    const textIds = await upsertTextRefs(qb, texts);
+    const row = {
+        page_id: pageId,
+        lang: flat.lang,
+        dir: flat.dir,
+        charset: flat.charset,
+        base_href: flat.baseHref,
+        viewport_raw: flat.viewport_raw,
+        theme_color: flat.themeColor,
+        application_name: flat.applicationName,
+        author: flat.author,
+        generator: flat.generator,
+        publisher: flat.publisher,
+        robots_noindex: flat.robots_noindex,
+        robots_nofollow: flat.robots_nofollow,
+        robots_noarchive: flat.robots_noarchive,
+        robots_noimageindex: flat.robots_noimageindex,
+        googlebot: flat.googlebot,
+        og_type: flat.og_type,
+        og_site_name: flat.og_site_name,
+        og_image_alt: flat.og_image_alt,
+        og_image_width: flat.og_image_width,
+        og_image_height: flat.og_image_height,
+        og_locale: flat.og_locale,
+        og_article_published_time: flat.og_article_published_time,
+        og_article_modified_time: flat.og_article_modified_time,
+        twitter_card: flat.twitter_card,
+        twitter_site: flat.twitter_site,
+        twitter_creator: flat.twitter_creator,
+        fb_app_id: flat.fb_app_id,
+        verification_google: flat.verification_google,
+        format_detection_telephone: flat.formatDetection_telephone,
+        // Denormalised aggregates: written once at scrape time so list reads
+        // (Sheets, page-detail summary) can answer "how many JSON-LD
+        // entries?" and "which Wappalyzer providers?" by selecting a single
+        // column rather than running a GROUP BY join on every read.
+        tag_count: denorm.tag_count,
+        jsonld_count: denorm.jsonld_count,
+        tags_providers_csv: denorm.tags_providers_csv,
+    };
+    for (const { source, target } of PAGE_META_COLUMN_MAPS.text) {
+        const value = flat[source];
+        row[target] =
+            typeof value === 'string' && value !== '' ? (textIds.get(value) ?? null) : null;
+    }
+    for (const { source, target } of PAGE_META_COLUMN_MAPS.url) {
+        const value = flat[source];
+        if (typeof value !== 'string' || value === '' || isLargeDataUri(value)) {
+            // `page_meta` has no `*_blob_id` companion columns, so a large
+            // data URI in a URL slot (a base64 favicon, typically) is
+            // dropped rather than ballooning `url_refs` with an opaque
+            // payload — matching the archive-migration behaviour.
+            row[target] = null;
+            continue;
+        }
+        row[target] = await upsertUrlRef(qb, caches, value);
+    }
+    const extrasJson = JSON.stringify(extras);
+    row.meta_extras_json_id =
+        extrasJson === '' ? null : await upsertJsonRef(qb, caches, extrasJson);
+    await qb('page_meta').insert(row).onConflict('page_id').merge();
+}
+/**
+ * Returns `true` when `value` is a `data:` URI larger than the routing
+ * threshold — the same rule that routes image `src` values to
+ * `blob_refs` instead of `url_refs`.
+ * @param value - Raw URL column value.
+ */
+function isLargeDataUri(value) {
+    return value.startsWith('data:') && value.length > DATA_URI_URL_REFS_LIMIT;
+}

package/lib/archive/db-ops/pages/write/insert-tags.d.ts

@@ -0,0 +1,16 @@
+import type { PageData } from '../../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's Wappalyzer tag rows with the freshly captured set.
+ * Called inside `updatePage`'s transaction unconditionally — tag
+ * detection draws on `<script src>` / `<iframe src>` / window globals /
+ * response headers, not the HTML body, so external pages that skip
+ * rendering still contribute tags.
+ *
+ * Same empty-guard as `insertJsonLd`: an empty array does not wipe
+ * prior rows on a degraded re-scrape.
+ * @param pageId
+ * @param meta
+ * @param trx
+ */
+export declare function insertTags(pageId: number, meta: PageData['meta'], trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-tags.js

@@ -0,0 +1,34 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { extractTagsForArchive } from '../../../meta/extract-tags-for-archive.js';
+/**
+ * Replaces the page's Wappalyzer tag rows with the freshly captured set.
+ * Called inside `updatePage`'s transaction unconditionally — tag
+ * detection draws on `<script src>` / `<iframe src>` / window globals /
+ * response headers, not the HTML body, so external pages that skip
+ * rendering still contribute tags.
+ *
+ * Same empty-guard as `insertJsonLd`: an empty array does not wipe
+ * prior rows on a degraded re-scrape.
+ * @param pageId
+ * @param meta
+ * @param trx
+ */
+export async function insertTags(pageId, meta, trx) {
+    const partial = extractTagsForArchive(meta.tags);
+    if (partial.length === 0)
+        return;
+    const rows = partial.map((p) => ({
+        pageId,
+        provider: p.provider,
+        category: p.category,
+        externalId: p.externalId,
+        version: p.version,
+        confidence: p.confidence,
+        categories: JSON.stringify(p.categories),
+        sources: JSON.stringify(p.sources),
+    }));
+    await trx('page_tags').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_tags').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/link-redirect-sources.d.ts

@@ -0,0 +1,36 @@
+import type { PageSource } from '../../../types.js';
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Points each redirect-source URL at the destination page, marking it scraped
+ * and clearing any content it owned in a former life.
+ *
+ * Shared by `updatePage` (which also renders and stores the destination)
+ * and `recordRedirect` (which only records the edge for a destination
+ * rendered elsewhere). Self-redirects (source equal to the destination) are
+ * skipped so a page is never marked as redirecting to itself — that would
+ * exclude it from reports via the `whereNull('redirect_dest_id')` filter.
+ * @param trx - The active transaction. All SQL below is executed exclusively
+ *   through this transaction; there is no separate `knex` fallback because
+ *   the callers (`updatePage` / `recordRedirect`) always invoke this helper
+ *   from within a transaction.
+ * @param caches - The connection's write-side id caches.
+ * @param sources - Redirect-source URLs (normalised): the original URL plus
+ *   any intermediate hops. Empty when the page was not redirected.
+ * @param destId - `content_items.id` of the redirect destination page.
+ * @param destUrlNormalized - Normalised destination URL, used to detect and
+ *   skip self-redirects.
+ * @param isExternal - Whether the sources are external to the crawl scope.
+ * @param chainLineageSource - Lineage label propagated to each intermediate
+ *   hop's row (passed through to {@link resolveContentItemId}). Derived by
+ *   the caller from the **originating** page's source (`page.url`), not from
+ *   the destination — intermediates are reached transitively from the
+ *   originating render, so they inherit its lineage. Pass `'inventory-discovered'`
+ *   for chains rooted at inventory-seed/discovered pages so new intermediates
+ *   stay in the inventory chain; pass `'crawled'` for crawled chains so the
+ *   crawled-wins downgrade inside `resolveContentItemId` fires on existing
+ *   `'inventory-*'` intermediates a crawled chain reaches. Pass `undefined`
+ *   to fall back to the DB DEFAULT (`'crawled'`) on INSERT without
+ *   triggering the downgrade on existing rows.
+ */
+export declare function linkRedirectSources(trx: Knex.Transaction, caches: WriteRefCaches, sources: readonly string[], destId: number, destUrlNormalized: string, isExternal: boolean, chainLineageSource?: PageSource): Promise<void>;

package/lib/archive/db-ops/pages/write/link-redirect-sources.js

@@ -0,0 +1,93 @@
+import { dbLog } from '../../../debug.js';
+import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
+/**
+ * Points each redirect-source URL at the destination page, marking it scraped
+ * and clearing any content it owned in a former life.
+ *
+ * Shared by `updatePage` (which also renders and stores the destination)
+ * and `recordRedirect` (which only records the edge for a destination
+ * rendered elsewhere). Self-redirects (source equal to the destination) are
+ * skipped so a page is never marked as redirecting to itself — that would
+ * exclude it from reports via the `whereNull('redirect_dest_id')` filter.
+ * @param trx - The active transaction. All SQL below is executed exclusively
+ *   through this transaction; there is no separate `knex` fallback because
+ *   the callers (`updatePage` / `recordRedirect`) always invoke this helper
+ *   from within a transaction.
+ * @param caches - The connection's write-side id caches.
+ * @param sources - Redirect-source URLs (normalised): the original URL plus
+ *   any intermediate hops. Empty when the page was not redirected.
+ * @param destId - `content_items.id` of the redirect destination page.
+ * @param destUrlNormalized - Normalised destination URL, used to detect and
+ *   skip self-redirects.
+ * @param isExternal - Whether the sources are external to the crawl scope.
+ * @param chainLineageSource - Lineage label propagated to each intermediate
+ *   hop's row (passed through to {@link resolveContentItemId}). Derived by
+ *   the caller from the **originating** page's source (`page.url`), not from
+ *   the destination — intermediates are reached transitively from the
+ *   originating render, so they inherit its lineage. Pass `'inventory-discovered'`
+ *   for chains rooted at inventory-seed/discovered pages so new intermediates
+ *   stay in the inventory chain; pass `'crawled'` for crawled chains so the
+ *   crawled-wins downgrade inside `resolveContentItemId` fires on existing
+ *   `'inventory-*'` intermediates a crawled chain reaches. Pass `undefined`
+ *   to fall back to the DB DEFAULT (`'crawled'`) on INSERT without
+ *   triggering the downgrade on existing rows.
+ */
+export async function linkRedirectSources(trx, caches, sources, destId, destUrlNormalized, isExternal, chainLineageSource) {
+    for (const redirect of sources) {
+        if (redirect === destUrlNormalized) {
+            dbLog('Skip self-redirect: %s', redirect);
+            continue;
+        }
+        dbLog('Set redirected url: %s -> id:%d', redirect, destId);
+        // Pass `chainLineageSource` through so a brand-new intermediate
+        // hop INSERTed here inherits the originating page's lineage label
+        // (inventory-discovered when the originating chain is in the
+        // inventory chain, undefined otherwise). The crawled-wins
+        // downgrade inside `resolveContentItemId` still fires when this
+        // argument is `'crawled'`, matching the anchor-lineage propagation
+        // contract — an existing inventory-* intermediate that is later
+        // traversed by a `'crawled'` chain gets downgraded.
+        const redirectId = await resolveContentItemId(trx, caches, redirect, undefined, chainLineageSource);
+        await trx('content_items')
+            .where('id', redirectId)
+            .update({
+            scraped: 1,
+            redirect_dest_id: destId,
+            is_external: isExternal ? 1 : 0,
+        });
+        // Conditional `301 Moved Permanently` stamp — applied ONLY when the
+        // row carries no definitive status yet (NULL or the `-1`
+        // hard-failure sentinel). HEAD pre-flight does not retain each
+        // hop's individual status code (`redirectPaths` is a URL[] without
+        // statuses), so the only honest answer for an unknown-status hop is
+        // "some 3xx" — 301 is the canonical representative.
+        //
+        // We deliberately do NOT overwrite an existing definitive status
+        // (200 / 302 / 307 / etc.): a row that already captured a concrete
+        // status from a prior direct scrape would lose accuracy. The stamp
+        // only flips two cases:
+        // - NULL: a placeholder row created by `resolveContentItemId`
+        //   because the URL was reached only as a redirect target / source,
+        //   never directly scraped. Without the stamp the row is invisible
+        //   on the Errors view's status distribution.
+        // - -1: a row that recorded a hard scrape failure (e.g. a puppeteer
+        //   goto returned null on a HTTPS→HTTP downgrade redirect) BEFORE
+        //   the chain was understood. That `-1` then conflated "real
+        //   failure" with "actually a redirect source we now know about",
+        //   polluting the `-1` bucket AND inflating the `--retry-failed`
+        //   target (via the `whereNull('redirect_dest_id')` filter — the
+        //   redirect_dest_id update above already excludes the row from
+        //   retry; this stamp restores the visible identity).
+        await trx('content_items')
+            .where('id', redirectId)
+            .where((qb) => qb.whereNull('status').orWhere('status', -1))
+            .update({ status: 301, status_text: 'Moved Permanently' });
+        // A page that used to be scraped as content can later turn into a
+        // redirect source. It owns no content anymore, so drop any anchors /
+        // images it captured in its former life — otherwise they linger and
+        // leak into referrer / incoming-link reads (which do not filter out
+        // redirect sources).
+        await trx('anchor_edges').where('page_id', redirectId).delete();
+        await trx('image_items').where('page_id', redirectId).delete();
+    }
+}

package/lib/archive/db-ops/pages/write/record-redirect.d.ts

@@ -0,0 +1,35 @@
+import type { PageData } from '../../../../utils/types/types.js';
+import type { PageSource } from '../../../types.js';
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Records a redirect edge (source → destination) **without** re-storing the
+ * destination's content.
+ *
+ * The crawler renders a many-to-one redirect destination exactly once. For
+ * every subsequent source URL that redirects to that already-rendered
+ * destination, it calls this instead of {@link ./update-page.ts} (#73).
+ * Routing a content-less HEAD result through `updatePage` would funnel it
+ * into `insertPage` and overwrite the destination's good title / meta with
+ * empty values, so the dedicated edge-only path is required.
+ *
+ * The destination row is resolved (created on demand if a concurrent
+ * in-flight render has not committed it yet) so the edge always points at a
+ * valid id; the single render fills in the destination's content under that
+ * same id. The destination's existing anchors / images are never touched
+ * here.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param page - HEAD-resolved page data carrying the redirect chain. Its
+ *   `anchorList` / `imageList` are ignored (a redirect source owns no content).
+ * @param source - Inventory provenance forwarded by the orchestrator
+ *   (`Archive.setRedirect` → here) for the redirect-edge fast path. Used
+ *   as the fallback when the originating URL's row does NOT yet exist in
+ *   the archive (`#73` convergence on first sight, js-redirect rescue
+ *   before any prior write). When the originating row already exists
+ *   (e.g. anchor-lineage INSERT from a prior pass), its stored `source`
+ *   takes precedence so transitive lineage is preserved across resume /
+ *   retry-failed sessions. `undefined` keeps the DB DEFAULT `'crawled'`
+ *   on a brand-new destination row.
+ */
+export declare function recordRedirect(knex: Knex, caches: WriteRefCaches, page: PageData, source?: PageSource): Promise<void>;

package/lib/archive/db-ops/pages/write/record-redirect.js

@@ -0,0 +1,100 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { dbLog } from '../../../debug.js';
+import { deriveLineageFromParent } from '../../../derive-lineage-from-parent.js';
+import { resolveRedirectChain } from '../../../resolve-redirect-chain.js';
+import { clearWriteRefCaches } from '../../_shared/clear-write-ref-caches.js';
+import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
+import { linkRedirectSources } from './link-redirect-sources.js';
+/**
+ * Records a redirect edge (source → destination) **without** re-storing the
+ * destination's content.
+ *
+ * The crawler renders a many-to-one redirect destination exactly once. For
+ * every subsequent source URL that redirects to that already-rendered
+ * destination, it calls this instead of {@link ./update-page.ts} (#73).
+ * Routing a content-less HEAD result through `updatePage` would funnel it
+ * into `insertPage` and overwrite the destination's good title / meta with
+ * empty values, so the dedicated edge-only path is required.
+ *
+ * The destination row is resolved (created on demand if a concurrent
+ * in-flight render has not committed it yet) so the edge always points at a
+ * valid id; the single render fills in the destination's content under that
+ * same id. The destination's existing anchors / images are never touched
+ * here.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param page - HEAD-resolved page data carrying the redirect chain. Its
+ *   `anchorList` / `imageList` are ignored (a redirect source owns no content).
+ * @param source - Inventory provenance forwarded by the orchestrator
+ *   (`Archive.setRedirect` → here) for the redirect-edge fast path. Used
+ *   as the fallback when the originating URL's row does NOT yet exist in
+ *   the archive (`#73` convergence on first sight, js-redirect rescue
+ *   before any prior write). When the originating row already exists
+ *   (e.g. anchor-lineage INSERT from a prior pass), its stored `source`
+ *   takes precedence so transitive lineage is preserved across resume /
+ *   retry-failed sessions. `undefined` keeps the DB DEFAULT `'crawled'`
+ *   on a brand-new destination row.
+ */
+export async function recordRedirect(knex, caches, page, source) {
+    const { destUrl, sources } = resolveRedirectChain(page.url.withoutHashAndAuth, page.redirectPaths);
+    // No redirect chain (the URL is itself the already-rendered destination,
+    // reached both directly and via a redirect) → there is no edge to write.
+    // Returning here avoids opening a transaction and, crucially, avoids
+    // `resolveContentItemId` inserting a content-less placeholder row for a
+    // destination that may not have been written yet.
+    if (sources.length === 0) {
+        return;
+    }
+    const destUrlObject = parseUrl(destUrl);
+    if (!destUrlObject) {
+        // A malformed redirect target should not abort the whole crawl (this
+        // runs inside the WriteQueue, whose rejection aborts the run). Recording
+        // a single redirect edge is best-effort, so skip it and move on. Unlike
+        // `updatePage`, there is no page content at stake here.
+        dbLog('recordRedirect: skip malformed destination URL: %s', destUrl);
+        return;
+    }
+    try {
+        await knex.transaction(async (trx) => {
+            // Pass the caller-supplied `source` straight through so a brand-new
+            // destination row INSERTed here picks up the inventory lineage
+            // (instead of the DB DEFAULT `'crawled'`) when the caller is in the
+            // inventory chain — without the pass-through, inventory lineage
+            // would be laundered to `'crawled'` for js-redirect rescue / #73
+            // convergence destinations that have not yet been rendered.
+            const destId = await resolveContentItemId(trx, caches, destUrlObject.withoutHashAndAuth, undefined, source);
+            // Chain lineage propagates FROM the originating URL (`page.url`),
+            // NOT from the destination. The originating URL is what initiated
+            // the redirect chain, so its lineage is what every intermediate hop
+            // transitively inherits. Reading from the destination would
+            // mis-propagate in "inventory-seed → ... → existing crawled dest"
+            // chains: the intermediates are reached only via the inventory
+            // chain, so they belong to the inventory chain even though the
+            // chain happens to land on a crawled URL. The `'crawled'` fallback
+            // arms the crawled-wins downgrade for existing `'inventory-*'`
+            // intermediates that a crawled chain reaches.
+            const cachedOriginating = caches.contentItems.get(page.url.withoutHashAndAuth);
+            let originatingSource = cachedOriginating?.source;
+            if (originatingSource === undefined) {
+                const [originatingRow] = (await trx
+                    .select('ci.source')
+                    .from('content_items as ci')
+                    .join('url_refs as ur', 'ur.id', 'ci.url_id')
+                    .where('ur.url', page.url.withoutHashAndAuth));
+                originatingSource = originatingRow?.source ?? source;
+            }
+            const chainLineageSource = deriveLineageFromParent(originatingSource, 'crawled');
+            await linkRedirectSources(trx, caches, sources, destId, destUrlObject.withoutHashAndAuth, page.isExternal, chainLineageSource);
+        });
+    }
+    catch (error) {
+        // A rolled-back transaction can leave ids cached that no longer
+        // correspond to any row (AUTOINCREMENT never rewinds) — see
+        // `clearWriteRefCaches` for why a full clear, not a partial one, is
+        // required. `emitErrorAndRetry` may retry this whole call, so the
+        // cache must be clean before the next attempt (same guard as
+        // `updatePage`).
+        clearWriteRefCaches(caches);
+        throw error;
+    }
+}

package/lib/archive/db-ops/pages/write/set-skipped-page.d.ts

@@ -0,0 +1,13 @@
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Marks a page as skipped in the database with the given reason.
+ * Creates the `content_items` row (with its `url_refs` entry) if it does
+ * not already exist.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param url - The URL of the skipped page.
+ * @param reason - The reason the page was skipped.
+ * @param isExternal - Whether the page is on an external domain. Defaults to `false`.
+ */
+export declare function setSkippedPage(knex: Knex, caches: WriteRefCaches, url: string, reason: string, isExternal?: boolean): Promise<void>;

package/lib/archive/db-ops/pages/write/set-skipped-page.js

@@ -0,0 +1,22 @@
+import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
+/**
+ * Marks a page as skipped in the database with the given reason.
+ * Creates the `content_items` row (with its `url_refs` entry) if it does
+ * not already exist.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param url - The URL of the skipped page.
+ * @param reason - The reason the page was skipped.
+ * @param isExternal - Whether the page is on an external domain. Defaults to `false`.
+ */
+export async function setSkippedPage(knex, caches, url, reason, isExternal = false) {
+    const pageId = await resolveContentItemId(knex, caches, url, isExternal ? 1 : 0);
+    await knex('content_items')
+        .where('id', pageId)
+        .update({
+        scraped: 1,
+        is_external: isExternal ? 1 : 0,
+        is_skipped: 1,
+        skip_reason: reason,
+    });
+}

package/lib/archive/db-ops/pages/write/update-page.d.ts

@@ -0,0 +1,29 @@
+import type { PageDataWithDomPaths, PageSource } from '../../../types.js';
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Inserts or updates a crawled page — `content_items` + `page_meta` plus its
+ * redirect chain, `anchor_edges`, `image_items`, and (when `writeHtml`) its
+ * compressed HTML snapshot BLOB.
+ *
+ * Self-redirects (where the source URL equals the destination URL after
+ * normalization) are skipped to avoid marking a page as redirected to itself
+ * — a situation caused by authentication challenges (e.g. Basic Auth 302)
+ * that would otherwise exclude the page from reports via the
+ * `whereNull('redirect_dest_id')` filter.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param page - The page data to store, optionally carrying the in-browser
+ *   dom-path capture (see {@link PageDataWithDomPaths}).
+ * @param writeHtml - When `true`, this call is allowed to insert (or clear)
+ *   the page's HTML blob. `setExternalPage` passes `false` because external
+ *   metadata-only scrapes never carry HTML and must not perturb an already
+ *   stored body.
+ * @param isTarget - Whether this page is a crawl target.
+ * @param source - Provenance label written ONLY when the row is freshly
+ *   inserted. Existing rows keep their original `source` (this is why a
+ *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
+ *   that was discovered earlier).
+ * @returns The database `pageId` (`content_items.id`) of the inserted or updated row.
+ */
+export declare function updatePage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, writeHtml: boolean, isTarget: boolean, source?: PageSource): Promise<number>;