@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/db-ops/pages/write/insert-technologies.js

@@ -0,0 +1,45 @@
+import { extractTechnologiesForArchive } from '../../../meta/technologies/extract-technologies-for-archive.js';
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `technology_signals` + `page_technologies` rows with
+ * the freshly extracted set. Called inside `updatePage`'s transaction
+ * unconditionally — the `wappalyzer` signal source draws on `<script src>` /
+ * `<iframe src>` / window globals / response headers, not the HTML body, so
+ * external pages that skip rendering still contribute technology signals
+ * (structural signals naturally find nothing against an empty `html`
+ * string, which is exactly what a skipped render leaves it as).
+ *
+ * Direct replacement for `insertTags` in the live crawl write path (that
+ * file is untouched — `scripts/migrate-to-0.10.mjs` still depends on it).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does
+ * not wipe prior rows on a degraded re-scrape. Both tables are always
+ * written together in this one call — see ARCHITECTURE.md's invariant
+ * that `technology_signals` and `page_technologies` are never updated
+ * independently.
+ * @param pageId
+ * @param html - The page's raw HTML string (empty for external/metadata-only scrapes).
+ * @param meta
+ * @param trx
+ */
+export async function insertTechnologies(pageId, html, meta, trx) {
+    const { signals, technologies } = extractTechnologiesForArchive(html, meta);
+    if (signals.length === 0)
+        return;
+    await trx('technology_signals').where('pageId', pageId).delete();
+    await trx('page_technologies').where('pageId', pageId).delete();
+    await eachSplitted(signals, 100, async (chunk) => {
+        await trx('technology_signals').insert(chunk.map((s) => ({
+            pageId,
+            technology: s.technology,
+            signalType: s.signalType,
+            evidence: s.evidence,
+            weight: s.weight,
+        })));
+    });
+    if (technologies.length > 0) {
+        await eachSplitted(technologies, 100, async (chunk) => {
+            await trx('page_technologies').insert(chunk.map((t) => ({ pageId, ...t })));
+        });
+    }
+}

package/lib/db-ops/pages/write/insert-videos.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_videos` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertVideos(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/db-ops/pages/write/insert-videos.js

@@ -0,0 +1,28 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_videos` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertVideos(pageId, mainContents, trx) {
+    if (mainContents.videos.length === 0)
+        return;
+    const rows = mainContents.videos.map((video, order) => ({
+        pageId,
+        order,
+        src: video.src,
+        poster: video.poster,
+        width: video.width,
+        height: video.height,
+    }));
+    await trx('page_main_content_videos').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_videos').insert(chunk);
+    });
+}

package/lib/db-ops/pages/write/link-redirect-sources.d.ts

@@ -0,0 +1,42 @@
+import type { PageSource } from '../../../types.js';
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Points each redirect-source URL at the destination page, marking it scraped
+ * and clearing any content it owned in a former life.
+ *
+ * Shared by `updatePage` (which also renders and stores the destination)
+ * and `recordRedirect` (which only records the edge for a destination
+ * rendered elsewhere). Self-redirects (source equal to the destination) are
+ * skipped so a page is never marked as redirecting to itself — that would
+ * exclude it from reports via the `whereNull('redirect_dest_id')` filter.
+ * @param trx - The active transaction. All SQL below is executed exclusively
+ *   through this transaction; there is no separate `knex` fallback because
+ *   the callers (`updatePage` / `recordRedirect`) always invoke this helper
+ *   from within a transaction.
+ * @param caches - The connection's write-side id caches.
+ * @param sources - Redirect-source URLs (normalised): the original URL plus
+ *   any intermediate hops. Empty when the page was not redirected.
+ * @param destId - `content_items.id` of the redirect destination page.
+ * @param destUrlNormalized - Normalised destination URL, used to detect and
+ *   skip self-redirects.
+ * @param isExternal - Whether the sources are external to the crawl scope, as
+ *   a single value applied to every hop in `sources` — the same
+ *   requester-describes-a-different-row shape `insertPage` guards against
+ *   (see its docs in `insert-page.ts`), since a hop can independently be a
+ *   real, already-scraped internal page in its own right (reached earlier as
+ *   a direct crawl target) before it is later observed as an intermediate hop
+ *   in some other chain. The UPDATE below carries the same CASE guard.
+ * @param chainLineageSource - Lineage label propagated to each intermediate
+ *   hop's row (passed through to {@link resolveContentItemId}). Derived by
+ *   the caller from the **originating** page's source (`page.url`), not from
+ *   the destination — intermediates are reached transitively from the
+ *   originating render, so they inherit its lineage. Pass `'inventory-discovered'`
+ *   for chains rooted at inventory-seed/discovered pages so new intermediates
+ *   stay in the inventory chain; pass `'crawled'` for crawled chains so the
+ *   crawled-wins downgrade inside `resolveContentItemId` fires on existing
+ *   `'inventory-*'` intermediates a crawled chain reaches. Pass `undefined`
+ *   to fall back to the DB DEFAULT (`'crawled'`) on INSERT without
+ *   triggering the downgrade on existing rows.
+ */
+export declare function linkRedirectSources(trx: Knex.Transaction, caches: WriteRefCaches, sources: readonly string[], destId: number, destUrlNormalized: string, isExternal: boolean, chainLineageSource?: PageSource): Promise<void>;

package/lib/db-ops/pages/write/link-redirect-sources.js

@@ -0,0 +1,101 @@
+import { dbLog } from '../../../debug.js';
+import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
+/**
+ * Points each redirect-source URL at the destination page, marking it scraped
+ * and clearing any content it owned in a former life.
+ *
+ * Shared by `updatePage` (which also renders and stores the destination)
+ * and `recordRedirect` (which only records the edge for a destination
+ * rendered elsewhere). Self-redirects (source equal to the destination) are
+ * skipped so a page is never marked as redirecting to itself — that would
+ * exclude it from reports via the `whereNull('redirect_dest_id')` filter.
+ * @param trx - The active transaction. All SQL below is executed exclusively
+ *   through this transaction; there is no separate `knex` fallback because
+ *   the callers (`updatePage` / `recordRedirect`) always invoke this helper
+ *   from within a transaction.
+ * @param caches - The connection's write-side id caches.
+ * @param sources - Redirect-source URLs (normalised): the original URL plus
+ *   any intermediate hops. Empty when the page was not redirected.
+ * @param destId - `content_items.id` of the redirect destination page.
+ * @param destUrlNormalized - Normalised destination URL, used to detect and
+ *   skip self-redirects.
+ * @param isExternal - Whether the sources are external to the crawl scope, as
+ *   a single value applied to every hop in `sources` — the same
+ *   requester-describes-a-different-row shape `insertPage` guards against
+ *   (see its docs in `insert-page.ts`), since a hop can independently be a
+ *   real, already-scraped internal page in its own right (reached earlier as
+ *   a direct crawl target) before it is later observed as an intermediate hop
+ *   in some other chain. The UPDATE below carries the same CASE guard.
+ * @param chainLineageSource - Lineage label propagated to each intermediate
+ *   hop's row (passed through to {@link resolveContentItemId}). Derived by
+ *   the caller from the **originating** page's source (`page.url`), not from
+ *   the destination — intermediates are reached transitively from the
+ *   originating render, so they inherit its lineage. Pass `'inventory-discovered'`
+ *   for chains rooted at inventory-seed/discovered pages so new intermediates
+ *   stay in the inventory chain; pass `'crawled'` for crawled chains so the
+ *   crawled-wins downgrade inside `resolveContentItemId` fires on existing
+ *   `'inventory-*'` intermediates a crawled chain reaches. Pass `undefined`
+ *   to fall back to the DB DEFAULT (`'crawled'`) on INSERT without
+ *   triggering the downgrade on existing rows.
+ */
+export async function linkRedirectSources(trx, caches, sources, destId, destUrlNormalized, isExternal, chainLineageSource) {
+    for (const redirect of sources) {
+        if (redirect === destUrlNormalized) {
+            dbLog('Skip self-redirect: %s', redirect);
+            continue;
+        }
+        dbLog('Set redirected url: %s -> id:%d', redirect, destId);
+        // Pass `chainLineageSource` through so a brand-new intermediate
+        // hop INSERTed here inherits the originating page's lineage label
+        // (inventory-discovered when the originating chain is in the
+        // inventory chain, undefined otherwise). The crawled-wins
+        // downgrade inside `resolveContentItemId` still fires when this
+        // argument is `'crawled'`, matching the anchor-lineage propagation
+        // contract — an existing inventory-* intermediate that is later
+        // traversed by a `'crawled'` chain gets downgraded.
+        const redirectId = await resolveContentItemId(trx, caches, redirect, {
+            source: chainLineageSource,
+        });
+        await trx('content_items')
+            .where('id', redirectId)
+            .update({
+            scraped: 1,
+            redirect_dest_id: destId,
+            is_external: trx.raw('CASE WHEN scraped = 1 AND is_external = 0 THEN 0 ELSE ? END', [isExternal ? 1 : 0]),
+        });
+        // Conditional `301 Moved Permanently` stamp — applied ONLY when the
+        // row carries no definitive status yet (NULL or the `-1`
+        // hard-failure sentinel). HEAD pre-flight does not retain each
+        // hop's individual status code (`redirectPaths` is a URL[] without
+        // statuses), so the only honest answer for an unknown-status hop is
+        // "some 3xx" — 301 is the canonical representative.
+        //
+        // We deliberately do NOT overwrite an existing definitive status
+        // (200 / 302 / 307 / etc.): a row that already captured a concrete
+        // status from a prior direct scrape would lose accuracy. The stamp
+        // only flips two cases:
+        // - NULL: a placeholder row created by `resolveContentItemId`
+        //   because the URL was reached only as a redirect target / source,
+        //   never directly scraped. Without the stamp the row is invisible
+        //   on the Errors view's status distribution.
+        // - -1: a row that recorded a hard scrape failure (e.g. a puppeteer
+        //   goto returned null on a HTTPS→HTTP downgrade redirect) BEFORE
+        //   the chain was understood. That `-1` then conflated "real
+        //   failure" with "actually a redirect source we now know about",
+        //   polluting the `-1` bucket AND inflating the `--retry-failed`
+        //   target (via the `whereNull('redirect_dest_id')` filter — the
+        //   redirect_dest_id update above already excludes the row from
+        //   retry; this stamp restores the visible identity).
+        await trx('content_items')
+            .where('id', redirectId)
+            .where((qb) => qb.whereNull('status').orWhere('status', -1))
+            .update({ status: 301, status_text: 'Moved Permanently' });
+        // A page that used to be scraped as content can later turn into a
+        // redirect source. It owns no content anymore, so drop any anchors /
+        // images it captured in its former life — otherwise they linger and
+        // leak into referrer / incoming-link reads (which do not filter out
+        // redirect sources).
+        await trx('anchor_edges').where('page_id', redirectId).delete();
+        await trx('image_items').where('page_id', redirectId).delete();
+    }
+}

package/lib/db-ops/pages/write/record-redirect.d.ts

@@ -0,0 +1,35 @@
+import type { PageSource } from '../../../types.js';
+import type { PageData } from '../../../utils/types/types.js';
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Records a redirect edge (source → destination) **without** re-storing the
+ * destination's content.
+ *
+ * The crawler renders a many-to-one redirect destination exactly once. For
+ * every subsequent source URL that redirects to that already-rendered
+ * destination, it calls this instead of {@link ./update-page.ts} (#73).
+ * Routing a content-less HEAD result through `updatePage` would funnel it
+ * into `insertPage` and overwrite the destination's good title / meta with
+ * empty values, so the dedicated edge-only path is required.
+ *
+ * The destination row is resolved (created on demand if a concurrent
+ * in-flight render has not committed it yet) so the edge always points at a
+ * valid id; the single render fills in the destination's content under that
+ * same id. The destination's existing anchors / images are never touched
+ * here.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param page - HEAD-resolved page data carrying the redirect chain. Its
+ *   `anchorList` / `imageList` are ignored (a redirect source owns no content).
+ * @param source - Inventory provenance forwarded by the orchestrator
+ *   (`Archive.setRedirect` → here) for the redirect-edge fast path. Used
+ *   as the fallback when the originating URL's row does NOT yet exist in
+ *   the archive (`#73` convergence on first sight, js-redirect rescue
+ *   before any prior write). When the originating row already exists
+ *   (e.g. anchor-lineage INSERT from a prior pass), its stored `source`
+ *   takes precedence so transitive lineage is preserved across resume /
+ *   retry-failed sessions. `undefined` keeps the DB DEFAULT `'crawled'`
+ *   on a brand-new destination row.
+ */
+export declare function recordRedirect(knex: Knex, caches: WriteRefCaches, page: PageData, source?: PageSource): Promise<void>;

package/lib/db-ops/pages/write/record-redirect.js

@@ -0,0 +1,100 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { dbLog } from '../../../debug.js';
+import { deriveLineageFromParent } from '../../../derive-lineage-from-parent.js';
+import { resolveRedirectChain } from '../../../resolve-redirect-chain.js';
+import { clearWriteRefCaches } from '../../_shared/clear-write-ref-caches.js';
+import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
+import { linkRedirectSources } from './link-redirect-sources.js';
+/**
+ * Records a redirect edge (source → destination) **without** re-storing the
+ * destination's content.
+ *
+ * The crawler renders a many-to-one redirect destination exactly once. For
+ * every subsequent source URL that redirects to that already-rendered
+ * destination, it calls this instead of {@link ./update-page.ts} (#73).
+ * Routing a content-less HEAD result through `updatePage` would funnel it
+ * into `insertPage` and overwrite the destination's good title / meta with
+ * empty values, so the dedicated edge-only path is required.
+ *
+ * The destination row is resolved (created on demand if a concurrent
+ * in-flight render has not committed it yet) so the edge always points at a
+ * valid id; the single render fills in the destination's content under that
+ * same id. The destination's existing anchors / images are never touched
+ * here.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param page - HEAD-resolved page data carrying the redirect chain. Its
+ *   `anchorList` / `imageList` are ignored (a redirect source owns no content).
+ * @param source - Inventory provenance forwarded by the orchestrator
+ *   (`Archive.setRedirect` → here) for the redirect-edge fast path. Used
+ *   as the fallback when the originating URL's row does NOT yet exist in
+ *   the archive (`#73` convergence on first sight, js-redirect rescue
+ *   before any prior write). When the originating row already exists
+ *   (e.g. anchor-lineage INSERT from a prior pass), its stored `source`
+ *   takes precedence so transitive lineage is preserved across resume /
+ *   retry-failed sessions. `undefined` keeps the DB DEFAULT `'crawled'`
+ *   on a brand-new destination row.
+ */
+export async function recordRedirect(knex, caches, page, source) {
+    const { destUrl, sources } = resolveRedirectChain(page.url.withoutHashAndAuth, page.redirectPaths);
+    // No redirect chain (the URL is itself the already-rendered destination,
+    // reached both directly and via a redirect) → there is no edge to write.
+    // Returning here avoids opening a transaction and, crucially, avoids
+    // `resolveContentItemId` inserting a content-less placeholder row for a
+    // destination that may not have been written yet.
+    if (sources.length === 0) {
+        return;
+    }
+    const destUrlObject = parseUrl(destUrl);
+    if (!destUrlObject) {
+        // A malformed redirect target should not abort the whole crawl (this
+        // runs inside the WriteQueue, whose rejection aborts the run). Recording
+        // a single redirect edge is best-effort, so skip it and move on. Unlike
+        // `updatePage`, there is no page content at stake here.
+        dbLog('recordRedirect: skip malformed destination URL: %s', destUrl);
+        return;
+    }
+    try {
+        await knex.transaction(async (trx) => {
+            // Pass the caller-supplied `source` straight through so a brand-new
+            // destination row INSERTed here picks up the inventory lineage
+            // (instead of the DB DEFAULT `'crawled'`) when the caller is in the
+            // inventory chain — without the pass-through, inventory lineage
+            // would be laundered to `'crawled'` for js-redirect rescue / #73
+            // convergence destinations that have not yet been rendered.
+            const destId = await resolveContentItemId(trx, caches, destUrlObject.withoutHashAndAuth, { source });
+            // Chain lineage propagates FROM the originating URL (`page.url`),
+            // NOT from the destination. The originating URL is what initiated
+            // the redirect chain, so its lineage is what every intermediate hop
+            // transitively inherits. Reading from the destination would
+            // mis-propagate in "inventory-seed → ... → existing crawled dest"
+            // chains: the intermediates are reached only via the inventory
+            // chain, so they belong to the inventory chain even though the
+            // chain happens to land on a crawled URL. The `'crawled'` fallback
+            // arms the crawled-wins downgrade for existing `'inventory-*'`
+            // intermediates that a crawled chain reaches.
+            const cachedOriginating = caches.contentItems.get(page.url.withoutHashAndAuth);
+            let originatingSource = cachedOriginating?.source;
+            if (originatingSource === undefined) {
+                const [originatingRow] = (await trx
+                    .select('ci.source')
+                    .from('content_items as ci')
+                    .join('url_refs as ur', 'ur.id', 'ci.url_id')
+                    .where('ur.url', page.url.withoutHashAndAuth));
+                originatingSource = originatingRow?.source ?? source;
+            }
+            const chainLineageSource = deriveLineageFromParent(originatingSource, 'crawled');
+            await linkRedirectSources(trx, caches, sources, destId, destUrlObject.withoutHashAndAuth, page.isExternal, chainLineageSource);
+        });
+    }
+    catch (error) {
+        // A rolled-back transaction can leave ids cached that no longer
+        // correspond to any row (AUTOINCREMENT never rewinds) — see
+        // `clearWriteRefCaches` for why a full clear, not a partial one, is
+        // required. `emitErrorAndRetry` may retry this whole call, so the
+        // cache must be clean before the next attempt (same guard as
+        // `updatePage`).
+        clearWriteRefCaches(caches);
+        throw error;
+    }
+}

package/lib/db-ops/pages/write/set-skipped-page.d.ts

@@ -0,0 +1,13 @@
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Marks a page as skipped in the database with the given reason.
+ * Creates the `content_items` row (with its `url_refs` entry) if it does
+ * not already exist.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param url - The URL of the skipped page.
+ * @param reason - The reason the page was skipped.
+ * @param isExternal - Whether the page is on an external domain. Defaults to `false`.
+ */
+export declare function setSkippedPage(knex: Knex, caches: WriteRefCaches, url: string, reason: string, isExternal?: boolean): Promise<void>;

package/lib/db-ops/pages/write/set-skipped-page.js

@@ -0,0 +1,24 @@
+import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
+/**
+ * Marks a page as skipped in the database with the given reason.
+ * Creates the `content_items` row (with its `url_refs` entry) if it does
+ * not already exist.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param url - The URL of the skipped page.
+ * @param reason - The reason the page was skipped.
+ * @param isExternal - Whether the page is on an external domain. Defaults to `false`.
+ */
+export async function setSkippedPage(knex, caches, url, reason, isExternal = false) {
+    const pageId = await resolveContentItemId(knex, caches, url, {
+        isExternal: isExternal ? 1 : 0,
+    });
+    await knex('content_items')
+        .where('id', pageId)
+        .update({
+        scraped: 1,
+        is_external: isExternal ? 1 : 0,
+        is_skipped: 1,
+        skip_reason: reason,
+    });
+}

package/lib/db-ops/pages/write/update-page.d.ts

@@ -0,0 +1,40 @@
+import type { PageDataWithDomPaths, PageSource } from '../../../types.js';
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Inserts or updates a crawled page — `content_items` + `page_meta` plus its
+ * redirect chain, `anchor_edges`, `image_items`, and (when `writeHtml`) its
+ * compressed HTML snapshot BLOB.
+ *
+ * Self-redirects (where the source URL equals the destination URL after
+ * normalization) are skipped to avoid marking a page as redirected to itself
+ * — a situation caused by authentication challenges (e.g. Basic Auth 302)
+ * that would otherwise exclude the page from reports via the
+ * `whereNull('redirect_dest_id')` filter.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param page - The page data to store, optionally carrying the in-browser
+ *   dom-path capture (see {@link PageDataWithDomPaths}).
+ * @param writeHtml - When `true`, this call is allowed to insert (or clear)
+ *   the page's HTML blob. `setExternalPage` passes `false` because external
+ *   metadata-only scrapes never carry HTML and must not perturb an already
+ *   stored body.
+ * @param isTarget - Whether this page is a crawl target.
+ * @param source - Provenance label written ONLY when the row is freshly
+ *   inserted. Existing rows keep their original `source` (this is why a
+ *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
+ *   that was discovered earlier).
+ * @param bodyHash - Precomputed body hash for the page's HTML (see
+ *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
+ *   computing it from the HTML instead.
+ * @param recursive - The crawl session's `recursive` option, forwarded to
+ *   `replaceAnchorEdges` so it can compute each discovered anchor's
+ *   `is_metadata_only` value the same way `processAnchors`
+ *   (`handle-scrape-end.ts`) decides whether to queue it for a full or
+ *   metadata-only scrape (#369). Defaults to `true` (the historical,
+ *   pre-#369 behaviour: never persist an anchor as metadata-only) for
+ *   callers that do not track the option, such as `setExternalPage`, whose
+ *   anchor list is always empty anyway.
+ * @returns The database `pageId` (`content_items.id`) of the inserted or updated row.
+ */
+export declare function updatePage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, writeHtml: boolean, isTarget: boolean, source?: PageSource, bodyHash?: Buffer | null, recursive?: boolean): Promise<number>;