@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/db-ops/pages/write/update-page.js

@@ -0,0 +1,419 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { computeBodyHash } from '../../../body-hash/compute-body-hash.js';
+import { isHtmlContentType } from '../../../content-type/is-html-content-type.js';
+import { dbLog } from '../../../debug.js';
+import { deriveLineageFromParent } from '../../../derive-lineage-from-parent.js';
+import { matchImagesToDomPaths } from '../../../populate-entity-tables/match-images-to-dom-paths.js';
+import { upsertTextRefs } from '../../../populate-entity-tables/upsert-text-refs.js';
+import { resolveRedirectChain } from '../../../resolve-redirect-chain.js';
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+import { clearWriteRefCaches } from '../../_shared/clear-write-ref-caches.js';
+import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
+import { resolveUrlOrBlob } from '../../_shared/resolve-url-or-blob.js';
+import { insertAudios } from './insert-audios.js';
+import { insertButtons } from './insert-buttons.js';
+import { insertCanvases } from './insert-canvases.js';
+import { insertCustomElements } from './insert-custom-elements.js';
+import { insertHeadings } from './insert-headings.js';
+import { insertIframes } from './insert-iframes.js';
+import { insertJsonLd } from './insert-jsonld.js';
+import { insertMainContentImages } from './insert-main-content-images.js';
+import { insertMainContentTables } from './insert-main-content-tables.js';
+import { insertPage } from './insert-page.js';
+import { insertTechnologies } from './insert-technologies.js';
+import { insertVideos } from './insert-videos.js';
+import { linkRedirectSources } from './link-redirect-sources.js';
+import { writePageHtmlBlob } from './write-page-html-blob.js';
+/**
+ * Inserts or updates a crawled page — `content_items` + `page_meta` plus its
+ * redirect chain, `anchor_edges`, `image_items`, and (when `writeHtml`) its
+ * compressed HTML snapshot BLOB.
+ *
+ * Self-redirects (where the source URL equals the destination URL after
+ * normalization) are skipped to avoid marking a page as redirected to itself
+ * — a situation caused by authentication challenges (e.g. Basic Auth 302)
+ * that would otherwise exclude the page from reports via the
+ * `whereNull('redirect_dest_id')` filter.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param page - The page data to store, optionally carrying the in-browser
+ *   dom-path capture (see {@link PageDataWithDomPaths}).
+ * @param writeHtml - When `true`, this call is allowed to insert (or clear)
+ *   the page's HTML blob. `setExternalPage` passes `false` because external
+ *   metadata-only scrapes never carry HTML and must not perturb an already
+ *   stored body.
+ * @param isTarget - Whether this page is a crawl target.
+ * @param source - Provenance label written ONLY when the row is freshly
+ *   inserted. Existing rows keep their original `source` (this is why a
+ *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
+ *   that was discovered earlier).
+ * @param bodyHash - Precomputed body hash for the page's HTML (see
+ *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
+ *   computing it from the HTML instead.
+ * @param recursive - The crawl session's `recursive` option, forwarded to
+ *   `replaceAnchorEdges` so it can compute each discovered anchor's
+ *   `is_metadata_only` value the same way `processAnchors`
+ *   (`handle-scrape-end.ts`) decides whether to queue it for a full or
+ *   metadata-only scrape (#369). Defaults to `true` (the historical,
+ *   pre-#369 behaviour: never persist an anchor as metadata-only) for
+ *   callers that do not track the option, such as `setExternalPage`, whose
+ *   anchor list is always empty anyway.
+ * @returns The database `pageId` (`content_items.id`) of the inserted or updated row.
+ */
+export async function updatePage(knex, caches, page, writeHtml, isTarget, source, bodyHash, recursive = true) {
+    const { destUrl, sources } = resolveRedirectChain(page.url.withoutHashAndAuth, page.redirectPaths);
+    const destUrlObject = parseUrl(destUrl);
+    if (!destUrlObject) {
+        throw new Error(`Failed to parse URL: ${destUrl}`);
+    }
+    try {
+        return await knex.transaction(async (trx) => {
+            return await updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash, recursive);
+        });
+    }
+    catch (error) {
+        // A rolled-back transaction can leave ids cached that no longer
+        // correspond to any row (AUTOINCREMENT never rewinds) — see
+        // `clearWriteRefCaches` for why a full clear, not a partial one,
+        // is required. `emitErrorAndRetry` may retry this whole call, so
+        // the cache must be clean before the next attempt.
+        clearWriteRefCaches(caches);
+        throw error;
+    }
+}
+/**
+ * The transactional body of {@link updatePage}, extracted so the
+ * transaction boundary in the caller can wrap it in a try/catch for
+ * cache invalidation on rollback.
+ * @param trx - The active transaction.
+ * @param knex - Knex query builder connected to the archive DB (passed
+ *   through to `insertPage` for its `trx ?? knex` fallback contract).
+ * @param caches - The connection's write-side id caches.
+ * @param page - The page data to store.
+ * @param destUrlObject - The parsed redirect-chain destination URL.
+ * @param sources - Redirect-source URLs from the chain resolution.
+ * @param writeHtml - See {@link updatePage}.
+ * @param isTarget - See {@link updatePage}.
+ * @param source - See {@link updatePage}.
+ * @param bodyHash - See {@link updatePage}.
+ * @param recursive - See {@link updatePage}.
+ * @returns The `content_items.id` of the inserted or updated row.
+ */
+async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash, recursive) {
+    const pageId = await insertPage(knex, caches, {
+        ...page,
+        url: destUrlObject,
+    }, isTarget, trx, source);
+    // Technology detection combines a Wappalyzer signal source (HTML-body
+    // independent — relies on `<script src>` / `<iframe src>` / window
+    // globals / response headers, so it fires for every page including
+    // external / metadata-only) with structural signals that scan
+    // `page.html` (naturally a no-op against the empty string an
+    // external/metadata-only scrape leaves it as). Both run unconditionally,
+    // unlike JSON-LD below, which lives inside the rendered HTML body and
+    // is only written when there is HTML to scrape — see the same
+    // `writeHtml` gate as `writePageHtmlBlob` below.
+    await insertTechnologies(pageId, page.html, page.meta, trx);
+    if (writeHtml) {
+        await insertJsonLd(pageId, page.meta, trx);
+    }
+    // beholder's MainContentsData is captured in the same render pass as the
+    // HTML body / JSON-LD, so it shares the same `writeHtml` + non-null gate:
+    // external / non-HTML / metadata-only scrapes never populate it.
+    if (writeHtml && page.mainContents) {
+        await insertHeadings(pageId, page.mainContents, trx);
+        await insertMainContentImages(pageId, page.mainContents, trx);
+        await insertMainContentTables(pageId, page.mainContents, trx);
+        await insertButtons(pageId, page.mainContents, trx);
+        await insertIframes(pageId, page.mainContents, trx);
+        await insertVideos(pageId, page.mainContents, trx);
+        await insertAudios(pageId, page.mainContents, trx);
+        await insertCanvases(pageId, page.mainContents, trx);
+        // Unlike the eight calls above, the source here is nitpicker's own
+        // `capture-custom-elements.ts` capture, not `page.mainContents` — see
+        // `insert-custom-elements.ts`. That capture can fail independently
+        // of `page.mainContents` (a separate `page.evaluate()` call), so
+        // `undefined` (capture failed) must skip the call entirely — passing
+        // a defaulted `[]` would look identical to "found zero" and, per
+        // `insertCustomElements`'s docs, wipe stale-but-still-good rows.
+        if (page.mainContentCustomElements !== undefined) {
+            await insertCustomElements(pageId, page.mainContentCustomElements, trx);
+        }
+    }
+    // Chain lineage propagates FROM the originating URL
+    // (`page.url`), NOT from the destination. See the matching
+    // rationale in `recordRedirect`: intermediates are reached
+    // transitively from the originating URL's render, so they
+    // inherit its lineage. The `source` argument is the
+    // authoritative origin label when inventoryMode is live; fall
+    // through to a DB lookup of `page.url` for the resume /
+    // retry-failed path where the call-site has no source.
+    let originatingSource = source;
+    if (originatingSource === undefined) {
+        originatingSource = await readSourceByUrl(trx, caches, page.url.withoutHashAndAuth);
+    }
+    const chainLineageSource = deriveLineageFromParent(originatingSource, 'crawled');
+    await linkRedirectSources(trx, caches, sources, pageId, destUrlObject.withoutHashAndAuth, page.isExternal, chainLineageSource);
+    // Only insert a snapshot blob when there is actual HTML to write.
+    // `page.html.length > 0` is the precise signal: the scraper returns
+    // `html: ''` for everything that is not a rendered `text/html` document
+    // (non-HTML responses, metadata-only, external, degraded renders), so a
+    // non-empty `html` is exactly "a rendered HTML body exists". Gating on
+    // `isTarget` alone would store an empty body for every internal non-HTML
+    // resource — PDF / zip / images are isTarget=1 (#72).
+    //
+    // `isTarget` is intentionally NOT part of this condition: it is implied by
+    // `html.length > 0` (only in-scope target pages are browser-rendered into a
+    // non-empty body; metadata-only and external pages carry `html: ''`), so the
+    // content check alone expresses the intent without a redundant term.
+    if (writeHtml && page.html.length > 0) {
+        await writePageHtmlBlob(pageId, page.html, trx);
+        // Prefer the caller's precomputed hash (the crawler already hashed
+        // this exact `page.html` once, before this write path ever runs) —
+        // falling back to computing it here only when the caller has not
+        // (direct callers such as tests, or a future write path that has not
+        // adopted the precompute). Either way this stays derived from the
+        // same `page.html` written above, in the same transaction, so
+        // `page_meta.body_hash` never observably lags the snapshot it was
+        // derived from.
+        await trx('page_meta')
+            .where('page_id', pageId)
+            .update({ body_hash: bodyHash ?? computeBodyHash(page.html) });
+    }
+    else if (writeHtml &&
+        page.contentType !== null &&
+        !isHtmlContentType(page.contentType)) {
+        // The page is now a *known* non-HTML type. If a previous scrape stored
+        // an HTML body for this URL (e.g. it served HTML then was replaced by
+        // a PDF across `crawl --resume` / `--append`), drop the stale ref so
+        // `page_html_ref` never contradicts the content type. A degraded HTML
+        // re-scrape (text/html or unknown content type with empty html) is NOT
+        // cleared — the last good snapshot is preserved, mirroring the
+        // anchors / images empty-guard below. Gated on `writeHtml` because a
+        // stale ref can only have been written by a snapshot-capable call
+        // (`setPage`); `setExternalPage` passes `writeHtml = false` and never
+        // sets `html`, so it has nothing to clear.
+        //
+        // `page_meta.body_hash` is cleared alongside it for the same reason —
+        // it is derived from the same HTML `page_html_ref` points at, so it
+        // must not outlive the ref it was computed from.
+        await trx('page_html_ref').where('page_id', pageId).delete();
+        await trx('page_meta').where('page_id', pageId).update({ body_hash: null });
+    }
+    // Re-scrape semantics: the same URL can be scraped more than once
+    // (e.g. `crawl --resume`, re-visits, `--append` re-promotion). Edges
+    // are replaced wholesale per page on every non-empty scrape — the
+    // delete is paired with, and guarded by, a non-empty new list: a
+    // degraded re-scrape (navigation timeout / partial render) can return
+    // an empty `anchorList` for a page that previously had links, and
+    // wiping the prior good data in that case would be destructive. We
+    // cannot tell a transient empty result apart from a page that has
+    // legitimately lost all its links, so we err on the side of keeping
+    // what we already had. The accepted trade-off is that a page which
+    // genuinely dropped to zero links keeps its stale rows until the next
+    // non-empty re-scrape replaces them (#70).
+    //
+    // Lineage propagation: read the current page's merged source
+    // (post-UPDATE by `insertPage`) so anchor placeholder rows inherit a
+    // label that reflects the parent's chain. A `'crawled'`-lineage
+    // parent passes `'crawled'` explicitly so the crawled-wins downgrade
+    // in `resolveContentItemId` fires when an anchor hits an existing
+    // `'inventory-*'` row. An inventory-lineage parent passes
+    // `'inventory-discovered'` to label transitively-reached URLs
+    // correctly without the orchestrator needing to rehydrate
+    // `inventoryMode` from disk.
+    const [parentRow] = (await trx
+        .select('source')
+        .from('content_items')
+        .where('id', pageId));
+    const anchorLineageSource = deriveLineageFromParent(parentRow?.source, 'crawled');
+    await replaceAnchorEdges(trx, caches, pageId, page, anchorLineageSource, recursive);
+    await replaceImageItems(trx, caches, pageId, page);
+    // Clear this page's resource_ref_edges unconditionally (no non-empty
+    // guard, unlike anchors/images above): the crawler always emits this
+    // page's `responseReferrers` events right after its `page` event (see
+    // `Crawler#handleResources`, called immediately after `#handleResult`
+    // for the same scrape), through the same serialized WriteQueue, so the
+    // fresh INSERT is guaranteed to follow this DELETE in commit order —
+    // no writer can interleave a stale re-insert between them. This is a
+    // write-ordering guarantee only: the DELETE and the follow-up INSERT
+    // are still two separate transactions, so a concurrent read-only
+    // connection (viewer / MCP open on the same archive during an active
+    // `--append` / `--retry-failed` run) can observe a momentary window
+    // with zero rows for this page and misclassify its resources as
+    // unused. The window closes as soon as the next transaction commits,
+    // so this is a transient display artifact, not a durable data loss —
+    // accepted rather than merging the two writes into one transaction.
+    // A degraded re-scrape that legitimately captures zero sub-resources
+    // leaves this page referrer-less until its next non-empty re-scrape —
+    // accepted, since resource_items rows for no-longer-referenced
+    // resources are themselves allowed to become orphaned (no cross-page
+    // cleanup is attempted for those either).
+    await trx('resource_ref_edges').where('page_id', pageId).delete();
+    return pageId;
+}
+/**
+ * Reads the `source` of the row identified by `url` without creating a
+ * placeholder — the cache-first mirror of the legacy by-URL SELECT.
+ * Returns `undefined` when no row exists yet.
+ * @param trx - The active transaction.
+ * @param caches - The connection's write-side id caches (read-only here;
+ *   a DB fallback does not populate the identity cache because the id is
+ *   not needed).
+ * @param url - Normalised URL key.
+ */
+async function readSourceByUrl(trx, caches, url) {
+    const cached = caches.contentItems.get(url);
+    if (cached !== undefined) {
+        return cached.source;
+    }
+    const [row] = (await trx
+        .select('ci.source')
+        .from('content_items as ci')
+        .join('url_refs as ur', 'ur.id', 'ci.url_id')
+        .where('ur.url', url));
+    return row?.source;
+}
+/**
+ * Replaces the page's `anchor_edges` rows with the aggregate of the new
+ * scrape's `anchorList` (empty-guarded; see the caller's re-scrape
+ * comment). Instances sharing the same resolved `href_page_id` collapse
+ * into one edge whose `count` is the instance count and whose
+ * `first_hash` / `first_text_id` capture the first instance encountered
+ * in list order — the same first-wins rule the archive migration applies
+ * to legacy `anchors` rows.
+ * @param trx - The active transaction.
+ * @param caches - The connection's write-side id caches.
+ * @param pageId - The owning `content_items.id`.
+ * @param page - The scraped page payload.
+ * @param anchorLineageSource - Lineage label for anchor-target
+ *   placeholder rows (see the caller's lineage comment).
+ * @param recursive - The crawl session's `recursive` option. Combined with
+ *   each anchor's `isExternal` (already resolved by `processAnchors` before
+ *   this ever runs) to compute `is_metadata_only` the same way
+ *   `processAnchors` (`handle-scrape-end.ts`) decides whether to queue the
+ *   anchor for a full or metadata-only scrape: `!recursive || isExternal`.
+ *   Both `recursive` and a given URL's `isExternal` are constant for the
+ *   duration of one `CrawlerOrchestrator#crawling()` invocation (this is a
+ *   pure recomputation of that call's own decision, not a second source of
+ *   truth, #369) — NOT necessarily across the whole session:
+ *   `#crawlUntilPendingClears`'s auto-retry loop (issue #350) always
+ *   re-invokes `crawling([], { recursive: false, isRetryContinuation: true })`
+ *   for a retry round regardless of the session's original `recursive`
+ *   value, so the SAME URL anchored again during a retry round can compute
+ *   a different `is_metadata_only` verdict than it did on the original
+ *   pass. `resolveContentItemId`'s ratchet lets the later call's opinion
+ *   win, same as it does for any other repeat discovery.
+ */
+async function replaceAnchorEdges(trx, caches, pageId, page, anchorLineageSource, recursive) {
+    if (page.anchorList.length === 0) {
+        return;
+    }
+    const edges = new Map();
+    for (const anchor of page.anchorList) {
+        const isExternal = anchor.isExternal ?? false;
+        const hrefId = await resolveContentItemId(trx, caches, anchor.href.withoutHashAndAuth, {
+            isExternal: isExternal ? 1 : 0,
+            source: anchorLineageSource,
+            isMetadataOnly: !recursive || isExternal ? 1 : 0,
+        });
+        const existing = edges.get(hrefId);
+        if (existing !== undefined) {
+            existing.count += 1;
+            continue;
+        }
+        edges.set(hrefId, {
+            count: 1,
+            firstHash: anchor.href.hash || null,
+            firstText: anchor.textContent || null,
+        });
+    }
+    const texts = new Set();
+    for (const edge of edges.values()) {
+        if (edge.firstText !== null) {
+            texts.add(edge.firstText);
+        }
+    }
+    const textIds = await upsertTextRefs(trx, texts);
+    dbLog('Replace anchor_edges: %d edges (%d instances)', edges.size, page.anchorList.length);
+    await trx('anchor_edges').where('page_id', pageId).delete();
+    const rows = [...edges.entries()].map(([hrefPageId, edge]) => ({
+        page_id: pageId,
+        href_page_id: hrefPageId,
+        count: edge.count,
+        first_hash: edge.firstHash,
+        first_text_id: edge.firstText === null ? null : (textIds.get(edge.firstText) ?? null),
+    }));
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('anchor_edges').insert(chunk);
+    });
+}
+/**
+ * Replaces the page's `image_items` rows with the new scrape's
+ * `imageList` (empty-guarded; see the caller's re-scrape comment).
+ * `src` / `currentSrc` route to `url_refs` or (for large `data:` URIs)
+ * `blob_refs`; `dom_path_text_id` resolves by matching each image's
+ * `sourceCode` against the in-browser capture carried on
+ * `page.imageDomPaths`, falling back to the synthetic `unknown/<n>`
+ * marker per image when the capture is absent or ambiguous.
+ * @param trx - The active transaction.
+ * @param caches - The connection's write-side id caches.
+ * @param pageId - The owning `content_items.id`.
+ * @param page - The scraped page payload.
+ */
+async function replaceImageItems(trx, caches, pageId, page) {
+    if (page.imageList.length === 0) {
+        return;
+    }
+    // Match by list index: ids only need to be unique within this call and
+    // ordered in insertion order, which the list order is by construction.
+    const domPaths = matchImagesToDomPaths(page.imageList.map((image, index) => ({
+        id: index,
+        sourceCode: image.sourceCode || null,
+    })), page.imageDomPaths ?? []);
+    const texts = new Set();
+    for (const image of page.imageList) {
+        if (typeof image.alt === 'string' && image.alt !== '') {
+            texts.add(image.alt);
+        }
+    }
+    for (const entry of domPaths.values()) {
+        texts.add(entry.path);
+    }
+    const textIds = await upsertTextRefs(trx, texts);
+    const rows = [];
+    for (const [index, image] of page.imageList.entries()) {
+        const domPath = domPaths.get(index);
+        if (domPath === undefined) {
+            throw new Error(`replaceImageItems: dom_path not resolved for image #${index}`);
+        }
+        const domPathId = textIds.get(domPath.path);
+        if (domPathId === undefined) {
+            throw new Error(`replaceImageItems: text_refs.id not resolved for dom_path=${domPath.path}`);
+        }
+        const srcSlot = await resolveUrlOrBlob(trx, caches, image.src);
+        const currentSrcSlot = await resolveUrlOrBlob(trx, caches, image.currentSrc);
+        rows.push({
+            page_id: pageId,
+            src_url_id: srcSlot.url,
+            current_src_url_id: currentSrcSlot.url,
+            src_blob_id: srcSlot.blob,
+            current_src_blob_id: currentSrcSlot.blob,
+            alt_text_id: typeof image.alt === 'string' && image.alt !== ''
+                ? (textIds.get(image.alt) ?? null)
+                : null,
+            width: image.width,
+            height: image.height,
+            natural_width: image.naturalWidth,
+            natural_height: image.naturalHeight,
+            is_lazy: image.isLazy == null ? null : image.isLazy ? 1 : 0,
+            viewport_width: image.viewportWidth,
+            dom_path_text_id: domPathId,
+        });
+    }
+    dbLog('Replace image_items: %d rows', rows.length);
+    await trx('image_items').where('page_id', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('image_items').insert(chunk);
+    });
+}

package/lib/db-ops/pages/write/write-page-html-blob.d.ts

@@ -0,0 +1,19 @@
+import type { Knex } from 'knex';
+/**
+ * Encodes, dedups, and persists a page's HTML snapshot.
+ *
+ * Computes SHA-256 over the raw UTF-8 bytes, compresses them with zstd,
+ * inserts into `page_html_blobs` only if the hash is new (so identical
+ * bodies — 404 templates, error pages, redirect destinations — share a
+ * single row), and then upserts `page_html_ref(page_id → hash)` so the
+ * latest scrape always points at the right body.
+ *
+ * Runs entirely inside the caller's transaction; a failure here rolls
+ * back the rest of `updatePage`, which is the desired semantics (an
+ * archive that lost its HTML for a page would otherwise serve stale
+ * meta against a missing body).
+ * @param pageId - The database id of the page.
+ * @param html - The raw HTML string (UTF-8).
+ * @param trx - The active transaction.
+ */
+export declare function writePageHtmlBlob(pageId: number, html: string, trx: Knex.Transaction): Promise<void>;

package/lib/db-ops/pages/write/write-page-html-blob.js

@@ -0,0 +1,41 @@
+import { createHash } from 'node:crypto';
+import { compressPayload } from '../../_shared/compress-payload.js';
+/**
+ * Encodes, dedups, and persists a page's HTML snapshot.
+ *
+ * Computes SHA-256 over the raw UTF-8 bytes, compresses them with zstd,
+ * inserts into `page_html_blobs` only if the hash is new (so identical
+ * bodies — 404 templates, error pages, redirect destinations — share a
+ * single row), and then upserts `page_html_ref(page_id → hash)` so the
+ * latest scrape always points at the right body.
+ *
+ * Runs entirely inside the caller's transaction; a failure here rolls
+ * back the rest of `updatePage`, which is the desired semantics (an
+ * archive that lost its HTML for a page would otherwise serve stale
+ * meta against a missing body).
+ * @param pageId - The database id of the page.
+ * @param html - The raw HTML string (UTF-8).
+ * @param trx - The active transaction.
+ */
+export async function writePageHtmlBlob(pageId, html, trx) {
+    const rawBytes = Buffer.from(html, 'utf8');
+    const hash = createHash('sha256').update(rawBytes).digest();
+    const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
+    await trx('page_html_blobs')
+        .insert({
+        hash,
+        body,
+        codec,
+        size_raw: sizeRaw,
+        size_stored: sizeStored,
+    })
+        .onConflict('hash')
+        .ignore();
+    // Upsert so a re-scrape's body cleanly supersedes the prior pointer.
+    // The old blob row is intentionally left in place — a future #23 GC
+    // pass will sweep unreachable hashes.
+    await trx('page_html_ref')
+        .insert({ page_id: pageId, hash })
+        .onConflict('page_id')
+        .merge(['hash']);
+}

package/lib/db-ops/referrers/get-redirects-for-pages.d.ts

@@ -0,0 +1,9 @@
+import type { DB_Redirect } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves redirect sources for the given page IDs in bulk.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageIds - The database IDs of the destination pages.
+ * @returns An array of {@link DB_Redirect} records mapping destination pages to their redirect sources.
+ */
+export declare function getRedirectsForPages(knex: Knex, pageIds: number[]): Promise<DB_Redirect[]>;

package/lib/db-ops/referrers/get-redirects-for-pages.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves redirect sources for the given page IDs in bulk.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageIds - The database IDs of the destination pages.
+ * @returns An array of {@link DB_Redirect} records mapping destination pages to their redirect sources.
+ */
+export async function getRedirectsForPages(knex, pageIds) {
+    if (pageIds.length === 0)
+        return [];
+    return knex
+        .select('content_items.redirect_dest_id as pageId', 'url_refs.url as from', 'content_items.id as fromId')
+        .from('content_items')
+        .join('url_refs', 'url_refs.id', '=', 'content_items.url_id')
+        .whereIn('content_items.redirect_dest_id', pageIds);
+}

package/lib/db-ops/referrers/get-referrers-of-page.d.ts

@@ -0,0 +1,17 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves pages that link to a specific page (incoming links / referrers).
+ *
+ * Incoming links are resolved **through redirects**: an anchor pointing at a
+ * redirect source (e.g. `http://x` that 301s to `https://x`) counts as a
+ * referrer of the redirect's final destination, not of the source. This keeps
+ * backlinks merged on the canonical page instead of splitting them across the
+ * `http`/`https` (or any redirect source/dest) pair. The resolution mirrors
+ * `getPagesWithRels`' redirect handling — `redirect_dest_id` is
+ * pre-flattened to the final destination, so
+ * `COALESCE(target.redirect_dest_id, target.id)` is a single hop.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the target page.
+ * @returns An array of referrer records with URL, hash, and text content.
+ */
+export declare function getReferrersOfPage(knex: Knex, pageId: number): Promise<any[]>;

package/lib/db-ops/referrers/get-referrers-of-page.js

@@ -0,0 +1,32 @@
+/**
+ * Retrieves pages that link to a specific page (incoming links / referrers).
+ *
+ * Incoming links are resolved **through redirects**: an anchor pointing at a
+ * redirect source (e.g. `http://x` that 301s to `https://x`) counts as a
+ * referrer of the redirect's final destination, not of the source. This keeps
+ * backlinks merged on the canonical page instead of splitting them across the
+ * `http`/`https` (or any redirect source/dest) pair. The resolution mirrors
+ * `getPagesWithRels`' redirect handling — `redirect_dest_id` is
+ * pre-flattened to the final destination, so
+ * `COALESCE(target.redirect_dest_id, target.id)` is a single hop.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the target page.
+ * @returns An array of referrer records with URL, hash, and text content.
+ */
+export async function getReferrersOfPage(knex, pageId) {
+    const res = await knex
+        .select('referrer_url.url as url', 
+    // `through` / `throughId` = the URL the anchor actually pointed at (the
+    // redirect source, e.g. `http://x`), mirroring `getPagesWithRels`'
+    // `redirect.from` / `redirect.fromId`. Lets report code print the
+    // "[REDIRECTED FROM]" note even on this (non-preloaded) referrer path.
+    'target_url.url as through', 'target.id as throughId', 'anchor_edges.first_hash as hash', 'first_text.text as textContent')
+        .from('anchor_edges')
+        .join('content_items as referrer', 'anchor_edges.page_id', '=', 'referrer.id')
+        .join('url_refs as referrer_url', 'referrer.url_id', '=', 'referrer_url.id')
+        .join('content_items as target', 'anchor_edges.href_page_id', '=', 'target.id')
+        .join('url_refs as target_url', 'target.url_id', '=', 'target_url.id')
+        .leftJoin('text_refs as first_text', 'anchor_edges.first_text_id', '=', 'first_text.id')
+        .whereRaw('coalesce("target"."redirect_dest_id", "target"."id") = ?', [pageId]);
+    return res;
+}

package/lib/db-ops/referrers/get-referrers-of-resource.d.ts

@@ -0,0 +1,8 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves the page URLs that reference a specific resource.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The database ID of the resource.
+ * @returns An array of page URL strings that reference the resource.
+ */
+export declare function getReferrersOfResource(knex: Knex, id: number): Promise<string[]>;

package/lib/db-ops/referrers/get-referrers-of-resource.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves the page URLs that reference a specific resource.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The database ID of the resource.
+ * @returns An array of page URL strings that reference the resource.
+ */
+export async function getReferrersOfResource(knex, id) {
+    const res = await knex
+        .select('url_refs.url as url')
+        .from('resource_ref_edges')
+        .join('content_items', 'content_items.id', '=', 'resource_ref_edges.page_id')
+        .join('url_refs', 'url_refs.id', '=', 'content_items.url_id')
+        .where('resource_ref_edges.resource_id', id);
+    return res.map((r) => r.url);
+}

package/lib/db-ops/resources/build-resource-query.d.ts

@@ -0,0 +1,25 @@
+import type { Knex } from 'knex';
+/**
+ * Builds a `resource_items` query joined against `url_refs` and
+ * `content_type_refs`, reconstructing a flat, legacy-shaped resource row.
+ * Selected columns are aliased to `DB_Resource` field names. `responseHeaders`
+ * is NOT reconstructed here (it needs a second, batched pass — see
+ * {@link ../resources/reconstruct-resource-rows.js}) so this query stays a
+ * single-pass join with no N+1 subqueries; `headerSetId` is selected as an
+ * intermediate for that second pass.
+ *
+ * The `url_refs` join is a LEFT JOIN, not INNER: a resource whose own
+ * identity URL is a large `data:` URI has `url_id = NULL` (routed to
+ * `url_blob_id` / `blob_refs` instead — see `create-entity-tables.ts`'s
+ * `resource_items url / blob mutual-exclusion CHECK`). An INNER JOIN
+ * would silently drop such rows from every reader. `url` reads back as
+ * `null` for those rows, matching the established `image_items` src /
+ * blob convention (`list-images.ts`) — recovering the raw data URI would
+ * require zstd-decompression in JS, which no current caller needs.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns A query builder pre-configured with the joins `get-resources.ts` /
+ *   `get-resource-by-url.ts` need; callers add `.where()` on top.
+ * @example
+ * const rows = await buildResourceQuery(knex).whereIn('ur.url', urls);
+ */
+export declare function buildResourceQuery(knex: Knex): Knex.QueryBuilder;

package/lib/db-ops/resources/build-resource-query.js

@@ -0,0 +1,29 @@
+/**
+ * Builds a `resource_items` query joined against `url_refs` and
+ * `content_type_refs`, reconstructing a flat, legacy-shaped resource row.
+ * Selected columns are aliased to `DB_Resource` field names. `responseHeaders`
+ * is NOT reconstructed here (it needs a second, batched pass — see
+ * {@link ../resources/reconstruct-resource-rows.js}) so this query stays a
+ * single-pass join with no N+1 subqueries; `headerSetId` is selected as an
+ * intermediate for that second pass.
+ *
+ * The `url_refs` join is a LEFT JOIN, not INNER: a resource whose own
+ * identity URL is a large `data:` URI has `url_id = NULL` (routed to
+ * `url_blob_id` / `blob_refs` instead — see `create-entity-tables.ts`'s
+ * `resource_items url / blob mutual-exclusion CHECK`). An INNER JOIN
+ * would silently drop such rows from every reader. `url` reads back as
+ * `null` for those rows, matching the established `image_items` src /
+ * blob convention (`list-images.ts`) — recovering the raw data URI would
+ * require zstd-decompression in JS, which no current caller needs.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns A query builder pre-configured with the joins `get-resources.ts` /
+ *   `get-resource-by-url.ts` need; callers add `.where()` on top.
+ * @example
+ * const rows = await buildResourceQuery(knex).whereIn('ur.url', urls);
+ */
+export function buildResourceQuery(knex) {
+    return knex('resource_items as ri')
+        .leftJoin('url_refs as ur', 'ur.id', 'ri.url_id')
+        .leftJoin('content_type_refs as ctr', 'ctr.id', 'ri.content_type_id')
+        .select('ri.id as id', 'ur.url as url', 'ri.is_external as isExternal', 'ri.status as status', 'ri.status_text as statusText', 'ctr.raw as contentType', 'ri.content_length as contentLength', 'ri.header_set_id as headerSetId', 'ri.compress as compress', 'ri.cdn as cdn', 'ri.source as source');
+}

package/lib/db-ops/resources/get-existing-resource-urls.d.ts

@@ -0,0 +1,9 @@
+import type { Knex } from 'knex';
+/**
+ * Return the subset of `urls` that already exist as `resource_items` rows.
+ * See `getExistingPageUrls` — same chunking strategy.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param urls - URL strings to probe.
+ * @returns URLs found among `resource_items`.
+ */
+export declare function getExistingResourceUrls(knex: Knex, urls: readonly string[]): Promise<string[]>;