@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/db-ops/lifecycle/checkpoint.js

@@ -0,0 +1,9 @@
+/**
+ * Forces a WAL checkpoint, writing all pending WAL data back to the main database file.
+ * Uses TRUNCATE mode to reset the WAL file to zero bytes after checkpointing.
+ * This ensures the database is fully self-contained in `db.sqlite` before archiving.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export async function checkpoint(knex) {
+    await knex.raw('PRAGMA wal_checkpoint(TRUNCATE)');
+}

package/lib/db-ops/lifecycle/destroy.d.ts

@@ -0,0 +1,6 @@
+import type { Knex } from 'knex';
+/**
+ * Destroys the database connection, releasing all pooled resources.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export declare function destroy(knex: Knex): Promise<void>;

package/lib/db-ops/lifecycle/destroy.js

@@ -0,0 +1,7 @@
+/**
+ * Destroys the database connection, releasing all pooled resources.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export async function destroy(knex) {
+    await knex.destroy();
+}

package/lib/db-ops/lifecycle/init.d.ts

@@ -0,0 +1,58 @@
+import type { Knex } from 'knex';
+/**
+ * Initializes the database schema if tables do not exist, then runs the
+ * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
+ * `page_meta.main_content_*`, `list_reconcile_runs.invalid_skipped`,
+ * `list_reconcile_runs.exclude_skipped`).
+ *
+ * There is deliberately no per-table *table-creation* migration chain here:
+ * `assertCompatibleVersion` (called below, before any schema work) rejects
+ * every archive older than the current format, so a connection that reaches
+ * `initSchema` is either brand new (`initSchema` provisions the full schema)
+ * or was produced by `scripts/migrate-to-0.13.mjs` (which guarantees the
+ * full table set before it repacks) — and `initSchema` itself re-runs
+ * `createEntityTables` / `createAdjunctTables` unconditionally on every
+ * open, self-healing any *missing table* via their internal
+ * `IF NOT EXISTS` / `hasTable` guards. What that self-healing cannot do is
+ * retrofit a *new column* onto an entity table that already exists —
+ * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
+ * additions to an existing 0.13 table are therefore the one case that still
+ * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
+ * `migrateInfoCreatedCwd`, `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
+ * `migratePageMetaConsoleErrorCount`, `migratePageMetaCustomElementCount`,
+ * `migratePageMetaImageScan`, `migrateContentItemsAliasOfId`,
+ * `migrateContentItemsIsMetadataOnly`,
+ *
+ * `migratePageTagsToPageTechnologies` is the one exception to "column adds
+ * only": it converts `page_tags` (removed) rows into `technology_signals`/
+ * `page_technologies` and DROPS the old table — a one-time table-level ETL,
+ * not a column add, but it belongs in this same boot phase for the same
+ * reason (self-healing an old archive's schema before any reader runs).
+ *
+ * `migrateContentItemsDedupeCapEventId`, `migrateListReconcileRunsInvalidSkipped`,
+ * `migrateListReconcileRunsExcludeSkipped`) rather than a DDL-string change alone.
+ *
+ * `migrateInventoryRunsToListReconcileRuns` runs BEFORE `initSchema`
+ * (unlike every other migration here) — see its own JSDoc for why: a
+ * plain rename must land before `createAdjunctTables`'s
+ * `IF NOT EXISTS` guard would otherwise create an empty `list_reconcile_runs`
+ * and strand the old table's rows.
+ *
+ * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
+ * change) but belongs at this same boot phase for the same reason the
+ * others do: it must run before ANY reader (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, …) can observe a `network_outages` row
+ * left `ended_at = NULL` by a crawl process that was killed mid-outage in a
+ * prior session.
+ *
+ * In read-only mode schema init + migration are SKIPPED so the same DB
+ * can be opened safely by a viewer attached to a live (or interrupted)
+ * crawl without rewriting the user's tmpDir.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param readOnly - When true, skip schema init + migrations.
+ * @param onLog - Forwarded to every `migrate*` call below (issue #294) —
+ *   see {@link import('../../migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ *   docs for why a bare `console.error` at this point is unsafe (it can
+ *   fire while a caller's `Lanes`/`TaskList` display is mid-redraw).
+ */
+export declare function init(knex: Knex, readOnly: boolean, onLog?: (message: string) => void): Promise<void>;

package/lib/db-ops/lifecycle/init.js

@@ -0,0 +1,122 @@
+import { applyConnectionPragmas } from '../../apply-connection-pragmas.js';
+import { initSchema } from '../../init-schema.js';
+import { assertCompatibleVersion } from '../../meta/assert-compatible-version.js';
+import { migrateContentItemsAliasOfId } from '../../migrate-content-items-alias-of-id.js';
+import { migrateContentItemsDedupeCapEventId } from '../../migrate-content-items-dedupe-cap-event-id.js';
+import { migrateContentItemsIsMetadataOnly } from '../../migrate-content-items-is-metadata-only.js';
+import { migrateInfoCreatedCwd } from '../../migrate-info-created-cwd.js';
+import { migrateInfoMainContentSelector } from '../../migrate-info-main-content-selector.js';
+import { migrateInfoRequestHeaderNames } from '../../migrate-info-request-header-names.js';
+import { migrateInfoRoots } from '../../migrate-info-roots.js';
+import { migrateInventoryRunsToListReconcileRuns } from '../../migrate-inventory-runs-to-list-reconcile-runs.js';
+import { migrateListReconcileRunsExcludeSkipped } from '../../migrate-list-reconcile-runs-exclude-skipped.js';
+import { migrateListReconcileRunsInvalidSkipped } from '../../migrate-list-reconcile-runs-invalid-skipped.js';
+import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.js';
+import { migratePageMetaBodyHash } from '../../migrate-page-meta-body-hash.js';
+import { migratePageMetaConsoleErrorCount } from '../../migrate-page-meta-console-error-count.js';
+import { migratePageMetaCustomElementCount } from '../../migrate-page-meta-custom-element-count.js';
+import { migratePageMetaImageScan } from '../../migrate-page-meta-image-scan.js';
+import { migratePageTagsToPageTechnologies } from '../../migrate-page-tags-to-page-technologies.js';
+import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-network-outages.js';
+/**
+ * Initializes the database schema if tables do not exist, then runs the
+ * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
+ * `page_meta.main_content_*`, `list_reconcile_runs.invalid_skipped`,
+ * `list_reconcile_runs.exclude_skipped`).
+ *
+ * There is deliberately no per-table *table-creation* migration chain here:
+ * `assertCompatibleVersion` (called below, before any schema work) rejects
+ * every archive older than the current format, so a connection that reaches
+ * `initSchema` is either brand new (`initSchema` provisions the full schema)
+ * or was produced by `scripts/migrate-to-0.13.mjs` (which guarantees the
+ * full table set before it repacks) — and `initSchema` itself re-runs
+ * `createEntityTables` / `createAdjunctTables` unconditionally on every
+ * open, self-healing any *missing table* via their internal
+ * `IF NOT EXISTS` / `hasTable` guards. What that self-healing cannot do is
+ * retrofit a *new column* onto an entity table that already exists —
+ * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
+ * additions to an existing 0.13 table are therefore the one case that still
+ * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
+ * `migrateInfoCreatedCwd`, `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
+ * `migratePageMetaConsoleErrorCount`, `migratePageMetaCustomElementCount`,
+ * `migratePageMetaImageScan`, `migrateContentItemsAliasOfId`,
+ * `migrateContentItemsIsMetadataOnly`,
+ *
+ * `migratePageTagsToPageTechnologies` is the one exception to "column adds
+ * only": it converts `page_tags` (removed) rows into `technology_signals`/
+ * `page_technologies` and DROPS the old table — a one-time table-level ETL,
+ * not a column add, but it belongs in this same boot phase for the same
+ * reason (self-healing an old archive's schema before any reader runs).
+ *
+ * `migrateContentItemsDedupeCapEventId`, `migrateListReconcileRunsInvalidSkipped`,
+ * `migrateListReconcileRunsExcludeSkipped`) rather than a DDL-string change alone.
+ *
+ * `migrateInventoryRunsToListReconcileRuns` runs BEFORE `initSchema`
+ * (unlike every other migration here) — see its own JSDoc for why: a
+ * plain rename must land before `createAdjunctTables`'s
+ * `IF NOT EXISTS` guard would otherwise create an empty `list_reconcile_runs`
+ * and strand the old table's rows.
+ *
+ * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
+ * change) but belongs at this same boot phase for the same reason the
+ * others do: it must run before ANY reader (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, …) can observe a `network_outages` row
+ * left `ended_at = NULL` by a crawl process that was killed mid-outage in a
+ * prior session.
+ *
+ * In read-only mode schema init + migration are SKIPPED so the same DB
+ * can be opened safely by a viewer attached to a live (or interrupted)
+ * crawl without rewriting the user's tmpDir.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param readOnly - When true, skip schema init + migrations.
+ * @param onLog - Forwarded to every `migrate*` call below (issue #294) —
+ *   see {@link import('../../migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ *   docs for why a bare `console.error` at this point is unsafe (it can
+ *   fire while a caller's `Lanes`/`TaskList` display is mid-redraw).
+ */
+export async function init(knex, readOnly, onLog) {
+    // Connection-level PRAGMAs (foreign_keys, mmap_size, …) must be
+    // reapplied on every connect — they are not persisted across opens.
+    // They are safe in read-only mode because they don't write to the
+    // user's tmpDir, just configure the libsql connection.
+    await applyConnectionPragmas(knex);
+    // Reject incompatible archives before any further work. Runs for both
+    // writer and read-only (stub viewer) connections so old
+    // `._nitpicker-*` stubs surface a clear error instead of
+    // dereferencing missing columns at query time. New archives (no
+    // `info` table yet) pass through; the schema is filled in by
+    // `initSchema` below.
+    await assertCompatibleVersion(knex);
+    if (readOnly) {
+        return;
+    }
+    // Must run before `initSchema`: see the function's own JSDoc for why a
+    // plain table rename has to land before `createAdjunctTables`'s
+    // `IF NOT EXISTS` guard would otherwise create an empty replacement.
+    await migrateInventoryRunsToListReconcileRuns(knex, onLog);
+    await initSchema(knex);
+    await migrateInfoRoots(knex, onLog);
+    await migrateInfoMainContentSelector(knex, onLog);
+    await migrateInfoRequestHeaderNames(knex, onLog);
+    await migrateInfoCreatedCwd(knex, onLog);
+    await migrateMainContentsColumns(knex, onLog);
+    await migratePageMetaBodyHash(knex, onLog);
+    await migratePageMetaConsoleErrorCount(knex, onLog);
+    await migratePageMetaCustomElementCount(knex, onLog);
+    await migratePageMetaImageScan(knex, onLog);
+    // Table-level migration (converts + drops page_tags), not a column
+    // add — see its own JSDoc for why it still belongs in this boot phase.
+    await migratePageTagsToPageTechnologies(knex, onLog);
+    await migrateContentItemsAliasOfId(knex, onLog);
+    // Runs after `initSchema` above, which already created
+    // `dedupe_cap_events` (an adjunct table) unconditionally — so the new
+    // column's `REFERENCES dedupe_cap_events(id)` target always exists by
+    // this point, for both fresh and legacy archives.
+    await migrateContentItemsDedupeCapEventId(knex, onLog);
+    // A plain `hasColumn`-guarded ADD COLUMN, same as `migrateContentItemsAliasOfId`
+    // above — no adjunct-table dependency, so ordering relative to it doesn't matter.
+    await migrateContentItemsIsMetadataOnly(knex, onLog);
+    await migrateListReconcileRunsInvalidSkipped(knex, onLog);
+    await migrateListReconcileRunsExcludeSkipped(knex, onLog);
+    await closeStaleOpenNetworkOutages(knex);
+}

package/lib/db-ops/list-reconcile/record-list-reconcile-run.d.ts

@@ -0,0 +1,22 @@
+import type { ListReconcileRunMeta } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `list_reconcile_runs` audit log.
+ *
+ * Called by `CrawlerOrchestrator.inventory` and `CrawlerOrchestrator.recrawl`
+ * on every successful `--inventory <list>` / `--recrawl <list>` invocation
+ * so the archive carries a durable record of which list was applied when
+ * and at what scale — the operational question "did we apply last month's
+ * list" the archive itself can answer without consulting external
+ * bookkeeping.
+ *
+ * Append-only. There is intentionally no UPDATE path and no UNIQUE
+ * constraint on `source_file_sha256`; two applies of the same list
+ * each get their own row. Duplicate detection is left to readers —
+ * the hash is recorded as the content-identity key they would use.
+ * Field-level NULL semantics live on {@link ListReconcileRunMeta}.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param meta - The run metadata to record. Only `ran_at` is required.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function recordListReconcileRun(knex: Knex, meta: ListReconcileRunMeta): Promise<number>;

package/lib/db-ops/list-reconcile/record-list-reconcile-run.js

@@ -0,0 +1,41 @@
+/**
+ * Appends one row to the `list_reconcile_runs` audit log.
+ *
+ * Called by `CrawlerOrchestrator.inventory` and `CrawlerOrchestrator.recrawl`
+ * on every successful `--inventory <list>` / `--recrawl <list>` invocation
+ * so the archive carries a durable record of which list was applied when
+ * and at what scale — the operational question "did we apply last month's
+ * list" the archive itself can answer without consulting external
+ * bookkeeping.
+ *
+ * Append-only. There is intentionally no UPDATE path and no UNIQUE
+ * constraint on `source_file_sha256`; two applies of the same list
+ * each get their own row. Duplicate detection is left to readers —
+ * the hash is recorded as the content-identity key they would use.
+ * Field-level NULL semantics live on {@link ListReconcileRunMeta}.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param meta - The run metadata to record. Only `ran_at` is required.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function recordListReconcileRun(knex, meta) {
+    const inserted = await knex
+        .from('list_reconcile_runs')
+        .insert({
+        ran_at: meta.ran_at,
+        list_label: meta.list_label ?? null,
+        source_file_sha256: meta.source_file_sha256 ?? null,
+        total_lines: meta.total_lines ?? null,
+        new_pages: meta.new_pages ?? null,
+        new_resources: meta.new_resources ?? null,
+        scope_skipped: meta.scope_skipped ?? null,
+        exclude_skipped: meta.exclude_skipped ?? null,
+        invalid_skipped: meta.invalid_skipped ?? null,
+        notes: meta.notes ?? null,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('recordListReconcileRun: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/db-ops/meta/get-audios-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentAudioRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_audios` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertAudios`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getAudiosOfPage(knex: Knex, pageId: number): Promise<MainContentAudioRow[]>;

package/lib/db-ops/meta/get-audios-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_audios` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertAudios`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getAudiosOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src')
+        .from('page_main_content_audios')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/db-ops/meta/get-buttons-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentButtonRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_buttons` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertButtons`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getButtonsOfPage(knex: Knex, pageId: number): Promise<MainContentButtonRow[]>;

package/lib/db-ops/meta/get-buttons-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_buttons` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertButtons`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getButtonsOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'nodeName', 'role', 'type', 'text', 'disabled')
+        .from('page_main_content_buttons')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/db-ops/meta/get-canvases-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentCanvasRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_canvases` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertCanvases`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getCanvasesOfPage(knex: Knex, pageId: number): Promise<MainContentCanvasRow[]>;

package/lib/db-ops/meta/get-canvases-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_canvases` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertCanvases`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getCanvasesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'width', 'height')
+        .from('page_main_content_canvases')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/db-ops/meta/get-custom-elements-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentCustomElementRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_custom_elements` rows for the given page
+ * id, in DOM traversal order.
+ *
+ * Read-side counterpart to `insertCustomElements`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getCustomElementsOfPage(knex: Knex, pageId: number): Promise<MainContentCustomElementRow[]>;

package/lib/db-ops/meta/get-custom-elements-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_custom_elements` rows for the given page
+ * id, in DOM traversal order.
+ *
+ * Read-side counterpart to `insertCustomElements`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getCustomElementsOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'nodeName', 'elementId', 'classList')
+        .from('page_main_content_custom_elements')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/db-ops/meta/get-headings-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentHeadingRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_headings` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertHeadings`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getHeadingsOfPage(knex: Knex, pageId: number): Promise<MainContentHeadingRow[]>;

package/lib/db-ops/meta/get-headings-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_headings` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertHeadings`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getHeadingsOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'text', 'level')
+        .from('page_main_content_headings')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/db-ops/meta/get-iframes-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentIframeRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_iframes` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertIframes`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getIframesOfPage(knex: Knex, pageId: number): Promise<MainContentIframeRow[]>;

package/lib/db-ops/meta/get-iframes-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_iframes` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertIframes`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getIframesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src', 'title', 'width', 'height')
+        .from('page_main_content_iframes')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/db-ops/meta/get-jsonld-of-page.d.ts

@@ -0,0 +1,13 @@
+import type { JsonLdRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_jsonld` rows for the given page id, parsed back into
+ * {@link JsonLdRow} shape (with `parsed` deserialised from its JSON column).
+ *
+ * Read-side counterpart to `insertJsonLd`. Returns rows in insertion order
+ * by `id` so the order observed by `get-page-jsonld` matches the order the
+ * scraper saw them.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getJsonLdOfPage(knex: Knex, pageId: number): Promise<JsonLdRow[]>;

package/lib/db-ops/meta/get-jsonld-of-page.js

@@ -0,0 +1,27 @@
+import { safeParseJson } from '../_shared/safe-parse-json.js';
+/**
+ * Retrieves all `page_jsonld` rows for the given page id, parsed back into
+ * {@link JsonLdRow} shape (with `parsed` deserialised from its JSON column).
+ *
+ * Read-side counterpart to `insertJsonLd`. Returns rows in insertion order
+ * by `id` so the order observed by `get-page-jsonld` matches the order the
+ * scraper saw them.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getJsonLdOfPage(knex, pageId) {
+    const rows = await knex
+        .select('id', 'pageId', 'kind', 'type', 'raw', 'parsed', 'parseError')
+        .from('page_jsonld')
+        .where('pageId', pageId)
+        .orderBy('id', 'asc');
+    return rows.map((r) => ({
+        id: r.id,
+        pageId: r.pageId,
+        kind: r.kind === 'speculationrules' ? 'speculationrules' : 'ld+json',
+        type: r.type,
+        raw: r.raw,
+        parsed: r.parsed === null ? null : safeParseJson(r.parsed),
+        parseError: r.parseError,
+    }));
+}

package/lib/db-ops/meta/get-main-content-images-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentImageRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_images` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentImages`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getMainContentImagesOfPage(knex: Knex, pageId: number): Promise<MainContentImageRow[]>;

package/lib/db-ops/meta/get-main-content-images-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_images` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentImages`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getMainContentImagesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src', 'alt')
+        .from('page_main_content_images')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/db-ops/meta/get-main-content-tables-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentTableRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_tables` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentTables`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getMainContentTablesOfPage(knex: Knex, pageId: number): Promise<MainContentTableRow[]>;

package/lib/db-ops/meta/get-main-content-tables-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_tables` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentTables`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getMainContentTablesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'rows', 'cols', 'hasHeader', 'hasFooter', 'hasMergedCell')
+        .from('page_main_content_tables')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/db-ops/meta/get-page-technologies-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { PageTechnologyRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_technologies` rows for the given page id — the
+ * confidence-combined roll-up (one row per detected technology).
+ *
+ * Read-side counterpart to `insertTechnologies`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getPageTechnologiesOfPage(knex: Knex, pageId: number): Promise<PageTechnologyRow[]>;

package/lib/db-ops/meta/get-page-technologies-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_technologies` rows for the given page id — the
+ * confidence-combined roll-up (one row per detected technology).
+ *
+ * Read-side counterpart to `insertTechnologies`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getPageTechnologiesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'technology', 'category', 'version', 'confidence', 'signalCount')
+        .from('page_technologies')
+        .where('pageId', pageId)
+        .orderBy('confidence', 'desc');
+}

package/lib/db-ops/meta/get-technology-signals-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { TechnologySignalRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `technology_signals` rows for the given page id — the raw,
+ * per-signal evidence behind its `page_technologies` roll-up.
+ *
+ * Read-side counterpart to `insertTechnologies`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getTechnologySignalsOfPage(knex: Knex, pageId: number): Promise<TechnologySignalRow[]>;

package/lib/db-ops/meta/get-technology-signals-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `technology_signals` rows for the given page id — the raw,
+ * per-signal evidence behind its `page_technologies` roll-up.
+ *
+ * Read-side counterpart to `insertTechnologies`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getTechnologySignalsOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'technology', 'signalType', 'evidence', 'weight')
+        .from('technology_signals')
+        .where('pageId', pageId)
+        .orderBy('id', 'asc');
+}

package/lib/db-ops/meta/get-videos-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentVideoRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_videos` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertVideos`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getVideosOfPage(knex: Knex, pageId: number): Promise<MainContentVideoRow[]>;

package/lib/db-ops/meta/get-videos-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_videos` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertVideos`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getVideosOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src', 'poster', 'width', 'height')
+        .from('page_main_content_videos')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}