@nitpicker/crawler 0.8.0 → 0.11.0

package/lib/archive/database.js

@@ -32,27 +32,90 @@ var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn,
     if (target) Object.defineProperty(target, contextIn.name, descriptor);
     done = true;
 };
+import { createHash } from 'node:crypto';
+import { existsSync } from 'node:fs';
 import path from 'node:path';
+import { zstdCompressSync, zstdDecompressSync } from 'node:zlib';
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { retry } from '@d-zero/shared/retry';
 import { pathComparator } from '@d-zero/shared/sort/path';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import knex from 'knex';
 import { findScopeEntry } from '../crawler/find-scope-entry.js';
+import { isHtmlContentType } from '../crawler/is-html-content-type.js';
+import { normalizeContentType } from '../crawler/normalize-content-type.js';
 import { eachSplitted } from '../utils/array/each-splitted.js';
 import { ErrorEmitter } from '../utils/error/error-emitter.js';
 import { dbLog } from './debug.js';
 import { mkdir } from './filesystem/mkdir.js';
 import { getJSON } from './get-json.js';
-import { initSchema } from './init-schema.js';
+import { applyConnectionPragmas, initSchema } from './init-schema.js';
 import { LibsqlDialect } from './libsql-dialect.js';
 import { limitedPageIds } from './limited-page-ids.js';
+import { assertCompatibleVersion } from './meta/assert-compatible-version.js';
+import { classifyJsonLdType } from './meta/classify-jsonld-type.js';
+import { computePageDenormalized } from './meta/compute-page-denormalized.js';
+import { deriveFlatFromMeta } from './meta/derive-flat-from-meta.js';
+import { deriveMetaExtras } from './meta/derive-meta-extras.js';
+import { extractTagsForArchive } from './meta/extract-tags-for-archive.js';
+import { migrateCrawlErrors } from './migrate-crawl-errors.js';
+import { migrateHtmlBlobTables } from './migrate-html-blob-tables.js';
 import { migrateInfoRoots } from './migrate-info-roots.js';
+import { migratePageErrors } from './migrate-page-errors.js';
+import { migratePagesResourcesSource } from './migrate-pages-resources-source.js';
 import { redirectTable } from './redirect-table.js';
+import { resolveRedirectChain } from './resolve-redirect-chain.js';
 const retrySetting = {
     interval: 300,
     retries: 3,
 };
+/**
+ * Decodes a stored HTML body BLOB according to its codec marker. The codec
+ * column on `page_html_blobs` exists so individual rows can be migrated to
+ * a future encoder without rewriting the whole table; readers must dispatch
+ * on it. The body is typed `Uint8Array` (not `Buffer`) because libsql
+ * returns BLOB columns as bare `Uint8Array`; `Buffer.from` wraps it
+ * zero-copy.
+ * @param body - Raw bytes as stored in `page_html_blobs.body`.
+ * @param codec - The `codec` column value (e.g. `'zstd'`, `'none'`).
+ * @returns UTF-8 decoded HTML string.
+ * @throws {Error} If the codec is not recognised.
+ */
+/**
+ * Parses a JSON column value, returning `null` on parse failure rather than
+ * throwing. JSON columns in `page_jsonld` (`parsed`) and `page_tags`
+ * (`categories`, `sources`) are written by `JSON.stringify` and round-trip
+ * cleanly under normal conditions; a hand-edited archive that has
+ * malformed JSON in those columns should degrade gracefully rather than
+ * propagate a parse error up to the consumer.
+ * @param value - JSON-encoded text.
+ */
+function safeParseJson(value) {
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ *
+ * @param body
+ * @param codec
+ */
+function decodeStoredBlob(body, codec) {
+    // `Buffer.from(buffer)` accepts Uint8Array, Buffer, and array-like
+    // shapes uniformly; libsql may hand back any of these for a BLOB
+    // column depending on the row encoding.
+    const buffer = Buffer.from(body);
+    if (codec === 'zstd') {
+        return zstdDecompressSync(buffer).toString('utf8');
+    }
+    if (codec === 'none') {
+        return buffer.toString('utf8');
+    }
+    throw new Error(`Unknown page_html_blobs.codec: ${codec}`);
+}
 /**
  * Columns of the `info` table that `setConfig` / `updateConfig` are allowed to
  * write. Any key outside this set is silently dropped so callers can splat a
@@ -89,13 +152,99 @@ const INFO_JSON_COLUMNS = new Set([
     'excludeKeywords',
     'excludeUrls',
 ]);
+/**
+ * Columns of the `pages` table that should be reset to `null` whenever a
+ * previously-scraped row is demoted back to "pending" (i.e. by
+ * `resetFailedPages` and `repromoteExternalPages`).
+ *
+ * Includes all flat meta columns, the denormalised aggregates, and the
+ * `meta_extras` JSON catch-all. **Excludes** `firstCrawledAt` / `lastCrawledAt`
+ * by design — failure reset must not erase the last-success timestamp, which
+ * is the within-archive observation axis for #11 / #17 / #19 use cases.
+ *
+ * Centralised in one constant so schema growth and reset logic stay in lock-
+ * step: adding a flat meta column without updating this list would leave
+ * stale data after a reset.
+ */
+const META_NULLABLE_COLUMNS = [
+    // Document basics
+    'lang',
+    'dir',
+    'charset',
+    'baseHref',
+    'viewport_raw',
+    'themeColor',
+    'applicationName',
+    'author',
+    'generator',
+    'publisher',
+    // Title / description / keywords
+    'title',
+    'description',
+    'keywords',
+    // Robots
+    'robots_raw',
+    'robots_noindex',
+    'robots_nofollow',
+    'robots_noarchive',
+    'robots_noimageindex',
+    'googlebot',
+    // Link (1:1)
+    'canonical',
+    'amphtml',
+    'manifest',
+    'icon_href',
+    'appleTouchIcon_href',
+    // Open Graph
+    'og_type',
+    'og_title',
+    'og_url',
+    'og_site_name',
+    'og_description',
+    'og_image',
+    'og_image_alt',
+    'og_image_width',
+    'og_image_height',
+    'og_locale',
+    'og_article_published_time',
+    'og_article_modified_time',
+    // Twitter
+    'twitter_card',
+    'twitter_site',
+    'twitter_creator',
+    'twitter_title',
+    'twitter_description',
+    'twitter_image',
+    // One-offs
+    'fb_app_id',
+    'verification_google',
+    'formatDetection_telephone',
+    // Denormalised aggregates
+    'tag_count',
+    'jsonld_count',
+    'tags_providers_csv',
+    // Catch-all
+    'meta_extras',
+];
+/**
+ * Builds the reset payload for {@link META_NULLABLE_COLUMNS} as a plain object
+ * suitable for `knex.update(...)`. All listed columns are mapped to `null`.
+ */
+function makeMetaResetPayload() {
+    const payload = {};
+    for (const col of META_NULLABLE_COLUMNS) {
+        payload[col] = null;
+    }
+    return payload;
+}
 /**
  * Low-level database abstraction layer for the archive's SQLite database.
  *
- * Manages the `pages`, `anchors`, `images`, `resources`, and `resources-referrers`
- * tables. All public methods that perform database queries use the `@retryable`
- * decorator for automatic retry on transient failures, and `@ErrorEmitter` to
- * propagate errors as events.
+ * Public methods that perform database queries use the `@retryable`
+ * decorator for automatic retry on transient failures, and `@ErrorEmitter`
+ * to propagate errors as events. The set of tables this layer manages is
+ * defined by `init-schema.ts` (the source of truth — query that file for
+ * the canonical list).
  *
  * Use the static {@link Database.connect} factory method to create instances.
  * The constructor is private.
@@ -103,12 +252,14 @@ const INFO_JSON_COLUMNS = new Set([
 let Database = (() => {
     let _classSuper = EventEmitter;
     let _instanceExtraInitializers = [];
-    let _clearHtmlPath_decorators;
     let _getAnchorsOnPage_decorators;
     let _getBaseUrl_decorators;
     let _getConfig_decorators;
     let _getCrawlingState_decorators;
-    let _getHtmlPathOnPage_decorators;
+    let _getExistingPageUrls_decorators;
+    let _getExistingResourceUrls_decorators;
+    let _getHtmlOfPageById_decorators;
+    let _getJsonLdOfPage_decorators;
     let _getName_decorators;
     let _getPageCount_decorators;
     let _getPages_decorators;
@@ -116,11 +267,18 @@ let Database = (() => {
     let _getRedirectsForPages_decorators;
     let _getReferrersOfPage_decorators;
     let _getReferrersOfResource_decorators;
+    let _getResourceByUrl_decorators;
     let _getResources_decorators;
     let _getResourceUrlList_decorators;
+    let _getScrapedHtmlPageCount_decorators;
+    let _getTagsOfPage_decorators;
+    let _insertCrawlError_decorators;
+    let _insertPageError_decorators;
     let _insertResource_decorators;
     let _insertResourceReferrers_decorators;
+    let _recordRedirect_decorators;
     let _repromoteExternalPages_decorators;
+    let _resetFailedPages_decorators;
     let _setConfig_decorators;
     let _setSkippedPage_decorators;
     let _updateConfig_decorators;
@@ -128,12 +286,14 @@ let Database = (() => {
     return class Database extends _classSuper {
         static {
             const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
-            _clearHtmlPath_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getAnchorsOnPage_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getBaseUrl_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getConfig_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getCrawlingState_decorators = [ErrorEmitter(), retry(retrySetting)];
-            _getHtmlPathOnPage_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getExistingPageUrls_decorators = [ErrorEmitter()];
+            _getExistingResourceUrls_decorators = [ErrorEmitter()];
+            _getHtmlOfPageById_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getJsonLdOfPage_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getName_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getPageCount_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getPages_decorators = [ErrorEmitter(), retry(retrySetting)];
@@ -141,21 +301,30 @@ let Database = (() => {
             _getRedirectsForPages_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getReferrersOfPage_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getReferrersOfResource_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getResourceByUrl_decorators = [retry(retrySetting)];
             _getResources_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getResourceUrlList_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getScrapedHtmlPageCount_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getTagsOfPage_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _insertCrawlError_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _insertPageError_decorators = [ErrorEmitter(), retry(retrySetting)];
             _insertResource_decorators = [ErrorEmitter(), retry(retrySetting)];
             _insertResourceReferrers_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _recordRedirect_decorators = [ErrorEmitter(), retry(retrySetting)];
             _repromoteExternalPages_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _resetFailedPages_decorators = [ErrorEmitter(), retry(retrySetting)];
             _setConfig_decorators = [ErrorEmitter(), retry(retrySetting)];
             _setSkippedPage_decorators = [ErrorEmitter(), retry(retrySetting)];
             _updateConfig_decorators = [ErrorEmitter(), retry(retrySetting)];
             _updatePage_decorators = [ErrorEmitter(), retry(retrySetting)];
-            __esDecorate(this, null, _clearHtmlPath_decorators, { kind: "method", name: "clearHtmlPath", static: false, private: false, access: { has: obj => "clearHtmlPath" in obj, get: obj => obj.clearHtmlPath }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getAnchorsOnPage_decorators, { kind: "method", name: "getAnchorsOnPage", static: false, private: false, access: { has: obj => "getAnchorsOnPage" in obj, get: obj => obj.getAnchorsOnPage }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getBaseUrl_decorators, { kind: "method", name: "getBaseUrl", static: false, private: false, access: { has: obj => "getBaseUrl" in obj, get: obj => obj.getBaseUrl }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getConfig_decorators, { kind: "method", name: "getConfig", static: false, private: false, access: { has: obj => "getConfig" in obj, get: obj => obj.getConfig }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getCrawlingState_decorators, { kind: "method", name: "getCrawlingState", static: false, private: false, access: { has: obj => "getCrawlingState" in obj, get: obj => obj.getCrawlingState }, metadata: _metadata }, null, _instanceExtraInitializers);
-            __esDecorate(this, null, _getHtmlPathOnPage_decorators, { kind: "method", name: "getHtmlPathOnPage", static: false, private: false, access: { has: obj => "getHtmlPathOnPage" in obj, get: obj => obj.getHtmlPathOnPage }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getExistingPageUrls_decorators, { kind: "method", name: "getExistingPageUrls", static: false, private: false, access: { has: obj => "getExistingPageUrls" in obj, get: obj => obj.getExistingPageUrls }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getExistingResourceUrls_decorators, { kind: "method", name: "getExistingResourceUrls", static: false, private: false, access: { has: obj => "getExistingResourceUrls" in obj, get: obj => obj.getExistingResourceUrls }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getHtmlOfPageById_decorators, { kind: "method", name: "getHtmlOfPageById", static: false, private: false, access: { has: obj => "getHtmlOfPageById" in obj, get: obj => obj.getHtmlOfPageById }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getJsonLdOfPage_decorators, { kind: "method", name: "getJsonLdOfPage", static: false, private: false, access: { has: obj => "getJsonLdOfPage" in obj, get: obj => obj.getJsonLdOfPage }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getName_decorators, { kind: "method", name: "getName", static: false, private: false, access: { has: obj => "getName" in obj, get: obj => obj.getName }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getPageCount_decorators, { kind: "method", name: "getPageCount", static: false, private: false, access: { has: obj => "getPageCount" in obj, get: obj => obj.getPageCount }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getPages_decorators, { kind: "method", name: "getPages", static: false, private: false, access: { has: obj => "getPages" in obj, get: obj => obj.getPages }, metadata: _metadata }, null, _instanceExtraInitializers);
@@ -163,11 +332,18 @@ let Database = (() => {
             __esDecorate(this, null, _getRedirectsForPages_decorators, { kind: "method", name: "getRedirectsForPages", static: false, private: false, access: { has: obj => "getRedirectsForPages" in obj, get: obj => obj.getRedirectsForPages }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getReferrersOfPage_decorators, { kind: "method", name: "getReferrersOfPage", static: false, private: false, access: { has: obj => "getReferrersOfPage" in obj, get: obj => obj.getReferrersOfPage }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getReferrersOfResource_decorators, { kind: "method", name: "getReferrersOfResource", static: false, private: false, access: { has: obj => "getReferrersOfResource" in obj, get: obj => obj.getReferrersOfResource }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getResourceByUrl_decorators, { kind: "method", name: "getResourceByUrl", static: false, private: false, access: { has: obj => "getResourceByUrl" in obj, get: obj => obj.getResourceByUrl }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getResources_decorators, { kind: "method", name: "getResources", static: false, private: false, access: { has: obj => "getResources" in obj, get: obj => obj.getResources }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getResourceUrlList_decorators, { kind: "method", name: "getResourceUrlList", static: false, private: false, access: { has: obj => "getResourceUrlList" in obj, get: obj => obj.getResourceUrlList }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getScrapedHtmlPageCount_decorators, { kind: "method", name: "getScrapedHtmlPageCount", static: false, private: false, access: { has: obj => "getScrapedHtmlPageCount" in obj, get: obj => obj.getScrapedHtmlPageCount }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getTagsOfPage_decorators, { kind: "method", name: "getTagsOfPage", static: false, private: false, access: { has: obj => "getTagsOfPage" in obj, get: obj => obj.getTagsOfPage }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _insertCrawlError_decorators, { kind: "method", name: "insertCrawlError", static: false, private: false, access: { has: obj => "insertCrawlError" in obj, get: obj => obj.insertCrawlError }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _insertPageError_decorators, { kind: "method", name: "insertPageError", static: false, private: false, access: { has: obj => "insertPageError" in obj, get: obj => obj.insertPageError }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _insertResource_decorators, { kind: "method", name: "insertResource", static: false, private: false, access: { has: obj => "insertResource" in obj, get: obj => obj.insertResource }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _insertResourceReferrers_decorators, { kind: "method", name: "insertResourceReferrers", static: false, private: false, access: { has: obj => "insertResourceReferrers" in obj, get: obj => obj.insertResourceReferrers }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _recordRedirect_decorators, { kind: "method", name: "recordRedirect", static: false, private: false, access: { has: obj => "recordRedirect" in obj, get: obj => obj.recordRedirect }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _repromoteExternalPages_decorators, { kind: "method", name: "repromoteExternalPages", static: false, private: false, access: { has: obj => "repromoteExternalPages" in obj, get: obj => obj.repromoteExternalPages }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _resetFailedPages_decorators, { kind: "method", name: "resetFailedPages", static: false, private: false, access: { has: obj => "resetFailedPages" in obj, get: obj => obj.resetFailedPages }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _setConfig_decorators, { kind: "method", name: "setConfig", static: false, private: false, access: { has: obj => "setConfig" in obj, get: obj => obj.setConfig }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _setSkippedPage_decorators, { kind: "method", name: "setSkippedPage", static: false, private: false, access: { has: obj => "setSkippedPage" in obj, get: obj => obj.setSkippedPage }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _updateConfig_decorators, { kind: "method", name: "updateConfig", static: false, private: false, access: { has: obj => "updateConfig" in obj, get: obj => obj.updateConfig }, metadata: _metadata }, null, _instanceExtraInitializers);
@@ -176,12 +352,9 @@ let Database = (() => {
         }
         /** The Knex query builder instance connected to the SQLite database. */
         #instance = __runInitializers(this, _instanceExtraInitializers);
-        /** Absolute path to the working directory, used for resolving relative snapshot paths. */
-        #workingDir;
         // eslint-disable-next-line no-restricted-syntax
         constructor(options) {
             super();
-            this.#workingDir = options.workingDir;
             this.#instance = knex({
                 client: LibsqlDialect,
                 connection: {
@@ -216,14 +389,6 @@ let Database = (() => {
         async checkpoint() {
             await this.#instance.raw('PRAGMA wal_checkpoint(TRUNCATE)');
         }
-        /**
-         * Clears the HTML snapshot path for a page.
-         * Used to roll back the snapshot reference when the snapshot file write fails.
-         * @param pageId - The database ID of the page whose HTML path should be cleared.
-         */
-        async clearHtmlPath(pageId) {
-            await this.#instance('pages').where('id', pageId).update({ html: null });
-        }
         /**
          * Destroys the database connection, releasing all pooled resources.
          */
@@ -303,18 +468,106 @@ let Database = (() => {
             };
         }
         /**
-         * Retrieves the HTML snapshot file path for a specific page.
-         * @param pageId - The database ID of the page.
-         * @returns The relative file path to the HTML snapshot, or null if not saved.
+         * Return the subset of `urls` that already exist in the `pages` table.
+         * Chunked into batches so SQLite's `IN (?, ?, …)` parameter limit
+         * (`SQLITE_MAX_VARIABLE_NUMBER`, default 999) cannot be hit even when the
+         * inventory list contains tens of thousands of URLs.
+         *
+         * Read-only — no transaction, no lock contention with the crawler write
+         * pipeline (callers run this BEFORE the `<archive>.bak` is taken and the
+         * crawl is started).
+         * @param urls - URL strings to probe (already in `withoutHashAndAuth` form).
+         * @returns URLs found in `pages`. Order is not preserved.
          */
-        async getHtmlPathOnPage(pageId) {
-            return await this.#instance.transaction(async (trx) => {
-                const [{ html }] = await trx
-                    .select('html')
+        async getExistingPageUrls(urls) {
+            if (urls.length === 0) {
+                return [];
+            }
+            const found = [];
+            await eachSplitted([...urls], 500, async (chunk) => {
+                const rows = await this.#instance
+                    .select('url')
                     .from('pages')
-                    .where('id', pageId);
-                return html || null;
+                    .whereIn('url', chunk);
+                for (const row of rows) {
+                    found.push(row.url);
+                }
+            });
+            return found;
+        }
+        /**
+         * Return the subset of `urls` that already exist in the `resources` table.
+         * See {@link Database.getExistingPageUrls} — same chunking strategy.
+         * @param urls - URL strings to probe.
+         * @returns URLs found in `resources`.
+         */
+        async getExistingResourceUrls(urls) {
+            if (urls.length === 0) {
+                return [];
+            }
+            const found = [];
+            await eachSplitted([...urls], 500, async (chunk) => {
+                const rows = await this.#instance
+                    .select('url')
+                    .from('resources')
+                    .whereIn('url', chunk);
+                for (const row of rows) {
+                    found.push(row.url);
+                }
             });
+            return found;
+        }
+        /**
+         * Reads the HTML snapshot stored as a zstd-compressed BLOB for the given page.
+         *
+         * Joins `page_html_ref` → `page_html_blobs` and decompresses inline. Returns
+         * `null` when the page has no stored body (a non-HTML resource, a redirect
+         * source, a degraded render). Read works identically on read-only / stub
+         * connections — the special-cased "do we have a loose dir vs zip?" branching
+         * the previous file-backed layout required is gone.
+         *
+         * Tables `page_html_ref` and `page_html_blobs` are created by `initSchema`.
+         * Older `.nitpicker` archives that predate this migration must be passed
+         * through `scripts/migrate-to-0.10.mjs` before they can be read.
+         * @param pageId - The database ID of the page.
+         * @returns The decompressed HTML string, or `null` if no snapshot is stored.
+         */
+        async getHtmlOfPageById(pageId) {
+            const row = await this.#instance
+                .from('page_html_ref')
+                .join('page_html_blobs', 'page_html_ref.hash', '=', 'page_html_blobs.hash')
+                .select('page_html_blobs.body as body', 'page_html_blobs.codec as codec')
+                .where('page_html_ref.page_id', pageId)
+                .first();
+            if (!row) {
+                return null;
+            }
+            return decodeStoredBlob(row.body, row.codec);
+        }
+        /**
+         * Retrieves all `page_jsonld` rows for the given page id, parsed back into
+         * {@link JsonLdRow} shape (with `parsed` deserialised from its JSON column).
+         *
+         * Read-side counterpart to `#insertJsonLd`. Returns rows in insertion order
+         * by `id` so the order observed by `get-page-jsonld` matches the order the
+         * scraper saw them.
+         * @param pageId
+         */
+        async getJsonLdOfPage(pageId) {
+            const rows = await this.#instance
+                .select('id', 'pageId', 'kind', 'type', 'raw', 'parsed', 'parseError')
+                .from('page_jsonld')
+                .where('pageId', pageId)
+                .orderBy('id', 'asc');
+            return rows.map((r) => ({
+                id: r.id,
+                pageId: r.pageId,
+                kind: r.kind === 'speculationrules' ? 'speculationrules' : 'ld+json',
+                type: r.type,
+                raw: r.raw,
+                parsed: r.parsed === null ? null : safeParseJson(r.parsed),
+                parseError: r.parseError,
+            }));
         }
         /**
          * Returns the underlying Knex query builder instance for direct SQL access.
@@ -524,15 +777,29 @@ let Database = (() => {
         }
         /**
          * Retrieves pages that link to a specific page (incoming links / referrers).
+         *
+         * Incoming links are resolved **through redirects**: an anchor pointing at a
+         * redirect source (e.g. `http://x` that 301s to `https://x`) counts as a
+         * referrer of the redirect's final destination, not of the source. This keeps
+         * backlinks merged on the canonical page instead of splitting them across the
+         * `http`/`https` (or any redirect source/dest) pair. The resolution mirrors
+         * `redirectTable()` — `redirectDestId` is pre-flattened to the final
+         * destination, so `COALESCE(target.redirectDestId, target.id)` is a single hop.
          * @param pageId - The database ID of the target page.
          * @returns An array of referrer records with URL, hash, and text content.
          */
         async getReferrersOfPage(pageId) {
             const res = await this.#instance
-                .select('pages.url', 'anchors.hash', 'anchors.textContent')
+                .select('referrer.url', 
+            // `through` / `throughId` = the URL the anchor actually pointed at (the
+            // redirect source, e.g. `http://x`), mirroring `getPagesWithRels`'
+            // `redirect.from` / `redirect.fromId`. Lets report code print the
+            // "[REDIRECTED FROM]" note even on this (non-preloaded) referrer path.
+            'target.url as through', 'target.id as throughId', 'anchors.hash', 'anchors.textContent')
                 .from('anchors')
-                .join('pages', 'anchors.pageId', '=', 'pages.id')
-                .where('anchors.hrefId', pageId);
+                .join('pages as referrer', 'anchors.pageId', '=', 'referrer.id')
+                .join('pages as target', 'anchors.hrefId', '=', 'target.id')
+                .whereRaw('coalesce("target"."redirectDestId", "target"."id") = ?', [pageId]);
             return res;
         }
         /**
@@ -549,6 +816,29 @@ let Database = (() => {
                 .where('resources.id', id);
             return res.map((r) => r.url);
         }
+        /**
+         * Retrieves a single sub-resource from the `resources` table by its URL.
+         *
+         * Accepts multiple URL candidates because the stored key is the resource's
+         * `href` while callers may only know the hash-stripped form; the first match
+         * wins.
+         *
+         * Deliberately NOT decorated with `@ErrorEmitter`: the only caller (the
+         * crawler's resource-reuse hook) has a full fallback (the HEAD pre-flight),
+         * so a read failure here must not surface as a database `error` event —
+         * the orchestrator aborts the whole crawl on that event, which is the
+         * correct reaction to write failures but not to a recoverable read.
+         * @param urls - URL candidates to match against the `url` column.
+         * @returns The raw {@link DB_Resource} row, or `null` if none match.
+         */
+        async getResourceByUrl(urls) {
+            const res = await this.#instance
+                .select('*')
+                .from('resources')
+                .whereIn('url', [...urls])
+                .first();
+            return res ?? null;
+        }
         /**
          * Retrieves all sub-resources from the `resources` table.
          * @returns An array of raw {@link DB_Resource} rows.
@@ -564,12 +854,113 @@ let Database = (() => {
             const res = await this.#instance.select('url').from('resources');
             return res.map((r) => r.url);
         }
+        /**
+         * Counts pages that were scraped as crawl targets (full HTML render).
+         *
+         * Used by the crawler to seed its `pagesScraped` counter on resume so the
+         * progress display reflects all browser-rendered HTML pages across sessions,
+         * not just the current one.
+         *
+         * "HTML page" is guaranteed by `contentType = 'text/html'`, NOT by `isTarget`
+         * alone: `isTarget` means "in-scope crawl target" and is set for in-scope
+         * non-HTML resources too (e.g. a PDF reached via the HEAD pre-flight is
+         * `isTarget = 1`). Counting those would over-report the HTML page total, so
+         * page-ness is asserted at the read layer here rather than by trusting
+         * `isTarget`.
+         * @returns The number of `text/html` rows with `isTarget = 1` and `scraped = 1`.
+         */
+        async getScrapedHtmlPageCount() {
+            const [row] = await this.#instance
+                .from('pages')
+                .where('isTarget', 1)
+                .andWhere('scraped', 1)
+                .andWhere('contentType', 'text/html')
+                .count('* as count');
+            return row ? Number(row.count) : 0;
+        }
+        /**
+         * Retrieves all `page_tags` rows for the given page id, parsed back into
+         * {@link TagRow} shape (with `categories` and `sources` JSON columns
+         * deserialised).
+         *
+         * Read-side counterpart to `#insertTags`.
+         * @param pageId
+         */
+        async getTagsOfPage(pageId) {
+            const rows = await this.#instance
+                .select('id', 'pageId', 'provider', 'category', 'externalId', 'version', 'confidence', 'categories', 'sources')
+                .from('page_tags')
+                .where('pageId', pageId)
+                .orderBy('id', 'asc');
+            return rows.map((r) => ({
+                id: r.id,
+                pageId: r.pageId,
+                provider: r.provider,
+                category: r.category,
+                externalId: r.externalId,
+                version: r.version,
+                confidence: r.confidence,
+                categories: r.categories === null ? [] : (safeParseJson(r.categories) ?? []),
+                sources: r.sources === null ? [] : (safeParseJson(r.sources) ?? []),
+            }));
+        }
+        /**
+         * Records a crawler-level (`error` channel) failure into `crawl_errors`.
+         *
+         * Unlike {@link insertPageError} this is not tied to a scraped page: `url`
+         * may be an external link that never became a page row, or `null` for a
+         * process-level error. The cause is intentionally not stored — it is derived
+         * on read so that older archives (which only have `error.log`) and freshly
+         * captured rows classify identically.
+         * @param url - The URL the error is about, or `null` for a process-level error.
+         * @param message - The error message (one line is enough for classification).
+         * @param isExternal - Whether the URL is external to the crawl scope.
+         */
+        async insertCrawlError(url, message, isExternal = false) {
+            await this.#instance('crawl_errors').insert({
+                url,
+                isExternal: isExternal ? 1 : 0,
+                message,
+                createdAt: Date.now(),
+            });
+        }
+        /**
+         * Records a partial scrape failure against the page identified by `url`.
+         *
+         * The page row is resolved (or inserted as a stub) via
+         * {@link Database.#getIdByUrl} so the error can be recorded even before
+         * `setPage` has run — useful when the failure fires during scraping
+         * (e.g. mid-`scrapeStart`) and the orchestrator enqueues this write
+         * before the success write for the same URL.
+         *
+         * A single page can have multiple `page_errors` rows (e.g. both
+         * `desktop-compact` and `mobile-small` viewports failing).
+         * @param url - URL of the page being scraped.
+         * @param phase - Scrape phase name (typically `'retryExhausted'`).
+         * @param message - Human-readable failure message.
+         * @param isExternal - Whether the URL is external. Defaults to `false`.
+         */
+        async insertPageError(url, phase, message, isExternal = false) {
+            const pageId = await this.#getIdByUrl(url, isExternal ? 1 : 0);
+            await this.#instance('page_errors').insert({
+                pageId,
+                phase,
+                message,
+                createdAt: Date.now(),
+            });
+        }
         /**
          * Inserts a sub-resource into the `resources` table.
          * Ignores duplicate URLs (uses `ON CONFLICT IGNORE`).
+         *
+         * The `source` provenance label is written ONLY on insert; an
+         * `ON CONFLICT IGNORE` collision leaves an existing row's source untouched
+         * (this is what makes a second `crawl --inventory` non-destructive — see
+         * the inventory plan).
          * @param resource - The resource data to insert.
+         * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
          */
-        async insertResource(resource) {
+        async insertResource(resource, source) {
             await this.#instance
                 .from('resources')
                 .insert({
@@ -577,11 +968,14 @@ let Database = (() => {
                 isExternal: resource.isExternal ? 1 : 0,
                 status: resource.status,
                 statusText: resource.statusText,
-                contentType: resource.contentType,
+                // Canonicalize like `pages.contentType` (see #insertPage) so resource
+                // content-type filters / dedupe keys are case- and whitespace-stable.
+                contentType: normalizeContentType(resource.contentType),
                 contentLength: resource.contentLength,
                 compress: resource.compress || 0,
                 cdn: resource.cdn || 0,
                 responseHeaders: JSON.stringify(resource.headers),
+                ...(source === undefined ? {} : { source }),
             })
                 .onConflict('url')
                 .ignore();
@@ -611,6 +1005,48 @@ let Database = (() => {
                 .onConflict(['resourceId', 'pageId'])
                 .ignore();
         }
+        /**
+         * Records a redirect edge (source → destination) **without** re-storing the
+         * destination's content.
+         *
+         * The crawler renders a many-to-one redirect destination exactly once. For
+         * every subsequent source URL that redirects to that already-rendered
+         * destination, it calls this instead of {@link updatePage} (#73). Routing a
+         * content-less HEAD result through `updatePage` would funnel it into
+         * `#insertPage` and overwrite the destination's good title / meta with empty
+         * values, so the dedicated edge-only path is required.
+         *
+         * The destination row is resolved (created on demand if a concurrent in-flight
+         * render has not committed it yet) so the edge always points at a valid id;
+         * the single render fills in the destination's content under that same id.
+         * The destination's existing anchors / images are never touched here.
+         * @param page - HEAD-resolved page data carrying the redirect chain. Its
+         *   `anchorList` / `imageList` are ignored (a redirect source owns no content).
+         */
+        async recordRedirect(page) {
+            const { destUrl, sources } = resolveRedirectChain(page.url.withoutHashAndAuth, page.redirectPaths);
+            // No redirect chain (the URL is itself the already-rendered destination,
+            // reached both directly and via a redirect) → there is no edge to write.
+            // Returning here avoids opening a transaction and, crucially, avoids
+            // `#getIdByUrl` inserting a content-less placeholder row for a destination
+            // that may not have been written yet.
+            if (sources.length === 0) {
+                return;
+            }
+            const destUrlObject = parseUrl(destUrl);
+            if (!destUrlObject) {
+                // A malformed redirect target should not abort the whole crawl (this
+                // runs inside the WriteQueue, whose rejection aborts the run). Recording
+                // a single redirect edge is best-effort, so skip it and move on. Unlike
+                // `updatePage`, there is no page content at stake here.
+                dbLog('recordRedirect: skip malformed destination URL: %s', destUrl);
+                return;
+            }
+            await this.#instance.transaction(async (trx) => {
+                const destId = await this.#getIdByUrl(destUrlObject.withoutHashAndAuth, undefined, trx);
+                await this.#linkRedirectSources(trx, sources, destId, destUrlObject.withoutHashAndAuth, page.isExternal);
+            });
+        }
         /**
          * Promote previously-external pages whose URL falls under any of the new scope
          * entries back to a "needs scraping" state so that the next crawl picks them up
@@ -654,28 +1090,137 @@ let Database = (() => {
                 return [];
             }
             const chunkSize = 500;
+            const metaReset = makeMetaResetPayload();
             for (let i = 0; i < promotedIds.length; i += chunkSize) {
                 const chunk = promotedIds.slice(i, i + chunkSize);
-                await this.#instance('pages').whereIn('id', chunk).update({
+                await this.#instance('pages')
+                    .whereIn('id', chunk)
+                    .update({
                     scraped: 0,
                     isExternal: 0,
                     isSkipped: 0,
                     skipReason: null,
-                    html: null,
                     status: null,
                     statusText: null,
                     contentType: null,
                     contentLength: null,
                     responseHeaders: '{}',
                     redirectDestId: null,
+                    // Null every flat meta column + denormalised aggregates +
+                    // meta_extras. `firstCrawledAt` / `lastCrawledAt` are
+                    // deliberately omitted from META_NULLABLE_COLUMNS — the
+                    // last-success timestamp survives the demotion.
+                    ...metaReset,
                 });
+                // Clear the prior crawl's data for the repromoted pages. `updatePage`
+                // also replaces anchors/images/tags/jsonld when it re-scrapes them, but
+                // only when the new scrape is non-empty — so this pre-clear is still
+                // load-bearing for pages that get repromoted but then re-scrape to
+                // nothing (or are never reached again), and it is the only place
+                // `resources-referrers` is cleared. The HTML body ref is also cleared
+                // so a repromoted page whose re-scrape ends up degraded does not keep
+                // its old external-render snapshot. `page_tags` / `page_jsonld` are
+                // cleared explicitly even though both tables also carry ON DELETE
+                // CASCADE — we keep the existing pattern of explicit chunked DELETEs
+                // rather than relying on CASCADE indirectly (and would not cascade
+                // anyway: the parent `pages` row is updated, not deleted). Orphan
+                // blobs in `page_html_blobs` are left behind; #23 will add GC.
                 await this.#instance('anchors').whereIn('pageId', chunk).delete();
                 await this.#instance('images').whereIn('pageId', chunk).delete();
                 await this.#instance('resources-referrers').whereIn('pageId', chunk).delete();
+                await this.#instance('page_html_ref').whereIn('page_id', chunk).delete();
+                await this.#instance('page_tags').whereIn('pageId', chunk).delete();
+                await this.#instance('page_jsonld').whereIn('pageId', chunk).delete();
             }
             dbLog('Repromoted %d external pages back to pending', promotedUrls.length);
             return promotedUrls;
         }
+        /**
+         * Reset previously-attempted pages that ended in a recoverable failure so a
+         * follow-up crawl can re-fetch them from scratch.
+         *
+         * A page qualifies as a recoverable failure when it was already scraped
+         * (`scraped = 1`), is not a redirect source (`redirectDestId IS NULL`), was
+         * not intentionally skipped (`isSkipped` is not `1`), and one of the
+         * following holds:
+         *
+         * - `status = -1` — the sentinel a hard scrape failure (network error,
+         *   timeout, browser crash) is recorded with (see `handle-scrape-error.ts`);
+         * - `status IS NULL` — no status was ever stored for the row;
+         * - `contentType IS NULL` — the content type could not be determined;
+         * - `status` is in the `5xx` range — a (frequently transient) server error.
+         *
+         * Definitive `4xx` responses are intentionally excluded: re-fetching a 404
+         * almost always yields the same answer. Matching rows — internal and
+         * external alike — are demoted back to pending (`scraped = 0`) and have their
+         * stale scrape metadata cleared. The page row itself is kept (id preserved)
+         * so existing `anchors.hrefId` referrers stay valid, and `isExternal` is left
+         * untouched so the next pass re-classifies each page from the crawl scope.
+         * Related `anchors`, `images`, `resources-referrers`, and `page_errors` rows
+         * are deleted so the re-scrape can re-insert fresh data without duplicates.
+         *
+         * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
+         * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+         * @returns The URLs of the pages that were reset to pending.
+         */
+        async resetFailedPages() {
+            const candidates = await this.#instance
+                .select('id', 'url')
+                .from('pages')
+                .where('scraped', 1)
+                .whereNull('redirectDestId')
+                .where((qb) => {
+                qb.where('isSkipped', 0).orWhereNull('isSkipped');
+            })
+                .where((qb) => {
+                qb.whereNull('status')
+                    .orWhere('status', -1)
+                    .orWhereNull('contentType')
+                    .orWhereBetween('status', [500, 599]);
+            });
+            if (candidates.length === 0) {
+                return [];
+            }
+            const ids = candidates.map((row) => row.id);
+            const urls = candidates.map((row) => row.url);
+            const chunkSize = 500;
+            const metaReset = makeMetaResetPayload();
+            for (let i = 0; i < ids.length; i += chunkSize) {
+                const chunk = ids.slice(i, i + chunkSize);
+                await this.#instance('pages')
+                    .whereIn('id', chunk)
+                    .update({
+                    scraped: 0,
+                    status: null,
+                    statusText: null,
+                    contentType: null,
+                    contentLength: null,
+                    responseHeaders: '{}',
+                    // Null every flat meta column + denormalised aggregates +
+                    // meta_extras. `firstCrawledAt` / `lastCrawledAt` are
+                    // deliberately omitted from META_NULLABLE_COLUMNS so the
+                    // last-success timestamp records survive the demotion (the
+                    // within-archive observation axis for #11/#17/#19).
+                    ...metaReset,
+                });
+                // Clear the prior crawl's per-page data so the re-scrape starts clean.
+                // `updatePage` only replaces anchors/images/tags/jsonld when the new
+                // scrape is non-empty, so this pre-clear is load-bearing for pages that
+                // reset but then fail again (or are never reached), and it is the only
+                // place `resources-referrers` and `page_errors` are cleared. The HTML
+                // body ref is also cleared so a previously-rendered page that now fails
+                // to re-scrape does not keep its old snapshot.
+                await this.#instance('anchors').whereIn('pageId', chunk).delete();
+                await this.#instance('images').whereIn('pageId', chunk).delete();
+                await this.#instance('resources-referrers').whereIn('pageId', chunk).delete();
+                await this.#instance('page_errors').whereIn('pageId', chunk).delete();
+                await this.#instance('page_html_ref').whereIn('page_id', chunk).delete();
+                await this.#instance('page_tags').whereIn('pageId', chunk).delete();
+                await this.#instance('page_jsonld').whereIn('pageId', chunk).delete();
+            }
+            dbLog('Reset %d failed pages back to pending', urls.length);
+            return urls;
+        }
         /**
          * Stores the crawl configuration in the `info` table.
          * Only fields in {@link INFO_COLUMN_ALLOWLIST} are forwarded — any extra
@@ -774,24 +1319,26 @@ let Database = (() => {
         }
         /**
          * Inserts or updates a crawled page in the database, including its redirect chain,
-         * anchors, and images. Optionally creates an HTML snapshot file path entry.
+         * anchors, images, and (when `writeHtml`) its compressed HTML snapshot BLOB.
          *
          * Self-redirects (where the source URL equals the destination URL after normalization)
          * are skipped to avoid marking a page as redirected to itself — a situation caused by
          * authentication challenges (e.g. Basic Auth 302) that would otherwise exclude the page
          * from reports via the `whereNull('redirectDestId')` filter.
          * @param page - The page data to store.
-         * @param snapshotDir - The directory for saving HTML snapshots, or null to skip snapshots.
+         * @param writeHtml - When `true`, this call is allowed to insert (or clear)
+         *   the page's HTML blob. `setExternalPage` passes `false` because external
+         *   metadata-only scrapes never carry HTML and must not perturb an already
+         *   stored body.
          * @param isTarget - Whether this page is a crawl target.
-         * @returns An object with the optional `html` snapshot file path and the page's database `pageId`.
+         * @param source - Provenance label written ONLY when the row is freshly
+         *   inserted. Existing rows keep their original `source` (this is why a
+         *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
+         *   that was discovered earlier).
+         * @returns The database `pageId` of the inserted/updated row.
          */
-        async updatePage(page, snapshotDir, isTarget) {
-            let destUrl = page.url.withoutHashAndAuth;
-            const redirectPaths = [...page.redirectPaths];
-            if (redirectPaths.length > 0) {
-                destUrl = redirectPaths.pop();
-                redirectPaths.unshift(page.url.withoutHashAndAuth);
-            }
+        async updatePage(page, writeHtml, isTarget, source) {
+            const { destUrl, sources } = resolveRedirectChain(page.url.withoutHashAndAuth, page.redirectPaths);
             const destUrlObject = parseUrl(destUrl);
             if (!destUrlObject) {
                 throw new Error(`Failed to parse URL: ${destUrl}`);
@@ -800,27 +1347,70 @@ let Database = (() => {
                 const pageId = await this.#insertPage({
                     ...page,
                     url: destUrlObject,
-                }, isTarget, trx);
-                const destUrlNormalized = destUrlObject.withoutHashAndAuth;
-                for (const redirect of redirectPaths) {
-                    if (redirect === destUrlNormalized) {
-                        dbLog('Skip self-redirect: %s', redirect);
-                        continue;
-                    }
-                    dbLog('Set redirected url: %s -> %s', redirect, destUrl);
-                    const redirectId = await this.#getIdByUrl(redirect, undefined, trx);
-                    await trx('pages')
-                        .where('id', redirectId)
-                        .update({
-                        scraped: 1,
-                        redirectDestId: pageId,
-                        isExternal: page.isExternal ? 1 : 0,
-                    });
+                }, isTarget, trx, source);
+                // Wappalyzer tag detection is HTML-body independent (relies on
+                // `<script src>` / `<iframe src>` / window globals / response
+                // headers) so it runs for every page including external /
+                // metadata-only. JSON-LD on the other hand lives inside the
+                // rendered HTML body, so we only write it when there is HTML to
+                // scrape — see the same `writeHtml` gate as `#writePageHtmlBlob`
+                // below.
+                await this.#insertTags(pageId, page.meta, trx);
+                if (writeHtml) {
+                    await this.#insertJsonLd(pageId, page.meta, trx);
+                }
+                await this.#linkRedirectSources(trx, sources, pageId, destUrlObject.withoutHashAndAuth, page.isExternal);
+                // Only insert a snapshot blob when there is actual HTML to write.
+                // `page.html.length > 0` is the precise signal: the scraper returns
+                // `html: ''` for everything that is not a rendered `text/html` document
+                // (non-HTML responses, metadata-only, external, degraded renders), so a
+                // non-empty `html` is exactly "a rendered HTML body exists". Gating on
+                // `isTarget` alone would store an empty body for every internal non-HTML
+                // resource — PDF / zip / images are isTarget=1 (#72).
+                //
+                // `isTarget` is intentionally NOT part of this condition: it is implied by
+                // `html.length > 0` (only in-scope target pages are browser-rendered into a
+                // non-empty body; metadata-only and external pages carry `html: ''`), so the
+                // content check alone expresses the intent without a redundant term.
+                if (writeHtml && page.html.length > 0) {
+                    await this.#writePageHtmlBlob(pageId, page.html, trx);
                 }
-                let snapshot = { pageId };
-                if (isTarget && snapshotDir) {
-                    snapshot = await this.#updateSnapshotPath(pageId, snapshotDir, trx);
+                else if (writeHtml &&
+                    page.contentType !== null &&
+                    !isHtmlContentType(page.contentType)) {
+                    // The page is now a *known* non-HTML type. If a previous scrape stored
+                    // an HTML body for this URL (e.g. it served HTML then was replaced by
+                    // a PDF across `crawl --resume` / `--append`), drop the stale ref so
+                    // `page_html_ref` never contradicts `contentType`. A degraded HTML
+                    // re-scrape (text/html or unknown content type with empty html) is NOT
+                    // cleared — the last good snapshot is preserved, mirroring the
+                    // anchors / images empty-guard below. Gated on `writeHtml` because a
+                    // stale ref can only have been written by a snapshot-capable call
+                    // (`setPage`); `setExternalPage` passes `writeHtml = false` and never
+                    // sets `html`, so it has nothing to clear.
+                    await trx('page_html_ref').where('page_id', pageId).delete();
                 }
+                // Re-scrape semantics: the same URL can be scraped more than once
+                // (e.g. `crawl --resume`, re-visits, `--append` re-promotion). The
+                // `anchors` / `images` tables have no uniqueness constraint, so
+                // re-inserting without clearing would accumulate a full duplicate set
+                // on every re-scrape (the bug fixed in #70). So we delete-then-insert
+                // to *replace* the previous rows.
+                //
+                // The delete is paired with — and guarded by — a non-empty new list:
+                // a degraded re-scrape (navigation timeout / partial render) can return
+                // an empty `anchorList` for a page that previously had links, and
+                // wiping the prior good data in that case would be destructive. We
+                // cannot tell a transient empty result apart from a page that has
+                // legitimately lost all its links, so we err on the side of keeping
+                // what we already had. The accepted trade-off is that a page which
+                // genuinely dropped to zero links keeps its stale rows until the next
+                // non-empty re-scrape replaces them.
+                //
+                // (A DB-level unique constraint + `onConflict` would also prevent
+                // duplication, but multiple distinct anchors can share the same
+                // hrefId/hash/textContent legitimately, so there is no natural unique
+                // key to enforce — replace-on-write is the correct mechanism here.)
                 const anchors = await Promise.all(page.anchorList.map(async (anchor) => {
                     const hrefId = await this.#getIdByUrl(anchor.href.withoutHashAndAuth, anchor.isExternal ? 1 : 0, trx);
                     return {
@@ -832,6 +1422,7 @@ let Database = (() => {
                 }));
                 dbLog('Insert anchors.length: %d', anchors.length);
                 if (anchors.length > 0) {
+                    await trx('anchors').where('pageId', pageId).delete();
                     await eachSplitted(anchors, 100, async (_anchors) => {
                         await trx('anchors').insert(_anchors);
                     });
@@ -842,21 +1433,29 @@ let Database = (() => {
                 }));
                 dbLog('Insert images.length: %d', images.length);
                 if (images.length > 0) {
+                    await trx('images').where('pageId', pageId).delete();
                     await eachSplitted(images, 100, async (_images) => {
                         await trx('images').insert(_images);
                     });
                 }
-                return snapshot;
+                return pageId;
             });
         }
         /**
          * Returns the database ID for a URL, creating a new page row if needed.
          * Uses `ON CONFLICT IGNORE` to handle race conditions in concurrent inserts.
+         *
+         * `source` is written ONLY on the INSERT path — when the row already
+         * exists, we never reach the INSERT and the existing row's `source`
+         * stays untouched. This is what keeps a second `crawl --inventory` from
+         * "demoting" a page that was first labelled `'inventory-seed'` back to
+         * `'inventory-discovered'` on later passes.
          * @param url
          * @param isExternal
          * @param trx
+         * @param source - Provenance label to put on the newly-inserted row. `undefined` lets the DB DEFAULT (`'crawled'`) apply.
          */
-        async #getIdByUrl(url, isExternal, trx) {
+        async #getIdByUrl(url, isExternal, trx, source) {
             const qb = trx ?? this.#instance;
             const [record] = await qb.select('id').from('pages').where('url', url);
             // Must use `?` because it may be `undefined`
@@ -870,6 +1469,7 @@ let Database = (() => {
                 scraped: 0,
                 isTarget: 0,
                 ...(isExternal != null && { isExternal }),
+                ...(source === undefined ? {} : { source }),
             })
                 .onConflict('url')
                 .ignore();
@@ -888,22 +1488,115 @@ let Database = (() => {
          * Initializes the database schema if tables do not exist, then runs lightweight
          * migrations that bring older archives up to the current schema.
          *
-         * Migrations are idempotent and run on every {@link Database.connect}, so the
-         * same DB can be opened safely from both writer and reader code paths.
+         * Migrations are idempotent and run on every writer-side {@link Database.connect};
+         * in read-only mode they are SKIPPED so the same DB can be opened safely
+         * by a viewer attached to a live (or interrupted) crawl without rewriting
+         * the user's tmpDir.
+         * @param readOnly - When true, skip schema init + migrations.
          */
-        async #init() {
+        async #init(readOnly) {
+            // Connection-level PRAGMAs (foreign_keys, mmap_size, …) must be
+            // reapplied on every connect — they are not persisted across opens.
+            // They are safe in read-only mode because they don't write to the
+            // user's tmpDir, just configure the libsql connection.
+            await applyConnectionPragmas(this.#instance);
+            // Reject pre-0.10 archives before any further work. Runs for both
+            // writer and read-only (stub viewer) connections so old
+            // `._nitpicker-*` stubs surface a clear error instead of
+            // dereferencing missing columns at query time. New archives (no
+            // `info` table yet) pass through; the schema is filled in by
+            // `initSchema` below.
+            await assertCompatibleVersion(this.#instance);
+            if (readOnly) {
+                return;
+            }
             await initSchema(this.#instance);
             await migrateInfoRoots(this.#instance);
+            await migratePageErrors(this.#instance);
+            await migrateCrawlErrors(this.#instance);
+            await migrateHtmlBlobTables(this.#instance);
+            await migratePagesResourcesSource(this.#instance);
+        }
+        /**
+         * Replaces the page's JSON-LD / SpeculationRules rows with the freshly
+         * captured set. Called inside `updatePage`'s transaction.
+         *
+         * `writeHtml = false` branches (`setExternalPage`, metadata-only) skip
+         * this entirely — JSON-LD lives inside the HTML body, so external pages
+         * that are not rendered have no entries to write. An empty array on a
+         * normally-rendered page is treated as a degraded re-scrape: prior rows
+         * are kept (same `delete-only-when-replacing` invariant as `anchors` /
+         * `images`).
+         * @param pageId
+         * @param meta
+         * @param trx
+         */
+        async #insertJsonLd(pageId, meta, trx) {
+            // `??` guards tolerate the legacy "minimal meta" shape from older test
+            // fixtures. Real beholder 3.0.0 always populates these required fields.
+            const jsonLd = meta.jsonLd ?? [];
+            const speculationRules = meta.speculationRules ?? [];
+            const rows = [];
+            for (const entry of jsonLd) {
+                rows.push({
+                    pageId,
+                    kind: 'ld+json',
+                    type: classifyJsonLdType(entry),
+                    raw: entry.raw,
+                    parsed: entry.parsed === undefined ? null : JSON.stringify(entry.parsed),
+                    parseError: entry.parseError ?? null,
+                });
+            }
+            for (const entry of speculationRules) {
+                rows.push({
+                    pageId,
+                    kind: 'speculationrules',
+                    type: classifyJsonLdType(entry),
+                    raw: entry.raw,
+                    parsed: entry.parsed === undefined ? null : JSON.stringify(entry.parsed),
+                    parseError: entry.parseError ?? null,
+                });
+            }
+            if (rows.length === 0)
+                return;
+            await trx('page_jsonld').where('pageId', pageId).delete();
+            await eachSplitted(rows, 100, async (chunk) => {
+                await trx('page_jsonld').insert(chunk);
+            });
         }
         /**
          * Upserts page data into the `pages` table (inserts if new, updates if existing).
+         *
+         * `source` is intentionally NOT in the UPDATE clause — provenance is set
+         * once at INSERT time inside `#getIdByUrl`, and existing rows keep
+         * whatever label they were first inserted with.
          * @param page
          * @param isTarget
          * @param trx
+         * @param source - Inventory provenance for the INSERT path. Ignored on UPDATE.
          */
-        async #insertPage(page, isTarget, trx) {
+        async #insertPage(page, isTarget, trx, source) {
             const qb = trx ?? this.#instance;
-            const pageId = await this.#getIdByUrl(page.url.withoutHashAndAuth, undefined, trx);
+            const pageId = await this.#getIdByUrl(page.url.withoutHashAndAuth, undefined, trx, source);
+            const flat = deriveFlatFromMeta(page.meta, page.url.href);
+            const denorm = computePageDenormalized(page.meta);
+            const extras = deriveMetaExtras(page.meta);
+            const now = Date.now();
+            // Source promotion on UPDATE: when an inventory-mode scrape lands on
+            // a row that was created earlier as a placeholder (e.g. an anchor
+            // from a seed page pointed at this URL and `#getIdByUrl` inserted a
+            // row with the DB DEFAULT `'crawled'`), bump the label to the
+            // inventory variant. But never demote an already-inventoried row —
+            // `CASE WHEN source = 'crawled' THEN ? ELSE source END` keeps a
+            // previously labelled `'inventory-seed'` or `'inventory-discovered'`
+            // row intact on a second pass.
+            const sourceUpdate = source === undefined
+                ? {}
+                : {
+                    source: qb.raw("CASE WHEN source = 'crawled' THEN ? ELSE source END", [
+                        source,
+                    ]),
+                };
             await qb('pages')
                 .where('id', pageId)
                 .update({
@@ -912,58 +1605,185 @@ let Database = (() => {
                 isExternal: page.isExternal,
                 status: page.status,
                 statusText: page.statusText,
-                contentType: page.contentType,
+                // Canonicalize so the stored value matches the exact-string page-ness
+                // predicate (`WHERE contentType = 'text/html'`) used by the read layer
+                // and the case-insensitive `isHtmlContentType` used in code. Responses
+                // are recorded verbatim upstream, so `Text/HTML` / `text/html ` can
+                // otherwise be stored and silently misclassified.
+                contentType: normalizeContentType(page.contentType),
                 contentLength: page.contentLength,
                 responseHeaders: JSON.stringify(page.responseHeaders),
-                lang: page.meta.lang,
-                title: page.meta.title,
-                description: page.meta.description,
-                keywords: page.meta.keywords,
-                noindex: page.meta.noindex,
-                nofollow: page.meta.nofollow,
-                noarchive: page.meta.noarchive,
-                canonical: page.meta.canonical,
-                alternate: page.meta.alternate,
-                og_type: page.meta['og:type'],
-                og_title: page.meta['og:title'],
-                og_site_name: page.meta['og:site_name'],
-                og_description: page.meta['og:description'],
-                og_url: page.meta['og:url'],
-                og_image: page.meta['og:image'],
-                twitter_card: page.meta['twitter:card'],
+                // Flat meta columns derived from beholder 3.0.0 nested Meta.
+                // URL-shaped columns (canonical / og_url / og_image / amphtml / manifest /
+                // icon_href / appleTouchIcon_href / twitter_image) are already absolutised
+                // by `deriveFlatFromMeta` against the page URL — `find-mismatches` compares
+                // `canonical != url` directly, so storing the raw `getAttribute('href')`
+                // would generate false positives for sites using relative canonicals.
+                ...flat,
+                // Denormalised aggregates: written once at scrape time so list reads
+                // (Sheets, page-detail summary) can answer "how many JSON-LD entries?"
+                // and "which Wappalyzer providers?" by selecting a single pages column
+                // rather than running a GROUP BY join on every read.
+                tag_count: denorm.tag_count,
+                jsonld_count: denorm.jsonld_count,
+                tags_providers_csv: denorm.tags_providers_csv,
+                // JSON catch-all for nested Meta sub-objects not flattened above.
+                meta_extras: JSON.stringify(extras),
+                // Timestamps: `firstCrawledAt` is set only on first INSERT — `COALESCE`
+                // preserves the existing value so a re-scrape (`--append`, `--retry-failed`)
+                // does not erase the discovery time. `lastCrawledAt` is updated every
+                // successful scrape.
+                firstCrawledAt: qb.raw('COALESCE(firstCrawledAt, ?)', [now]),
+                lastCrawledAt: now,
                 isSkipped: page.isSkipped,
+                ...sourceUpdate,
             });
             return pageId;
         }
         /**
-         * Assigns and persists the HTML snapshot file path for a page.
+         * Replaces the page's Wappalyzer tag rows with the freshly captured set.
+         * Called inside `updatePage`'s transaction unconditionally — tag
+         * detection draws on `<script src>` / `<iframe src>` / window globals /
+         * response headers, not the HTML body, so external pages that skip
+         * rendering still contribute tags.
+         *
+         * Same empty-guard as `#insertJsonLd`: an empty array does not wipe
+         * prior rows on a degraded re-scrape.
          * @param pageId
-         * @param snapshotDir
+         * @param meta
          * @param trx
          */
-        async #updateSnapshotPath(pageId, snapshotDir, trx) {
-            const qb = trx ?? this.#instance;
-            const snapshotHtmlPath = path.resolve(snapshotDir, `${pageId}.html`);
-            const snapshotRelHtmlPath = path.relative(this.#workingDir, snapshotHtmlPath);
-            await qb('pages').where('id', pageId).update({
-                html: snapshotRelHtmlPath,
-            });
-            return {
-                html: snapshotHtmlPath,
+        async #insertTags(pageId, meta, trx) {
+            const partial = extractTagsForArchive(meta.tags);
+            if (partial.length === 0)
+                return;
+            const rows = partial.map((p) => ({
                 pageId,
-            };
+                provider: p.provider,
+                category: p.category,
+                externalId: p.externalId,
+                version: p.version,
+                confidence: p.confidence,
+                categories: JSON.stringify(p.categories),
+                sources: JSON.stringify(p.sources),
+            }));
+            await trx('page_tags').where('pageId', pageId).delete();
+            await eachSplitted(rows, 100, async (chunk) => {
+                await trx('page_tags').insert(chunk);
+            });
+        }
+        /**
+         * Points each redirect-source URL at the destination page, marking it scraped
+         * and clearing any content it owned in a former life.
+         *
+         * Shared by {@link updatePage} (which also renders and stores the destination)
+         * and {@link recordRedirect} (which only records the edge for a destination
+         * rendered elsewhere). Self-redirects (source equal to the destination) are
+         * skipped so a page is never marked as redirecting to itself — that would
+         * exclude it from reports via the `whereNull('redirectDestId')` filter.
+         * @param trx - The active transaction.
+         * @param sources - Redirect-source URLs (normalised): the original URL plus
+         *   any intermediate hops. Empty when the page was not redirected.
+         * @param destId - Database id of the redirect destination page.
+         * @param destUrlNormalized - Normalised destination URL, used to detect and
+         *   skip self-redirects.
+         * @param isExternal - Whether the sources are external to the crawl scope.
+         */
+        async #linkRedirectSources(trx, sources, destId, destUrlNormalized, isExternal) {
+            for (const redirect of sources) {
+                if (redirect === destUrlNormalized) {
+                    dbLog('Skip self-redirect: %s', redirect);
+                    continue;
+                }
+                dbLog('Set redirected url: %s -> id:%d', redirect, destId);
+                const redirectId = await this.#getIdByUrl(redirect, undefined, trx);
+                await trx('pages')
+                    .where('id', redirectId)
+                    .update({
+                    scraped: 1,
+                    redirectDestId: destId,
+                    isExternal: isExternal ? 1 : 0,
+                });
+                // A page that used to be scraped as content can later turn into a
+                // redirect source. It owns no content anymore, so drop any anchors /
+                // images it captured in its former life — otherwise they linger and
+                // leak into referrer / incoming-link reads (which do not filter out
+                // redirect sources).
+                await trx('anchors').where('pageId', redirectId).delete();
+                await trx('images').where('pageId', redirectId).delete();
+            }
+        }
+        /**
+         * Encodes, dedups, and persists a page's HTML snapshot.
+         *
+         * Computes SHA-256 over the raw UTF-8 bytes, compresses them with zstd,
+         * inserts into `page_html_blobs` only if the hash is new (so identical
+         * bodies — 404 templates, error pages, redirect destinations — share a
+         * single row), and then upserts `page_html_ref(page_id → hash)` so the
+         * latest scrape always points at the right body.
+         *
+         * Runs entirely inside the caller's transaction; a failure here rolls
+         * back the rest of `updatePage`, which is the desired semantics (an
+         * archive that lost its HTML for a page would otherwise serve stale
+         * meta against a missing body).
+         * @param pageId - The database id of the page.
+         * @param html - The raw HTML string (UTF-8).
+         * @param trx - The active transaction.
+         */
+        async #writePageHtmlBlob(pageId, html, trx) {
+            const rawBytes = Buffer.from(html, 'utf8');
+            const hash = createHash('sha256').update(rawBytes).digest();
+            const compressed = zstdCompressSync(rawBytes);
+            await trx('page_html_blobs')
+                .insert({
+                hash,
+                body: compressed,
+                codec: 'zstd',
+                size_raw: rawBytes.byteLength,
+                size_stored: compressed.byteLength,
+            })
+                .onConflict('hash')
+                .ignore();
+            // Upsert so a re-scrape's body cleanly supersedes the prior pointer.
+            // The old blob row is intentionally left in place — a future #23 GC
+            // pass will sweep unreachable hashes.
+            await trx('page_html_ref')
+                .insert({ page_id: pageId, hash })
+                .onConflict('page_id')
+                .merge(['hash']);
         }
         /**
          * Creates and initializes a new Database instance.
-         * Creates the parent directory for the database file if needed,
-         * establishes the connection, and initializes tables if they do not exist.
-         * @param options - Database connection options (working directory + SQLite file path).
+         *
+         * **Writer mode (default)**: creates the parent directory for the
+         * database file if needed, establishes the connection, and initializes
+         * the schema + migrations.
+         *
+         * **Read-only mode** (`options.readOnly`): refuses to resurrect a
+         * missing parent directory or db file — throws if either is absent at
+         * the time of the call. Skips schema init and migrations entirely so
+         * the user's tmpDir is never modified. Required by viewer / MCP
+         * stub-mode opens, where a TOCTOU window between classification and
+         * `connect()` could otherwise leave behind a phantom empty tmpDir.
+         * @param options - Database connection options.
          * @returns A fully initialized Database instance.
+         * @throws {Error} In read-only mode, if the parent directory or db
+         *   file does not exist when `connect()` runs.
          */
         static async connect(options) {
-            mkdir(options.filename);
+            if (options.readOnly) {
+                if (!existsSync(path.dirname(options.filename))) {
+                    throw new Error(`Cannot open archive read-only: parent directory disappeared (${path.dirname(options.filename)}). The source may have been removed by another process.`);
+                }
+                if (!existsSync(options.filename)) {
+                    throw new Error(`Cannot open archive read-only: database file missing (${options.filename}). The source may have been removed by another process.`);
+                }
+            }
+            else {
+                mkdir(options.filename);
+            }
             const db = new Database(options);
-            await db.#init();
+            await db.#init(options.readOnly ?? false);
             return db;
         }
     };