@nitpicker/crawler 0.8.0 → 0.11.0

package/lib/archive/database.d.ts

@@ -1,4 +1,5 @@
-import type { Config, DatabaseOption, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, PageFilter } from './types.js';
+import type { JsonLdRow, TagRow } from './meta/types.js';
+import type { Config, DatabaseOption, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, PageFilter, PageSource } from './types.js';
 import type { PageData, Resource } from '../utils/types/types.js';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import type { Knex } from 'knex';
@@ -6,10 +7,11 @@ import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-awa
 /**
  * Low-level database abstraction layer for the archive's SQLite database.
  *
- * Manages the `pages`, `anchors`, `images`, `resources`, and `resources-referrers`
- * tables. All public methods that perform database queries use the `@retryable`
- * decorator for automatic retry on transient failures, and `@ErrorEmitter` to
- * propagate errors as events.
+ * Public methods that perform database queries use the `@retryable`
+ * decorator for automatic retry on transient failures, and `@ErrorEmitter`
+ * to propagate errors as events. The set of tables this layer manages is
+ * defined by `init-schema.ts` (the source of truth — query that file for
+ * the canonical list).
  *
  * Use the static {@link Database.connect} factory method to create instances.
  * The constructor is private.
@@ -30,12 +32,6 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * This ensures the database is fully self-contained in `db.sqlite` before archiving.
      */
     checkpoint(): Promise<void>;
-    /**
-     * Clears the HTML snapshot path for a page.
-     * Used to roll back the snapshot reference when the snapshot file write fails.
-     * @param pageId - The database ID of the page whose HTML path should be cleared.
-     */
-    clearHtmlPath(pageId: number): Promise<void>;
     /**
      * Destroys the database connection, releasing all pooled resources.
      */
@@ -69,11 +65,51 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
         pending: string[];
     }>;
     /**
-     * Retrieves the HTML snapshot file path for a specific page.
+     * Return the subset of `urls` that already exist in the `pages` table.
+     * Chunked into batches so SQLite's `IN (?, ?, …)` parameter limit
+     * (`SQLITE_MAX_VARIABLE_NUMBER`, default 999) cannot be hit even when the
+     * inventory list contains tens of thousands of URLs.
+     *
+     * Read-only — no transaction, no lock contention with the crawler write
+     * pipeline (callers run this BEFORE the `<archive>.bak` is taken and the
+     * crawl is started).
+     * @param urls - URL strings to probe (already in `withoutHashAndAuth` form).
+     * @returns URLs found in `pages`. Order is not preserved.
+     */
+    getExistingPageUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Return the subset of `urls` that already exist in the `resources` table.
+     * See {@link Database.getExistingPageUrls} — same chunking strategy.
+     * @param urls - URL strings to probe.
+     * @returns URLs found in `resources`.
+     */
+    getExistingResourceUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Reads the HTML snapshot stored as a zstd-compressed BLOB for the given page.
+     *
+     * Joins `page_html_ref` → `page_html_blobs` and decompresses inline. Returns
+     * `null` when the page has no stored body (a non-HTML resource, a redirect
+     * source, a degraded render). Read works identically on read-only / stub
+     * connections — the special-cased "do we have a loose dir vs zip?" branching
+     * the previous file-backed layout required is gone.
+     *
+     * Tables `page_html_ref` and `page_html_blobs` are created by `initSchema`.
+     * Older `.nitpicker` archives that predate this migration must be passed
+     * through `scripts/migrate-to-0.10.mjs` before they can be read.
      * @param pageId - The database ID of the page.
-     * @returns The relative file path to the HTML snapshot, or null if not saved.
+     * @returns The decompressed HTML string, or `null` if no snapshot is stored.
+     */
+    getHtmlOfPageById(pageId: number): Promise<string | null>;
+    /**
+     * Retrieves all `page_jsonld` rows for the given page id, parsed back into
+     * {@link JsonLdRow} shape (with `parsed` deserialised from its JSON column).
+     *
+     * Read-side counterpart to `#insertJsonLd`. Returns rows in insertion order
+     * by `id` so the order observed by `get-page-jsonld` matches the order the
+     * scraper saw them.
+     * @param pageId
      */
-    getHtmlPathOnPage(pageId: number): Promise<any>;
+    getJsonLdOfPage(pageId: number): Promise<JsonLdRow[]>;
     /**
      * Returns the underlying Knex query builder instance for direct SQL access.
      * This enables advanced queries (GROUP BY, HAVING, JOINs) at the database
@@ -122,6 +158,14 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     getRedirectsForPages(pageIds: number[]): Promise<DB_Redirect[]>;
     /**
      * Retrieves pages that link to a specific page (incoming links / referrers).
+     *
+     * Incoming links are resolved **through redirects**: an anchor pointing at a
+     * redirect source (e.g. `http://x` that 301s to `https://x`) counts as a
+     * referrer of the redirect's final destination, not of the source. This keeps
+     * backlinks merged on the canonical page instead of splitting them across the
+     * `http`/`https` (or any redirect source/dest) pair. The resolution mirrors
+     * `redirectTable()` — `redirectDestId` is pre-flattened to the final
+     * destination, so `COALESCE(target.redirectDestId, target.id)` is a single hop.
      * @param pageId - The database ID of the target page.
      * @returns An array of referrer records with URL, hash, and text content.
      */
@@ -132,6 +176,22 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns An array of page URL strings that reference the resource.
      */
     getReferrersOfResource(id: number): Promise<string[]>;
+    /**
+     * Retrieves a single sub-resource from the `resources` table by its URL.
+     *
+     * Accepts multiple URL candidates because the stored key is the resource's
+     * `href` while callers may only know the hash-stripped form; the first match
+     * wins.
+     *
+     * Deliberately NOT decorated with `@ErrorEmitter`: the only caller (the
+     * crawler's resource-reuse hook) has a full fallback (the HEAD pre-flight),
+     * so a read failure here must not surface as a database `error` event —
+     * the orchestrator aborts the whole crawl on that event, which is the
+     * correct reaction to write failures but not to a recoverable read.
+     * @param urls - URL candidates to match against the `url` column.
+     * @returns The raw {@link DB_Resource} row, or `null` if none match.
+     */
+    getResourceByUrl(urls: readonly string[]): Promise<DB_Resource | null>;
     /**
      * Retrieves all sub-resources from the `resources` table.
      * @returns An array of raw {@link DB_Resource} rows.
@@ -142,12 +202,73 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns An array of resource URL strings.
      */
     getResourceUrlList(): Promise<any[]>;
+    /**
+     * Counts pages that were scraped as crawl targets (full HTML render).
+     *
+     * Used by the crawler to seed its `pagesScraped` counter on resume so the
+     * progress display reflects all browser-rendered HTML pages across sessions,
+     * not just the current one.
+     *
+     * "HTML page" is guaranteed by `contentType = 'text/html'`, NOT by `isTarget`
+     * alone: `isTarget` means "in-scope crawl target" and is set for in-scope
+     * non-HTML resources too (e.g. a PDF reached via the HEAD pre-flight is
+     * `isTarget = 1`). Counting those would over-report the HTML page total, so
+     * page-ness is asserted at the read layer here rather than by trusting
+     * `isTarget`.
+     * @returns The number of `text/html` rows with `isTarget = 1` and `scraped = 1`.
+     */
+    getScrapedHtmlPageCount(): Promise<number>;
+    /**
+     * Retrieves all `page_tags` rows for the given page id, parsed back into
+     * {@link TagRow} shape (with `categories` and `sources` JSON columns
+     * deserialised).
+     *
+     * Read-side counterpart to `#insertTags`.
+     * @param pageId
+     */
+    getTagsOfPage(pageId: number): Promise<TagRow[]>;
+    /**
+     * Records a crawler-level (`error` channel) failure into `crawl_errors`.
+     *
+     * Unlike {@link insertPageError} this is not tied to a scraped page: `url`
+     * may be an external link that never became a page row, or `null` for a
+     * process-level error. The cause is intentionally not stored — it is derived
+     * on read so that older archives (which only have `error.log`) and freshly
+     * captured rows classify identically.
+     * @param url - The URL the error is about, or `null` for a process-level error.
+     * @param message - The error message (one line is enough for classification).
+     * @param isExternal - Whether the URL is external to the crawl scope.
+     */
+    insertCrawlError(url: string | null, message: string, isExternal?: boolean): Promise<void>;
+    /**
+     * Records a partial scrape failure against the page identified by `url`.
+     *
+     * The page row is resolved (or inserted as a stub) via
+     * {@link Database.#getIdByUrl} so the error can be recorded even before
+     * `setPage` has run — useful when the failure fires during scraping
+     * (e.g. mid-`scrapeStart`) and the orchestrator enqueues this write
+     * before the success write for the same URL.
+     *
+     * A single page can have multiple `page_errors` rows (e.g. both
+     * `desktop-compact` and `mobile-small` viewports failing).
+     * @param url - URL of the page being scraped.
+     * @param phase - Scrape phase name (typically `'retryExhausted'`).
+     * @param message - Human-readable failure message.
+     * @param isExternal - Whether the URL is external. Defaults to `false`.
+     */
+    insertPageError(url: string, phase: string, message: string, isExternal?: boolean): Promise<void>;
     /**
      * Inserts a sub-resource into the `resources` table.
      * Ignores duplicate URLs (uses `ON CONFLICT IGNORE`).
+     *
+     * The `source` provenance label is written ONLY on insert; an
+     * `ON CONFLICT IGNORE` collision leaves an existing row's source untouched
+     * (this is what makes a second `crawl --inventory` non-destructive — see
+     * the inventory plan).
      * @param resource - The resource data to insert.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      */
-    insertResource(resource: Resource): Promise<void>;
+    insertResource(resource: Resource, source?: PageSource): Promise<void>;
     /**
      * Inserts a referrer relationship between a resource and a page into the
      * `resources-referrers` table. Silently skips if the resource is not found.
@@ -155,6 +276,25 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageUrl - The URL of the page that references the resource.
      */
     insertResourceReferrers(src: string, pageUrl: string): Promise<void>;
+    /**
+     * Records a redirect edge (source → destination) **without** re-storing the
+     * destination's content.
+     *
+     * The crawler renders a many-to-one redirect destination exactly once. For
+     * every subsequent source URL that redirects to that already-rendered
+     * destination, it calls this instead of {@link updatePage} (#73). Routing a
+     * content-less HEAD result through `updatePage` would funnel it into
+     * `#insertPage` and overwrite the destination's good title / meta with empty
+     * values, so the dedicated edge-only path is required.
+     *
+     * The destination row is resolved (created on demand if a concurrent in-flight
+     * render has not committed it yet) so the edge always points at a valid id;
+     * the single render fills in the destination's content under that same id.
+     * The destination's existing anchors / images are never touched here.
+     * @param page - HEAD-resolved page data carrying the redirect chain. Its
+     *   `anchorList` / `imageList` are ignored (a redirect source owns no content).
+     */
+    recordRedirect(page: PageData): Promise<void>;
     /**
      * Promote previously-external pages whose URL falls under any of the new scope
      * entries back to a "needs scraping" state so that the next crawl picks them up
@@ -174,6 +314,35 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns The URLs of the pages that were promoted.
      */
     repromoteExternalPages(scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<string[]>;
+    /**
+     * Reset previously-attempted pages that ended in a recoverable failure so a
+     * follow-up crawl can re-fetch them from scratch.
+     *
+     * A page qualifies as a recoverable failure when it was already scraped
+     * (`scraped = 1`), is not a redirect source (`redirectDestId IS NULL`), was
+     * not intentionally skipped (`isSkipped` is not `1`), and one of the
+     * following holds:
+     *
+     * - `status = -1` — the sentinel a hard scrape failure (network error,
+     *   timeout, browser crash) is recorded with (see `handle-scrape-error.ts`);
+     * - `status IS NULL` — no status was ever stored for the row;
+     * - `contentType IS NULL` — the content type could not be determined;
+     * - `status` is in the `5xx` range — a (frequently transient) server error.
+     *
+     * Definitive `4xx` responses are intentionally excluded: re-fetching a 404
+     * almost always yields the same answer. Matching rows — internal and
+     * external alike — are demoted back to pending (`scraped = 0`) and have their
+     * stale scrape metadata cleared. The page row itself is kept (id preserved)
+     * so existing `anchors.hrefId` referrers stay valid, and `isExternal` is left
+     * untouched so the next pass re-classifies each page from the crawl scope.
+     * Related `anchors`, `images`, `resources-referrers`, and `page_errors` rows
+     * are deleted so the re-scrape can re-insert fresh data without duplicates.
+     *
+     * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
+     * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+     * @returns The URLs of the pages that were reset to pending.
+     */
+    resetFailedPages(): Promise<string[]>;
     /**
      * Stores the crawl configuration in the `info` table.
      * Only fields in {@link INFO_COLUMN_ALLOWLIST} are forwarded — any extra
@@ -212,27 +381,42 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     updateConfig(patch: Partial<Config>): Promise<void>;
     /**
      * Inserts or updates a crawled page in the database, including its redirect chain,
-     * anchors, and images. Optionally creates an HTML snapshot file path entry.
+     * anchors, images, and (when `writeHtml`) its compressed HTML snapshot BLOB.
      *
      * Self-redirects (where the source URL equals the destination URL after normalization)
      * are skipped to avoid marking a page as redirected to itself — a situation caused by
      * authentication challenges (e.g. Basic Auth 302) that would otherwise exclude the page
      * from reports via the `whereNull('redirectDestId')` filter.
      * @param page - The page data to store.
-     * @param snapshotDir - The directory for saving HTML snapshots, or null to skip snapshots.
+     * @param writeHtml - When `true`, this call is allowed to insert (or clear)
+     *   the page's HTML blob. `setExternalPage` passes `false` because external
+     *   metadata-only scrapes never carry HTML and must not perturb an already
+     *   stored body.
      * @param isTarget - Whether this page is a crawl target.
-     * @returns An object with the optional `html` snapshot file path and the page's database `pageId`.
+     * @param source - Provenance label written ONLY when the row is freshly
+     *   inserted. Existing rows keep their original `source` (this is why a
+     *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
+     *   that was discovered earlier).
+     * @returns The database `pageId` of the inserted/updated row.
      */
-    updatePage(page: PageData, snapshotDir: string | null, isTarget: boolean): Promise<{
-        html?: string | undefined;
-        pageId: number;
-    }>;
+    updatePage(page: PageData, writeHtml: boolean, isTarget: boolean, source?: PageSource): Promise<number>;
     /**
      * Creates and initializes a new Database instance.
-     * Creates the parent directory for the database file if needed,
-     * establishes the connection, and initializes tables if they do not exist.
-     * @param options - Database connection options (working directory + SQLite file path).
+     *
+     * **Writer mode (default)**: creates the parent directory for the
+     * database file if needed, establishes the connection, and initializes
+     * the schema + migrations.
+     *
+     * **Read-only mode** (`options.readOnly`): refuses to resurrect a
+     * missing parent directory or db file — throws if either is absent at
+     * the time of the call. Skips schema init and migrations entirely so
+     * the user's tmpDir is never modified. Required by viewer / MCP
+     * stub-mode opens, where a TOCTOU window between classification and
+     * `connect()` could otherwise leave behind a phantom empty tmpDir.
+     * @param options - Database connection options.
      * @returns A fully initialized Database instance.
+     * @throws {Error} In read-only mode, if the parent directory or db
+     *   file does not exist when `connect()` runs.
      */
     static connect(options: DatabaseOption): Promise<Database>;
 }