@nitpicker/crawler 0.11.0 → 0.12.0

package/lib/archive/database.d.ts

@@ -1,5 +1,5 @@
 import type { JsonLdRow, TagRow } from './meta/types.js';
-import type { Config, DatabaseOption, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, PageFilter, PageSource } from './types.js';
+import type { Config, DatabaseOption, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, InventoryRunMeta, PageFilter, PageSource } from './types.js';
 import type { PageData, Resource } from '../utils/types/types.js';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import type { Knex } from 'knex';
@@ -7,12 +7,20 @@ import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-awa
 /**
  * Low-level database abstraction layer for the archive's SQLite database.
  *
- * Public methods that perform database queries use the `@retryable`
- * decorator for automatic retry on transient failures, and `@ErrorEmitter`
- * to propagate errors as events. The set of tables this layer manages is
+ * Public methods that perform database queries use the `emitErrorAndRetry`
+ * HOF for automatic retry on transient failures combined with error-event
+ * propagation, or `emitError` when retry is not appropriate. The set of
+ * tables this layer manages is
  * defined by `init-schema.ts` (the source of truth — query that file for
  * the canonical list).
  *
+ * **Label sync caveat**: each `emitError` / `emitErrorAndRetry` call passes
+ * the method name as a string literal (e.g. `'Database.getAnchorsOnPage'`).
+ * TypeScript cannot check that the string matches the enclosing method's
+ * real name — the two-way sync is manual. Renaming a method here **must**
+ * update the literal string too, otherwise debug logs and `RetryTimeoutError`
+ * messages will silently report the old name.
+ *
  * Use the static {@link Database.connect} factory method to create instances.
  * The constructor is private.
  */
@@ -58,7 +66,67 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     getConfig(): Promise<Config>;
     /**
      * Retrieves the current crawling state by listing scraped and pending URLs.
-     * @returns An object with `scraped` (completed URLs) and `pending` (remaining URLs) arrays.
+     *
+     * `scraped` is straightforward: every page row whose `scraped` flag is `1`
+     * — that is, every URL the crawl reached a terminal state on, including
+     * setSkippedPage / setExternalPage / outright setPage success or failure.
+     *
+     * `pending` is intentionally STRICT — not "every `scraped = 0` row".
+     * Three filters apply:
+     *
+     * 1. `scraped = 0` — work still incomplete.
+     * 2. `isExternal = 0` — only in-scope work. External URLs go through a
+     *    HEAD-only path that always lands on `scraped = 1` (either setPage or
+     *    setExternalPage). A row with `isExternal = 1 AND scraped = 0` is
+     *    therefore a data anomaly, and resume / inventory / append have no
+     *    business retrying it on the next session.
+     * 3. `EXISTS (anchor with hrefId = pages.id) OR source != 'crawled'` —
+     *    the row was either discovered as an anchor destination during a
+     *    previous scrape OR was explicitly tagged with a non-default
+     *    source label (`'inventory-seed'`, `'inventory-discovered'`, …).
+     *    Both halves of the OR represent "deliberately enqueued, expected
+     *    to be processed", which is exactly what `resume` should pick up.
+     *
+     *    The orphan filter targets the **predicted-discard leak** in
+     *    `crawler.ts` where `shouldDiscardPredicted` returns true but no
+     *    `emit('skip')` follows. Such placeholders are inserted with the
+     *    DB DEFAULT `source = 'crawled'` (no caller explicitly labels
+     *    them) AND have no anchor referrer (predicted URLs are
+     *    synthesised from pagination patterns, never anchored from a
+     *    rendered page) — both halves of the OR are therefore false and
+     *    the leak is excluded.
+     *
+     *    The `source != 'crawled'` clause specifically saves the
+     *    `--inventory` × `--retry-failed` interaction: an inventory-seed
+     *    URL came from the operator's URL list (no anchor referrer) and
+     *    `resetFailedPages` puts it back at `scraped = 0`. Without this
+     *    clause those legitimate retries would be dropped on resume.
+     *
+     * The defensive shape is on purpose: the data source can drift into
+     * anomalous states under interruption, but the reader must never throw
+     * or feed garbage back into the dealer. A real in-scope URL that was
+     * truly interrupted mid-crawl will always have at least one anchor
+     * referrer (otherwise the dealer would not have queued it), so the
+     * strict filter loses no legitimate pending work.
+     *
+     * Seeds passed directly to `Crawler.start()` are NOT in the strict
+     * pending set when they were never picked by the dealer — they have no
+     * DB row at all in that case (`linkList.add` is purely in-memory until
+     * `setPage` runs). A Ctrl-C between dealer pick and `setPage` likewise
+     * leaves no row to recover. Recovery of un-picked seeds is the
+     * responsibility of the caller (e.g. re-running `--inventory ./list.txt`
+     * with the same URL list).
+     *
+     * The query uses an explicit `p` alias on the `pages` table so the
+     * correlated `EXISTS` subquery can join via `whereRaw('anchors.hrefId =
+     * p.id')`. A future refactor that renames the alias must update both
+     * sites — the raw string in the subquery cannot be grep-resolved
+     * automatically. Read-only / stub viewer connections never call this
+     * method (they do not need to know about pending state), so the EXISTS
+     * shape is safe to use without the `migrate*` guards that other writer
+     * methods carry.
+     * @returns An object with `scraped` (completed URLs) and `pending` (the
+     *   strict set of in-scope, anchor-referenced, unfinished URLs).
      */
     getCrawlingState(): Promise<{
         scraped: string[];
@@ -137,6 +205,27 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns An array of raw {@link DB_Page} rows.
      */
     getPages(filter?: PageFilter, offset?: number, limit?: number): Promise<DB_Page[]>;
+    /**
+     * Look up the `source` column of a single page by its URL key. Used by
+     * the orchestrator's `PageSourceLookup` injection so the Crawler can
+     * resolve a parent page's lineage on `--resume` / `--retry-failed`
+     * sessions, where the in-memory `inventoryMode` is no longer
+     * available but the DB still remembers what label was last persisted.
+     *
+     * Returns `undefined` when the URL has no `pages` row (e.g. a brand-new
+     * URL that has not been seen yet) so the caller can fall through to
+     * its default behaviour without distinguishing "row absent" from "row
+     * present with NULL source" — the schema's `NOT NULL DEFAULT 'crawled'`
+     * makes a NULL value impossible in practice.
+     *
+     * Read-only — no transaction, single PK-equivalent lookup on
+     * `pages.url` (a UNIQUE column), so the cost is constant per call. The
+     * Crawler calls this at most once per page render, NOT per
+     * sub-resource, so the N+1 risk does not apply.
+     * @param url - URL key in `url.withoutHashAndAuth` form.
+     * @returns The recorded `source`, or `undefined` when no row exists.
+     */
+    getPageSourceByUrl(url: string): Promise<PageSource | undefined>;
     /**
      * Retrieves pages along with their related redirect, anchor, and referrer data.
      * Results are ordered by the natural URL sort order. Only non-redirected pages are returned.
@@ -183,7 +272,7 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * `href` while callers may only know the hash-stripped form; the first match
      * wins.
      *
-     * Deliberately NOT decorated with `@ErrorEmitter`: the only caller (the
+     * Deliberately NOT wrapped with `emitError`/`emitErrorAndRetry`: the only caller (the
      * crawler's resource-reuse hook) has a full fallback (the HEAD pre-flight),
      * so a read failure here must not surface as a database `error` event —
      * the orchestrator aborts the whole crawl on that event, which is the
@@ -240,6 +329,58 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param isExternal - Whether the URL is external to the crawl scope.
      */
     insertCrawlError(url: string | null, message: string, isExternal?: boolean): Promise<void>;
+    /**
+     * Pre-insert inventory non-HTML URLs into `resources` as placeholder rows
+     * with `source = 'inventory-seed'` and all metadata columns NULL — the
+     * non-HTML counterpart of {@link Database.insertInventorySeeds}. Used by
+     * `CrawlerOrchestrator.inventory` so the ingestion phase commits all of
+     * its non-HTML URLs in one chunked round-trip per 500 instead of N
+     * sequential `insertResource` awaits. On a 50k-URL inventory list the
+     * old per-URL loop spent minutes inside the `.bak`-protected window;
+     * the bulk path finishes in seconds.
+     *
+     * Idempotent: `onConflict('url').ignore()` leaves existing rows untouched
+     * (the orchestrator's `getExistingResourceUrls` filter is what keeps a
+     * crawled-lineage `resources` row from being downgraded to the
+     * inventory label here).
+     *
+     * Chunked at 500 to stay well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER`
+     * (default 999) — every row binds the URL plus the `responseHeaders`
+     * JSON null, so the per-chunk bound budget is well within limits.
+     * @param urls - URL strings (already in `withoutHashAndAuth` form).
+     */
+    insertInventoryResources(urls: readonly string[]): Promise<void>;
+    /**
+     * Pre-insert inventory HTML seeds into `pages` as `scraped = 0`,
+     * `source = 'inventory-seed'` placeholders so the URL's existence in the
+     * archive is **durable before the scrape phase starts**.
+     *
+     * Why this is the linchpin of `--inventory` Ctrl+C tolerance: HTML seeds
+     * used to live only in the Crawler's in-memory `LinkList` until the
+     * dealer eventually called `setPage`. A Ctrl+C / crash before that point
+     * lost the seed without trace, and `--resume` could not recover it
+     * because `getCrawlingState`'s strict pending set requires a `pages` row.
+     * Pre-inserting fills exactly that gap: the strict pending set picks
+     * these rows up via its `OR p.source != 'crawled'` clause, so
+     * `--resume` after an interrupted inventory pass picks every seed back
+     * up. See {@link Database.getCrawlingState} for the strict-set rationale.
+     *
+     * Idempotent: `onConflict('url').ignore()` keeps existing rows intact.
+     * The {@link Database.#getIdByUrl} crawled-wins downgrade still fires
+     * later when a crawled-lineage anchor reaches one of these seeds —
+     * that's the right behaviour (a seed that turned out to be reachable
+     * is not an orphan and should not retain the inventory label).
+     *
+     * Chunked into 500-URL batches so SQLite's bound-parameter limit
+     * (`SQLITE_MAX_VARIABLE_NUMBER`, default 999) cannot be hit even on a
+     * tens-of-thousands inventory list.
+     *
+     * Called by {@link CrawlerOrchestrator.inventory} during the
+     * `.bak`-protected ingestion phase, so any failure here aborts the run
+     * and restores from backup — the operator reruns from scratch.
+     * @param urls - URL strings already in `withoutHashAndAuth` form.
+     */
+    insertInventorySeeds(urls: readonly string[]): Promise<void>;
     /**
      * Records a partial scrape failure against the page identified by `url`.
      *
@@ -276,6 +417,44 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageUrl - The URL of the page that references the resource.
      */
     insertResourceReferrers(src: string, pageUrl: string): Promise<void>;
+    /**
+     * Hostnames whose `crawl_errors` history is consistently DNS failures and
+     * for which no recent 2xx-3xx page or resource is recorded — i.e. hosts
+     * the previous crawl already proved unreachable. Returned in lower-cased
+     * form. Used by `CrawlerOrchestrator.#preloadDnsBurnedHostCache` so the
+     * next session short-circuits HEAD pre-flight on these hosts.
+     *
+     * Implementation: a coarse `LIKE` filter over `crawl_errors.message`
+     * narrows the row set, then `classifyErrorKind` confirms `'dns'` in JS
+     * (the regex is the single truth source — DB-side filters never narrow
+     * it). Exclusion bags are built from a single `pages` and a single
+     * `resources` scan: any host with a 2xx-3xx page, a 2xx-3xx resource, or
+     * a `pages.lastCrawledAt` newer than its latest DNS error is dropped
+     * (the host probably recovered between the failure and the last crawl).
+     *
+     * Returns `[]` on legacy archives that pre-date the `crawl_errors`
+     * table — the `hasTable` guard keeps the call non-destructive.
+     * @returns Lower-cased hostnames safe to short-circuit.
+     */
+    listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Appends one row to the `inventory_runs` audit log.
+     *
+     * Called by {@link CrawlerOrchestrator.inventory} on every successful
+     * `--inventory <list>` invocation so the archive carries a durable
+     * record of which deploy list was applied when and at what scale —
+     * the operational question "did we apply last month's list" the
+     * archive itself can answer without consulting external bookkeeping.
+     *
+     * Append-only at Phase 1. There is intentionally no UPDATE path and
+     * no UNIQUE constraint on `source_file_sha256`; two applies of the
+     * same list each get their own row, and `Phase 3 --refresh` is where
+     * dedupe / pre-flight against the hash will land. Field-level NULL
+     * semantics live on {@link InventoryRunMeta}.
+     * @param meta - The run metadata to record. Only `ran_at` is required.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    recordInventoryRun(meta: InventoryRunMeta): Promise<number>;
     /**
      * Records a redirect edge (source → destination) **without** re-storing the
      * destination's content.
@@ -293,8 +472,17 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * The destination's existing anchors / images are never touched here.
      * @param page - HEAD-resolved page data carrying the redirect chain. Its
      *   `anchorList` / `imageList` are ignored (a redirect source owns no content).
-     */
-    recordRedirect(page: PageData): Promise<void>;
+     * @param source - Inventory provenance forwarded by the orchestrator
+     *   (`Archive.setRedirect` → here) for the redirect-edge fast path. Used
+     *   as the fallback when the originating URL's row does NOT yet exist in
+     *   the archive (`#73` convergence on first sight, js-redirect rescue
+     *   before any prior write). When the originating row already exists
+     *   (e.g. anchor-lineage INSERT from a prior pass), its stored `source`
+     *   takes precedence so transitive lineage is preserved across resume /
+     *   retry-failed sessions. `undefined` keeps the DB DEFAULT `'crawled'`
+     *   on a brand-new destination row.
+     */
+    recordRedirect(page: PageData, source?: PageSource): Promise<void>;
     /**
      * Promote previously-external pages whose URL falls under any of the new scope
      * entries back to a "needs scraping" state so that the next crawl picks them up
@@ -330,13 +518,26 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * - `status` is in the `5xx` range — a (frequently transient) server error.
      *
      * Definitive `4xx` responses are intentionally excluded: re-fetching a 404
-     * almost always yields the same answer. Matching rows — internal and
-     * external alike — are demoted back to pending (`scraped = 0`) and have their
-     * stale scrape metadata cleared. The page row itself is kept (id preserved)
-     * so existing `anchors.hrefId` referrers stay valid, and `isExternal` is left
-     * untouched so the next pass re-classifies each page from the crawl scope.
-     * Related `anchors`, `images`, `resources-referrers`, and `page_errors` rows
-     * are deleted so the re-scrape can re-insert fresh data without duplicates.
+     * almost always yields the same answer.
+     *
+     * A second exclusion runs in JS after the SQL candidate scan: any page whose
+     * latest recorded `page_errors` / `crawl_errors` message classifies into a
+     * permanent {@link PERMANENT_ERROR_KINDS} kind (dns / tls / client-blocked /
+     * parse-error / connection-refused) is left as-is rather than reset to
+     * pending. Without this filter, `--retry-failed` never converges: NXDOMAIN
+     * hosts, expired-cert hosts, and `ERR_BLOCKED_BY_CLIENT` ad pixels would be
+     * reset every iteration, re-attempted, fail identically, and rejoin the
+     * candidate pool for the next iteration. The exclusion keeps the retry
+     * target shrinking across `--retry-failed` passes by leaving deterministic
+     * dead-ends alone.
+     *
+     * Matching rows — internal and external alike — are demoted back to pending
+     * (`scraped = 0`) and have their stale scrape metadata cleared. The page row
+     * itself is kept (id preserved) so existing `anchors.hrefId` referrers stay
+     * valid, and `isExternal` is left untouched so the next pass re-classifies
+     * each page from the crawl scope. Related `anchors`, `images`,
+     * `resources-referrers`, and `page_errors` rows are deleted so the re-scrape
+     * can re-insert fresh data without duplicates.
      *
      * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
      * `SQLITE_LIMIT_VARIABLE_NUMBER`.