@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/crawler/types.d.ts

@@ -1,6 +1,93 @@
+import type { PageSource } from '../archive/types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
-import type { ChangePhaseEvent } from '@d-zero/beholder';
+import type { ChangePhaseEvent, ScrapeResult } from '@d-zero/beholder';
 import type { ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Result of resolving a URL that redirects to a destination already rendered
+ * during this crawl (#73). The crawler records the redirect edge only and skips
+ * launching the browser, so the destination is never re-rendered.
+ */
+export interface RedirectEdgeResult {
+    /** Discriminant marking this as a redirect-edge-only outcome. */
+    type: 'redirect-edge';
+    /** HEAD-resolved page data carrying the redirect chain (source → destination). */
+    pageData: PageData;
+    /**
+     * Where the chain came from. The caller needs this to decide whether the
+     * URLs in `pageData.redirectPaths` are already-known (HTTP chain — every
+     * hop was followed by the browser/HEAD pre-flight and the destination is
+     * already rendered) or brand-new (JS redirect — only the source was
+     * processed, the destination came out of `page.url()` and has never been
+     * touched).
+     *
+     * - `'http-chain'` — Returned when the HEAD pre-flight resolved a real
+     *   3xx chain and the destination has already been claimed via
+     *   `#scrapedDestinations`. The crawler folds every URL in
+     *   `redirectPaths` into the link-list's done-set; the destination is not
+     *   re-enqueued because it is already in the archive.
+     * - `'js-redirect'` — Returned when `scraper.scrapeStart` threw because
+     *   `page.goto()` resolved to `null` (client-side
+     *   `window.location.replace()` / `<meta http-equiv="refresh">`) and
+     *   `page.url()` exposed a different destination. `redirectPaths`
+     *   contains exactly one URL: the JS-redirect target. The crawler MUST
+     *   enqueue that destination so it reaches the browser, and MUST NOT
+     *   fold it into the done-set (otherwise the dealer's `seen` rejects
+     *   the push and the destination is silently lost from the archive).
+     */
+    source: 'http-chain' | 'js-redirect';
+}
+/**
+ * The outcome of {@link Crawler.#scrapePage}: either a normal scrape result from
+ * the browser/HEAD pipeline, or a {@link RedirectEdgeResult} when the URL's
+ * redirect destination was already rendered and only the edge needs recording.
+ */
+export type ScrapeOutcome = ScrapeResult | RedirectEdgeResult;
+/**
+ * Internal envelope returned by {@link Crawler.#launchBrowserAndScrape} that
+ * augments beholder's {@link ScrapeResult} with the puppeteer-side
+ * post-navigation URL.
+ *
+ * **Why:** when `scraper.scrapeStart` throws because `page.goto()` resolved to
+ * `null` (the classic puppeteer symptom of a client-side
+ * `window.location.replace()` / meta-refresh firing mid-navigation), the only
+ * authoritative source for the URL the browser actually landed on is
+ * `page.url()` — neither the HEAD pre-flight nor the thrown error carries it.
+ * Capturing it here lets `#scrapePage` fold the source into a redirect edge
+ * instead of recording a hard `status = -1` that `--retry-failed` would chase
+ * forever (`Page.goto returned null` classifies as `protocol`, which is neither
+ * permanent nor a puppeteer-fallback kind — so the SQL filter resets it every
+ * pass and the next pass replays the same failure).
+ *
+ * `postNavigationUrl` is optional because:
+ * - successful / skipped outcomes do not need it (the success path already
+ *   exposes the final URL via `pageData.url` + `redirectPaths`);
+ * - capturing can itself fail when the underlying browser context is already
+ *   torn down (target closed, session killed) — we treat that as "no extra
+ *   information" and fall through to the existing error path.
+ */
+export type BrowserScrapeResult = ScrapeResult & {
+    /**
+     * URL puppeteer reports via `page.url()` *after* a thrown navigation.
+     *
+     * Semantically only meaningful when the parent result is `type: 'error'`
+     * — `#launchBrowserAndScrape` sets it from inside its catch arm, and the
+     * success / skipped paths never write to it. The field is typed as
+     * optional on the whole envelope rather than narrowed to the error
+     * variant because beholder's `ScrapeResult` is not a discriminated
+     * union (all variants share the same shape and disambiguate via
+     * `type`), so narrowing here would force a parallel ad-hoc union with
+     * no compile-time payoff. Consumers MUST therefore check
+     * `result.type === 'error'` before reading `postNavigationUrl` — and in
+     * practice the only consumer is the JS-redirect rescue, which does
+     * exactly that.
+     *
+     * Consumers should also confirm the URL is meaningful via
+     * {@link deriveJsRedirectTarget} — `about:blank`, identity values,
+     * case-only or trailing-slash variants are all filtered there, not
+     * here.
+     */
+    postNavigationUrl?: string;
+};
 /**
  * Configuration options that control crawler behavior.
  *
@@ -42,6 +129,89 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
     userAgent: string;
     /** Whether to ignore robots.txt restrictions. */
     ignoreRobots: boolean;
+    /**
+     * Lookup for previously captured sub-resources, or `null` to disable the
+     * resource-reuse optimization. See {@link ResourceLookup}.
+     */
+    lookupResource: ResourceLookup | null;
+    /**
+     * Lookup for an already-persisted page's `source` column, or `null` when
+     * lineage propagation across sessions is not required. See
+     * {@link PageSourceLookup}.
+     *
+     * Injected by the orchestrator so that `#scrapePage` can resolve the
+     * parent's source on `--resume` / `--retry-failed` paths, where
+     * `inventoryMode` is not persisted but the page's `source` column is.
+     * Without this, sub-resources captured during a re-render of an
+     * inventory-labelled page would fall back to the DB DEFAULT `'crawled'`
+     * and lose their `'inventory-discovered'` provenance.
+     */
+    lookupPageSource: PageSourceLookup | null;
+    /**
+     * When non-null, the crawler is running in `--inventory` mode. New page
+     * rows whose URL matches `seedUrls` are labelled `'inventory-seed'`;
+     * every other newly-inserted page or sub-resource is labelled
+     * `'inventory-discovered'`. When `null`, no source label is emitted —
+     * the DB DEFAULT `'crawled'` applies.
+     */
+    inventoryMode: InventoryMode | null;
+}
+/**
+ * Inventory-mode runtime configuration. Passed from
+ * `CrawlerOrchestrator.inventory` into the Crawler so the emit pipeline can
+ * label new rows with the correct {@link PageSource}.
+ */
+export interface InventoryMode {
+    /**
+     * URLs explicitly listed in the user-supplied URL file, keyed by their
+     * `withoutHashAndAuth` form (so credentials in the URL don't break the
+     * match). Membership decides `inventory-seed` vs `inventory-discovered`
+     * for HTML pages.
+     */
+    seedUrls: ReadonlySet<string>;
+}
+/**
+ * Looks up a previously captured sub-resource by URL.
+ *
+ * Injected by the orchestrator so that the crawler can reuse network data
+ * recorded during page rendering instead of issuing a redundant HEAD
+ * pre-flight request. Implementations must serialize the read against any
+ * pending resource writes (e.g., via the orchestrator's WriteQueue).
+ * @param urls - URL candidates to match (e.g., with and without auth credentials).
+ * @returns The recorded resource data, or `null` when no row matches.
+ */
+export type ResourceLookup = (urls: readonly string[]) => Promise<ResourceLookupResult | null>;
+/**
+ * Looks up the `source` column of a previously persisted page by URL.
+ *
+ * Returns `undefined` when no row matches (e.g. a freshly-discovered URL
+ * that has not been INSERTed yet) so the caller can fall through to its
+ * default behaviour.
+ *
+ * Used by {@link Crawler} during sub-resource lineage propagation:
+ * `#scrapePage` consults this once per page to resolve the parent's
+ * lineage when the in-memory `inventoryMode` is unavailable (i.e. on
+ * `--resume` / `--retry-failed` sessions where inventory state lives only
+ * in the DB).
+ * @param urlWithoutHashAndAuth - The URL key (`url.withoutHashAndAuth` form) to look up.
+ * @returns The recorded `source`, or `undefined` when no matching row exists.
+ */
+export type PageSourceLookup = (urlWithoutHashAndAuth: string) => Promise<PageSource | undefined>;
+/**
+ * Minimal sub-resource data needed to synthesize {@link PageData}
+ * without performing a network fetch.
+ */
+export interface ResourceLookupResult {
+    /** HTTP status code of the recorded response, or `null` if unknown. */
+    status: number | null;
+    /** HTTP status text of the recorded response, or `null` if unknown. */
+    statusText: string | null;
+    /** The Content-Type header value (media type only), or `null` if unknown. */
+    contentType: string | null;
+    /** The Content-Length header value in bytes, or `null` if unknown. */
+    contentLength: number | null;
+    /** Raw HTTP response headers, or `null` if unavailable. */
+    responseHeaders: Record<string, string | string[] | undefined> | null;
 }
 /**
  * Describes a detected pagination pattern between two consecutive URLs.
@@ -67,6 +237,12 @@ export interface CrawlerEventTypes {
     page: {
         /** The scraped page data including HTML, metadata, anchors, and images. */
         result: PageData;
+        /**
+         * Inventory provenance to write to `pages.source` when this row is new.
+         * `undefined` means the DB default (`'crawled'`) applies, which is the
+         * common case outside `crawl --inventory`. See {@link PageSource}.
+         */
+        source?: PageSource;
     };
     /**
      * Emitted when an external page (outside the crawl scope) has been scraped.
@@ -74,6 +250,8 @@ export interface CrawlerEventTypes {
     externalPage: {
         /** The scraped page data for the external page. */
         result: PageData;
+        /** Inventory provenance for new rows — see {@link CrawlerEventTypes.page.source}. */
+        source?: PageSource;
     };
     /**
      * Emitted when a URL is skipped due to exclusion rules, robots.txt restrictions,
@@ -93,6 +271,14 @@ export interface CrawlerEventTypes {
     response: {
         /** The captured resource data including URL, status, content type, and headers. */
         resource: Resource;
+        /**
+         * Inventory provenance to write to `resources.source` when this row is new.
+         * Sub-resources discovered while puppeteer renders an inventory-seed
+         * page are always `'inventory-discovered'` (a sub-resource is never
+         * itself a seed). `undefined` means the DB default (`'crawled'`)
+         * applies. See {@link PageSource}.
+         */
+        source?: PageSource;
     };
     /**
      * Emitted to record the relationship between a page and a resource it references.
@@ -116,4 +302,46 @@ export interface CrawlerEventTypes {
      * (e.g., scrapeStart, headRequest, openPage, success).
      */
     changePhase: ChangePhaseEvent;
+    /**
+     * Emitted when a secondary scrape step fails for a URL but the page itself
+     * is otherwise scraped successfully (e.g. a viewport switch in
+     * `#fetchImages` detaches the frame and `@retryable` gives up). The
+     * orchestrator persists these as `page_errors` rows so the failure is
+     * visible in the archive instead of being lost to stdout logs.
+     *
+     * For ordering, this event is always emitted AFTER `page` / `externalPage`
+     * for the same URL, so the orchestrator's WriteQueue serialises the
+     * `pages` upsert before the `page_errors` insert and the FK resolution
+     * via URL succeeds.
+     */
+    pageError: {
+        /** URL of the affected page. */
+        url: string;
+        /** Scrape phase name (typically `'retryExhausted'`). */
+        phase: string;
+        /** Human-readable failure message. */
+        message: string;
+        /** Whether the URL is external to the crawl scope. */
+        isExternal: boolean;
+    };
+    /**
+     * Emitted when a URL redirects to a destination that has already been
+     * rendered during this crawl, so only the redirect edge is recorded and the
+     * destination is not re-rendered (#73). The orchestrator persists this via
+     * `Archive.setRedirect`, which writes the edge without overwriting the
+     * destination's content.
+     */
+    redirect: {
+        /** HEAD-resolved page data carrying the redirect chain (source → destination). */
+        result: PageData;
+        /**
+         * Inventory provenance for the redirect-edge call. Forwarded by the
+         * orchestrator to `Archive.setRedirect` → `Database.recordRedirect`
+         * so brand-new destination rows INSERTed by the edge-only path pick
+         * up the inventory label (and propagate it to intermediates) when
+         * the originating chain is in the inventory chain. `undefined` keeps
+         * the DB DEFAULT `'crawled'`.
+         */
+        source: PageSource | undefined;
+    };
 }

package/lib/crawler-orchestrator.d.ts

@@ -1,4 +1,5 @@
 import type { Config } from './archive/types.js';
+import type { InventoryMode } from './crawler/types.js';
 import type { CrawlEvent } from './types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
@@ -38,6 +39,12 @@ interface CrawlConfig extends Config {
     userAgent: string;
     /** Whether to ignore robots.txt restrictions. */
     ignoreRobots: boolean;
+    /**
+     * Inventory-mode runtime configuration (see {@link InventoryMode}). Set
+     * by {@link CrawlerOrchestrator.inventory}; the default crawl path leaves
+     * this `null` so new rows are labelled `'crawled'` by the DB DEFAULT.
+     */
+    inventoryMode: InventoryMode | null;
 }
 /**
  * Callback invoked after the CrawlerOrchestrator instance is fully initialized
@@ -82,11 +89,17 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * Sets up event listeners on the crawler, starts crawling, and resolves
      * when the crawl completes. Discovered pages, external pages, skipped pages,
      * and resources are forwarded to the archive for storage.
-     * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
+     * @param list - The list of parsed URLs to crawl. May be empty when a resumed
+     *   session already has pending pages queued (for example `--retry-failed`).
+     * @param opts - Optional crawl overrides.
+     * @param opts.recursive - Whether discovered URLs are followed. Defaults to
+     *   `!fromList` (recursive unless the archive was created from a URL list), so
+     *   existing callers keep their behaviour; the retry flow passes it explicitly.
      * @returns A promise that resolves when crawling is complete.
-     * @throws {Error} If the URL list is empty.
      */
-    crawling(list: ExURL[]): Promise<void>;
+    crawling(list: ExURL[], opts?: {
+        recursive?: boolean;
+    }): Promise<void>;
     /**
      * Kill any zombie Chromium processes that were not properly cleaned up.
      *
@@ -146,6 +159,95 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @throws {Error} When `newUrls` is empty, the archive is in list mode, or it cannot be parsed.
      */
     static append(archivePath: string, newUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    /**
+     * Inventory mode: cross-reference a user-supplied URL list against an
+     * existing `.nitpicker` archive and import ONLY the URLs that are not yet
+     * tracked there. Designed to surface "orphan" landing pages that link
+     * graph traversal could not reach, and "unused" server-side files that
+     * no crawled page references — both of which the
+     * `listIsolatedPages` / `listUnusedResources` queries can then list.
+     *
+     * Flow:
+     *
+     * 1. Open the archive (writer mode, takes the archive lock).
+     * 2. Reject list-mode archives — they hold metadata-only rows that
+     *    inventory has no business touching.
+     * 3. Reject archives with unfinished `pending` URLs — those would inherit
+     *    the inventory `source` label by mistake. Operator must resume /
+     *    retry-failed first.
+     * 4. Parse the URL list. Anything outside the archived scope is warned
+     *    and skipped (inventory is per-server by design).
+     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     *    second (and N-th) inventory pass is a no-op for known rows — keeps
+     *    `'inventory-seed'` rows from being silently demoted.
+     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     *    from the backup.
+     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
+     *    in `resources` directly as `'inventory-seed'` (no browser launch).
+     * 8. If any HTML seeds exist, start a Crawler with
+     *    `inventoryMode = { seedUrls }` so the rendered page and every newly
+     *    discovered downstream link is labelled correctly. `resume` is fed
+     *    the existing `scraped` / `resources` sets so links into already-
+     *    crawled pages stop at the seen-gate without re-rendering.
+     * 9. Drop the backup on success; restore it on any throw.
+     *
+     * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
+     * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
+     * CLI dispatch enforces this; this method assumes the caller honoured
+     * the contract.
+     * @param archivePath - Absolute or cwd-relative path to the `.nitpicker` archive.
+     * @param inventoryUrls - Pre-read URL list (one URL per element).
+     * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
+     * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
+     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
+     *   source URL list. The orchestrator deliberately does NOT receive
+     *   the file path: the path is privacy-sensitive (leaks user-home /
+     *   OS structure when archives are shared) and we want it lifted off
+     *   this boundary so no future log line / breadcrumb / error message
+     *   inside the orchestrator can accidentally re-leak it. The CLI
+     *   computes the digest via `computeFileSha256(resolvedListFile)`
+     *   and passes it through here. Pass `null` for programmatic
+     *   callers that built `inventoryUrls` in-memory; the audit row's
+     *   `source_file_sha256` column will be `NULL`.
+     * @returns The orchestrator instance after a successful inventory pass.
+     * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
+     */
+    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, sourceFileSha256?: string | null): Promise<CrawlerOrchestrator>;
+    /**
+     * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
+     *
+     * Opens the archive, resets every page whose previous attempt ended in a
+     * recoverable failure (missing status / content type, or a 5xx status — see
+     * {@link Archive.resetFailedPages}) back to pending, and resumes crawling.
+     * The archived crawl configuration is reused — scopes, excludes, keywords,
+     * user agent, etc. — so the retry honours the original crawl boundaries
+     * unless a field is explicitly overridden via `options`. The exception is
+     * `recursive`: it is taken from `options` (the CLI flag defaults it to
+     * `true`) rather than inherited from the archive, so a retry decides afresh
+     * whether to follow newly-discovered URLs regardless of how the original
+     * crawl was run.
+     *
+     * When `recursive` is enabled (the default), newly-discovered URLs from the
+     * re-fetched pages are followed and crawled from scratch; when disabled, only
+     * the failed pages themselves are re-fetched. The archived roots seed the
+     * crawl scope while the reset pages are picked up through the resumed pending
+     * set, so failed external pages stay external (metadata-only) instead of being
+     * promoted into scope, and a failed root is re-fetched in place.
+     *
+     * A `<archive>.bak` is created before any DB mutation and removed on success;
+     * if the crawl throws, the backup is restored to keep the original archive
+     * intact.
+     *
+     * List-mode archives (`info.fromList === true`) are rejected for the same
+     * reason as {@link CrawlerOrchestrator.append}: their pages are metadata-only.
+     * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
+     * @param options - Optional config overrides applied on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @returns The orchestrator instance after the retry crawl completes.
+     * @throws {Error} When the archive is in list mode or has no parseable roots.
+     */
+    static retryFailed(archivePath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
     /**
      * Resume a previously interrupted crawl from an existing archive file.
      *