@nitpicker/crawler 0.8.0 → 0.11.0

package/lib/crawler/types.d.ts

@@ -1,6 +1,24 @@
+import type { PageSource } from '../archive/types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
-import type { ChangePhaseEvent } from '@d-zero/beholder';
+import type { ChangePhaseEvent, ScrapeResult } from '@d-zero/beholder';
 import type { ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Result of resolving a URL that redirects to a destination already rendered
+ * during this crawl (#73). The crawler records the redirect edge only and skips
+ * launching the browser, so the destination is never re-rendered.
+ */
+export interface RedirectEdgeResult {
+    /** Discriminant marking this as a redirect-edge-only outcome. */
+    type: 'redirect-edge';
+    /** HEAD-resolved page data carrying the redirect chain (source → destination). */
+    pageData: PageData;
+}
+/**
+ * The outcome of {@link Crawler.#scrapePage}: either a normal scrape result from
+ * the browser/HEAD pipeline, or a {@link RedirectEdgeResult} when the URL's
+ * redirect destination was already rendered and only the edge needs recording.
+ */
+export type ScrapeOutcome = ScrapeResult | RedirectEdgeResult;
 /**
  * Configuration options that control crawler behavior.
  *
@@ -42,6 +60,60 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
     userAgent: string;
     /** Whether to ignore robots.txt restrictions. */
     ignoreRobots: boolean;
+    /**
+     * Lookup for previously captured sub-resources, or `null` to disable the
+     * resource-reuse optimization. See {@link ResourceLookup}.
+     */
+    lookupResource: ResourceLookup | null;
+    /**
+     * When non-null, the crawler is running in `--inventory` mode. New page
+     * rows whose URL matches `seedUrls` are labelled `'inventory-seed'`;
+     * every other newly-inserted page or sub-resource is labelled
+     * `'inventory-discovered'`. When `null`, no source label is emitted —
+     * the DB DEFAULT `'crawled'` applies.
+     */
+    inventoryMode: InventoryMode | null;
+}
+/**
+ * Inventory-mode runtime configuration. Passed from
+ * `CrawlerOrchestrator.inventory` into the Crawler so the emit pipeline can
+ * label new rows with the correct {@link PageSource}.
+ */
+export interface InventoryMode {
+    /**
+     * URLs explicitly listed in the user-supplied URL file, keyed by their
+     * `withoutHashAndAuth` form (so credentials in the URL don't break the
+     * match). Membership decides `inventory-seed` vs `inventory-discovered`
+     * for HTML pages.
+     */
+    seedUrls: ReadonlySet<string>;
+}
+/**
+ * Looks up a previously captured sub-resource by URL.
+ *
+ * Injected by the orchestrator so that the crawler can reuse network data
+ * recorded during page rendering instead of issuing a redundant HEAD
+ * pre-flight request. Implementations must serialize the read against any
+ * pending resource writes (e.g., via the orchestrator's WriteQueue).
+ * @param urls - URL candidates to match (e.g., with and without auth credentials).
+ * @returns The recorded resource data, or `null` when no row matches.
+ */
+export type ResourceLookup = (urls: readonly string[]) => Promise<ResourceLookupResult | null>;
+/**
+ * Minimal sub-resource data needed to synthesize {@link PageData}
+ * without performing a network fetch.
+ */
+export interface ResourceLookupResult {
+    /** HTTP status code of the recorded response, or `null` if unknown. */
+    status: number | null;
+    /** HTTP status text of the recorded response, or `null` if unknown. */
+    statusText: string | null;
+    /** The Content-Type header value (media type only), or `null` if unknown. */
+    contentType: string | null;
+    /** The Content-Length header value in bytes, or `null` if unknown. */
+    contentLength: number | null;
+    /** Raw HTTP response headers, or `null` if unavailable. */
+    responseHeaders: Record<string, string | string[] | undefined> | null;
 }
 /**
  * Describes a detected pagination pattern between two consecutive URLs.
@@ -67,6 +139,12 @@ export interface CrawlerEventTypes {
     page: {
         /** The scraped page data including HTML, metadata, anchors, and images. */
         result: PageData;
+        /**
+         * Inventory provenance to write to `pages.source` when this row is new.
+         * `undefined` means the DB default (`'crawled'`) applies, which is the
+         * common case outside `crawl --inventory`. See {@link PageSource}.
+         */
+        source?: PageSource;
     };
     /**
      * Emitted when an external page (outside the crawl scope) has been scraped.
@@ -74,6 +152,8 @@ export interface CrawlerEventTypes {
     externalPage: {
         /** The scraped page data for the external page. */
         result: PageData;
+        /** Inventory provenance for new rows — see {@link CrawlerEventTypes.page.source}. */
+        source?: PageSource;
     };
     /**
      * Emitted when a URL is skipped due to exclusion rules, robots.txt restrictions,
@@ -93,6 +173,14 @@ export interface CrawlerEventTypes {
     response: {
         /** The captured resource data including URL, status, content type, and headers. */
         resource: Resource;
+        /**
+         * Inventory provenance to write to `resources.source` when this row is new.
+         * Sub-resources discovered while puppeteer renders an inventory-seed
+         * page are always `'inventory-discovered'` (a sub-resource is never
+         * itself a seed). `undefined` means the DB default (`'crawled'`)
+         * applies. See {@link PageSource}.
+         */
+        source?: PageSource;
     };
     /**
      * Emitted to record the relationship between a page and a resource it references.
@@ -116,4 +204,37 @@ export interface CrawlerEventTypes {
      * (e.g., scrapeStart, headRequest, openPage, success).
      */
     changePhase: ChangePhaseEvent;
+    /**
+     * Emitted when a secondary scrape step fails for a URL but the page itself
+     * is otherwise scraped successfully (e.g. a viewport switch in
+     * `#fetchImages` detaches the frame and `@retryable` gives up). The
+     * orchestrator persists these as `page_errors` rows so the failure is
+     * visible in the archive instead of being lost to stdout logs.
+     *
+     * For ordering, this event is always emitted AFTER `page` / `externalPage`
+     * for the same URL, so the orchestrator's WriteQueue serialises the
+     * `pages` upsert before the `page_errors` insert and the FK resolution
+     * via URL succeeds.
+     */
+    pageError: {
+        /** URL of the affected page. */
+        url: string;
+        /** Scrape phase name (typically `'retryExhausted'`). */
+        phase: string;
+        /** Human-readable failure message. */
+        message: string;
+        /** Whether the URL is external to the crawl scope. */
+        isExternal: boolean;
+    };
+    /**
+     * Emitted when a URL redirects to a destination that has already been
+     * rendered during this crawl, so only the redirect edge is recorded and the
+     * destination is not re-rendered (#73). The orchestrator persists this via
+     * `Archive.setRedirect`, which writes the edge without overwriting the
+     * destination's content.
+     */
+    redirect: {
+        /** HEAD-resolved page data carrying the redirect chain (source → destination). */
+        result: PageData;
+    };
 }

package/lib/crawler-orchestrator.d.ts

@@ -1,4 +1,5 @@
 import type { Config } from './archive/types.js';
+import type { InventoryMode } from './crawler/types.js';
 import type { CrawlEvent } from './types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
@@ -38,6 +39,12 @@ interface CrawlConfig extends Config {
     userAgent: string;
     /** Whether to ignore robots.txt restrictions. */
     ignoreRobots: boolean;
+    /**
+     * Inventory-mode runtime configuration (see {@link InventoryMode}). Set
+     * by {@link CrawlerOrchestrator.inventory}; the default crawl path leaves
+     * this `null` so new rows are labelled `'crawled'` by the DB DEFAULT.
+     */
+    inventoryMode: InventoryMode | null;
 }
 /**
  * Callback invoked after the CrawlerOrchestrator instance is fully initialized
@@ -83,10 +90,16 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * when the crawl completes. Discovered pages, external pages, skipped pages,
      * and resources are forwarded to the archive for storage.
      * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
+     * @param opts - Optional crawl overrides.
+     * @param opts.recursive - Whether discovered URLs are followed. Defaults to
+     *   `!fromList` (recursive unless the archive was created from a URL list), so
+     *   existing callers keep their behaviour; the retry flow passes it explicitly.
      * @returns A promise that resolves when crawling is complete.
      * @throws {Error} If the URL list is empty.
      */
-    crawling(list: ExURL[]): Promise<void>;
+    crawling(list: ExURL[], opts?: {
+        recursive?: boolean;
+    }): Promise<void>;
     /**
      * Kill any zombie Chromium processes that were not properly cleaned up.
      *
@@ -146,6 +159,85 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @throws {Error} When `newUrls` is empty, the archive is in list mode, or it cannot be parsed.
      */
     static append(archivePath: string, newUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    /**
+     * Inventory mode: cross-reference a user-supplied URL list against an
+     * existing `.nitpicker` archive and import ONLY the URLs that are not yet
+     * tracked there. Designed to surface "orphan" landing pages that link
+     * graph traversal could not reach, and "unused" server-side files that
+     * no crawled page references — both of which the
+     * `listIsolatedPages` / `listUnusedResources` queries can then list.
+     *
+     * Flow:
+     *
+     * 1. Open the archive (writer mode, takes the archive lock).
+     * 2. Reject list-mode archives — they hold metadata-only rows that
+     *    inventory has no business touching.
+     * 3. Reject archives with unfinished `pending` URLs — those would inherit
+     *    the inventory `source` label by mistake. Operator must resume /
+     *    retry-failed first.
+     * 4. Parse the URL list. Anything outside the archived scope is warned
+     *    and skipped (inventory is per-server by design).
+     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     *    second (and N-th) inventory pass is a no-op for known rows — keeps
+     *    `'inventory-seed'` rows from being silently demoted.
+     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     *    from the backup.
+     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
+     *    in `resources` directly as `'inventory-seed'` (no browser launch).
+     * 8. If any HTML seeds exist, start a Crawler with
+     *    `inventoryMode = { seedUrls }` so the rendered page and every newly
+     *    discovered downstream link is labelled correctly. `resume` is fed
+     *    the existing `scraped` / `resources` sets so links into already-
+     *    crawled pages stop at the seen-gate without re-rendering.
+     * 9. Drop the backup on success; restore it on any throw.
+     *
+     * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
+     * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
+     * CLI dispatch enforces this; this method assumes the caller honoured
+     * the contract.
+     * @param archivePath - Absolute or cwd-relative path to the `.nitpicker` archive.
+     * @param inventoryUrls - Pre-read URL list (one URL per element).
+     * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
+     * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
+     * @returns The orchestrator instance after a successful inventory pass.
+     * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
+     */
+    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    /**
+     * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
+     *
+     * Opens the archive, resets every page whose previous attempt ended in a
+     * recoverable failure (missing status / content type, or a 5xx status — see
+     * {@link Archive.resetFailedPages}) back to pending, and resumes crawling.
+     * The archived crawl configuration is reused — scopes, excludes, keywords,
+     * user agent, etc. — so the retry honours the original crawl boundaries
+     * unless a field is explicitly overridden via `options`. The exception is
+     * `recursive`: it is taken from `options` (the CLI flag defaults it to
+     * `true`) rather than inherited from the archive, so a retry decides afresh
+     * whether to follow newly-discovered URLs regardless of how the original
+     * crawl was run.
+     *
+     * When `recursive` is enabled (the default), newly-discovered URLs from the
+     * re-fetched pages are followed and crawled from scratch; when disabled, only
+     * the failed pages themselves are re-fetched. The archived roots seed the
+     * crawl scope while the reset pages are picked up through the resumed pending
+     * set, so failed external pages stay external (metadata-only) instead of being
+     * promoted into scope, and a failed root is re-fetched in place.
+     *
+     * A `<archive>.bak` is created before any DB mutation and removed on success;
+     * if the crawl throws, the backup is restored to keep the original archive
+     * intact.
+     *
+     * List-mode archives (`info.fromList === true`) are rejected for the same
+     * reason as {@link CrawlerOrchestrator.append}: their pages are metadata-only.
+     * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
+     * @param options - Optional config overrides applied on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @returns The orchestrator instance after the retry crawl completes.
+     * @throws {Error} When the archive is in list mode or has no parseable roots.
+     */
+    static retryFailed(archivePath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
     /**
      * Resume a previously interrupted crawl from an existing archive file.
      *