@nitpicker/crawler 0.11.0 → 0.12.0

package/lib/crawler/types.d.ts

@@ -12,6 +12,29 @@ export interface RedirectEdgeResult {
     type: 'redirect-edge';
     /** HEAD-resolved page data carrying the redirect chain (source → destination). */
     pageData: PageData;
+    /**
+     * Where the chain came from. The caller needs this to decide whether the
+     * URLs in `pageData.redirectPaths` are already-known (HTTP chain — every
+     * hop was followed by the browser/HEAD pre-flight and the destination is
+     * already rendered) or brand-new (JS redirect — only the source was
+     * processed, the destination came out of `page.url()` and has never been
+     * touched).
+     *
+     * - `'http-chain'` — Returned when the HEAD pre-flight resolved a real
+     *   3xx chain and the destination has already been claimed via
+     *   `#scrapedDestinations`. The crawler folds every URL in
+     *   `redirectPaths` into the link-list's done-set; the destination is not
+     *   re-enqueued because it is already in the archive.
+     * - `'js-redirect'` — Returned when `scraper.scrapeStart` threw because
+     *   `page.goto()` resolved to `null` (client-side
+     *   `window.location.replace()` / `<meta http-equiv="refresh">`) and
+     *   `page.url()` exposed a different destination. `redirectPaths`
+     *   contains exactly one URL: the JS-redirect target. The crawler MUST
+     *   enqueue that destination so it reaches the browser, and MUST NOT
+     *   fold it into the done-set (otherwise the dealer's `seen` rejects
+     *   the push and the destination is silently lost from the archive).
+     */
+    source: 'http-chain' | 'js-redirect';
 }
 /**
  * The outcome of {@link Crawler.#scrapePage}: either a normal scrape result from
@@ -19,6 +42,52 @@ export interface RedirectEdgeResult {
  * redirect destination was already rendered and only the edge needs recording.
  */
 export type ScrapeOutcome = ScrapeResult | RedirectEdgeResult;
+/**
+ * Internal envelope returned by {@link Crawler.#launchBrowserAndScrape} that
+ * augments beholder's {@link ScrapeResult} with the puppeteer-side
+ * post-navigation URL.
+ *
+ * **Why:** when `scraper.scrapeStart` throws because `page.goto()` resolved to
+ * `null` (the classic puppeteer symptom of a client-side
+ * `window.location.replace()` / meta-refresh firing mid-navigation), the only
+ * authoritative source for the URL the browser actually landed on is
+ * `page.url()` — neither the HEAD pre-flight nor the thrown error carries it.
+ * Capturing it here lets `#scrapePage` fold the source into a redirect edge
+ * instead of recording a hard `status = -1` that `--retry-failed` would chase
+ * forever (`Page.goto returned null` classifies as `protocol`, which is neither
+ * permanent nor a puppeteer-fallback kind — so the SQL filter resets it every
+ * pass and the next pass replays the same failure).
+ *
+ * `postNavigationUrl` is optional because:
+ * - successful / skipped outcomes do not need it (the success path already
+ *   exposes the final URL via `pageData.url` + `redirectPaths`);
+ * - capturing can itself fail when the underlying browser context is already
+ *   torn down (target closed, session killed) — we treat that as "no extra
+ *   information" and fall through to the existing error path.
+ */
+export type BrowserScrapeResult = ScrapeResult & {
+    /**
+     * URL puppeteer reports via `page.url()` *after* a thrown navigation.
+     *
+     * Semantically only meaningful when the parent result is `type: 'error'`
+     * — `#launchBrowserAndScrape` sets it from inside its catch arm, and the
+     * success / skipped paths never write to it. The field is typed as
+     * optional on the whole envelope rather than narrowed to the error
+     * variant because beholder's `ScrapeResult` is not a discriminated
+     * union (all variants share the same shape and disambiguate via
+     * `type`), so narrowing here would force a parallel ad-hoc union with
+     * no compile-time payoff. Consumers MUST therefore check
+     * `result.type === 'error'` before reading `postNavigationUrl` — and in
+     * practice the only consumer is the JS-redirect rescue, which does
+     * exactly that.
+     *
+     * Consumers should also confirm the URL is meaningful via
+     * {@link deriveJsRedirectTarget} — `about:blank`, identity values,
+     * case-only or trailing-slash variants are all filtered there, not
+     * here.
+     */
+    postNavigationUrl?: string;
+};
 /**
  * Configuration options that control crawler behavior.
  *
@@ -65,6 +134,19 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
      * resource-reuse optimization. See {@link ResourceLookup}.
      */
     lookupResource: ResourceLookup | null;
+    /**
+     * Lookup for an already-persisted page's `source` column, or `null` when
+     * lineage propagation across sessions is not required. See
+     * {@link PageSourceLookup}.
+     *
+     * Injected by the orchestrator so that `#scrapePage` can resolve the
+     * parent's source on `--resume` / `--retry-failed` paths, where
+     * `inventoryMode` is not persisted but the page's `source` column is.
+     * Without this, sub-resources captured during a re-render of an
+     * inventory-labelled page would fall back to the DB DEFAULT `'crawled'`
+     * and lose their `'inventory-discovered'` provenance.
+     */
+    lookupPageSource: PageSourceLookup | null;
     /**
      * When non-null, the crawler is running in `--inventory` mode. New page
      * rows whose URL matches `seedUrls` are labelled `'inventory-seed'`;
@@ -99,6 +181,22 @@ export interface InventoryMode {
  * @returns The recorded resource data, or `null` when no row matches.
  */
 export type ResourceLookup = (urls: readonly string[]) => Promise<ResourceLookupResult | null>;
+/**
+ * Looks up the `source` column of a previously persisted page by URL.
+ *
+ * Returns `undefined` when no row matches (e.g. a freshly-discovered URL
+ * that has not been INSERTed yet) so the caller can fall through to its
+ * default behaviour.
+ *
+ * Used by {@link Crawler} during sub-resource lineage propagation:
+ * `#scrapePage` consults this once per page to resolve the parent's
+ * lineage when the in-memory `inventoryMode` is unavailable (i.e. on
+ * `--resume` / `--retry-failed` sessions where inventory state lives only
+ * in the DB).
+ * @param urlWithoutHashAndAuth - The URL key (`url.withoutHashAndAuth` form) to look up.
+ * @returns The recorded `source`, or `undefined` when no matching row exists.
+ */
+export type PageSourceLookup = (urlWithoutHashAndAuth: string) => Promise<PageSource | undefined>;
 /**
  * Minimal sub-resource data needed to synthesize {@link PageData}
  * without performing a network fetch.
@@ -236,5 +334,14 @@ export interface CrawlerEventTypes {
     redirect: {
         /** HEAD-resolved page data carrying the redirect chain (source → destination). */
         result: PageData;
+        /**
+         * Inventory provenance for the redirect-edge call. Forwarded by the
+         * orchestrator to `Archive.setRedirect` → `Database.recordRedirect`
+         * so brand-new destination rows INSERTed by the edge-only path pick
+         * up the inventory label (and propagate it to intermediates) when
+         * the originating chain is in the inventory chain. `undefined` keeps
+         * the DB DEFAULT `'crawled'`.
+         */
+        source: PageSource | undefined;
     };
 }

package/lib/crawler-orchestrator.d.ts

@@ -89,13 +89,13 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * Sets up event listeners on the crawler, starts crawling, and resolves
      * when the crawl completes. Discovered pages, external pages, skipped pages,
      * and resources are forwarded to the archive for storage.
-     * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
+     * @param list - The list of parsed URLs to crawl. May be empty when a resumed
+     *   session already has pending pages queued (for example `--retry-failed`).
      * @param opts - Optional crawl overrides.
      * @param opts.recursive - Whether discovered URLs are followed. Defaults to
      *   `!fromList` (recursive unless the archive was created from a URL list), so
      *   existing callers keep their behaviour; the retry flow passes it explicitly.
      * @returns A promise that resolves when crawling is complete.
-     * @throws {Error} If the URL list is empty.
      */
     crawling(list: ExURL[], opts?: {
         recursive?: boolean;
@@ -200,10 +200,20 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param inventoryUrls - Pre-read URL list (one URL per element).
      * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
      * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
+     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
+     *   source URL list. The orchestrator deliberately does NOT receive
+     *   the file path: the path is privacy-sensitive (leaks user-home /
+     *   OS structure when archives are shared) and we want it lifted off
+     *   this boundary so no future log line / breadcrumb / error message
+     *   inside the orchestrator can accidentally re-leak it. The CLI
+     *   computes the digest via `computeFileSha256(resolvedListFile)`
+     *   and passes it through here. Pass `null` for programmatic
+     *   callers that built `inventoryUrls` in-memory; the audit row's
+     *   `source_file_sha256` column will be `NULL`.
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
      */
-    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, sourceFileSha256?: string | null): Promise<CrawlerOrchestrator>;
     /**
      * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
      *