@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/crawler/should-get-fallback-on-head-failure.js

@@ -0,0 +1,46 @@
+import { classifyErrorKind } from '../classify-error-kind.js';
+import NetTimeoutError from './net-timeout-error.js';
+/**
+ * Decide whether a HEAD pre-flight failure should trigger a GET retry and
+ * stay OUT of `destinationCache`.
+ *
+ * The two contracts are intentionally tied to one helper: anything we'll GET
+ * later because the HEAD answer might be wrong is also the same thing we
+ * must NOT freeze into the per-session cache (or the second attempt would
+ * hit the stale cached failure and skip the live retry that
+ * `Crawler.#sendHeadRequest`'s `HEAD_TIMEOUT_ESCALATION_MS` is supposed to
+ * pay for).
+ *
+ * The eligible kinds are:
+ *
+ * - **`NetTimeoutError`** — the HEAD pre-flight race fired without a
+ *   server response. The escalating retry can still succeed against a
+ *   slow-but-reachable host.
+ * - **`parse-error`** — `Parse Error` / `Expected HTTP/` / `Unexpected end
+ *   of stream`. Usually a middlebox rewriting / truncating the HEAD reply
+ *   the GET path traverses differently.
+ * - **`connection-reset`** — `ECONNRESET` / `ERR_CONNECTION_RESET` etc.
+ *   middlebox dropping the connection mid-response; a retry frequently
+ *   succeeds.
+ *
+ * Everything else — DNS, TLS, refused, blocked, plain timeout — is treated
+ * as a persistent within-session verdict and IS cached so repeated calls on
+ * the same host pay the network cost once.
+ * @param error - The `Error` raised by the HEAD attempt.
+ * @returns `true` when the error warrants a GET fallback AND a cache skip.
+ * @example
+ * ```ts
+ * shouldGetFallbackOnHeadFailure(new NetTimeoutError(url)); // true
+ * shouldGetFallbackOnHeadFailure(new Error('read ECONNRESET')); // true
+ * shouldGetFallbackOnHeadFailure(new Error('Parse Error')); // true
+ * shouldGetFallbackOnHeadFailure(new Error('getaddrinfo ENOTFOUND host')); // false
+ * shouldGetFallbackOnHeadFailure(new Error('ERR_CERT_DATE_INVALID')); // false
+ * ```
+ */
+export function shouldGetFallbackOnHeadFailure(error) {
+    if (error instanceof NetTimeoutError) {
+        return true;
+    }
+    const kind = classifyErrorKind(error.message);
+    return kind === 'parse-error' || kind === 'connection-reset';
+}

package/lib/crawler/types.d.ts

@@ -12,6 +12,29 @@ export interface RedirectEdgeResult {
     type: 'redirect-edge';
     /** HEAD-resolved page data carrying the redirect chain (source → destination). */
     pageData: PageData;
+    /**
+     * Where the chain came from. The caller needs this to decide whether the
+     * URLs in `pageData.redirectPaths` are already-known (HTTP chain — every
+     * hop was followed by the browser/HEAD pre-flight and the destination is
+     * already rendered) or brand-new (JS redirect — only the source was
+     * processed, the destination came out of `page.url()` and has never been
+     * touched).
+     *
+     * - `'http-chain'` — Returned when the HEAD pre-flight resolved a real
+     *   3xx chain and the destination has already been claimed via
+     *   `#scrapedDestinations`. The crawler folds every URL in
+     *   `redirectPaths` into the link-list's done-set; the destination is not
+     *   re-enqueued because it is already in the archive.
+     * - `'js-redirect'` — Returned when `scraper.scrapeStart` threw because
+     *   `page.goto()` resolved to `null` (client-side
+     *   `window.location.replace()` / `<meta http-equiv="refresh">`) and
+     *   `page.url()` exposed a different destination. `redirectPaths`
+     *   contains exactly one URL: the JS-redirect target. The crawler MUST
+     *   enqueue that destination so it reaches the browser, and MUST NOT
+     *   fold it into the done-set (otherwise the dealer's `seen` rejects
+     *   the push and the destination is silently lost from the archive).
+     */
+    source: 'http-chain' | 'js-redirect';
 }
 /**
  * The outcome of {@link Crawler.#scrapePage}: either a normal scrape result from
@@ -19,6 +42,52 @@ export interface RedirectEdgeResult {
  * redirect destination was already rendered and only the edge needs recording.
  */
 export type ScrapeOutcome = ScrapeResult | RedirectEdgeResult;
+/**
+ * Internal envelope returned by {@link Crawler.#launchBrowserAndScrape} that
+ * augments beholder's {@link ScrapeResult} with the puppeteer-side
+ * post-navigation URL.
+ *
+ * **Why:** when `scraper.scrapeStart` throws because `page.goto()` resolved to
+ * `null` (the classic puppeteer symptom of a client-side
+ * `window.location.replace()` / meta-refresh firing mid-navigation), the only
+ * authoritative source for the URL the browser actually landed on is
+ * `page.url()` — neither the HEAD pre-flight nor the thrown error carries it.
+ * Capturing it here lets `#scrapePage` fold the source into a redirect edge
+ * instead of recording a hard `status = -1` that `--retry-failed` would chase
+ * forever (`Page.goto returned null` classifies as `protocol`, which is neither
+ * permanent nor a puppeteer-fallback kind — so the SQL filter resets it every
+ * pass and the next pass replays the same failure).
+ *
+ * `postNavigationUrl` is optional because:
+ * - successful / skipped outcomes do not need it (the success path already
+ *   exposes the final URL via `pageData.url` + `redirectPaths`);
+ * - capturing can itself fail when the underlying browser context is already
+ *   torn down (target closed, session killed) — we treat that as "no extra
+ *   information" and fall through to the existing error path.
+ */
+export type BrowserScrapeResult = ScrapeResult & {
+    /**
+     * URL puppeteer reports via `page.url()` *after* a thrown navigation.
+     *
+     * Semantically only meaningful when the parent result is `type: 'error'`
+     * — `#launchBrowserAndScrape` sets it from inside its catch arm, and the
+     * success / skipped paths never write to it. The field is typed as
+     * optional on the whole envelope rather than narrowed to the error
+     * variant because beholder's `ScrapeResult` is not a discriminated
+     * union (all variants share the same shape and disambiguate via
+     * `type`), so narrowing here would force a parallel ad-hoc union with
+     * no compile-time payoff. Consumers MUST therefore check
+     * `result.type === 'error'` before reading `postNavigationUrl` — and in
+     * practice the only consumer is the JS-redirect rescue, which does
+     * exactly that.
+     *
+     * Consumers should also confirm the URL is meaningful via
+     * `deriveJsRedirectTarget` — `about:blank`, identity values,
+     * case-only or trailing-slash variants are all filtered there, not
+     * here.
+     */
+    postNavigationUrl?: string;
+};
 /**
  * Configuration options that control crawler behavior.
  *
@@ -65,6 +134,19 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
      * resource-reuse optimization. See {@link ResourceLookup}.
      */
     lookupResource: ResourceLookup | null;
+    /**
+     * Lookup for an already-persisted page's `source` column, or `null` when
+     * lineage propagation across sessions is not required. See
+     * {@link PageSourceLookup}.
+     *
+     * Injected by the orchestrator so that `#scrapePage` can resolve the
+     * parent's source on `--resume` / `--retry-failed` paths, where
+     * `inventoryMode` is not persisted but the page's `source` column is.
+     * Without this, sub-resources captured during a re-render of an
+     * inventory-labelled page would fall back to the DB DEFAULT `'crawled'`
+     * and lose their `'inventory-discovered'` provenance.
+     */
+    lookupPageSource: PageSourceLookup | null;
     /**
      * When non-null, the crawler is running in `--inventory` mode. New page
      * rows whose URL matches `seedUrls` are labelled `'inventory-seed'`;
@@ -99,6 +181,22 @@ export interface InventoryMode {
  * @returns The recorded resource data, or `null` when no row matches.
  */
 export type ResourceLookup = (urls: readonly string[]) => Promise<ResourceLookupResult | null>;
+/**
+ * Looks up the `source` column of a previously persisted page by URL.
+ *
+ * Returns `undefined` when no row matches (e.g. a freshly-discovered URL
+ * that has not been INSERTed yet) so the caller can fall through to its
+ * default behaviour.
+ *
+ * Used by `Crawler` during sub-resource lineage propagation:
+ * `#scrapePage` consults this once per page to resolve the parent's
+ * lineage when the in-memory `inventoryMode` is unavailable (i.e. on
+ * `--resume` / `--retry-failed` sessions where inventory state lives only
+ * in the DB).
+ * @param urlWithoutHashAndAuth - The URL key (`url.withoutHashAndAuth` form) to look up.
+ * @returns The recorded `source`, or `undefined` when no matching row exists.
+ */
+export type PageSourceLookup = (urlWithoutHashAndAuth: string) => Promise<PageSource | undefined>;
 /**
  * Minimal sub-resource data needed to synthesize {@link PageData}
  * without performing a network fetch.
@@ -152,7 +250,7 @@ export interface CrawlerEventTypes {
     externalPage: {
         /** The scraped page data for the external page. */
         result: PageData;
-        /** Inventory provenance for new rows — see {@link CrawlerEventTypes.page.source}. */
+        /** Inventory provenance for new rows — see `CrawlerEventTypes.page.source`. */
         source?: PageSource;
     };
     /**
@@ -236,5 +334,14 @@ export interface CrawlerEventTypes {
     redirect: {
         /** HEAD-resolved page data carrying the redirect chain (source → destination). */
         result: PageData;
+        /**
+         * Inventory provenance for the redirect-edge call. Forwarded by the
+         * orchestrator to `Archive.setRedirect` → `Database.recordRedirect`
+         * so brand-new destination rows INSERTed by the edge-only path pick
+         * up the inventory label (and propagate it to intermediates) when
+         * the originating chain is in the inventory chain. `undefined` keeps
+         * the DB DEFAULT `'crawled'`.
+         */
+        source: PageSource | undefined;
     };
 }

package/lib/crawler-orchestrator.d.ts

@@ -8,6 +8,9 @@ import Archive from './archive/archive.js';
  * Default list of external URL prefixes excluded from crawling.
  * Includes social media sharing endpoints that are commonly linked
  * but provide no useful crawl data.
+ * @example
+ * // Merged ahead of user-supplied excludeUrls when a crawl starts:
+ * const excludeUrls = [...DEFAULT_EXCLUDED_EXTERNAL_URLS, 'https://ads.example.com'];
  */
 export declare const DEFAULT_EXCLUDED_EXTERNAL_URLS: string[];
 /**
@@ -89,13 +92,13 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * Sets up event listeners on the crawler, starts crawling, and resolves
      * when the crawl completes. Discovered pages, external pages, skipped pages,
      * and resources are forwarded to the archive for storage.
-     * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
+     * @param list - The list of parsed URLs to crawl. May be empty when a resumed
+     *   session already has pending pages queued (for example `--retry-failed`).
      * @param opts - Optional crawl overrides.
      * @param opts.recursive - Whether discovered URLs are followed. Defaults to
      *   `!fromList` (recursive unless the archive was created from a URL list), so
      *   existing callers keep their behaviour; the retry flow passes it explicitly.
      * @returns A promise that resolves when crawling is complete.
-     * @throws {Error} If the URL list is empty.
      */
     crawling(list: ExURL[], opts?: {
         recursive?: boolean;
@@ -117,6 +120,12 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
     /**
      * Write the archive to its configured file path.
      *
+     * The crawler's write path inserts directly into the 0.13 entity
+     * tables (`content_items` / `page_meta` / `anchor_edges` / …) during
+     * `crawling` / `append` / `resume` / `retryFailed` / `inventory`, so by
+     * the time `write()` is called those tables are already populated.
+     * This method just tars.
+     *
      * Emits `writeFileStart` before writing and `writeFileEnd` after
      * the write completes successfully.
      */
@@ -200,10 +209,20 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param inventoryUrls - Pre-read URL list (one URL per element).
      * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
      * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
+     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
+     *   source URL list. The orchestrator deliberately does NOT receive
+     *   the file path: the path is privacy-sensitive (leaks user-home /
+     *   OS structure when archives are shared) and we want it lifted off
+     *   this boundary so no future log line / breadcrumb / error message
+     *   inside the orchestrator can accidentally re-leak it. The CLI
+     *   computes the digest via `computeFileSha256(resolvedListFile)`
+     *   and passes it through here. Pass `null` for programmatic
+     *   callers that built `inventoryUrls` in-memory; the audit row's
+     *   `source_file_sha256` column will be `NULL`.
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
      */
-    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, sourceFileSha256?: string | null): Promise<CrawlerOrchestrator>;
     /**
      * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
      *