@nitpicker/crawler 0.9.0 → 0.11.0

package/lib/archive/types.d.ts

@@ -46,6 +46,28 @@ export interface Config extends Required<Pick<ParseURLOptions, 'disableQueries'>
     /** Whether robots.txt restrictions were ignored during crawling. */
     ignoreRobots: boolean;
 }
+/**
+ * Provenance of a page or resource row — which crawler channel originally
+ * inserted it. Stored as `pages.source` / `resources.source` in the
+ * SQLite schema (NOT NULL DEFAULT `'crawled'`).
+ *
+ * - `'crawled'` — discovered via the recursive crawl rooted at `info.roots`.
+ *   Default for pre-`--inventory` archives after the
+ *   `migratePagesResourcesSource` runtime migration.
+ * - `'inventory-seed'` — supplied directly by a `crawl --inventory` URL
+ *   list. For pages this is the HTML URL that was rendered; for resources
+ *   this is a non-HTML URL handed in by the list (HEAD-fetched without
+ *   rendering).
+ * - `'inventory-discovered'` — found by following links from an
+ *   `inventory-seed` page, OR (for resources) loaded by puppeteer while
+ *   rendering one of those pages.
+ *
+ * Used by the viewer as a badge and to indicate why a row was added.
+ * Isolation queries (`listIsolatedPages` / `listUnusedResources`) judge
+ * orphans by `referrer = 0`, NOT by this value — `source` only labels
+ * the row.
+ */
+export type PageSource = 'crawled' | 'inventory-seed' | 'inventory-discovered';
 /**
  * Filter type for querying pages from the database.
  *
@@ -59,7 +81,13 @@ export interface Config extends Required<Pick<ParseURLOptions, 'disableQueries'>
  */
 export type PageFilter = 'page' | 'page-included-no-target' | 'external-page' | 'internal-page' | 'no-page' | 'external-no-page' | 'internal-no-page';
 /**
- * Raw database row representing a crawled page in the `pages` table.
+ * Raw database row representing a crawled page in the `pages` table (v2 schema).
+ *
+ * Maps 1:1 to the columns defined by `archive/init-schema.ts`. Most meta
+ * fields are derived from beholder 3.0.0's nested {@link import('@d-zero/beholder').Meta}
+ * via `archive/meta/derive-flat-from-meta.ts` and are stored as plain
+ * scalars for SQL-level filter / projection. The catch-all `meta_extras`
+ * JSON column preserves nested sub-objects not flattened above.
  */
 export interface DB_Page {
     /** Auto-incremented primary key. */
@@ -84,48 +112,118 @@ export interface DB_Page {
     contentLength: number | null;
     /** JSON-serialized HTTP response headers. */
     responseHeaders: string;
-    /** The `lang` attribute value from the HTML element, or null if not present. */
+    /** The `lang` attribute value from the HTML element, or null. */
     lang: string | null;
-    /** The page title from the `<title>` element, or null if not present. */
+    /** The `dir` attribute value, or null. */
+    dir: string | null;
+    /** The `<meta charset>` value, or null. */
+    charset: string | null;
+    /** Absolutised `<base href>`, or null. */
+    baseHref: string | null;
+    /** The raw `<meta name="viewport">` content, or null. */
+    viewport_raw: string | null;
+    /** The primary `<meta name="theme-color">` (no media), or null. */
+    themeColor: string | null;
+    /** `<meta name="application-name">`, or null. */
+    applicationName: string | null;
+    /** `<meta name="author">`, or null. */
+    author: string | null;
+    /** `<meta name="generator">`, or null. */
+    generator: string | null;
+    /** `<meta name="publisher">`, or null. */
+    publisher: string | null;
+    /** The page title from the `<title>` element, or null. */
     title: string | null;
-    /** The meta description content, or null if not present. */
+    /** The meta description content, or null. */
     description: string | null;
-    /** The meta keywords content, or null if not present. */
+    /** The meta keywords content, or null. */
     keywords: string | null;
-    /** Whether the noindex robots directive is set (SQLite INTEGER 0/1). */
-    noindex: number | null;
-    /** Whether the nofollow robots directive is set (SQLite INTEGER 0/1). */
-    nofollow: number | null;
-    /** Whether the noarchive robots directive is set (SQLite INTEGER 0/1). */
-    noarchive: number | null;
-    /** The canonical URL from `<link rel="canonical">`, or null if not present. */
+    /** The raw `<meta name="robots">` content, or null. */
+    robots_raw: string | null;
+    /** Whether the noindex directive is set (SQLite INTEGER 0/1). */
+    robots_noindex: number | null;
+    /** Whether the nofollow directive is set (SQLite INTEGER 0/1). */
+    robots_nofollow: number | null;
+    /** Whether the noarchive directive is set (SQLite INTEGER 0/1). */
+    robots_noarchive: number | null;
+    /** Whether the noimageindex directive is set (SQLite INTEGER 0/1). */
+    robots_noimageindex: number | null;
+    /** `<meta name="googlebot">` content, or null. */
+    googlebot: string | null;
+    /** Absolutised `<link rel="canonical">` href, or null. */
     canonical: string | null;
-    /** The alternate URL from `<link rel="alternate">`, or null if not present. */
-    alternate: string | null;
-    /** The Open Graph type (`og:type`), or null if not present. */
+    /** Absolutised `<link rel="amphtml">` href, or null. */
+    amphtml: string | null;
+    /** Absolutised `<link rel="manifest">` href, or null. */
+    manifest: string | null;
+    /** Absolutised `<link rel="icon">` href, or null. */
+    icon_href: string | null;
+    /** Absolutised `<link rel="apple-touch-icon">` href, or null. */
+    appleTouchIcon_href: string | null;
+    /** og:type, or null. */
     og_type: string | null;
-    /** The Open Graph title (`og:title`), or null if not present. */
+    /** og:title, or null. */
     og_title: string | null;
-    /** The Open Graph site name (`og:site_name`), or null if not present. */
+    /** Absolutised og:url, or null. */
+    og_url: string | null;
+    /** og:site_name, or null. */
     og_site_name: string | null;
-    /** The Open Graph description (`og:description`), or null if not present. */
+    /** og:description, or null. */
     og_description: string | null;
-    /** The Open Graph URL (`og:url`), or null if not present. */
-    og_url: string | null;
-    /** The Open Graph image URL (`og:image`), or null if not present. */
+    /** Absolutised og:image (first if multiple), or null. */
     og_image: string | null;
-    /** The Twitter Card type (`twitter:card`), or null if not present. */
+    /** og:image:alt, or null. */
+    og_image_alt: string | null;
+    /** og:image:width as a string (per spec), or null. */
+    og_image_width: string | null;
+    /** og:image:height as a string (per spec), or null. */
+    og_image_height: string | null;
+    /** og:locale, or null. */
+    og_locale: string | null;
+    /** og:article:published_time, or null. */
+    og_article_published_time: string | null;
+    /** og:article:modified_time, or null. */
+    og_article_modified_time: string | null;
+    /** twitter:card, or null. */
     twitter_card: string | null;
+    /** twitter:site, or null. */
+    twitter_site: string | null;
+    /** twitter:creator, or null. */
+    twitter_creator: string | null;
+    /** twitter:title, or null. */
+    twitter_title: string | null;
+    /** twitter:description, or null. */
+    twitter_description: string | null;
+    /** Absolutised twitter:image (or twitter:image:src fallback), or null. */
+    twitter_image: string | null;
+    /** Facebook app id (`fb:app_id`), or null. */
+    fb_app_id: string | null;
+    /** Google site verification token, or null. */
+    verification_google: string | null;
+    /** `format-detection` telephone (SQLite INTEGER 0/1), or null. */
+    formatDetection_telephone: number | null;
+    /** First time this page row was inserted (UNIX ms), or null on legacy rows. */
+    firstCrawledAt: number | null;
+    /** Last successful re-scrape time (UNIX ms), or null on legacy rows. */
+    lastCrawledAt: number | null;
+    /** Number of `page_tags` rows belonging to this page. */
+    tag_count: number | null;
+    /** `meta.jsonLd.length + meta.speculationRules.length` at scrape time. */
+    jsonld_count: number | null;
+    /** Sorted unique provider names, comma-separated (empty string when none). */
+    tags_providers_csv: string | null;
+    /** JSON-serialised nested Meta sub-objects not flattened above. */
+    meta_extras: string | null;
     /** JSON-serialized network logs captured during scraping, or null if not collected. */
     networkLogs: string | null;
-    /** Relative file path to the saved HTML snapshot, or null if not saved. */
-    html: string | null;
     /** Whether the page was skipped during crawling (1) or processed normally (0). */
     isSkipped: 0 | 1;
     /** The reason the page was skipped, or null if it was not skipped. */
     skipReason: string | null;
     /** The natural URL sort order index, or null if not yet assigned. */
     order: number | null;
+    /** Provenance of the row — see {@link PageSource}. */
+    source: PageSource;
 }
 /**
  * Raw database row representing a redirect relationship.
@@ -283,13 +381,24 @@ export interface DB_Resource {
     cdn: string | 0;
     /** JSON-serialized HTTP response headers, or null if not available. */
     responseHeaders: string | null;
+    /** Provenance of the row — see {@link PageSource}. */
+    source: PageSource;
 }
 /**
  * Connection options for the archive's libsql-backed database.
  */
 export interface DatabaseOption {
-    /** The working directory for the database (used for resolving relative paths). */
-    workingDir: string;
     /** The absolute file path to the SQLite database file. */
     filename: string;
+    /**
+     * When `true`, open the database for read-only inspection:
+     *
+     * - Skip schema migrations (no `ALTER TABLE` / `UPDATE` on the user's file).
+     * - Refuse to mkdir/create the parent dir or the db file — fail loudly if
+     *   either is missing instead of resurrecting them.
+     *
+     * Used by the viewer / MCP server when attaching to an in-progress crawl's
+     * tmpDir, where any write would race the live crawler.
+     */
+    readOnly?: boolean;
 }

package/lib/crawler/close-browser-safely.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Minimal structural subset of a Puppeteer `Browser` required to close it safely.
+ *
+ * Declared structurally (rather than importing Puppeteer's `Browser`) so the
+ * function stays unit-testable with a lightweight stub and free of a Puppeteer
+ * import.
+ */
+export interface ClosableBrowser {
+    /** Gracefully closes the browser and all of its pages over the CDP connection. */
+    close(): Promise<void>;
+    /**
+     * Returns the underlying Chromium child process, or `null` when the browser
+     * was connected to (rather than launched) and therefore owns no process.
+     */
+    process(): {
+        /** PID of the Chromium parent process, or `undefined` before spawn settles. */
+        readonly pid?: number;
+        /** Sends a signal to the process; returns whether it was delivered. */
+        kill(signal?: NodeJS.Signals | number): boolean;
+        /** Whether a signal has already been successfully sent to the process. */
+        readonly killed: boolean;
+    } | null;
+}
+/**
+ * Dependency overrides for {@link closeBrowserSafely}. Used only by tests
+ * to substitute the tree-kill orchestration.
+ */
+export interface CloseBrowserSafelyDeps {
+    /**
+     * Kills a process and every descendant. Defaults to {@link killProcessTree}.
+     */
+    killTree?: (pid: number, signal: NodeJS.Signals | number) => Promise<void>;
+}
+/**
+ * Closes a Puppeteer browser, falling back to a hard tree-kill if the graceful
+ * close hangs.
+ *
+ * WHY: When a page's Chromium session dies mid-scrape (e.g. a viewport change
+ * detaches the frame, surfacing `Attempted to use detached Frame` or
+ * `Session closed`), the CDP connection can be left wedged. A bare
+ * `await browser.close()` then never settles, stalling the `deal()` worker and
+ * hanging the whole crawl. Racing the close against a timeout and SIGKILLing
+ * the Chromium process tree (parent + renderer/network/zygote children) on
+ * expiry guarantees the worker always completes and no orphan subprocesses are
+ * left behind.
+ *
+ * The losing timer is cleared explicitly in `.finally()` so it never keeps the
+ * event loop alive after the race settles (a plain `delay()` in `Promise.race`
+ * would leak the timer until it fires).
+ *
+ * The tree-kill happens via {@link killProcessTree}, which enumerates
+ * descendants through `ps` (POSIX) or delegates to `taskkill /T /F` (Windows).
+ * `childProcess.kill('SIGKILL')` is still invoked on the parent up-front
+ * because Node's `ChildProcess.killed` flag governs how Node treats the
+ * spawn handle (reaping etc.); without it the parent would linger in Node's
+ * process table even after the OS-level kill.
+ * @param browser - The browser to close.
+ * @param timeoutMs - Milliseconds to wait for a graceful close before force-killing.
+ *   Defaults to {@link DEFAULT_CLOSE_TIMEOUT_MS}.
+ * @param deps - Test-time overrides (default-free for production callers).
+ * @returns `true` if the graceful close timed out (and a tree-kill was
+ *   attempted), `false` if `close()` settled in time.
+ */
+export declare function closeBrowserSafely(browser: ClosableBrowser, timeoutMs?: number, deps?: CloseBrowserSafelyDeps): Promise<boolean>;

package/lib/crawler/close-browser-safely.js

@@ -0,0 +1,73 @@
+import { crawlerLog } from '../debug.js';
+import { killProcessTree } from './kill-process-tree.js';
+/**
+ * Default time to wait for a graceful `browser.close()` before force-killing
+ * the underlying Chromium process, in milliseconds.
+ */
+const DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000;
+/**
+ * Closes a Puppeteer browser, falling back to a hard tree-kill if the graceful
+ * close hangs.
+ *
+ * WHY: When a page's Chromium session dies mid-scrape (e.g. a viewport change
+ * detaches the frame, surfacing `Attempted to use detached Frame` or
+ * `Session closed`), the CDP connection can be left wedged. A bare
+ * `await browser.close()` then never settles, stalling the `deal()` worker and
+ * hanging the whole crawl. Racing the close against a timeout and SIGKILLing
+ * the Chromium process tree (parent + renderer/network/zygote children) on
+ * expiry guarantees the worker always completes and no orphan subprocesses are
+ * left behind.
+ *
+ * The losing timer is cleared explicitly in `.finally()` so it never keeps the
+ * event loop alive after the race settles (a plain `delay()` in `Promise.race`
+ * would leak the timer until it fires).
+ *
+ * The tree-kill happens via {@link killProcessTree}, which enumerates
+ * descendants through `ps` (POSIX) or delegates to `taskkill /T /F` (Windows).
+ * `childProcess.kill('SIGKILL')` is still invoked on the parent up-front
+ * because Node's `ChildProcess.killed` flag governs how Node treats the
+ * spawn handle (reaping etc.); without it the parent would linger in Node's
+ * process table even after the OS-level kill.
+ * @param browser - The browser to close.
+ * @param timeoutMs - Milliseconds to wait for a graceful close before force-killing.
+ *   Defaults to {@link DEFAULT_CLOSE_TIMEOUT_MS}.
+ * @param deps - Test-time overrides (default-free for production callers).
+ * @returns `true` if the graceful close timed out (and a tree-kill was
+ *   attempted), `false` if `close()` settled in time.
+ */
+export async function closeBrowserSafely(browser, timeoutMs = DEFAULT_CLOSE_TIMEOUT_MS, deps = {}) {
+    // Capture the process up-front: after a successful close() puppeteer
+    // releases its internal reference and process() returns null, so we would
+    // have no handle to tree-kill on timeout.
+    const childProcess = browser.process();
+    let timeoutHandle;
+    const timedOut = await Promise.race([
+        browser
+            .close()
+            .then(() => false)
+            .catch(() => false),
+        new Promise((resolve) => {
+            timeoutHandle = setTimeout(() => resolve(true), timeoutMs);
+        }),
+    ]).finally(() => {
+        if (timeoutHandle) {
+            clearTimeout(timeoutHandle);
+        }
+    });
+    if (timedOut && childProcess && !childProcess.killed) {
+        // Mark the Node ChildProcess as killed so Node's reaping logic treats
+        // it correctly; then walk the OS process tree.
+        childProcess.kill('SIGKILL');
+        // Capture pid once: ChildProcess.pid is technically `number | undefined`
+        // (undefined before spawn settles), and reading it twice across the
+        // `await` below would force the second read to re-widen back to
+        // `number | undefined` regardless of the typeof guard. Snapshotting
+        // makes the type and the runtime value match.
+        const pid = childProcess.pid;
+        if (typeof pid === 'number') {
+            const killTree = deps.killTree ?? ((p, sig) => killProcessTree(p, sig, { log: crawlerLog }));
+            await killTree(pid, 'SIGKILL');
+        }
+    }
+    return timedOut;
+}

package/lib/crawler/crawler.d.ts

@@ -53,8 +53,11 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
      * @param pending - URLs that were pending (not yet scraped) in the previous session.
      * @param scraped - URLs that were already scraped in the previous session.
      * @param resources - Resource URLs that were already captured in the previous session.
+     * @param pagesScrapedOffset - Number of HTML pages already rendered in previous
+     *   sessions, used to seed the session-spanning progress counter. Defaults to 0
+     *   for callers that don't need cross-session accuracy in the progress display.
      */
-    resume(pending: string[], scraped: string[], resources: string[]): void;
+    resume(pending: string[], scraped: string[], resources: string[], pagesScrapedOffset?: number): void;
     /**
      * Start crawling from one or more root URLs.
      *