@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/archive/page.js

@@ -1,9 +1,78 @@
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { isHtmlContentType } from '../crawler/is-html-content-type.js';
+import { parseResponseHeaders } from '../utils/object/parse-response-headers.js';
+import { summarizeJsonLd } from './meta/summarize-jsonld.js';
+import { summarizeTags } from './meta/summarize-tags.js';
+/**
+ * Subset of {@link DB_Page} that maps to the flat meta columns derived from
+ * beholder's nested Meta. Used by {@link Page.metaFlat} so consumers can
+ * iterate every meta column without enumerating each one.
+ *
+ * Keep in sync with {@link import('./meta/types.js').FlatPageMetaColumns} —
+ * one row of `pages` has the same shape.
+ */
+const FLAT_META_COLUMNS = [
+    'lang',
+    'dir',
+    'charset',
+    'baseHref',
+    'viewport_raw',
+    'themeColor',
+    'applicationName',
+    'author',
+    'generator',
+    'publisher',
+    'title',
+    'description',
+    'keywords',
+    'robots_raw',
+    'robots_noindex',
+    'robots_nofollow',
+    'robots_noarchive',
+    'robots_noimageindex',
+    'googlebot',
+    'canonical',
+    'amphtml',
+    'manifest',
+    'icon_href',
+    'appleTouchIcon_href',
+    'og_type',
+    'og_title',
+    'og_url',
+    'og_site_name',
+    'og_description',
+    'og_image',
+    'og_image_alt',
+    'og_image_width',
+    'og_image_height',
+    'og_locale',
+    'og_article_published_time',
+    'og_article_modified_time',
+    'twitter_card',
+    'twitter_site',
+    'twitter_creator',
+    'twitter_title',
+    'twitter_description',
+    'twitter_image',
+    'fb_app_id',
+    'verification_google',
+    'formatDetection_telephone',
+    'tag_count',
+    'jsonld_count',
+    'tags_providers_csv',
+];
 /**
  * Represents a crawled page stored in the archive.
  *
- * Provides access to the page's metadata (title, status, SEO tags, etc.),
- * its relationships (anchors, referrers, redirects), and its HTML snapshot.
+ * Provides typed getters for the most-used meta columns (title, canonical,
+ * og:*, twitter_card, robots flags, lang), plus {@link metaFlat} as an
+ * iterable view over all ~47 flat meta columns and {@link metaExtras} for
+ * the JSON catch-all of nested sub-objects not flattened to columns.
+ *
+ * JSON-LD entries and Wappalyzer tag rows live in dedicated tables and are
+ * fetched on demand via {@link jsonLd} / {@link tags} (lazy reads, same
+ * pattern as {@link getAnchors}).
+ *
  * Instances are created by {@link ArchiveAccessor.getPages} or
  * {@link ArchiveAccessor.getPagesWithRefs}.
  */
@@ -20,16 +89,17 @@ export default class Page {
     #rawAnchors;
     #rawReferrers;
     /**
-     * The alternate URL from the `<link rel="alternate">` tag, or null if not present.
+     * The canonical URL from `<link rel="canonical">` (absolutised against the
+     * page URL at write time), or null if not present.
      */
-    get alternate() {
-        return this.#raw.alternate;
+    get canonical() {
+        return this.#raw.canonical;
     }
     /**
-     * The canonical URL from the `<link rel="canonical">` tag, or null if not present.
+     * The `<meta charset>` value, or null if not present.
      */
-    get canonical() {
-        return this.#raw.canonical;
+    get charset() {
+        return this.#raw.charset;
     }
     /**
      * The content length of the HTTP response in bytes, or null if unknown.
@@ -49,6 +119,14 @@ export default class Page {
     get description() {
         return this.#raw.description;
     }
+    /**
+     * UNIX ms timestamp of the first time this page row was inserted, or
+     * null for legacy rows. Survives `resetFailedPages` so the discovery
+     * time of a page is preserved across retries.
+     */
+    get firstCrawledAt() {
+        return this.#raw.firstCrawledAt;
+    }
     /**
      * Whether this page is on an external domain (outside the crawl scope).
      */
@@ -68,10 +146,11 @@ export default class Page {
         return !!this.#raw.isTarget;
     }
     /**
-     * The reason this page was skipped during crawling, or null if it was not skipped.
+     * Number of JSON-LD + SpeculationRules entries detected on this page
+     * (denormalised aggregate written at scrape time).
      */
-    get skipReason() {
-        return this.#raw.skipReason;
+    get jsonldCount() {
+        return this.#raw.jsonld_count;
     }
     /**
      * The meta keywords content, or null if not present.
@@ -86,70 +165,126 @@ export default class Page {
         return this.#raw.lang;
     }
     /**
-     * Whether the noarchive robots directive is set.
+     * UNIX ms timestamp of the most recent successful scrape for this page,
+     * or null for legacy rows / never-scraped pages.
      */
-    get noarchive() {
-        return !!this.#raw.noarchive;
+    get lastCrawledAt() {
+        return this.#raw.lastCrawledAt;
     }
     /**
-     * Whether the nofollow robots directive is set.
+     * Iterable view over every flat meta column (~47 fields). Returns a frozen
+     * record so consumers can pick fields by name without re-enumerating
+     * typed getters.
+     *
+     * Use the typed getters for high-frequency fields (title, canonical, og_*
+     * etc.); use `metaFlat` for bulk projection (Sheets row generation,
+     * `toJSON`, debug dumps).
      */
-    get nofollow() {
-        return !!this.#raw.nofollow;
+    get metaFlat() {
+        const out = {};
+        for (const col of FLAT_META_COLUMNS) {
+            out[col] = this.#raw[col];
+        }
+        return Object.freeze(out);
     }
     /**
-     * Whether the noindex robots directive is set.
+     * Parsed `meta_extras` JSON: nested Meta sub-objects (referrer, viewport,
+     * httpEquiv, og.image[], twitter.*, apple.*, msapplication.*, geo,
+     * citation, link.alternateHreflang[], others.*, etc.) that were not
+     * flattened to dedicated columns.
+     *
+     * Returns an empty object when the column is null or invalid JSON.
      */
-    get noindex() {
-        return !!this.#raw.noindex;
+    get metaExtras() {
+        const raw = this.#raw.meta_extras;
+        if (raw === null)
+            return {};
+        try {
+            const parsed = JSON.parse(raw);
+            if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {
+                return parsed;
+            }
+            return {};
+        }
+        catch {
+            return {};
+        }
     }
     /**
-     * The Open Graph description (`og:description`), or null if not present.
+     * Open Graph description, or null if not present.
      */
     get og_description() {
         return this.#raw.og_description;
     }
     /**
-     * The Open Graph image URL (`og:image`), or null if not present.
+     * Open Graph image URL (first image when og:image is multi-valued;
+     * absolutised against the page URL at write time), or null if not present.
      */
     get og_image() {
         return this.#raw.og_image;
     }
     /**
-     * The Open Graph site name (`og:site_name`), or null if not present.
+     * Open Graph site name, or null if not present.
      */
     get og_site_name() {
         return this.#raw.og_site_name;
     }
     /**
-     * The Open Graph title (`og:title`), or null if not present.
+     * Open Graph title, or null if not present.
      */
     get og_title() {
         return this.#raw.og_title;
     }
     /**
-     * The Open Graph type (`og:type`), or null if not present.
+     * Open Graph type, or null if not present.
      */
     get og_type() {
         return this.#raw.og_type;
     }
     /**
-     * The Open Graph URL (`og:url`), or null if not present.
+     * Open Graph URL (absolutised), or null if not present.
      */
     get og_url() {
         return this.#raw.og_url;
     }
     /**
      * The parsed HTTP response headers as a key-value record.
-     * Returns an empty object if headers cannot be parsed.
+     * Header values may be arrays for multi-value headers (e.g. `set-cookie`).
+     * Returns an empty object if headers are absent or cannot be parsed.
      */
     get responseHeaders() {
-        try {
-            return JSON.parse(this.#raw.responseHeaders);
-        }
-        catch {
-            return {};
-        }
+        return parseResponseHeaders(this.#raw.responseHeaders) ?? {};
+    }
+    /**
+     * Whether the robots:noarchive directive is set.
+     */
+    get robots_noarchive() {
+        return !!this.#raw.robots_noarchive;
+    }
+    /**
+     * Whether the robots:nofollow directive is set.
+     */
+    get robots_nofollow() {
+        return !!this.#raw.robots_nofollow;
+    }
+    /**
+     * Whether the robots:noindex directive is set.
+     */
+    get robots_noindex() {
+        return !!this.#raw.robots_noindex;
+    }
+    /**
+     * Raw `<meta name="robots">` content, or null if not present.
+     * Use for diagnostics; specific directive flags live on `robots_*` getters.
+     */
+    get robots_raw() {
+        return this.#raw.robots_raw;
+    }
+    /**
+     * The reason this page was skipped during crawling, or null if it was not skipped.
+     */
+    get skipReason() {
+        return this.#raw.skipReason;
     }
     /**
      * The HTTP response status code, or null if the page has not been fetched.
@@ -163,6 +298,21 @@ export default class Page {
     get statusText() {
         return this.#raw.statusText;
     }
+    /**
+     * Number of Wappalyzer tag entries detected on this page (denormalised
+     * aggregate written at scrape time).
+     */
+    get tagCount() {
+        return this.#raw.tag_count;
+    }
+    /**
+     * Sorted unique Wappalyzer provider names, comma-separated, empty string
+     * when no tags. Denormalised aggregate; for the structured form fetch
+     * {@link tags} (lazy).
+     */
+    get tagsProvidersCsv() {
+        return this.#raw.tags_providers_csv ?? '';
+    }
     /**
      * The page title from the `<title>` element.
      * Returns an empty string if no title is set.
@@ -171,7 +321,7 @@ export default class Page {
         return this.#raw.title || '';
     }
     /**
-     * The Twitter Card type (`twitter:card`), or null if not present.
+     * Twitter Card type (`twitter:card`), or null if not present.
      */
     get twitter_card() {
         return this.#raw.twitter_card;
@@ -227,11 +377,28 @@ export default class Page {
         return this.#archive.getAnchorsOnPage(this.#raw.id);
     }
     /**
-     * Reads the HTML snapshot content of this page from the archive.
-     * @returns The HTML content as a string, or null if no snapshot was saved.
+     * Thin wrapper that forwards this page's id to the accessor's
+     * BLOB-read path. Lets callers hold a `Page` reference and ask for its
+     * HTML without having to thread the page id through.
+     * @returns The HTML content, or `null` if no snapshot was saved.
+     * @see {@link ArchiveAccessor.getHtmlOfPage} for the resolution rules.
+     * @example
+     * for (const page of await archive.getPages()) {
+     *   const html = await page.getHtml();
+     *   if (html !== null) processBody(html);
+     * }
      */
     async getHtml() {
-        return this.#archive.getHtmlOfPage(this.#raw.html);
+        return this.#archive.getHtmlOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves the JSON-LD entries for this page from `page_jsonld`.
+     * Lazy — runs a single SELECT per call. Returns entries in insertion
+     * order (matches the scraper's traversal order).
+     * @returns Ordered JSON-LD / SpeculationRules rows.
+     */
+    async getJsonLd() {
+        return this.#archive.getJsonLdOfPage(this.#raw.id);
     }
     /**
      * Retrieves the referrers (incoming links) pointing to this page.
@@ -248,7 +415,14 @@ export default class Page {
                 textContent: r.textContent || '',
             }));
         }
-        return this.#archive.getReferrersOfPage(this.#raw.id);
+        const refs = await this.#archive.getReferrersOfPage(this.#raw.id);
+        return refs.map((r) => ({
+            url: r.url,
+            through: r.through,
+            throughId: r.throughId,
+            hash: r.hash,
+            textContent: r.textContent || '',
+        }));
     }
     /**
      * Retrieves all request referrers for this page directly from the database.
@@ -256,7 +430,22 @@ export default class Page {
      * @returns An array of {@link Referrer} objects.
      */
     async getRequests() {
-        return this.#archive.getReferrersOfPage(this.#raw.id);
+        const refs = await this.#archive.getReferrersOfPage(this.#raw.id);
+        return refs.map((r) => ({
+            url: r.url,
+            through: r.through,
+            throughId: r.throughId,
+            hash: r.hash,
+            textContent: r.textContent || '',
+        }));
+    }
+    /**
+     * Retrieves the Wappalyzer tag rows for this page from `page_tags`.
+     * Lazy — runs a single SELECT per call.
+     * @returns Ordered tag rows.
+     */
+    async getTags() {
+        return this.#archive.getTagsOfPage(this.#raw.id);
     }
     /**
      * Checks whether this page is an internal HTML page (not external and has `text/html` content type).
@@ -270,18 +459,33 @@ export default class Page {
      * @returns `true` if the content type is `text/html`, `false` otherwise.
      */
     isPage() {
-        const type = this.contentType || '';
-        return type.toLowerCase().trim() === 'text/html';
-    }
-    /**
-     * Serializes the page data to a plain JSON object,
-     * including resolved anchors and referrers.
+        return isHtmlContentType(this.contentType);
+    }
+    /**
+     * Serializes the page data to a plain JSON object including the full flat
+     * meta column set, the `meta_extras` catch-all, and **summaries** of the
+     * JSON-LD and tag rows.
+     *
+     * Summaries (not raw entries) are inlined so a Page detail payload stays
+     * token-bounded for MCP / LLM consumers — the full `raw` JSON-LD payload
+     * is fetched separately via `getJsonLd()` / the dedicated CLI/MCP
+     * endpoints.
+     *
+     * Anchors and referrers are still resolved eagerly because consumers
+     * (Sheets `eachPage`, viewer detail) depend on having them inline.
      * @returns A plain object containing all page metadata and relationships.
      */
     async toJSON() {
+        const [anchors, referrers, jsonLdRows, tagRows] = await Promise.all([
+            this.getAnchors(),
+            this.getReferrers(),
+            this.getJsonLd(),
+            this.getTags(),
+        ]);
+        const jsonLdSummary = summarizeJsonLd(jsonLdRows);
+        const tagsSummary = summarizeTags(tagRows);
         return {
             url: this.url.href,
-            title: this.title,
             status: this.status,
             statusText: this.statusText,
             contentType: this.contentType,
@@ -291,26 +495,17 @@ export default class Page {
             isSkipped: this.isSkipped,
             skipReason: this.skipReason,
             isTarget: this.isTarget,
-            lang: this.lang,
-            description: this.description,
-            keywords: this.keywords,
-            noindex: this.noindex,
-            nofollow: this.nofollow,
-            noarchive: this.noarchive,
-            canonical: this.canonical,
-            alternate: this.alternate,
-            twitter_card: this.twitter_card,
-            og_site_name: this.og_site_name,
-            og_url: this.og_url,
-            og_title: this.og_title,
-            og_description: this.og_description,
-            og_type: this.og_type,
-            og_image: this.og_image,
+            firstCrawledAt: this.firstCrawledAt,
+            lastCrawledAt: this.lastCrawledAt,
+            ...this.metaFlat,
+            metaExtras: this.metaExtras,
+            jsonLd: jsonLdSummary,
+            tags: tagsSummary,
             redirectFrom: this.redirectFrom,
             isPage: this.isPage(),
             isInternalPage: this.isInternalPage(),
-            getAnchors: await this.getAnchors(),
-            getReferrers: await this.getReferrers(),
+            getAnchors: anchors,
+            getReferrers: referrers,
         };
     }
 }

package/lib/archive/peek-archive-lock.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Result of probing an archive's `.lock` sibling without acquiring it.
+ *
+ * Used by read-only consumers (the viewer, MCP `open_archive`) that want
+ * to surface "a crawler is currently writing this archive" to the user
+ * without blocking or competing for the lock.
+ */
+export interface ArchiveLockHolder {
+    /** Path of the lock directory (`<tmpDir>.lock`) that was probed. */
+    readonly lockPath: string;
+    /** PID of the process whose lock is recorded in `pid.txt`. */
+    readonly pid: number;
+    /**
+     * Whether that PID is currently alive on the OS (signal-0 probe).
+     *
+     * **PID-recycling caveat**: this only proves *some* process owns that PID,
+     * not necessarily the original crawler. Callers should treat
+     * `{ alive: true }` as "likely a live crawler" rather than a hard
+     * guarantee.
+     */
+    readonly alive: boolean;
+}
+/**
+ * Probe `<tmpDir>.lock/pid.txt` without acquiring the lock.
+ *
+ * Mirror of the alive-check inside {@link acquireArchiveLock}, exposed for
+ * read-only consumers so they can detect (and surface) a concurrent crawler
+ * without competing for the lock. Returns `null` when no lock directory is
+ * present or the pid file is missing/malformed — callers should treat that
+ * as "no detectable crawler" rather than as an error.
+ *
+ * Co-located with {@link acquireArchiveLock} so the writer side and the
+ * read-only probe stay in lockstep when the lock format evolves (e.g.
+ * adding a hostname/timestamp field).
+ * @param tmpDir - The archive's temporary working directory whose
+ *   `${tmpDir}.lock` sibling will be probed.
+ * @returns Lock-holder metadata when a parseable lock exists, otherwise
+ *   `null`.
+ */
+export declare function peekArchiveLockHolder(tmpDir: string): ArchiveLockHolder | null;

package/lib/archive/peek-archive-lock.js

@@ -0,0 +1,62 @@
+import { existsSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+/**
+ * Probe `<tmpDir>.lock/pid.txt` without acquiring the lock.
+ *
+ * Mirror of the alive-check inside {@link acquireArchiveLock}, exposed for
+ * read-only consumers so they can detect (and surface) a concurrent crawler
+ * without competing for the lock. Returns `null` when no lock directory is
+ * present or the pid file is missing/malformed — callers should treat that
+ * as "no detectable crawler" rather than as an error.
+ *
+ * Co-located with {@link acquireArchiveLock} so the writer side and the
+ * read-only probe stay in lockstep when the lock format evolves (e.g.
+ * adding a hostname/timestamp field).
+ * @param tmpDir - The archive's temporary working directory whose
+ *   `${tmpDir}.lock` sibling will be probed.
+ * @returns Lock-holder metadata when a parseable lock exists, otherwise
+ *   `null`.
+ */
+export function peekArchiveLockHolder(tmpDir) {
+    const lockPath = `${tmpDir}.lock`;
+    if (!existsSync(lockPath)) {
+        return null;
+    }
+    const pid = readPidFile(lockPath);
+    if (pid === null) {
+        return null;
+    }
+    return { lockPath, pid, alive: isProcessAlive(pid) };
+}
+/**
+ * Read the PID stored in the lock's `pid.txt`. Returns `null` for any
+ * missing/unparsable/non-positive content so the probe degrades gracefully
+ * on partially-written or corrupted lock dirs.
+ * @param lockPath - The lock directory path.
+ */
+function readPidFile(lockPath) {
+    try {
+        const raw = readFileSync(path.join(lockPath, 'pid.txt'), 'utf8');
+        const pid = Number.parseInt(raw.trim(), 10);
+        return Number.isFinite(pid) && pid > 0 ? pid : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Check whether a PID is currently alive using a signal-0 probe.
+ *
+ * Treats `EPERM` as alive (the process exists but is owned by another
+ * user). Any other errno (notably `ESRCH`) means dead.
+ * @param pid - The process id to probe.
+ */
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === 'EPERM';
+    }
+}

package/lib/archive/resolve-redirect-chain.d.ts

@@ -0,0 +1,33 @@
+/**
+ * The destination of a redirect chain and the source URLs that point at it.
+ */
+export interface RedirectChain {
+    /** The final destination URL the chain lands on (normalised). */
+    destUrl: string;
+    /**
+     * The source URLs that redirect to {@link destUrl}: the originally requested
+     * URL followed by every intermediate hop. Empty when the page was not
+     * redirected at all.
+     */
+    sources: string[];
+}
+/**
+ * Splits a page's redirect chain into its final destination and the list of
+ * source URLs that point at it.
+ *
+ * The archive stores redirects as edges: every source URL carries a
+ * `redirectDestId` pointing at the destination page, and only the destination
+ * holds content. This helper applies the shared convention — the last entry of
+ * `redirectPaths` is the destination, while the originally requested URL plus
+ * every intermediate hop are the sources. When the page was not redirected,
+ * the destination is the page URL itself and there are no sources.
+ *
+ * Used by both {@link Database.updatePage} (which renders and stores the
+ * destination) and {@link Database.recordRedirect} (the #73 convergence
+ * optimisation, which records the edge for a destination already rendered
+ * elsewhere without touching its content).
+ * @param pageUrl - The originally requested URL (normalised, without hash/auth).
+ * @param redirectPaths - The redirect hop URLs captured during fetch, in order.
+ * @returns The destination URL and the source URLs pointing to it.
+ */
+export declare function resolveRedirectChain(pageUrl: string, redirectPaths: readonly string[]): RedirectChain;

package/lib/archive/resolve-redirect-chain.js

@@ -0,0 +1,27 @@
+/**
+ * Splits a page's redirect chain into its final destination and the list of
+ * source URLs that point at it.
+ *
+ * The archive stores redirects as edges: every source URL carries a
+ * `redirectDestId` pointing at the destination page, and only the destination
+ * holds content. This helper applies the shared convention — the last entry of
+ * `redirectPaths` is the destination, while the originally requested URL plus
+ * every intermediate hop are the sources. When the page was not redirected,
+ * the destination is the page URL itself and there are no sources.
+ *
+ * Used by both {@link Database.updatePage} (which renders and stores the
+ * destination) and {@link Database.recordRedirect} (the #73 convergence
+ * optimisation, which records the edge for a destination already rendered
+ * elsewhere without touching its content).
+ * @param pageUrl - The originally requested URL (normalised, without hash/auth).
+ * @param redirectPaths - The redirect hop URLs captured during fetch, in order.
+ * @returns The destination URL and the source URLs pointing to it.
+ */
+export function resolveRedirectChain(pageUrl, redirectPaths) {
+    if (redirectPaths.length === 0) {
+        return { destUrl: pageUrl, sources: [] };
+    }
+    const paths = [...redirectPaths];
+    const destUrl = paths.pop();
+    return { destUrl, sources: [pageUrl, ...paths] };
+}