@nitpicker/crawler 0.8.0 → 0.11.0

package/lib/archive/page.d.ts

@@ -1,10 +1,27 @@
 import type { ArchiveAccessor } from './archive-accessor.js';
+import type { JsonLdRow, JsonLdSummary, TagRow, TagsSummary } from './meta/types.js';
 import type { Anchor, Redirect, Referrer, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer } from './types.js';
+/**
+ * Subset of {@link DB_Page} that maps to the flat meta columns derived from
+ * beholder's nested Meta. Used by {@link Page.metaFlat} so consumers can
+ * iterate every meta column without enumerating each one.
+ *
+ * Keep in sync with {@link import('./meta/types.js').FlatPageMetaColumns} —
+ * one row of `pages` has the same shape.
+ */
+declare const FLAT_META_COLUMNS: readonly ["lang", "dir", "charset", "baseHref", "viewport_raw", "themeColor", "applicationName", "author", "generator", "publisher", "title", "description", "keywords", "robots_raw", "robots_noindex", "robots_nofollow", "robots_noarchive", "robots_noimageindex", "googlebot", "canonical", "amphtml", "manifest", "icon_href", "appleTouchIcon_href", "og_type", "og_title", "og_url", "og_site_name", "og_description", "og_image", "og_image_alt", "og_image_width", "og_image_height", "og_locale", "og_article_published_time", "og_article_modified_time", "twitter_card", "twitter_site", "twitter_creator", "twitter_title", "twitter_description", "twitter_image", "fb_app_id", "verification_google", "formatDetection_telephone", "tag_count", "jsonld_count", "tags_providers_csv"];
 /**
  * Represents a crawled page stored in the archive.
  *
- * Provides access to the page's metadata (title, status, SEO tags, etc.),
- * its relationships (anchors, referrers, redirects), and its HTML snapshot.
+ * Provides typed getters for the most-used meta columns (title, canonical,
+ * og:*, twitter_card, robots flags, lang), plus {@link metaFlat} as an
+ * iterable view over all ~47 flat meta columns and {@link metaExtras} for
+ * the JSON catch-all of nested sub-objects not flattened to columns.
+ *
+ * JSON-LD entries and Wappalyzer tag rows live in dedicated tables and are
+ * fetched on demand via {@link jsonLd} / {@link tags} (lazy reads, same
+ * pattern as {@link getAnchors}).
+ *
  * Instances are created by {@link ArchiveAccessor.getPages} or
  * {@link ArchiveAccessor.getPagesWithRefs}.
  */
@@ -17,13 +34,14 @@ export default class Page {
      */
     readonly redirectFrom: Redirect[];
     /**
-     * The alternate URL from the `<link rel="alternate">` tag, or null if not present.
+     * The canonical URL from `<link rel="canonical">` (absolutised against the
+     * page URL at write time), or null if not present.
      */
-    get alternate(): string | null;
+    get canonical(): string | null;
     /**
-     * The canonical URL from the `<link rel="canonical">` tag, or null if not present.
+     * The `<meta charset>` value, or null if not present.
      */
-    get canonical(): string | null;
+    get charset(): string | null;
     /**
      * The content length of the HTTP response in bytes, or null if unknown.
      */
@@ -36,6 +54,12 @@ export default class Page {
      * The meta description content, or null if not present.
      */
     get description(): string | null;
+    /**
+     * UNIX ms timestamp of the first time this page row was inserted, or
+     * null for legacy rows. Survives `resetFailedPages` so the discovery
+     * time of a page is preserved across retries.
+     */
+    get firstCrawledAt(): number | null;
     /**
      * Whether this page is on an external domain (outside the crawl scope).
      */
@@ -49,9 +73,10 @@ export default class Page {
      */
     get isTarget(): boolean;
     /**
-     * The reason this page was skipped during crawling, or null if it was not skipped.
+     * Number of JSON-LD + SpeculationRules entries detected on this page
+     * (denormalised aggregate written at scrape time).
      */
-    get skipReason(): string | null;
+    get jsonldCount(): number | null;
     /**
      * The meta keywords content, or null if not present.
      */
@@ -61,46 +86,81 @@ export default class Page {
      */
     get lang(): string | null;
     /**
-     * Whether the noarchive robots directive is set.
+     * UNIX ms timestamp of the most recent successful scrape for this page,
+     * or null for legacy rows / never-scraped pages.
      */
-    get noarchive(): boolean;
+    get lastCrawledAt(): number | null;
     /**
-     * Whether the nofollow robots directive is set.
+     * Iterable view over every flat meta column (~47 fields). Returns a frozen
+     * record so consumers can pick fields by name without re-enumerating
+     * typed getters.
+     *
+     * Use the typed getters for high-frequency fields (title, canonical, og_*
+     * etc.); use `metaFlat` for bulk projection (Sheets row generation,
+     * `toJSON`, debug dumps).
      */
-    get nofollow(): boolean;
+    get metaFlat(): Readonly<Record<(typeof FLAT_META_COLUMNS)[number], string | number | null>>;
     /**
-     * Whether the noindex robots directive is set.
+     * Parsed `meta_extras` JSON: nested Meta sub-objects (referrer, viewport,
+     * httpEquiv, og.image[], twitter.*, apple.*, msapplication.*, geo,
+     * citation, link.alternateHreflang[], others.*, etc.) that were not
+     * flattened to dedicated columns.
+     *
+     * Returns an empty object when the column is null or invalid JSON.
      */
-    get noindex(): boolean;
+    get metaExtras(): Record<string, unknown>;
     /**
-     * The Open Graph description (`og:description`), or null if not present.
+     * Open Graph description, or null if not present.
      */
     get og_description(): string | null;
     /**
-     * The Open Graph image URL (`og:image`), or null if not present.
+     * Open Graph image URL (first image when og:image is multi-valued;
+     * absolutised against the page URL at write time), or null if not present.
      */
     get og_image(): string | null;
     /**
-     * The Open Graph site name (`og:site_name`), or null if not present.
+     * Open Graph site name, or null if not present.
      */
     get og_site_name(): string | null;
     /**
-     * The Open Graph title (`og:title`), or null if not present.
+     * Open Graph title, or null if not present.
      */
     get og_title(): string | null;
     /**
-     * The Open Graph type (`og:type`), or null if not present.
+     * Open Graph type, or null if not present.
      */
     get og_type(): string | null;
     /**
-     * The Open Graph URL (`og:url`), or null if not present.
+     * Open Graph URL (absolutised), or null if not present.
      */
     get og_url(): string | null;
     /**
      * The parsed HTTP response headers as a key-value record.
-     * Returns an empty object if headers cannot be parsed.
+     * Header values may be arrays for multi-value headers (e.g. `set-cookie`).
+     * Returns an empty object if headers are absent or cannot be parsed.
      */
-    get responseHeaders(): Record<string, string>;
+    get responseHeaders(): Record<string, string | string[] | undefined>;
+    /**
+     * Whether the robots:noarchive directive is set.
+     */
+    get robots_noarchive(): boolean;
+    /**
+     * Whether the robots:nofollow directive is set.
+     */
+    get robots_nofollow(): boolean;
+    /**
+     * Whether the robots:noindex directive is set.
+     */
+    get robots_noindex(): boolean;
+    /**
+     * Raw `<meta name="robots">` content, or null if not present.
+     * Use for diagnostics; specific directive flags live on `robots_*` getters.
+     */
+    get robots_raw(): string | null;
+    /**
+     * The reason this page was skipped during crawling, or null if it was not skipped.
+     */
+    get skipReason(): string | null;
     /**
      * The HTTP response status code, or null if the page has not been fetched.
      */
@@ -109,13 +169,24 @@ export default class Page {
      * The HTTP response status text (e.g., `"OK"`, `"Not Found"`), or null if not fetched.
      */
     get statusText(): string | null;
+    /**
+     * Number of Wappalyzer tag entries detected on this page (denormalised
+     * aggregate written at scrape time).
+     */
+    get tagCount(): number | null;
+    /**
+     * Sorted unique Wappalyzer provider names, comma-separated, empty string
+     * when no tags. Denormalised aggregate; for the structured form fetch
+     * {@link tags} (lazy).
+     */
+    get tagsProvidersCsv(): string;
     /**
      * The page title from the `<title>` element.
      * Returns an empty string if no title is set.
      */
     get title(): string;
     /**
-     * The Twitter Card type (`twitter:card`), or null if not present.
+     * Twitter Card type (`twitter:card`), or null if not present.
      */
     get twitter_card(): string | null;
     /**
@@ -140,10 +211,25 @@ export default class Page {
      */
     getAnchors(): Promise<Anchor[]>;
     /**
-     * Reads the HTML snapshot content of this page from the archive.
-     * @returns The HTML content as a string, or null if no snapshot was saved.
+     * Thin wrapper that forwards this page's id to the accessor's
+     * BLOB-read path. Lets callers hold a `Page` reference and ask for its
+     * HTML without having to thread the page id through.
+     * @returns The HTML content, or `null` if no snapshot was saved.
+     * @see {@link ArchiveAccessor.getHtmlOfPage} for the resolution rules.
+     * @example
+     * for (const page of await archive.getPages()) {
+     *   const html = await page.getHtml();
+     *   if (html !== null) processBody(html);
+     * }
      */
     getHtml(): Promise<string | null>;
+    /**
+     * Retrieves the JSON-LD entries for this page from `page_jsonld`.
+     * Lazy — runs a single SELECT per call. Returns entries in insertion
+     * order (matches the scraper's traversal order).
+     * @returns Ordered JSON-LD / SpeculationRules rows.
+     */
+    getJsonLd(): Promise<readonly JsonLdRow[]>;
     /**
      * Retrieves the referrers (incoming links) pointing to this page.
      * Uses pre-loaded data if available, otherwise queries the database.
@@ -156,6 +242,12 @@ export default class Page {
      * @returns An array of {@link Referrer} objects.
      */
     getRequests(): Promise<Referrer[]>;
+    /**
+     * Retrieves the Wappalyzer tag rows for this page from `page_tags`.
+     * Lazy — runs a single SELECT per call.
+     * @returns Ordered tag rows.
+     */
+    getTags(): Promise<readonly TagRow[]>;
     /**
      * Checks whether this page is an internal HTML page (not external and has `text/html` content type).
      * @returns `true` if this is an internal HTML page, `false` otherwise.
@@ -167,42 +259,88 @@ export default class Page {
      */
     isPage(): boolean;
     /**
-     * Serializes the page data to a plain JSON object,
-     * including resolved anchors and referrers.
+     * Serializes the page data to a plain JSON object including the full flat
+     * meta column set, the `meta_extras` catch-all, and **summaries** of the
+     * JSON-LD and tag rows.
+     *
+     * Summaries (not raw entries) are inlined so a Page detail payload stays
+     * token-bounded for MCP / LLM consumers — the full `raw` JSON-LD payload
+     * is fetched separately via `getJsonLd()` / the dedicated CLI/MCP
+     * endpoints.
+     *
+     * Anchors and referrers are still resolved eagerly because consumers
+     * (Sheets `eachPage`, viewer detail) depend on having them inline.
      * @returns A plain object containing all page metadata and relationships.
      */
     toJSON(): Promise<{
+        metaExtras: Record<string, unknown>;
+        jsonLd: JsonLdSummary;
+        tags: TagsSummary;
+        redirectFrom: Redirect[];
+        isPage: boolean;
+        isInternalPage: boolean;
+        getAnchors: Anchor[];
+        getReferrers: Referrer[];
+        dir: string | number | null;
+        lang: string | number | null;
+        charset: string | number | null;
+        baseHref: string | number | null;
+        viewport_raw: string | number | null;
+        themeColor: string | number | null;
+        applicationName: string | number | null;
+        author: string | number | null;
+        generator: string | number | null;
+        publisher: string | number | null;
+        title: string | number | null;
+        description: string | number | null;
+        keywords: string | number | null;
+        robots_raw: string | number | null;
+        robots_noindex: string | number | null;
+        robots_nofollow: string | number | null;
+        robots_noarchive: string | number | null;
+        robots_noimageindex: string | number | null;
+        googlebot: string | number | null;
+        canonical: string | number | null;
+        amphtml: string | number | null;
+        manifest: string | number | null;
+        icon_href: string | number | null;
+        appleTouchIcon_href: string | number | null;
+        og_type: string | number | null;
+        og_title: string | number | null;
+        og_url: string | number | null;
+        og_site_name: string | number | null;
+        og_description: string | number | null;
+        og_image: string | number | null;
+        og_image_alt: string | number | null;
+        og_image_width: string | number | null;
+        og_image_height: string | number | null;
+        og_locale: string | number | null;
+        og_article_published_time: string | number | null;
+        og_article_modified_time: string | number | null;
+        twitter_card: string | number | null;
+        twitter_site: string | number | null;
+        twitter_creator: string | number | null;
+        twitter_title: string | number | null;
+        twitter_description: string | number | null;
+        twitter_image: string | number | null;
+        fb_app_id: string | number | null;
+        verification_google: string | number | null;
+        formatDetection_telephone: string | number | null;
+        tag_count: string | number | null;
+        jsonld_count: string | number | null;
+        tags_providers_csv: string | number | null;
         url: string;
-        title: string;
         status: number | null;
         statusText: string | null;
         contentType: string | null;
         contentLength: number | null;
-        responseHeaders: Record<string, string>;
+        responseHeaders: Record<string, string | string[] | undefined>;
         isExternal: boolean;
         isSkipped: boolean;
         skipReason: string | null;
         isTarget: boolean;
-        lang: string | null;
-        description: string | null;
-        keywords: string | null;
-        noindex: boolean;
-        nofollow: boolean;
-        noarchive: boolean;
-        canonical: string | null;
-        alternate: string | null;
-        twitter_card: string | null;
-        og_site_name: string | null;
-        og_url: string | null;
-        og_title: string | null;
-        og_description: string | null;
-        og_type: string | null;
-        og_image: string | null;
-        redirectFrom: Redirect[];
-        isPage: boolean;
-        isInternalPage: boolean;
-        getAnchors: Anchor[];
-        getReferrers: Referrer[];
+        firstCrawledAt: number | null;
+        lastCrawledAt: number | null;
     }>;
 }
 /**