@nitpicker/archive 0.0.0 → 0.25.0

package/lib/page.d.ts

@@ -0,0 +1,546 @@
+import type { ArchiveAccessor } from './archive-accessor.js';
+import type { JsonLdRow, JsonLdSummary, PageTechnologyRow, TechnologySignalRow } from './meta/types.js';
+import type { Anchor, Redirect, Referrer, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer } from './types.js';
+/**
+ * Subset of {@link DB_Page} that maps to the flat meta columns derived from
+ * beholder's nested Meta. Used by {@link Page.metaFlat} so consumers can
+ * iterate every meta column without enumerating each one.
+ *
+ * Keep in sync with {@link import('./meta/types.js').FlatPageMetaColumns} —
+ * one row of `pages` has the same shape.
+ */
+declare const FLAT_META_COLUMNS: readonly ["lang", "dir", "charset", "baseHref", "viewport_raw", "themeColor", "applicationName", "author", "generator", "publisher", "title", "description", "keywords", "robots_raw", "robots_noindex", "robots_nofollow", "robots_noarchive", "robots_noimageindex", "googlebot", "canonical", "amphtml", "manifest", "icon_href", "appleTouchIcon_href", "og_type", "og_title", "og_url", "og_site_name", "og_description", "og_image", "og_image_alt", "og_image_width", "og_image_height", "og_locale", "og_article_published_time", "og_article_modified_time", "twitter_card", "twitter_site", "twitter_creator", "twitter_title", "twitter_description", "twitter_image", "fb_app_id", "verification_google", "formatDetection_telephone", "tag_count", "jsonld_count", "tags_providers_csv", "main_content_node_name", "main_content_id", "main_content_role", "main_content_selector", "main_content_class_list", "main_content_word_count", "main_content_body_word_count", "main_content_heading_count", "main_content_image_count", "main_content_table_count", "main_content_button_count", "main_content_iframe_count", "main_content_video_count", "main_content_audio_count", "main_content_canvas_count", "scroll_height_desktop", "scroll_height_mobile"];
+/**
+ * Represents a crawled page stored in the archive.
+ *
+ * Provides typed getters for the most-used meta columns (title, canonical,
+ * og:*, twitter_card, robots flags, lang), plus {@link Page.metaFlat} as an
+ * iterable view over all ~47 flat meta columns and {@link Page.metaExtras} for
+ * the JSON catch-all of nested sub-objects not flattened to columns.
+ *
+ * JSON-LD entries and detected technology signals live in dedicated tables
+ * and are fetched on demand via {@link Page.getJsonLd} /
+ * {@link Page.getTechnologySignals} / {@link Page.getPageTechnologies} (lazy
+ * reads, same pattern as {@link Page.getAnchors}).
+ *
+ * Instances are created by {@link ArchiveAccessor.getPages} or
+ * {@link ArchiveAccessor.getPagesWithRefs}.
+ */
+export default class Page {
+    #private;
+    /**
+     * An array of URLs that redirect to this page.
+     * Each entry contains the source URL and its page ID.
+     * Returns an empty array if no redirects exist.
+     */
+    readonly redirectFrom: Redirect[];
+    /**
+     * The canonical URL from `<link rel="canonical">` (absolutised against the
+     * page URL at write time), or null if not present.
+     */
+    get canonical(): string | null;
+    /**
+     * The `<meta charset>` value, or null if not present.
+     */
+    get charset(): string | null;
+    /**
+     * The content length of the HTTP response in bytes, or null if unknown.
+     */
+    get contentLength(): number | null;
+    /**
+     * The MIME content type of the HTTP response (e.g., `"text/html"`), or null if unknown.
+     */
+    get contentType(): string | null;
+    /**
+     * The meta description content, or null if not present.
+     */
+    get description(): string | null;
+    /**
+     * UNIX ms timestamp of the first time this page row was inserted, or
+     * null for legacy rows. Survives `resetFailedPages` so the discovery
+     * time of a page is preserved across retries.
+     */
+    get firstCrawledAt(): number | null;
+    /**
+     * The page's row id in the archive (`content_items.id`): stable within one
+     * archive, the key the page's own tables (`page_html_ref`, `page_templates`,
+     * …) reference it by. Not comparable across archives.
+     * @example
+     * ```ts
+     * const pageIdsWithHtml = await collectPageIdsWithHtml(archive);
+     * const hasHtml = pageIdsWithHtml.has(page.id);
+     * ```
+     */
+    get id(): number;
+    /**
+     * Whether this page is on an external domain (outside the crawl scope).
+     */
+    get isExternal(): boolean;
+    /**
+     * Whether this page was skipped during crawling.
+     */
+    get isSkipped(): boolean;
+    /**
+     * Whether this page was a crawl target (as opposed to being discovered incidentally).
+     */
+    get isTarget(): boolean;
+    /**
+     * Number of JSON-LD + SpeculationRules entries detected on this page
+     * (denormalised aggregate written at scrape time).
+     */
+    get jsonldCount(): number | null;
+    /**
+     * The meta keywords content, or null if not present.
+     */
+    get keywords(): string | null;
+    /**
+     * The `lang` attribute value from the HTML element, or null if not present.
+     */
+    get lang(): string | null;
+    /**
+     * UNIX ms timestamp of the most recent successful scrape for this page,
+     * or null for legacy rows / never-scraped pages.
+     */
+    get lastCrawledAt(): number | null;
+    /**
+     * Detected main-content element's `nodeName` (e.g. `'MAIN'`), or null when
+     * no main region was found or the page was not fully rendered.
+     */
+    get mainContentNodeName(): string | null;
+    /**
+     * Detected main-content element's `id`, or null.
+     */
+    get mainContentId(): string | null;
+    /**
+     * Detected main-content element's `role` attribute, or null.
+     */
+    get mainContentRole(): string | null;
+    /**
+     * Diagnostic tag+id+class selector for the detected main-content element, or null.
+     */
+    get mainContentSelector(): string | null;
+    /**
+     * Detected main-content element's CSS classes, or null when no main region
+     * was found. Parsed from the JSON-encoded `main_content_class_list` column.
+     * @throws {Error} When the stored value is not a JSON array of strings — a
+     *   corrupt archive; a silently dropped class list would change which
+     *   template the page is classified into.
+     */
+    get mainContentClassList(): string[] | null;
+    /**
+     * Character count of the main region's text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentWordCount(): number | null;
+    /**
+     * Character count of `document.body`'s text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentBodyWordCount(): number | null;
+    /**
+     * Number of headings within the main region (denormalised aggregate), or null.
+     */
+    get mainContentHeadingCount(): number | null;
+    /**
+     * Number of images within the main region (denormalised aggregate), or null.
+     */
+    get mainContentImageCount(): number | null;
+    /**
+     * Number of tables within the main region (denormalised aggregate), or null.
+     */
+    get mainContentTableCount(): number | null;
+    /**
+     * Number of button-like elements within the main region (denormalised
+     * aggregate), or null.
+     */
+    get mainContentButtonCount(): number | null;
+    /**
+     * Number of iframes within the main region (denormalised aggregate), or null.
+     */
+    get mainContentIframeCount(): number | null;
+    /**
+     * Number of videos within the main region (denormalised aggregate), or null.
+     */
+    get mainContentVideoCount(): number | null;
+    /**
+     * Number of audios within the main region (denormalised aggregate), or null.
+     */
+    get mainContentAudioCount(): number | null;
+    /**
+     * Number of canvases within the main region (denormalised aggregate), or null.
+     */
+    get mainContentCanvasCount(): number | null;
+    /**
+     * Number of Web Components (custom elements) within the main region
+     * (denormalised aggregate), `null` when unknown (page not rendered, or
+     * nitpicker's own capture failed — see `compute-main-contents-denormalized.ts`).
+     */
+    get mainContentCustomElementCount(): number | null;
+    /**
+     * Iterable view over every flat meta column (~47 fields). Returns a frozen
+     * record so consumers can pick fields by name without re-enumerating
+     * typed getters.
+     *
+     * Use the typed getters for high-frequency fields (title, canonical, og_*
+     * etc.); use `metaFlat` for bulk projection (Sheets row generation,
+     * `toJSON`, debug dumps).
+     */
+    get metaFlat(): Readonly<Record<(typeof FLAT_META_COLUMNS)[number], string | number | null>>;
+    /**
+     * Parsed `meta_extras` JSON: nested Meta sub-objects (referrer, viewport,
+     * httpEquiv, og.image[], twitter.*, apple.*, msapplication.*, geo,
+     * citation, link.alternateHreflang[], others.*, etc.) that were not
+     * flattened to dedicated columns.
+     *
+     * Returns an empty object when the column is null or invalid JSON.
+     */
+    get metaExtras(): Record<string, unknown>;
+    /**
+     * Open Graph description, or null if not present.
+     */
+    get og_description(): string | null;
+    /**
+     * Open Graph image URL (first image when og:image is multi-valued;
+     * absolutised against the page URL at write time), or null if not present.
+     */
+    get og_image(): string | null;
+    /**
+     * Open Graph site name, or null if not present.
+     */
+    get og_site_name(): string | null;
+    /**
+     * Open Graph title, or null if not present.
+     */
+    get og_title(): string | null;
+    /**
+     * Open Graph type, or null if not present.
+     */
+    get og_type(): string | null;
+    /**
+     * Open Graph URL (absolutised), or null if not present.
+     */
+    get og_url(): string | null;
+    /**
+     * The parsed HTTP response headers as a key-value record.
+     * Header values may be arrays for multi-value headers (e.g. `set-cookie`).
+     * Returns an empty object if headers are absent or cannot be parsed.
+     */
+    get responseHeaders(): Record<string, string | string[] | undefined>;
+    /**
+     * Whether the robots:noarchive directive is set.
+     */
+    get robots_noarchive(): boolean;
+    /**
+     * Whether the robots:nofollow directive is set.
+     */
+    get robots_nofollow(): boolean;
+    /**
+     * Whether the robots:noindex directive is set.
+     */
+    get robots_noindex(): boolean;
+    /**
+     * Raw `<meta name="robots">` content, or null if not present.
+     * Use for diagnostics; specific directive flags live on `robots_*` getters.
+     */
+    get robots_raw(): string | null;
+    /**
+     * `document.body.scrollHeight` at the desktop-compact preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightDesktop(): number | null;
+    /**
+     * `document.body.scrollHeight` at the mobile-small preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightMobile(): number | null;
+    /**
+     * `@d-zero/beholder`'s `IMAGE_SCAN_CODE` outcome for the desktop-compact
+     * `<img>` element scan (denormalised aggregate written at scrape time), or
+     * null when not attempted.
+     */
+    get imageScanDesktop(): number | null;
+    /**
+     * `@d-zero/beholder`'s `IMAGE_SCAN_CODE` outcome for the mobile-small
+     * `<img>` element scan (denormalised aggregate written at scrape time), or
+     * null when not attempted.
+     */
+    get imageScanMobile(): number | null;
+    /**
+     * The reason this page was skipped during crawling, or null if it was not skipped.
+     */
+    get skipReason(): string | null;
+    /**
+     * The HTTP response status code, or null if the page has not been fetched.
+     */
+    get status(): number | null;
+    /**
+     * The HTTP response status text (e.g., `"OK"`, `"Not Found"`), or null if not fetched.
+     */
+    get statusText(): string | null;
+    /**
+     * Number of Wappalyzer tag entries detected on this page (denormalised
+     * aggregate written at scrape time).
+     */
+    get tagCount(): number | null;
+    /**
+     * Sorted unique Wappalyzer provider names, comma-separated, empty string
+     * when no tags. Denormalised aggregate; for the structured form (with
+     * confidence, category, and every raw signal) fetch
+     * {@link Page.getTechnologySignals} / {@link Page.getPageTechnologies} (lazy).
+     */
+    get tagsProvidersCsv(): string;
+    /**
+     * The page title from the `<title>` element.
+     * Returns an empty string if no title is set.
+     */
+    get title(): string;
+    /**
+     * Twitter Card type (`twitter:card`), or null if not present.
+     */
+    get twitter_card(): string | null;
+    /**
+     * The parsed URL of this page as an ExURL object.
+     * Respects the `disableQueries` option for query string handling.
+     */
+    get url(): import("@d-zero/shared/parse-url").ExURL;
+    /**
+     * Creates a new Page instance.
+     * @param archive - The ArchiveAccessor used for lazy-loading relationships.
+     * @param raw - The raw database row for this page.
+     * @param rawRedirects - Pre-loaded redirect records, or undefined for lazy loading.
+     * @param rawAnchors - Pre-loaded anchor records, or undefined for lazy loading.
+     * @param rawReferrers - Pre-loaded referrer records, or undefined for lazy loading.
+     * @param disableQueries - Whether to strip query strings from the URL.
+     */
+    constructor(archive: ArchiveAccessor, raw: DB_Page, rawRedirects?: DB_Redirect[], rawAnchors?: DB_Anchor[], rawReferrers?: DB_Referrer[], disableQueries?: boolean);
+    /**
+     * Retrieves the anchors (outgoing links) found on this page.
+     * Uses pre-loaded data if available, otherwise queries the database.
+     * @returns An array of {@link Anchor} objects representing the links on this page.
+     */
+    getAnchors(): Promise<Anchor[]>;
+    /**
+     * Retrieves the audios within this page's detected main content region
+     * from `page_main_content_audios`. Lazy — runs a single SELECT per call.
+     * @returns Ordered audio rows.
+     */
+    getAudios(): Promise<import("./meta/types.js").MainContentAudioRow[]>;
+    /**
+     * Retrieves the button-like elements within this page's detected main
+     * content region from `page_main_content_buttons`. Lazy — runs a single
+     * SELECT per call.
+     * @returns Ordered button rows.
+     */
+    getButtons(): Promise<import("./meta/types.js").MainContentButtonRow[]>;
+    /**
+     * Retrieves the canvases within this page's detected main content region
+     * from `page_main_content_canvases`. Lazy — runs a single SELECT per call.
+     * @returns Ordered canvas rows.
+     */
+    getCanvases(): Promise<import("./meta/types.js").MainContentCanvasRow[]>;
+    /**
+     * Retrieves the Web Components (custom elements) within this page's
+     * detected main content region from `page_main_content_custom_elements`.
+     * Lazy — runs a single SELECT per call.
+     * @returns Ordered custom-element rows.
+     */
+    getCustomElements(): Promise<import("./meta/types.js").MainContentCustomElementRow[]>;
+    /**
+     * Retrieves the headings within this page's detected main content region
+     * from `page_main_content_headings`. Lazy — runs a single SELECT per call.
+     * @returns Ordered heading rows.
+     */
+    getHeadings(): Promise<import("./meta/types.js").MainContentHeadingRow[]>;
+    /**
+     * Thin wrapper that forwards this page's id to the accessor's
+     * BLOB-read path. Lets callers hold a `Page` reference and ask for its
+     * HTML without having to thread the page id through.
+     * @returns The HTML content, or `null` if no snapshot was saved.
+     * @see {@link ArchiveAccessor.getHtmlOfPage} for the resolution rules.
+     * @example
+     * for (const page of await archive.getPages()) {
+     *   const html = await page.getHtml();
+     *   if (html !== null) processBody(html);
+     * }
+     */
+    getHtml(): Promise<string | null>;
+    /**
+     * Retrieves the iframes within this page's detected main content region
+     * from `page_main_content_iframes`. Lazy — runs a single SELECT per call.
+     * @returns Ordered iframe rows.
+     */
+    getIframes(): Promise<import("./meta/types.js").MainContentIframeRow[]>;
+    /**
+     * Retrieves the JSON-LD entries for this page from `page_jsonld`.
+     * Lazy — runs a single SELECT per call. Returns entries in insertion
+     * order (matches the scraper's traversal order).
+     * @returns Ordered JSON-LD / SpeculationRules rows.
+     */
+    getJsonLd(): Promise<readonly JsonLdRow[]>;
+    /**
+     * Retrieves the images within this page's detected main content region
+     * from `page_main_content_images`. Lazy — runs a single SELECT per call.
+     * Distinct from the whole-page image scan (`image_items`).
+     * @returns Ordered image rows.
+     */
+    getMainContentImages(): Promise<import("./meta/types.js").MainContentImageRow[]>;
+    /**
+     * Retrieves the tables within this page's detected main content region
+     * from `page_main_content_tables`. Lazy — runs a single SELECT per call.
+     * @returns Ordered table rows.
+     */
+    getMainContentTables(): Promise<import("./meta/types.js").MainContentTableRow[]>;
+    /**
+     * Retrieves the confidence-combined technology roll-up for this page
+     * from `page_technologies`. Lazy — runs a single SELECT per call.
+     * @returns Technology rows, highest confidence first.
+     */
+    getPageTechnologies(): Promise<readonly PageTechnologyRow[]>;
+    /**
+     * Retrieves the referrers (incoming links) pointing to this page.
+     * Uses pre-loaded data if available, otherwise queries the database.
+     * @returns An array of {@link Referrer} objects representing pages that link to this page.
+     */
+    getReferrers(): Promise<Referrer[]>;
+    /**
+     * Retrieves all request referrers for this page directly from the database.
+     * Unlike {@link getReferrers}, this always queries the database and does not use pre-loaded data.
+     * @returns An array of {@link Referrer} objects.
+     */
+    getRequests(): Promise<Referrer[]>;
+    /**
+     * Retrieves the raw technology-detection signals for this page from
+     * `technology_signals`. Lazy — runs a single SELECT per call.
+     * @returns Ordered signal rows.
+     */
+    getTechnologySignals(): Promise<readonly TechnologySignalRow[]>;
+    /**
+     * Retrieves the videos within this page's detected main content region
+     * from `page_main_content_videos`. Lazy — runs a single SELECT per call.
+     * @returns Ordered video rows.
+     */
+    getVideos(): Promise<import("./meta/types.js").MainContentVideoRow[]>;
+    /**
+     * Checks whether this page is an internal HTML page (not external and has `text/html` content type).
+     * @returns `true` if this is an internal HTML page, `false` otherwise.
+     */
+    isInternalPage(): boolean;
+    /**
+     * Checks whether this entry represents an HTML page (content type is `text/html`).
+     * @returns `true` if the content type is `text/html`, `false` otherwise.
+     */
+    isPage(): boolean;
+    /**
+     * Serializes the page data to a plain JSON object including the full flat
+     * meta column set, the `meta_extras` catch-all, a **summary** of the
+     * JSON-LD rows, and the confidence-combined technology roll-up
+     * (`page_technologies` — already compact, one row per detected
+     * technology, so no further summarization is needed the way `page_tags`'
+     * per-`externalId` granularity used to require).
+     *
+     * Summaries (not raw entries) are inlined so a Page detail payload stays
+     * token-bounded for MCP / LLM consumers — the full `raw` JSON-LD payload
+     * is fetched separately via `getJsonLd()` / the dedicated CLI/MCP
+     * endpoints.
+     *
+     * Anchors and referrers are still resolved eagerly because consumers
+     * (Sheets `eachPage`, viewer detail) depend on having them inline.
+     * @returns A plain object containing all page metadata and relationships.
+     */
+    toJSON(): Promise<{
+        metaExtras: Record<string, unknown>;
+        jsonLd: JsonLdSummary;
+        technologies: readonly PageTechnologyRow[];
+        redirectFrom: Redirect[];
+        isPage: boolean;
+        isInternalPage: boolean;
+        getAnchors: Anchor[];
+        getReferrers: Referrer[];
+        title: string | number | null;
+        main_content_word_count: string | number | null;
+        main_content_node_name: string | number | null;
+        main_content_id: string | number | null;
+        main_content_role: string | number | null;
+        main_content_selector: string | number | null;
+        main_content_class_list: string | number | null;
+        main_content_body_word_count: string | number | null;
+        main_content_heading_count: string | number | null;
+        main_content_image_count: string | number | null;
+        main_content_table_count: string | number | null;
+        main_content_button_count: string | number | null;
+        main_content_iframe_count: string | number | null;
+        main_content_video_count: string | number | null;
+        main_content_audio_count: string | number | null;
+        main_content_canvas_count: string | number | null;
+        scroll_height_desktop: string | number | null;
+        scroll_height_mobile: string | number | null;
+        lang: string | number | null;
+        dir: string | number | null;
+        charset: string | number | null;
+        baseHref: string | number | null;
+        viewport_raw: string | number | null;
+        themeColor: string | number | null;
+        applicationName: string | number | null;
+        author: string | number | null;
+        generator: string | number | null;
+        publisher: string | number | null;
+        description: string | number | null;
+        keywords: string | number | null;
+        robots_raw: string | number | null;
+        robots_noindex: string | number | null;
+        robots_nofollow: string | number | null;
+        robots_noarchive: string | number | null;
+        robots_noimageindex: string | number | null;
+        googlebot: string | number | null;
+        canonical: string | number | null;
+        amphtml: string | number | null;
+        manifest: string | number | null;
+        icon_href: string | number | null;
+        appleTouchIcon_href: string | number | null;
+        og_type: string | number | null;
+        og_title: string | number | null;
+        og_url: string | number | null;
+        og_site_name: string | number | null;
+        og_description: string | number | null;
+        og_image: string | number | null;
+        og_image_alt: string | number | null;
+        og_image_width: string | number | null;
+        og_image_height: string | number | null;
+        og_locale: string | number | null;
+        og_article_published_time: string | number | null;
+        og_article_modified_time: string | number | null;
+        twitter_card: string | number | null;
+        twitter_site: string | number | null;
+        twitter_creator: string | number | null;
+        twitter_title: string | number | null;
+        twitter_description: string | number | null;
+        twitter_image: string | number | null;
+        fb_app_id: string | number | null;
+        verification_google: string | number | null;
+        formatDetection_telephone: string | number | null;
+        tag_count: string | number | null;
+        jsonld_count: string | number | null;
+        tags_providers_csv: string | number | null;
+        url: string;
+        status: number | null;
+        statusText: string | null;
+        contentType: string | null;
+        contentLength: number | null;
+        responseHeaders: Record<string, string | string[] | undefined>;
+        isExternal: boolean;
+        isSkipped: boolean;
+        skipReason: string | null;
+        isTarget: boolean;
+        firstCrawledAt: number | null;
+        lastCrawledAt: number | null;
+    }>;
+}
+/**
+ * Utility type that extracts the resolved type from a Promise.
+ */
+type PromiseType<T> = T extends PromiseLike<infer U> ? U : T;
+/**
+ * The static (serialized) representation of a Page, as returned by {@link Page.toJSON}.
+ */
+export type StaticPageData = PromiseType<ReturnType<Page['toJSON']>>;
+export {};