@nitpicker/crawler 0.13.0 → 0.14.0

package/lib/archive/page.js

@@ -60,6 +60,23 @@ const FLAT_META_COLUMNS = [
     'tag_count',
     'jsonld_count',
     'tags_providers_csv',
+    'main_content_node_name',
+    'main_content_id',
+    'main_content_role',
+    'main_content_selector',
+    'main_content_class_list',
+    'main_content_word_count',
+    'main_content_body_word_count',
+    'main_content_heading_count',
+    'main_content_image_count',
+    'main_content_table_count',
+    'main_content_button_count',
+    'main_content_iframe_count',
+    'main_content_video_count',
+    'main_content_audio_count',
+    'main_content_canvas_count',
+    'scroll_height_desktop',
+    'scroll_height_mobile',
 ];
 /**
  * Represents a crawled page stored in the archive.
@@ -171,6 +188,104 @@ export default class Page {
     get lastCrawledAt() {
         return this.#raw.lastCrawledAt;
     }
+    /**
+     * Detected main-content element's `nodeName` (e.g. `'MAIN'`), or null when
+     * no main region was found or the page was not fully rendered.
+     */
+    get mainContentNodeName() {
+        return this.#raw.main_content_node_name;
+    }
+    /**
+     * Detected main-content element's `id`, or null.
+     */
+    get mainContentId() {
+        return this.#raw.main_content_id;
+    }
+    /**
+     * Detected main-content element's `role` attribute, or null.
+     */
+    get mainContentRole() {
+        return this.#raw.main_content_role;
+    }
+    /**
+     * Diagnostic tag+id+class selector for the detected main-content element, or null.
+     */
+    get mainContentSelector() {
+        return this.#raw.main_content_selector;
+    }
+    /**
+     * Detected main-content element's CSS classes, or null when no main region
+     * was found. Parsed from the JSON-encoded `main_content_class_list` column.
+     */
+    get mainContentClassList() {
+        if (this.#raw.main_content_class_list === null) {
+            return null;
+        }
+        return JSON.parse(this.#raw.main_content_class_list);
+    }
+    /**
+     * Character count of the main region's text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentWordCount() {
+        return this.#raw.main_content_word_count;
+    }
+    /**
+     * Character count of `document.body`'s text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentBodyWordCount() {
+        return this.#raw.main_content_body_word_count;
+    }
+    /**
+     * Number of headings within the main region (denormalised aggregate), or null.
+     */
+    get mainContentHeadingCount() {
+        return this.#raw.main_content_heading_count;
+    }
+    /**
+     * Number of images within the main region (denormalised aggregate), or null.
+     */
+    get mainContentImageCount() {
+        return this.#raw.main_content_image_count;
+    }
+    /**
+     * Number of tables within the main region (denormalised aggregate), or null.
+     */
+    get mainContentTableCount() {
+        return this.#raw.main_content_table_count;
+    }
+    /**
+     * Number of button-like elements within the main region (denormalised
+     * aggregate), or null.
+     */
+    get mainContentButtonCount() {
+        return this.#raw.main_content_button_count;
+    }
+    /**
+     * Number of iframes within the main region (denormalised aggregate), or null.
+     */
+    get mainContentIframeCount() {
+        return this.#raw.main_content_iframe_count;
+    }
+    /**
+     * Number of videos within the main region (denormalised aggregate), or null.
+     */
+    get mainContentVideoCount() {
+        return this.#raw.main_content_video_count;
+    }
+    /**
+     * Number of audios within the main region (denormalised aggregate), or null.
+     */
+    get mainContentAudioCount() {
+        return this.#raw.main_content_audio_count;
+    }
+    /**
+     * Number of canvases within the main region (denormalised aggregate), or null.
+     */
+    get mainContentCanvasCount() {
+        return this.#raw.main_content_canvas_count;
+    }
     /**
      * Iterable view over every flat meta column (~47 fields). Returns a frozen
      * record so consumers can pick fields by name without re-enumerating
@@ -280,6 +395,20 @@ export default class Page {
     get robots_raw() {
         return this.#raw.robots_raw;
     }
+    /**
+     * `document.body.scrollHeight` at the desktop-compact preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightDesktop() {
+        return this.#raw.scroll_height_desktop;
+    }
+    /**
+     * `document.body.scrollHeight` at the mobile-small preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightMobile() {
+        return this.#raw.scroll_height_mobile;
+    }
     /**
      * The reason this page was skipped during crawling, or null if it was not skipped.
      */
@@ -376,6 +505,39 @@ export default class Page {
         }
         return this.#archive.getAnchorsOnPage(this.#raw.id);
     }
+    /**
+     * Retrieves the audios within this page's detected main content region
+     * from `page_main_content_audios`. Lazy — runs a single SELECT per call.
+     * @returns Ordered audio rows.
+     */
+    async getAudios() {
+        return this.#archive.getAudiosOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves the button-like elements within this page's detected main
+     * content region from `page_main_content_buttons`. Lazy — runs a single
+     * SELECT per call.
+     * @returns Ordered button rows.
+     */
+    async getButtons() {
+        return this.#archive.getButtonsOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves the canvases within this page's detected main content region
+     * from `page_main_content_canvases`. Lazy — runs a single SELECT per call.
+     * @returns Ordered canvas rows.
+     */
+    async getCanvases() {
+        return this.#archive.getCanvasesOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves the headings within this page's detected main content region
+     * from `page_main_content_headings`. Lazy — runs a single SELECT per call.
+     * @returns Ordered heading rows.
+     */
+    async getHeadings() {
+        return this.#archive.getHeadingsOfPage(this.#raw.id);
+    }
     /**
      * Thin wrapper that forwards this page's id to the accessor's
      * BLOB-read path. Lets callers hold a `Page` reference and ask for its
@@ -391,6 +553,14 @@ export default class Page {
     async getHtml() {
         return this.#archive.getHtmlOfPage(this.#raw.id);
     }
+    /**
+     * Retrieves the iframes within this page's detected main content region
+     * from `page_main_content_iframes`. Lazy — runs a single SELECT per call.
+     * @returns Ordered iframe rows.
+     */
+    async getIframes() {
+        return this.#archive.getIframesOfPage(this.#raw.id);
+    }
     /**
      * Retrieves the JSON-LD entries for this page from `page_jsonld`.
      * Lazy — runs a single SELECT per call. Returns entries in insertion
@@ -400,6 +570,23 @@ export default class Page {
     async getJsonLd() {
         return this.#archive.getJsonLdOfPage(this.#raw.id);
     }
+    /**
+     * Retrieves the images within this page's detected main content region
+     * from `page_main_content_images`. Lazy — runs a single SELECT per call.
+     * Distinct from the whole-page image scan (`image_items`).
+     * @returns Ordered image rows.
+     */
+    async getMainContentImages() {
+        return this.#archive.getMainContentImagesOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves the tables within this page's detected main content region
+     * from `page_main_content_tables`. Lazy — runs a single SELECT per call.
+     * @returns Ordered table rows.
+     */
+    async getMainContentTables() {
+        return this.#archive.getMainContentTablesOfPage(this.#raw.id);
+    }
     /**
      * Retrieves the referrers (incoming links) pointing to this page.
      * Uses pre-loaded data if available, otherwise queries the database.
@@ -447,6 +634,14 @@ export default class Page {
     async getTags() {
         return this.#archive.getTagsOfPage(this.#raw.id);
     }
+    /**
+     * Retrieves the videos within this page's detected main content region
+     * from `page_main_content_videos`. Lazy — runs a single SELECT per call.
+     * @returns Ordered video rows.
+     */
+    async getVideos() {
+        return this.#archive.getVideosOfPage(this.#raw.id);
+    }
     /**
      * Checks whether this page is an internal HTML page (not external and has `text/html` content type).
      * @returns `true` if this is an internal HTML page, `false` otherwise.

package/lib/archive/types.d.ts

@@ -67,6 +67,11 @@ export interface Config extends Required<Pick<ParseURLOptions, 'disableQueries'>
     userAgent: string;
     /** Whether robots.txt restrictions were ignored during crawling. */
     ignoreRobots: boolean;
+    /**
+     * CSS selector overriding beholder's automatic main-content-region
+     * detection, or `null`/omitted to use the automatic heuristic.
+     */
+    mainContentSelector?: string | null;
 }
 /**
  * Provenance of a page or resource row — which crawler channel originally
@@ -292,6 +297,40 @@ export interface DB_Page {
     jsonld_count: number | null;
     /** Sorted unique provider names, comma-separated (empty string when none). */
     tags_providers_csv: string | null;
+    /** Detected main-content element's `nodeName` (e.g. `'MAIN'`), or null. */
+    main_content_node_name: string | null;
+    /** Detected main-content element's `id`, or null. */
+    main_content_id: string | null;
+    /** Detected main-content element's `role` attribute, or null. */
+    main_content_role: string | null;
+    /** Diagnostic tag+id+class selector for the detected element, or null. */
+    main_content_selector: string | null;
+    /** JSON-encoded array of the detected element's CSS classes, or null. */
+    main_content_class_list: string | null;
+    /** Character count of the main region's text content, or null. */
+    main_content_word_count: number | null;
+    /** Character count of `document.body`'s text content, or null. */
+    main_content_body_word_count: number | null;
+    /** Number of headings within the main region, or null. */
+    main_content_heading_count: number | null;
+    /** Number of images within the main region, or null. */
+    main_content_image_count: number | null;
+    /** Number of tables within the main region, or null. */
+    main_content_table_count: number | null;
+    /** Number of button-like elements within the main region, or null. */
+    main_content_button_count: number | null;
+    /** Number of iframes within the main region, or null. */
+    main_content_iframe_count: number | null;
+    /** Number of videos within the main region, or null. */
+    main_content_video_count: number | null;
+    /** Number of audios within the main region, or null. */
+    main_content_audio_count: number | null;
+    /** Number of canvases within the main region, or null. */
+    main_content_canvas_count: number | null;
+    /** `document.body.scrollHeight` at the desktop-compact preset, or null. */
+    scroll_height_desktop: number | null;
+    /** `document.body.scrollHeight` at the mobile-small preset, or null. */
+    scroll_height_mobile: number | null;
     /** JSON-serialised nested Meta sub-objects not flattened above. */
     meta_extras: string | null;
     /** JSON-serialized network logs captured during scraping, or null if not collected. */

package/lib/crawler/crawler.js

@@ -149,6 +149,7 @@ class Crawler extends EventEmitter {
             verbose: options?.verbose ?? false,
             userAgent: options?.userAgent || `Nitpicker/${pkg.version}`,
             ignoreRobots: options?.ignoreRobots ?? false,
+            mainContentSelector: options?.mainContentSelector ?? null,
             lookupResource: options?.lookupResource ?? null,
             lookupPageSource: options?.lookupPageSource ?? null,
             inventoryMode: options?.inventoryMode ?? null,
@@ -1355,8 +1356,8 @@ class Crawler extends EventEmitter {
             //    sub-resource requests issued from the same page —
             //    including cross-origin requests to a different hostname
             //    sharing the same IP / port (e.g. an embedded
-            //    `<img src="http://127.0.0.1:8010/…">` loaded from a
-            //    `localhost:8010` page) — get the cached `Authorization`
+            //    `<img src="http://127.0.0.1:PORT/…">` loaded from a
+            //    `localhost:PORT` page) — get the cached `Authorization`
             //    header re-attached by the network stack. The
             //    `Fetch.authRequired` event never fires for these
             //    pre-emptive attachments, so neither `page.authenticate`
@@ -1399,6 +1400,7 @@ class Crawler extends EventEmitter {
                 metadataOnly,
                 retries: this.#options.retry,
                 headCheckResult,
+                mainContentSelector: this.#options.mainContentSelector,
             });
             // Image dom-path capture runs here — after the scrape completed but
             // while `page` is still alive — because beholder's image metadata

package/lib/crawler/fetch-destination.js

@@ -196,6 +196,8 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent, t
                     imageList: [],
                     anchorList: [],
                     html: '',
+                    mainContents: null,
+                    scrollHeight: null,
                     isSkipped: false,
                 };
             };

package/lib/crawler/link-to-page-data.js

@@ -48,6 +48,8 @@ export function linkToPageData(link) {
         anchorList: [],
         imageList: [],
         html: '',
+        mainContents: null,
+        scrollHeight: null,
         isSkipped: false,
     };
 }

package/lib/crawler/resource-to-page-data.js

@@ -54,6 +54,8 @@ export function resourceToPageData(params) {
         anchorList: [],
         imageList: [],
         html: '',
+        mainContents: null,
+        scrollHeight: null,
         isSkipped: false,
     };
 }

package/lib/crawler/types.d.ts

@@ -129,6 +129,11 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
     userAgent: string;
     /** Whether to ignore robots.txt restrictions. */
     ignoreRobots: boolean;
+    /**
+     * CSS selector overriding beholder's automatic main-content-region
+     * detection, or `null`/undefined to use the automatic heuristic.
+     */
+    mainContentSelector?: string | null;
     /**
      * Lookup for previously captured sub-resources, or `null` to disable the
      * resource-reuse optimization. See {@link ResourceLookup}.

package/lib/crawler-orchestrator.js

@@ -109,6 +109,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             verbose: options?.verbose ?? false,
             userAgent: options?.userAgent || defaultUserAgent,
             ignoreRobots: options?.ignoreRobots ?? false,
+            mainContentSelector: options?.mainContentSelector ?? null,
             // Let the crawler reuse sub-resource data captured during page
             // rendering instead of issuing a redundant HEAD pre-flight.
             lookupResource: async (urls) => {
@@ -337,6 +338,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             disableQueries,
             userAgent: options?.userAgent || defaultUserAgent,
             ignoreRobots: options?.ignoreRobots ?? false,
+            mainContentSelector: options?.mainContentSelector ?? null,
         });
         const orchestrator = new CrawlerOrchestrator(archive, {
             ...options,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.13.0",
+	"version": "0.14.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,7 +27,7 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "3.1.5",
+		"@d-zero/beholder": "4.0.0",
 		"@d-zero/dealer": "1.9.4",
 		"@d-zero/fs": "0.2.6",
 		"@d-zero/shared": "0.22.2",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "2cd462ea6b183f4462b7caba27b4db04aaba7c70"
+	"gitHead": "98222c2bbce55ab6d253787a251f040b51c7a9a0"
 }