page-markdown 0.1.5 → 0.3.0

package/dist/types/index.d.ts

@@ -3,10 +3,14 @@
  * @property {string} [url] page URL for absolute links (default: the document's URL)
  * @property {"full"|"no_links"|"text_only"} [mode="full"]
  * @property {"readable"|"full"|import("./extract.js").Extractor} [extract="readable"]
- *   "readable": Readability with full-page fallback; "full": the whole body;
+ *   "readable": Readability with main-content or full-page fallback; "full": the whole body;
  *   or a custom extractor receiving a copy of the standardized document.
- * @property {number} [minExtractChars=100] below this, fall back to the full page
+ * @property {number} [minExtractChars=100] below this, fall back to the page's <main>
+ *   if it marks one and it holds enough, else to the full page
+ * @property {number} [minExtractShare=0.4] when Readability keeps less than this
+ *   share of the page's main content, use the main content instead; 0 turns it off
  * @property {boolean} [includeFullPage=false] also return the full-page Markdown
+ * @property {boolean} [detectConcealedText=false] opt-in live computed-style check, default false
  * @property {boolean} [includeTitle=true] start readable output with "# <headline>"
  * @property {boolean} [recoverLead=true] restore a dropped lead paragraph that matches the meta description
  * @property {boolean} [pageFacts=false] append "## Page facts": short label/value pairs
@@ -27,7 +31,7 @@
  * @returns {import("./types.js").PageMarkdownResult}
  */
 export function pageToMarkdown(input: Document | Element, options?: PageMarkdownOptions): import("./types.js").PageMarkdownResult;
-export const VERSION: "0.1.5";
+export const VERSION: "0.3.0";
 export { standardizeDocument } from "./standardize.js";
 export { collectMetadata } from "./metadata.js";
 export * from "./types.js";
@@ -38,18 +42,28 @@ export type PageMarkdownOptions = {
     url?: string | undefined;
     mode?: "full" | "no_links" | "text_only" | undefined;
     /**
-     * "readable": Readability with full-page fallback; "full": the whole body;
+     * "readable": Readability with main-content or full-page fallback; "full": the whole body;
      * or a custom extractor receiving a copy of the standardized document.
      */
     extract?: import("./extract.js").Extractor | "readable" | "full" | undefined;
     /**
-     * below this, fall back to the full page
+     * below this, fall back to the page's <main>
+     * if it marks one and it holds enough, else to the full page
      */
     minExtractChars?: number | undefined;
+    /**
+     * when Readability keeps less than this
+     * share of the page's main content, use the main content instead; 0 turns it off
+     */
+    minExtractShare?: number | undefined;
     /**
      * also return the full-page Markdown
      */
     includeFullPage?: boolean | undefined;
+    /**
+     * opt-in live computed-style check, default false
+     */
+    detectConcealedText?: boolean | undefined;
     /**
      * start readable output with "# <headline>"
      */
@@ -90,7 +104,7 @@ export type PageMarkdownOptions = {
     } | undefined;
 };
 export { sanitizeToInertDocument, DEFAULT_SANITIZE_OPTIONS, isAdvertisementElement, createInertDocument } from "./sanitize.js";
-export { extractContent, DEFAULT_MIN_EXTRACT_CHARS } from "./extract.js";
+export { extractContent, DEFAULT_MIN_EXTRACT_CHARS, DEFAULT_MIN_EXTRACT_SHARE } from "./extract.js";
 export { htmlToMarkdown, createTurndown, TURNDOWN_OPTIONS } from "./convert.js";
 export { cleanMarkdown, applyMode, mapOutsideFences } from "./clean.js";
 export { recoverLeadParagraph, chooseHeadline } from "./refine.js";

package/dist/types/sanitize.d.ts

@@ -9,18 +9,37 @@ export function createInertDocument(doc: Document): Document;
 /**
  * Clones `root` (a Document or Element) into a new inert document.
  * Open shadow roots replace their host's light children (native slots supply
- * assigned children). Limits apply only to shadow content; a host over the
- * limit is emptied and counted as an omission.
+ * assigned children). Limits apply only to shadow content, per host in the
+ * page's own tree, so slotted page content is not charged: docs sites put a
+ * whole article in one host's slot. A host over the limit keeps what fits and
+ * is counted as an omission.
  *
  * @param {Document|Element} root
  * @param {import("./types.js").SanitizeOptions} [options]
- * @returns {{ document: Document, omissions: { shadowHosts: number } }}
  */
 export function sanitizeToInertDocument(root: Document | Element, options?: import("./types.js").SanitizeOptions): {
     document: Document;
     omissions: {
         shadowHosts: number;
     };
+    concealment: {
+        process: (el: any, copy: any, work: any, pairs: any) => void;
+        initial: (source: any) => any;
+        inherit: (work: any, child: any, clone: any) => any;
+        exempt: (el: any) => void;
+        lift: (inert: any) => void;
+        entries: import("./types.js").ConcealedText[];
+        summary: {
+            count: number;
+            chars: number;
+            truncated: boolean;
+        };
+        readonly counters: {
+            [k: string]: number;
+        };
+        readonly check: import("./types.js").ConcealmentCheck;
+        readonly pendingOpacity: number;
+    };
 };
 /** @type {Readonly<Required<import("./types.js").SanitizeOptions>>} */
 export const DEFAULT_SANITIZE_OPTIONS: Readonly<Required<import("./types.js").SanitizeOptions>>;

package/dist/types/standardize.d.ts

@@ -1,15 +1,29 @@
+/**
+ * Tabs read as one document: inactive panels are shown, each under its tab's
+ * name, in place of a tab strip that points nowhere. A long inactive panel is
+ * left out, and named in stats.omittedTabs.
+ */
+export function prepareConcealment(doc: any, concealment: any): {
+    panels: any[];
+    hasProvenance: any;
+    inactive: Map<any, boolean>;
+    panelLengths: Map<any, any>;
+};
 /**
  * @param {Document} doc the inert clone (mutated in place)
- * @param {{ url?: string }} [options]
+ * @param {{ url?: string, concealment?: ReturnType<typeof import('./concealed.js').createConcealmentWalk> }} [options]
  */
 export function standardizeDocument(doc: Document, options?: {
     url?: string;
+    concealment?: ReturnType<typeof import("./concealed.js").createConcealmentWalk>;
 }): {
     lazyImagesResolved: number;
     inlineImagesDropped: number;
     scriptLinksUnwrapped: number;
+    invalidLinksUnwrapped: number;
     emptyLinksRemoved: number;
     preWrapped: number;
     headingWidgetsRemoved: number;
     headingsUnwrapped: number;
+    omittedTabs: string[];
 };

package/dist/types/types.d.ts

@@ -7,6 +7,10 @@ export type SanitizeOptions = {
     stripMeta?: boolean | undefined;
     stripLinkTags?: boolean | undefined;
     stripAds?: boolean | undefined;
+    /**
+     * opt-in computed-style check, default false
+     */
+    detectConcealedText?: boolean | undefined;
     /**
      * app-specific overlays or widgets to drop, e.g. "[data-app-overlay]"
      */
@@ -37,7 +41,7 @@ export type PageMarkdownMetadata = CollectedMetadata & ExtractedMetadata;
 export type PageMarkdownExtraction = {
     method: Extraction["method"];
     /**
-     * why extraction fell back to the full page
+     * why extraction fell back to the full page or the main content
      */
     fallbackReason?: string | undefined;
     /**
@@ -97,7 +101,8 @@ export type HiddenTextOmission = {
     note: string;
 };
 /**
- * Check `kind` before reading `chars`: only hidden-text omissions have it.
+ * Check `kind` before reading `chars`: hidden Unicode omissions use code points;
+ * concealed_text uses UTF-16 units. Other kinds do not have it.
  */
 export type PageMarkdownOmission = {
     kind: "shadow_content";
@@ -107,6 +112,19 @@ export type PageMarkdownOmission = {
     kind: "json_line";
     count: number;
     note: string;
+} | {
+    kind: "inactive_tabs";
+    count: number;
+    note: string;
+} | {
+    kind: "comments";
+    count: number;
+    note: string;
+} | {
+    kind: "concealed_text";
+    count: number;
+    chars: number;
+    note: string;
 } | HiddenTextOmission;
 export type StandardizationStats = ReturnType<typeof import("./standardize.js").standardizeDocument>;
 export type PageMarkdownStats = {
@@ -114,6 +132,11 @@ export type PageMarkdownStats = {
     extractedHtmlChars: number;
     convertedChars: number;
     markdownChars: number;
+    /**
+     * the page's text at capture time: a capture is a
+     * snapshot of the DOM, so a caller can re-capture until this number settles
+     */
+    pageTextChars: number;
     standardization: StandardizationStats;
 };
 export type PageMarkdownResult = {
@@ -121,6 +144,18 @@ export type PageMarkdownResult = {
      * the package version that made this result
      */
     version: string;
+    /**
+     * capped captured blocks, raw text in UTF-16 units
+     */
+    concealedText: ConcealedText[];
+    /**
+     * source computed-style capability
+     */
+    concealmentCheck: ConcealmentCheck;
+    /**
+     * true if any block, text or entry cap omitted report text
+     */
+    concealedTextTruncated: boolean;
     markdown: string;
     metadata: PageMarkdownMetadata;
     extraction: PageMarkdownExtraction;
@@ -136,6 +171,20 @@ export type PageFact = {
     label: string;
     value: string;
 };
+/**
+ * The one-time probe establishes that the document's view can return every
+ * required field and exposed optional field. Afterwards, incomplete means a
+ * CONSULTED value was empty or threw; unread irrelevant values are not checked.
+ * Computed means no such failure; static-only means no computed-style API;
+ * disabled means detection was turned off.
+ */
+export type ConcealmentCheck = "computed" | "static-only" | "disabled" | "incomplete";
+export type ConcealmentReason = "display_none" | "visibility_hidden" | "opacity_zero" | "font_size_zero" | "clipped" | "transparent_text" | "same_color";
+export type ConcealedText = {
+    reason: ConcealmentReason;
+    chars: number;
+    text: string;
+};
 export type TurndownFilter = string | string[] | ((node: HTMLElement, options: Record<string, unknown>) => boolean);
 /**
  * Turndown's public methods; Turndown ships no types of its own.

package/package.json

@@ -1,68 +1,69 @@
-{
-  "name": "page-markdown",
-  "version": "0.1.5",
-  "description": "Faithful page-to-Markdown for humans and agents: sanitize, extract the readable article, convert with GFM, and clean without losing content.",
-  "type": "module",
-  "exports": {
-    ".": {
-      "types": "./dist/types/index.d.ts",
-      "default": "./src/index.js"
-    }
-  },
-  "types": "./dist/types/index.d.ts",
-  "files": [
-    "src",
-    "dist/types",
-    "LICENSE",
-    "ADDITIONAL_TERMS.md",
-    "THIRD_PARTY_NOTICES.md",
-    "README.md",
-    "CHANGELOG.md",
-    "robots.txt"
-  ],
-  "sideEffects": false,
-  "packageManager": "pnpm@10.33.0",
-  "engines": {
-    "node": ">=20"
-  },
-  "scripts": {
-    "test": "node --test \"test/**/*.test.mjs\"",
-    "build:types": "tsc -p tsconfig.json",
-    "typecheck": "npm run build:types && node test/types/real-output.mjs && tsc -p test/types && tsc -p test/types/linkedom",
-    "size": "node bench/size.mjs",
-    "test:browser": "node test/browser/run.mjs",
-    "ci": "npm test && npm run typecheck && npm run test:browser && npm run size",
-    "prepack": "npm run build:types",
-    "prepublishOnly": "npm run ci",
-    "bench": "node bench/compare.mjs",
-    "bench:write": "node bench/compare.mjs --write"
-  },
-  "keywords": [
-    "markdown",
-    "readability",
-    "html-to-markdown",
-    "reader-mode",
-    "web-clipper",
-    "browser-extension"
-  ],
-  "author": "WATT3D <WATT3D@protonmail.com>",
-  "license": "SEE LICENSE IN LICENSE",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/iWhatty/page-markdown.git"
-  },
-  "homepage": "https://github.com/iWhatty/page-markdown#readme",
-  "bugs": {
-    "url": "https://github.com/iWhatty/page-markdown/issues"
-  },
-  "dependencies": {
-    "@mozilla/readability": "0.6.0",
-    "text-sanctifier": "1.3.8",
-    "turndown": "7.2.4"
-  },
-  "devDependencies": {
-    "esbuild": "0.27.0",
-    "linkedom": "0.18.12",
-    "typescript": "5.9.3"
-  }
-}
+{
+  "name": "page-markdown",
+  "version": "0.3.0",
+  "description": "Faithful page-to-Markdown for humans and agents: sanitize, extract the readable article, convert with GFM, and clean without losing content.",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./src/index.js"
+    }
+  },
+  "types": "./dist/types/index.d.ts",
+  "files": [
+    "src",
+    "dist/types",
+    "LICENSE",
+    "ADDITIONAL_TERMS.md",
+    "THIRD_PARTY_NOTICES.md",
+    "README.md",
+    "CHANGELOG.md",
+    "robots.txt"
+  ],
+  "sideEffects": false,
+  "packageManager": "pnpm@10.33.0",
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "test": "node --test \"test/**/*.test.mjs\"",
+    "build:types": "tsc -p tsconfig.json",
+    "typecheck": "npm run build:types && node test/types/real-output.mjs && tsc -p test/types && tsc -p test/types/linkedom",
+    "size": "node bench/size.mjs",
+    "test:browser": "node test/browser/run.mjs",
+    "report:build": "node test/browser/build-report.mjs",
+    "ci": "npm test && npm run typecheck && npm run size",
+    "prepack": "npm run build:types",
+    "prepublishOnly": "npm run ci",
+    "bench": "node bench/compare.mjs",
+    "bench:write": "node bench/compare.mjs --write"
+  },
+  "keywords": [
+    "markdown",
+    "readability",
+    "html-to-markdown",
+    "reader-mode",
+    "web-clipper",
+    "browser-extension"
+  ],
+  "author": "WATT3D <WATT3D@protonmail.com>",
+  "license": "SEE LICENSE IN LICENSE",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/iWhatty/page-markdown.git"
+  },
+  "homepage": "https://github.com/iWhatty/page-markdown#readme",
+  "bugs": {
+    "url": "https://github.com/iWhatty/page-markdown/issues"
+  },
+  "dependencies": {
+    "@mozilla/readability": "0.6.0",
+    "text-sanctifier": "1.3.8",
+    "turndown": "7.2.4"
+  },
+  "devDependencies": {
+    "esbuild": "0.27.0",
+    "linkedom": "0.18.12",
+    "typescript": "5.9.3"
+  }
+}

package/robots.txt

@@ -1,79 +1,79 @@
-# WATT3D — AI scraping policy
-#
-# This file enumerates known AI / LLM training and inference crawlers
-# that the WATT3D AI Training Rider does not consent to. See
-# ./ADDITIONAL_TERMS.md for the legal scope and definitions.
-#
-# Honoring robots.txt is voluntary. The rider applies whether or not
-# a bot respects this file. The enumeration is non-exhaustive — bots
-# not listed here are still bound by the rider if they perform
-# Training Use as defined in ADDITIONAL_TERMS.md §1.
-
-User-agent: GPTBot
-Disallow: /
-
-User-agent: ChatGPT-User
-Disallow: /
-
-User-agent: OAI-SearchBot
-Disallow: /
-
-User-agent: anthropic-ai
-Disallow: /
-
-User-agent: ClaudeBot
-Disallow: /
-
-User-agent: Claude-Web
-Disallow: /
-
-User-agent: Claude-SearchBot
-Disallow: /
-
-User-agent: Google-Extended
-Disallow: /
-
-User-agent: Applebot-Extended
-Disallow: /
-
-User-agent: Meta-ExternalAgent
-Disallow: /
-
-User-agent: Meta-ExternalFetcher
-Disallow: /
-
-User-agent: FacebookBot
-Disallow: /
-
-User-agent: CCBot
-Disallow: /
-
-User-agent: PerplexityBot
-Disallow: /
-
-User-agent: Perplexity-User
-Disallow: /
-
-User-agent: cohere-ai
-Disallow: /
-
-User-agent: Bytespider
-Disallow: /
-
-User-agent: Amazonbot
-Disallow: /
-
-User-agent: Omgilibot
-Disallow: /
-
-User-agent: AI2Bot
-Disallow: /
-
-User-agent: Diffbot
-Disallow: /
-
-# Everyone else: full access. Search engines and human-driven crawlers
-# remain welcome.
-
-User-agent: *
-Allow: /
+# WATT3D — AI scraping policy
+#
+# This file enumerates known AI / LLM training and inference crawlers
+# that the WATT3D AI Training Rider does not consent to. See
+# ./ADDITIONAL_TERMS.md for the legal scope and definitions.
+#
+# Honoring robots.txt is voluntary. The rider applies whether or not
+# a bot respects this file. The enumeration is non-exhaustive — bots
+# not listed here are still bound by the rider if they perform
+# Training Use as defined in ADDITIONAL_TERMS.md §1.
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: OAI-SearchBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: Claude-Web
+Disallow: /
+
+User-agent: Claude-SearchBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: Meta-ExternalAgent
+Disallow: /
+
+User-agent: Meta-ExternalFetcher
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: Perplexity-User
+Disallow: /
+
+User-agent: cohere-ai
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-agent: AI2Bot
+Disallow: /
+
+User-agent: Diffbot
+Disallow: /
+
+# Everyone else: full access. Search engines and human-driven crawlers
+# remain welcome.
+
+User-agent: *
+Allow: /

package/src/clean.js

@@ -103,7 +103,7 @@ const stripEmphasis = (md) => replaceOutsideCode(
   /(?<![\w*])([*_])(?!\s)(.+?)(?<!\s)\1(?![\w*])/,
   (all, mark, text) => text,
 );
-const stripImages = (md) => replaceOutsideCode(md, /!\[([^\]]*)\]\([^)]*\)/, (all, alt) => alt);
+const stripImages = (md) => replaceOutsideCode(md, INLINE_LINK, (all, bang, text) => text);
 
 /**
  * Output modes. "no_links" keeps link text. "text_only" also drops headings,