page-markdown 0.1.4 → 0.2.0

package/CHANGELOG.md

@@ -1,5 +1,154 @@
 # Changelog
 
+## 0.2.0 — 2026-11-07
+
+Fixes for silent content loss, found by live runs in a real extension (HELIOS, in
+Brave): pages lost their main content and reported a normal extraction.
+
+A minor version because results change in ways callers can see. `omissions` has two
+new kinds, `inactive_tabs` and `comments`, so an exhaustive `switch` on `kind` in
+TypeScript stops compiling. `extraction.method` can be `main_content` on pages that
+used to report `readability`, those pages come out longer, and some outputs gain a
+closing line.
+
+- **Threads and product grids keep their content.** When Readability keeps less
+  than `minExtractShare` (default 0.4) of the page's main content and misses at
+  least 500 characters, the main content is used, without navigation, asides and
+  forms. `extraction.method` is then `main_content`, `fallbackReason` is
+  `extracted_small_share_of_page`, and `warnings` says so. Word recall on a Lobsters
+  thread goes from 36% to 100%, and on an IKEA category page from 50% to 95%. These
+  pages keep more clutter: IKEA's precision falls from 92% to 30%.
+- **Layout class names no longer cost a block its place.** Readability's clutter
+  list matches "scroll" and "hidden" in Tailwind's `scroll-mt-2`, `overflow-hidden`
+  and `md:hidden`, and it removed what carried them: every code block in Docker's
+  install guide and the plan cards on Tailscale's pricing page. Proven live on both.
+- **Icon-only table cells say something.** A cell holding only an SVG came out
+  empty, so a comparison table couldn't say which plan has what. It now shows the
+  icon's accessible name, else `✓`, `–` or `✗` for those simple shapes, else
+  `[icon]`. On Tailscale, 114 ticks and 25 dashes where there were empty cells.
+- **More cases fall back to the main content,** and say so with their own
+  `fallbackReason`. `extracted_site_menu`: Readability chose a container above
+  `<main>` and kept the site's menu (BBC News). `extracted_few_sections_of_page`:
+  it kept under 1.5 times `minExtractShare` of the main content and lost most of the
+  page's h2 sections, as on Adafruit's product page, which lost its price, stock and
+  technical details. Headings that are a link to another page don't count, so story
+  cards around an article, as on The Register, don't trigger it. The product page
+  keeps more clutter this way, including its "May We Also Suggest" list. Proven live
+  on all three.
+- **The fallback keeps collapsed answers.** Hidden text of 40 characters or more
+  straight after a button or summary is kept, so an FAQ isn't lost when a page
+  hides its answers with inline styles. Proven live on Allbirds' FAQ.
+- **Tab panels hidden by a class name keep their text.** Readability removes
+  `class="hidden"`, so Docker's "Specific version" install tab was lost live with
+  no closing line. Inactive panels are now recognized by that class too, and Docker's
+  `aria-role` tab strips are read as tabs. Proven live on Docker.
+- **Shadow limits no longer erase a whole host.** A host over the limit keeps what
+  fits, limits apply per host in the page's own tree so a large site header can't
+  use up the article's budget, and `warnings` now reports the cut.
+- **Slotted page content no longer counts against the shadow limit.** Material Web
+  puts its whole article in one host's slot, so the limit cut it off partway
+  through. Proven live on Material Web, Adobe Spectrum and Red Hat (no cut), though
+  no live page has yet gone over the limit.
+- **Headings survive clutter-like names.** Readability removed headings named like
+  Docusaurus's `anchorTargetHideOnScrollNavbar`. Permalink text ("Section titled",
+  "Direct link to") goes too.
+- **Permalinks are recognised by where they point.** A link to a heading's own id,
+  inside or beside it, is removed whatever its class or wording. Short headings
+  beside one ("Types", "Usage") were being dropped by Readability as clutter:
+  five on Material Web's button page.
+- **Page facts skip more junk:** headings inside or holding a link (another story's
+  teaser), values holding form controls, and cookie settings.
+- **An article above its comments stays an article.** Comments outweighing the
+  article made the main-content fallback fire, four times the length on a real
+  page. Readability's result now stands when what it kept isn't itself a comment,
+  and `omissions` gets `comments`.
+- **Hidden text no longer counts or comes through in the fallback,** except code
+  behind a "Show code" toggle.
+- **Tabs read as one document.** Inactive panels are kept under their tab's name,
+  in place of a tab strip of dead links. A panel over 5,000 characters is left out
+  and named (`stats.standardization.omittedTabs`, `omissions` kind `inactive_tabs`).
+- **Note asides survive,** as blockquotes: Readability removes every `<aside>`.
+- **What changes the content says so in the Markdown too:** a fallback, a removed
+  JSON line, left-out tabs and comments each add a closing line, since a caller may
+  pass on only the Markdown. The shadow notice stays last.
+- **Too little from Readability uses the page's `<main>`,** when it marks one with
+  enough text, before the full page.
+- **`stats.pageTextChars`** counts the page's text. A capture is a snapshot of the
+  DOM at call time: a caller can re-capture until the number settles.
+- **Headlines:** a site-wide `og:title` no longer replaces the page's H1, and a badge
+  after an H1's title is dropped. A heading that is wholly a link to itself becomes
+  plain text.
+- A link around block content stays on one line, and `.ad-label` counts as an ad.
+- The browser bundle grows to about 75.4 KB minified; the size ceiling is now
+  76 KB.
+- **The browser run no longer launches by default.** Launching browsers from agent
+  sessions has crashed this machine. `npm run test:browser` now needs
+  `PAGE_MARKDOWN_ALLOW_BROWSER_LAUNCH=1`, set by a person in their own shell;
+  without it the run builds its page, launches nothing and exits with an error.
+  It is no longer part of `npm run ci` or `prepublishOnly`.
+
+## 0.1.5 — 2026-11-04
+
+Content-fidelity fixes, each with a new fixture run in Node and in the browser:
+
+- **Code blocks keep their lines.** Lines marked up with `<br>` or block
+  elements (Docusaurus and similar highlighters) came out joined on one line.
+- **Copy buttons stay out of code.** A `<button>` inside `<pre>` leaked its
+  label into the code and lost the language. Whitespace around the `<code>`
+  no longer loses the fence.
+- **Text beside a heading is kept.** A `<div>` holding a heading and bare text
+  was replaced by the heading, deleting the text.
+- **Row spans keep columns aligned.** Later cells shifted left. A spanned cell
+  now repeats in each row of a GFM table, and block-rendered tables label cells
+  by their true column.
+- **Large tables no longer freeze the page.** Table checks ran once per cell,
+  so time grew with the square of the size: 500 rows took about 8 seconds in
+  Chrome.
+- **Modes leave more code alone.** Fences in list items and blockquotes, and
+  inline code, were edited by `no_links`, `text_only` and blank-line cleanup.
+- **A lead paragraph is no longer doubled** when it starts with a link or holds
+  an entity such as `&lt;`.
+- **Script links are caught however they are written.** `javascript:` with an
+  embedded newline, tab or control character got through, as did `vbscript:`
+  and `data:`. These links keep only their text. Images with such a source are
+  dropped.
+- **`<base href>` is respected** for links and images.
+- **A linked image with empty alt is kept.**
+- **An ad marker on `<body>` no longer empties the page.** Sites put page-wide ad
+  settings there (`data-ad-unit` on heise.de), and the ad filter removed the whole
+  body. `<html>` and `<body>` are never treated as ads; ads inside them still are.
+- `stats.markdownChars` now matches the returned Markdown.
+- The browser bundle grows to about 67.2 KB minified; the size ceiling is now
+  68 KB.
+
+- **TypeScript types.** Declarations for `pageToMarkdown`, its options, the full
+  result and every stage export are generated from the JSDoc in `src` into
+  `dist/types`, through a `types` condition in `exports`. The result is
+  documented in a README table. `npm run typecheck` is part of `npm run ci`: it
+  checks `src` against its JSDoc, checks a consumer file with `tsc`, and checks
+  the declarations against real output from every fixture, in both directions.
+  Only comments change in the existing modules; the bundle is the same size.
+- **Bench.** `npm run bench` adds a Readability + Turndown baseline, word recall
+  and precision, and 11 held-out pages whose probes were committed before any
+  engine ran on them. The headline changes from 32/32 against 11/32 to 84/100
+  against 74/100 and 61/100 on the held-out pages; see `bench/RESULTS.md` for the
+  failures. `bench/fetch-pages.mjs` now decodes by declared charset and keeps the
+  last good copy when a fetch fails. `npm run bench:write` updates
+  `bench/RESULTS.md`; `npm run bench` only prints.
+- **Licence summary.** The README now says in plain words who can use the package
+  free and who needs a commercial licence, and how to ask for one. The licence
+  itself is unchanged.
+- **Browser run.** `npm run test:browser` runs the golden fixtures in
+  headless Chrome or Edge, with the browser's own DOM, and checks what Node
+  can't show:
+  - the live document is left unmodified;
+  - real open shadow roots are captured, nested roots and slotted children
+    included;
+  - closed shadow roots stay out.
+
+  It is part of `npm run ci` and needs no new dependencies. The package's output
+  in Chrome and Edge matches its Node output.
+
 ## 0.1.4 — 2026-11-02
 
 - **text-sanctifier 1.3.8** (was 1.3.3). `removeHiddenText` now also removes

package/README.md

@@ -3,8 +3,10 @@
 Faithful page-to-Markdown for humans and agents. Give it a web page's DOM and
 it returns clean Markdown plus metadata. It is written for browsers, for
 example an extension's content script, and runs in Node with a DOM library.
-So far it is tested only in Node (see [Known limits](#known-limits-v01)). It
-was built for WATT3D's browser extensions.
+It is tested in Node, in headless Chrome and Edge, and in a real extension
+(HELIOS, in Brave), but not yet in Firefox or Safari (see
+[Known limits](#known-limits-v02)). It was built for WATT3D's
+browser extensions.
 
 ```
 live Document
@@ -12,8 +14,10 @@ live Document
   → standardize   <pre> → fenced code with language; heading wrappers unwrapped; empty and
                   javascript: links removed; absolute links and images; lazy images resolved
   → extract       Mozilla Readability on a copy; full-page fallback when it finds too little;
+                  main-content fallback when it keeps only a small share of the page;
                   a dropped lead paragraph restored; a clean title heading
-  → convert       Turndown + GitHub-flavoured tables, strikethrough and task lists; never raw HTML
+  → convert       Turndown + GitHub-flavoured tables, strikethrough and task lists; tables never
+                  come out as raw HTML
   → clean         fence-safe; keeps short lines and <br> hard breaks; never cuts text
   → hidden text   opt-in (removeHiddenText): tag-character text, variation-selector runs and
                   invisible operators removed, fenced code included; counted in omissions
@@ -28,6 +32,9 @@ The live page is only read, never modified.
 npm install page-markdown
 ```
 
+Free for personal, non-commercial academic and fully open-source use. Other
+commercial use needs a licence: see [Licence](#licence).
+
 ## Usage
 
 ```js
@@ -52,19 +59,52 @@ pageToMarkdown(doc, { url: "https://example.com/article" });
 | `url` | the document's URL | Base for absolute links and images |
 | `extract` | `"readable"` | `"readable"` (Readability, full-page fallback), `"full"`, or a function `(docCopy) => { html, method }` |
 | `minExtractChars` | `100` | Below this, readable extraction falls back to the full page |
+| `minExtractShare` | `0.4` | When Readability keeps less than this share of the page's main content (and misses at least 500 characters), the main content is used instead: forum threads and product grids. `0` turns it off |
 | `includeFullPage` | `false` | Also return `fullPageMarkdown` |
 | `includeTitle` | `true` | Start readable output with `# <headline>`: the page's H1, else its title without the site name |
 | `recoverLead` | `true` | Restore a lead paragraph the extractor dropped, when it matches the meta description |
 | `pageFacts` | `false` | Append `## Page facts`: short label/value pairs outside the article (a sidebar's version or license, spec rows, infobox rows) that the extractor dropped. `extraction.pageFacts` counts them |
 | `removeHiddenText` | `false` | Remove text hidden from people but read by models: runs of tag characters, variation selectors, invisible operators and reserved tag-plane code points, found by [text-sanctifier](https://github.com/iWhatty/text-sanctifier)'s `findHiddenRuns`. Applies to `markdown` (fenced code included), `fullPageMarkdown` and `metadata`. The England, Scotland and Wales flags and a lone emoji or text presentation selector are kept. `omissions` gets one entry per kind (`hidden_tag`, `hidden_variation_selector`, `hidden_invisible_operator`, `hidden_reserved_tag_plane`) with `count` runs and `chars` code points removed from `markdown` |
 | `mode` | `"full"` | `"no_links"` keeps link text; `"text_only"` also drops structure. Fenced code is never changed |
-| `sanitize` | see `DEFAULT_SANITIZE_OPTIONS` | `stripAds`, `extraStripSelectors` (e.g. `["[data-app-overlay]"]`), shadow limits |
+| `sanitize` | see `DEFAULT_SANITIZE_OPTIONS` | `stripAds`, `extraStripSelectors` (e.g. `["[data-app-overlay]"]`), shadow limits, which apply per host in the page's own tree and don't charge slotted page content |
 | `readability` | `{}` | Passed to Readability |
 
 The stages are also exported for advanced use: `sanitizeToInertDocument`,
 `standardizeDocument`, `extractContent`, `htmlToMarkdown`, `cleanMarkdown`,
 `applyMode`, `collectMetadata`, `collectPageFacts` and `removeHiddenText`.
 
+### Result
+
+`pageToMarkdown` returns:
+
+| Field | Type | Meaning |
+|---|---|---|
+| `version` | `string` | The package version that made this result |
+| `markdown` | `string` | The Markdown |
+| `fullPageMarkdown` | `string?` | The whole page as Markdown. Present only with `includeFullPage` |
+| `metadata.url`, `canonicalUrl` | `string` | The page URL; the canonical link, else the page URL |
+| `metadata.documentTitle`, `title` | `string` | The `<title>`; og:title or twitter:title, else the JSON-LD headline, else the `<title>` |
+| `metadata.description`, `author`, `published`, `modified`, `siteName`, `lang` | `string` | From meta tags and JSON-LD, `""` when the page doesn't state them. `siteName` falls back to the hostname |
+| `metadata.headline` | `string?` | The heading chosen for the capture |
+| `metadata.extractedTitle`, `byline` | `string?` | What the extractor reported |
+| `extraction.method` | `string` | `"readability"`, `"main_content"`, `"full_page"`, or the name a custom extractor gave |
+| `extraction.fallbackReason` | `string?` | Why extraction fell back: `extracted_small_share_of_page`, `extracted_site_menu` or `extracted_few_sections_of_page` (to the main content), or `extractor_found_nothing`, `extracted_text_below_minimum` or `extractor_error: …` (to the full page) |
+| `extraction.leadRecovered` | `true?` | A dropped lead paragraph was restored |
+| `extraction.pageFacts` | `number?` | How many facts `pageFacts` appended |
+| `omissions[]` | `{ kind, count, note, chars? }` | What was left out or removed. `kind` is `shadow_content`, `json_line`, `inactive_tabs`, `comments` or a `hidden_*` kind; only `hidden_*` entries have `chars`. All but `hidden_*` also add a closing line to `markdown`, as a fallback does, so a caller that passes on only the Markdown still says so |
+| `warnings[]` | `string` | Read these before trusting a capture: a fallback, or shadow content cut off at the capture limit, in which case the main content may be missing |
+| `stats.mode` | `string` | The output mode used |
+| `stats.extractedHtmlChars`, `convertedChars`, `markdownChars` | `number` | Sizes at each step. `markdownChars` is the returned Markdown's length |
+| `stats.standardization` | `object` | Counts of what standardizing changed: `lazyImagesResolved`, `inlineImagesDropped`, `scriptLinksUnwrapped`, `emptyLinksRemoved`, `preWrapped`, `headingWidgetsRemoved`, `headingsUnwrapped` |
+
+A `?` marks a field that is absent, not empty, when it doesn't apply.
+
+### TypeScript
+
+Types ship with the package, generated from the JSDoc in `src`: `pageToMarkdown`,
+the stages, and `PageMarkdownOptions`, `PageMarkdownResult` and the types they use.
+They bring in TypeScript's `dom` lib, so a Node project needs no `lib` change.
+
 ### Recipes
 
 ```js
@@ -87,7 +127,7 @@ const { markdown: safe, omissions } = pageToMarkdown(document, { removeHiddenTex
 ## Quality
 
 `npm test` runs the contract suite. It covers:
-- golden outputs for 26 fixtures;
+- golden outputs for 34 fixtures;
 - regressions for known converter bugs, including dropped lead paragraphs,
   raw HTML tables, flattened code in tables, site-suffixed titles and
   unescaped table pipes;
@@ -95,31 +135,97 @@ const { markdown: safe, omissions } = pageToMarkdown(document, { removeHiddenTex
 - the shadow DOM limits and their omission notice;
 - leaving the live page unmodified.
 
-`npm run bench` scores the package against a naive baseline (Turndown over the
-page body). It runs 15 fixture checks and 17 probes on five real pages, which
-`node bench/fetch-pages.mjs` fetches first. See
-[bench/RESULTS.md](bench/RESULTS.md): currently **32/32 against 11/32**.
-
-Browser bundle: about 65 KB minified, 23 KB gzip (`npm run size`, ceiling 66 KB
+`npm run typecheck` checks `src` against its JSDoc with `tsc` and generates the
+declarations in `dist/types`. It then checks those from a consumer's side
+(`test/types/consumer.ts`) and against real output: it runs the pipeline over the
+fixtures and fails when a result or the export list no longer matches the
+declarations, in either direction.
+
+`npm run test:browser` runs the same golden fixtures in headless Chrome (or
+Edge), parsed by the browser's own DOM, with no extra dependencies. It also
+checks what Node can't show:
+- a live document with layout is left unmodified;
+- real open shadow roots are captured, including nested roots and slotted
+  children in rendered order;
+- closed shadow roots stay out of the capture.
+
+It launches a browser, so it runs only when a person sets
+`PAGE_MARKDOWN_ALLOW_BROWSER_LAUNCH=1` in their own shell; agents must not set it.
+Without the flag it builds `.browser-test/index.html`, which you can open in a
+browser that is already running, and exits with an error. Set
+`PAGE_MARKDOWN_BROWSER` to choose the browser.
+
+`npm run bench` scores the package against two baselines: Turndown over the page
+body, and Readability then Turndown. It runs 15 fixture checks and 117 probes on
+16 real pages, which `node bench/fetch-pages.mjs` fetches first, and measures word
+recall against a hand-picked reference for each page. `npm run bench` only prints;
+`npm run bench:write` also updates [bench/RESULTS.md](bench/RESULTS.md). Currently:
+
+| Tier | page-markdown | Readability + Turndown | Turndown only |
+|---|---|---|---|
+| Fixtures, written with the package | 15/15 | 7/15 | 3/15 |
+| Five pages the fixes were tuned on | 17/17 | 9/17 | 8/17 |
+| **11 pages, once held out** | **98/100** | **74/100** | **61/100** |
+| Their word recall / precision | 97.6% / 89.1% | 85.4% / 95.6% | 98.2% / 72.1% |
+
+Don't read 98 as a score on unseen pages. These probes were written before any
+engine ran and the package first scored 75/100, but most of the gain since comes
+from fixes for these same pages' failures. New pages are the real test.
+
+It gains recall (97.6% against 85.4%) by keeping more, and pays in precision
+(89.1% against 95.6%). The cost is mostly on the IKEA category page: 29.7% precision,
+because the main-content fallback keeps the page's banners and filters along with
+its products. What it still fails on these pages:
+
+- **Promo boxes kept** inside a BBC Arabic article.
+
+Browser bundle: about 75 KB minified, 26 KB gzip (`npm run size`, ceiling 76 KB
 minified). Readability is most of it.
 
-`npm run ci` runs the contract suite and the size check, and must exit 0
-before a commit. It runs locally; the repo deliberately has no cloud CI.
-
-## Known limits (v0.1)
-
-- **Node only so far.** Tested in Node with linkedom, not yet in a browser. The
-  open shadow DOM path is exercised only with a stand-in host in tests.
+`npm run ci` runs the contract suite, the type check and the size check, and must
+exit 0 before a commit. It launches no browser, so run the browser check yourself
+before a release when the DOM handling has changed. It runs locally; the repo
+deliberately has no cloud CI.
+
+## Known limits (v0.2)
+
+- **Chromium only so far.** Tested in Node (linkedom), in headless Chrome and
+  Edge, and on live pages in a real extension (HELIOS, in Brave), but not
+  in Firefox or Safari.
+- **Extraction can still be wrong without saying so.** The main-content fallback
+  and its warning catch a small share of the page, a kept site menu and lost
+  sections, not every wrong choice. If Readability picks the wrong part of a page
+  and that part is large, nothing flags it. No closing line doesn't mean nothing
+  was lost. A short post among long replies can be dropped: a Mastodon status lost
+  the post itself, and its cause isn't known yet.
+- **Answers hidden on an ordinary article can be lost.** Collapsed FAQ answers are
+  kept by the main-content fallback only; Readability removes hidden text.
+- **Bylines and dates can be missing from the Markdown.** Readability often leaves
+  them out of the article. Look in `metadata.author` and `metadata.published`.
+- **Note labels can be lost.** On Docker's docs, "Warning", "Note" and "Tip" are
+  dropped, though the blockquotes stay.
 - **Some legacy markup loses paragraph breaks.** Readability can drop runs of
   `<br>` between inline elements in old HTML. The text survives; a paragraph
   break may not.
-- **Hidden content stays.** The pipeline reads the static DOM, so elements
-  hidden only by CSS remain unless extraction drops them.
+- **A capture is a snapshot.** It reads the DOM as it is at call time, and it
+  can't tell whether the page has finished rendering. A page read too early looks
+  like a normal, short result. To wait, re-capture until `stats.pageTextChars`
+  settles.
+- **Captures hold whatever is in the DOM,** including text that other extensions
+  add. Use `sanitize.extraStripSelectors` for what you know about.
+- **Hidden content stays where only CSS hides it.** The pipeline reads the static
+  DOM. The `hidden` attribute and inline `display: none` are understood; a tab
+  panel hidden by a stylesheet alone is kept unless its group names the selected
+  one. Tabs marked with neither `role` nor `aria-role` and hidden with
+  `class="hidden"` can be lost, with no closing line.
 - **`removeHiddenText` covers hidden runs only.** Zero-width spaces, bidi
   controls and look-alike spaces pass through. It also removes runs that are
   legitimate in some scripts: ideographic variation selectors in CJK names and
   the Mongolian free variation selectors.
 - **Iframes and closed shadow roots are not captured.**
+- **Text that spells out HTML stays literal.** `&lt;img onerror=…&gt;` in page
+  text comes out as `<img onerror=…>`. Sanitize the Markdown before rendering it
+  as HTML.
 
 ## Licence
 
@@ -127,6 +233,19 @@ page-markdown is under the WATT3D Source License: AGPL-3.0 with the Additional
 Terms in [ADDITIONAL_TERMS.md](ADDITIONAL_TERMS.md) (see [LICENSE](LICENSE)).
 [robots.txt](robots.txt) states the AI-crawler policy.
 
+In short (the licence files govern, and this is not legal advice):
+
+| Use | Terms |
+|---|---|
+| Personal, non-commercial | Free |
+| Non-commercial academic research | Free |
+| Commercial use, including inside a for-profit company | Only if everything you own or operate is open source under a licence at least as protective as AGPL-3.0; otherwise you need a commercial licence |
+| AI training, fine-tuning or a RAG corpus built from this software | Counts as commercial use, and the resulting model must be released with open weights |
+
+AGPL-3.0 applies throughout, including its source-sharing duties for networked
+services. For a commercial licence, write to
+[WATT3D@protonmail.com](mailto:WATT3D@protonmail.com).
+
 It bundles text-sanctifier's run finder (same licence) and Mozilla Readability
 (Apache-2.0) and Turndown (MIT), whose licences are in
 [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). Those notices must accompany

package/dist/types/clean.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Applies `transform` to the prose between fenced code blocks only.
+ * A shorter embedded fence does not close a block; an unterminated fence
+ * keeps the rest verbatim.
+ * @param {string} markdown
+ * @param {(prose: string) => string} transform
+ */
+export function mapOutsideFences(markdown: string, transform: (prose: string) => string): string;
+/**
+ * @param {string} markdown
+ * @returns {{ markdown: string, removedJsonLines: number }}
+ */
+export function cleanMarkdown(markdown: string): {
+    markdown: string;
+    removedJsonLines: number;
+};
+/**
+ * Output modes. "no_links" keeps link text. "text_only" also drops headings,
+ * list markers, emphasis and images. Both leave fenced code verbatim.
+ * @param {string} markdown
+ * @param {import("./types.js").OutputMode} [mode]
+ */
+export function applyMode(markdown: string, mode?: import("./types.js").OutputMode): string;

package/dist/types/convert.d.ts

@@ -0,0 +1,19 @@
+/**
+ * A configured Turndown instance (cached).
+ * @returns {import("./types.js").TurndownInstance}
+ */
+export function createTurndown(): import("./types.js").TurndownInstance;
+/**
+ * @param {string} html
+ * @returns {string}
+ */
+export function htmlToMarkdown(html: string): string;
+export const TURNDOWN_OPTIONS: Readonly<{
+    headingStyle: "atx";
+    bulletListMarker: "-";
+    codeBlockStyle: "fenced";
+    emDelimiter: "_";
+    strongDelimiter: "**";
+    linkStyle: "inlined";
+    hr: "---";
+}>;

package/dist/types/extract.d.ts

@@ -0,0 +1,42 @@
+export function isHiddenElement(el: any): any;
+/**
+ * @typedef {{ html: string, method: "readability"|"main_content"|"full_page"|(string & {}), fallbackReason?: string, title?: string,
+ *   byline?: string, published?: string, siteName?: string, lang?: string, excerpt?: string, commentsOmitted?: boolean }} Extraction
+ */
+/**
+ * Receives a copy of the standardized document. Returning nothing, empty HTML
+ * or too little text falls back to the full page; so does throwing.
+ * @typedef {(doc: Document) => Extraction|null|undefined} Extractor
+ */
+/**
+ * @param {Document} doc standardized inert document (not mutated)
+ * @param {{ extract?: "readable"|"full"|Extractor,
+ *   minExtractChars?: number, minExtractShare?: number, readability?: import("./types.js").ReadabilityOptions }} [options]
+ * @returns {Extraction}
+ */
+export function extractContent(doc: Document, options?: {
+    extract?: "readable" | "full" | Extractor;
+    minExtractChars?: number;
+    minExtractShare?: number;
+    readability?: import("./types.js").ReadabilityOptions;
+}): Extraction;
+export const DEFAULT_MIN_EXTRACT_CHARS: 100;
+export const DEFAULT_MIN_EXTRACT_SHARE: 0.4;
+export const PAGE_CHROME: "nav, footer, form, [role='navigation'], [role='banner'], [role='contentinfo'], [role='search']";
+export type Extraction = {
+    html: string;
+    method: "readability" | "main_content" | "full_page" | (string & {});
+    fallbackReason?: string;
+    title?: string;
+    byline?: string;
+    published?: string;
+    siteName?: string;
+    lang?: string;
+    excerpt?: string;
+    commentsOmitted?: boolean;
+};
+/**
+ * Receives a copy of the standardized document. Returning nothing, empty HTML
+ * or too little text falls back to the full page; so does throwing.
+ */
+export type Extractor = (doc: Document) => Extraction | null | undefined;

package/dist/types/facts.d.ts

@@ -0,0 +1,15 @@
+/**
+ * @param {Document} doc standardized clone
+ * @param {string} articleHtml the extracted article
+ * @returns {Array<{ label: string, value: string }>} facts absent from the article
+ */
+export function collectPageFacts(doc: Document, articleHtml: string): Array<{
+    label: string;
+    value: string;
+}>;
+export function escapeMarkdown(text: any): any;
+/**
+ * "## Page facts" list, or "" when there are none.
+ * @param {readonly import("./types.js").PageFact[]} facts
+ */
+export function pageFactsMarkdown(facts: readonly import("./types.js").PageFact[]): string;

package/dist/types/hidden.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @param {string} text
+ * @returns {{ text: string, runs: import("./types.js").HiddenRun[] }}
+ */
+export function removeHiddenText(text: string): {
+    text: string;
+    runs: import("./types.js").HiddenRun[];
+};
+/**
+ * One omission per kind of run: `count` runs, `chars` code points.
+ * @param {ReadonlyArray<Pick<import("./types.js").HiddenRun, "kind"|"count">>} runs
+ * @returns {import("./types.js").HiddenTextOmission[]}
+ */
+export function hiddenTextOmissions(runs: ReadonlyArray<Pick<import("./types.js").HiddenRun, "kind" | "count">>): import("./types.js").HiddenTextOmission[];

package/dist/types/index.d.ts

@@ -0,0 +1,107 @@
+/**
+ * @typedef {object} PageMarkdownOptions
+ * @property {string} [url] page URL for absolute links (default: the document's URL)
+ * @property {"full"|"no_links"|"text_only"} [mode="full"]
+ * @property {"readable"|"full"|import("./extract.js").Extractor} [extract="readable"]
+ *   "readable": Readability with full-page fallback; "full": the whole body;
+ *   or a custom extractor receiving a copy of the standardized document.
+ * @property {number} [minExtractChars=100] below this, fall back to the page's <main>
+ *   if it marks one and it holds enough, else to the full page
+ * @property {number} [minExtractShare=0.4] when Readability keeps less than this
+ *   share of the page's main content, use the main content instead; 0 turns it off
+ * @property {boolean} [includeFullPage=false] also return the full-page Markdown
+ * @property {boolean} [includeTitle=true] start readable output with "# <headline>"
+ * @property {boolean} [recoverLead=true] restore a dropped lead paragraph that matches the meta description
+ * @property {boolean} [pageFacts=false] append "## Page facts": short label/value pairs
+ *   (sidebar versions, prices, spec rows) that the extracted article left out
+ * @property {boolean} [removeHiddenText=false] remove tag-character text,
+ *   variation-selector runs and invisible operators from the Markdown (fenced
+ *   code included), fullPageMarkdown and metadata; omissions count the runs
+ *   removed from the Markdown
+ * @property {import("./types.js").SanitizeOptions} [sanitize] see DEFAULT_SANITIZE_OPTIONS
+ * @property {import("./types.js").ReadabilityOptions} [readability] options passed to Readability
+ */
+/**
+ * Converts a page (or an element of it) to Markdown.
+ * Pure with respect to the input: the live document is only read.
+ *
+ * @param {Document|Element} input
+ * @param {PageMarkdownOptions} [options]
+ * @returns {import("./types.js").PageMarkdownResult}
+ */
+export function pageToMarkdown(input: Document | Element, options?: PageMarkdownOptions): import("./types.js").PageMarkdownResult;
+export const VERSION: "0.2.0";
+export { standardizeDocument } from "./standardize.js";
+export { collectMetadata } from "./metadata.js";
+export * from "./types.js";
+export type PageMarkdownOptions = {
+    /**
+     * page URL for absolute links (default: the document's URL)
+     */
+    url?: string | undefined;
+    mode?: "full" | "no_links" | "text_only" | undefined;
+    /**
+     * "readable": Readability with full-page fallback; "full": the whole body;
+     * or a custom extractor receiving a copy of the standardized document.
+     */
+    extract?: import("./extract.js").Extractor | "readable" | "full" | undefined;
+    /**
+     * below this, fall back to the page's <main>
+     * if it marks one and it holds enough, else to the full page
+     */
+    minExtractChars?: number | undefined;
+    /**
+     * when Readability keeps less than this
+     * share of the page's main content, use the main content instead; 0 turns it off
+     */
+    minExtractShare?: number | undefined;
+    /**
+     * also return the full-page Markdown
+     */
+    includeFullPage?: boolean | undefined;
+    /**
+     * start readable output with "# <headline>"
+     */
+    includeTitle?: boolean | undefined;
+    /**
+     * restore a dropped lead paragraph that matches the meta description
+     */
+    recoverLead?: boolean | undefined;
+    /**
+     * append "## Page facts": short label/value pairs
+     * (sidebar versions, prices, spec rows) that the extracted article left out
+     */
+    pageFacts?: boolean | undefined;
+    /**
+     * remove tag-character text,
+     * variation-selector runs and invisible operators from the Markdown (fenced
+     * code included), fullPageMarkdown and metadata; omissions count the runs
+     * removed from the Markdown
+     */
+    removeHiddenText?: boolean | undefined;
+    /**
+     * see DEFAULT_SANITIZE_OPTIONS
+     */
+    sanitize?: import("./types.js").SanitizeOptions | undefined;
+    /**
+     * options passed to Readability
+     */
+    readability?: {
+        debug?: boolean;
+        maxElemsToParse?: number;
+        nbTopCandidates?: number;
+        charThreshold?: number;
+        classesToPreserve?: string[];
+        keepClasses?: boolean;
+        serializer?: ((node: Node) => string) | undefined;
+        disableJSONLD?: boolean;
+        allowedVideoRegex?: RegExp;
+    } | undefined;
+};
+export { sanitizeToInertDocument, DEFAULT_SANITIZE_OPTIONS, isAdvertisementElement, createInertDocument } from "./sanitize.js";
+export { extractContent, DEFAULT_MIN_EXTRACT_CHARS, DEFAULT_MIN_EXTRACT_SHARE } from "./extract.js";
+export { htmlToMarkdown, createTurndown, TURNDOWN_OPTIONS } from "./convert.js";
+export { cleanMarkdown, applyMode, mapOutsideFences } from "./clean.js";
+export { recoverLeadParagraph, chooseHeadline } from "./refine.js";
+export { collectPageFacts, pageFactsMarkdown } from "./facts.js";
+export { removeHiddenText, hiddenTextOmissions } from "./hidden.js";

package/dist/types/metadata.d.ts

@@ -0,0 +1,18 @@
+/**
+ * @param {Document} doc the original page document
+ * @param {{ url?: string }} [options]
+ */
+export function collectMetadata(doc: Document, options?: {
+    url?: string;
+}): {
+    url: string;
+    canonicalUrl: string;
+    documentTitle: string;
+    title: string;
+    description: string;
+    author: string;
+    published: string;
+    modified: string;
+    siteName: string;
+    lang: string;
+};