page-markdown 0.1.4 → 0.1.5

package/CHANGELOG.md

@@ -1,124 +1,186 @@
-# Changelog
-
-## 0.1.4 — 2026-11-02
-
-- **text-sanctifier 1.3.8** (was 1.3.3). `removeHiddenText` now also removes
-  runs of reserved tag-plane code points (U+E0080–U+E00FF, U+E01F0–U+E0FFF),
-  which 1.3.3 did not find, and reports them as `hidden_reserved_tag_plane`.
-  The browser bundle grows to about 65.6 KB minified.
-- A test checks that every run kind text-sanctifier can report has an
-  omission note, so a new kind fails `npm run ci` instead of reaching users
-  with no note.
-- A test fails when any source, test, fixture or doc file holds a raw
-  invisible or control character. A test regex in 0.1.3 held four, written
-  by an editing tool that expanded `\u` escapes; they never shipped, since
-  tests are not in the package.
-
-## 0.1.3 — 2026-10-30
-
-First release on npm. `npm run ci` (contract suite and size check) is the
-local gate before a commit; there is no cloud CI. The size ceiling is 66 KB
-minified.
-
-Content-fidelity fixes:
-
-- **Title heading kept beside shell comments.** A `# comment` line in fenced
-  code no longer counts as the article's top-level heading, which had
-  suppressed the `# <headline>` line.
-- **Every pipe in a table cell is escaped.** `a || b` escaped only its first
-  pipe, which added a column and broke the table.
-- **Images beside a heading survive.** A heading wrapper that also holds an
-  image (or picture, video, audio) is no longer unwrapped to the bare heading.
-- **`no_links` keeps linked images intact.** `[![alt](img)](href)` became
-  `![alt](href)`, an image pointing at the link target; the image is now kept
-  as it was.
-- **Code in tables stays fenced.** A GFM cell is one line, so a data table
-  holding `<pre>` code (Wikipedia's syntax examples) was flattened to inline
-  code, losing its line breaks and indentation. Such a table is now rendered
-  as blocks: each cell under a bold label from its column header (or its row
-  header, for infoboxes), with code as verbatim fences. Tables without code
-  are still GFM grids.
-- Code language from MediaWiki's `mw-highlight-lang-*` class.
-- At most one blank line beside a fenced block.
-- Five new regression fixtures, one per fix.
-
-Hidden text (opt-in):
-
-- **`removeHiddenText: true`** removes runs of tag characters, variation
-  selectors and invisible operators, which people can't see but models read
-  (hidden prompt instructions). It applies to the Markdown, fenced code
-  included, and to `fullPageMarkdown` and `metadata`. The England, Scotland
-  and Wales flags and a lone emoji or text presentation selector are kept.
-  `omissions` gets `hidden_tag`, `hidden_variation_selector` and
-  `hidden_invisible_operator` entries with `count` (runs) and `chars` (code
-  points). `removeHiddenText` is exported. Default output is unchanged.
-- New runtime dependency: text-sanctifier 1.3.3 (first-party, no
-  dependencies), for its `text-sanctifier/hidden` run finder. About 1.5 KB
-  minified in the browser bundle.
-- New contract fixture `hidden-text.html`, written with character references
-  so it holds no raw hidden characters.
-
-## 0.1.2 — 2026-10-23
-
-- **Page facts (opt-in).** `pageFacts: true` appends a `## Page facts` list
-  of short label/value pairs that sit outside the extracted article: a
-  package sidebar's version, license and downloads, spec and price rows,
-  infobox rows. Readability drops these asides, which suits reading but
-  loses the answer to "which version?". Pairs come from a heading followed by
-  a short value, `<dt>`/`<dd>`, and two-cell table rows. Values already in
-  the article, sentences, link lists, lone non-data links and anything in
-  navigation, header, footer or forms are skipped. `extraction.pageFacts`
-  counts them; `collectPageFacts` is exported. Default output is unchanged.
-- New regression fixture: a package page whose version lives only in the
-  sidebar.
-
-## 0.1.1 — 2026-10-22
-
-Fixes from the first in-browser comparison (MDN, Wikipedia, the Python docs,
-GitHub):
-
-- **Tables never come out as raw HTML.** Own GFM table rules replace the
-  plugin's:
-  - a header row is used even when the table has none;
-  - colspans are padded with empty cells;
-  - cell content is flattened to one line (code, lists, line breaks);
-  - pipes are escaped;
-  - layout and nested tables are rendered as blocks.
-- **Dropped lead paragraph restored.** When the extractor leaves out a
-  paragraph that matches the meta description (MDN's heading-less first
-  section), it is put back first. `extraction.leadRecovered` reports it, and
-  `recoverLead: false` turns it off.
-- **Clean title heading.** The page's own H1 (permalink sign removed) when
-  the title contains it, else `og:title` or the title, without a site-name
-  segment at either end. It is exposed as `metadata.headline`.
-- **Smaller bundle.** Strikethrough and task-list rules are now built in, and
-  `@joplin/turndown-plugin-gfm` is no longer a dependency. The browser bundle
-  is 60.2 KB minified, down from 61.7 KB.
-
-## 0.1.0 — 2026-10-22
-
-First release.
-
-- `pageToMarkdown(document, options)`: sanitize → standardize → extract →
-  convert → clean, returning Markdown, metadata, extraction details,
-  omissions, warnings and stats.
-- **Sanitize.** Builds an inert clone and inlines open shadow DOM within node,
-  character and depth limits, with an omission notice past them. Removes
-  scripts, styles, `noscript`, meta and link tags, and ads, plus any
-  `extraStripSelectors`.
-- **Standardize.**
-  - Turns `<pre>` into fenced code, with the language from
-    `language-`/`lang-`/`highlight-` classes or `data-lang`.
-  - Unwraps heading wrappers and removes edit and permalink widgets.
-  - Removes empty and `javascript:` links.
-  - Makes links and images absolute and resolves lazy images; drops inline
-    `data:` images.
-- **Extract.** Mozilla Readability on a copy of the clone, falling back to the
-  full page when it finds nothing or too little. Optionally restores the title
-  heading.
-- **Convert.** Turndown with GitHub-flavoured tables and strikethrough.
-- **Clean.** Fence-safe. Keeps short lines and `<br>` hard breaks, removes only
-  verified single-line JSON documents, and offers `full`, `no_links` and
-  `text_only` modes.
-- **Metadata.** Meta tags, JSON-LD (`@graph` included), canonical URL and
-  language.
+# Changelog
+
+## 0.1.5 — 2026-11-04
+
+Content-fidelity fixes, each with a new fixture run in Node and in the browser:
+
+- **Code blocks keep their lines.** Lines marked up with `<br>` or block
+  elements (Docusaurus and similar highlighters) came out joined on one line.
+- **Copy buttons stay out of code.** A `<button>` inside `<pre>` leaked its
+  label into the code and lost the language. Whitespace around the `<code>`
+  no longer loses the fence.
+- **Text beside a heading is kept.** A `<div>` holding a heading and bare text
+  was replaced by the heading, deleting the text.
+- **Row spans keep columns aligned.** Later cells shifted left. A spanned cell
+  now repeats in each row of a GFM table, and block-rendered tables label cells
+  by their true column.
+- **Large tables no longer freeze the page.** Table checks ran once per cell,
+  so time grew with the square of the size: 500 rows took about 8 seconds in
+  Chrome.
+- **Modes leave more code alone.** Fences in list items and blockquotes, and
+  inline code, were edited by `no_links`, `text_only` and blank-line cleanup.
+- **A lead paragraph is no longer doubled** when it starts with a link or holds
+  an entity such as `&lt;`.
+- **Script links are caught however they are written.** `javascript:` with an
+  embedded newline, tab or control character got through, as did `vbscript:`
+  and `data:`. These links keep only their text. Images with such a source are
+  dropped.
+- **`<base href>` is respected** for links and images.
+- **A linked image with empty alt is kept.**
+- **An ad marker on `<body>` no longer empties the page.** Sites put page-wide ad
+  settings there (`data-ad-unit` on heise.de), and the ad filter removed the whole
+  body. `<html>` and `<body>` are never treated as ads; ads inside them still are.
+- `stats.markdownChars` now matches the returned Markdown.
+- The browser bundle grows to about 67.2 KB minified; the size ceiling is now
+  68 KB.
+
+- **TypeScript types.** Declarations for `pageToMarkdown`, its options, the full
+  result and every stage export are generated from the JSDoc in `src` into
+  `dist/types`, through a `types` condition in `exports`. The result is
+  documented in a README table. `npm run typecheck` is part of `npm run ci`: it
+  checks `src` against its JSDoc, checks a consumer file with `tsc`, and checks
+  the declarations against real output from every fixture, in both directions.
+  Only comments change in the existing modules; the bundle is the same size.
+- **Bench.** `npm run bench` adds a Readability + Turndown baseline, word recall
+  and precision, and 11 held-out pages whose probes were committed before any
+  engine ran on them. The headline changes from 32/32 against 11/32 to 84/100
+  against 74/100 and 61/100 on the held-out pages; see `bench/RESULTS.md` for the
+  failures. `bench/fetch-pages.mjs` now decodes by declared charset and keeps the
+  last good copy when a fetch fails. `npm run bench:write` updates
+  `bench/RESULTS.md`; `npm run bench` only prints.
+- **Licence summary.** The README now says in plain words who can use the package
+  free and who needs a commercial licence, and how to ask for one. The licence
+  itself is unchanged.
+- **Browser run.** `npm run test:browser` runs the golden fixtures in
+  headless Chrome or Edge, with the browser's own DOM, and checks what Node
+  can't show:
+  - the live document is left unmodified;
+  - real open shadow roots are captured, nested roots and slotted children
+    included;
+  - closed shadow roots stay out.
+
+  It is part of `npm run ci` and needs no new dependencies. The package's output
+  in Chrome and Edge matches its Node output.
+
+## 0.1.4 — 2026-11-02
+
+- **text-sanctifier 1.3.8** (was 1.3.3). `removeHiddenText` now also removes
+  runs of reserved tag-plane code points (U+E0080–U+E00FF, U+E01F0–U+E0FFF),
+  which 1.3.3 did not find, and reports them as `hidden_reserved_tag_plane`.
+  The browser bundle grows to about 65.6 KB minified.
+- A test checks that every run kind text-sanctifier can report has an
+  omission note, so a new kind fails `npm run ci` instead of reaching users
+  with no note.
+- A test fails when any source, test, fixture or doc file holds a raw
+  invisible or control character. A test regex in 0.1.3 held four, written
+  by an editing tool that expanded `\u` escapes; they never shipped, since
+  tests are not in the package.
+
+## 0.1.3 — 2026-10-30
+
+First release on npm. `npm run ci` (contract suite and size check) is the
+local gate before a commit; there is no cloud CI. The size ceiling is 66 KB
+minified.
+
+Content-fidelity fixes:
+
+- **Title heading kept beside shell comments.** A `# comment` line in fenced
+  code no longer counts as the article's top-level heading, which had
+  suppressed the `# <headline>` line.
+- **Every pipe in a table cell is escaped.** `a || b` escaped only its first
+  pipe, which added a column and broke the table.
+- **Images beside a heading survive.** A heading wrapper that also holds an
+  image (or picture, video, audio) is no longer unwrapped to the bare heading.
+- **`no_links` keeps linked images intact.** `[![alt](img)](href)` became
+  `![alt](href)`, an image pointing at the link target; the image is now kept
+  as it was.
+- **Code in tables stays fenced.** A GFM cell is one line, so a data table
+  holding `<pre>` code (Wikipedia's syntax examples) was flattened to inline
+  code, losing its line breaks and indentation. Such a table is now rendered
+  as blocks: each cell under a bold label from its column header (or its row
+  header, for infoboxes), with code as verbatim fences. Tables without code
+  are still GFM grids.
+- Code language from MediaWiki's `mw-highlight-lang-*` class.
+- At most one blank line beside a fenced block.
+- Five new regression fixtures, one per fix.
+
+Hidden text (opt-in):
+
+- **`removeHiddenText: true`** removes runs of tag characters, variation
+  selectors and invisible operators, which people can't see but models read
+  (hidden prompt instructions). It applies to the Markdown, fenced code
+  included, and to `fullPageMarkdown` and `metadata`. The England, Scotland
+  and Wales flags and a lone emoji or text presentation selector are kept.
+  `omissions` gets `hidden_tag`, `hidden_variation_selector` and
+  `hidden_invisible_operator` entries with `count` (runs) and `chars` (code
+  points). `removeHiddenText` is exported. Default output is unchanged.
+- New runtime dependency: text-sanctifier 1.3.3 (first-party, no
+  dependencies), for its `text-sanctifier/hidden` run finder. About 1.5 KB
+  minified in the browser bundle.
+- New contract fixture `hidden-text.html`, written with character references
+  so it holds no raw hidden characters.
+
+## 0.1.2 — 2026-10-23
+
+- **Page facts (opt-in).** `pageFacts: true` appends a `## Page facts` list
+  of short label/value pairs that sit outside the extracted article: a
+  package sidebar's version, license and downloads, spec and price rows,
+  infobox rows. Readability drops these asides, which suits reading but
+  loses the answer to "which version?". Pairs come from a heading followed by
+  a short value, `<dt>`/`<dd>`, and two-cell table rows. Values already in
+  the article, sentences, link lists, lone non-data links and anything in
+  navigation, header, footer or forms are skipped. `extraction.pageFacts`
+  counts them; `collectPageFacts` is exported. Default output is unchanged.
+- New regression fixture: a package page whose version lives only in the
+  sidebar.
+
+## 0.1.1 — 2026-10-22
+
+Fixes from the first in-browser comparison (MDN, Wikipedia, the Python docs,
+GitHub):
+
+- **Tables never come out as raw HTML.** Own GFM table rules replace the
+  plugin's:
+  - a header row is used even when the table has none;
+  - colspans are padded with empty cells;
+  - cell content is flattened to one line (code, lists, line breaks);
+  - pipes are escaped;
+  - layout and nested tables are rendered as blocks.
+- **Dropped lead paragraph restored.** When the extractor leaves out a
+  paragraph that matches the meta description (MDN's heading-less first
+  section), it is put back first. `extraction.leadRecovered` reports it, and
+  `recoverLead: false` turns it off.
+- **Clean title heading.** The page's own H1 (permalink sign removed) when
+  the title contains it, else `og:title` or the title, without a site-name
+  segment at either end. It is exposed as `metadata.headline`.
+- **Smaller bundle.** Strikethrough and task-list rules are now built in, and
+  `@joplin/turndown-plugin-gfm` is no longer a dependency. The browser bundle
+  is 60.2 KB minified, down from 61.7 KB.
+
+## 0.1.0 — 2026-10-22
+
+First release.
+
+- `pageToMarkdown(document, options)`: sanitize → standardize → extract →
+  convert → clean, returning Markdown, metadata, extraction details,
+  omissions, warnings and stats.
+- **Sanitize.** Builds an inert clone and inlines open shadow DOM within node,
+  character and depth limits, with an omission notice past them. Removes
+  scripts, styles, `noscript`, meta and link tags, and ads, plus any
+  `extraStripSelectors`.
+- **Standardize.**
+  - Turns `<pre>` into fenced code, with the language from
+    `language-`/`lang-`/`highlight-` classes or `data-lang`.
+  - Unwraps heading wrappers and removes edit and permalink widgets.
+  - Removes empty and `javascript:` links.
+  - Makes links and images absolute and resolves lazy images; drops inline
+    `data:` images.
+- **Extract.** Mozilla Readability on a copy of the clone, falling back to the
+  full page when it finds nothing or too little. Optionally restores the title
+  heading.
+- **Convert.** Turndown with GitHub-flavoured tables and strikethrough.
+- **Clean.** Fence-safe. Keeps short lines and `<br>` hard breaks, removes only
+  verified single-line JSON documents, and offers `full`, `no_links` and
+  `text_only` modes.
+- **Metadata.** Meta tags, JSON-LD (`@graph` included), canonical URL and
+  language.

package/README.md

@@ -3,8 +3,9 @@
 Faithful page-to-Markdown for humans and agents. Give it a web page's DOM and
 it returns clean Markdown plus metadata. It is written for browsers, for
 example an extension's content script, and runs in Node with a DOM library.
-So far it is tested only in Node (see [Known limits](#known-limits-v01)). It
-was built for WATT3D's browser extensions.
+It is tested in Node and in headless Chrome and Edge, but not yet in Firefox or
+Safari (see [Known limits](#known-limits-v01)). It was built for WATT3D's
+browser extensions.
 
 ```
 live Document
@@ -13,7 +14,8 @@ live Document
                   javascript: links removed; absolute links and images; lazy images resolved
   → extract       Mozilla Readability on a copy; full-page fallback when it finds too little;
                   a dropped lead paragraph restored; a clean title heading
-  → convert       Turndown + GitHub-flavoured tables, strikethrough and task lists; never raw HTML
+  → convert       Turndown + GitHub-flavoured tables, strikethrough and task lists; tables never
+                  come out as raw HTML
   → clean         fence-safe; keeps short lines and <br> hard breaks; never cuts text
   → hidden text   opt-in (removeHiddenText): tag-character text, variation-selector runs and
                   invisible operators removed, fenced code included; counted in omissions
@@ -28,6 +30,9 @@ The live page is only read, never modified.
 npm install page-markdown
 ```
 
+Free for personal, non-commercial academic and fully open-source use. Other
+commercial use needs a licence: see [Licence](#licence).
+
 ## Usage
 
 ```js
@@ -65,6 +70,38 @@ The stages are also exported for advanced use: `sanitizeToInertDocument`,
 `standardizeDocument`, `extractContent`, `htmlToMarkdown`, `cleanMarkdown`,
 `applyMode`, `collectMetadata`, `collectPageFacts` and `removeHiddenText`.
 
+### Result
+
+`pageToMarkdown` returns:
+
+| Field | Type | Meaning |
+|---|---|---|
+| `version` | `string` | The package version that made this result |
+| `markdown` | `string` | The Markdown |
+| `fullPageMarkdown` | `string?` | The whole page as Markdown. Present only with `includeFullPage` |
+| `metadata.url`, `canonicalUrl` | `string` | The page URL; the canonical link, else the page URL |
+| `metadata.documentTitle`, `title` | `string` | The `<title>`; og:title or twitter:title, else the JSON-LD headline, else the `<title>` |
+| `metadata.description`, `author`, `published`, `modified`, `siteName`, `lang` | `string` | From meta tags and JSON-LD, `""` when the page doesn't state them. `siteName` falls back to the hostname |
+| `metadata.headline` | `string?` | The heading chosen for the capture |
+| `metadata.extractedTitle`, `byline` | `string?` | What the extractor reported |
+| `extraction.method` | `string` | `"readability"`, `"full_page"`, or the name a custom extractor gave |
+| `extraction.fallbackReason` | `string?` | Why extraction fell back to the full page: `extractor_found_nothing`, `extracted_text_below_minimum` or `extractor_error: …` |
+| `extraction.leadRecovered` | `true?` | A dropped lead paragraph was restored |
+| `extraction.pageFacts` | `number?` | How many facts `pageFacts` appended |
+| `omissions[]` | `{ kind, count, note, chars? }` | What was left out or removed. `kind` is `shadow_content`, `json_line` or a `hidden_*` kind; only `hidden_*` entries have `chars` |
+| `warnings[]` | `string` | Currently the full-page fallback, in words |
+| `stats.mode` | `string` | The output mode used |
+| `stats.extractedHtmlChars`, `convertedChars`, `markdownChars` | `number` | Sizes at each step. `markdownChars` is measured before `removeHiddenText` and the shadow omission notice |
+| `stats.standardization` | `object` | Counts of what standardizing changed: `lazyImagesResolved`, `inlineImagesDropped`, `scriptLinksUnwrapped`, `emptyLinksRemoved`, `preWrapped`, `headingWidgetsRemoved`, `headingsUnwrapped` |
+
+A `?` marks a field that is absent, not empty, when it doesn't apply.
+
+### TypeScript
+
+Types ship with the package, generated from the JSDoc in `src`: `pageToMarkdown`,
+the stages, and `PageMarkdownOptions`, `PageMarkdownResult` and the types they use.
+They bring in TypeScript's `dom` lib, so a Node project needs no `lib` change.
+
 ### Recipes
 
 ```js
@@ -87,7 +124,7 @@ const { markdown: safe, omissions } = pageToMarkdown(document, { removeHiddenTex
 ## Quality
 
 `npm test` runs the contract suite. It covers:
-- golden outputs for 26 fixtures;
+- golden outputs for 34 fixtures;
 - regressions for known converter bugs, including dropped lead paragraphs,
   raw HTML tables, flattened code in tables, site-suffixed titles and
   unescaped table pipes;
@@ -95,21 +132,60 @@ const { markdown: safe, omissions } = pageToMarkdown(document, { removeHiddenTex
 - the shadow DOM limits and their omission notice;
 - leaving the live page unmodified.
 
-`npm run bench` scores the package against a naive baseline (Turndown over the
-page body). It runs 15 fixture checks and 17 probes on five real pages, which
-`node bench/fetch-pages.mjs` fetches first. See
-[bench/RESULTS.md](bench/RESULTS.md): currently **32/32 against 11/32**.
-
-Browser bundle: about 65 KB minified, 23 KB gzip (`npm run size`, ceiling 66 KB
+`npm run typecheck` checks `src` against its JSDoc with `tsc` and generates the
+declarations in `dist/types`. It then checks those from a consumer's side
+(`test/types/consumer.ts`) and against real output: it runs the pipeline over the
+fixtures and fails when a result or the export list no longer matches the
+declarations, in either direction.
+
+`npm run test:browser` runs the same golden fixtures in headless Chrome (or
+Edge), parsed by the browser's own DOM, with no extra dependencies. It also
+checks what Node can't show:
+- a live document with layout is left unmodified;
+- real open shadow roots are captured, including nested roots and slotted
+  children in rendered order;
+- closed shadow roots stay out of the capture.
+
+Set `PAGE_MARKDOWN_BROWSER` to choose the browser. The built page is kept at
+`.browser-test/index.html` for opening by hand.
+
+`npm run bench` scores the package against two baselines: Turndown over the page
+body, and Readability then Turndown. It runs 15 fixture checks and 117 probes on
+16 real pages, which `node bench/fetch-pages.mjs` fetches first, and measures word
+recall against a hand-picked reference for each page. `npm run bench` only prints;
+`npm run bench:write` also updates [bench/RESULTS.md](bench/RESULTS.md). Currently:
+
+| Tier | page-markdown | Readability + Turndown | Turndown only |
+|---|---|---|---|
+| Fixtures, written with the package | 15/15 | 7/15 | 3/15 |
+| Five pages the fixes were tuned on | 17/17 | 9/17 | 8/17 |
+| **11 held-out pages** | **84/100** | **74/100** | **61/100** |
+| Held-out word recall / precision | 86.3% / 95.5% | 85.4% / 95.6% | 98.2% / 72.1% |
+
+The held-out row is the one to weigh. On those pages the package leads plain
+Readability + Turndown on probes, and the two are level on word recall and
+precision. The pages are no longer fully unseen: the package scored 75/100 before
+two fixes that came from their failures, so read 84 as a little generous. What it
+still fails there:
+
+- **Most H2s dropped** on Docusaurus and Starlight, by Readability; Starlight also
+  keeps "Section titled" anchor text.
+- **Non-default tab panels dropped** (Starlight).
+- **Comments and product grids lost**, by Readability: 36% word recall on a
+  Lobsters thread, 50% on an IKEA category page.
+- **Promo boxes kept** inside a BBC Arabic article.
+
+Browser bundle: about 67 KB minified, 24 KB gzip (`npm run size`, ceiling 68 KB
 minified). Readability is most of it.
 
-`npm run ci` runs the contract suite and the size check, and must exit 0
+`npm run ci` runs the contract suite, the type check, the browser run and the size check, and must exit 0
 before a commit. It runs locally; the repo deliberately has no cloud CI.
 
 ## Known limits (v0.1)
 
-- **Node only so far.** Tested in Node with linkedom, not yet in a browser. The
-  open shadow DOM path is exercised only with a stand-in host in tests.
+- **Chromium only so far.** Tested in Node (linkedom) and in headless Chrome
+  and Edge, but not in Firefox or Safari, and not yet inside a real extension's
+  content script.
 - **Some legacy markup loses paragraph breaks.** Readability can drop runs of
   `<br>` between inline elements in old HTML. The text survives; a paragraph
   break may not.
@@ -120,6 +196,9 @@ before a commit. It runs locally; the repo deliberately has no cloud CI.
   legitimate in some scripts: ideographic variation selectors in CJK names and
   the Mongolian free variation selectors.
 - **Iframes and closed shadow roots are not captured.**
+- **Text that spells out HTML stays literal.** `&lt;img onerror=…&gt;` in page
+  text comes out as `<img onerror=…>`. Sanitize the Markdown before rendering it
+  as HTML.
 
 ## Licence
 
@@ -127,6 +206,19 @@ page-markdown is under the WATT3D Source License: AGPL-3.0 with the Additional
 Terms in [ADDITIONAL_TERMS.md](ADDITIONAL_TERMS.md) (see [LICENSE](LICENSE)).
 [robots.txt](robots.txt) states the AI-crawler policy.
 
+In short (the licence files govern, and this is not legal advice):
+
+| Use | Terms |
+|---|---|
+| Personal, non-commercial | Free |
+| Non-commercial academic research | Free |
+| Commercial use, including inside a for-profit company | Only if everything you own or operate is open source under a licence at least as protective as AGPL-3.0; otherwise you need a commercial licence |
+| AI training, fine-tuning or a RAG corpus built from this software | Counts as commercial use, and the resulting model must be released with open weights |
+
+AGPL-3.0 applies throughout, including its source-sharing duties for networked
+services. For a commercial licence, write to
+[WATT3D@protonmail.com](mailto:WATT3D@protonmail.com).
+
 It bundles text-sanctifier's run finder (same licence) and Mozilla Readability
 (Apache-2.0) and Turndown (MIT), whose licences are in
 [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). Those notices must accompany

package/dist/types/clean.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Applies `transform` to the prose between fenced code blocks only.
+ * A shorter embedded fence does not close a block; an unterminated fence
+ * keeps the rest verbatim.
+ * @param {string} markdown
+ * @param {(prose: string) => string} transform
+ */
+export function mapOutsideFences(markdown: string, transform: (prose: string) => string): string;
+/**
+ * @param {string} markdown
+ * @returns {{ markdown: string, removedJsonLines: number }}
+ */
+export function cleanMarkdown(markdown: string): {
+    markdown: string;
+    removedJsonLines: number;
+};
+/**
+ * Output modes. "no_links" keeps link text. "text_only" also drops headings,
+ * list markers, emphasis and images. Both leave fenced code verbatim.
+ * @param {string} markdown
+ * @param {import("./types.js").OutputMode} [mode]
+ */
+export function applyMode(markdown: string, mode?: import("./types.js").OutputMode): string;

package/dist/types/convert.d.ts

@@ -0,0 +1,19 @@
+/**
+ * A configured Turndown instance (cached).
+ * @returns {import("./types.js").TurndownInstance}
+ */
+export function createTurndown(): import("./types.js").TurndownInstance;
+/**
+ * @param {string} html
+ * @returns {string}
+ */
+export function htmlToMarkdown(html: string): string;
+export const TURNDOWN_OPTIONS: Readonly<{
+    headingStyle: "atx";
+    bulletListMarker: "-";
+    codeBlockStyle: "fenced";
+    emDelimiter: "_";
+    strongDelimiter: "**";
+    linkStyle: "inlined";
+    hr: "---";
+}>;

package/dist/types/extract.d.ts

@@ -0,0 +1,37 @@
+/**
+ * @typedef {{ html: string, method: "readability"|"full_page"|(string & {}), fallbackReason?: string, title?: string,
+ *   byline?: string, published?: string, siteName?: string, lang?: string, excerpt?: string }} Extraction
+ */
+/**
+ * Receives a copy of the standardized document. Returning nothing, empty HTML
+ * or too little text falls back to the full page; so does throwing.
+ * @typedef {(doc: Document) => Extraction|null|undefined} Extractor
+ */
+/**
+ * @param {Document} doc standardized inert document (not mutated)
+ * @param {{ extract?: "readable"|"full"|Extractor,
+ *   minExtractChars?: number, readability?: import("./types.js").ReadabilityOptions }} [options]
+ * @returns {Extraction}
+ */
+export function extractContent(doc: Document, options?: {
+    extract?: "readable" | "full" | Extractor;
+    minExtractChars?: number;
+    readability?: import("./types.js").ReadabilityOptions;
+}): Extraction;
+export const DEFAULT_MIN_EXTRACT_CHARS: 100;
+export type Extraction = {
+    html: string;
+    method: "readability" | "full_page" | (string & {});
+    fallbackReason?: string;
+    title?: string;
+    byline?: string;
+    published?: string;
+    siteName?: string;
+    lang?: string;
+    excerpt?: string;
+};
+/**
+ * Receives a copy of the standardized document. Returning nothing, empty HTML
+ * or too little text falls back to the full page; so does throwing.
+ */
+export type Extractor = (doc: Document) => Extraction | null | undefined;

package/dist/types/facts.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @param {Document} doc standardized clone
+ * @param {string} articleHtml the extracted article
+ * @returns {Array<{ label: string, value: string }>} facts absent from the article
+ */
+export function collectPageFacts(doc: Document, articleHtml: string): Array<{
+    label: string;
+    value: string;
+}>;
+/**
+ * "## Page facts" list, or "" when there are none.
+ * @param {readonly import("./types.js").PageFact[]} facts
+ */
+export function pageFactsMarkdown(facts: readonly import("./types.js").PageFact[]): string;

package/dist/types/hidden.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @param {string} text
+ * @returns {{ text: string, runs: import("./types.js").HiddenRun[] }}
+ */
+export function removeHiddenText(text: string): {
+    text: string;
+    runs: import("./types.js").HiddenRun[];
+};
+/**
+ * One omission per kind of run: `count` runs, `chars` code points.
+ * @param {ReadonlyArray<Pick<import("./types.js").HiddenRun, "kind"|"count">>} runs
+ * @returns {import("./types.js").HiddenTextOmission[]}
+ */
+export function hiddenTextOmissions(runs: ReadonlyArray<Pick<import("./types.js").HiddenRun, "kind" | "count">>): import("./types.js").HiddenTextOmission[];