@page2ai/core 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -45,13 +45,14 @@ console.log(result.markdown);
45
45
 
46
46
  Converts an HTML string to Markdown. Synchronous.
47
47
 
48
- - `html` (string) — the HTML to parse
48
+ - `html` (string) — the HTML to parse: a whole document or a fragment (an HTMX partial, a template snippet, plain text)
49
49
  - `opts.baseUrl` (string) — URL used to resolve relative links (default: `'about:blank'`)
50
- - `opts.profile` (string) — extraction profile: `'auto' | 'docs' | 'marketing' | 'research' | 'dashboard' | 'wordpress-marketing'` (default: `'auto'`, v0.2)
50
+ - `opts.profile` (string) — accepted, but **ignored by the Node build in this version**: `'dashboard'`, `'marketing'` and the rest produce the same output here. The profiles switch the browser extension's live-page pipeline (clicking tabs, waiting for lazy content), which the Node renderer does not run
51
51
  - `opts.includeFrontmatter` (boolean) — emit YAML frontmatter block (default: `true`)
52
- - `opts.includeImages` (boolean) — emit image links (default: `true`)
52
+ - `opts.includeImages` (boolean) — emit image links (default: `true`). Inside a table cell an image is written as its alt text either way, and an image with an empty alt or `aria-hidden="true"` adds nothing
53
+ - `opts.includeHidden` (boolean) — also convert content under the HTML `hidden` attribute, which a browser does not show (default: `false`). Tab panels, `hidden="until-found"` sections and React streaming segments are converted either way, and a hidden block that holds code contributes its code blocks and nothing else. Code, link names and tab labels never take text from a hidden element. A `<main>` under `hidden` is not taken as the page's main content while a visible one exists, and one inside a hidden tab panel or `until-found` section never is. The default is not a security boundary: those kept kinds are set by the page's own attributes, so a page can still put text a reader does not see into the output that way, and the converter reads no CSS (`display: none` is converted)
53
54
 
54
- Returns `{ markdown, title, baseUrl, charCount, extractedAt }`.
55
+ Returns `{ markdown, title, baseUrl, charCount, extractedAt }`. When nothing can be extracted the Markdown says so with `<!-- AI: page has no extractable body content -->` instead of coming back empty.
55
56
 
56
57
  ### `fetchAndConvert(url, opts?)`
57
58
 
@@ -60,22 +61,26 @@ Fetches a URL and converts it to Markdown. Async.
60
61
  - `url` (string) — the URL to fetch
61
62
  - `opts.timeoutMs` (number) — abort after N ms (default: `15000`)
62
63
  - `opts.userAgent` (string) — custom user agent (default: `page2ai-core/0.1`)
64
+ - `opts.tryMdSuffixFirst` (boolean) — first ask for `<url>.md` (or `<url>index.md` for a URL ending in `/`), the Markdown twin that Mintlify and other docs platforms serve, and return it when it answers with Markdown (default: `true`; one extra request per call, through the same guard)
63
65
  - ...all `htmlToMarkdown` options
64
66
 
65
- Uses Node 18+ built-in `fetch()`. Follows redirects; passes the final URL to `htmlToMarkdown` as `baseUrl` unless overridden.
67
+ Uses Node 18+ built-in `fetch()`. Follows redirects; passes the final URL to `htmlToMarkdown` as `baseUrl` unless overridden. A server that answers `text/markdown` (content negotiation: Fumadocs, Mintlify, Vercel, Supabase) gets its Markdown returned as-is; `source` in the result says which way it came (`md-suffix`, `content-negotiation` or `html-parse`), and `includeFrontmatter` applies on every path. The body is decoded as UTF-8 whatever charset the server declares, so a page in another encoding comes out garbled.
68
+
69
+ The fetch refuses anything that is not public, on the URL and on every redirect hop, after DNS resolution: `localhost`, loopback `127.0.0.0/8` and `::1`, `0.0.0.0/8`, private `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, link-local `169.254.0.0/16` (cloud metadata) and `fe80::/10`, CGNAT `100.64.0.0/10` (this includes Tailscale addresses), IPv6 unique-local `fc00::/7`, IPv4-mapped IPv6, the unspecified address `::`, and the hostnames `metadata` and `metadata.google.internal`. So a page on your own intranet or tailnet cannot be fetched this way: fetch the HTML yourself and pass it to `htmlToMarkdown`. `assertSafeUrlResolved(url)` runs the same check without fetching, and `findIpBlockReason(address)` applies its address rule; both are exported.
66
70
 
67
71
  ## What it does
68
72
 
69
73
  - Parses HTML with [linkedom](https://github.com/WebReflection/linkedom) (fast, browser-DOM-fidelity, no layout engine)
70
- - Picks a content root (`main article` > `article` > `main` > `[role="main"]` > `body`)
71
- - Emits: headings (`# `), paragraphs, lists (ordered/unordered, nested), links `[text](url)`, images `![alt](src)`, code blocks (with language tag when detectable), tables (`| a | b |`), blockquotes, inline formatting (bold, italic, code)
74
+ - Places React streaming content first. A page streamed by React (Next.js App Router, Remix, Mintlify) sends a fallback such as "Loading..." where content will go and the content itself later, hidden at the end of the page; the page's own inline scripts (`$RC`, `$RR`, `$RS`) move it into place in a browser. The converter runs no scripts, so it makes those moves itself, in document order, and drops the fallback, only where a browser runs the call: in a classic script (not one with `src`, `nomodule` or a non-JavaScript `type`, not inside `<noscript>` or `<template>`), as a statement of its own, on nodes parsed before that script and on the markup React writes for them. What a script would do when run is not checked: call text in a script that redefines `$RC` still moves a segment. A segment whose boundary is gone is dropped, as React drops it. A hidden segment that no call places stays where it stands: inside the content root it is converted there, outside it is left out. React's external runtime (`<template data-rci>`, an opt-in that no page we sampled uses) is not replayed
75
+ - Picks a content root: `<main>` (else `[role="main"]`, else `<body>`), replaced by its largest `<article>` when that article holds at least half of the root's text or is three times larger than every other `<article>`. Text outside the root (beside `<main>`, or beside such an article) is not converted
76
+ - Emits: headings (`# `), paragraphs, lists (ordered/unordered, nested), links `[text](url)`, images `![alt](src)`, code blocks (with language tag when detectable), tables (`| a | b |`; a cell spanning columns or rows leaves empty cells, so every value stays under its header; a span ends with its row group, as browsers lay it out, and a table whose spans would cover more than 100,000 cells is written in the plain layout, cells in source order), blockquotes, definition lists (`**term**` then `: definition`), inline formatting (bold, italic, code). Text sitting directly in a `<div>`, `<section>` or `<span>` is kept as its own paragraph, so dashboards and admin pages built from bare `<div>`s convert too
72
77
  - Extracts metadata into YAML frontmatter: `title`, `source`, `captured_at`, `language`, `description`, `canonical`, `og_title`, `og_description`, `author`, `published`, `extractor`, `extractor_version`
73
78
 
74
79
  ## What it does NOT do (v0.1)
75
80
 
76
81
  - No tab/dropdown expansion (browser-only interactive machinery)
77
82
  - No lazy-load activation (Node has no runtime images)
78
- - No `<details>` auto-expansion
83
+ - No `<details>` clicking, and none is needed: the body of a closed `<details>` is in the HTML and is converted with the rest of the page
79
84
  - No computed-style-based visibility filtering (linkedom has no layout engine)
80
85
  - No badge detection via font weight
81
86
  - No shadow DOM traversal (linkedom has no shadow DOM support)
@@ -1,8 +1,13 @@
1
1
  import type { PageAdapter } from '../shared/page-adapter.js';
2
+ export declare const documentTitle: (doc: Document) => string;
2
3
  interface RenderOpts {
3
4
  baseUrl: string;
4
5
  includeFrontmatter: boolean;
5
6
  includeImages: boolean;
7
+ includeHidden?: boolean;
8
+ inLink?: boolean;
9
+ tight?: boolean;
10
+ imageAlt?: boolean;
6
11
  }
7
12
  export declare const renderBasicMarkdown: (doc: Document, adapter: PageAdapter, opts: RenderOpts) => string;
8
13
  export {};
@@ -1 +1 @@
1
- {"version":3,"file":"basic-renderer.d.ts","sourceRoot":"","sources":["../../src/node/basic-renderer.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAU7D,UAAU,UAAU;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,aAAa,EAAE,OAAO,CAAC;CACxB;AAkBD,eAAO,MAAM,mBAAmB,GAC9B,KAAK,QAAQ,EACb,SAAS,WAAW,EACpB,MAAM,UAAU,KACf,MA8BF,CAAC"}
1
+ {"version":3,"file":"basic-renderer.d.ts","sourceRoot":"","sources":["../../src/node/basic-renderer.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAkC7D,eAAO,MAAM,aAAa,GAAI,KAAK,QAAQ,KAAG,MACwC,CAAC;AAEvF,UAAU,UAAU;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,aAAa,EAAE,OAAO,CAAC;IAEvB,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,MAAM,CAAC,EAAE,OAAO,CAAC;IAGjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAkBD,eAAO,MAAM,mBAAmB,GAC9B,KAAK,QAAQ,EACb,SAAS,WAAW,EACpB,MAAM,UAAU,KACf,MAiCF,CAAC"}