@page2ai/core 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -10,7 +10,8 @@
10
10
  Works in Node.js (via linkedom) and in browsers.
11
11
 
12
12
  - **Node.js**: `htmlToMarkdown(html, opts)` or `fetchAndConvert(url, opts)` — zero external API calls, uses linkedom for parsing
13
- - **Browser**: exports `BrowserAdapter` and the `PageAdapter` interface; the full extract pipeline lives in the extension repo (a migration is planned)
13
+ - **Browser tab or Web Worker**: `import { htmlToMarkdown } from '@page2ai/core/html'` — the same converter with no network code and no Node.js built-in, so a bundler ships it as is and the HTML never leaves the page (the [online converter](https://igorsaevets.github.io/page2ai-docs/convert/) runs it this way)
14
+ - **Browser, live page**: exports `BrowserAdapter` and the `PageAdapter` interface; the full extract pipeline lives in the extension repo (a migration is planned)
14
15
  - **Zero data collection**: no telemetry, no analytics, no third-party calls
15
16
  - **MIT** licensed
16
17
 
@@ -55,6 +56,13 @@ Converts an HTML string to Markdown. Synchronous.
55
56
 
56
57
  Returns `{ markdown, title, baseUrl, charCount, extractedAt }`. When nothing can be extracted the Markdown says so with `<!-- AI: page has no extractable body content -->` instead of coming back empty.
57
58
 
59
+ The same function is published alone as `@page2ai/core/html` (since 0.4.5). That entry imports only linkedom, css.escape and the renderer: no `fetch`, no undici, no `node:` module. Bundled for a browser tab or a Web Worker it gives the Markdown the package root gives in Node for the same HTML and options, and it loads nothing the HTML names: no script runs and no image, stylesheet or font is requested, because linkedom only parses.
60
+
61
+ ```javascript
62
+ import { htmlToMarkdown } from '@page2ai/core/html';
63
+ const { markdown } = htmlToMarkdown(await file.text(), { baseUrl: 'https://example.com/page' });
64
+ ```
65
+
58
66
  ### `fetchAndConvert(url, opts?)`
59
67
 
60
68
  Fetches a URL and converts it to Markdown. Async.
@@ -114,7 +122,8 @@ src/
114
122
  │ ├── constants.ts # DEFAULTS, BLOCK_TAGS, PII patterns
115
123
  │ └── profiles.ts # profile presets
116
124
  ├── node/ # LinkedomAdapter + htmlToMarkdown + basic-renderer
117
- │ ├── index.ts
125
+ │ ├── index.ts # package root: re-exports convert.ts, adds the guarded fetch
126
+ │ ├── convert.ts # htmlToMarkdown alone = '@page2ai/core/html' (no network code)
118
127
  │ ├── linkedom-adapter.ts
119
128
  │ └── basic-renderer.ts
120
129
  └── browser/ # BrowserAdapter; extract() comes with the port
@@ -0,0 +1,18 @@
1
+ import type { ProfileName } from '../shared/types.js';
2
+ export interface HtmlToMarkdownOptions {
3
+ baseUrl?: string;
4
+ profile?: ProfileName | 'auto';
5
+ includeFrontmatter?: boolean;
6
+ includeImages?: boolean;
7
+ includeHidden?: boolean;
8
+ }
9
+ export interface ConversionResult {
10
+ markdown: string;
11
+ title: string;
12
+ baseUrl: string;
13
+ charCount: number;
14
+ extractedAt: string;
15
+ source: 'html-parse' | 'markdown-response';
16
+ }
17
+ export declare const htmlToMarkdown: (html: string, opts?: HtmlToMarkdownOptions) => ConversionResult;
18
+ //# sourceMappingURL=convert.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../../src/node/convert.ts"],"names":[],"mappings":"AAuBA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,WAAW,qBAAqB;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC;IAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IAEpB,MAAM,EAAE,YAAY,GAAG,mBAAmB,CAAC;CAC5C;AAuLD,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,OAAM,qBAA0B,KAC/B,gBAgDF,CAAC"}
@@ -0,0 +1,270 @@
1
+ // @page2ai/core/html — an HTML string in, Markdown out.
2
+ //
3
+ // Public API:
4
+ // htmlToMarkdown(html, opts?) — convert an HTML string to Markdown
5
+ //
6
+ // No network and no Node.js built-in: this module's import graph is linkedom, css.escape and the
7
+ // renderer, so a bundler can ship it to a browser tab or a Web Worker (the online converter of the
8
+ // docs site runs it there). tests/node/public-api.test.ts walks that graph. The package root
9
+ // (index.ts) re-exports htmlToMarkdown and adds the guarded fetch: fetchAndConvert hands the page it
10
+ // fetched to this same function.
11
+ //
12
+ // SCOPE: basic renderer only (headings, paragraphs, lists, links, images,
13
+ // code blocks, tables, frontmatter, blockquotes, inline formatting) + STATIC
14
+ // tab discovery (aria-controls / role=tablist) that emits '## Tab: {label}'
15
+ // sections instead of concatenating panels. No interactive tab-click, no
16
+ // lazy-load, no <details> auto-expand. For the full pipeline see the browser build.
17
+ //
18
+ // A later release will port the full lib/core/html-to-md.ts renderer via PageAdapter.
19
+ import { parseHTML } from 'linkedom';
20
+ import { LinkedomAdapter } from './linkedom-adapter.js';
21
+ import { documentTitle, renderBasicMarkdown } from './basic-renderer.js';
22
+ import { CORE_VERSION } from '../shared/constants.js';
23
+ // linkedom builds the document of a FRAGMENT around its first element: `<style>..</style><div>..`
24
+ // gets <style> as documentElement, an empty synthetic <body>, and every later top-level node as
25
+ // a sibling outside both, so the walk found nothing and returned "" with no error (P2A-003: an
26
+ // HTMX-style partial; `<h1>T</h1><p>x</p>` loses even the heading). Plain text or "" gets no
27
+ // documentElement at all and linkedom's own `body` getter throws. Any parse whose root is not
28
+ // <html> is redone inside an html/body shell.
29
+ // A browser keeps what follows a stray </body> or </html>: HTML parsing reprocesses it in the body
30
+ // (spec "after body" modes). linkedom leaves it beside <body> or beside <html>, where the walk never
31
+ // looks, so `<body><p>a</p></body><p>b</p>` lost "b" (BL-37). Those nodes are moved into the body, a
32
+ // second <body> or <html> by its children. T110 deleted the two end tags from the source instead,
33
+ // which also deleted them inside <title>, <xmp> and <plaintext> text (T110 panel: grok, spark, agy).
34
+ const adoptAfterBody = (doc) => {
35
+ const body = doc.body;
36
+ const root = doc.documentElement;
37
+ if (!body || !root || body.parentNode !== root)
38
+ return doc;
39
+ const strays = [];
40
+ for (let n = body.nextSibling; n; n = n.nextSibling)
41
+ strays.push(n);
42
+ for (let n = root.nextSibling; n; n = n.nextSibling)
43
+ strays.push(n);
44
+ const adopt = (n) => {
45
+ if (n.nodeType === 1 && /^(?:BODY|HTML)$/.test(n.tagName)) {
46
+ for (const c of [...n.childNodes])
47
+ adopt(c);
48
+ n.parentNode?.removeChild(n);
49
+ }
50
+ else if (n.nodeType === 1 || (n.nodeType === 3 && (n.textContent || '').trim())) {
51
+ body.appendChild(n);
52
+ }
53
+ };
54
+ for (const n of strays)
55
+ adopt(n);
56
+ return doc;
57
+ };
58
+ // React streaming SSR (Fizz: Next.js App Router, Remix, Mintlify) sends the shell first, with a Suspense fallback
59
+ // where late content goes, `<!--$?--><template id="B:0"></template>fallback<!--/$-->`, and the content later, at the
60
+ // end of <body>: `<div hidden id="S:0">` and `<script>$RC("B:0","S:0")</script>`, which moves it into the boundary in
61
+ // place of the fallback. With no script run, the article stayed hidden beside <main> and the root printed the shell
62
+ // (the benchmark's docs.anthropic.com page: 212 characters of "Loading..." for a 12,270-character article, BL-39; 7 of
63
+ // 16 fresh Next.js / Mintlify / Remix pages stream, T126). So the calls are replayed in document order, as a browser
64
+ // runs them: $RC, and $RR (the same once stylesheets load), fill a boundary; $RS moves a segment onto its
65
+ // `<template id="P:1">` placeholder; $RX (client rendering) keeps the fallback. Chromium running only a page's inline
66
+ // scripts gave the converter the same page (8 of 8 pages, T126).
67
+ //
68
+ // A call counts only where a browser would run it (T127, the T126 panel; each case run in Chromium 149 with React's
69
+ // own runtime): call text in a `type="application/json"` script, a comment, a string or a <noscript> had moved a hidden
70
+ // segment into the root. So: a classic script (no `src`, no `nomodule`, a JavaScript `type` or none, outside <template>
71
+ // and <noscript>; React never writes a module script, and one runs after the classic ones), the call at a statement
72
+ // start as React writes it (all 23 calls on the 8 streaming pages begin the script or follow `;`), segment and boundary
73
+ // parsed before the script, as getElementById sees them when it runs, and only on markup React writes: one of its
74
+ // segment wrappers and a pending boundary closed by its own comment. What a script does is not checked: `$RC` defined
75
+ // as a no-op, or not at all, still moves segments; no real page streams without React's runtime. The external runtime
76
+ // (`<template data-rci>`, `unstable_externalRuntimeSrc`) is not replayed: none of the 16 pages uses it.
77
+ // Ids as React prints them: an optional identifierPrefix, then B:, S: or P: and hex.
78
+ const FILL_BOUNDARY = /\$R[CR]\("([^"\\]*B:[0-9a-f]+)","([^"\\]*S:[0-9a-f]+)"/;
79
+ const FILL_PLACEHOLDER = /\$RS\("([^"\\]*S:[0-9a-f]+)","([^"\\]*P:[0-9a-f]+)"/;
80
+ const STREAM_CALL = new RegExp(`(?<![^;])(?:${FILL_BOUNDARY.source}|${FILL_PLACEHOLDER.source})`, 'g');
81
+ const BOUNDARY_PENDING = /^\$[?~]$/; // `$~` = queued by React 19.2's batched reveal
82
+ const BOUNDARY_START = /^(?:\$[?~!]?|&)$/; // `&` = an <Activity> boundary
83
+ const REACT_ID = /[BSP]:[0-9a-f]+$/;
84
+ // The HTML standard's JavaScript MIME types ("JavaScript MIME type essence match", no parameters).
85
+ const JS_TYPE = /^((application|text)\/(x-)?(ecma|java)script|text\/(javascript1\.[0-5]|jscript|livescript))$/i;
86
+ const runsAsClassicScript = (script) => {
87
+ if (script.hasAttribute('src') || script.hasAttribute('nomodule'))
88
+ return false;
89
+ if (script.parentElement?.closest('template, noscript'))
90
+ return false;
91
+ const type = script.getAttribute('type');
92
+ if (type !== null)
93
+ return type === '' || JS_TYPE.test(type.trim());
94
+ const language = script.getAttribute('language');
95
+ return !language || JS_TYPE.test(`text/${language}`);
96
+ };
97
+ // React's segment wrappers (ReactFizzConfigDOM.js, writeStartSegment): `<div hidden>`, `<svg>` and `<math>` with
98
+ // aria-hidden="true" and display:none, `<table hidden>`, or the <tbody>, <tr> or <colgroup> of a `<table hidden>`.
99
+ const isSegment = (el) => {
100
+ if (!el)
101
+ return false;
102
+ const tag = el.tagName.toUpperCase();
103
+ if (tag === 'DIV' || tag === 'TABLE')
104
+ return el.hasAttribute('hidden');
105
+ if (tag === 'SVG' || tag === 'MATH') {
106
+ return el.getAttribute('aria-hidden') === 'true' && /display:\s*none/.test(el.getAttribute('style') || '');
107
+ }
108
+ return /^(?:TBODY|TR|COLGROUP)$/.test(tag) && !!el.closest('table')?.hasAttribute('hidden');
109
+ };
110
+ const scanDocument = (doc) => {
111
+ const order = new Map();
112
+ const ids = new Map();
113
+ for (const el of doc.querySelectorAll('*')) {
114
+ order.set(el, order.size);
115
+ if (!el.id || !REACT_ID.test(el.id))
116
+ continue;
117
+ const list = ids.get(el.id);
118
+ if (list)
119
+ list.push(el);
120
+ else
121
+ ids.set(el.id, [el]);
122
+ }
123
+ return { order, ids };
124
+ };
125
+ // getElementById as the script sees it when it runs: the first element with that id still in the document, if the
126
+ // parser had reached it before the script.
127
+ const lookup = (scan, id, script) => {
128
+ const list = scan.ids.get(id);
129
+ while (list?.length && !list[0].isConnected)
130
+ list.shift();
131
+ const el = list?.[0];
132
+ return el && (scan.order.get(el) ?? Infinity) < (scan.order.get(script) ?? -1) ? el : null;
133
+ };
134
+ const fillBoundary = (scan, script, boundaryId, segmentId) => {
135
+ const segment = lookup(scan, segmentId, script);
136
+ if (!isSegment(segment))
137
+ return;
138
+ const marker = lookup(scan, boundaryId, script);
139
+ if (!marker) {
140
+ segment.remove(); // React's $RC drops a segment whose boundary is gone: the browser never shows it
141
+ return;
142
+ }
143
+ const open = marker.previousSibling;
144
+ if (marker.tagName !== 'TEMPLATE' || open?.nodeType !== 8 || !BOUNDARY_PENDING.test(open.data))
145
+ return;
146
+ const parent = marker.parentNode;
147
+ if (!parent || segment.contains(marker))
148
+ return;
149
+ let close = marker.nextSibling;
150
+ for (let depth = 0; close; close = close.nextSibling) {
151
+ if (close.nodeType !== 8)
152
+ continue;
153
+ const data = close.data;
154
+ if (data === '/$' || data === '/&') {
155
+ if (!depth)
156
+ break;
157
+ depth--;
158
+ }
159
+ else if (BOUNDARY_START.test(data))
160
+ depth++;
161
+ }
162
+ if (!close)
163
+ return;
164
+ segment.remove();
165
+ for (let n = marker; n && n !== close;) {
166
+ const next = n.nextSibling;
167
+ parent.removeChild(n);
168
+ n = next;
169
+ }
170
+ while (segment.firstChild)
171
+ parent.insertBefore(segment.firstChild, close);
172
+ open.data = '$';
173
+ };
174
+ const fillPlaceholder = (scan, script, segmentId, placeholderId) => {
175
+ const segment = lookup(scan, segmentId, script);
176
+ if (!isSegment(segment))
177
+ return;
178
+ const placeholder = lookup(scan, placeholderId, script);
179
+ if (!placeholder) {
180
+ segment.remove(); // React's $RS removes the segment, then fails on the missing placeholder
181
+ return;
182
+ }
183
+ const parent = placeholder.parentNode;
184
+ if (placeholder.tagName !== 'TEMPLATE' || !parent || segment.contains(placeholder))
185
+ return;
186
+ segment.remove();
187
+ while (segment.firstChild)
188
+ parent.insertBefore(segment.firstChild, placeholder);
189
+ placeholder.remove();
190
+ };
191
+ const revealStreamedSegments = (doc) => {
192
+ let scan;
193
+ for (const script of [...doc.querySelectorAll('script')]) {
194
+ const text = script.textContent || '';
195
+ if (!text.includes('$R') || !runsAsClassicScript(script))
196
+ continue;
197
+ for (const m of text.matchAll(STREAM_CALL)) {
198
+ scan ??= scanDocument(doc);
199
+ if (m[1])
200
+ fillBoundary(scan, script, m[1], m[2]);
201
+ else
202
+ fillPlaceholder(scan, script, m[3], m[4]);
203
+ }
204
+ }
205
+ return doc;
206
+ };
207
+ const parseDocument = (source) => {
208
+ const first = parseHTML(source).document;
209
+ if (first.documentElement?.tagName === 'HTML')
210
+ return adoptAfterBody(first);
211
+ const inner = source.replace(/^/, '').replace(/^\s*<\?xml[^>]*>/i, '').replace(/^\s*<!doctype[^>]*>/i, '');
212
+ // Only a fragment that STARTS with <head> or <body> brings its own; one met later is wrapped like any
213
+ // other content. Matching `<body` anywhere left `the <body> tag is required` outside the body, losing
214
+ // "the " (mimo, T109).
215
+ const shell = /^\s*(?:<!--[\s\S]*?-->\s*)*<(?:head|body)[\s>]/i.test(inner)
216
+ ? `<!DOCTYPE html><html>${inner}</html>`
217
+ : `<!DOCTYPE html><html><head></head><body>${inner}</body></html>`;
218
+ return adoptAfterBody(parseHTML(shell).document);
219
+ };
220
+ export const htmlToMarkdown = (html, opts = {}) => {
221
+ const injectedBase = opts.baseUrl ?? 'about:blank';
222
+ try {
223
+ const document = revealStreamedSegments(parseDocument(html));
224
+ const adapter = new LinkedomAdapter(document, injectedBase);
225
+ const title = documentTitle(document).trim();
226
+ const markdown = renderBasicMarkdown(document, adapter, {
227
+ baseUrl: adapter.baseUrl(),
228
+ includeFrontmatter: opts.includeFrontmatter ?? true,
229
+ includeImages: opts.includeImages ?? true,
230
+ includeHidden: opts.includeHidden ?? false,
231
+ });
232
+ return {
233
+ markdown,
234
+ title,
235
+ baseUrl: adapter.baseUrl(),
236
+ charCount: markdown.length,
237
+ extractedAt: new Date().toISOString(),
238
+ source: 'html-parse',
239
+ };
240
+ }
241
+ catch (e) {
242
+ const errMsg = e instanceof Error ? e.message : String(e);
243
+ const fallback = [
244
+ '---',
245
+ `title: "Page2AI extraction failed"`,
246
+ `source: "${injectedBase}"`,
247
+ `captured_at: "${new Date().toISOString()}"`,
248
+ `extractor: "page2ai-core"`,
249
+ `extractor_version: "${CORE_VERSION}"`,
250
+ `error: "${errMsg.replace(/"/g, '\\"')}"`,
251
+ '---',
252
+ '',
253
+ '# Page2AI extraction failed',
254
+ '',
255
+ `Could not parse HTML from ${injectedBase}.`,
256
+ '',
257
+ `Error: ${errMsg}`,
258
+ '',
259
+ ].join('\n');
260
+ return {
261
+ markdown: fallback,
262
+ title: 'Page2AI extraction failed',
263
+ baseUrl: injectedBase,
264
+ charCount: fallback.length,
265
+ extractedAt: new Date().toISOString(),
266
+ source: 'html-parse',
267
+ };
268
+ }
269
+ };
270
+ //# sourceMappingURL=convert.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"convert.js","sourceRoot":"","sources":["../../src/node/convert.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,EAAE;AACF,cAAc;AACd,sEAAsE;AACtE,EAAE;AACF,iGAAiG;AACjG,mGAAmG;AACnG,6FAA6F;AAC7F,qGAAqG;AACrG,iCAAiC;AACjC,EAAE;AACF,0EAA0E;AAC1E,6EAA6E;AAC7E,4EAA4E;AAC5E,yEAAyE;AACzE,oFAAoF;AACpF,EAAE;AACF,sFAAsF;AAEtF,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAsBtD,kGAAkG;AAClG,gGAAgG;AAChG,+FAA+F;AAC/F,6FAA6F;AAC7F,8FAA8F;AAC9F,8CAA8C;AAC9C,mGAAmG;AACnG,qGAAqG;AACrG,qGAAqG;AACrG,kGAAkG;AAClG,qGAAqG;AACrG,MAAM,cAAc,GAAG,CAAC,GAAa,EAAY,EAAE;IACjD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,IAAI,GAAG,GAAG,CAAC,eAAe,CAAC;IACjC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IAC3D,MAAM,MAAM,GAAW,EAAE,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW;QAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW;QAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,CAAC,CAAO,EAAQ,EAAE;QAC9B,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAE,CAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YACvE,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC;gBAAE,KAAK,CAAC,CAAC,CAAC,CAAC;YAC5C,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;aAAM,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAClF,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACjC,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,kHAAkH;AAClH,qHAAqH;AACrH,sHAAsH;AACtH,oHAAoH;AACpH,uHAAuH;AACvH,qHAAqH;AACrH,0GAA0G;AAC1G,sHAAsH;AACtH,iEAAiE;AACjE,EAAE;AACF,oHAAoH;AACpH,wHAAwH;AACxH,wHAAwH;AACxH,oHAAoH;AACpH,wHAAwH;AACxH,kHAAkH;AAClH,sHAAsH;AACtH,sHAAsH;AACtH,wGAAwG;AACxG,qFAAqF;AACrF,MAAM,aAAa,GAAG,wDAAwD,CAAC;AAC/E,MAAM,gBAAgB,GAAG,qDAAqD,CAAC;AAC/E,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,eAAe,aAAa,CAAC,MAAM,IAAI,gBAAgB,CAAC,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC;AACvG,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,+CAA+C;AACpF,MAAM,cAAc,GAAG,kBAAkB,CAAC,CAAC,+BAA+B;AAC1E,MAAM,QAAQ,GAAG,kBAAkB,CAAC;AACpC,mGAAmG;AACnG,MAAM,OAAO,GAAG,+FAA+F,CAAC;AAEhH,MAAM,mBAAmB,GAAG,CAAC,MAAe,EAAW,EAAE;IACvD,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAChF,IAAI,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,oBAAoB,CAAC;QAAE,OAAO,KAAK,CAAC;IACtE,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,IAAI,KAAK,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;IACjD,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,QAAQ,EAAE,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,iHAAiH;AACjH,mHAAmH;AACnH,MAAM,SAAS,GAAG,CAAC,EAAkB,EAAiB,EAAE;IACtD,IAAI,CAAC,EAAE;QAAE,OAAO,KAAK,CAAC;IACtB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACrC,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,OAAO;QAAE,OAAO,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvE,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACpC,OAAO,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,MAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7G,CAAC;IACD,OAAO,yBAAyB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;AAC9F,CAAC,CAAC;AAOF,MAAM,YAAY,GAAG,CAAC,GAAa,EAAc,EAAE;IACjD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmB,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAqB,CAAC;IACzC,KAAK,MAAM,EAAE,IAAI,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3C,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAAE,SAAS;QAC9C,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;;YACnB,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;AACxB,CAAC,CAAC;AAEF,kHAAkH;AAClH,2CAA2C;AAC3C,MAAM,MAAM,GAAG,CAAC,IAAgB,EAAE,EAAU,EAAE,MAAe,EAAkB,EAAE;IAC/E,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9B,OAAO,IAAI,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW;QAAE,IAAI,CAAC,KAAK,EAAE,CAAC;IAC1D,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACrB,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7F,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,IAAgB,EAAE,MAAe,EAAE,UAAkB,EAAE,SAAiB,EAAQ,EAAE;IACtG,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAChD,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAChC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAChD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,iFAAiF;QACnG,OAAO;IACT,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,eAAe,CAAC;IACpC,IAAI,MAAM,CAAC,OAAO,KAAK,UAAU,IAAI,IAAI,EAAE,QAAQ,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAE,IAAgB,CAAC,IAAI,CAAC;QAAE,OAAO;IACpH,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC;IACjC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO;IAChD,IAAI,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC;IAC/B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QACrD,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC;YAAE,SAAS;QACnC,MAAM,IAAI,GAAI,KAAiB,CAAC,IAAI,CAAC;QACrC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,KAAK;gBAAE,MAAM;YAClB,KAAK,EAAE,CAAC;QACV,CAAC;aAAM,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,KAAK,EAAE,CAAC;IAChD,CAAC;IACD,IAAI,CAAC,KAAK;QAAE,OAAO;IACnB,OAAO,CAAC,MAAM,EAAE,CAAC;IACjB,KAAK,IAAI,CAAC,GAAqB,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,GAAI,CAAC;QAC1D,MAAM,IAAI,GAAqB,CAAC,CAAC,WAAW,CAAC;QAC7C,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC,GAAG,IAAI,CAAC;IACX,CAAC;IACD,OAAO,OAAO,CAAC,UAAU;QAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACzE,IAAgB,CAAC,IAAI,GAAG,GAAG,CAAC;AAC/B,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,IAAgB,EAAE,MAAe,EAAE,SAAiB,EAAE,aAAqB,EAAQ,EAAE;IAC5G,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAChD,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAChC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;IACxD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,yEAAyE;QAC3F,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC;IACtC,IAAI,WAAW,CAAC,OAAO,KAAK,UAAU,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO;IAC3F,OAAO,CAAC,MAAM,EAAE,CAAC;IACjB,OAAO,OAAO,CAAC,UAAU;QAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAChF,WAAW,CAAC,MAAM,EAAE,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,GAAa,EAAY,EAAE;IACzD,IAAI,IAA4B,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;YAAE,SAAS;QACnE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3C,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAE,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;gBAC5C,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,MAAc,EAAY,EAAE;IACjD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAA+B,CAAC;IAChE,IAAI,KAAK,CAAC,eAAe,EAAE,OAAO,KAAK,MAAM;QAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;IAC5E,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC;IAC5G,sGAAsG;IACtG,sGAAsG;IACtG,uBAAuB;IACvB,MAAM,KAAK,GAAG,iDAAiD,CAAC,IAAI,CAAC,KAAK,CAAC;QACzE,CAAC,CAAC,wBAAwB,KAAK,SAAS;QACxC,CAAC,CAAC,2CAA2C,KAAK,gBAAgB,CAAC;IACrE,OAAO,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,QAA+B,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,IAAY,EACZ,OAA8B,EAAE,EACd,EAAE;IACpB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,IAAI,aAAa,CAAC;IACnD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,sBAAsB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,OAAO,EAAE;YACtD,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;YAC1B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,IAAI,IAAI;YACnD,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;YACzC,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,KAAK;SAC3C,CAAC,CAAC;QACH,OAAO;YACL,QAAQ;YACR,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;YAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;IAAC,OAAO,CAAU,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG;YACf,KAAK;YACL,oCAAoC;YACpC,YAAY,YAAY,GAAG;YAC3B,iBAAiB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG;YAC5C,2BAA2B;YAC3B,uBAAuB,YAAY,GAAG;YACtC,WAAW,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG;YACzC,KAAK;YACL,EAAE;YACF,6BAA6B;YAC7B,EAAE;YACF,6BAA6B,YAAY,GAAG;YAC5C,EAAE;YACF,UAAU,MAAM,EAAE;YAClB,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,2BAA2B;YAClC,OAAO,EAAE,YAAY;YACrB,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;AACH,CAAC,CAAC"}
@@ -1,3 +1,4 @@
1
+ import { Agent } from 'undici';
1
2
  import { type FetchPolicyOptions } from './fetch-policy.js';
2
3
  export interface FetchProtectedOptions extends FetchPolicyOptions {
3
4
  timeoutMs?: number;
@@ -14,13 +15,21 @@ export interface FetchProtectedResult {
14
15
  bytesRead: number;
15
16
  statusCode: number;
16
17
  }
17
- export declare const DEFAULT_UA = "page2ai-core/0.4.3 (+https://github.com/igorsaevets/page2ai-core)";
18
+ export declare const DEFAULT_UA = "page2ai-core/0.4.5 (+https://github.com/igorsaevets/page2ai-core)";
18
19
  export declare const BROWSER_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
20
+ export declare const TLS_CIPHERS: string;
21
+ export type TlsProfile = 'mozilla' | 'node-default';
22
+ export declare const isTlsNegotiationFailure: (err: unknown) => boolean;
19
23
  export declare class FetchProtectionError extends Error {
20
24
  readonly url: string;
21
25
  constructor(reason: string, url: string);
22
26
  }
23
27
  export declare const getProxyForUrl: (target: URL) => string | null;
28
+ /** Test-only: the dispatcher a hostname's hop uses without a proxy. */
29
+ export declare const _createPinnedAgent: (addrs: {
30
+ address: string;
31
+ family: number;
32
+ }[], profile: TlsProfile) => Agent;
24
33
  export declare const _waitForHost: (host: string) => Promise<void>;
25
34
  export declare const _resetHostThrottle: () => void;
26
35
  export declare const fetchProtected: (rawUrl: string, opts?: FetchProtectedOptions) => Promise<FetchProtectedResult>;
@@ -1 +1 @@
1
- {"version":3,"file":"fetch-protected.d.ts","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"AAuCA,OAAO,EAKL,KAAK,kBAAkB,EACxB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,WAAW,qBAAsB,SAAQ,kBAAkB;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uGAAuG;IACvG,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,UAAU,sEAAgF,CAAC;AAOxG,eAAO,MAAM,cAAc,oEAAoE,CAAC;AAGhG,qBAAa,oBAAqB,SAAQ,KAAK;aACD,GAAG,EAAE,MAAM;gBAA3C,MAAM,EAAE,MAAM,EAAkB,GAAG,EAAE,MAAM;CAIxD;AAgBD,eAAO,MAAM,cAAc,GAAI,QAAQ,GAAG,KAAG,MAAM,GAAG,IAwBrD,CAAC;AA6BF,eAAO,MAAM,YAAY,GAAU,MAAM,MAAM,KAAG,OAAO,CAAC,IAAI,CAiB7D,CAAC;AAEF,eAAO,MAAM,kBAAkB,QAAO,IAAkC,CAAC;AAEzE,eAAO,MAAM,cAAc,GACzB,QAAQ,MAAM,EACd,OAAM,qBAA0B,KAC/B,OAAO,CAAC,oBAAoB,CA6I9B,CAAC"}
1
+ {"version":3,"file":"fetch-protected.d.ts","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"AAoCA,OAAO,EAAE,KAAK,EAAc,MAAM,QAAQ,CAAC;AAM3C,OAAO,EAKL,KAAK,kBAAkB,EACxB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,WAAW,qBAAsB,SAAQ,kBAAkB;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uGAAuG;IACvG,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,UAAU,sEAAgF,CAAC;AAOxG,eAAO,MAAM,cAAc,oEAAoE,CAAC;AAUhG,eAAO,MAAM,WAAW,QAUb,CAAC;AAGZ,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,cAAc,CAAC;AAWpD,eAAO,MAAM,uBAAuB,GAAI,KAAK,OAAO,KAAG,OAMtD,CAAC;AAEF,qBAAa,oBAAqB,SAAQ,KAAK;aACD,GAAG,EAAE,MAAM;gBAA3C,MAAM,EAAE,MAAM,EAAkB,GAAG,EAAE,MAAM;CAIxD;AAgBD,eAAO,MAAM,cAAc,GAAI,QAAQ,GAAG,KAAG,MAAM,GAAG,IAwBrD,CAAC;AAuBF,uEAAuE;AACvE,eAAO,MAAM,kBAAkB,UArBtB;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,WACnC,UAAU,KAClB,KAmBgD,CAAC;AAUpD,eAAO,MAAM,YAAY,GAAU,MAAM,MAAM,KAAG,OAAO,CAAC,IAAI,CAiB7D,CAAC;AAEF,eAAO,MAAM,kBAAkB,QAAO,IAAkC,CAAC;AAEzE,eAAO,MAAM,cAAc,GACzB,QAAQ,MAAM,EACd,OAAM,qBAA0B,KAC/B,OAAO,CAAC,oBAAoB,CAuJ9B,CAAC"}
@@ -28,6 +28,9 @@
28
28
  // is NOT subject to the SSRF guard. When NO_PROXY matches the target, the
29
29
  // direct DNS-pinned path is used as before.
30
30
  //
31
+ // - TLS: the cipher list of Mozilla's "intermediate" configuration (TLS_CIPHERS), not Node's
32
+ // default one; a server that speaks none of it gets one retry with Node's defaults.
33
+ //
31
34
  // KNOWN LIMITATION: no charset detection for non-UTF-8 pages (Windows-1251 docs will
32
35
  // mojibake); a later release may add it via `fetch-charset-detection`.
33
36
  import { Agent, ProxyAgent } from 'undici';
@@ -43,6 +46,39 @@ const DEFAULT_ACCEPT = /^(text\/html|text\/markdown|text\/x-markdown|text\/plain
43
46
  // A server that answered this header with Markdown would answer every Firefox user with it.
44
47
  export const BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
45
48
  const MAX_REDIRECTS = 10;
49
+ // TLS cipher list for every outgoing connection: Mozilla's "intermediate" configuration, guidelines 6.0
50
+ // (https://ssl-config.mozilla.org/guidelines/6.0.json, read 2026-11-12): the three TLS 1.3 suites, then the six
51
+ // TLS 1.2 suites with ECDHE and AEAD (GCM, ChaCha20). Node's default list (24 entries on Node 24) also offers CBC
52
+ // and DHE suites, and Cloudflare's bot checks challenge that handshake whatever the User-Agent says. Measured
53
+ // 2026-11-12 from one PC with this module's own User-Agent: npmjs.com, openai.com and medium.com answered 403
54
+ // `cf-mitigated: challenge` to Node's default list and 200 with the page to this one, as they did to curl. The
55
+ // User-Agent does not change: a site still sees page2ai by name and can refuse it by name.
56
+ export const TLS_CIPHERS = [
57
+ 'TLS_AES_128_GCM_SHA256',
58
+ 'TLS_AES_256_GCM_SHA384',
59
+ 'TLS_CHACHA20_POLY1305_SHA256',
60
+ 'ECDHE-ECDSA-AES128-GCM-SHA256',
61
+ 'ECDHE-RSA-AES128-GCM-SHA256',
62
+ 'ECDHE-ECDSA-AES256-GCM-SHA384',
63
+ 'ECDHE-RSA-AES256-GCM-SHA384',
64
+ 'ECDHE-ECDSA-CHACHA20-POLY1305',
65
+ 'ECDHE-RSA-CHACHA20-POLY1305',
66
+ ].join(':');
67
+ const tlsOptionsFor = (profile) => profile === 'mozilla' ? { ciphers: TLS_CIPHERS } : {};
68
+ // The handshake failed on the protocol or the cipher list: the server sent a handshake_failure,
69
+ // protocol_version or insufficient_security alert, or nothing on offer matched. OpenSSL 3.0 (Node 20, 22)
70
+ // names the first alert `ERR_SSL_SSLV3_ALERT_HANDSHAKE_FAILURE`, OpenSSL 3.5 (Node 24)
71
+ // `ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE` (measured 2026-11-12). Certificate errors (CERT_HAS_EXPIRED,
72
+ // ERR_TLS_CERT_ALTNAME_INVALID, ...) do not match: another cipher list cannot fix them.
73
+ const TLS_NEGOTIATION_CODE = /^ERR_SSL_(?:(?:SSLV3|SSL\/TLS|TLSV1)_ALERT_(?:HANDSHAKE_FAILURE|PROTOCOL_VERSION|INSUFFICIENT_SECURITY)|NO_SHARED_CIPHER|NO_CIPHERS_AVAILABLE|NO_SUITABLE_SIGNATURE_ALGORITHM)$/;
74
+ export const isTlsNegotiationFailure = (err) => {
75
+ for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) {
76
+ const code = e.code;
77
+ if (typeof code === 'string' && TLS_NEGOTIATION_CODE.test(code))
78
+ return true;
79
+ }
80
+ return false;
81
+ };
46
82
  export class FetchProtectionError extends Error {
47
83
  url;
48
84
  constructor(reason, url) {
@@ -88,8 +124,9 @@ export const getProxyForUrl = (target) => {
88
124
  : process.env.http_proxy || process.env.HTTP_PROXY;
89
125
  return proxy || null;
90
126
  };
91
- const createPinnedAgent = (addrs) => new Agent({
127
+ const createPinnedAgent = (addrs, profile) => new Agent({
92
128
  connect: {
129
+ ...tlsOptionsFor(profile),
93
130
  lookup: (_hostname, options, callback) => {
94
131
  if (options.all) {
95
132
  callback(null, addrs);
@@ -100,6 +137,8 @@ const createPinnedAgent = (addrs) => new Agent({
100
137
  },
101
138
  },
102
139
  });
140
+ /** Test-only: the dispatcher a hostname's hop uses without a proxy. */
141
+ export const _createPinnedAgent = createPinnedAgent;
103
142
  // Per-host rate limiting: prevents a looping agent from hammering one site.
104
143
  // The Map survives across tool calls because the MCP server is a long-running
105
144
  // process. Redirect hops within one fetchProtected call are not throttled
@@ -159,36 +198,50 @@ export const fetchProtected = async (rawUrl, opts = {}) => {
159
198
  assertPortAllowed(parsed, policy, currentUrl);
160
199
  const host = normalizeHost(parsed.hostname);
161
200
  const proxyUrl = getProxyForUrl(parsed);
162
- let dispatcher;
201
+ let addrs = null;
163
202
  if (isIpv4Literal(host) || host.includes(':')) {
164
203
  assertAddressAllowed(host, host, policy, currentUrl);
165
204
  }
166
205
  else {
167
- const addrs = await resolveAndValidate(host, currentUrl);
206
+ addrs = await resolveAndValidate(host, currentUrl);
168
207
  for (const { address } of addrs)
169
208
  assertAddressAllowed(address, host, policy, currentUrl);
170
- if (!proxyUrl) {
171
- dispatcher = createPinnedAgent(addrs);
172
- }
173
- }
174
- if (proxyUrl && !dispatcher) {
175
- dispatcher = new ProxyAgent({ uri: proxyUrl });
176
209
  }
177
- try {
178
- resp = await fetch(currentUrl, {
179
- signal: controller.signal,
180
- headers: {
181
- 'user-agent': ua,
182
- accept: BROWSER_ACCEPT,
183
- },
184
- redirect: 'manual',
185
- ...(dispatcher ? { dispatcher } : {}),
186
- });
187
- }
188
- finally {
189
- if (dispatcher) {
210
+ // Through a proxy the proxy resolves the target (see the header); otherwise the socket is pinned
211
+ // to the validated addresses. An IP literal needs no pinning, only the TLS options.
212
+ const makeDispatcher = (profile) => {
213
+ if (proxyUrl)
214
+ return new ProxyAgent({ uri: proxyUrl, requestTls: tlsOptionsFor(profile) });
215
+ if (addrs)
216
+ return createPinnedAgent(addrs, profile);
217
+ return new Agent({ connect: tlsOptionsFor(profile) });
218
+ };
219
+ const send = async (profile) => {
220
+ const dispatcher = makeDispatcher(profile);
221
+ try {
222
+ return await fetch(currentUrl, {
223
+ signal: controller.signal,
224
+ headers: {
225
+ 'user-agent': ua,
226
+ accept: BROWSER_ACCEPT,
227
+ },
228
+ redirect: 'manual',
229
+ dispatcher,
230
+ });
231
+ }
232
+ finally {
190
233
  dispatcher.close().catch(() => { });
191
234
  }
235
+ };
236
+ try {
237
+ resp = await send('mozilla');
238
+ }
239
+ catch (err) {
240
+ // A server that speaks none of TLS_CIPHERS (TLS 1.2 with CBC suites only) refuses the
241
+ // handshake before any HTTP is sent: the same hop once more with Node's default list.
242
+ if (controller.signal.aborted || !isTlsNegotiationFailure(err))
243
+ throw err;
244
+ resp = await send('node-default');
192
245
  }
193
246
  if (resp.status >= 300 && resp.status < 400) {
194
247
  const location = resp.headers.get('location');
@@ -1 +1 @@
1
- {"version":3,"file":"fetch-protected.js","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,6EAA6E;AAC7E,gFAAgF;AAChF,4EAA4E;AAC5E,wBAAwB;AACxB,wEAAwE;AACxE,sEAAsE;AACtE,2EAA2E;AAC3E,6DAA6D;AAC7D,wCAAwC;AACxC,+FAA+F;AAC/F,gGAAgG;AAChG,iBAAiB;AACjB,kGAAkG;AAClG,kGAAkG;AAClG,iGAAiG;AACjG,mGAAmG;AACnG,wFAAwF;AACxF,kGAAkG;AAClG,qGAAqG;AACrG,EAAE;AACF,gFAAgF;AAChF,+EAA+E;AAC/E,8EAA8E;AAC9E,2EAA2E;AAC3E,gFAAgF;AAChF,iFAAiF;AACjF,4EAA4E;AAC5E,8CAA8C;AAC9C,EAAE;AACF,qFAAqF;AACrF,uEAAuE;AAEvE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,oBAAoB,EACpB,iBAAiB,EACjB,kBAAkB,EAClB,kBAAkB,GAEnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAmBtD,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,YAAY,iDAAiD,CAAC;AACxG,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAE,QAAQ;AACrD,MAAM,cAAc,GAAG,oFAAoF,CAAC;AAC5G,sGAAsG;AACtG,+GAA+G;AAC/G,4FAA4F;AAC5F,MAAM,CAAC,MAAM,cAAc,GAAG,iEAAiE,CAAC;AAChG,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACD;IAA5C,YAAY,MAAc,EAAkB,GAAW;QACrD,KAAK,CAAC,oBAAoB,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC;QADR,QAAG,GAAH,GAAG,CAAQ;QAErD,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAE5D,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAU,EAAE;IACjD,IAAI,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACvD,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,sCAAsC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEpD,6EAA6E;AAC7E,8DAA8D;AAC9D,oEAAoE;AACpE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAW,EAAiB,EAAE;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;IACnE,IAAI,OAAO,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAEjC,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAE1E,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,WAAW,IAAI,WAAW,KAAK,IAAI;YAAE,SAAS;QAClD,IAAI,CAAC,WAAW;YAAE,SAAS;QAC3B,IAAI,QAAQ,KAAK,WAAW;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC;QAC7E,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GACT,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAC1B,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW;QACpD,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;IAEvD,OAAO,KAAK,IAAI,IAAI,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CACxB,KAA4C,EACrC,EAAE,CACT,IAAI,KAAK,CAAC;IACR,OAAO,EAAE;QACP,MAAM,EAAE,CACN,SAAiB,EACjB,OAA0B,EAC1B,QAAkB,EAClB,EAAE;YACF,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChB,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;KACF;CACF,CAAC,CAAC;AAEL,4EAA4E;AAC5E,8EAA8E;AAC9E,0EAA0E;AAC1E,kEAAkE;AAClE,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;AAChD,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,YAAY,GAAG,GAAG,CAAC;AAEzB,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,IAAY,EAAiB,EAAE;IAChE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC;QAC3B,IAAI,OAAO,GAAG,oBAAoB,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,oBAAoB,GAAG,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;YACnE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IACD,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpC,IAAI,aAAa,CAAC,IAAI,GAAG,YAAY,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC;aACxC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,IAAI,GAAG,YAAY,CAAC,CAAC;QAC/C,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM;YAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAS,EAAE,GAAG,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;AAEzE,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,MAAc,EACd,OAA8B,EAAE,EACD,EAAE;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,eAAe,CAAC;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACpD,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,IAAI,UAAU,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,IAAI,cAAc,CAAC;IACzD,6FAA6F;IAC7F,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAExC,8EAA8E;IAC9E,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,6EAA6E;IAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;IAClD,MAAM,YAAY,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACpD,IAAI,KAAK,EAAE,OAAO;QAAE,UAAU,CAAC,KAAK,EAAE,CAAC;;QAClC,KAAK,EAAE,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,qEAAqE;QACrE,uEAAuE;QACvE,uEAAuE;QACvE,qCAAqC;QACrC,IAAI,UAAU,GAAG,MAAM,CAAC;QACxB,IAAI,IAAc,CAAC;QACnB,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,SAAS,CAAC;YACR,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC5C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,UAA0C,CAAC;YAE/C,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9C,oBAAoB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACzD,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,KAAK;oBAAE,oBAAoB,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;gBACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACxC,CAAC;YACH,CAAC;YACD,IAAI,QAAQ,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC5B,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;YACjD,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;oBAC7B,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,YAAY,EAAE,EAAE;wBAChB,MAAM,EAAE,cAAc;qBACvB;oBACD,QAAQ,EAAE,QAAQ;oBAClB,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAA6B,CAAC,CAAC,CAAC,EAAE,CAAC;iBAClD,CAAC,CAAC;YACpB,CAAC;oBAAS,CAAC;gBACT,IAAI,UAAU,EAAE,CAAC;oBACf,UAAU,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,IAAI,CAAC,MAAM,qCAAqC,EACxD,MAAM,CACP,CAAC;gBACJ,CAAC;gBACD,IAAI,CAAC;oBAAC,MAAM,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC9D,IAAI,EAAE,CAAC;gBACP,IAAI,IAAI,GAAG,aAAa,EAAE,CAAC;oBACzB,MAAM,IAAI,oBAAoB,CAAC,aAAa,aAAa,YAAY,EAAE,MAAM,CAAC,CAAC;gBACjF,CAAC;gBACD,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACrD,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,IAAI,oBAAoB,CAAC,kCAAkC,QAAQ,GAAG,EAAE,MAAM,CAAC,CAAC;gBACxF,CAAC;gBACD,UAAU,GAAG,OAAO,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,MAAM;QACR,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC;QAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC3D,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,kBAAkB,WAAW,EAAE,EACrE,MAAM,CACP,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,oBAAoB,CAC5B,iBAAiB,WAAW,uBAAuB,EACnD,MAAM,CACP,CAAC;QACJ,CAAC;QAED,wEAAwE;QACxE,4EAA4E;QAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,oBAAoB,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM;YAChB,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC;YAC9B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;gBACzB,IAAI,CAAC;oBAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC1D,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,QAAQ,YAAY,EAC7C,MAAM,CACP,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChG,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAE7C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,2EAA2E;QAC3E,IAAI,CAAC,YAAY,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,oBAAoB,CAAC,0BAA0B,CAAC,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;QAChF,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,KAAK,EAAE,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACpD,CAAC;AACH,CAAC,CAAC"}
1
+ {"version":3,"file":"fetch-protected.js","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,6EAA6E;AAC7E,gFAAgF;AAChF,4EAA4E;AAC5E,wBAAwB;AACxB,wEAAwE;AACxE,sEAAsE;AACtE,2EAA2E;AAC3E,6DAA6D;AAC7D,wCAAwC;AACxC,+FAA+F;AAC/F,gGAAgG;AAChG,iBAAiB;AACjB,kGAAkG;AAClG,kGAAkG;AAClG,iGAAiG;AACjG,mGAAmG;AACnG,wFAAwF;AACxF,kGAAkG;AAClG,qGAAqG;AACrG,EAAE;AACF,gFAAgF;AAChF,+EAA+E;AAC/E,8EAA8E;AAC9E,2EAA2E;AAC3E,gFAAgF;AAChF,iFAAiF;AACjF,4EAA4E;AAC5E,8CAA8C;AAC9C,EAAE;AACF,6FAA6F;AAC7F,sFAAsF;AACtF,EAAE;AACF,qFAAqF;AACrF,uEAAuE;AAEvE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,oBAAoB,EACpB,iBAAiB,EACjB,kBAAkB,EAClB,kBAAkB,GAEnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAmBtD,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,YAAY,iDAAiD,CAAC;AACxG,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAE,QAAQ;AACrD,MAAM,cAAc,GAAG,oFAAoF,CAAC;AAC5G,sGAAsG;AACtG,+GAA+G;AAC/G,4FAA4F;AAC5F,MAAM,CAAC,MAAM,cAAc,GAAG,iEAAiE,CAAC;AAChG,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,wGAAwG;AACxG,gHAAgH;AAChH,kHAAkH;AAClH,8GAA8G;AAC9G,8GAA8G;AAC9G,+GAA+G;AAC/G,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,wBAAwB;IACxB,wBAAwB;IACxB,8BAA8B;IAC9B,+BAA+B;IAC/B,6BAA6B;IAC7B,+BAA+B;IAC/B,6BAA6B;IAC7B,+BAA+B;IAC/B,6BAA6B;CAC9B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAIZ,MAAM,aAAa,GAAG,CAAC,OAAmB,EAAwB,EAAE,CAClE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAExD,gGAAgG;AAChG,0GAA0G;AAC1G,uFAAuF;AACvF,yGAAyG;AACzG,wFAAwF;AACxF,MAAM,oBAAoB,GACxB,iLAAiL,CAAC;AACpL,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAY,EAAW,EAAE;IAC/D,KAAK,IAAI,CAAC,GAAY,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC,GAAI,CAAyB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;QACpG,MAAM,IAAI,GAAI,CAAwB,CAAC,IAAI,CAAC;QAC5C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;IAC/E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACD;IAA5C,YAAY,MAAc,EAAkB,GAAW;QACrD,KAAK,CAAC,oBAAoB,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC;QADR,QAAG,GAAH,GAAG,CAAQ;QAErD,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAE5D,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAU,EAAE;IACjD,IAAI,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACvD,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,sCAAsC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEpD,6EAA6E;AAC7E,8DAA8D;AAC9D,oEAAoE;AACpE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAW,EAAiB,EAAE;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;IACnE,IAAI,OAAO,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAEjC,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAE1E,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,WAAW,IAAI,WAAW,KAAK,IAAI;YAAE,SAAS;QAClD,IAAI,CAAC,WAAW;YAAE,SAAS;QAC3B,IAAI,QAAQ,KAAK,WAAW;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC;QAC7E,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GACT,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAC1B,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW;QACpD,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;IAEvD,OAAO,KAAK,IAAI,IAAI,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CACxB,KAA4C,EAC5C,OAAmB,EACZ,EAAE,CACT,IAAI,KAAK,CAAC;IACR,OAAO,EAAE;QACP,GAAG,aAAa,CAAC,OAAO,CAAC;QACzB,MAAM,EAAE,CACN,SAAiB,EACjB,OAA0B,EAC1B,QAAkB,EAClB,EAAE;YACF,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChB,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;KACF;CACF,CAAC,CAAC;AAEL,uEAAuE;AACvE,MAAM,CAAC,MAAM,kBAAkB,GAAG,iBAAiB,CAAC;AAEpD,4EAA4E;AAC5E,8EAA8E;AAC9E,0EAA0E;AAC1E,kEAAkE;AAClE,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;AAChD,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,YAAY,GAAG,GAAG,CAAC;AAEzB,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,IAAY,EAAiB,EAAE;IAChE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC;QAC3B,IAAI,OAAO,GAAG,oBAAoB,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,oBAAoB,GAAG,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;YACnE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IACD,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpC,IAAI,aAAa,CAAC,IAAI,GAAG,YAAY,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC;aACxC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,IAAI,GAAG,YAAY,CAAC,CAAC;QAC/C,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM;YAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAS,EAAE,GAAG,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;AAEzE,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,MAAc,EACd,OAA8B,EAAE,EACD,EAAE;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,eAAe,CAAC;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACpD,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,IAAI,UAAU,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,IAAI,cAAc,CAAC;IACzD,6FAA6F;IAC7F,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAExC,8EAA8E;IAC9E,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,6EAA6E;IAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;IAClD,MAAM,YAAY,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACpD,IAAI,KAAK,EAAE,OAAO;QAAE,UAAU,CAAC,KAAK,EAAE,CAAC;;QAClC,KAAK,EAAE,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,qEAAqE;QACrE,uEAAuE;QACvE,uEAAuE;QACvE,qCAAqC;QACrC,IAAI,UAAU,GAAG,MAAM,CAAC;QACxB,IAAI,IAAc,CAAC;QACnB,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,SAAS,CAAC;YACR,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC5C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,KAAK,GAAiD,IAAI,CAAC;YAE/D,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9C,oBAAoB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACnD,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,KAAK;oBAAE,oBAAoB,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC3F,CAAC;YACD,iGAAiG;YACjG,oFAAoF;YACpF,MAAM,cAAc,GAAG,CAAC,OAAmB,EAAsB,EAAE;gBACjE,IAAI,QAAQ;oBAAE,OAAO,IAAI,UAAU,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC3F,IAAI,KAAK;oBAAE,OAAO,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;gBACpD,OAAO,IAAI,KAAK,CAAC,EAAE,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACxD,CAAC,CAAC;YACF,MAAM,IAAI,GAAG,KAAK,EAAE,OAAmB,EAAqB,EAAE;gBAC5D,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;gBAC3C,IAAI,CAAC;oBACH,OAAO,MAAM,KAAK,CAAC,UAAU,EAAE;wBAC7B,MAAM,EAAE,UAAU,CAAC,MAAM;wBACzB,OAAO,EAAE;4BACP,YAAY,EAAE,EAAE;4BAChB,MAAM,EAAE,cAAc;yBACvB;wBACD,QAAQ,EAAE,QAAQ;wBAClB,UAAU;qBACI,CAAC,CAAC;gBACpB,CAAC;wBAAS,CAAC;oBACT,UAAU,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,sFAAsF;gBACtF,sFAAsF;gBACtF,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC;oBAAE,MAAM,GAAG,CAAC;gBAC1E,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,CAAC;YACpC,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,IAAI,CAAC,MAAM,qCAAqC,EACxD,MAAM,CACP,CAAC;gBACJ,CAAC;gBACD,IAAI,CAAC;oBAAC,MAAM,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC9D,IAAI,EAAE,CAAC;gBACP,IAAI,IAAI,GAAG,aAAa,EAAE,CAAC;oBACzB,MAAM,IAAI,oBAAoB,CAAC,aAAa,aAAa,YAAY,EAAE,MAAM,CAAC,CAAC;gBACjF,CAAC;gBACD,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACrD,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,IAAI,oBAAoB,CAAC,kCAAkC,QAAQ,GAAG,EAAE,MAAM,CAAC,CAAC;gBACxF,CAAC;gBACD,UAAU,GAAG,OAAO,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,MAAM;QACR,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC;QAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC3D,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,kBAAkB,WAAW,EAAE,EACrE,MAAM,CACP,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,oBAAoB,CAC5B,iBAAiB,WAAW,uBAAuB,EACnD,MAAM,CACP,CAAC;QACJ,CAAC;QAED,wEAAwE;QACxE,4EAA4E;QAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,oBAAoB,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM;YAChB,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC;YAC9B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;gBACzB,IAAI,CAAC;oBAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC1D,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,QAAQ,YAAY,EAC7C,MAAM,CACP,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChG,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAE7C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,2EAA2E;QAC3E,IAAI,CAAC,YAAY,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,oBAAoB,CAAC,0BAA0B,CAAC,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;QAChF,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,KAAK,EAAE,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACpD,CAAC;AACH,CAAC,CAAC"}
@@ -1,28 +1,14 @@
1
+ import type { ConversionResult, HtmlToMarkdownOptions } from './convert.js';
1
2
  import type { FetchPolicyOptions } from './fetch-policy.js';
2
- import type { ProfileName } from '../shared/types.js';
3
- export interface HtmlToMarkdownOptions {
4
- baseUrl?: string;
5
- profile?: ProfileName | 'auto';
6
- includeFrontmatter?: boolean;
7
- includeImages?: boolean;
8
- includeHidden?: boolean;
9
- }
10
3
  export interface FetchAndConvertOptions extends HtmlToMarkdownOptions, FetchPolicyOptions {
11
4
  timeoutMs?: number;
12
5
  maxBytes?: number;
13
6
  userAgent?: string;
14
7
  signal?: AbortSignal;
15
8
  }
16
- export interface ConversionResult {
17
- markdown: string;
18
- title: string;
19
- baseUrl: string;
20
- charCount: number;
21
- extractedAt: string;
22
- source: 'html-parse' | 'markdown-response';
23
- }
24
- export declare const htmlToMarkdown: (html: string, opts?: HtmlToMarkdownOptions) => ConversionResult;
25
9
  export declare const fetchAndConvert: (url: string, opts?: FetchAndConvertOptions) => Promise<ConversionResult>;
10
+ export { htmlToMarkdown } from './convert.js';
11
+ export type { ConversionResult, HtmlToMarkdownOptions } from './convert.js';
26
12
  export type { ProfileName } from '../shared/types.js';
27
13
  export { LinkedomAdapter } from './linkedom-adapter.js';
28
14
  export { SsrfBlockedError, assertSafeUrl, assertSafeUrlResolved, findIpBlockReason } from './ssrf-guard.js';
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/node/index.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAG5D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,WAAW,qBAAqB;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC;IAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,sBAAuB,SAAQ,qBAAqB,EAAE,kBAAkB;IACvF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IAEpB,MAAM,EAAE,YAAY,GAAG,mBAAmB,CAAC;CAC5C;AAuLD,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,OAAM,qBAA0B,KAC/B,gBAgDF,CAAC;AA8DF,eAAO,MAAM,eAAe,GAC1B,KAAK,MAAM,EACX,OAAM,sBAA2B,KAChC,OAAO,CAAC,gBAAgB,CAgD1B,CAAC;AAGF,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAIxD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC5G,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,cAAc,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC5H,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAC1F,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/node/index.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAE5E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAI5D,MAAM,WAAW,sBAAuB,SAAQ,qBAAqB,EAAE,kBAAkB;IACvF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AA8DD,eAAO,MAAM,eAAe,GAC1B,KAAK,MAAM,EACX,OAAM,sBAA2B,KAChC,OAAO,CAAC,gBAAgB,CAgD1B,CAAC;AAGF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAE5E,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAIxD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC5G,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,cAAc,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC5H,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAC1F,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC"}
@@ -1,269 +1,15 @@
1
1
  // @page2ai/core — Node.js entry point.
2
2
  //
3
3
  // Public API:
4
- // htmlToMarkdown(html, opts?) — convert an HTML string to Markdown
4
+ // htmlToMarkdown(html, opts?) — convert an HTML string to Markdown. It lives in convert.ts, which
5
+ // is also published alone as '@page2ai/core/html' (no network code).
5
6
  // fetchAndConvert(url, opts?) — SSRF-guarded fetch + convert
6
7
  //
7
- // SCOPE: basic renderer only (headings, paragraphs, lists, links, images,
8
- // code blocks, tables, frontmatter, blockquotes, inline formatting) + STATIC
9
- // tab discovery (aria-controls / role=tablist) that emits '## Tab: {label}'
10
- // sections instead of concatenating panels. No interactive tab-click, no
11
- // lazy-load, no <details> auto-expand. For the full pipeline see the browser build.
12
- //
13
- // A later release will port the full lib/core/html-to-md.ts renderer via PageAdapter.
14
- import { parseHTML } from 'linkedom';
15
- import { LinkedomAdapter } from './linkedom-adapter.js';
16
- import { documentTitle, renderBasicMarkdown } from './basic-renderer.js';
8
+ // What the conversion covers: the SCOPE note at the top of convert.ts.
9
+ import { htmlToMarkdown } from './convert.js';
17
10
  import { FetchProtectionError, fetchProtected } from './fetch-protected.js';
18
11
  import { assertSafeUrl } from './ssrf-guard.js';
19
12
  import { CORE_VERSION } from '../shared/constants.js';
20
- // linkedom builds the document of a FRAGMENT around its first element: `<style>..</style><div>..`
21
- // gets <style> as documentElement, an empty synthetic <body>, and every later top-level node as
22
- // a sibling outside both, so the walk found nothing and returned "" with no error (P2A-003: an
23
- // HTMX-style partial; `<h1>T</h1><p>x</p>` loses even the heading). Plain text or "" gets no
24
- // documentElement at all and linkedom's own `body` getter throws. Any parse whose root is not
25
- // <html> is redone inside an html/body shell.
26
- // A browser keeps what follows a stray </body> or </html>: HTML parsing reprocesses it in the body
27
- // (spec "after body" modes). linkedom leaves it beside <body> or beside <html>, where the walk never
28
- // looks, so `<body><p>a</p></body><p>b</p>` lost "b" (BL-37). Those nodes are moved into the body, a
29
- // second <body> or <html> by its children. T110 deleted the two end tags from the source instead,
30
- // which also deleted them inside <title>, <xmp> and <plaintext> text (T110 panel: grok, spark, agy).
31
- const adoptAfterBody = (doc) => {
32
- const body = doc.body;
33
- const root = doc.documentElement;
34
- if (!body || !root || body.parentNode !== root)
35
- return doc;
36
- const strays = [];
37
- for (let n = body.nextSibling; n; n = n.nextSibling)
38
- strays.push(n);
39
- for (let n = root.nextSibling; n; n = n.nextSibling)
40
- strays.push(n);
41
- const adopt = (n) => {
42
- if (n.nodeType === 1 && /^(?:BODY|HTML)$/.test(n.tagName)) {
43
- for (const c of [...n.childNodes])
44
- adopt(c);
45
- n.parentNode?.removeChild(n);
46
- }
47
- else if (n.nodeType === 1 || (n.nodeType === 3 && (n.textContent || '').trim())) {
48
- body.appendChild(n);
49
- }
50
- };
51
- for (const n of strays)
52
- adopt(n);
53
- return doc;
54
- };
55
- // React streaming SSR (Fizz: Next.js App Router, Remix, Mintlify) sends the shell first, with a Suspense fallback
56
- // where late content goes, `<!--$?--><template id="B:0"></template>fallback<!--/$-->`, and the content later, at the
57
- // end of <body>: `<div hidden id="S:0">` and `<script>$RC("B:0","S:0")</script>`, which moves it into the boundary in
58
- // place of the fallback. With no script run, the article stayed hidden beside <main> and the root printed the shell
59
- // (the benchmark's docs.anthropic.com page: 212 characters of "Loading..." for a 12,270-character article, BL-39; 7 of
60
- // 16 fresh Next.js / Mintlify / Remix pages stream, T126). So the calls are replayed in document order, as a browser
61
- // runs them: $RC, and $RR (the same once stylesheets load), fill a boundary; $RS moves a segment onto its
62
- // `<template id="P:1">` placeholder; $RX (client rendering) keeps the fallback. Chromium running only a page's inline
63
- // scripts gave the converter the same page (8 of 8 pages, T126).
64
- //
65
- // A call counts only where a browser would run it (T127, the T126 panel; each case run in Chromium 149 with React's
66
- // own runtime): call text in a `type="application/json"` script, a comment, a string or a <noscript> had moved a hidden
67
- // segment into the root. So: a classic script (no `src`, no `nomodule`, a JavaScript `type` or none, outside <template>
68
- // and <noscript>; React never writes a module script, and one runs after the classic ones), the call at a statement
69
- // start as React writes it (all 23 calls on the 8 streaming pages begin the script or follow `;`), segment and boundary
70
- // parsed before the script, as getElementById sees them when it runs, and only on markup React writes: one of its
71
- // segment wrappers and a pending boundary closed by its own comment. What a script does is not checked: `$RC` defined
72
- // as a no-op, or not at all, still moves segments; no real page streams without React's runtime. The external runtime
73
- // (`<template data-rci>`, `unstable_externalRuntimeSrc`) is not replayed: none of the 16 pages uses it.
74
- // Ids as React prints them: an optional identifierPrefix, then B:, S: or P: and hex.
75
- const FILL_BOUNDARY = /\$R[CR]\("([^"\\]*B:[0-9a-f]+)","([^"\\]*S:[0-9a-f]+)"/;
76
- const FILL_PLACEHOLDER = /\$RS\("([^"\\]*S:[0-9a-f]+)","([^"\\]*P:[0-9a-f]+)"/;
77
- const STREAM_CALL = new RegExp(`(?<![^;])(?:${FILL_BOUNDARY.source}|${FILL_PLACEHOLDER.source})`, 'g');
78
- const BOUNDARY_PENDING = /^\$[?~]$/; // `$~` = queued by React 19.2's batched reveal
79
- const BOUNDARY_START = /^(?:\$[?~!]?|&)$/; // `&` = an <Activity> boundary
80
- const REACT_ID = /[BSP]:[0-9a-f]+$/;
81
- // The HTML standard's JavaScript MIME types ("JavaScript MIME type essence match", no parameters).
82
- const JS_TYPE = /^((application|text)\/(x-)?(ecma|java)script|text\/(javascript1\.[0-5]|jscript|livescript))$/i;
83
- const runsAsClassicScript = (script) => {
84
- if (script.hasAttribute('src') || script.hasAttribute('nomodule'))
85
- return false;
86
- if (script.parentElement?.closest('template, noscript'))
87
- return false;
88
- const type = script.getAttribute('type');
89
- if (type !== null)
90
- return type === '' || JS_TYPE.test(type.trim());
91
- const language = script.getAttribute('language');
92
- return !language || JS_TYPE.test(`text/${language}`);
93
- };
94
- // React's segment wrappers (ReactFizzConfigDOM.js, writeStartSegment): `<div hidden>`, `<svg>` and `<math>` with
95
- // aria-hidden="true" and display:none, `<table hidden>`, or the <tbody>, <tr> or <colgroup> of a `<table hidden>`.
96
- const isSegment = (el) => {
97
- if (!el)
98
- return false;
99
- const tag = el.tagName.toUpperCase();
100
- if (tag === 'DIV' || tag === 'TABLE')
101
- return el.hasAttribute('hidden');
102
- if (tag === 'SVG' || tag === 'MATH') {
103
- return el.getAttribute('aria-hidden') === 'true' && /display:\s*none/.test(el.getAttribute('style') || '');
104
- }
105
- return /^(?:TBODY|TR|COLGROUP)$/.test(tag) && !!el.closest('table')?.hasAttribute('hidden');
106
- };
107
- const scanDocument = (doc) => {
108
- const order = new Map();
109
- const ids = new Map();
110
- for (const el of doc.querySelectorAll('*')) {
111
- order.set(el, order.size);
112
- if (!el.id || !REACT_ID.test(el.id))
113
- continue;
114
- const list = ids.get(el.id);
115
- if (list)
116
- list.push(el);
117
- else
118
- ids.set(el.id, [el]);
119
- }
120
- return { order, ids };
121
- };
122
- // getElementById as the script sees it when it runs: the first element with that id still in the document, if the
123
- // parser had reached it before the script.
124
- const lookup = (scan, id, script) => {
125
- const list = scan.ids.get(id);
126
- while (list?.length && !list[0].isConnected)
127
- list.shift();
128
- const el = list?.[0];
129
- return el && (scan.order.get(el) ?? Infinity) < (scan.order.get(script) ?? -1) ? el : null;
130
- };
131
- const fillBoundary = (scan, script, boundaryId, segmentId) => {
132
- const segment = lookup(scan, segmentId, script);
133
- if (!isSegment(segment))
134
- return;
135
- const marker = lookup(scan, boundaryId, script);
136
- if (!marker) {
137
- segment.remove(); // React's $RC drops a segment whose boundary is gone: the browser never shows it
138
- return;
139
- }
140
- const open = marker.previousSibling;
141
- if (marker.tagName !== 'TEMPLATE' || open?.nodeType !== 8 || !BOUNDARY_PENDING.test(open.data))
142
- return;
143
- const parent = marker.parentNode;
144
- if (!parent || segment.contains(marker))
145
- return;
146
- let close = marker.nextSibling;
147
- for (let depth = 0; close; close = close.nextSibling) {
148
- if (close.nodeType !== 8)
149
- continue;
150
- const data = close.data;
151
- if (data === '/$' || data === '/&') {
152
- if (!depth)
153
- break;
154
- depth--;
155
- }
156
- else if (BOUNDARY_START.test(data))
157
- depth++;
158
- }
159
- if (!close)
160
- return;
161
- segment.remove();
162
- for (let n = marker; n && n !== close;) {
163
- const next = n.nextSibling;
164
- parent.removeChild(n);
165
- n = next;
166
- }
167
- while (segment.firstChild)
168
- parent.insertBefore(segment.firstChild, close);
169
- open.data = '$';
170
- };
171
- const fillPlaceholder = (scan, script, segmentId, placeholderId) => {
172
- const segment = lookup(scan, segmentId, script);
173
- if (!isSegment(segment))
174
- return;
175
- const placeholder = lookup(scan, placeholderId, script);
176
- if (!placeholder) {
177
- segment.remove(); // React's $RS removes the segment, then fails on the missing placeholder
178
- return;
179
- }
180
- const parent = placeholder.parentNode;
181
- if (placeholder.tagName !== 'TEMPLATE' || !parent || segment.contains(placeholder))
182
- return;
183
- segment.remove();
184
- while (segment.firstChild)
185
- parent.insertBefore(segment.firstChild, placeholder);
186
- placeholder.remove();
187
- };
188
- const revealStreamedSegments = (doc) => {
189
- let scan;
190
- for (const script of [...doc.querySelectorAll('script')]) {
191
- const text = script.textContent || '';
192
- if (!text.includes('$R') || !runsAsClassicScript(script))
193
- continue;
194
- for (const m of text.matchAll(STREAM_CALL)) {
195
- scan ??= scanDocument(doc);
196
- if (m[1])
197
- fillBoundary(scan, script, m[1], m[2]);
198
- else
199
- fillPlaceholder(scan, script, m[3], m[4]);
200
- }
201
- }
202
- return doc;
203
- };
204
- const parseDocument = (source) => {
205
- const first = parseHTML(source).document;
206
- if (first.documentElement?.tagName === 'HTML')
207
- return adoptAfterBody(first);
208
- const inner = source.replace(/^/, '').replace(/^\s*<\?xml[^>]*>/i, '').replace(/^\s*<!doctype[^>]*>/i, '');
209
- // Only a fragment that STARTS with <head> or <body> brings its own; one met later is wrapped like any
210
- // other content. Matching `<body` anywhere left `the <body> tag is required` outside the body, losing
211
- // "the " (mimo, T109).
212
- const shell = /^\s*(?:<!--[\s\S]*?-->\s*)*<(?:head|body)[\s>]/i.test(inner)
213
- ? `<!DOCTYPE html><html>${inner}</html>`
214
- : `<!DOCTYPE html><html><head></head><body>${inner}</body></html>`;
215
- return adoptAfterBody(parseHTML(shell).document);
216
- };
217
- export const htmlToMarkdown = (html, opts = {}) => {
218
- const injectedBase = opts.baseUrl ?? 'about:blank';
219
- try {
220
- const document = revealStreamedSegments(parseDocument(html));
221
- const adapter = new LinkedomAdapter(document, injectedBase);
222
- const title = documentTitle(document).trim();
223
- const markdown = renderBasicMarkdown(document, adapter, {
224
- baseUrl: adapter.baseUrl(),
225
- includeFrontmatter: opts.includeFrontmatter ?? true,
226
- includeImages: opts.includeImages ?? true,
227
- includeHidden: opts.includeHidden ?? false,
228
- });
229
- return {
230
- markdown,
231
- title,
232
- baseUrl: adapter.baseUrl(),
233
- charCount: markdown.length,
234
- extractedAt: new Date().toISOString(),
235
- source: 'html-parse',
236
- };
237
- }
238
- catch (e) {
239
- const errMsg = e instanceof Error ? e.message : String(e);
240
- const fallback = [
241
- '---',
242
- `title: "Page2AI extraction failed"`,
243
- `source: "${injectedBase}"`,
244
- `captured_at: "${new Date().toISOString()}"`,
245
- `extractor: "page2ai-core"`,
246
- `extractor_version: "${CORE_VERSION}"`,
247
- `error: "${errMsg.replace(/"/g, '\\"')}"`,
248
- '---',
249
- '',
250
- '# Page2AI extraction failed',
251
- '',
252
- `Could not parse HTML from ${injectedBase}.`,
253
- '',
254
- `Error: ${errMsg}`,
255
- '',
256
- ].join('\n');
257
- return {
258
- markdown: fallback,
259
- title: 'Page2AI extraction failed',
260
- baseUrl: injectedBase,
261
- charCount: fallback.length,
262
- extractedAt: new Date().toISOString(),
263
- source: 'html-parse',
264
- };
265
- }
266
- };
267
13
  // Wrap a raw markdown response in the frontmatter block so downstream consumers
268
14
  // see the same shape as html-to-md output. `extractor_source` says that no
269
15
  // conversion happened, because a caller measuring conversion quality needs to know.
@@ -356,6 +102,8 @@ export const fetchAndConvert = async (url, opts = {}) => {
356
102
  }
357
103
  return htmlToMarkdown(resp.text, { ...opts, baseUrl: opts.baseUrl ?? resp.finalUrl });
358
104
  };
105
+ // htmlToMarkdown and its types come from convert.ts, the network-free module behind '@page2ai/core/html'.
106
+ export { htmlToMarkdown } from './convert.js';
359
107
  export { LinkedomAdapter } from './linkedom-adapter.js';
360
108
  // assertSafeUrlResolved is the exact check fetchAndConvert runs on the URL and on every redirect
361
109
  // hop, so a caller can learn before fetching whether a URL will be refused; findIpBlockReason is
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/node/index.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,EAAE;AACF,cAAc;AACd,sEAAsE;AACtE,gEAAgE;AAChE,EAAE;AACF,0EAA0E;AAC1E,6EAA6E;AAC7E,4EAA4E;AAC5E,yEAAyE;AACzE,oFAAoF;AACpF,EAAE;AACF,sFAAsF;AAEtF,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE5E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AA8BtD,kGAAkG;AAClG,gGAAgG;AAChG,+FAA+F;AAC/F,6FAA6F;AAC7F,8FAA8F;AAC9F,8CAA8C;AAC9C,mGAAmG;AACnG,qGAAqG;AACrG,qGAAqG;AACrG,kGAAkG;AAClG,qGAAqG;AACrG,MAAM,cAAc,GAAG,CAAC,GAAa,EAAY,EAAE;IACjD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,IAAI,GAAG,GAAG,CAAC,eAAe,CAAC;IACjC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IAC3D,MAAM,MAAM,GAAW,EAAE,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW;QAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW;QAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,CAAC,CAAO,EAAQ,EAAE;QAC9B,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAE,CAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YACvE,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC;gBAAE,KAAK,CAAC,CAAC,CAAC,CAAC;YAC5C,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;aAAM,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAClF,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACjC,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,kHAAkH;AAClH,qHAAqH;AACrH,sHAAsH;AACtH,oHAAoH;AACpH,uHAAuH;AACvH,qHAAqH;AACrH,0GAA0G;AAC1G,sHAAsH;AACtH,iEAAiE;AACjE,EAAE;AACF,oHAAoH;AACpH,wHAAwH;AACxH,wHAAwH;AACxH,oHAAoH;AACpH,wHAAwH;AACxH,kHAAkH;AAClH,sHAAsH;AACtH,sHAAsH;AACtH,wGAAwG;AACxG,qFAAqF;AACrF,MAAM,aAAa,GAAG,wDAAwD,CAAC;AAC/E,MAAM,gBAAgB,GAAG,qDAAqD,CAAC;AAC/E,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,eAAe,aAAa,CAAC,MAAM,IAAI,gBAAgB,CAAC,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC;AACvG,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,+CAA+C;AACpF,MAAM,cAAc,GAAG,kBAAkB,CAAC,CAAC,+BAA+B;AAC1E,MAAM,QAAQ,GAAG,kBAAkB,CAAC;AACpC,mGAAmG;AACnG,MAAM,OAAO,GAAG,+FAA+F,CAAC;AAEhH,MAAM,mBAAmB,GAAG,CAAC,MAAe,EAAW,EAAE;IACvD,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAChF,IAAI,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,oBAAoB,CAAC;QAAE,OAAO,KAAK,CAAC;IACtE,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,IAAI,KAAK,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;IACjD,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,QAAQ,EAAE,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,iHAAiH;AACjH,mHAAmH;AACnH,MAAM,SAAS,GAAG,CAAC,EAAkB,EAAiB,EAAE;IACtD,IAAI,CAAC,EAAE;QAAE,OAAO,KAAK,CAAC;IACtB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACrC,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,OAAO;QAAE,OAAO,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvE,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACpC,OAAO,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,MAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7G,CAAC;IACD,OAAO,yBAAyB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;AAC9F,CAAC,CAAC;AAOF,MAAM,YAAY,GAAG,CAAC,GAAa,EAAc,EAAE;IACjD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmB,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAqB,CAAC;IACzC,KAAK,MAAM,EAAE,IAAI,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3C,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAAE,SAAS;QAC9C,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;;YACnB,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;AACxB,CAAC,CAAC;AAEF,kHAAkH;AAClH,2CAA2C;AAC3C,MAAM,MAAM,GAAG,CAAC,IAAgB,EAAE,EAAU,EAAE,MAAe,EAAkB,EAAE;IAC/E,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9B,OAAO,IAAI,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW;QAAE,IAAI,CAAC,KAAK,EAAE,CAAC;IAC1D,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACrB,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7F,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,IAAgB,EAAE,MAAe,EAAE,UAAkB,EAAE,SAAiB,EAAQ,EAAE;IACtG,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAChD,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAChC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAChD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,iFAAiF;QACnG,OAAO;IACT,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,eAAe,CAAC;IACpC,IAAI,MAAM,CAAC,OAAO,KAAK,UAAU,IAAI,IAAI,EAAE,QAAQ,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAE,IAAgB,CAAC,IAAI,CAAC;QAAE,OAAO;IACpH,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC;IACjC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO;IAChD,IAAI,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC;IAC/B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QACrD,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC;YAAE,SAAS;QACnC,MAAM,IAAI,GAAI,KAAiB,CAAC,IAAI,CAAC;QACrC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,KAAK;gBAAE,MAAM;YAClB,KAAK,EAAE,CAAC;QACV,CAAC;aAAM,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,KAAK,EAAE,CAAC;IAChD,CAAC;IACD,IAAI,CAAC,KAAK;QAAE,OAAO;IACnB,OAAO,CAAC,MAAM,EAAE,CAAC;IACjB,KAAK,IAAI,CAAC,GAAqB,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,GAAI,CAAC;QAC1D,MAAM,IAAI,GAAqB,CAAC,CAAC,WAAW,CAAC;QAC7C,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC,GAAG,IAAI,CAAC;IACX,CAAC;IACD,OAAO,OAAO,CAAC,UAAU;QAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACzE,IAAgB,CAAC,IAAI,GAAG,GAAG,CAAC;AAC/B,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,IAAgB,EAAE,MAAe,EAAE,SAAiB,EAAE,aAAqB,EAAQ,EAAE;IAC5G,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAChD,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAChC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;IACxD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,yEAAyE;QAC3F,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC;IACtC,IAAI,WAAW,CAAC,OAAO,KAAK,UAAU,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO;IAC3F,OAAO,CAAC,MAAM,EAAE,CAAC;IACjB,OAAO,OAAO,CAAC,UAAU;QAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAChF,WAAW,CAAC,MAAM,EAAE,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,GAAa,EAAY,EAAE;IACzD,IAAI,IAA4B,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;YAAE,SAAS;QACnE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3C,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAE,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;gBAC5C,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,MAAc,EAAY,EAAE;IACjD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAA+B,CAAC;IAChE,IAAI,KAAK,CAAC,eAAe,EAAE,OAAO,KAAK,MAAM;QAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;IAC5E,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC;IAC5G,sGAAsG;IACtG,sGAAsG;IACtG,uBAAuB;IACvB,MAAM,KAAK,GAAG,iDAAiD,CAAC,IAAI,CAAC,KAAK,CAAC;QACzE,CAAC,CAAC,wBAAwB,KAAK,SAAS;QACxC,CAAC,CAAC,2CAA2C,KAAK,gBAAgB,CAAC;IACrE,OAAO,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,QAA+B,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,IAAY,EACZ,OAA8B,EAAE,EACd,EAAE;IACpB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,IAAI,aAAa,CAAC;IACnD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,sBAAsB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,OAAO,EAAE;YACtD,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;YAC1B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,IAAI,IAAI;YACnD,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;YACzC,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,KAAK;SAC3C,CAAC,CAAC;QACH,OAAO;YACL,QAAQ;YACR,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;YAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;IAAC,OAAO,CAAU,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG;YACf,KAAK;YACL,oCAAoC;YACpC,YAAY,YAAY,GAAG;YAC3B,iBAAiB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG;YAC5C,2BAA2B;YAC3B,uBAAuB,YAAY,GAAG;YACtC,WAAW,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG;YACzC,KAAK;YACL,EAAE;YACF,6BAA6B;YAC7B,EAAE;YACF,6BAA6B,YAAY,GAAG;YAC5C,EAAE;YACF,UAAU,MAAM,EAAE;YAClB,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,2BAA2B;YAClC,OAAO,EAAE,YAAY;YACrB,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAEF,gFAAgF;AAChF,2EAA2E;AAC3E,oFAAoF;AACpF,MAAM,qBAAqB,GAAG,CAAC,GAAW,EAAE,GAAW,EAAE,KAAa,EAAU,EAAE;IAChF,MAAM,EAAE,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC;IAC7E,MAAM,EAAE,GAAG;QACT,KAAK;QACL,UAAU,EAAE,CAAC,KAAK,CAAC,EAAE;QACrB,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE;QACpB,gBAAgB,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,EAAE;QAC9C,cAAc,EAAE,CAAC,cAAc,CAAC,EAAE;QAClC,sBAAsB,EAAE,CAAC,YAAY,CAAC,EAAE;QACxC,qBAAqB,EAAE,CAAC,mBAAmB,CAAC,EAAE;QAC9C,KAAK;QACL,EAAE;KACH,CAAC;IACF,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;AAC7D,CAAC,CAAC;AAEF,4FAA4F;AAC5F,2FAA2F;AAC3F,uFAAuF;AACvF,uBAAuB;AACvB,MAAM,kBAAkB,GAAG,CACzB,GAAW,EACX,GAAW,EACX,KAAa,EACb,IAA4B,EACpB,EAAE,CACV,CAAC,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC;IAC/B,CAAC,CAAC,qBAAqB,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC;IACxC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;AAE1C,iFAAiF;AACjF,gFAAgF;AAChF,iFAAiF;AACjF,4FAA4F;AAC5F,kGAAkG;AAClG,0FAA0F;AAC1F,MAAM,qBAAqB,GAAG,qCAAqC,CAAC;AACpE,MAAM,iBAAiB,GAAG,wCAAwC,CAAC;AAEnE,oGAAoG;AACpG,8FAA8F;AAC9F,kGAAkG;AAClG,kDAAkD;AAClD,MAAM,cAAc,GAAG,uCAAuC,CAAC;AAC/D,MAAM,eAAe,GACnB,mIAAmI,CAAC;AAEtI,MAAM,MAAM,GAAG,CAAC,CAAS,EAAU,EAAE;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,CAAS,EAAW,EAAE,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAClC,GAAW,EACX,OAA+B,EAAE,EACN,EAAE;IAC7B,aAAa,CAAC,GAAG,CAAC,CAAC;IACnB,4FAA4F;IAC5F,+EAA+E;IAC/E,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE;QACrC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,kBAAkB,EAAE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;KACnF,CAAC,CAAC;IAEH,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3E,8FAA8F;QAC9F,wDAAwD;QACxD,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC/C,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,oBAAoB,CAAC,kDAAkD,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACpG,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACrE,OAAO;gBACL,QAAQ;gBACR,KAAK;gBACL,OAAO,EAAE,IAAI,CAAC,QAAQ;gBACtB,SAAS,EAAE,QAAQ,CAAC,MAAM;gBAC1B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,MAAM,EAAE,mBAAmB;aAC5B,CAAC;QACJ,CAAC;QACD,0FAA0F;QAC1F,gGAAgG;QAChG,8FAA8F;QAC9F,mDAAmD;QACnD,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,oBAAoB,CAC5B,sCAAsC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,uBAAuB;gBACjG,gGAAgG;gBAChG,0EAA0E,EAC5E,IAAI,CAAC,QAAQ,CACd,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;AACxF,CAAC,CAAC;AAIF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,iGAAiG;AACjG,iGAAiG;AACjG,wFAAwF;AACxF,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC5G,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,cAAc,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC5H,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/node/index.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,EAAE;AACF,cAAc;AACd,qGAAqG;AACrG,sGAAsG;AACtG,gEAAgE;AAChE,EAAE;AACF,uEAAuE;AAEvE,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE5E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAUtD,gFAAgF;AAChF,2EAA2E;AAC3E,oFAAoF;AACpF,MAAM,qBAAqB,GAAG,CAAC,GAAW,EAAE,GAAW,EAAE,KAAa,EAAU,EAAE;IAChF,MAAM,EAAE,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC;IAC7E,MAAM,EAAE,GAAG;QACT,KAAK;QACL,UAAU,EAAE,CAAC,KAAK,CAAC,EAAE;QACrB,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE;QACpB,gBAAgB,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,EAAE;QAC9C,cAAc,EAAE,CAAC,cAAc,CAAC,EAAE;QAClC,sBAAsB,EAAE,CAAC,YAAY,CAAC,EAAE;QACxC,qBAAqB,EAAE,CAAC,mBAAmB,CAAC,EAAE;QAC9C,KAAK;QACL,EAAE;KACH,CAAC;IACF,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;AAC7D,CAAC,CAAC;AAEF,4FAA4F;AAC5F,2FAA2F;AAC3F,uFAAuF;AACvF,uBAAuB;AACvB,MAAM,kBAAkB,GAAG,CACzB,GAAW,EACX,GAAW,EACX,KAAa,EACb,IAA4B,EACpB,EAAE,CACV,CAAC,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC;IAC/B,CAAC,CAAC,qBAAqB,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC;IACxC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;AAE1C,iFAAiF;AACjF,gFAAgF;AAChF,iFAAiF;AACjF,4FAA4F;AAC5F,kGAAkG;AAClG,0FAA0F;AAC1F,MAAM,qBAAqB,GAAG,qCAAqC,CAAC;AACpE,MAAM,iBAAiB,GAAG,wCAAwC,CAAC;AAEnE,oGAAoG;AACpG,8FAA8F;AAC9F,kGAAkG;AAClG,kDAAkD;AAClD,MAAM,cAAc,GAAG,uCAAuC,CAAC;AAC/D,MAAM,eAAe,GACnB,mIAAmI,CAAC;AAEtI,MAAM,MAAM,GAAG,CAAC,CAAS,EAAU,EAAE;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,CAAS,EAAW,EAAE,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAClC,GAAW,EACX,OAA+B,EAAE,EACN,EAAE;IAC7B,aAAa,CAAC,GAAG,CAAC,CAAC;IACnB,4FAA4F;IAC5F,+EAA+E;IAC/E,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE;QACrC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,kBAAkB,EAAE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;KACnF,CAAC,CAAC;IAEH,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3E,8FAA8F;QAC9F,wDAAwD;QACxD,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC/C,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,oBAAoB,CAAC,kDAAkD,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACpG,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACrE,OAAO;gBACL,QAAQ;gBACR,KAAK;gBACL,OAAO,EAAE,IAAI,CAAC,QAAQ;gBACtB,SAAS,EAAE,QAAQ,CAAC,MAAM;gBAC1B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,MAAM,EAAE,mBAAmB;aAC5B,CAAC;QACJ,CAAC;QACD,0FAA0F;QAC1F,gGAAgG;QAChG,8FAA8F;QAC9F,mDAAmD;QACnD,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,oBAAoB,CAC5B,sCAAsC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,uBAAuB;gBACjG,gGAAgG;gBAChG,0EAA0E,EAC5E,IAAI,CAAC,QAAQ,CACd,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;AACxF,CAAC,CAAC;AAEF,0GAA0G;AAC1G,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAI9C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,iGAAiG;AACjG,iGAAiG;AACjG,wFAAwF;AACxF,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC5G,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,cAAc,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC5H,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC"}
@@ -1,5 +1,5 @@
1
1
  import type { ExtractorConfig } from './types.js';
2
- export declare const CORE_VERSION = "0.4.3";
2
+ export declare const CORE_VERSION = "0.4.5";
3
3
  export declare const INDENT = " ";
4
4
  export declare const DEFAULTS: ExtractorConfig;
5
5
  export declare const BLOCK_TAGS: Set<string>;
@@ -6,7 +6,7 @@
6
6
  // in @page2ai/mcp, caught T51) — never add another literal. This copy itself drifted at 0.1.9
7
7
  // (frontmatter said 0.1.8 in live ChatGPT output); tests/shared/version-sync.test.ts now fails
8
8
  // the suite whenever the two copies disagree.
9
- export const CORE_VERSION = '0.4.3';
9
+ export const CORE_VERSION = '0.4.5';
10
10
  // Indent constant: 3 spaces for CommonMark-compatible nested lists.
11
11
  export const INDENT = ' ';
12
12
  export const DEFAULTS = {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@page2ai/core",
3
- "version": "0.4.3",
3
+ "version": "0.4.5",
4
4
  "description": "Extract clean Markdown from any web page HTML. Works in Node.js (via linkedom) and browsers. Powers the @page2ai/mcp server.",
5
5
  "type": "module",
6
6
  "exports": {
@@ -12,6 +12,10 @@
12
12
  "types": "./dist/node/index.d.ts",
13
13
  "import": "./dist/node/index.js"
14
14
  },
15
+ "./html": {
16
+ "types": "./dist/node/convert.d.ts",
17
+ "import": "./dist/node/convert.js"
18
+ },
15
19
  "./browser": {
16
20
  "types": "./dist/browser/index.d.ts",
17
21
  "import": "./dist/browser/index.js"