dsh-web-fetch-playwright 0.2.18 → 0.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/README.zh-CN.md +17 -1
- package/lib/client.js +18 -18
- package/lib/index.js +1004 -5
- package/lib/types/index.d.ts +2 -0
- package/lib/types/pdf.d.ts +143 -0
- package/lib/types/provider.d.ts +9 -0
- package/lib/types/types.d.ts +5 -0
- package/package.json +3 -2
- package/src/index.ts +2 -0
- package/src/pdf.ts +1114 -0
- package/src/provider.ts +117 -8
- package/src/session-window.ts +8 -0
- package/src/types.ts +5 -0
package/lib/index.js
CHANGED
|
@@ -1067,6 +1067,7 @@ async function deepCaptureWalk(page, budgetMs) {
|
|
|
1067
1067
|
//#region src/session-window.ts
|
|
1068
1068
|
const ownership = /* @__PURE__ */ new WeakMap();
|
|
1069
1069
|
const unavailable = /* @__PURE__ */ new WeakSet();
|
|
1070
|
+
const identities = /* @__PURE__ */ new WeakMap();
|
|
1070
1071
|
const CLEANUP_GRACE_MS = 100;
|
|
1071
1072
|
function claim(browser, context, keys) {
|
|
1072
1073
|
let owners = ownership.get(browser);
|
|
@@ -1163,6 +1164,8 @@ async function createSessionPage(context, browser, timeoutMs) {
|
|
|
1163
1164
|
return session;
|
|
1164
1165
|
};
|
|
1165
1166
|
const identify = async (page) => {
|
|
1167
|
+
const known = identities.get(page);
|
|
1168
|
+
if (known) return known;
|
|
1166
1169
|
const session = await newSession(() => ctx.newCDPSession(page));
|
|
1167
1170
|
let identificationFailed = false;
|
|
1168
1171
|
try {
|
|
@@ -1212,6 +1215,7 @@ async function createSessionPage(context, browser, timeoutMs) {
|
|
|
1212
1215
|
targetId = actual.targetId;
|
|
1213
1216
|
if (actual.browserContextId !== identity.browserContextId || await windowOf(actual.targetId) !== expectedWindow) throw new Error("Chromium did not place the session tab in its dedicated window");
|
|
1214
1217
|
claim(browser, context, [`context:${actual.browserContextId}`, `window:${expectedWindow}`]);
|
|
1218
|
+
identities.set(created, actual);
|
|
1215
1219
|
succeeded = true;
|
|
1216
1220
|
return created;
|
|
1217
1221
|
}
|
|
@@ -1259,6 +1263,10 @@ async function createSessionPage(context, browser, timeoutMs) {
|
|
|
1259
1263
|
const explicitWindow = await windowOf(explicitTargetId);
|
|
1260
1264
|
if (explicitWindow === bootstrapWindow) throw new Error("Chromium ignored the dedicated session window request");
|
|
1261
1265
|
claim(browser, context, [`context:${identity.browserContextId}`, `window:${explicitWindow}`]);
|
|
1266
|
+
identities.set(created, {
|
|
1267
|
+
targetId: explicitTargetId,
|
|
1268
|
+
browserContextId: identity.browserContextId
|
|
1269
|
+
});
|
|
1262
1270
|
await deadline.run(() => bootstrap.close());
|
|
1263
1271
|
bootstrap = void 0;
|
|
1264
1272
|
succeeded = true;
|
|
@@ -2210,6 +2218,902 @@ function htmlToMarkdown(html, url) {
|
|
|
2210
2218
|
};
|
|
2211
2219
|
}
|
|
2212
2220
|
//#endregion
|
|
2221
|
+
//#region src/pdf.ts
|
|
2222
|
+
/**
|
|
2223
|
+
* PDF → markdown. A fetched URL that turns out to be a PDF (served inline as
|
|
2224
|
+
* `application/pdf`, as a `Content-Disposition: attachment` download, or as
|
|
2225
|
+
* an anonymous `application/octet-stream` whose bytes start with `%PDF-`) is
|
|
2226
|
+
* routed here instead of failing with `WEB_UNSUPPORTED_CONTENT_TYPE`.
|
|
2227
|
+
*
|
|
2228
|
+
* Two layers, kept apart so the heuristics are testable without pdf.js:
|
|
2229
|
+
*
|
|
2230
|
+
* 1. {@link readPdfDocument} — the pdf.js boundary (through `unpdf`'s
|
|
2231
|
+
* serverless build, loaded lazily so plugin start never pays for it).
|
|
2232
|
+
* Per page it reads the positioned text runs (`getTextContent`) plus the
|
|
2233
|
+
* real font names (`getOperatorList` + `fontExtraProperties`, the only way
|
|
2234
|
+
* pdf.js exposes "NimbusRomNo9L-Medi" instead of a generic family), so the
|
|
2235
|
+
* layout pass can see bold and monospace. Bounded by the fetch deadline,
|
|
2236
|
+
* the caller's abort signal, a page cap, and a collected-character cap.
|
|
2237
|
+
* 2. {@link layoutPdfMarkdown} — pure heuristics over those runs:
|
|
2238
|
+
* - runs → lines (baseline grouping in content-stream order, x-sorted
|
|
2239
|
+
* inside a line; rotated runs such as arXiv's side stamp are dropped);
|
|
2240
|
+
* - running headers/footers (text repeated in the page margins across
|
|
2241
|
+
* pages, digits normalized) and bare page numbers are removed;
|
|
2242
|
+
* - the dominant (body) font size and line pitch are measured;
|
|
2243
|
+
* - headings come from, strongest first: the PDF outline (bookmarks),
|
|
2244
|
+
* section numbering ("3.1 Pre-training", "第二章", "一、"), larger font
|
|
2245
|
+
* sizes (ranked into levels), and standalone bold lines;
|
|
2246
|
+
* - lines are merged into paragraphs (vertical gap, first-line indent,
|
|
2247
|
+
* short terminal line, font change; continuation across columns and
|
|
2248
|
+
* pages when the sentence is unfinished), with dictionary-checked
|
|
2249
|
+
* de-hyphenation and CJK-aware joining;
|
|
2250
|
+
* - bullets / numbered items become markdown lists, monospace blocks
|
|
2251
|
+
* become fenced code (unless the whole document is monospace), lines
|
|
2252
|
+
* with wide column gaps become GFM tables when they line up;
|
|
2253
|
+
* - `<!-- page N -->` markers keep page provenance for citations.
|
|
2254
|
+
*
|
|
2255
|
+
* Scanned PDFs without a text layer, encrypted PDFs, and broken files fail
|
|
2256
|
+
* with dedicated, provider-specific error codes (OCR is out of scope).
|
|
2257
|
+
*
|
|
2258
|
+
* @module dsh-web-fetch-playwright/pdf
|
|
2259
|
+
*/
|
|
2260
|
+
/** Largest PDF this provider downloads and parses (bytes). */
|
|
2261
|
+
const MAX_PDF_BYTES = 52428800;
|
|
2262
|
+
/** Most pages read from one document; later pages are reported as truncated. */
|
|
2263
|
+
const MAX_PDF_PAGES = 500;
|
|
2264
|
+
/** No text layer (a scanned / image-only PDF); OCR is not supported. */
|
|
2265
|
+
const WEB_PDF_NO_TEXT_CODE = "WEB_PDF_NO_TEXT";
|
|
2266
|
+
/** Password-protected PDF the parser cannot open. */
|
|
2267
|
+
const WEB_PDF_ENCRYPTED_CODE = "WEB_PDF_ENCRYPTED";
|
|
2268
|
+
/** Bytes claimed (or sniffed) to be a PDF that pdf.js cannot parse. */
|
|
2269
|
+
const WEB_PDF_INVALID_CODE = "WEB_PDF_INVALID";
|
|
2270
|
+
/** PDF larger than {@link MAX_PDF_BYTES}. */
|
|
2271
|
+
const WEB_PDF_TOO_LARGE_CODE = "WEB_PDF_TOO_LARGE";
|
|
2272
|
+
const PDF_MIME_TYPES = /* @__PURE__ */ new Set([
|
|
2273
|
+
"application/pdf",
|
|
2274
|
+
"application/x-pdf",
|
|
2275
|
+
"application/acrobat",
|
|
2276
|
+
"applications/vnd.pdf",
|
|
2277
|
+
"text/pdf",
|
|
2278
|
+
"text/x-pdf"
|
|
2279
|
+
]);
|
|
2280
|
+
/** Generic binary types a server may use for a PDF; their bytes get sniffed. */
|
|
2281
|
+
const OPAQUE_BINARY_TYPES = /* @__PURE__ */ new Set([
|
|
2282
|
+
"application/octet-stream",
|
|
2283
|
+
"binary/octet-stream",
|
|
2284
|
+
"application/binary",
|
|
2285
|
+
"application/download",
|
|
2286
|
+
"application/x-download",
|
|
2287
|
+
"application/force-download",
|
|
2288
|
+
"application/unknown"
|
|
2289
|
+
]);
|
|
2290
|
+
/** The bare media type of a `Content-Type` header value. */
|
|
2291
|
+
function mediaType(contentType) {
|
|
2292
|
+
return (contentType ?? "").replace(/;.*$/s, "").trim().toLowerCase();
|
|
2293
|
+
}
|
|
2294
|
+
/** Whether a `Content-Type` names a PDF. */
|
|
2295
|
+
function isPdfContentType(contentType) {
|
|
2296
|
+
return PDF_MIME_TYPES.has(mediaType(contentType));
|
|
2297
|
+
}
|
|
2298
|
+
/**
|
|
2299
|
+
* Whether a response that is not otherwise decodable is worth downloading
|
|
2300
|
+
* to sniff for PDF bytes: a PDF content type, a `.pdf` attachment filename,
|
|
2301
|
+
* a `.pdf` URL path, or an opaque binary type (servers commonly hand PDFs out
|
|
2302
|
+
* as `application/octet-stream`). The magic-byte check then decides.
|
|
2303
|
+
*/
|
|
2304
|
+
function mayBePdf(headers, url) {
|
|
2305
|
+
if (isPdfContentType(headers["content-type"])) return true;
|
|
2306
|
+
const disposition = headers["content-disposition"] ?? "";
|
|
2307
|
+
if (/filename\*?\s*=\s*(?:[\w-]+'[^']*')?"?[^";]*\.pdf\b/i.test(disposition)) return true;
|
|
2308
|
+
try {
|
|
2309
|
+
if (/\.pdf$/i.test(new URL(url).pathname)) return true;
|
|
2310
|
+
} catch {}
|
|
2311
|
+
return OPAQUE_BINARY_TYPES.has(mediaType(headers["content-type"]));
|
|
2312
|
+
}
|
|
2313
|
+
/**
|
|
2314
|
+
* Whether the bytes are a PDF: `%PDF-` within the first 1024 bytes (the
|
|
2315
|
+
* spec-tolerated leading-garbage window every mainstream reader accepts).
|
|
2316
|
+
*/
|
|
2317
|
+
function isPdfBytes(bytes) {
|
|
2318
|
+
const limit = Math.min(bytes.length - 5, 1024);
|
|
2319
|
+
for (let i = 0; i <= limit; i++) if (bytes[i] === 37 && bytes[i + 1] === 80 && bytes[i + 2] === 68 && bytes[i + 3] === 70 && bytes[i + 4] === 45) return true;
|
|
2320
|
+
return false;
|
|
2321
|
+
}
|
|
2322
|
+
/** Bold-looking PostScript font names (subset prefix already stripped). */
|
|
2323
|
+
const BOLD_FONT_RE = /bold|black|heavy|semibold|demibold|demi\b|[-,]?medi(um)?\b|-B$|,B$|-SB$|CMBX|CMB\d|BX\d|\bHei\b|SimHei|Heiti|HTJ-|HT-|黑体/i;
|
|
2324
|
+
/** Monospace-looking font names. */
|
|
2325
|
+
const MONO_FONT_RE = /mono|courier|consol|menlo|inconsolata|NimbusMonL|CMTT|SFTT|LMMono|SourceCodePro|FiraCode|DejaVuSansMono|LucidaConsole|Code/i;
|
|
2326
|
+
/**
|
|
2327
|
+
* Classify a pdf.js font object by its real (PostScript) name. pdf.js only
|
|
2328
|
+
* exposes the name after the page's operator list loaded the font, and only
|
|
2329
|
+
* with `fontExtraProperties`; anything unreadable classifies as regular.
|
|
2330
|
+
*/
|
|
2331
|
+
function classifyFont(page, loadedName, cache) {
|
|
2332
|
+
if (loadedName === void 0) return {
|
|
2333
|
+
bold: false,
|
|
2334
|
+
mono: false
|
|
2335
|
+
};
|
|
2336
|
+
const cached = cache.get(loadedName);
|
|
2337
|
+
if (cached !== void 0) return cached;
|
|
2338
|
+
let name = "";
|
|
2339
|
+
let flagsBold = false;
|
|
2340
|
+
try {
|
|
2341
|
+
const font = page.commonObjs.get(loadedName);
|
|
2342
|
+
name = (font?.name ?? "").replace(/^[A-Z]{6}\+/, "");
|
|
2343
|
+
flagsBold = font?.bold === true || font?.black === true;
|
|
2344
|
+
} catch {}
|
|
2345
|
+
const result = {
|
|
2346
|
+
bold: flagsBold || BOLD_FONT_RE.test(name),
|
|
2347
|
+
mono: MONO_FONT_RE.test(name)
|
|
2348
|
+
};
|
|
2349
|
+
cache.set(loadedName, result);
|
|
2350
|
+
return result;
|
|
2351
|
+
}
|
|
2352
|
+
/** Flatten the outline tree (bounded) into `{ title, depth }` entries. */
|
|
2353
|
+
function flattenOutline(nodes, depth = 0, out = []) {
|
|
2354
|
+
if (nodes === null) return out;
|
|
2355
|
+
for (const node of nodes) {
|
|
2356
|
+
if (out.length >= 2e3) break;
|
|
2357
|
+
const title = (node.title ?? "").replace(/\s+/g, " ").trim();
|
|
2358
|
+
if (title !== "") out.push({
|
|
2359
|
+
title,
|
|
2360
|
+
depth
|
|
2361
|
+
});
|
|
2362
|
+
if (node.items !== void 0 && node.items.length > 0 && depth < 8) flattenOutline(node.items, depth + 1, out);
|
|
2363
|
+
}
|
|
2364
|
+
return out;
|
|
2365
|
+
}
|
|
2366
|
+
/** Map a pdf.js load failure onto the provider's PDF error codes. */
|
|
2367
|
+
function translatePdfError(error) {
|
|
2368
|
+
if (error instanceof WebError) return error;
|
|
2369
|
+
const name = error instanceof Error ? error.name : "";
|
|
2370
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2371
|
+
if (name === "PasswordException" || /password/i.test(message)) return new WebError("the PDF is password-protected and cannot be read", WEB_PDF_ENCRYPTED_CODE, { cause: error });
|
|
2372
|
+
return new WebError(`the PDF could not be parsed: ${message.split("\n")[0].slice(0, 200)}`, WEB_PDF_INVALID_CODE, { cause: error });
|
|
2373
|
+
}
|
|
2374
|
+
/**
|
|
2375
|
+
* Map Kangxi / CJK radical code points to the unified ideographs they look
|
|
2376
|
+
* like. Some producers (Chromium printing Noto CJK among them) write a
|
|
2377
|
+
* ToUnicode CMap that resolves 文 / 用 / 长 glyphs to U+2F42 ⽂ / U+2F64 ⽤
|
|
2378
|
+
* / U+2EA7 ⻓ — visually identical, but they break search and reading.
|
|
2379
|
+
* NFKC maps exactly these; it is applied to nothing else (fullwidth forms
|
|
2380
|
+
* stay as the document wrote them).
|
|
2381
|
+
*/
|
|
2382
|
+
function normalizeRadicals(text) {
|
|
2383
|
+
return text.replace(/[\u2E80-\u2FDF]/g, (ch) => SIMPLIFIED_RADICALS[ch] ?? ch.normalize("NFKC"));
|
|
2384
|
+
}
|
|
2385
|
+
/**
|
|
2386
|
+
* The C-simplified radicals (CJK Radicals Supplement) have no NFKC mapping;
|
|
2387
|
+
* these are the ones that double as everyday simplified characters.
|
|
2388
|
+
*/
|
|
2389
|
+
const SIMPLIFIED_RADICALS = {
|
|
2390
|
+
"⻅": "见",
|
|
2391
|
+
"⻆": "角",
|
|
2392
|
+
"⻈": "讠",
|
|
2393
|
+
"⻉": "贝",
|
|
2394
|
+
"⻋": "车",
|
|
2395
|
+
"⻐": "钅",
|
|
2396
|
+
"⻓": "长",
|
|
2397
|
+
"⻔": "门",
|
|
2398
|
+
"⻙": "韦",
|
|
2399
|
+
"⻚": "页",
|
|
2400
|
+
"⻛": "风",
|
|
2401
|
+
"⻜": "飞",
|
|
2402
|
+
"⻠": "饣",
|
|
2403
|
+
"⻢": "马",
|
|
2404
|
+
"⻥": "鱼",
|
|
2405
|
+
"⻦": "鸟",
|
|
2406
|
+
"⻧": "卤",
|
|
2407
|
+
"⻨": "麦",
|
|
2408
|
+
"⻩": "黄",
|
|
2409
|
+
"⻪": "黾",
|
|
2410
|
+
"⻬": "齐",
|
|
2411
|
+
"⻮": "齿",
|
|
2412
|
+
"⻰": "龙"
|
|
2413
|
+
};
|
|
2414
|
+
/** A run is horizontal when the text matrix has (almost) no rotation/skew. */
|
|
2415
|
+
function isHorizontal(t) {
|
|
2416
|
+
const [a = 0, b = 0, c = 0, d = 0] = t;
|
|
2417
|
+
return Math.abs(b) <= Math.abs(a) * .1 + 1e-6 && Math.abs(c) <= Math.abs(d) * .1 + 1e-6 && a > 0 && d > 0;
|
|
2418
|
+
}
|
|
2419
|
+
/**
|
|
2420
|
+
* Read a PDF's positioned text with pdf.js, page by page, inside the given
|
|
2421
|
+
* limits. Text items become {@link PdfTextRun}s (horizontal only — rotated
|
|
2422
|
+
* stamps and vertical margin text are dropped as layout noise).
|
|
2423
|
+
*
|
|
2424
|
+
* @returns the text model, plus whether a limit stopped reading early.
|
|
2425
|
+
*/
|
|
2426
|
+
async function readPdfDocument(bytes, options = {}) {
|
|
2427
|
+
const startedAt = Date.now();
|
|
2428
|
+
const budget = options.timeBudgetMs ?? Number.POSITIVE_INFINITY;
|
|
2429
|
+
const maxChars = options.maxChars ?? Number.POSITIVE_INFINITY;
|
|
2430
|
+
const maxPages = options.maxPages ?? 500;
|
|
2431
|
+
const { getDocumentProxy } = await import("unpdf");
|
|
2432
|
+
const data = new Uint8Array(bytes.byteLength);
|
|
2433
|
+
data.set(bytes);
|
|
2434
|
+
let pdf;
|
|
2435
|
+
try {
|
|
2436
|
+
pdf = await getDocumentProxy(data, {
|
|
2437
|
+
fontExtraProperties: true,
|
|
2438
|
+
verbosity: 0,
|
|
2439
|
+
stopAtErrors: false
|
|
2440
|
+
});
|
|
2441
|
+
} catch (error) {
|
|
2442
|
+
throw translatePdfError(error);
|
|
2443
|
+
}
|
|
2444
|
+
try {
|
|
2445
|
+
let title;
|
|
2446
|
+
try {
|
|
2447
|
+
const meta = await pdf.getMetadata();
|
|
2448
|
+
const xmpTitle = meta.metadata?.get("dc:title");
|
|
2449
|
+
const infoTitle = meta.info?.Title;
|
|
2450
|
+
const candidate = typeof xmpTitle === "string" && xmpTitle.trim() !== "" ? xmpTitle : typeof infoTitle === "string" ? infoTitle : "";
|
|
2451
|
+
if (candidate.trim() !== "") title = candidate.replace(/\s+/g, " ").trim();
|
|
2452
|
+
} catch {}
|
|
2453
|
+
let outline = [];
|
|
2454
|
+
try {
|
|
2455
|
+
outline = flattenOutline(await pdf.getOutline());
|
|
2456
|
+
} catch {}
|
|
2457
|
+
const pages = [];
|
|
2458
|
+
const fontCache = /* @__PURE__ */ new Map();
|
|
2459
|
+
let chars = 0;
|
|
2460
|
+
let truncated = false;
|
|
2461
|
+
const pageCount = pdf.numPages;
|
|
2462
|
+
for (let n = 1; n <= pageCount; n++) {
|
|
2463
|
+
if (options.signal?.aborted) throw options.signal.reason instanceof Error ? options.signal.reason : /* @__PURE__ */ new Error("pdf extraction aborted");
|
|
2464
|
+
if (n > maxPages || chars >= maxChars || Date.now() - startedAt >= budget) {
|
|
2465
|
+
truncated = true;
|
|
2466
|
+
break;
|
|
2467
|
+
}
|
|
2468
|
+
const page = await pdf.getPage(n);
|
|
2469
|
+
const content = await page.getTextContent();
|
|
2470
|
+
if (Date.now() - startedAt < budget * .8) try {
|
|
2471
|
+
await page.getOperatorList();
|
|
2472
|
+
} catch {}
|
|
2473
|
+
const [, viewBottom = 0, , viewTop = 0] = page.view;
|
|
2474
|
+
const runs = [];
|
|
2475
|
+
for (const item of content.items) {
|
|
2476
|
+
if (typeof item.str !== "string" || item.str === "" || item.transform === void 0) continue;
|
|
2477
|
+
if (!isHorizontal(item.transform)) continue;
|
|
2478
|
+
const t = item.transform;
|
|
2479
|
+
const size = Math.hypot(t[2] ?? 0, t[3] ?? 0) || item.height || 0;
|
|
2480
|
+
if (!(size > 0)) continue;
|
|
2481
|
+
const font = classifyFont(page, item.fontName, fontCache);
|
|
2482
|
+
runs.push({
|
|
2483
|
+
text: normalizeRadicals(item.str),
|
|
2484
|
+
x: t[4] ?? 0,
|
|
2485
|
+
y: t[5] ?? 0,
|
|
2486
|
+
width: item.width ?? 0,
|
|
2487
|
+
size,
|
|
2488
|
+
bold: font.bold,
|
|
2489
|
+
mono: font.mono
|
|
2490
|
+
});
|
|
2491
|
+
chars += item.str.length;
|
|
2492
|
+
}
|
|
2493
|
+
pages.push({
|
|
2494
|
+
number: n,
|
|
2495
|
+
bottom: Math.min(viewBottom, viewTop),
|
|
2496
|
+
height: Math.abs(viewTop - viewBottom) || 792,
|
|
2497
|
+
runs
|
|
2498
|
+
});
|
|
2499
|
+
page.cleanup();
|
|
2500
|
+
}
|
|
2501
|
+
return {
|
|
2502
|
+
doc: {
|
|
2503
|
+
...title !== void 0 ? { title } : {},
|
|
2504
|
+
outline,
|
|
2505
|
+
pages,
|
|
2506
|
+
totalPages: pageCount
|
|
2507
|
+
},
|
|
2508
|
+
truncated
|
|
2509
|
+
};
|
|
2510
|
+
} catch (error) {
|
|
2511
|
+
if (options.signal?.aborted) throw error;
|
|
2512
|
+
throw translatePdfError(error);
|
|
2513
|
+
} finally {
|
|
2514
|
+
pdf.loadingTask?.destroy().catch(() => {});
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
2517
|
+
/**
|
|
2518
|
+
* Extract a PDF as markdown: read it with pdf.js inside the limits, lay it
|
|
2519
|
+
* out, and fail with {@link WEB_PDF_NO_TEXT_CODE} when there is no text layer
|
|
2520
|
+
* to speak of (a scan). Sizes beyond {@link MAX_PDF_BYTES} are refused.
|
|
2521
|
+
*/
|
|
2522
|
+
async function extractPdfMarkdown(bytes, options = {}) {
|
|
2523
|
+
if (bytes.length > 52428800) throw new WebError(`the PDF is ${String(Math.round(bytes.length / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`, WEB_PDF_TOO_LARGE_CODE);
|
|
2524
|
+
const { doc, truncated } = await readPdfDocument(bytes, options);
|
|
2525
|
+
const textChars = doc.pages.reduce((sum, page) => sum + page.runs.reduce((s, run) => s + run.text.replace(/\s+/g, "").length, 0), 0);
|
|
2526
|
+
if (doc.pages.length > 0 && textChars < Math.max(1, doc.pages.length * 5)) throw new WebError(`the PDF has no extractable text layer (${String(doc.totalPages)} page(s), likely scanned images); OCR is not supported`, WEB_PDF_NO_TEXT_CODE);
|
|
2527
|
+
let markdown = layoutPdfMarkdown(doc);
|
|
2528
|
+
if (truncated) markdown += `\n\n> [PDF extraction stopped after page ${String(doc.pages.length)} of ${String(doc.totalPages)} (size/time limit).]\n`;
|
|
2529
|
+
return {
|
|
2530
|
+
markdown,
|
|
2531
|
+
totalPages: doc.totalPages,
|
|
2532
|
+
pagesRead: doc.pages.length,
|
|
2533
|
+
truncated
|
|
2534
|
+
};
|
|
2535
|
+
}
|
|
2536
|
+
/** CJK ideographs, kana, hangul, and fullwidth/CJK punctuation. */
|
|
2537
|
+
const CJK_RE = /[⺀- -〿-ヿ-ㇿ㐀-䶿一-鿿가-豈--]/;
|
|
2538
|
+
function isCjk(ch) {
|
|
2539
|
+
return ch !== void 0 && CJK_RE.test(ch);
|
|
2540
|
+
}
|
|
2541
|
+
/** Letter-spaced CJK ("预 防 和 治 理") reads as one word. */
|
|
2542
|
+
function collapseCjkSpacing(text) {
|
|
2543
|
+
return text.replace(new RegExp(`(?<=${CJK_RE.source}) +(?=${CJK_RE.source})`, "g"), "");
|
|
2544
|
+
}
|
|
2545
|
+
/** Group a page's runs into lines (content-stream order, baseline match). */
|
|
2546
|
+
function buildLines(page) {
|
|
2547
|
+
const groups = [];
|
|
2548
|
+
let current = [];
|
|
2549
|
+
let baseY = 0;
|
|
2550
|
+
let baseSize = 0;
|
|
2551
|
+
for (const run of page.runs) {
|
|
2552
|
+
if (current.length > 0) {
|
|
2553
|
+
const tolerance = Math.max(.6 * Math.max(run.size, baseSize), 1);
|
|
2554
|
+
if (Math.abs(run.y - baseY) <= tolerance) {
|
|
2555
|
+
current.push(run);
|
|
2556
|
+
if (run.text.trim() !== "" && run.size > baseSize) {
|
|
2557
|
+
baseSize = run.size;
|
|
2558
|
+
baseY = run.y;
|
|
2559
|
+
}
|
|
2560
|
+
continue;
|
|
2561
|
+
}
|
|
2562
|
+
groups.push(current);
|
|
2563
|
+
}
|
|
2564
|
+
current = [run];
|
|
2565
|
+
baseY = run.y;
|
|
2566
|
+
baseSize = run.size;
|
|
2567
|
+
}
|
|
2568
|
+
if (current.length > 0) groups.push(current);
|
|
2569
|
+
const lines = [];
|
|
2570
|
+
for (const group of groups) {
|
|
2571
|
+
const line = composeLine(group, page);
|
|
2572
|
+
if (line !== void 0) lines.push(line);
|
|
2573
|
+
}
|
|
2574
|
+
return lines;
|
|
2575
|
+
}
|
|
2576
|
+
/** Join one baseline group's runs (x-sorted) into a {@link Line}. */
|
|
2577
|
+
function composeLine(group, page) {
|
|
2578
|
+
const runs = [...group].sort((a, b) => a.x - b.x);
|
|
2579
|
+
const visible = runs.filter((run) => run.text.trim() !== "");
|
|
2580
|
+
if (visible.length === 0) return void 0;
|
|
2581
|
+
const weight = /* @__PURE__ */ new Map();
|
|
2582
|
+
for (const run of visible) {
|
|
2583
|
+
const key = Math.round(run.size * 10) / 10;
|
|
2584
|
+
weight.set(key, (weight.get(key) ?? 0) + run.text.trim().length);
|
|
2585
|
+
}
|
|
2586
|
+
let size = visible[0].size;
|
|
2587
|
+
let best = -1;
|
|
2588
|
+
for (const [key, w] of weight) if (w > best) {
|
|
2589
|
+
best = w;
|
|
2590
|
+
size = key;
|
|
2591
|
+
}
|
|
2592
|
+
const baseline = visible.find((run) => Math.abs(run.size - size) < .11)?.y ?? visible[0].y;
|
|
2593
|
+
const segments = [];
|
|
2594
|
+
const cells = [""];
|
|
2595
|
+
const cellX = [];
|
|
2596
|
+
let text = "";
|
|
2597
|
+
let prevEnd;
|
|
2598
|
+
let boldChars = 0;
|
|
2599
|
+
let monoChars = 0;
|
|
2600
|
+
let chars = 0;
|
|
2601
|
+
for (const run of runs) {
|
|
2602
|
+
const content = run.text;
|
|
2603
|
+
const trimmed = content.trim();
|
|
2604
|
+
if (trimmed === "") continue;
|
|
2605
|
+
let sep = "";
|
|
2606
|
+
if (prevEnd !== void 0 && text !== "") {
|
|
2607
|
+
const gap = run.x - prevEnd;
|
|
2608
|
+
const last = text[text.length - 1];
|
|
2609
|
+
const first = trimmed[0];
|
|
2610
|
+
if (gap > run.size * 1.5 && !(isCjk(last) && isCjk(first) && gap < run.size * 3)) {
|
|
2611
|
+
cells.push("");
|
|
2612
|
+
cellX.push(run.x);
|
|
2613
|
+
sep = " ";
|
|
2614
|
+
} else if (gap > run.size * .15 || /\s$/.test(content.slice(0, content.length - content.trimStart().length)) || /^\s/.test(content)) sep = isCjk(last) && isCjk(first) ? "" : " ";
|
|
2615
|
+
if (/\s$/.test(text)) sep = "";
|
|
2616
|
+
}
|
|
2617
|
+
const piece = sep + trimmed;
|
|
2618
|
+
if (cellX.length === 0) cellX.push(run.x);
|
|
2619
|
+
text += piece;
|
|
2620
|
+
cells[cells.length - 1] += cells[cells.length - 1] === "" ? trimmed : piece;
|
|
2621
|
+
const prev = segments[segments.length - 1];
|
|
2622
|
+
if (prev !== void 0 && prev.bold === run.bold) prev.text += piece;
|
|
2623
|
+
else segments.push({
|
|
2624
|
+
text: piece,
|
|
2625
|
+
bold: run.bold
|
|
2626
|
+
});
|
|
2627
|
+
prevEnd = Math.max(prevEnd ?? -Infinity, run.x + run.width);
|
|
2628
|
+
const n = trimmed.replace(/\s+/g, "").length;
|
|
2629
|
+
chars += n;
|
|
2630
|
+
if (run.bold) boldChars += n;
|
|
2631
|
+
if (run.mono) monoChars += n;
|
|
2632
|
+
}
|
|
2633
|
+
text = collapseCjkSpacing(text.replace(/\s+/g, " ").trim());
|
|
2634
|
+
if (text === "") return void 0;
|
|
2635
|
+
for (const segment of segments) segment.text = collapseCjkSpacing(segment.text);
|
|
2636
|
+
const x0 = visible.reduce((m, run) => Math.min(m, run.x), Infinity);
|
|
2637
|
+
const x1 = visible.reduce((m, run) => Math.max(m, run.x + run.width), -Infinity);
|
|
2638
|
+
return {
|
|
2639
|
+
page: page.number,
|
|
2640
|
+
text,
|
|
2641
|
+
segments,
|
|
2642
|
+
x0,
|
|
2643
|
+
x1,
|
|
2644
|
+
y: baseline,
|
|
2645
|
+
size,
|
|
2646
|
+
bold: chars > 0 && boldChars / chars >= .8,
|
|
2647
|
+
mono: chars > 0 && monoChars / chars >= .8,
|
|
2648
|
+
cells: cells.map((cell) => collapseCjkSpacing(cell.replace(/\s+/g, " ").trim())),
|
|
2649
|
+
cellX,
|
|
2650
|
+
chars,
|
|
2651
|
+
relY: page.height > 0 ? (baseline - page.bottom) / page.height : .5
|
|
2652
|
+
};
|
|
2653
|
+
}
|
|
2654
|
+
/** Bare page-number shapes found in page margins. */
|
|
2655
|
+
const PAGE_NUMBER_RES = [
|
|
2656
|
+
/^[-–—\s]*(?:page|p\.|pág\.|seite)?\s*\d{1,4}(?:\s*(?:\/|of|von|de)\s*\d{1,4})?[-–—\s]*$/i,
|
|
2657
|
+
/^[-–—\s]*[ivxlcdm]{1,7}[-–—\s]*$/i,
|
|
2658
|
+
/^第\s*\d{1,4}\s*页(?:\s*[,,//]?\s*共\s*\d{1,4}\s*页)?$/,
|
|
2659
|
+
/^[-–—一-―·•]{1,2}\s*\d{1,4}\s*[-–—一-―·•]{1,2}$/
|
|
2660
|
+
];
|
|
2661
|
+
const isMarginLine = (line) => line.relY > .9 || line.relY < .12;
|
|
2662
|
+
/** Drop running headers/footers and page numbers. */
|
|
2663
|
+
function removeMarginNoise(pages) {
|
|
2664
|
+
const keyOf = (line) => line.text.toLowerCase().replace(/\d+/g, "#").replace(/\s+/g, " ").trim();
|
|
2665
|
+
const pagesByKey = /* @__PURE__ */ new Map();
|
|
2666
|
+
for (const lines of pages) for (const line of lines) {
|
|
2667
|
+
if (!isMarginLine(line)) continue;
|
|
2668
|
+
const key = keyOf(line);
|
|
2669
|
+
let set = pagesByKey.get(key);
|
|
2670
|
+
if (set === void 0) {
|
|
2671
|
+
set = /* @__PURE__ */ new Set();
|
|
2672
|
+
pagesByKey.set(key, set);
|
|
2673
|
+
}
|
|
2674
|
+
set.add(line.page);
|
|
2675
|
+
}
|
|
2676
|
+
const repeatThreshold = Math.max(3, Math.ceil(pages.length * .3));
|
|
2677
|
+
return pages.map((lines) => lines.filter((line) => {
|
|
2678
|
+
if (!isMarginLine(line)) return true;
|
|
2679
|
+
const plain = line.text.normalize("NFKC");
|
|
2680
|
+
if (PAGE_NUMBER_RES.some((re) => re.test(plain))) return false;
|
|
2681
|
+
if (pages.length >= 3 && (pagesByKey.get(keyOf(line))?.size ?? 0) >= repeatThreshold) return false;
|
|
2682
|
+
return true;
|
|
2683
|
+
}));
|
|
2684
|
+
}
|
|
2685
|
+
/** Character-weighted mode of the line font sizes (0.5pt buckets). */
|
|
2686
|
+
function bodySizeOf(lines) {
|
|
2687
|
+
const weight = /* @__PURE__ */ new Map();
|
|
2688
|
+
for (const line of lines) {
|
|
2689
|
+
const key = Math.round(line.size * 2) / 2;
|
|
2690
|
+
weight.set(key, (weight.get(key) ?? 0) + line.chars);
|
|
2691
|
+
}
|
|
2692
|
+
let body = 10;
|
|
2693
|
+
let best = -1;
|
|
2694
|
+
for (const [key, w] of weight) if (w > best) {
|
|
2695
|
+
best = w;
|
|
2696
|
+
body = key;
|
|
2697
|
+
}
|
|
2698
|
+
return body;
|
|
2699
|
+
}
|
|
2700
|
+
/** Median baseline-to-baseline distance between consecutive body lines. */
|
|
2701
|
+
function lineSpacingOf(pages, body) {
|
|
2702
|
+
const deltas = [];
|
|
2703
|
+
for (const lines of pages) for (let i = 1; i < lines.length; i++) {
|
|
2704
|
+
const a = lines[i - 1];
|
|
2705
|
+
const b = lines[i];
|
|
2706
|
+
if (Math.abs(a.size - body) > body * .1 || Math.abs(b.size - body) > body * .1) continue;
|
|
2707
|
+
const dy = a.y - b.y;
|
|
2708
|
+
if (dy > body * .6 && dy < body * 3) deltas.push(dy);
|
|
2709
|
+
}
|
|
2710
|
+
if (deltas.length === 0) return body * 1.2;
|
|
2711
|
+
deltas.sort((p, q) => p - q);
|
|
2712
|
+
return deltas[Math.floor(deltas.length / 2)];
|
|
2713
|
+
}
|
|
2714
|
+
/** Normalization for comparing headings to outline titles. */
|
|
2715
|
+
function headingKey(text) {
|
|
2716
|
+
return text.normalize("NFKC").toLowerCase().replace(/[\s\p{P}\p{S}]/gu, "");
|
|
2717
|
+
}
|
|
2718
|
+
/** Leading section number ("3", "3.1", "A.2") stripped for outline matching. */
|
|
2719
|
+
function stripNumbering(text) {
|
|
2720
|
+
return text.replace(/^(?:\d+(?:\.\d+)*|[A-Z](?:\.\d+)+)\.?\s+/, "").replace(/^(?:第[一二三四五六七八九十百千零〇两\d]+[章节篇部]|[一二三四五六七八九十]+[、..]|[((][一二三四五六七八九十]+[))])\s*/, "");
|
|
2721
|
+
}
|
|
2722
|
+
const NUMBERED_HEADING_RE = /^(\d{1,2}(?:\.\d{1,2}){0,4}|[A-Z](?:\.\d{1,2}){1,4})\.?\s+(?=\S)/;
|
|
2723
|
+
const CN_CHAPTER_RE = /^第[一二三四五六七八九十百千零〇两\d]+[章篇部]/;
|
|
2724
|
+
const CN_SECTION_RE = /^第[一二三四五六七八九十百千零〇两\d]+节/;
|
|
2725
|
+
const CN_ENUM_RE = /^[一二三四五六七八九十]{1,3}[、.]/;
|
|
2726
|
+
const CN_PAREN_ENUM_RE = /^[((][一二三四五六七八九十]{1,3}[))]/;
|
|
2727
|
+
/** A table-of-contents entry: leader dots (or a wide gap) then a page reference. */
|
|
2728
|
+
const TOC_ENTRY_RE = /(?:[.…·.]\s*){4,}[^.…·.]{0,16}?[((]?\s*[\d0-9ivxlc]+\s*[))]?$/i;
|
|
2729
|
+
const EXPLICIT_HEADING_RES = [
|
|
2730
|
+
NUMBERED_HEADING_RE,
|
|
2731
|
+
CN_CHAPTER_RE,
|
|
2732
|
+
CN_SECTION_RE,
|
|
2733
|
+
CN_ENUM_RE,
|
|
2734
|
+
CN_PAREN_ENUM_RE
|
|
2735
|
+
];
|
|
2736
|
+
const TERMINAL_RE = /[.!?。!?…::;;"”’))\]】]$/;
|
|
2737
|
+
const SENTENCE_END_RE = /[.!?。!?…"”’]$/;
|
|
2738
|
+
const BULLET_RE = /^(?:[•◦▪▫●○■□►▸‣⁃∙·*❖➢✓✔]|[])\s*/;
|
|
2739
|
+
const DASH_BULLET_RE = /^[-–—]\s+(?=\S)/;
|
|
2740
|
+
const ORDERED_RE = /^(\d{1,3})[.)]\s+(?=\S)/;
|
|
2741
|
+
const PAREN_ITEM_RE = /^(?:[((](?:\d{1,3}|[a-zA-Z]|[ivx]{1,4})[))]|[a-z][.)]|[①-⑳])\s*(?=\S)/;
|
|
2742
|
+
/** Heading level placeholder: one below the most recent heading. */
|
|
2743
|
+
const RELATIVE_HEADING = -1;
|
|
2744
|
+
/** Decide a line's heading level, if it is one. */
|
|
2745
|
+
function headingLevel(line, groupLines, stats, standalone, styleRules) {
|
|
2746
|
+
const text = groupLines.map((l) => l.text).join(" ");
|
|
2747
|
+
const chars = groupLines.reduce((s, l) => s + l.chars, 0);
|
|
2748
|
+
if (chars === 0 || chars > 200 || !/\p{L}/u.test(text)) return void 0;
|
|
2749
|
+
const words = text.split(/\s+/).length;
|
|
2750
|
+
const sentenceLike = /[,,;;]$/.test(text) || /[.。]$/.test(text) && !/^\d+(\.\d+)*\.?$/.test(text.split(" ")[0] ?? "");
|
|
2751
|
+
if (stats.titleSize !== void 0 && Math.abs(line.size - stats.titleSize) < .3 && groupLines.length <= 3 && styleRules) return 1;
|
|
2752
|
+
const outlineDepth = stats.outlineKeys.get(headingKey(text)) ?? stats.outlineKeys.get(headingKey(stripNumbering(text)));
|
|
2753
|
+
if (outlineDepth !== void 0 && chars <= 160) return Math.min(6, Math.max(2, outlineDepth + 2 - stats.outlineOffset));
|
|
2754
|
+
const bigger = line.size >= stats.body * 1.12;
|
|
2755
|
+
const emphasized = bigger || line.bold;
|
|
2756
|
+
const styleOk = styleRules && groupLines.every((l) => l.cells.length < 2);
|
|
2757
|
+
const numbered = NUMBERED_HEADING_RE.exec(text);
|
|
2758
|
+
if (numbered !== null && emphasized && !sentenceLike && words <= 16 && chars <= 120) {
|
|
2759
|
+
const depth = numbered[1].split(".").length;
|
|
2760
|
+
return Math.min(6, depth + 1);
|
|
2761
|
+
}
|
|
2762
|
+
if (CN_CHAPTER_RE.test(text) && chars <= 60 && !sentenceLike) return 2;
|
|
2763
|
+
if (CN_SECTION_RE.test(text) && chars <= 60 && !sentenceLike) return 3;
|
|
2764
|
+
if (CN_ENUM_RE.test(text) && chars <= 40 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 2;
|
|
2765
|
+
if (CN_PAREN_ENUM_RE.test(text) && chars <= 30 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 3;
|
|
2766
|
+
if (!styleOk) return void 0;
|
|
2767
|
+
if (bigger && groupLines.length <= 3 && !sentenceLike) {
|
|
2768
|
+
if (stats.titleSize !== void 0 && Math.abs(line.size - stats.titleSize) < .3) return 1;
|
|
2769
|
+
const rank = stats.headingSizes.findIndex((size) => Math.abs(size - Math.round(line.size * 2) / 2) < .3);
|
|
2770
|
+
return Math.min(4, 2 + Math.max(0, rank));
|
|
2771
|
+
}
|
|
2772
|
+
const lastSegment = groupLines[groupLines.length - 1].segments.at(-1);
|
|
2773
|
+
if (line.bold && lastSegment?.bold === true && standalone && groupLines.length <= 2 && chars <= 100 && words <= 14 && !sentenceLike && !TERMINAL_RE.test(text.replace(/[))]$/, ""))) return RELATIVE_HEADING;
|
|
2774
|
+
}
|
|
2775
|
+
/** Group lines into markdown blocks. */
|
|
2776
|
+
function buildBlocks(pages, stats) {
|
|
2777
|
+
const all = pages.flat();
|
|
2778
|
+
const blocks = [];
|
|
2779
|
+
const rightEdge = /* @__PURE__ */ new Map();
|
|
2780
|
+
for (const lines of pages) for (const line of lines) {
|
|
2781
|
+
let right = line.x1;
|
|
2782
|
+
for (const other of lines) if (Math.abs(other.x0 - line.x0) < line.size * 3 && Math.abs(other.size - line.size) < line.size * .15) right = Math.max(right, other.x1);
|
|
2783
|
+
rightEdge.set(line, right);
|
|
2784
|
+
}
|
|
2785
|
+
const pitch = (line) => stats.spacing * (line.size / stats.body);
|
|
2786
|
+
const gapBefore = (i) => {
|
|
2787
|
+
const prev = all[i - 1];
|
|
2788
|
+
const line = all[i];
|
|
2789
|
+
if (prev === void 0 || prev.page !== line.page) return Infinity;
|
|
2790
|
+
return prev.y - line.y;
|
|
2791
|
+
};
|
|
2792
|
+
const isStandalone = (i, span) => {
|
|
2793
|
+
const before = gapBefore(i);
|
|
2794
|
+
const after = i + span < all.length ? gapBefore(i + span) : Infinity;
|
|
2795
|
+
const first = all[i];
|
|
2796
|
+
const sepBefore = before === Infinity || before < 0 || before > pitch(first) * 1.3 || all[i - 1].size !== first.size || all[i - 1].bold !== first.bold;
|
|
2797
|
+
const next = all[i + span];
|
|
2798
|
+
const sepAfter = after === Infinity || after < 0 || after > pitch(first) * 1.3 || next === void 0 || Math.abs(next.size - first.size) > .3;
|
|
2799
|
+
const short = all[i + span - 1].x1 < (rightEdge.get(all[i + span - 1]) ?? Infinity) - first.size * 2;
|
|
2800
|
+
return sepBefore && (sepAfter || short);
|
|
2801
|
+
};
|
|
2802
|
+
const isInlineMono = (index) => {
|
|
2803
|
+
const line = all[index];
|
|
2804
|
+
const prev = all[index - 1];
|
|
2805
|
+
const next = all[index + 1];
|
|
2806
|
+
if (prev === void 0 || prev.mono || next?.mono === true && next.page === line.page && gapBefore(index + 1) > 0 && gapBefore(index + 1) < pitch(line) * 2.2) return false;
|
|
2807
|
+
const dy = gapBefore(index);
|
|
2808
|
+
return dy > 0 && dy < pitch(line) * 1.45 && !TERMINAL_RE.test(prev.text) && line.chars <= 80;
|
|
2809
|
+
};
|
|
2810
|
+
const tableEnd = /* @__PURE__ */ new Map();
|
|
2811
|
+
const inTable = /* @__PURE__ */ new Set();
|
|
2812
|
+
for (let start = 0; start < all.length;) {
|
|
2813
|
+
if (all[start].cells.length < 2 || all[start].mono) {
|
|
2814
|
+
start++;
|
|
2815
|
+
continue;
|
|
2816
|
+
}
|
|
2817
|
+
let end = start + 1;
|
|
2818
|
+
while (end < all.length && all[end].cells.length >= 2 && !all[end].mono && all[end].page === all[start].page && gapBefore(end) > 0 && gapBefore(end) < pitch(all[end]) * 2.5) end++;
|
|
2819
|
+
let first = start;
|
|
2820
|
+
let last = end;
|
|
2821
|
+
while (last - first >= 2 && !rowAligned(all[last - 1], all.slice(first, last - 1))) last--;
|
|
2822
|
+
while (last - first >= 2 && !rowAligned(all[first], all.slice(first + 1, last))) first++;
|
|
2823
|
+
if (last - first >= 2 && columnsAlign(all.slice(first, last))) {
|
|
2824
|
+
tableEnd.set(first, last);
|
|
2825
|
+
for (let k = first; k < last; k++) inTable.add(k);
|
|
2826
|
+
}
|
|
2827
|
+
start = end;
|
|
2828
|
+
}
|
|
2829
|
+
let lastLevel = 1;
|
|
2830
|
+
let i = 0;
|
|
2831
|
+
while (i < all.length) {
|
|
2832
|
+
const line = all[i];
|
|
2833
|
+
let span = 1;
|
|
2834
|
+
while (span < 4 && i + span < all.length && all[i + span].page === line.page && Math.abs(all[i + span].size - line.size) < .3 && all[i + span].bold === line.bold && gapBefore(i + span) > 0 && gapBefore(i + span) < pitch(line) * 1.6 && !EXPLICIT_HEADING_RES.some((re) => re.test(all[i + span].text))) span++;
|
|
2835
|
+
let level;
|
|
2836
|
+
let used = 0;
|
|
2837
|
+
if (span <= 3) {
|
|
2838
|
+
level = headingLevel(line, all.slice(i, i + span), stats, isStandalone(i, span), true);
|
|
2839
|
+
if (level !== void 0) used = span;
|
|
2840
|
+
}
|
|
2841
|
+
if (level === void 0 && span > 1) {
|
|
2842
|
+
level = headingLevel(line, [line], stats, isStandalone(i, 1), false);
|
|
2843
|
+
if (level !== void 0) used = 1;
|
|
2844
|
+
}
|
|
2845
|
+
if (level !== void 0 && line.mono && !stats.monoIsBody) level = void 0;
|
|
2846
|
+
if (level === RELATIVE_HEADING) level = Math.min(6, Math.max(2, lastLevel + 1));
|
|
2847
|
+
else if (level !== void 0) lastLevel = level;
|
|
2848
|
+
if (level !== void 0) {
|
|
2849
|
+
blocks.push({
|
|
2850
|
+
kind: "heading",
|
|
2851
|
+
page: line.page,
|
|
2852
|
+
level,
|
|
2853
|
+
lines: all.slice(i, i + used)
|
|
2854
|
+
});
|
|
2855
|
+
i += used;
|
|
2856
|
+
continue;
|
|
2857
|
+
}
|
|
2858
|
+
const start = i;
|
|
2859
|
+
const kind = tableEnd.has(i) ? "table" : lineKind(line, stats);
|
|
2860
|
+
const block = {
|
|
2861
|
+
kind: kind === "code" && isInlineMono(i) ? "text" : kind,
|
|
2862
|
+
page: line.page,
|
|
2863
|
+
lines: [line]
|
|
2864
|
+
};
|
|
2865
|
+
i++;
|
|
2866
|
+
if (kind === "table") {
|
|
2867
|
+
const end = tableEnd.get(start) ?? start + 1;
|
|
2868
|
+
while (i < end) {
|
|
2869
|
+
block.lines.push(all[i]);
|
|
2870
|
+
i++;
|
|
2871
|
+
}
|
|
2872
|
+
blocks.push(block);
|
|
2873
|
+
continue;
|
|
2874
|
+
}
|
|
2875
|
+
if (kind === "code") {
|
|
2876
|
+
while (i < all.length && all[i].mono && gapBefore(i) > 0 && gapBefore(i) < pitch(all[i]) * 2.2) {
|
|
2877
|
+
block.lines.push(all[i]);
|
|
2878
|
+
i++;
|
|
2879
|
+
}
|
|
2880
|
+
blocks.push(block);
|
|
2881
|
+
continue;
|
|
2882
|
+
}
|
|
2883
|
+
let blockRight = rightEdge.get(line) ?? line.x1;
|
|
2884
|
+
while (i < all.length) {
|
|
2885
|
+
const next = all[i];
|
|
2886
|
+
const prev = block.lines[block.lines.length - 1];
|
|
2887
|
+
if (inTable.has(i)) break;
|
|
2888
|
+
const nextKind = lineKind(next, stats);
|
|
2889
|
+
if (nextKind !== "text" && !(nextKind === "code" && isInlineMono(i))) break;
|
|
2890
|
+
if (Math.abs(next.size - prev.size) > Math.max(next.size, prev.size) * .12) break;
|
|
2891
|
+
const dy = next.page === prev.page ? prev.y - next.y : -Infinity;
|
|
2892
|
+
if (!(dy > 0 && dy < pitch(next) * 1.45)) {
|
|
2893
|
+
if (dy > 0) break;
|
|
2894
|
+
if (TERMINAL_RE.test(prev.text)) break;
|
|
2895
|
+
if (headingStartLike(next)) break;
|
|
2896
|
+
if (next.page !== prev.page && block.kind !== "text") break;
|
|
2897
|
+
} else {
|
|
2898
|
+
if (TOC_ENTRY_RE.test(prev.text)) break;
|
|
2899
|
+
if (next.x0 - prev.x0 > next.size * .8 && block.kind === "text" && next.x1 >= blockRight - next.size * 2 && (TERMINAL_RE.test(prev.text) || prev.x1 < blockRight - prev.size * 2)) break;
|
|
2900
|
+
if (block.kind !== "text" && next.x0 < block.lines[0].x0 + next.size * .3 && TERMINAL_RE.test(prev.text)) break;
|
|
2901
|
+
if (prev.x1 < blockRight - prev.size * 2.5 && SENTENCE_END_RE.test(prev.text)) break;
|
|
2902
|
+
if (next.bold !== prev.bold && next.bold && next.chars < 100 && SENTENCE_END_RE.test(prev.text)) break;
|
|
2903
|
+
}
|
|
2904
|
+
block.lines.push(next);
|
|
2905
|
+
blockRight = Math.max(blockRight, rightEdge.get(next) ?? next.x1);
|
|
2906
|
+
i++;
|
|
2907
|
+
}
|
|
2908
|
+
blocks.push(block);
|
|
2909
|
+
}
|
|
2910
|
+
return blocks;
|
|
2911
|
+
}
|
|
2912
|
+
/**
|
|
2913
|
+
* Whether table-candidate rows share column starts: at least 60% of the rows
|
|
2914
|
+
* have two or more cell left edges within one em of another row's.
|
|
2915
|
+
*/
|
|
2916
|
+
function columnsAlign(rows) {
|
|
2917
|
+
const firstCells = rows.map((row) => row.cells.find((cell) => cell !== "") ?? "").map((cell) => cell.length).sort((a, b) => a - b);
|
|
2918
|
+
if (Math.max(...rows.map((row) => row.cells.filter((cell) => cell !== "").length)) <= 2 && (firstCells[Math.floor(firstCells.length / 2)] ?? 0) > 50) return false;
|
|
2919
|
+
return rows.filter((row) => rowAligned(row, rows.filter((other) => other !== row))).length / rows.length >= .6;
|
|
2920
|
+
}
|
|
2921
|
+
/** Whether a row shares at least two column starts with some other row. */
|
|
2922
|
+
function rowAligned(row, others) {
|
|
2923
|
+
return others.some((other) => row.cellX.filter((x) => other.cellX.some((o) => Math.abs(o - x) < row.size)).length >= 2);
|
|
2924
|
+
}
|
|
2925
|
+
/** A line that looks like the start of a new structural element. */
|
|
2926
|
+
function headingStartLike(line) {
|
|
2927
|
+
return BULLET_RE.test(line.text) || ORDERED_RE.test(line.text) || CN_CHAPTER_RE.test(line.text);
|
|
2928
|
+
}
|
|
2929
|
+
/** Classify a non-heading line. */
|
|
2930
|
+
function lineKind(line, stats) {
|
|
2931
|
+
if (line.mono && !stats.monoIsBody) return "code";
|
|
2932
|
+
if (BULLET_RE.test(line.text) && line.text.replace(BULLET_RE, "").trim() !== "") return "bullet";
|
|
2933
|
+
if (DASH_BULLET_RE.test(line.text)) return "bullet";
|
|
2934
|
+
if (ORDERED_RE.test(line.text)) return "ordered";
|
|
2935
|
+
if (PAREN_ITEM_RE.test(line.text)) return "item";
|
|
2936
|
+
return "text";
|
|
2937
|
+
}
|
|
2938
|
+
/** Words seen in the document (lowercased), for de-hyphenation decisions. */
|
|
2939
|
+
function vocabularyOf(lines) {
|
|
2940
|
+
const words = /* @__PURE__ */ new Set();
|
|
2941
|
+
for (const line of lines) for (const match of line.text.matchAll(/[A-Za-z]+(?:-[A-Za-z]+)*/g)) {
|
|
2942
|
+
const word = match[0].toLowerCase();
|
|
2943
|
+
words.add(word);
|
|
2944
|
+
if (word.includes("-")) words.add(word.replace(/-/g, "\0"));
|
|
2945
|
+
}
|
|
2946
|
+
return words;
|
|
2947
|
+
}
|
|
2948
|
+
/** Join two consecutive lines of one paragraph. */
|
|
2949
|
+
function joinText(acc, next, vocabulary) {
|
|
2950
|
+
if (acc === "") return next;
|
|
2951
|
+
const hyphen = /([A-Za-z]+)[-\u2010\u00AD]$/.exec(acc);
|
|
2952
|
+
const lead = /^([a-z]+)/.exec(next);
|
|
2953
|
+
if (hyphen !== null && lead !== null && !/\s[-\u2010\u00AD]$/.test(acc)) {
|
|
2954
|
+
const head = hyphen[1].toLowerCase();
|
|
2955
|
+
const tail = lead[1];
|
|
2956
|
+
if (vocabulary.has(head + tail)) return acc.slice(0, -1) + next;
|
|
2957
|
+
if (vocabulary.has(`${head}\u0000${tail}`)) return `${acc.slice(0, -1)}-${next}`;
|
|
2958
|
+
return acc.slice(0, -1) + next;
|
|
2959
|
+
}
|
|
2960
|
+
const last = acc[acc.length - 1];
|
|
2961
|
+
const first = next[0];
|
|
2962
|
+
if (isCjk(last) || isCjk(first)) return acc + next;
|
|
2963
|
+
if (/[-/]$/.test(acc)) return acc + next;
|
|
2964
|
+
return `${acc} ${next}`;
|
|
2965
|
+
}
|
|
2966
|
+
/** Escape what would otherwise read as markdown structure at a block start. */
|
|
2967
|
+
function escapeLeading(text) {
|
|
2968
|
+
return text.replace(/^(#{1,6}\s|>|[-+*]\s|\d{1,9}[.)]\s)/, "\\$1");
|
|
2969
|
+
}
|
|
2970
|
+
/** Render a line with inline bold (when only part of it is bold). */
|
|
2971
|
+
function renderLine(line, inlineBold) {
|
|
2972
|
+
if (!inlineBold || !line.segments.some((s) => s.bold) || line.segments.every((s) => s.bold)) return line.text;
|
|
2973
|
+
let out = "";
|
|
2974
|
+
for (const segment of line.segments) {
|
|
2975
|
+
const text = segment.text;
|
|
2976
|
+
const core = text.trim();
|
|
2977
|
+
if (segment.bold && core.replace(/[^\p{L}\p{N}]/gu, "").length >= 2 && !/^\*|\*$/.test(core)) {
|
|
2978
|
+
const lead = text.slice(0, text.length - text.trimStart().length);
|
|
2979
|
+
const trail = text.slice(text.trimEnd().length);
|
|
2980
|
+
out += `${lead}**${core}**${trail}`;
|
|
2981
|
+
} else out += text;
|
|
2982
|
+
}
|
|
2983
|
+
return out.replace(/\s+/g, " ").trim();
|
|
2984
|
+
}
|
|
2985
|
+
/**
|
|
2986
|
+
* Merge bold spans split only by whitespace or punctuation set in the
|
|
2987
|
+
* regular face ("**一是…会议**,**推动会议** **精神**" → one span): the
|
|
2988
|
+
* separator moves inside, and a line-break space between CJK spans goes.
|
|
2989
|
+
*/
|
|
2990
|
+
function mergeBoldRuns(text) {
|
|
2991
|
+
return text.replace(/\*\*(\s*)([,、,;;::]?)(\s*)\*\*/g, (match, a, punct, b, offset, whole) => {
|
|
2992
|
+
const before = whole[offset - 1];
|
|
2993
|
+
const after = whole[offset + match.length];
|
|
2994
|
+
const space = (a !== "" || b !== "") && !(isCjk(before) || isCjk(after) || isCjk(punct)) ? " " : "";
|
|
2995
|
+
return punct === "" ? space : `${punct}${b !== "" && !isCjk(punct) ? " " : ""}`;
|
|
2996
|
+
});
|
|
2997
|
+
}
|
|
2998
|
+
/** Render a table block as GFM when its rows line up, else as plain rows. */
|
|
2999
|
+
function renderTable(block) {
|
|
3000
|
+
const rows = block.lines.map((line) => line.cells.filter((cell) => cell !== "").map((cell) => cell.replace(/\|/g, "\\|")));
|
|
3001
|
+
const width = Math.max(...rows.map((row) => row.length));
|
|
3002
|
+
const aligned = rows.filter((row) => row.length === width).length;
|
|
3003
|
+
if (rows.length >= 2 && width >= 2 && aligned / rows.length >= .6) {
|
|
3004
|
+
const pad = (row) => `| ${[...row, ...Array(width - row.length).fill("")].slice(0, width).join(" | ")} |`;
|
|
3005
|
+
return [
|
|
3006
|
+
pad(rows[0]),
|
|
3007
|
+
`| ${Array(width).fill("---").join(" | ")} |`,
|
|
3008
|
+
...rows.slice(1).map(pad)
|
|
3009
|
+
].join("\n");
|
|
3010
|
+
}
|
|
3011
|
+
return block.lines.map((line) => escapeLeading(line.cells.join(" "))).join(" \n");
|
|
3012
|
+
}
|
|
3013
|
+
/** Whether a metadata title is worth printing as the document heading. */
|
|
3014
|
+
function plausibleTitle(title) {
|
|
3015
|
+
if (title === void 0) return false;
|
|
3016
|
+
const t = title.trim();
|
|
3017
|
+
if (t.length < 3 || t.length > 250) return false;
|
|
3018
|
+
if (/^(?:microsoft\s+(?:word|powerpoint|excel)\b|untitled|slide\s*\d|document\d*$|title$|no title|无标题)/i.test(t)) return false;
|
|
3019
|
+
if (/\.(?:docx?|pdf|pptx?|xlsx?|tex|dvi|indd|qxd|rtf|odt)$/i.test(t)) return false;
|
|
3020
|
+
if (/^[\w-]+$/.test(t) && !/\s/.test(t) && /\d/.test(t)) return false;
|
|
3021
|
+
return true;
|
|
3022
|
+
}
|
|
3023
|
+
/**
|
|
3024
|
+
* Lay a PDF's positioned text out as markdown (see the module docs for the
|
|
3025
|
+
* heuristics). Pure — no pdf.js — so the rules are unit-testable with
|
|
3026
|
+
* synthetic runs.
|
|
3027
|
+
*/
|
|
3028
|
+
function layoutPdfMarkdown(doc) {
|
|
3029
|
+
const pages = removeMarginNoise(doc.pages.map(buildLines));
|
|
3030
|
+
const all = pages.flat();
|
|
3031
|
+
if (all.length === 0) return plausibleTitle(doc.title) ? `# ${doc.title}\n` : "";
|
|
3032
|
+
const body = bodySizeOf(all);
|
|
3033
|
+
const totalChars = all.reduce((s, l) => s + l.chars, 0);
|
|
3034
|
+
const monoIsBody = all.reduce((s, l) => s + (l.mono ? l.chars : 0), 0) > totalChars * .5;
|
|
3035
|
+
const sizeChars = /* @__PURE__ */ new Map();
|
|
3036
|
+
for (const line of all) {
|
|
3037
|
+
if (line.size < body * 1.12) continue;
|
|
3038
|
+
const key = Math.round(line.size * 2) / 2;
|
|
3039
|
+
sizeChars.set(key, (sizeChars.get(key) ?? 0) + line.chars);
|
|
3040
|
+
}
|
|
3041
|
+
let headingSizes = [...sizeChars.keys()].filter((size) => (sizeChars.get(size) ?? 0) < Math.max(totalChars * .25, 400)).sort((a, b) => b - a);
|
|
3042
|
+
let titleSize;
|
|
3043
|
+
const firstPage = doc.pages[0]?.number ?? 1;
|
|
3044
|
+
const firstPageLines = all.filter((line) => line.page === firstPage && /\p{L}{2}/u.test(line.text));
|
|
3045
|
+
const firstLargest = firstPageLines.reduce((m, line) => Math.max(m, Math.round(line.size * 2) / 2), 0);
|
|
3046
|
+
if (firstLargest >= body * 1.3) {
|
|
3047
|
+
if (firstPageLines.filter((line) => Math.abs(Math.round(line.size * 2) / 2 - firstLargest) < .3).length <= 3) {
|
|
3048
|
+
titleSize = firstLargest;
|
|
3049
|
+
headingSizes = headingSizes.filter((size) => size < firstLargest - .25);
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
3052
|
+
headingSizes = headingSizes.slice(0, 3);
|
|
3053
|
+
const outlineKeys = /* @__PURE__ */ new Map();
|
|
3054
|
+
for (const entry of doc.outline) for (const key of [headingKey(entry.title), headingKey(stripNumbering(entry.title))]) if (key.length >= 2 && !outlineKeys.has(key)) outlineKeys.set(key, entry.depth);
|
|
3055
|
+
const roots = doc.outline.filter((entry) => entry.depth === 0);
|
|
3056
|
+
const titleKeys = new Set([doc.title ?? "", ...all.filter((line) => titleSize !== void 0 && Math.abs(line.size - titleSize) < .3).map((line) => line.text)].map(headingKey));
|
|
3057
|
+
const outlineOffset = roots.length === 1 && titleKeys.has(headingKey(roots[0].title)) ? 1 : 0;
|
|
3058
|
+
const blocks = buildBlocks(pages, {
|
|
3059
|
+
body,
|
|
3060
|
+
spacing: lineSpacingOf(pages, body),
|
|
3061
|
+
monoIsBody,
|
|
3062
|
+
headingSizes,
|
|
3063
|
+
...titleSize !== void 0 ? { titleSize } : {},
|
|
3064
|
+
outlineKeys,
|
|
3065
|
+
outlineOffset
|
|
3066
|
+
});
|
|
3067
|
+
const vocabulary = vocabularyOf(all);
|
|
3068
|
+
const out = [];
|
|
3069
|
+
const metaKey = headingKey(doc.title ?? "");
|
|
3070
|
+
if (!blocks.slice(0, 12).some((block) => block.kind === "heading" && (block.level === 1 || headingKey(block.lines.map((l) => l.text).join("")) === metaKey)) && plausibleTitle(doc.title)) out.push(`# ${doc.title.replace(/\s+/g, " ").trim()}`);
|
|
3071
|
+
let lastPage = 0;
|
|
3072
|
+
for (const block of blocks) {
|
|
3073
|
+
if (block.page !== lastPage) {
|
|
3074
|
+
out.push(`<!-- page ${String(block.page)} -->`);
|
|
3075
|
+
lastPage = block.page;
|
|
3076
|
+
}
|
|
3077
|
+
switch (block.kind) {
|
|
3078
|
+
case "heading": {
|
|
3079
|
+
const text = block.lines.map((l) => l.text).reduce((acc, t) => joinText(acc, t, vocabulary), "");
|
|
3080
|
+
out.push(`${"#".repeat(block.level ?? 2)} ${text}`);
|
|
3081
|
+
break;
|
|
3082
|
+
}
|
|
3083
|
+
case "code": {
|
|
3084
|
+
const left = Math.min(...block.lines.map((l) => l.x0));
|
|
3085
|
+
const advances = block.lines.filter((l) => l.text.length >= 4).map((l) => (l.x1 - l.x0) / l.text.length).sort((p, q) => p - q);
|
|
3086
|
+
const advance = advances[Math.floor(advances.length / 2)] ?? block.lines[0].size * .6;
|
|
3087
|
+
const body = block.lines.map((l) => {
|
|
3088
|
+
const indent = Math.max(0, Math.round((l.x0 - left) / advance));
|
|
3089
|
+
return " ".repeat(Math.min(indent, 40)) + l.text;
|
|
3090
|
+
}).join("\n");
|
|
3091
|
+
out.push(`\`\`\`\n${body.replace(/```/g, "```")}\n\`\`\``);
|
|
3092
|
+
break;
|
|
3093
|
+
}
|
|
3094
|
+
case "table":
|
|
3095
|
+
out.push(renderTable(block));
|
|
3096
|
+
break;
|
|
3097
|
+
default: {
|
|
3098
|
+
let text = "";
|
|
3099
|
+
for (const line of block.lines) text = joinText(text, renderLine(line, true), vocabulary);
|
|
3100
|
+
text = mergeBoldRuns(text);
|
|
3101
|
+
if (block.kind === "bullet") {
|
|
3102
|
+
text = text.replace(BULLET_RE, "").replace(DASH_BULLET_RE, "");
|
|
3103
|
+
const prev = out[out.length - 1];
|
|
3104
|
+
if (prev?.startsWith("- ") === true) out[out.length - 1] = `${prev}\n- ${text}`;
|
|
3105
|
+
else out.push(`- ${text}`);
|
|
3106
|
+
} else if (block.kind === "ordered") {
|
|
3107
|
+
const prev = out[out.length - 1];
|
|
3108
|
+
if (prev !== void 0 && /^\d{1,3}\.\s/.test(prev)) out[out.length - 1] = `${prev}\n${text.replace(/^(\d{1,3})\)/, "$1.")}`;
|
|
3109
|
+
else out.push(text.replace(/^(\d{1,3})\)/, "$1."));
|
|
3110
|
+
} else out.push(escapeLeading(text));
|
|
3111
|
+
}
|
|
3112
|
+
}
|
|
3113
|
+
}
|
|
3114
|
+
return `${out.join("\n\n")}\n`;
|
|
3115
|
+
}
|
|
3116
|
+
//#endregion
|
|
2213
3117
|
//#region src/playwright-resolve.ts
|
|
2214
3118
|
/**
|
|
2215
3119
|
* Locates a usable Playwright for the local backend. Resolution order:
|
|
@@ -2422,6 +3326,15 @@ function existsAsFile(path) {
|
|
|
2422
3326
|
* {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
|
|
2423
3327
|
* a queue and fail fast (rather than hang until abort) when no slot frees.
|
|
2424
3328
|
*
|
|
3329
|
+
* PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
|
|
3330
|
+
* handed to the download manager as an attachment, or served as an opaque
|
|
3331
|
+
* `application/octet-stream` whose bytes start with `%PDF-` — is read as
|
|
3332
|
+
* bytes (the navigation body, else a refetch through the page's API request
|
|
3333
|
+
* context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
|
|
3334
|
+
* plus layout heuristics: headings, paragraphs, lists, tables, page
|
|
3335
|
+
* markers). Scans without a text layer, encrypted, oversized, and broken
|
|
3336
|
+
* PDFs fail with the `WEB_PDF_*` codes.
|
|
3337
|
+
*
|
|
2425
3338
|
* Private-network and SSRF protection is not implemented (same stance as the
|
|
2426
3339
|
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
2427
3340
|
* browser can reach. Profile mode additionally acts WITH the remote
|
|
@@ -2620,6 +3533,7 @@ function validateFetchUrl(input) {
|
|
|
2620
3533
|
/** Classify a `Content-Type` into a decodable kind; undefined = unsupported. */
|
|
2621
3534
|
function classifyContentType(contentType) {
|
|
2622
3535
|
const mime = (contentType ?? "").replace(/;.*$/s, "").trim().toLowerCase();
|
|
3536
|
+
if (isPdfContentType(mime)) return "pdf";
|
|
2623
3537
|
if (mime === "" || mime === "text/html" || mime === "application/xhtml+xml") return "html";
|
|
2624
3538
|
if (mime.startsWith("text/")) return "text";
|
|
2625
3539
|
if (mime === "application/json" || mime === "application/xml" || mime.endsWith("+json") || mime.endsWith("+xml")) return "text";
|
|
@@ -2813,9 +3727,14 @@ var PlaywrightFetchProvider = class {
|
|
|
2813
3727
|
}
|
|
2814
3728
|
const finalResponse = (challengeWaitMs > 0 ? tracker.last() : void 0) ?? response;
|
|
2815
3729
|
const kind = classifyContentType(finalResponse?.headers()["content-type"]);
|
|
2816
|
-
if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
2817
3730
|
const finalUrl = page.url();
|
|
2818
3731
|
const statusCode = finalResponse !== null && !(challengeEntryResponse !== null && finalResponse === challengeEntryResponse) ? finalResponse.status() : 200;
|
|
3732
|
+
if (kind === "pdf" || kind === void 0 && finalResponse !== null && mayBePdf(finalResponse.headers(), finalUrl)) {
|
|
3733
|
+
const download = await downloadPdfBytes(page, finalResponse, finalResponse?.url?.() ?? finalUrl, deadline);
|
|
3734
|
+
if (download.pdf) return await pdfResult(download.bytes, download.url, statusCode, deadline);
|
|
3735
|
+
if (kind === "pdf") throw new WebError("the response claims to be a PDF but its bytes are not one", WEB_PDF_INVALID_CODE);
|
|
3736
|
+
}
|
|
3737
|
+
if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
2819
3738
|
if (kind === "text") return capResult(finalUrl, statusCode, {
|
|
2820
3739
|
kind: "text",
|
|
2821
3740
|
content: finalResponse !== null ? await finalResponse.text() : await page.content()
|
|
@@ -3024,7 +3943,7 @@ async function recoverUnrenderedStatus(error, tracker, url, denoise) {
|
|
|
3024
3943
|
} catch {
|
|
3025
3944
|
content = "";
|
|
3026
3945
|
}
|
|
3027
|
-
if (kind
|
|
3946
|
+
if (kind !== "html" || content === "") return capResult(finalUrl, status, {
|
|
3028
3947
|
kind: "text",
|
|
3029
3948
|
content
|
|
3030
3949
|
});
|
|
@@ -3073,7 +3992,9 @@ function isDownloadFailure(error, response) {
|
|
|
3073
3992
|
* download decision — which works identically on the local and CDP
|
|
3074
3993
|
* backends because it never touches the remote artifact.
|
|
3075
3994
|
*
|
|
3076
|
-
*
|
|
3995
|
+
* A PDF download (declared `application/pdf`, or an opaque binary whose
|
|
3996
|
+
* bytes sniff as `%PDF-`) is extracted to markdown (`src/pdf.ts`);
|
|
3997
|
+
* undecodable downloads (a ZIP) still fail with
|
|
3077
3998
|
* `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
|
|
3078
3999
|
* actually served; a playwright build without a request context surfaces a
|
|
3079
4000
|
* clear "the browser will not render this" error instead of the raw
|
|
@@ -3092,7 +4013,13 @@ async function recoverDownloadResponse(error, page, tracker, url, deadline, deno
|
|
|
3092
4013
|
const tracked = tracker.last();
|
|
3093
4014
|
if (!isDownloadFailure(error, tracked)) return void 0;
|
|
3094
4015
|
if (tracked !== null) {
|
|
3095
|
-
|
|
4016
|
+
const trackedKind = classifyContentType(tracked.headers()["content-type"]);
|
|
4017
|
+
if (trackedKind === "pdf" || trackedKind === void 0 && mayBePdf(tracked.headers(), tracked.url?.() ?? url.toString())) {
|
|
4018
|
+
const download = await downloadPdfBytes(page, null, tracked.url?.() ?? url.toString(), deadline, tracked.headers());
|
|
4019
|
+
if (download.pdf) return await pdfResult(download.bytes, download.url, download.status, deadline);
|
|
4020
|
+
throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download; not a PDF)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
4021
|
+
}
|
|
4022
|
+
if (trackedKind === void 0) throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
3096
4023
|
}
|
|
3097
4024
|
if (page.request === void 0) throw new WebError("the response is a file download (Content-Disposition: attachment) the browser will not render, and this playwright build exposes no request context to refetch it", "WEB_PROVIDER_ERROR", { cause: error });
|
|
3098
4025
|
const fetchUrl = tracked?.url?.() ?? url.toString();
|
|
@@ -3103,6 +4030,12 @@ async function recoverDownloadResponse(error, page, tracker, url, deadline, deno
|
|
|
3103
4030
|
const headers = refetched.headers();
|
|
3104
4031
|
const servedType = (headers["content-type"] ?? "").trim() !== "" ? headers["content-type"] : tracked?.headers()["content-type"];
|
|
3105
4032
|
const kind = classifyContentType(servedType);
|
|
4033
|
+
if (kind === "pdf") {
|
|
4034
|
+
assertPdfSize(headers);
|
|
4035
|
+
const bytes = refetched.body !== void 0 ? await refetched.body() : /* @__PURE__ */ new Uint8Array();
|
|
4036
|
+
if (!isPdfBytes(bytes)) throw new WebError("the response claims to be a PDF but its bytes are not one", WEB_PDF_INVALID_CODE);
|
|
4037
|
+
return await pdfResult(bytes, refetched.url?.() ?? fetchUrl, refetched.status(), deadline);
|
|
4038
|
+
}
|
|
3106
4039
|
if (kind === void 0) throw new WebError(`unsupported content type "${servedType ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
3107
4040
|
const finalUrl = refetched.url?.() ?? fetchUrl;
|
|
3108
4041
|
const statusCode = refetched.status();
|
|
@@ -3121,6 +4054,72 @@ async function recoverDownloadResponse(error, page, tracker, url, deadline, deno
|
|
|
3121
4054
|
content: markdown
|
|
3122
4055
|
});
|
|
3123
4056
|
}
|
|
4057
|
+
/** Time kept back from the PDF page walk for layout and result assembly. */
|
|
4058
|
+
const PDF_FINISH_RESERVE_MS = 1500;
|
|
4059
|
+
/** Refuse a PDF whose declared length is over the limit — before downloading it. */
|
|
4060
|
+
function assertPdfSize(headers) {
|
|
4061
|
+
const declared = Number.parseInt(headers?.["content-length"] ?? "", 10);
|
|
4062
|
+
if (Number.isFinite(declared) && declared > 52428800) throw new WebError(`the PDF is ${String(Math.round(declared / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`, WEB_PDF_TOO_LARGE_CODE);
|
|
4063
|
+
}
|
|
4064
|
+
/**
|
|
4065
|
+
* Get a (possibly) PDF resource's bytes. The navigation response's own body
|
|
4066
|
+
* is tried first — a PDF Chrome committed into its viewer still has it —
|
|
4067
|
+
* then the page's API request context refetches the URL (same cookie jar,
|
|
4068
|
+
* plain HTTP, no render-or-download decision; the only way to reach a body
|
|
4069
|
+
* Chromium streamed to its download manager). The `%PDF-` magic decides.
|
|
4070
|
+
*
|
|
4071
|
+
* @param page - the fetch's tab (for its request context).
|
|
4072
|
+
* @param response - the navigation response, when the browser committed one.
|
|
4073
|
+
* @param url - the URL to refetch.
|
|
4074
|
+
* @param deadline - the fetch budget.
|
|
4075
|
+
* @param servedHeaders - the navigation's headers (size pre-check) when no
|
|
4076
|
+
* response object is passed.
|
|
4077
|
+
* @returns the bytes, where they came from, and whether they are a PDF.
|
|
4078
|
+
*/
|
|
4079
|
+
async function downloadPdfBytes(page, response, url, deadline, servedHeaders) {
|
|
4080
|
+
assertPdfSize(response?.headers() ?? servedHeaders);
|
|
4081
|
+
if (response?.body !== void 0) try {
|
|
4082
|
+
const bytes = await response.body();
|
|
4083
|
+
if (isPdfBytes(bytes)) return {
|
|
4084
|
+
pdf: true,
|
|
4085
|
+
bytes,
|
|
4086
|
+
url: response.url?.() ?? url,
|
|
4087
|
+
status: response.status()
|
|
4088
|
+
};
|
|
4089
|
+
} catch {}
|
|
4090
|
+
if (page.request === void 0) throw new WebError("the response is a PDF the browser will not hand over, and this playwright build exposes no request context to refetch it", "WEB_PROVIDER_ERROR");
|
|
4091
|
+
const refetched = await page.request.get(url, {
|
|
4092
|
+
timeout: deadline.remainingMs(),
|
|
4093
|
+
signal: deadline.signal
|
|
4094
|
+
});
|
|
4095
|
+
assertPdfSize(refetched.headers());
|
|
4096
|
+
const bytes = refetched.body !== void 0 ? await refetched.body() : /* @__PURE__ */ new Uint8Array();
|
|
4097
|
+
return {
|
|
4098
|
+
pdf: isPdfBytes(bytes),
|
|
4099
|
+
bytes,
|
|
4100
|
+
url: refetched.url?.() ?? url,
|
|
4101
|
+
status: refetched.status()
|
|
4102
|
+
};
|
|
4103
|
+
}
|
|
4104
|
+
/**
|
|
4105
|
+
* Extract PDF bytes to a markdown text result inside the fetch budget.
|
|
4106
|
+
* A limit that stopped the page walk early flags the result truncated.
|
|
4107
|
+
*/
|
|
4108
|
+
async function pdfResult(bytes, finalUrl, statusCode, deadline) {
|
|
4109
|
+
const extraction = await extractPdfMarkdown(bytes, {
|
|
4110
|
+
signal: deadline.signal,
|
|
4111
|
+
timeBudgetMs: Math.max(1e3, deadline.remainingMs() - PDF_FINISH_RESERVE_MS),
|
|
4112
|
+
maxChars: MAX_BODY_CHARS
|
|
4113
|
+
});
|
|
4114
|
+
const result = capResult(finalUrl, statusCode, {
|
|
4115
|
+
kind: "text",
|
|
4116
|
+
content: extraction.markdown
|
|
4117
|
+
});
|
|
4118
|
+
return extraction.truncated && !result.truncated ? {
|
|
4119
|
+
...result,
|
|
4120
|
+
truncated: true
|
|
4121
|
+
} : result;
|
|
4122
|
+
}
|
|
3124
4123
|
/**
|
|
3125
4124
|
* Probe the live document for challenge markers. Prefers the in-page probe
|
|
3126
4125
|
* (`page.evaluate`) — the only thing that can see SPA-style clears — and
|
|
@@ -3283,4 +4282,4 @@ function apply(ctx, config) {
|
|
|
3283
4282
|
ctx.web.registerFetchProvider(provider);
|
|
3284
4283
|
}
|
|
3285
4284
|
//#endregion
|
|
3286
|
-
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, MAX_SCROLL_BUDGET_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint, resolveConfig, snapshotsOf };
|
|
4285
|
+
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, MAX_PDF_BYTES, MAX_PDF_PAGES, MAX_SCROLL_BUDGET_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE, apply, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, extractPdfMarkdown, htmlToMarkdown, inject, isChallengeCompatibleResponse, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, name, normalizeCdpEndpoint, readPdfDocument, resolveConfig, snapshotsOf };
|