dsh-web-fetch-playwright 0.2.18 → 0.2.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -1067,6 +1067,7 @@ async function deepCaptureWalk(page, budgetMs) {
1067
1067
  //#region src/session-window.ts
1068
1068
  const ownership = /* @__PURE__ */ new WeakMap();
1069
1069
  const unavailable = /* @__PURE__ */ new WeakSet();
1070
+ const identities = /* @__PURE__ */ new WeakMap();
1070
1071
  const CLEANUP_GRACE_MS = 100;
1071
1072
  function claim(browser, context, keys) {
1072
1073
  let owners = ownership.get(browser);
@@ -1163,6 +1164,8 @@ async function createSessionPage(context, browser, timeoutMs) {
1163
1164
  return session;
1164
1165
  };
1165
1166
  const identify = async (page) => {
1167
+ const known = identities.get(page);
1168
+ if (known) return known;
1166
1169
  const session = await newSession(() => ctx.newCDPSession(page));
1167
1170
  let identificationFailed = false;
1168
1171
  try {
@@ -1212,6 +1215,7 @@ async function createSessionPage(context, browser, timeoutMs) {
1212
1215
  targetId = actual.targetId;
1213
1216
  if (actual.browserContextId !== identity.browserContextId || await windowOf(actual.targetId) !== expectedWindow) throw new Error("Chromium did not place the session tab in its dedicated window");
1214
1217
  claim(browser, context, [`context:${actual.browserContextId}`, `window:${expectedWindow}`]);
1218
+ identities.set(created, actual);
1215
1219
  succeeded = true;
1216
1220
  return created;
1217
1221
  }
@@ -1259,6 +1263,10 @@ async function createSessionPage(context, browser, timeoutMs) {
1259
1263
  const explicitWindow = await windowOf(explicitTargetId);
1260
1264
  if (explicitWindow === bootstrapWindow) throw new Error("Chromium ignored the dedicated session window request");
1261
1265
  claim(browser, context, [`context:${identity.browserContextId}`, `window:${explicitWindow}`]);
1266
+ identities.set(created, {
1267
+ targetId: explicitTargetId,
1268
+ browserContextId: identity.browserContextId
1269
+ });
1262
1270
  await deadline.run(() => bootstrap.close());
1263
1271
  bootstrap = void 0;
1264
1272
  succeeded = true;
@@ -2210,6 +2218,902 @@ function htmlToMarkdown(html, url) {
2210
2218
  };
2211
2219
  }
2212
2220
  //#endregion
2221
+ //#region src/pdf.ts
2222
+ /**
2223
+ * PDF → markdown. A fetched URL that turns out to be a PDF (served inline as
2224
+ * `application/pdf`, as a `Content-Disposition: attachment` download, or as
2225
+ * an anonymous `application/octet-stream` whose bytes start with `%PDF-`) is
2226
+ * routed here instead of failing with `WEB_UNSUPPORTED_CONTENT_TYPE`.
2227
+ *
2228
+ * Two layers, kept apart so the heuristics are testable without pdf.js:
2229
+ *
2230
+ * 1. {@link readPdfDocument} — the pdf.js boundary (through `unpdf`'s
2231
+ * serverless build, loaded lazily so plugin start never pays for it).
2232
+ * Per page it reads the positioned text runs (`getTextContent`) plus the
2233
+ * real font names (`getOperatorList` + `fontExtraProperties`, the only way
2234
+ * pdf.js exposes "NimbusRomNo9L-Medi" instead of a generic family), so the
2235
+ * layout pass can see bold and monospace. Bounded by the fetch deadline,
2236
+ * the caller's abort signal, a page cap, and a collected-character cap.
2237
+ * 2. {@link layoutPdfMarkdown} — pure heuristics over those runs:
2238
+ * - runs → lines (baseline grouping in content-stream order, x-sorted
2239
+ * inside a line; rotated runs such as arXiv's side stamp are dropped);
2240
+ * - running headers/footers (text repeated in the page margins across
2241
+ * pages, digits normalized) and bare page numbers are removed;
2242
+ * - the dominant (body) font size and line pitch are measured;
2243
+ * - headings come from, strongest first: the PDF outline (bookmarks),
2244
+ * section numbering ("3.1 Pre-training", "第二章", "一、"), larger font
2245
+ * sizes (ranked into levels), and standalone bold lines;
2246
+ * - lines are merged into paragraphs (vertical gap, first-line indent,
2247
+ * short terminal line, font change; continuation across columns and
2248
+ * pages when the sentence is unfinished), with dictionary-checked
2249
+ * de-hyphenation and CJK-aware joining;
2250
+ * - bullets / numbered items become markdown lists, monospace blocks
2251
+ * become fenced code (unless the whole document is monospace), lines
2252
+ * with wide column gaps become GFM tables when they line up;
2253
+ * - `<!-- page N -->` markers keep page provenance for citations.
2254
+ *
2255
+ * Scanned PDFs without a text layer, encrypted PDFs, and broken files fail
2256
+ * with dedicated, provider-specific error codes (OCR is out of scope).
2257
+ *
2258
+ * @module dsh-web-fetch-playwright/pdf
2259
+ */
2260
+ /** Largest PDF this provider downloads and parses (bytes). */
2261
+ const MAX_PDF_BYTES = 52428800;
2262
+ /** Most pages read from one document; later pages are reported as truncated. */
2263
+ const MAX_PDF_PAGES = 500;
2264
+ /** No text layer (a scanned / image-only PDF); OCR is not supported. */
2265
+ const WEB_PDF_NO_TEXT_CODE = "WEB_PDF_NO_TEXT";
2266
+ /** Password-protected PDF the parser cannot open. */
2267
+ const WEB_PDF_ENCRYPTED_CODE = "WEB_PDF_ENCRYPTED";
2268
+ /** Bytes claimed (or sniffed) to be a PDF that pdf.js cannot parse. */
2269
+ const WEB_PDF_INVALID_CODE = "WEB_PDF_INVALID";
2270
+ /** PDF larger than {@link MAX_PDF_BYTES}. */
2271
+ const WEB_PDF_TOO_LARGE_CODE = "WEB_PDF_TOO_LARGE";
2272
+ const PDF_MIME_TYPES = /* @__PURE__ */ new Set([
2273
+ "application/pdf",
2274
+ "application/x-pdf",
2275
+ "application/acrobat",
2276
+ "applications/vnd.pdf",
2277
+ "text/pdf",
2278
+ "text/x-pdf"
2279
+ ]);
2280
+ /** Generic binary types a server may use for a PDF; their bytes get sniffed. */
2281
+ const OPAQUE_BINARY_TYPES = /* @__PURE__ */ new Set([
2282
+ "application/octet-stream",
2283
+ "binary/octet-stream",
2284
+ "application/binary",
2285
+ "application/download",
2286
+ "application/x-download",
2287
+ "application/force-download",
2288
+ "application/unknown"
2289
+ ]);
2290
+ /** The bare media type of a `Content-Type` header value. */
2291
+ function mediaType(contentType) {
2292
+ return (contentType ?? "").replace(/;.*$/s, "").trim().toLowerCase();
2293
+ }
2294
+ /** Whether a `Content-Type` names a PDF. */
2295
+ function isPdfContentType(contentType) {
2296
+ return PDF_MIME_TYPES.has(mediaType(contentType));
2297
+ }
2298
+ /**
2299
+ * Whether a response that is not otherwise decodable is worth downloading
2300
+ * to sniff for PDF bytes: a PDF content type, a `.pdf` attachment filename,
2301
+ * a `.pdf` URL path, or an opaque binary type (servers commonly hand PDFs out
2302
+ * as `application/octet-stream`). The magic-byte check then decides.
2303
+ */
2304
+ function mayBePdf(headers, url) {
2305
+ if (isPdfContentType(headers["content-type"])) return true;
2306
+ const disposition = headers["content-disposition"] ?? "";
2307
+ if (/filename\*?\s*=\s*(?:[\w-]+'[^']*')?"?[^";]*\.pdf\b/i.test(disposition)) return true;
2308
+ try {
2309
+ if (/\.pdf$/i.test(new URL(url).pathname)) return true;
2310
+ } catch {}
2311
+ return OPAQUE_BINARY_TYPES.has(mediaType(headers["content-type"]));
2312
+ }
2313
+ /**
2314
+ * Whether the bytes are a PDF: `%PDF-` within the first 1024 bytes (the
2315
+ * spec-tolerated leading-garbage window every mainstream reader accepts).
2316
+ */
2317
+ function isPdfBytes(bytes) {
2318
+ const limit = Math.min(bytes.length - 5, 1024);
2319
+ for (let i = 0; i <= limit; i++) if (bytes[i] === 37 && bytes[i + 1] === 80 && bytes[i + 2] === 68 && bytes[i + 3] === 70 && bytes[i + 4] === 45) return true;
2320
+ return false;
2321
+ }
2322
+ /** Bold-looking PostScript font names (subset prefix already stripped). */
2323
+ const BOLD_FONT_RE = /bold|black|heavy|semibold|demibold|demi\b|[-,]?medi(um)?\b|-B$|,B$|-SB$|CMBX|CMB\d|BX\d|\bHei\b|SimHei|Heiti|HTJ-|HT-|黑体/i;
2324
+ /** Monospace-looking font names. */
2325
+ const MONO_FONT_RE = /mono|courier|consol|menlo|inconsolata|NimbusMonL|CMTT|SFTT|LMMono|SourceCodePro|FiraCode|DejaVuSansMono|LucidaConsole|Code/i;
2326
+ /**
2327
+ * Classify a pdf.js font object by its real (PostScript) name. pdf.js only
2328
+ * exposes the name after the page's operator list loaded the font, and only
2329
+ * with `fontExtraProperties`; anything unreadable classifies as regular.
2330
+ */
2331
+ function classifyFont(page, loadedName, cache) {
2332
+ if (loadedName === void 0) return {
2333
+ bold: false,
2334
+ mono: false
2335
+ };
2336
+ const cached = cache.get(loadedName);
2337
+ if (cached !== void 0) return cached;
2338
+ let name = "";
2339
+ let flagsBold = false;
2340
+ try {
2341
+ const font = page.commonObjs.get(loadedName);
2342
+ name = (font?.name ?? "").replace(/^[A-Z]{6}\+/, "");
2343
+ flagsBold = font?.bold === true || font?.black === true;
2344
+ } catch {}
2345
+ const result = {
2346
+ bold: flagsBold || BOLD_FONT_RE.test(name),
2347
+ mono: MONO_FONT_RE.test(name)
2348
+ };
2349
+ cache.set(loadedName, result);
2350
+ return result;
2351
+ }
2352
+ /** Flatten the outline tree (bounded) into `{ title, depth }` entries. */
2353
+ function flattenOutline(nodes, depth = 0, out = []) {
2354
+ if (nodes === null) return out;
2355
+ for (const node of nodes) {
2356
+ if (out.length >= 2e3) break;
2357
+ const title = (node.title ?? "").replace(/\s+/g, " ").trim();
2358
+ if (title !== "") out.push({
2359
+ title,
2360
+ depth
2361
+ });
2362
+ if (node.items !== void 0 && node.items.length > 0 && depth < 8) flattenOutline(node.items, depth + 1, out);
2363
+ }
2364
+ return out;
2365
+ }
2366
+ /** Map a pdf.js load failure onto the provider's PDF error codes. */
2367
+ function translatePdfError(error) {
2368
+ if (error instanceof WebError) return error;
2369
+ const name = error instanceof Error ? error.name : "";
2370
+ const message = error instanceof Error ? error.message : String(error);
2371
+ if (name === "PasswordException" || /password/i.test(message)) return new WebError("the PDF is password-protected and cannot be read", WEB_PDF_ENCRYPTED_CODE, { cause: error });
2372
+ return new WebError(`the PDF could not be parsed: ${message.split("\n")[0].slice(0, 200)}`, WEB_PDF_INVALID_CODE, { cause: error });
2373
+ }
2374
+ /**
2375
+ * Map Kangxi / CJK radical code points to the unified ideographs they look
2376
+ * like. Some producers (Chromium printing Noto CJK among them) write a
2377
+ * ToUnicode CMap that resolves 文 / 用 / 长 glyphs to U+2F42 ⽂ / U+2F64 ⽤
2378
+ * / U+2EA7 ⻓ — visually identical, but they break search and reading.
2379
+ * NFKC maps exactly these; it is applied to nothing else (fullwidth forms
2380
+ * stay as the document wrote them).
2381
+ */
2382
+ function normalizeRadicals(text) {
2383
+ return text.replace(/[\u2E80-\u2FDF]/g, (ch) => SIMPLIFIED_RADICALS[ch] ?? ch.normalize("NFKC"));
2384
+ }
2385
+ /**
2386
+ * The C-simplified radicals (CJK Radicals Supplement) have no NFKC mapping;
2387
+ * these are the ones that double as everyday simplified characters.
2388
+ */
2389
+ const SIMPLIFIED_RADICALS = {
2390
+ "⻅": "见",
2391
+ "⻆": "角",
2392
+ "⻈": "讠",
2393
+ "⻉": "贝",
2394
+ "⻋": "车",
2395
+ "⻐": "钅",
2396
+ "⻓": "长",
2397
+ "⻔": "门",
2398
+ "⻙": "韦",
2399
+ "⻚": "页",
2400
+ "⻛": "风",
2401
+ "⻜": "飞",
2402
+ "⻠": "饣",
2403
+ "⻢": "马",
2404
+ "⻥": "鱼",
2405
+ "⻦": "鸟",
2406
+ "⻧": "卤",
2407
+ "⻨": "麦",
2408
+ "⻩": "黄",
2409
+ "⻪": "黾",
2410
+ "⻬": "齐",
2411
+ "⻮": "齿",
2412
+ "⻰": "龙"
2413
+ };
2414
+ /** A run is horizontal when the text matrix has (almost) no rotation/skew. */
2415
+ function isHorizontal(t) {
2416
+ const [a = 0, b = 0, c = 0, d = 0] = t;
2417
+ return Math.abs(b) <= Math.abs(a) * .1 + 1e-6 && Math.abs(c) <= Math.abs(d) * .1 + 1e-6 && a > 0 && d > 0;
2418
+ }
2419
+ /**
2420
+ * Read a PDF's positioned text with pdf.js, page by page, inside the given
2421
+ * limits. Text items become {@link PdfTextRun}s (horizontal only — rotated
2422
+ * stamps and vertical margin text are dropped as layout noise).
2423
+ *
2424
+ * @returns the text model, plus whether a limit stopped reading early.
2425
+ */
2426
+ async function readPdfDocument(bytes, options = {}) {
2427
+ const startedAt = Date.now();
2428
+ const budget = options.timeBudgetMs ?? Number.POSITIVE_INFINITY;
2429
+ const maxChars = options.maxChars ?? Number.POSITIVE_INFINITY;
2430
+ const maxPages = options.maxPages ?? 500;
2431
+ const { getDocumentProxy } = await import("unpdf");
2432
+ const data = new Uint8Array(bytes.byteLength);
2433
+ data.set(bytes);
2434
+ let pdf;
2435
+ try {
2436
+ pdf = await getDocumentProxy(data, {
2437
+ fontExtraProperties: true,
2438
+ verbosity: 0,
2439
+ stopAtErrors: false
2440
+ });
2441
+ } catch (error) {
2442
+ throw translatePdfError(error);
2443
+ }
2444
+ try {
2445
+ let title;
2446
+ try {
2447
+ const meta = await pdf.getMetadata();
2448
+ const xmpTitle = meta.metadata?.get("dc:title");
2449
+ const infoTitle = meta.info?.Title;
2450
+ const candidate = typeof xmpTitle === "string" && xmpTitle.trim() !== "" ? xmpTitle : typeof infoTitle === "string" ? infoTitle : "";
2451
+ if (candidate.trim() !== "") title = candidate.replace(/\s+/g, " ").trim();
2452
+ } catch {}
2453
+ let outline = [];
2454
+ try {
2455
+ outline = flattenOutline(await pdf.getOutline());
2456
+ } catch {}
2457
+ const pages = [];
2458
+ const fontCache = /* @__PURE__ */ new Map();
2459
+ let chars = 0;
2460
+ let truncated = false;
2461
+ const pageCount = pdf.numPages;
2462
+ for (let n = 1; n <= pageCount; n++) {
2463
+ if (options.signal?.aborted) throw options.signal.reason instanceof Error ? options.signal.reason : /* @__PURE__ */ new Error("pdf extraction aborted");
2464
+ if (n > maxPages || chars >= maxChars || Date.now() - startedAt >= budget) {
2465
+ truncated = true;
2466
+ break;
2467
+ }
2468
+ const page = await pdf.getPage(n);
2469
+ const content = await page.getTextContent();
2470
+ if (Date.now() - startedAt < budget * .8) try {
2471
+ await page.getOperatorList();
2472
+ } catch {}
2473
+ const [, viewBottom = 0, , viewTop = 0] = page.view;
2474
+ const runs = [];
2475
+ for (const item of content.items) {
2476
+ if (typeof item.str !== "string" || item.str === "" || item.transform === void 0) continue;
2477
+ if (!isHorizontal(item.transform)) continue;
2478
+ const t = item.transform;
2479
+ const size = Math.hypot(t[2] ?? 0, t[3] ?? 0) || item.height || 0;
2480
+ if (!(size > 0)) continue;
2481
+ const font = classifyFont(page, item.fontName, fontCache);
2482
+ runs.push({
2483
+ text: normalizeRadicals(item.str),
2484
+ x: t[4] ?? 0,
2485
+ y: t[5] ?? 0,
2486
+ width: item.width ?? 0,
2487
+ size,
2488
+ bold: font.bold,
2489
+ mono: font.mono
2490
+ });
2491
+ chars += item.str.length;
2492
+ }
2493
+ pages.push({
2494
+ number: n,
2495
+ bottom: Math.min(viewBottom, viewTop),
2496
+ height: Math.abs(viewTop - viewBottom) || 792,
2497
+ runs
2498
+ });
2499
+ page.cleanup();
2500
+ }
2501
+ return {
2502
+ doc: {
2503
+ ...title !== void 0 ? { title } : {},
2504
+ outline,
2505
+ pages,
2506
+ totalPages: pageCount
2507
+ },
2508
+ truncated
2509
+ };
2510
+ } catch (error) {
2511
+ if (options.signal?.aborted) throw error;
2512
+ throw translatePdfError(error);
2513
+ } finally {
2514
+ pdf.loadingTask?.destroy().catch(() => {});
2515
+ }
2516
+ }
2517
+ /**
2518
+ * Extract a PDF as markdown: read it with pdf.js inside the limits, lay it
2519
+ * out, and fail with {@link WEB_PDF_NO_TEXT_CODE} when there is no text layer
2520
+ * to speak of (a scan). Sizes beyond {@link MAX_PDF_BYTES} are refused.
2521
+ */
2522
+ async function extractPdfMarkdown(bytes, options = {}) {
2523
+ if (bytes.length > 52428800) throw new WebError(`the PDF is ${String(Math.round(bytes.length / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`, WEB_PDF_TOO_LARGE_CODE);
2524
+ const { doc, truncated } = await readPdfDocument(bytes, options);
2525
+ const textChars = doc.pages.reduce((sum, page) => sum + page.runs.reduce((s, run) => s + run.text.replace(/\s+/g, "").length, 0), 0);
2526
+ if (doc.pages.length > 0 && textChars < Math.max(1, doc.pages.length * 5)) throw new WebError(`the PDF has no extractable text layer (${String(doc.totalPages)} page(s), likely scanned images); OCR is not supported`, WEB_PDF_NO_TEXT_CODE);
2527
+ let markdown = layoutPdfMarkdown(doc);
2528
+ if (truncated) markdown += `\n\n> [PDF extraction stopped after page ${String(doc.pages.length)} of ${String(doc.totalPages)} (size/time limit).]\n`;
2529
+ return {
2530
+ markdown,
2531
+ totalPages: doc.totalPages,
2532
+ pagesRead: doc.pages.length,
2533
+ truncated
2534
+ };
2535
+ }
2536
+ /** CJK ideographs, kana, hangul, and fullwidth/CJK punctuation. */
2537
+ const CJK_RE = /[⺀-⿿ -〿぀-ヿ㄀-ㇿ㐀-䶿一-鿿가-힯豈-﫿＀-￯]/;
2538
+ function isCjk(ch) {
2539
+ return ch !== void 0 && CJK_RE.test(ch);
2540
+ }
2541
+ /** Letter-spaced CJK ("预 防 和 治 理") reads as one word. */
2542
+ function collapseCjkSpacing(text) {
2543
+ return text.replace(new RegExp(`(?<=${CJK_RE.source}) +(?=${CJK_RE.source})`, "g"), "");
2544
+ }
2545
+ /** Group a page's runs into lines (content-stream order, baseline match). */
2546
+ function buildLines(page) {
2547
+ const groups = [];
2548
+ let current = [];
2549
+ let baseY = 0;
2550
+ let baseSize = 0;
2551
+ for (const run of page.runs) {
2552
+ if (current.length > 0) {
2553
+ const tolerance = Math.max(.6 * Math.max(run.size, baseSize), 1);
2554
+ if (Math.abs(run.y - baseY) <= tolerance) {
2555
+ current.push(run);
2556
+ if (run.text.trim() !== "" && run.size > baseSize) {
2557
+ baseSize = run.size;
2558
+ baseY = run.y;
2559
+ }
2560
+ continue;
2561
+ }
2562
+ groups.push(current);
2563
+ }
2564
+ current = [run];
2565
+ baseY = run.y;
2566
+ baseSize = run.size;
2567
+ }
2568
+ if (current.length > 0) groups.push(current);
2569
+ const lines = [];
2570
+ for (const group of groups) {
2571
+ const line = composeLine(group, page);
2572
+ if (line !== void 0) lines.push(line);
2573
+ }
2574
+ return lines;
2575
+ }
2576
+ /** Join one baseline group's runs (x-sorted) into a {@link Line}. */
2577
+ function composeLine(group, page) {
2578
+ const runs = [...group].sort((a, b) => a.x - b.x);
2579
+ const visible = runs.filter((run) => run.text.trim() !== "");
2580
+ if (visible.length === 0) return void 0;
2581
+ const weight = /* @__PURE__ */ new Map();
2582
+ for (const run of visible) {
2583
+ const key = Math.round(run.size * 10) / 10;
2584
+ weight.set(key, (weight.get(key) ?? 0) + run.text.trim().length);
2585
+ }
2586
+ let size = visible[0].size;
2587
+ let best = -1;
2588
+ for (const [key, w] of weight) if (w > best) {
2589
+ best = w;
2590
+ size = key;
2591
+ }
2592
+ const baseline = visible.find((run) => Math.abs(run.size - size) < .11)?.y ?? visible[0].y;
2593
+ const segments = [];
2594
+ const cells = [""];
2595
+ const cellX = [];
2596
+ let text = "";
2597
+ let prevEnd;
2598
+ let boldChars = 0;
2599
+ let monoChars = 0;
2600
+ let chars = 0;
2601
+ for (const run of runs) {
2602
+ const content = run.text;
2603
+ const trimmed = content.trim();
2604
+ if (trimmed === "") continue;
2605
+ let sep = "";
2606
+ if (prevEnd !== void 0 && text !== "") {
2607
+ const gap = run.x - prevEnd;
2608
+ const last = text[text.length - 1];
2609
+ const first = trimmed[0];
2610
+ if (gap > run.size * 1.5 && !(isCjk(last) && isCjk(first) && gap < run.size * 3)) {
2611
+ cells.push("");
2612
+ cellX.push(run.x);
2613
+ sep = " ";
2614
+ } else if (gap > run.size * .15 || /\s$/.test(content.slice(0, content.length - content.trimStart().length)) || /^\s/.test(content)) sep = isCjk(last) && isCjk(first) ? "" : " ";
2615
+ if (/\s$/.test(text)) sep = "";
2616
+ }
2617
+ const piece = sep + trimmed;
2618
+ if (cellX.length === 0) cellX.push(run.x);
2619
+ text += piece;
2620
+ cells[cells.length - 1] += cells[cells.length - 1] === "" ? trimmed : piece;
2621
+ const prev = segments[segments.length - 1];
2622
+ if (prev !== void 0 && prev.bold === run.bold) prev.text += piece;
2623
+ else segments.push({
2624
+ text: piece,
2625
+ bold: run.bold
2626
+ });
2627
+ prevEnd = Math.max(prevEnd ?? -Infinity, run.x + run.width);
2628
+ const n = trimmed.replace(/\s+/g, "").length;
2629
+ chars += n;
2630
+ if (run.bold) boldChars += n;
2631
+ if (run.mono) monoChars += n;
2632
+ }
2633
+ text = collapseCjkSpacing(text.replace(/\s+/g, " ").trim());
2634
+ if (text === "") return void 0;
2635
+ for (const segment of segments) segment.text = collapseCjkSpacing(segment.text);
2636
+ const x0 = visible.reduce((m, run) => Math.min(m, run.x), Infinity);
2637
+ const x1 = visible.reduce((m, run) => Math.max(m, run.x + run.width), -Infinity);
2638
+ return {
2639
+ page: page.number,
2640
+ text,
2641
+ segments,
2642
+ x0,
2643
+ x1,
2644
+ y: baseline,
2645
+ size,
2646
+ bold: chars > 0 && boldChars / chars >= .8,
2647
+ mono: chars > 0 && monoChars / chars >= .8,
2648
+ cells: cells.map((cell) => collapseCjkSpacing(cell.replace(/\s+/g, " ").trim())),
2649
+ cellX,
2650
+ chars,
2651
+ relY: page.height > 0 ? (baseline - page.bottom) / page.height : .5
2652
+ };
2653
+ }
2654
+ /** Bare page-number shapes found in page margins. */
2655
+ const PAGE_NUMBER_RES = [
2656
+ /^[-–—\s]*(?:page|p\.|pág\.|seite)?\s*\d{1,4}(?:\s*(?:\/|of|von|de)\s*\d{1,4})?[-–—\s]*$/i,
2657
+ /^[-–—\s]*[ivxlcdm]{1,7}[-–—\s]*$/i,
2658
+ /^第\s*\d{1,4}\s*页(?:\s*[,,//]?\s*共\s*\d{1,4}\s*页)?$/,
2659
+ /^[-–—一-―·•]{1,2}\s*\d{1,4}\s*[-–—一-―·•]{1,2}$/
2660
+ ];
2661
+ const isMarginLine = (line) => line.relY > .9 || line.relY < .12;
2662
+ /** Drop running headers/footers and page numbers. */
2663
+ function removeMarginNoise(pages) {
2664
+ const keyOf = (line) => line.text.toLowerCase().replace(/\d+/g, "#").replace(/\s+/g, " ").trim();
2665
+ const pagesByKey = /* @__PURE__ */ new Map();
2666
+ for (const lines of pages) for (const line of lines) {
2667
+ if (!isMarginLine(line)) continue;
2668
+ const key = keyOf(line);
2669
+ let set = pagesByKey.get(key);
2670
+ if (set === void 0) {
2671
+ set = /* @__PURE__ */ new Set();
2672
+ pagesByKey.set(key, set);
2673
+ }
2674
+ set.add(line.page);
2675
+ }
2676
+ const repeatThreshold = Math.max(3, Math.ceil(pages.length * .3));
2677
+ return pages.map((lines) => lines.filter((line) => {
2678
+ if (!isMarginLine(line)) return true;
2679
+ const plain = line.text.normalize("NFKC");
2680
+ if (PAGE_NUMBER_RES.some((re) => re.test(plain))) return false;
2681
+ if (pages.length >= 3 && (pagesByKey.get(keyOf(line))?.size ?? 0) >= repeatThreshold) return false;
2682
+ return true;
2683
+ }));
2684
+ }
2685
+ /** Character-weighted mode of the line font sizes (0.5pt buckets). */
2686
+ function bodySizeOf(lines) {
2687
+ const weight = /* @__PURE__ */ new Map();
2688
+ for (const line of lines) {
2689
+ const key = Math.round(line.size * 2) / 2;
2690
+ weight.set(key, (weight.get(key) ?? 0) + line.chars);
2691
+ }
2692
+ let body = 10;
2693
+ let best = -1;
2694
+ for (const [key, w] of weight) if (w > best) {
2695
+ best = w;
2696
+ body = key;
2697
+ }
2698
+ return body;
2699
+ }
2700
+ /** Median baseline-to-baseline distance between consecutive body lines. */
2701
+ function lineSpacingOf(pages, body) {
2702
+ const deltas = [];
2703
+ for (const lines of pages) for (let i = 1; i < lines.length; i++) {
2704
+ const a = lines[i - 1];
2705
+ const b = lines[i];
2706
+ if (Math.abs(a.size - body) > body * .1 || Math.abs(b.size - body) > body * .1) continue;
2707
+ const dy = a.y - b.y;
2708
+ if (dy > body * .6 && dy < body * 3) deltas.push(dy);
2709
+ }
2710
+ if (deltas.length === 0) return body * 1.2;
2711
+ deltas.sort((p, q) => p - q);
2712
+ return deltas[Math.floor(deltas.length / 2)];
2713
+ }
2714
+ /** Normalization for comparing headings to outline titles. */
2715
+ function headingKey(text) {
2716
+ return text.normalize("NFKC").toLowerCase().replace(/[\s\p{P}\p{S}]/gu, "");
2717
+ }
2718
+ /** Leading section number ("3", "3.1", "A.2") stripped for outline matching. */
2719
+ function stripNumbering(text) {
2720
+ return text.replace(/^(?:\d+(?:\.\d+)*|[A-Z](?:\.\d+)+)\.?\s+/, "").replace(/^(?:第[一二三四五六七八九十百千零〇两\d]+[章节篇部]|[一二三四五六七八九十]+[、..]|[((][一二三四五六七八九十]+[))])\s*/, "");
2721
+ }
2722
+ const NUMBERED_HEADING_RE = /^(\d{1,2}(?:\.\d{1,2}){0,4}|[A-Z](?:\.\d{1,2}){1,4})\.?\s+(?=\S)/;
2723
+ const CN_CHAPTER_RE = /^第[一二三四五六七八九十百千零〇两\d]+[章篇部]/;
2724
+ const CN_SECTION_RE = /^第[一二三四五六七八九十百千零〇两\d]+节/;
2725
+ const CN_ENUM_RE = /^[一二三四五六七八九十]{1,3}[、.]/;
2726
+ const CN_PAREN_ENUM_RE = /^[((][一二三四五六七八九十]{1,3}[))]/;
2727
+ /** A table-of-contents entry: leader dots (or a wide gap) then a page reference. */
2728
+ const TOC_ENTRY_RE = /(?:[.…·.]\s*){4,}[^.…·.]{0,16}?[((]?\s*[\d0-9ivxlc]+\s*[))]?$/i;
2729
+ const EXPLICIT_HEADING_RES = [
2730
+ NUMBERED_HEADING_RE,
2731
+ CN_CHAPTER_RE,
2732
+ CN_SECTION_RE,
2733
+ CN_ENUM_RE,
2734
+ CN_PAREN_ENUM_RE
2735
+ ];
2736
+ const TERMINAL_RE = /[.!?。!?…::;;"”’))\]】]$/;
2737
+ const SENTENCE_END_RE = /[.!?。!?…"”’]$/;
2738
+ const BULLET_RE = /^(?:[•◦▪▫●○■□►▸‣⁃∙·*❖➢✓✔]|[])\s*/;
2739
+ const DASH_BULLET_RE = /^[-–—]\s+(?=\S)/;
2740
+ const ORDERED_RE = /^(\d{1,3})[.)]\s+(?=\S)/;
2741
+ const PAREN_ITEM_RE = /^(?:[((](?:\d{1,3}|[a-zA-Z]|[ivx]{1,4})[))]|[a-z][.)]|[①-⑳])\s*(?=\S)/;
2742
+ /** Heading level placeholder: one below the most recent heading. */
2743
+ const RELATIVE_HEADING = -1;
2744
+ /** Decide a line's heading level, if it is one. */
2745
+ function headingLevel(line, groupLines, stats, standalone, styleRules) {
2746
+ const text = groupLines.map((l) => l.text).join(" ");
2747
+ const chars = groupLines.reduce((s, l) => s + l.chars, 0);
2748
+ if (chars === 0 || chars > 200 || !/\p{L}/u.test(text)) return void 0;
2749
+ const words = text.split(/\s+/).length;
2750
+ const sentenceLike = /[,,;;]$/.test(text) || /[.。]$/.test(text) && !/^\d+(\.\d+)*\.?$/.test(text.split(" ")[0] ?? "");
2751
+ if (stats.titleSize !== void 0 && Math.abs(line.size - stats.titleSize) < .3 && groupLines.length <= 3 && styleRules) return 1;
2752
+ const outlineDepth = stats.outlineKeys.get(headingKey(text)) ?? stats.outlineKeys.get(headingKey(stripNumbering(text)));
2753
+ if (outlineDepth !== void 0 && chars <= 160) return Math.min(6, Math.max(2, outlineDepth + 2 - stats.outlineOffset));
2754
+ const bigger = line.size >= stats.body * 1.12;
2755
+ const emphasized = bigger || line.bold;
2756
+ const styleOk = styleRules && groupLines.every((l) => l.cells.length < 2);
2757
+ const numbered = NUMBERED_HEADING_RE.exec(text);
2758
+ if (numbered !== null && emphasized && !sentenceLike && words <= 16 && chars <= 120) {
2759
+ const depth = numbered[1].split(".").length;
2760
+ return Math.min(6, depth + 1);
2761
+ }
2762
+ if (CN_CHAPTER_RE.test(text) && chars <= 60 && !sentenceLike) return 2;
2763
+ if (CN_SECTION_RE.test(text) && chars <= 60 && !sentenceLike) return 3;
2764
+ if (CN_ENUM_RE.test(text) && chars <= 40 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 2;
2765
+ if (CN_PAREN_ENUM_RE.test(text) && chars <= 30 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 3;
2766
+ if (!styleOk) return void 0;
2767
+ if (bigger && groupLines.length <= 3 && !sentenceLike) {
2768
+ if (stats.titleSize !== void 0 && Math.abs(line.size - stats.titleSize) < .3) return 1;
2769
+ const rank = stats.headingSizes.findIndex((size) => Math.abs(size - Math.round(line.size * 2) / 2) < .3);
2770
+ return Math.min(4, 2 + Math.max(0, rank));
2771
+ }
2772
+ const lastSegment = groupLines[groupLines.length - 1].segments.at(-1);
2773
+ if (line.bold && lastSegment?.bold === true && standalone && groupLines.length <= 2 && chars <= 100 && words <= 14 && !sentenceLike && !TERMINAL_RE.test(text.replace(/[))]$/, ""))) return RELATIVE_HEADING;
2774
+ }
2775
+ /** Group lines into markdown blocks. */
2776
+ function buildBlocks(pages, stats) {
2777
+ const all = pages.flat();
2778
+ const blocks = [];
2779
+ const rightEdge = /* @__PURE__ */ new Map();
2780
+ for (const lines of pages) for (const line of lines) {
2781
+ let right = line.x1;
2782
+ for (const other of lines) if (Math.abs(other.x0 - line.x0) < line.size * 3 && Math.abs(other.size - line.size) < line.size * .15) right = Math.max(right, other.x1);
2783
+ rightEdge.set(line, right);
2784
+ }
2785
+ const pitch = (line) => stats.spacing * (line.size / stats.body);
2786
+ const gapBefore = (i) => {
2787
+ const prev = all[i - 1];
2788
+ const line = all[i];
2789
+ if (prev === void 0 || prev.page !== line.page) return Infinity;
2790
+ return prev.y - line.y;
2791
+ };
2792
+ const isStandalone = (i, span) => {
2793
+ const before = gapBefore(i);
2794
+ const after = i + span < all.length ? gapBefore(i + span) : Infinity;
2795
+ const first = all[i];
2796
+ const sepBefore = before === Infinity || before < 0 || before > pitch(first) * 1.3 || all[i - 1].size !== first.size || all[i - 1].bold !== first.bold;
2797
+ const next = all[i + span];
2798
+ const sepAfter = after === Infinity || after < 0 || after > pitch(first) * 1.3 || next === void 0 || Math.abs(next.size - first.size) > .3;
2799
+ const short = all[i + span - 1].x1 < (rightEdge.get(all[i + span - 1]) ?? Infinity) - first.size * 2;
2800
+ return sepBefore && (sepAfter || short);
2801
+ };
2802
+ const isInlineMono = (index) => {
2803
+ const line = all[index];
2804
+ const prev = all[index - 1];
2805
+ const next = all[index + 1];
2806
+ if (prev === void 0 || prev.mono || next?.mono === true && next.page === line.page && gapBefore(index + 1) > 0 && gapBefore(index + 1) < pitch(line) * 2.2) return false;
2807
+ const dy = gapBefore(index);
2808
+ return dy > 0 && dy < pitch(line) * 1.45 && !TERMINAL_RE.test(prev.text) && line.chars <= 80;
2809
+ };
2810
+ const tableEnd = /* @__PURE__ */ new Map();
2811
+ const inTable = /* @__PURE__ */ new Set();
2812
+ for (let start = 0; start < all.length;) {
2813
+ if (all[start].cells.length < 2 || all[start].mono) {
2814
+ start++;
2815
+ continue;
2816
+ }
2817
+ let end = start + 1;
2818
+ while (end < all.length && all[end].cells.length >= 2 && !all[end].mono && all[end].page === all[start].page && gapBefore(end) > 0 && gapBefore(end) < pitch(all[end]) * 2.5) end++;
2819
+ let first = start;
2820
+ let last = end;
2821
+ while (last - first >= 2 && !rowAligned(all[last - 1], all.slice(first, last - 1))) last--;
2822
+ while (last - first >= 2 && !rowAligned(all[first], all.slice(first + 1, last))) first++;
2823
+ if (last - first >= 2 && columnsAlign(all.slice(first, last))) {
2824
+ tableEnd.set(first, last);
2825
+ for (let k = first; k < last; k++) inTable.add(k);
2826
+ }
2827
+ start = end;
2828
+ }
2829
+ let lastLevel = 1;
2830
+ let i = 0;
2831
+ while (i < all.length) {
2832
+ const line = all[i];
2833
+ let span = 1;
2834
+ while (span < 4 && i + span < all.length && all[i + span].page === line.page && Math.abs(all[i + span].size - line.size) < .3 && all[i + span].bold === line.bold && gapBefore(i + span) > 0 && gapBefore(i + span) < pitch(line) * 1.6 && !EXPLICIT_HEADING_RES.some((re) => re.test(all[i + span].text))) span++;
2835
+ let level;
2836
+ let used = 0;
2837
+ if (span <= 3) {
2838
+ level = headingLevel(line, all.slice(i, i + span), stats, isStandalone(i, span), true);
2839
+ if (level !== void 0) used = span;
2840
+ }
2841
+ if (level === void 0 && span > 1) {
2842
+ level = headingLevel(line, [line], stats, isStandalone(i, 1), false);
2843
+ if (level !== void 0) used = 1;
2844
+ }
2845
+ if (level !== void 0 && line.mono && !stats.monoIsBody) level = void 0;
2846
+ if (level === RELATIVE_HEADING) level = Math.min(6, Math.max(2, lastLevel + 1));
2847
+ else if (level !== void 0) lastLevel = level;
2848
+ if (level !== void 0) {
2849
+ blocks.push({
2850
+ kind: "heading",
2851
+ page: line.page,
2852
+ level,
2853
+ lines: all.slice(i, i + used)
2854
+ });
2855
+ i += used;
2856
+ continue;
2857
+ }
2858
+ const start = i;
2859
+ const kind = tableEnd.has(i) ? "table" : lineKind(line, stats);
2860
+ const block = {
2861
+ kind: kind === "code" && isInlineMono(i) ? "text" : kind,
2862
+ page: line.page,
2863
+ lines: [line]
2864
+ };
2865
+ i++;
2866
+ if (kind === "table") {
2867
+ const end = tableEnd.get(start) ?? start + 1;
2868
+ while (i < end) {
2869
+ block.lines.push(all[i]);
2870
+ i++;
2871
+ }
2872
+ blocks.push(block);
2873
+ continue;
2874
+ }
2875
+ if (kind === "code") {
2876
+ while (i < all.length && all[i].mono && gapBefore(i) > 0 && gapBefore(i) < pitch(all[i]) * 2.2) {
2877
+ block.lines.push(all[i]);
2878
+ i++;
2879
+ }
2880
+ blocks.push(block);
2881
+ continue;
2882
+ }
2883
+ let blockRight = rightEdge.get(line) ?? line.x1;
2884
+ while (i < all.length) {
2885
+ const next = all[i];
2886
+ const prev = block.lines[block.lines.length - 1];
2887
+ if (inTable.has(i)) break;
2888
+ const nextKind = lineKind(next, stats);
2889
+ if (nextKind !== "text" && !(nextKind === "code" && isInlineMono(i))) break;
2890
+ if (Math.abs(next.size - prev.size) > Math.max(next.size, prev.size) * .12) break;
2891
+ const dy = next.page === prev.page ? prev.y - next.y : -Infinity;
2892
+ if (!(dy > 0 && dy < pitch(next) * 1.45)) {
2893
+ if (dy > 0) break;
2894
+ if (TERMINAL_RE.test(prev.text)) break;
2895
+ if (headingStartLike(next)) break;
2896
+ if (next.page !== prev.page && block.kind !== "text") break;
2897
+ } else {
2898
+ if (TOC_ENTRY_RE.test(prev.text)) break;
2899
+ if (next.x0 - prev.x0 > next.size * .8 && block.kind === "text" && next.x1 >= blockRight - next.size * 2 && (TERMINAL_RE.test(prev.text) || prev.x1 < blockRight - prev.size * 2)) break;
2900
+ if (block.kind !== "text" && next.x0 < block.lines[0].x0 + next.size * .3 && TERMINAL_RE.test(prev.text)) break;
2901
+ if (prev.x1 < blockRight - prev.size * 2.5 && SENTENCE_END_RE.test(prev.text)) break;
2902
+ if (next.bold !== prev.bold && next.bold && next.chars < 100 && SENTENCE_END_RE.test(prev.text)) break;
2903
+ }
2904
+ block.lines.push(next);
2905
+ blockRight = Math.max(blockRight, rightEdge.get(next) ?? next.x1);
2906
+ i++;
2907
+ }
2908
+ blocks.push(block);
2909
+ }
2910
+ return blocks;
2911
+ }
2912
+ /**
2913
+ * Whether table-candidate rows share column starts: at least 60% of the rows
2914
+ * have two or more cell left edges within one em of another row's.
2915
+ */
2916
+ function columnsAlign(rows) {
2917
+ const firstCells = rows.map((row) => row.cells.find((cell) => cell !== "") ?? "").map((cell) => cell.length).sort((a, b) => a - b);
2918
+ if (Math.max(...rows.map((row) => row.cells.filter((cell) => cell !== "").length)) <= 2 && (firstCells[Math.floor(firstCells.length / 2)] ?? 0) > 50) return false;
2919
+ return rows.filter((row) => rowAligned(row, rows.filter((other) => other !== row))).length / rows.length >= .6;
2920
+ }
2921
+ /** Whether a row shares at least two column starts with some other row. */
2922
+ function rowAligned(row, others) {
2923
+ return others.some((other) => row.cellX.filter((x) => other.cellX.some((o) => Math.abs(o - x) < row.size)).length >= 2);
2924
+ }
2925
+ /** A line that looks like the start of a new structural element. */
2926
+ function headingStartLike(line) {
2927
+ return BULLET_RE.test(line.text) || ORDERED_RE.test(line.text) || CN_CHAPTER_RE.test(line.text);
2928
+ }
2929
+ /** Classify a non-heading line. */
2930
+ function lineKind(line, stats) {
2931
+ if (line.mono && !stats.monoIsBody) return "code";
2932
+ if (BULLET_RE.test(line.text) && line.text.replace(BULLET_RE, "").trim() !== "") return "bullet";
2933
+ if (DASH_BULLET_RE.test(line.text)) return "bullet";
2934
+ if (ORDERED_RE.test(line.text)) return "ordered";
2935
+ if (PAREN_ITEM_RE.test(line.text)) return "item";
2936
+ return "text";
2937
+ }
2938
+ /** Words seen in the document (lowercased), for de-hyphenation decisions. */
2939
+ function vocabularyOf(lines) {
2940
+ const words = /* @__PURE__ */ new Set();
2941
+ for (const line of lines) for (const match of line.text.matchAll(/[A-Za-z]+(?:-[A-Za-z]+)*/g)) {
2942
+ const word = match[0].toLowerCase();
2943
+ words.add(word);
2944
+ if (word.includes("-")) words.add(word.replace(/-/g, "\0"));
2945
+ }
2946
+ return words;
2947
+ }
2948
+ /** Join two consecutive lines of one paragraph. */
2949
+ function joinText(acc, next, vocabulary) {
2950
+ if (acc === "") return next;
2951
+ const hyphen = /([A-Za-z]+)[-\u2010\u00AD]$/.exec(acc);
2952
+ const lead = /^([a-z]+)/.exec(next);
2953
+ if (hyphen !== null && lead !== null && !/\s[-\u2010\u00AD]$/.test(acc)) {
2954
+ const head = hyphen[1].toLowerCase();
2955
+ const tail = lead[1];
2956
+ if (vocabulary.has(head + tail)) return acc.slice(0, -1) + next;
2957
+ if (vocabulary.has(`${head}\u0000${tail}`)) return `${acc.slice(0, -1)}-${next}`;
2958
+ return acc.slice(0, -1) + next;
2959
+ }
2960
+ const last = acc[acc.length - 1];
2961
+ const first = next[0];
2962
+ if (isCjk(last) || isCjk(first)) return acc + next;
2963
+ if (/[-/]$/.test(acc)) return acc + next;
2964
+ return `${acc} ${next}`;
2965
+ }
2966
+ /** Escape what would otherwise read as markdown structure at a block start. */
2967
+ function escapeLeading(text) {
2968
+ return text.replace(/^(#{1,6}\s|>|[-+*]\s|\d{1,9}[.)]\s)/, "\\$1");
2969
+ }
2970
+ /** Render a line with inline bold (when only part of it is bold). */
2971
+ function renderLine(line, inlineBold) {
2972
+ if (!inlineBold || !line.segments.some((s) => s.bold) || line.segments.every((s) => s.bold)) return line.text;
2973
+ let out = "";
2974
+ for (const segment of line.segments) {
2975
+ const text = segment.text;
2976
+ const core = text.trim();
2977
+ if (segment.bold && core.replace(/[^\p{L}\p{N}]/gu, "").length >= 2 && !/^\*|\*$/.test(core)) {
2978
+ const lead = text.slice(0, text.length - text.trimStart().length);
2979
+ const trail = text.slice(text.trimEnd().length);
2980
+ out += `${lead}**${core}**${trail}`;
2981
+ } else out += text;
2982
+ }
2983
+ return out.replace(/\s+/g, " ").trim();
2984
+ }
2985
+ /**
2986
+ * Merge bold spans split only by whitespace or punctuation set in the
2987
+ * regular face ("**一是…会议**,**推动会议** **精神**" → one span): the
2988
+ * separator moves inside, and a line-break space between CJK spans goes.
2989
+ */
2990
+ function mergeBoldRuns(text) {
2991
+ return text.replace(/\*\*(\s*)([,、,;;::]?)(\s*)\*\*/g, (match, a, punct, b, offset, whole) => {
2992
+ const before = whole[offset - 1];
2993
+ const after = whole[offset + match.length];
2994
+ const space = (a !== "" || b !== "") && !(isCjk(before) || isCjk(after) || isCjk(punct)) ? " " : "";
2995
+ return punct === "" ? space : `${punct}${b !== "" && !isCjk(punct) ? " " : ""}`;
2996
+ });
2997
+ }
2998
+ /** Render a table block as GFM when its rows line up, else as plain rows. */
2999
+ function renderTable(block) {
3000
+ const rows = block.lines.map((line) => line.cells.filter((cell) => cell !== "").map((cell) => cell.replace(/\|/g, "\\|")));
3001
+ const width = Math.max(...rows.map((row) => row.length));
3002
+ const aligned = rows.filter((row) => row.length === width).length;
3003
+ if (rows.length >= 2 && width >= 2 && aligned / rows.length >= .6) {
3004
+ const pad = (row) => `| ${[...row, ...Array(width - row.length).fill("")].slice(0, width).join(" | ")} |`;
3005
+ return [
3006
+ pad(rows[0]),
3007
+ `| ${Array(width).fill("---").join(" | ")} |`,
3008
+ ...rows.slice(1).map(pad)
3009
+ ].join("\n");
3010
+ }
3011
+ return block.lines.map((line) => escapeLeading(line.cells.join(" "))).join(" \n");
3012
+ }
3013
+ /** Whether a metadata title is worth printing as the document heading. */
3014
+ function plausibleTitle(title) {
3015
+ if (title === void 0) return false;
3016
+ const t = title.trim();
3017
+ if (t.length < 3 || t.length > 250) return false;
3018
+ if (/^(?:microsoft\s+(?:word|powerpoint|excel)\b|untitled|slide\s*\d|document\d*$|title$|no title|无标题)/i.test(t)) return false;
3019
+ if (/\.(?:docx?|pdf|pptx?|xlsx?|tex|dvi|indd|qxd|rtf|odt)$/i.test(t)) return false;
3020
+ if (/^[\w-]+$/.test(t) && !/\s/.test(t) && /\d/.test(t)) return false;
3021
+ return true;
3022
+ }
3023
+ /**
3024
+ * Lay a PDF's positioned text out as markdown (see the module docs for the
3025
+ * heuristics). Pure — no pdf.js — so the rules are unit-testable with
3026
+ * synthetic runs.
3027
+ */
3028
+ function layoutPdfMarkdown(doc) {
3029
+ const pages = removeMarginNoise(doc.pages.map(buildLines));
3030
+ const all = pages.flat();
3031
+ if (all.length === 0) return plausibleTitle(doc.title) ? `# ${doc.title}\n` : "";
3032
+ const body = bodySizeOf(all);
3033
+ const totalChars = all.reduce((s, l) => s + l.chars, 0);
3034
+ const monoIsBody = all.reduce((s, l) => s + (l.mono ? l.chars : 0), 0) > totalChars * .5;
3035
+ const sizeChars = /* @__PURE__ */ new Map();
3036
+ for (const line of all) {
3037
+ if (line.size < body * 1.12) continue;
3038
+ const key = Math.round(line.size * 2) / 2;
3039
+ sizeChars.set(key, (sizeChars.get(key) ?? 0) + line.chars);
3040
+ }
3041
+ let headingSizes = [...sizeChars.keys()].filter((size) => (sizeChars.get(size) ?? 0) < Math.max(totalChars * .25, 400)).sort((a, b) => b - a);
3042
+ let titleSize;
3043
+ const firstPage = doc.pages[0]?.number ?? 1;
3044
+ const firstPageLines = all.filter((line) => line.page === firstPage && /\p{L}{2}/u.test(line.text));
3045
+ const firstLargest = firstPageLines.reduce((m, line) => Math.max(m, Math.round(line.size * 2) / 2), 0);
3046
+ if (firstLargest >= body * 1.3) {
3047
+ if (firstPageLines.filter((line) => Math.abs(Math.round(line.size * 2) / 2 - firstLargest) < .3).length <= 3) {
3048
+ titleSize = firstLargest;
3049
+ headingSizes = headingSizes.filter((size) => size < firstLargest - .25);
3050
+ }
3051
+ }
3052
+ headingSizes = headingSizes.slice(0, 3);
3053
+ const outlineKeys = /* @__PURE__ */ new Map();
3054
+ for (const entry of doc.outline) for (const key of [headingKey(entry.title), headingKey(stripNumbering(entry.title))]) if (key.length >= 2 && !outlineKeys.has(key)) outlineKeys.set(key, entry.depth);
3055
+ const roots = doc.outline.filter((entry) => entry.depth === 0);
3056
+ const titleKeys = new Set([doc.title ?? "", ...all.filter((line) => titleSize !== void 0 && Math.abs(line.size - titleSize) < .3).map((line) => line.text)].map(headingKey));
3057
+ const outlineOffset = roots.length === 1 && titleKeys.has(headingKey(roots[0].title)) ? 1 : 0;
3058
+ const blocks = buildBlocks(pages, {
3059
+ body,
3060
+ spacing: lineSpacingOf(pages, body),
3061
+ monoIsBody,
3062
+ headingSizes,
3063
+ ...titleSize !== void 0 ? { titleSize } : {},
3064
+ outlineKeys,
3065
+ outlineOffset
3066
+ });
3067
+ const vocabulary = vocabularyOf(all);
3068
+ const out = [];
3069
+ const metaKey = headingKey(doc.title ?? "");
3070
+ if (!blocks.slice(0, 12).some((block) => block.kind === "heading" && (block.level === 1 || headingKey(block.lines.map((l) => l.text).join("")) === metaKey)) && plausibleTitle(doc.title)) out.push(`# ${doc.title.replace(/\s+/g, " ").trim()}`);
3071
+ let lastPage = 0;
3072
+ for (const block of blocks) {
3073
+ if (block.page !== lastPage) {
3074
+ out.push(`<!-- page ${String(block.page)} -->`);
3075
+ lastPage = block.page;
3076
+ }
3077
+ switch (block.kind) {
3078
+ case "heading": {
3079
+ const text = block.lines.map((l) => l.text).reduce((acc, t) => joinText(acc, t, vocabulary), "");
3080
+ out.push(`${"#".repeat(block.level ?? 2)} ${text}`);
3081
+ break;
3082
+ }
3083
+ case "code": {
3084
+ const left = Math.min(...block.lines.map((l) => l.x0));
3085
+ const advances = block.lines.filter((l) => l.text.length >= 4).map((l) => (l.x1 - l.x0) / l.text.length).sort((p, q) => p - q);
3086
+ const advance = advances[Math.floor(advances.length / 2)] ?? block.lines[0].size * .6;
3087
+ const body = block.lines.map((l) => {
3088
+ const indent = Math.max(0, Math.round((l.x0 - left) / advance));
3089
+ return " ".repeat(Math.min(indent, 40)) + l.text;
3090
+ }).join("\n");
3091
+ out.push(`\`\`\`\n${body.replace(/```/g, "`​``")}\n\`\`\``);
3092
+ break;
3093
+ }
3094
+ case "table":
3095
+ out.push(renderTable(block));
3096
+ break;
3097
+ default: {
3098
+ let text = "";
3099
+ for (const line of block.lines) text = joinText(text, renderLine(line, true), vocabulary);
3100
+ text = mergeBoldRuns(text);
3101
+ if (block.kind === "bullet") {
3102
+ text = text.replace(BULLET_RE, "").replace(DASH_BULLET_RE, "");
3103
+ const prev = out[out.length - 1];
3104
+ if (prev?.startsWith("- ") === true) out[out.length - 1] = `${prev}\n- ${text}`;
3105
+ else out.push(`- ${text}`);
3106
+ } else if (block.kind === "ordered") {
3107
+ const prev = out[out.length - 1];
3108
+ if (prev !== void 0 && /^\d{1,3}\.\s/.test(prev)) out[out.length - 1] = `${prev}\n${text.replace(/^(\d{1,3})\)/, "$1.")}`;
3109
+ else out.push(text.replace(/^(\d{1,3})\)/, "$1."));
3110
+ } else out.push(escapeLeading(text));
3111
+ }
3112
+ }
3113
+ }
3114
+ return `${out.join("\n\n")}\n`;
3115
+ }
3116
+ //#endregion
2213
3117
  //#region src/playwright-resolve.ts
2214
3118
  /**
2215
3119
  * Locates a usable Playwright for the local backend. Resolution order:
@@ -2422,6 +3326,15 @@ function existsAsFile(path) {
2422
3326
  * {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
2423
3327
  * a queue and fail fast (rather than hang until abort) when no slot frees.
2424
3328
  *
3329
+ * PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
3330
+ * handed to the download manager as an attachment, or served as an opaque
3331
+ * `application/octet-stream` whose bytes start with `%PDF-` — is read as
3332
+ * bytes (the navigation body, else a refetch through the page's API request
3333
+ * context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
3334
+ * plus layout heuristics: headings, paragraphs, lists, tables, page
3335
+ * markers). Scans without a text layer, encrypted, oversized, and broken
3336
+ * PDFs fail with the `WEB_PDF_*` codes.
3337
+ *
2425
3338
  * Private-network and SSRF protection is not implemented (same stance as the
2426
3339
  * shipped HTTP provider); a page this provider can reach is whatever the
2427
3340
  * browser can reach. Profile mode additionally acts WITH the remote
@@ -2620,6 +3533,7 @@ function validateFetchUrl(input) {
2620
3533
  /** Classify a `Content-Type` into a decodable kind; undefined = unsupported. */
2621
3534
  function classifyContentType(contentType) {
2622
3535
  const mime = (contentType ?? "").replace(/;.*$/s, "").trim().toLowerCase();
3536
+ if (isPdfContentType(mime)) return "pdf";
2623
3537
  if (mime === "" || mime === "text/html" || mime === "application/xhtml+xml") return "html";
2624
3538
  if (mime.startsWith("text/")) return "text";
2625
3539
  if (mime === "application/json" || mime === "application/xml" || mime.endsWith("+json") || mime.endsWith("+xml")) return "text";
@@ -2813,9 +3727,14 @@ var PlaywrightFetchProvider = class {
2813
3727
  }
2814
3728
  const finalResponse = (challengeWaitMs > 0 ? tracker.last() : void 0) ?? response;
2815
3729
  const kind = classifyContentType(finalResponse?.headers()["content-type"]);
2816
- if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
2817
3730
  const finalUrl = page.url();
2818
3731
  const statusCode = finalResponse !== null && !(challengeEntryResponse !== null && finalResponse === challengeEntryResponse) ? finalResponse.status() : 200;
3732
+ if (kind === "pdf" || kind === void 0 && finalResponse !== null && mayBePdf(finalResponse.headers(), finalUrl)) {
3733
+ const download = await downloadPdfBytes(page, finalResponse, finalResponse?.url?.() ?? finalUrl, deadline);
3734
+ if (download.pdf) return await pdfResult(download.bytes, download.url, statusCode, deadline);
3735
+ if (kind === "pdf") throw new WebError("the response claims to be a PDF but its bytes are not one", WEB_PDF_INVALID_CODE);
3736
+ }
3737
+ if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
2819
3738
  if (kind === "text") return capResult(finalUrl, statusCode, {
2820
3739
  kind: "text",
2821
3740
  content: finalResponse !== null ? await finalResponse.text() : await page.content()
@@ -3024,7 +3943,7 @@ async function recoverUnrenderedStatus(error, tracker, url, denoise) {
3024
3943
  } catch {
3025
3944
  content = "";
3026
3945
  }
3027
- if (kind === "text" || content === "") return capResult(finalUrl, status, {
3946
+ if (kind !== "html" || content === "") return capResult(finalUrl, status, {
3028
3947
  kind: "text",
3029
3948
  content
3030
3949
  });
@@ -3073,7 +3992,9 @@ function isDownloadFailure(error, response) {
3073
3992
  * download decision — which works identically on the local and CDP
3074
3993
  * backends because it never touches the remote artifact.
3075
3994
  *
3076
- * Undecodable downloads (a PDF, a ZIP) still fail with
3995
+ * A PDF download (declared `application/pdf`, or an opaque binary whose
3996
+ * bytes sniff as `%PDF-`) is extracted to markdown (`src/pdf.ts`);
3997
+ * undecodable downloads (a ZIP) still fail with
3077
3998
  * `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
3078
3999
  * actually served; a playwright build without a request context surfaces a
3079
4000
  * clear "the browser will not render this" error instead of the raw
@@ -3092,7 +4013,13 @@ async function recoverDownloadResponse(error, page, tracker, url, deadline, deno
3092
4013
  const tracked = tracker.last();
3093
4014
  if (!isDownloadFailure(error, tracked)) return void 0;
3094
4015
  if (tracked !== null) {
3095
- if (classifyContentType(tracked.headers()["content-type"]) === void 0) throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
4016
+ const trackedKind = classifyContentType(tracked.headers()["content-type"]);
4017
+ if (trackedKind === "pdf" || trackedKind === void 0 && mayBePdf(tracked.headers(), tracked.url?.() ?? url.toString())) {
4018
+ const download = await downloadPdfBytes(page, null, tracked.url?.() ?? url.toString(), deadline, tracked.headers());
4019
+ if (download.pdf) return await pdfResult(download.bytes, download.url, download.status, deadline);
4020
+ throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download; not a PDF)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
4021
+ }
4022
+ if (trackedKind === void 0) throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
3096
4023
  }
3097
4024
  if (page.request === void 0) throw new WebError("the response is a file download (Content-Disposition: attachment) the browser will not render, and this playwright build exposes no request context to refetch it", "WEB_PROVIDER_ERROR", { cause: error });
3098
4025
  const fetchUrl = tracked?.url?.() ?? url.toString();
@@ -3103,6 +4030,12 @@ async function recoverDownloadResponse(error, page, tracker, url, deadline, deno
3103
4030
  const headers = refetched.headers();
3104
4031
  const servedType = (headers["content-type"] ?? "").trim() !== "" ? headers["content-type"] : tracked?.headers()["content-type"];
3105
4032
  const kind = classifyContentType(servedType);
4033
+ if (kind === "pdf") {
4034
+ assertPdfSize(headers);
4035
+ const bytes = refetched.body !== void 0 ? await refetched.body() : /* @__PURE__ */ new Uint8Array();
4036
+ if (!isPdfBytes(bytes)) throw new WebError("the response claims to be a PDF but its bytes are not one", WEB_PDF_INVALID_CODE);
4037
+ return await pdfResult(bytes, refetched.url?.() ?? fetchUrl, refetched.status(), deadline);
4038
+ }
3106
4039
  if (kind === void 0) throw new WebError(`unsupported content type "${servedType ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
3107
4040
  const finalUrl = refetched.url?.() ?? fetchUrl;
3108
4041
  const statusCode = refetched.status();
@@ -3121,6 +4054,72 @@ async function recoverDownloadResponse(error, page, tracker, url, deadline, deno
3121
4054
  content: markdown
3122
4055
  });
3123
4056
  }
4057
+ /** Time kept back from the PDF page walk for layout and result assembly. */
4058
+ const PDF_FINISH_RESERVE_MS = 1500;
4059
+ /** Refuse a PDF whose declared length is over the limit — before downloading it. */
4060
+ function assertPdfSize(headers) {
4061
+ const declared = Number.parseInt(headers?.["content-length"] ?? "", 10);
4062
+ if (Number.isFinite(declared) && declared > 52428800) throw new WebError(`the PDF is ${String(Math.round(declared / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`, WEB_PDF_TOO_LARGE_CODE);
4063
+ }
4064
+ /**
4065
+ * Get a (possibly) PDF resource's bytes. The navigation response's own body
4066
+ * is tried first — a PDF Chrome committed into its viewer still has it —
4067
+ * then the page's API request context refetches the URL (same cookie jar,
4068
+ * plain HTTP, no render-or-download decision; the only way to reach a body
4069
+ * Chromium streamed to its download manager). The `%PDF-` magic decides.
4070
+ *
4071
+ * @param page - the fetch's tab (for its request context).
4072
+ * @param response - the navigation response, when the browser committed one.
4073
+ * @param url - the URL to refetch.
4074
+ * @param deadline - the fetch budget.
4075
+ * @param servedHeaders - the navigation's headers (size pre-check) when no
4076
+ * response object is passed.
4077
+ * @returns the bytes, where they came from, and whether they are a PDF.
4078
+ */
4079
+ async function downloadPdfBytes(page, response, url, deadline, servedHeaders) {
4080
+ assertPdfSize(response?.headers() ?? servedHeaders);
4081
+ if (response?.body !== void 0) try {
4082
+ const bytes = await response.body();
4083
+ if (isPdfBytes(bytes)) return {
4084
+ pdf: true,
4085
+ bytes,
4086
+ url: response.url?.() ?? url,
4087
+ status: response.status()
4088
+ };
4089
+ } catch {}
4090
+ if (page.request === void 0) throw new WebError("the response is a PDF the browser will not hand over, and this playwright build exposes no request context to refetch it", "WEB_PROVIDER_ERROR");
4091
+ const refetched = await page.request.get(url, {
4092
+ timeout: deadline.remainingMs(),
4093
+ signal: deadline.signal
4094
+ });
4095
+ assertPdfSize(refetched.headers());
4096
+ const bytes = refetched.body !== void 0 ? await refetched.body() : /* @__PURE__ */ new Uint8Array();
4097
+ return {
4098
+ pdf: isPdfBytes(bytes),
4099
+ bytes,
4100
+ url: refetched.url?.() ?? url,
4101
+ status: refetched.status()
4102
+ };
4103
+ }
4104
+ /**
4105
+ * Extract PDF bytes to a markdown text result inside the fetch budget.
4106
+ * A limit that stopped the page walk early flags the result truncated.
4107
+ */
4108
+ async function pdfResult(bytes, finalUrl, statusCode, deadline) {
4109
+ const extraction = await extractPdfMarkdown(bytes, {
4110
+ signal: deadline.signal,
4111
+ timeBudgetMs: Math.max(1e3, deadline.remainingMs() - PDF_FINISH_RESERVE_MS),
4112
+ maxChars: MAX_BODY_CHARS
4113
+ });
4114
+ const result = capResult(finalUrl, statusCode, {
4115
+ kind: "text",
4116
+ content: extraction.markdown
4117
+ });
4118
+ return extraction.truncated && !result.truncated ? {
4119
+ ...result,
4120
+ truncated: true
4121
+ } : result;
4122
+ }
3124
4123
  /**
3125
4124
  * Probe the live document for challenge markers. Prefers the in-page probe
3126
4125
  * (`page.evaluate`) — the only thing that can see SPA-style clears — and
@@ -3283,4 +4282,4 @@ function apply(ctx, config) {
3283
4282
  ctx.web.registerFetchProvider(provider);
3284
4283
  }
3285
4284
  //#endregion
3286
- export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, MAX_SCROLL_BUDGET_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint, resolveConfig, snapshotsOf };
4285
+ export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, MAX_PDF_BYTES, MAX_PDF_PAGES, MAX_SCROLL_BUDGET_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE, apply, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, extractPdfMarkdown, htmlToMarkdown, inject, isChallengeCompatibleResponse, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, name, normalizeCdpEndpoint, readPdfDocument, resolveConfig, snapshotsOf };