dsh-web-fetch-playwright 0.2.18 → 0.2.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/pdf.ts ADDED
@@ -0,0 +1,1114 @@
1
+ /**
2
+ * PDF → markdown. A fetched URL that turns out to be a PDF (served inline as
3
+ * `application/pdf`, as a `Content-Disposition: attachment` download, or as
4
+ * an anonymous `application/octet-stream` whose bytes start with `%PDF-`) is
5
+ * routed here instead of failing with `WEB_UNSUPPORTED_CONTENT_TYPE`.
6
+ *
7
+ * Two layers, kept apart so the heuristics are testable without pdf.js:
8
+ *
9
+ * 1. {@link readPdfDocument} — the pdf.js boundary (through `unpdf`'s
10
+ * serverless build, loaded lazily so plugin start never pays for it).
11
+ * Per page it reads the positioned text runs (`getTextContent`) plus the
12
+ * real font names (`getOperatorList` + `fontExtraProperties`, the only way
13
+ * pdf.js exposes "NimbusRomNo9L-Medi" instead of a generic family), so the
14
+ * layout pass can see bold and monospace. Bounded by the fetch deadline,
15
+ * the caller's abort signal, a page cap, and a collected-character cap.
16
+ * 2. {@link layoutPdfMarkdown} — pure heuristics over those runs:
17
+ * - runs → lines (baseline grouping in content-stream order, x-sorted
18
+ * inside a line; rotated runs such as arXiv's side stamp are dropped);
19
+ * - running headers/footers (text repeated in the page margins across
20
+ * pages, digits normalized) and bare page numbers are removed;
21
+ * - the dominant (body) font size and line pitch are measured;
22
+ * - headings come from, strongest first: the PDF outline (bookmarks),
23
+ * section numbering ("3.1 Pre-training", "第二章", "一、"), larger font
24
+ * sizes (ranked into levels), and standalone bold lines;
25
+ * - lines are merged into paragraphs (vertical gap, first-line indent,
26
+ * short terminal line, font change; continuation across columns and
27
+ * pages when the sentence is unfinished), with dictionary-checked
28
+ * de-hyphenation and CJK-aware joining;
29
+ * - bullets / numbered items become markdown lists, monospace blocks
30
+ * become fenced code (unless the whole document is monospace), lines
31
+ * with wide column gaps become GFM tables when they line up;
32
+ * - `<!-- page N -->` markers keep page provenance for citations.
33
+ *
34
+ * Scanned PDFs without a text layer, encrypted PDFs, and broken files fail
35
+ * with dedicated, provider-specific error codes (OCR is out of scope).
36
+ *
37
+ * @module dsh-web-fetch-playwright/pdf
38
+ */
39
+
40
+ import { WebError } from '@deepseek-ai/dsh-web'
41
+
42
+ /** Largest PDF this provider downloads and parses (bytes). */
43
+ export const MAX_PDF_BYTES = 50 * 1024 * 1024
44
+
45
+ /** Most pages read from one document; later pages are reported as truncated. */
46
+ export const MAX_PDF_PAGES = 500
47
+
48
+ /** No text layer (a scanned / image-only PDF); OCR is not supported. */
49
+ export const WEB_PDF_NO_TEXT_CODE = 'WEB_PDF_NO_TEXT'
50
+ /** Password-protected PDF the parser cannot open. */
51
+ export const WEB_PDF_ENCRYPTED_CODE = 'WEB_PDF_ENCRYPTED'
52
+ /** Bytes claimed (or sniffed) to be a PDF that pdf.js cannot parse. */
53
+ export const WEB_PDF_INVALID_CODE = 'WEB_PDF_INVALID'
54
+ /** PDF larger than {@link MAX_PDF_BYTES}. */
55
+ export const WEB_PDF_TOO_LARGE_CODE = 'WEB_PDF_TOO_LARGE'
56
+
57
+ const PDF_MIME_TYPES = new Set(['application/pdf', 'application/x-pdf', 'application/acrobat', 'applications/vnd.pdf', 'text/pdf', 'text/x-pdf'])
58
+
59
+ /** Generic binary types a server may use for a PDF; their bytes get sniffed. */
60
+ const OPAQUE_BINARY_TYPES = new Set([
61
+ 'application/octet-stream',
62
+ 'binary/octet-stream',
63
+ 'application/binary',
64
+ 'application/download',
65
+ 'application/x-download',
66
+ 'application/force-download',
67
+ 'application/unknown',
68
+ ])
69
+
70
+ /** The bare media type of a `Content-Type` header value. */
71
+ function mediaType(contentType: string | undefined): string {
72
+ return (contentType ?? '').replace(/;.*$/s, '').trim().toLowerCase()
73
+ }
74
+
75
+ /** Whether a `Content-Type` names a PDF. */
76
+ export function isPdfContentType(contentType: string | undefined): boolean {
77
+ return PDF_MIME_TYPES.has(mediaType(contentType))
78
+ }
79
+
80
+ /**
81
+ * Whether a response that is not otherwise decodable is worth downloading
82
+ * to sniff for PDF bytes: a PDF content type, a `.pdf` attachment filename,
83
+ * a `.pdf` URL path, or an opaque binary type (servers commonly hand PDFs out
84
+ * as `application/octet-stream`). The magic-byte check then decides.
85
+ */
86
+ export function mayBePdf(headers: Record<string, string>, url: string): boolean {
87
+ if (isPdfContentType(headers['content-type'])) return true
88
+ const disposition = headers['content-disposition'] ?? ''
89
+ if (/filename\*?\s*=\s*(?:[\w-]+'[^']*')?"?[^";]*\.pdf\b/i.test(disposition)) return true
90
+ try {
91
+ if (/\.pdf$/i.test(new URL(url).pathname)) return true
92
+ } catch {
93
+ // not a URL — no path hint
94
+ }
95
+ return OPAQUE_BINARY_TYPES.has(mediaType(headers['content-type']))
96
+ }
97
+
98
+ /**
99
+ * Whether the bytes are a PDF: `%PDF-` within the first 1024 bytes (the
100
+ * spec-tolerated leading-garbage window every mainstream reader accepts).
101
+ */
102
+ export function isPdfBytes(bytes: Uint8Array): boolean {
103
+ const limit = Math.min(bytes.length - 5, 1024)
104
+ for (let i = 0; i <= limit; i++) {
105
+ if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true
106
+ }
107
+ return false
108
+ }
109
+
110
+ // ---------------------------------------------------------------------------
111
+ // Positioned text model (the layout pass's input)
112
+ // ---------------------------------------------------------------------------
113
+
114
+ /** One positioned, horizontal text run (pdf.js text item, normalized). */
115
+ export interface PdfTextRun {
116
+ text: string
117
+ /** Left edge in PDF user space. */
118
+ x: number
119
+ /** Baseline in PDF user space (origin bottom-left). */
120
+ y: number
121
+ width: number
122
+ /** Font size (vertical scale of the text matrix). */
123
+ size: number
124
+ bold: boolean
125
+ mono: boolean
126
+ }
127
+
128
+ /** One page's text runs in content-stream order. */
129
+ export interface PdfPageText {
130
+ /** 1-based page number. */
131
+ number: number
132
+ /** Page box bottom edge (user space). */
133
+ bottom: number
134
+ /** Page box height (user space). */
135
+ height: number
136
+ runs: PdfTextRun[]
137
+ }
138
+
139
+ /** One bookmark, flattened with its nesting depth (0 = top level). */
140
+ export interface PdfOutlineEntry {
141
+ title: string
142
+ depth: number
143
+ }
144
+
145
+ /** Everything the layout pass consumes. */
146
+ export interface PdfDocumentText {
147
+ /** Document info / XMP title, when present. */
148
+ title?: string
149
+ outline: PdfOutlineEntry[]
150
+ pages: PdfPageText[]
151
+ totalPages: number
152
+ }
153
+
154
+ /** The finished extraction. */
155
+ export interface PdfExtraction {
156
+ markdown: string
157
+ totalPages: number
158
+ /** Pages actually read (fewer than `totalPages` when a limit cut in). */
159
+ pagesRead: number
160
+ /** True when a page, size, or time limit stopped the extraction early. */
161
+ truncated: boolean
162
+ }
163
+
164
+ // ---------------------------------------------------------------------------
165
+ // pdf.js boundary
166
+ // ---------------------------------------------------------------------------
167
+
168
+ /** The structural pdf.js surface this module touches. */
169
+ interface PdfJsTextItem {
170
+ str?: string
171
+ transform?: number[]
172
+ width?: number
173
+ height?: number
174
+ fontName?: string
175
+ }
176
+ interface PdfJsPage {
177
+ view: number[]
178
+ getTextContent(): Promise<{ items: PdfJsTextItem[] }>
179
+ getOperatorList(): Promise<unknown>
180
+ commonObjs: { get(id: string): unknown }
181
+ cleanup(): unknown
182
+ }
183
+ interface PdfJsOutlineNode {
184
+ title?: string
185
+ items?: PdfJsOutlineNode[]
186
+ }
187
+ interface PdfJsDocument {
188
+ numPages: number
189
+ getPage(n: number): Promise<PdfJsPage>
190
+ getOutline(): Promise<PdfJsOutlineNode[] | null>
191
+ getMetadata(): Promise<{ info?: { Title?: unknown }; metadata?: { get(name: string): unknown } | null }>
192
+ loadingTask?: { destroy(): Promise<void> }
193
+ }
194
+
195
+ /** Limits one extraction runs under. */
196
+ export interface PdfExtractOptions {
197
+ /** Abort → stop between pages and fail. */
198
+ signal?: AbortSignal
199
+ /** Wall-clock budget for reading pages (ms); pages left over are truncated. */
200
+ timeBudgetMs?: number
201
+ /** Stop reading pages once this many characters were collected. */
202
+ maxChars?: number
203
+ /** Page cap (defaults to {@link MAX_PDF_PAGES}). */
204
+ maxPages?: number
205
+ }
206
+
207
+ /** Bold-looking PostScript font names (subset prefix already stripped). */
208
+ const BOLD_FONT_RE = /bold|black|heavy|semibold|demibold|demi\b|[-,]?medi(um)?\b|-B$|,B$|-SB$|CMBX|CMB\d|BX\d|\bHei\b|SimHei|Heiti|HTJ-|HT-|黑体/i
209
+ /** Monospace-looking font names. */
210
+ const MONO_FONT_RE = /mono|courier|consol|menlo|inconsolata|NimbusMonL|CMTT|SFTT|LMMono|SourceCodePro|FiraCode|DejaVuSansMono|LucidaConsole|Code/i
211
+
212
+ /**
213
+ * Classify a pdf.js font object by its real (PostScript) name. pdf.js only
214
+ * exposes the name after the page's operator list loaded the font, and only
215
+ * with `fontExtraProperties`; anything unreadable classifies as regular.
216
+ */
217
+ function classifyFont(page: PdfJsPage, loadedName: string | undefined, cache: Map<string, { bold: boolean; mono: boolean }>): { bold: boolean; mono: boolean } {
218
+ if (loadedName === undefined) return { bold: false, mono: false }
219
+ const cached = cache.get(loadedName)
220
+ if (cached !== undefined) return cached
221
+ let name = ''
222
+ let flagsBold = false
223
+ try {
224
+ const font = page.commonObjs.get(loadedName) as { name?: string; bold?: boolean; isMonospace?: boolean; black?: boolean } | undefined
225
+ name = (font?.name ?? '').replace(/^[A-Z]{6}\+/, '')
226
+ flagsBold = font?.bold === true || font?.black === true
227
+ } catch {
228
+ // font not resolved (op list skipped/failed) — regular
229
+ }
230
+ // Monospace by NAME only: CJK fonts routinely set the FixedPitch flag, and
231
+ // a Chinese paragraph is not a code block.
232
+ const result = { bold: flagsBold || BOLD_FONT_RE.test(name), mono: MONO_FONT_RE.test(name) }
233
+ cache.set(loadedName, result)
234
+ return result
235
+ }
236
+
237
+ /** Flatten the outline tree (bounded) into `{ title, depth }` entries. */
238
+ function flattenOutline(nodes: PdfJsOutlineNode[] | null, depth = 0, out: PdfOutlineEntry[] = []): PdfOutlineEntry[] {
239
+ if (nodes === null) return out
240
+ for (const node of nodes) {
241
+ if (out.length >= 2000) break
242
+ const title = (node.title ?? '').replace(/\s+/g, ' ').trim()
243
+ if (title !== '') out.push({ title, depth })
244
+ if (node.items !== undefined && node.items.length > 0 && depth < 8) flattenOutline(node.items, depth + 1, out)
245
+ }
246
+ return out
247
+ }
248
+
249
+ /** Map a pdf.js load failure onto the provider's PDF error codes. */
250
+ function translatePdfError(error: unknown): WebError {
251
+ if (error instanceof WebError) return error
252
+ const name = error instanceof Error ? error.name : ''
253
+ const message = error instanceof Error ? error.message : String(error)
254
+ if (name === 'PasswordException' || /password/i.test(message)) {
255
+ return new WebError('the PDF is password-protected and cannot be read', WEB_PDF_ENCRYPTED_CODE, { cause: error })
256
+ }
257
+ return new WebError(`the PDF could not be parsed: ${message.split('\n')[0]!.slice(0, 200)}`, WEB_PDF_INVALID_CODE, { cause: error })
258
+ }
259
+
260
+ /**
261
+ * Map Kangxi / CJK radical code points to the unified ideographs they look
262
+ * like. Some producers (Chromium printing Noto CJK among them) write a
263
+ * ToUnicode CMap that resolves 文 / 用 / 长 glyphs to U+2F42 ⽂ / U+2F64 ⽤
264
+ * / U+2EA7 ⻓ — visually identical, but they break search and reading.
265
+ * NFKC maps exactly these; it is applied to nothing else (fullwidth forms
266
+ * stay as the document wrote them).
267
+ */
268
+ function normalizeRadicals(text: string): string {
269
+ return text.replace(/[\u2E80-\u2FDF]/g, ch => SIMPLIFIED_RADICALS[ch] ?? ch.normalize('NFKC'))
270
+ }
271
+
272
+ /**
273
+ * The C-simplified radicals (CJK Radicals Supplement) have no NFKC mapping;
274
+ * these are the ones that double as everyday simplified characters.
275
+ */
276
+ const SIMPLIFIED_RADICALS: Record<string, string> = {
277
+ '\u2EC5': '见', '\u2EC6': '角', '\u2EC8': '讠', '\u2EC9': '贝', '\u2ECB': '车', '\u2ED0': '钅',
278
+ '\u2ED3': '长', '\u2ED4': '门', '\u2ED9': '韦', '\u2EDA': '页', '\u2EDB': '风', '\u2EDC': '飞',
279
+ '\u2EE0': '饣', '\u2EE2': '马', '\u2EE5': '鱼', '\u2EE6': '鸟', '\u2EE7': '卤', '\u2EE8': '麦',
280
+ '\u2EE9': '黄', '\u2EEA': '黾', '\u2EEC': '齐', '\u2EEE': '齿', '\u2EF0': '龙',
281
+ }
282
+
283
+ /** A run is horizontal when the text matrix has (almost) no rotation/skew. */
284
+ function isHorizontal(t: number[]): boolean {
285
+ const [a = 0, b = 0, c = 0, d = 0] = t
286
+ return Math.abs(b) <= Math.abs(a) * 0.1 + 1e-6 && Math.abs(c) <= Math.abs(d) * 0.1 + 1e-6 && a > 0 && d > 0
287
+ }
288
+
289
+ /**
290
+ * Read a PDF's positioned text with pdf.js, page by page, inside the given
291
+ * limits. Text items become {@link PdfTextRun}s (horizontal only — rotated
292
+ * stamps and vertical margin text are dropped as layout noise).
293
+ *
294
+ * @returns the text model, plus whether a limit stopped reading early.
295
+ */
296
+ export async function readPdfDocument(bytes: Uint8Array, options: PdfExtractOptions = {}): Promise<{ doc: PdfDocumentText; truncated: boolean }> {
297
+ const startedAt = Date.now()
298
+ const budget = options.timeBudgetMs ?? Number.POSITIVE_INFINITY
299
+ const maxChars = options.maxChars ?? Number.POSITIVE_INFINITY
300
+ const maxPages = options.maxPages ?? MAX_PDF_PAGES
301
+ const { getDocumentProxy } = await import('unpdf')
302
+ // pdf.js refuses Node `Buffer`s (and may detach what it is given): hand it
303
+ // a private plain `Uint8Array` copy.
304
+ const data = new Uint8Array(bytes.byteLength)
305
+ data.set(bytes)
306
+ let pdf: PdfJsDocument
307
+ try {
308
+ pdf = await getDocumentProxy(data, {
309
+ // Real font names (bold / monospace detection) after the operator list.
310
+ fontExtraProperties: true,
311
+ // (pdf.js 6 no longer compiles font programs with `new Function`, so
312
+ // there is no `isEvalSupported` switch left to turn off.)
313
+ verbosity: 0,
314
+ stopAtErrors: false,
315
+ }) as unknown as PdfJsDocument
316
+ } catch (error: unknown) {
317
+ throw translatePdfError(error)
318
+ }
319
+ try {
320
+ let title: string | undefined
321
+ try {
322
+ const meta = await pdf.getMetadata()
323
+ const xmpTitle = meta.metadata?.get('dc:title')
324
+ const infoTitle = meta.info?.Title
325
+ const candidate = typeof xmpTitle === 'string' && xmpTitle.trim() !== '' ? xmpTitle : typeof infoTitle === 'string' ? infoTitle : ''
326
+ if (candidate.trim() !== '') title = candidate.replace(/\s+/g, ' ').trim()
327
+ } catch {
328
+ // metadata is optional
329
+ }
330
+ let outline: PdfOutlineEntry[] = []
331
+ try {
332
+ outline = flattenOutline(await pdf.getOutline())
333
+ } catch {
334
+ // outline is optional
335
+ }
336
+
337
+ const pages: PdfPageText[] = []
338
+ const fontCache = new Map<string, { bold: boolean; mono: boolean }>()
339
+ let chars = 0
340
+ let truncated = false
341
+ const pageCount = pdf.numPages
342
+ for (let n = 1; n <= pageCount; n++) {
343
+ if (options.signal?.aborted) throw options.signal.reason instanceof Error ? options.signal.reason : new Error('pdf extraction aborted')
344
+ if (n > maxPages || chars >= maxChars || Date.now() - startedAt >= budget) {
345
+ truncated = true
346
+ break
347
+ }
348
+ const page = await pdf.getPage(n)
349
+ const content = await page.getTextContent()
350
+ // Fonts resolve only through the operator list; skip it when time is
351
+ // short (the layout then simply sees no bold/monospace on this page).
352
+ if (Date.now() - startedAt < budget * 0.8) {
353
+ try { await page.getOperatorList() } catch { /* fonts stay unknown */ }
354
+ }
355
+ const [, viewBottom = 0, , viewTop = 0] = page.view
356
+ const runs: PdfTextRun[] = []
357
+ for (const item of content.items) {
358
+ if (typeof item.str !== 'string' || item.str === '' || item.transform === undefined) continue
359
+ if (!isHorizontal(item.transform)) continue
360
+ const t = item.transform
361
+ const size = Math.hypot(t[2] ?? 0, t[3] ?? 0) || item.height || 0
362
+ if (!(size > 0)) continue
363
+ const font = classifyFont(page, item.fontName, fontCache)
364
+ runs.push({ text: normalizeRadicals(item.str), x: t[4] ?? 0, y: t[5] ?? 0, width: item.width ?? 0, size, bold: font.bold, mono: font.mono })
365
+ chars += item.str.length
366
+ }
367
+ pages.push({ number: n, bottom: Math.min(viewBottom, viewTop), height: Math.abs(viewTop - viewBottom) || 792, runs })
368
+ page.cleanup()
369
+ }
370
+ return { doc: { ...(title !== undefined ? { title } : {}), outline, pages, totalPages: pageCount }, truncated }
371
+ } catch (error: unknown) {
372
+ if (options.signal?.aborted) throw error
373
+ throw translatePdfError(error)
374
+ } finally {
375
+ // Destroying the loading task tears down the (in-process) worker too.
376
+ void pdf.loadingTask?.destroy().catch(() => {})
377
+ }
378
+ }
379
+
380
+ /**
381
+ * Extract a PDF as markdown: read it with pdf.js inside the limits, lay it
382
+ * out, and fail with {@link WEB_PDF_NO_TEXT_CODE} when there is no text layer
383
+ * to speak of (a scan). Sizes beyond {@link MAX_PDF_BYTES} are refused.
384
+ */
385
+ export async function extractPdfMarkdown(bytes: Uint8Array, options: PdfExtractOptions = {}): Promise<PdfExtraction> {
386
+ if (bytes.length > MAX_PDF_BYTES) {
387
+ throw new WebError(`the PDF is ${String(Math.round(bytes.length / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`, WEB_PDF_TOO_LARGE_CODE)
388
+ }
389
+ const { doc, truncated } = await readPdfDocument(bytes, options)
390
+ const textChars = doc.pages.reduce((sum, page) => sum + page.runs.reduce((s, run) => s + run.text.replace(/\s+/g, '').length, 0), 0)
391
+ // A scan carries no (or a few stray) characters per page; a genuinely
392
+ // short born-digital document ("Dummy PDF file") still clears this.
393
+ if (doc.pages.length > 0 && textChars < Math.max(1, doc.pages.length * 5)) {
394
+ throw new WebError(
395
+ `the PDF has no extractable text layer (${String(doc.totalPages)} page(s), likely scanned images); OCR is not supported`,
396
+ WEB_PDF_NO_TEXT_CODE,
397
+ )
398
+ }
399
+ let markdown = layoutPdfMarkdown(doc)
400
+ if (truncated) {
401
+ markdown += `\n\n> [PDF extraction stopped after page ${String(doc.pages.length)} of ${String(doc.totalPages)} (size/time limit).]\n`
402
+ }
403
+ return { markdown, totalPages: doc.totalPages, pagesRead: doc.pages.length, truncated }
404
+ }
405
+
406
+ // ---------------------------------------------------------------------------
407
+ // Layout heuristics (pure)
408
+ // ---------------------------------------------------------------------------
409
+
410
+ /** CJK ideographs, kana, hangul, and fullwidth/CJK punctuation. */
411
+ const CJK_RE = /[⺀-⿿ -〿぀-ヿ㄀-ㇿ㐀-䶿一-鿿가-힯豈-﫿＀-￯]/
412
+
413
+ function isCjk(ch: string | undefined): boolean {
414
+ return ch !== undefined && CJK_RE.test(ch)
415
+ }
416
+
417
+ /** Letter-spaced CJK ("预 防 和 治 理") reads as one word. */
418
+ function collapseCjkSpacing(text: string): string {
419
+ return text.replace(new RegExp(`(?<=${CJK_RE.source}) +(?=${CJK_RE.source})`, 'g'), '')
420
+ }
421
+
422
+ /** A styled fragment of a line (for inline bold). */
423
+ interface Segment {
424
+ text: string
425
+ bold: boolean
426
+ }
427
+
428
+ /** One visual line. */
429
+ interface Line {
430
+ page: number
431
+ text: string
432
+ segments: Segment[]
433
+ x0: number
434
+ x1: number
435
+ y: number
436
+ size: number
437
+ bold: boolean
438
+ mono: boolean
439
+ /** Text cells separated by wide horizontal gaps (table-ish rows). */
440
+ cells: string[]
441
+ /** Left edge of each cell in {@link Line.cells}. */
442
+ cellX: number[]
443
+ /** Non-whitespace character count. */
444
+ chars: number
445
+ /** Position relative to the page box (0 = bottom, 1 = top). */
446
+ relY: number
447
+ }
448
+
449
+ /** Group a page's runs into lines (content-stream order, baseline match). */
450
+ function buildLines(page: PdfPageText): Line[] {
451
+ const groups: PdfTextRun[][] = []
452
+ let current: PdfTextRun[] = []
453
+ let baseY = 0
454
+ let baseSize = 0
455
+ for (const run of page.runs) {
456
+ if (current.length > 0) {
457
+ const tolerance = Math.max(0.6 * Math.max(run.size, baseSize), 1)
458
+ if (Math.abs(run.y - baseY) <= tolerance) {
459
+ current.push(run)
460
+ if (run.text.trim() !== '' && run.size > baseSize) { baseSize = run.size; baseY = run.y }
461
+ continue
462
+ }
463
+ groups.push(current)
464
+ }
465
+ current = [run]
466
+ baseY = run.y
467
+ baseSize = run.size
468
+ }
469
+ if (current.length > 0) groups.push(current)
470
+
471
+ const lines: Line[] = []
472
+ for (const group of groups) {
473
+ const line = composeLine(group, page)
474
+ if (line !== undefined) lines.push(line)
475
+ }
476
+ return lines
477
+ }
478
+
479
+ /** Join one baseline group's runs (x-sorted) into a {@link Line}. */
480
+ function composeLine(group: PdfTextRun[], page: PdfPageText): Line | undefined {
481
+ const runs = [...group].sort((a, b) => a.x - b.x)
482
+ const visible = runs.filter(run => run.text.trim() !== '')
483
+ if (visible.length === 0) return undefined
484
+ // Dominant size and baseline: the size carrying the most characters.
485
+ const weight = new Map<number, number>()
486
+ for (const run of visible) {
487
+ const key = Math.round(run.size * 10) / 10
488
+ weight.set(key, (weight.get(key) ?? 0) + run.text.trim().length)
489
+ }
490
+ let size = visible[0]!.size
491
+ let best = -1
492
+ for (const [key, w] of weight) if (w > best) { best = w; size = key }
493
+ const baseline = visible.find(run => Math.abs(run.size - size) < 0.11)?.y ?? visible[0]!.y
494
+
495
+ const segments: Segment[] = []
496
+ const cells: string[] = ['']
497
+ const cellX: number[] = []
498
+ let text = ''
499
+ let prevEnd: number | undefined
500
+ let boldChars = 0
501
+ let monoChars = 0
502
+ let chars = 0
503
+ for (const run of runs) {
504
+ const content = run.text
505
+ const trimmed = content.trim()
506
+ if (trimmed === '') {
507
+ // Whitespace runs only matter as separators; gaps are measured between
508
+ // visible runs.
509
+ continue
510
+ }
511
+ let sep = ''
512
+ if (prevEnd !== undefined && text !== '') {
513
+ const gap = run.x - prevEnd
514
+ const last = text[text.length - 1]
515
+ const first = trimmed[0]
516
+ if (gap > run.size * 1.5 && !(isCjk(last) && isCjk(first) && gap < run.size * 3)) {
517
+ cells.push('')
518
+ cellX.push(run.x)
519
+ sep = ' '
520
+ } else if (gap > run.size * 0.15 || /\s$/.test(content.slice(0, content.length - content.trimStart().length)) || /^\s/.test(content)) {
521
+ sep = isCjk(last) && isCjk(first) ? '' : ' '
522
+ }
523
+ if (/\s$/.test(text)) sep = ''
524
+ }
525
+ const piece = sep + trimmed
526
+ if (cellX.length === 0) cellX.push(run.x)
527
+ text += piece
528
+ cells[cells.length - 1] += (cells[cells.length - 1] === '' ? trimmed : piece)
529
+ const prev = segments[segments.length - 1]
530
+ if (prev !== undefined && prev.bold === run.bold) prev.text += piece
531
+ else segments.push({ text: piece, bold: run.bold })
532
+ prevEnd = Math.max(prevEnd ?? -Infinity, run.x + run.width)
533
+ const n = trimmed.replace(/\s+/g, '').length
534
+ chars += n
535
+ if (run.bold) boldChars += n
536
+ if (run.mono) monoChars += n
537
+ }
538
+ text = collapseCjkSpacing(text.replace(/\s+/g, ' ').trim())
539
+ if (text === '') return undefined
540
+ for (const segment of segments) segment.text = collapseCjkSpacing(segment.text)
541
+ const x0 = visible.reduce((m, run) => Math.min(m, run.x), Infinity)
542
+ const x1 = visible.reduce((m, run) => Math.max(m, run.x + run.width), -Infinity)
543
+ return {
544
+ page: page.number,
545
+ text,
546
+ segments,
547
+ x0,
548
+ x1,
549
+ y: baseline,
550
+ size,
551
+ bold: chars > 0 && boldChars / chars >= 0.8,
552
+ mono: chars > 0 && monoChars / chars >= 0.8,
553
+ cells: cells.map(cell => collapseCjkSpacing(cell.replace(/\s+/g, ' ').trim())),
554
+ cellX,
555
+ chars,
556
+ relY: page.height > 0 ? (baseline - page.bottom) / page.height : 0.5,
557
+ }
558
+ }
559
+
560
+ /** Bare page-number shapes found in page margins. */
561
+ const PAGE_NUMBER_RES = [
562
+ /^[-–—\s]*(?:page|p\.|pág\.|seite)?\s*\d{1,4}(?:\s*(?:\/|of|von|de)\s*\d{1,4})?[-–—\s]*$/i,
563
+ /^[-–—\s]*[ivxlcdm]{1,7}[-–—\s]*$/i,
564
+ /^第\s*\d{1,4}\s*页(?:\s*[,,//]?\s*共\s*\d{1,4}\s*页)?$/,
565
+ // "— 12 —" (Chinese official documents), including the OCR misreading of
566
+ // the dashes as 一 and fullwidth dash variants.
567
+ /^[-–—一-―·•]{1,2}\s*\d{1,4}\s*[-–—一-―·•]{1,2}$/,
568
+ ]
569
+
570
+ const isMarginLine = (line: Line): boolean => line.relY > 0.9 || line.relY < 0.12
571
+
572
+ /** Drop running headers/footers and page numbers. */
573
+ function removeMarginNoise(pages: Line[][]): Line[][] {
574
+ const keyOf = (line: Line): string => line.text.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ').trim()
575
+ const pagesByKey = new Map<string, Set<number>>()
576
+ for (const lines of pages) {
577
+ for (const line of lines) {
578
+ if (!isMarginLine(line)) continue
579
+ const key = keyOf(line)
580
+ let set = pagesByKey.get(key)
581
+ if (set === undefined) { set = new Set(); pagesByKey.set(key, set) }
582
+ set.add(line.page)
583
+ }
584
+ }
585
+ const repeatThreshold = Math.max(3, Math.ceil(pages.length * 0.3))
586
+ return pages.map(lines => lines.filter((line) => {
587
+ if (!isMarginLine(line)) return true
588
+ const plain = line.text.normalize('NFKC')
589
+ if (PAGE_NUMBER_RES.some(re => re.test(plain))) return false
590
+ if (pages.length >= 3 && (pagesByKey.get(keyOf(line))?.size ?? 0) >= repeatThreshold) return false
591
+ return true
592
+ }))
593
+ }
594
+
595
+ /** Character-weighted mode of the line font sizes (0.5pt buckets). */
596
+ function bodySizeOf(lines: Line[]): number {
597
+ const weight = new Map<number, number>()
598
+ for (const line of lines) {
599
+ const key = Math.round(line.size * 2) / 2
600
+ weight.set(key, (weight.get(key) ?? 0) + line.chars)
601
+ }
602
+ let body = 10
603
+ let best = -1
604
+ for (const [key, w] of weight) if (w > best) { best = w; body = key }
605
+ return body
606
+ }
607
+
608
+ /** Median baseline-to-baseline distance between consecutive body lines. */
609
+ function lineSpacingOf(pages: Line[][], body: number): number {
610
+ const deltas: number[] = []
611
+ for (const lines of pages) {
612
+ for (let i = 1; i < lines.length; i++) {
613
+ const a = lines[i - 1]!
614
+ const b = lines[i]!
615
+ if (Math.abs(a.size - body) > body * 0.1 || Math.abs(b.size - body) > body * 0.1) continue
616
+ const dy = a.y - b.y
617
+ if (dy > body * 0.6 && dy < body * 3) deltas.push(dy)
618
+ }
619
+ }
620
+ if (deltas.length === 0) return body * 1.2
621
+ deltas.sort((p, q) => p - q)
622
+ return deltas[Math.floor(deltas.length / 2)]!
623
+ }
624
+
625
+ /** Normalization for comparing headings to outline titles. */
626
+ function headingKey(text: string): string {
627
+ return text.normalize('NFKC').toLowerCase().replace(/[\s\p{P}\p{S}]/gu, '')
628
+ }
629
+
630
+ /** Leading section number ("3", "3.1", "A.2") stripped for outline matching. */
631
+ function stripNumbering(text: string): string {
632
+ return text.replace(/^(?:\d+(?:\.\d+)*|[A-Z](?:\.\d+)+)\.?\s+/, '').replace(/^(?:第[一二三四五六七八九十百千零〇两\d]+[章节篇部]|[一二三四五六七八九十]+[、..]|[((][一二三四五六七八九十]+[))])\s*/, '')
633
+ }
634
+
635
+ const NUMBERED_HEADING_RE = /^(\d{1,2}(?:\.\d{1,2}){0,4}|[A-Z](?:\.\d{1,2}){1,4})\.?\s+(?=\S)/
636
+ const CN_CHAPTER_RE = /^第[一二三四五六七八九十百千零〇两\d]+[章篇部]/
637
+ const CN_SECTION_RE = /^第[一二三四五六七八九十百千零〇两\d]+节/
638
+ const CN_ENUM_RE = /^[一二三四五六七八九十]{1,3}[、.]/
639
+ const CN_PAREN_ENUM_RE = /^[((][一二三四五六七八九十]{1,3}[))]/
640
+ /** A table-of-contents entry: leader dots (or a wide gap) then a page reference. */
641
+ const TOC_ENTRY_RE = /(?:[.…·.]\s*){4,}[^.…·.]{0,16}?[((]?\s*[\d0-9ivxlc]+\s*[))]?$/i
642
+ const EXPLICIT_HEADING_RES = [NUMBERED_HEADING_RE, CN_CHAPTER_RE, CN_SECTION_RE, CN_ENUM_RE, CN_PAREN_ENUM_RE]
643
+ const TERMINAL_RE = /[.!?。!?…::;;"”’))\]】]$/
644
+ const SENTENCE_END_RE = /[.!?。!?…"”’]$/
645
+
646
+ const BULLET_RE = /^(?:[•◦▪▫●○■□►▸‣⁃∙·*❖➢✓✔]|[])\s*/
647
+ const DASH_BULLET_RE = /^[-–—]\s+(?=\S)/
648
+ const ORDERED_RE = /^(\d{1,3})[.)]\s+(?=\S)/
649
+ const PAREN_ITEM_RE = /^(?:[((](?:\d{1,3}|[a-zA-Z]|[ivx]{1,4})[))]|[a-z][.)]|[①-⑳])\s*(?=\S)/
650
+
651
+ /** Markdown kinds a line can open. */
652
+ type BlockKind = 'heading' | 'bullet' | 'ordered' | 'item' | 'code' | 'text' | 'table'
653
+
654
+ interface Block {
655
+ kind: BlockKind
656
+ page: number
657
+ level?: number
658
+ lines: Line[]
659
+ }
660
+
661
+ interface LayoutStats {
662
+ body: number
663
+ spacing: number
664
+ monoIsBody: boolean
665
+ /** Heading font sizes, largest first (each rank is one heading level). */
666
+ headingSizes: number[]
667
+ /** Whether a dedicated document-title size exists (rank 0 = h1). */
668
+ titleSize?: number
669
+ outlineKeys: Map<string, number>
670
+ /** 1 when the outline's single root is the document title (children → h2). */
671
+ outlineOffset: number
672
+ }
673
+
674
+ /** Heading level placeholder: one below the most recent heading. */
675
+ const RELATIVE_HEADING = -1
676
+
677
+ /** Decide a line's heading level, if it is one. */
678
+ function headingLevel(line: Line, groupLines: Line[], stats: LayoutStats, standalone: boolean, styleRules: boolean): number | undefined {
679
+ const text = groupLines.map(l => l.text).join(' ')
680
+ const chars = groupLines.reduce((s, l) => s + l.chars, 0)
681
+ if (chars === 0 || chars > 200 || !/\p{L}/u.test(text)) return undefined
682
+ const words = text.split(/\s+/).length
683
+ const sentenceLike = /[,,;;]$/.test(text) || (/[.。]$/.test(text) && !/^\d+(\.\d+)*\.?$/.test(text.split(' ')[0] ?? ''))
684
+ // 1. The outline (bookmarks) — the author's own structure.
685
+ if (stats.titleSize !== undefined && Math.abs(line.size - stats.titleSize) < 0.3 && groupLines.length <= 3 && styleRules) return 1
686
+ const outlineDepth = stats.outlineKeys.get(headingKey(text)) ?? stats.outlineKeys.get(headingKey(stripNumbering(text)))
687
+ if (outlineDepth !== undefined && chars <= 160) return Math.min(6, Math.max(2, outlineDepth + 2 - stats.outlineOffset))
688
+
689
+ const bigger = line.size >= stats.body * 1.12
690
+ const emphasized = bigger || line.bold
691
+ // Table rows (wide column gaps) are never style-only headings.
692
+ const styleOk = styleRules && groupLines.every(l => l.cells.length < 2)
693
+ // 2. Section numbering.
694
+ const numbered = NUMBERED_HEADING_RE.exec(text)
695
+ if (numbered !== null && emphasized && !sentenceLike && words <= 16 && chars <= 120) {
696
+ const depth = numbered[1]!.split('.').length
697
+ return Math.min(6, depth + 1)
698
+ }
699
+ if (CN_CHAPTER_RE.test(text) && chars <= 60 && !sentenceLike) return 2
700
+ if (CN_SECTION_RE.test(text) && chars <= 60 && !sentenceLike) return 3
701
+ if (CN_ENUM_RE.test(text) && chars <= 40 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 2
702
+ if (CN_PAREN_ENUM_RE.test(text) && chars <= 30 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 3
703
+ if (!styleOk) return undefined
704
+ // 3. Larger type, ranked.
705
+ if (bigger && groupLines.length <= 3 && !sentenceLike) {
706
+ if (stats.titleSize !== undefined && Math.abs(line.size - stats.titleSize) < 0.3) return 1
707
+ const rank = stats.headingSizes.findIndex(size => Math.abs(size - Math.round(line.size * 2) / 2) < 0.3)
708
+ return Math.min(4, 2 + Math.max(0, rank))
709
+ }
710
+ // 4. Standalone bold line at body size: a child of the current section
711
+ // (resolved by the caller). A run-in heading — bold lead, regular text
712
+ // after it on the same line — is a paragraph, not a heading.
713
+ const lastSegment = groupLines[groupLines.length - 1]!.segments.at(-1)
714
+ if (line.bold && lastSegment?.bold === true && standalone && groupLines.length <= 2 && chars <= 100 && words <= 14 && !sentenceLike && !TERMINAL_RE.test(text.replace(/[))]$/, ''))) {
715
+ return RELATIVE_HEADING
716
+ }
717
+ return undefined
718
+ }
719
+
720
+ /** Group lines into markdown blocks. */
721
+ function buildBlocks(pages: Line[][], stats: LayoutStats): Block[] {
722
+ const all = pages.flat()
723
+ const blocks: Block[] = []
724
+ // Column right edge for a line: the widest line on the page starting near it.
725
+ const rightEdge = new Map<Line, number>()
726
+ for (const lines of pages) {
727
+ for (const line of lines) {
728
+ let right = line.x1
729
+ for (const other of lines) {
730
+ if (Math.abs(other.x0 - line.x0) < line.size * 3 && Math.abs(other.size - line.size) < line.size * 0.15) right = Math.max(right, other.x1)
731
+ }
732
+ rightEdge.set(line, right)
733
+ }
734
+ }
735
+ const pitch = (line: Line): number => stats.spacing * (line.size / stats.body)
736
+ const gapBefore = (i: number): number => {
737
+ const prev = all[i - 1]
738
+ const line = all[i]!
739
+ if (prev === undefined || prev.page !== line.page) return Infinity
740
+ return prev.y - line.y
741
+ }
742
+ const isStandalone = (i: number, span: number): boolean => {
743
+ const before = gapBefore(i)
744
+ const after = i + span < all.length ? gapBefore(i + span) : Infinity
745
+ const first = all[i]!
746
+ const sepBefore = before === Infinity || before < 0 || before > pitch(first) * 1.3 || all[i - 1]!.size !== first.size || all[i - 1]!.bold !== first.bold
747
+ const next = all[i + span]
748
+ const sepAfter = after === Infinity || after < 0 || after > pitch(first) * 1.3 || next === undefined || Math.abs(next.size - first.size) > 0.3
749
+ const short = all[i + span - 1]!.x1 < (rightEdge.get(all[i + span - 1]!) ?? Infinity) - first.size * 2
750
+ return (sepBefore && (sepAfter || short))
751
+ }
752
+
753
+ // A lone monospace line wedged into running text (a URL or identifier
754
+ // the paragraph wrapped onto its own line) stays in the paragraph.
755
+ const isInlineMono = (index: number): boolean => {
756
+ const line = all[index]!
757
+ const prev = all[index - 1]
758
+ const next = all[index + 1]
759
+ if (prev === undefined || prev.mono || next?.mono === true && next.page === line.page && gapBefore(index + 1) > 0 && gapBefore(index + 1) < pitch(line) * 2.2) return false
760
+ const dy = gapBefore(index)
761
+ return dy > 0 && dy < pitch(line) * 1.45 && !TERMINAL_RE.test(prev.text) && line.chars <= 80
762
+ }
763
+ // Tables: runs of consecutive multi-cell lines whose columns line up.
764
+ // A justified line with wide word gaps has "cells" too, but no other row
765
+ // shares its column starts.
766
+ const tableEnd = new Map<number, number>()
767
+ const inTable = new Set<number>()
768
+ for (let start = 0; start < all.length;) {
769
+ if (all[start]!.cells.length < 2 || all[start]!.mono) { start++; continue }
770
+ let end = start + 1
771
+ while (end < all.length && all[end]!.cells.length >= 2 && !all[end]!.mono && all[end]!.page === all[start]!.page
772
+ && gapBefore(end) > 0 && gapBefore(end) < pitch(all[end]!) * 2.5) end++
773
+ // Rows at either end that share no columns with the rest are not part
774
+ // of the table (a justified line just above or below it).
775
+ let first = start
776
+ let last = end
777
+ while (last - first >= 2 && !rowAligned(all[last - 1]!, all.slice(first, last - 1))) last--
778
+ while (last - first >= 2 && !rowAligned(all[first]!, all.slice(first + 1, last))) first++
779
+ if (last - first >= 2 && columnsAlign(all.slice(first, last))) {
780
+ tableEnd.set(first, last)
781
+ for (let k = first; k < last; k++) inTable.add(k)
782
+ }
783
+ start = end
784
+ }
785
+ let lastLevel = 1
786
+ let i = 0
787
+ while (i < all.length) {
788
+ const line = all[i]!
789
+ // Candidate heading group: consecutive lines of the same size/weight.
790
+ let span = 1
791
+ while (
792
+ span < 4 && i + span < all.length
793
+ && all[i + span]!.page === line.page
794
+ && Math.abs(all[i + span]!.size - line.size) < 0.3
795
+ && all[i + span]!.bold === line.bold
796
+ && gapBefore(i + span) > 0 && gapBefore(i + span) < pitch(line) * 1.6
797
+ && !EXPLICIT_HEADING_RES.some(re => re.test(all[i + span]!.text))
798
+ ) span++
799
+ let level: number | undefined
800
+ let used = 0
801
+ // Style rules (size / bold) judge the WHOLE same-style run — a slice of
802
+ // a longer run is just its first lines (a 3-line notice in larger type
803
+ // is not a heading plus a paragraph). Explicit signals (outline,
804
+ // numbering) may still claim the first line alone.
805
+ if (span <= 3) {
806
+ level = headingLevel(line, all.slice(i, i + span), stats, isStandalone(i, span), true)
807
+ if (level !== undefined) used = span
808
+ }
809
+ if (level === undefined && span > 1) {
810
+ level = headingLevel(line, [line], stats, isStandalone(i, 1), false)
811
+ if (level !== undefined) used = 1
812
+ }
813
+ if (level !== undefined && line.mono && !stats.monoIsBody) level = undefined
814
+ if (level === RELATIVE_HEADING) level = Math.min(6, Math.max(2, lastLevel + 1))
815
+ else if (level !== undefined) lastLevel = level
816
+ if (level !== undefined) {
817
+ blocks.push({ kind: 'heading', page: line.page, level, lines: all.slice(i, i + used) })
818
+ i += used
819
+ continue
820
+ }
821
+
822
+ const start = i
823
+ const kind = tableEnd.has(i) ? 'table' : lineKind(line, stats)
824
+ const block: Block = { kind: kind === 'code' && isInlineMono(i) ? 'text' : kind, page: line.page, lines: [line] }
825
+ i++
826
+ if (kind === 'table') {
827
+ const end = tableEnd.get(start) ?? start + 1
828
+ while (i < end) {
829
+ block.lines.push(all[i]!)
830
+ i++
831
+ }
832
+ blocks.push(block)
833
+ continue
834
+ }
835
+ if (kind === 'code') {
836
+ while (i < all.length && all[i]!.mono && gapBefore(i) > 0 && gapBefore(i) < pitch(all[i]!) * 2.2) {
837
+ block.lines.push(all[i]!)
838
+ i++
839
+ }
840
+ blocks.push(block)
841
+ continue
842
+ }
843
+ let blockRight = rightEdge.get(line) ?? line.x1
844
+ while (i < all.length) {
845
+ const next = all[i]!
846
+ const prev = block.lines[block.lines.length - 1]!
847
+ if (inTable.has(i)) break
848
+ const nextKind = lineKind(next, stats)
849
+ if (nextKind !== 'text' && !(nextKind === 'code' && isInlineMono(i))) break
850
+ if (Math.abs(next.size - prev.size) > Math.max(next.size, prev.size) * 0.12) break
851
+ const dy = next.page === prev.page ? prev.y - next.y : -Infinity
852
+ const sameFlow = dy > 0 && dy < pitch(next) * 1.45
853
+ if (!sameFlow) {
854
+ // A page or column change: continue only an unfinished sentence.
855
+ if (dy > 0) break // a vertical gap inside the column
856
+ if (TERMINAL_RE.test(prev.text)) break
857
+ if (headingStartLike(next)) break
858
+ if (next.page !== prev.page && block.kind !== 'text') break
859
+ } else {
860
+ // A table-of-contents entry is a line of its own.
861
+ if (TOC_ENTRY_RE.test(prev.text)) break
862
+ // First-line indent opens a paragraph: a full-measure line starting
863
+ // further right after a line that finished (centered lines are inset
864
+ // on both sides; a hanging-indent continuation follows an unfinished
865
+ // line).
866
+ if (next.x0 - prev.x0 > next.size * 0.8 && block.kind === 'text' && next.x1 >= blockRight - next.size * 2
867
+ && (TERMINAL_RE.test(prev.text) || prev.x1 < blockRight - prev.size * 2)) break
868
+ // List items continue only on their hanging indent.
869
+ if (block.kind !== 'text' && next.x0 < block.lines[0]!.x0 + next.size * 0.3 && TERMINAL_RE.test(prev.text)) break
870
+ // A short line that ends a sentence closes its paragraph.
871
+ if (prev.x1 < blockRight - prev.size * 2.5 && SENTENCE_END_RE.test(prev.text)) break
872
+ if (next.bold !== prev.bold && next.bold && next.chars < 100 && SENTENCE_END_RE.test(prev.text)) break
873
+ }
874
+ block.lines.push(next)
875
+ blockRight = Math.max(blockRight, rightEdge.get(next) ?? next.x1)
876
+ i++
877
+ }
878
+ blocks.push(block)
879
+ }
880
+ return blocks
881
+ }
882
+
883
+ /**
884
+ * Whether table-candidate rows share column starts: at least 60% of the rows
885
+ * have two or more cell left edges within one em of another row's.
886
+ */
887
+ function columnsAlign(rows: Line[]): boolean {
888
+ // Prose with a right-hand field label (forms: "Enter the amount . . . 3b $")
889
+ // has two "columns", the first of them sentences — not a table.
890
+ const firstCells = rows.map(row => row.cells.find(cell => cell !== '') ?? '').map(cell => cell.length).sort((a, b) => a - b)
891
+ const maxCells = Math.max(...rows.map(row => row.cells.filter(cell => cell !== '').length))
892
+ if (maxCells <= 2 && (firstCells[Math.floor(firstCells.length / 2)] ?? 0) > 50) return false
893
+ const aligned = rows.filter(row => rowAligned(row, rows.filter(other => other !== row))).length
894
+ return aligned / rows.length >= 0.6
895
+ }
896
+
897
+ /** Whether a row shares at least two column starts with some other row. */
898
+ function rowAligned(row: Line, others: Line[]): boolean {
899
+ return others.some(other => row.cellX.filter(x => other.cellX.some(o => Math.abs(o - x) < row.size)).length >= 2)
900
+ }
901
+
902
+ /** A line that looks like the start of a new structural element. */
903
+ function headingStartLike(line: Line): boolean {
904
+ return BULLET_RE.test(line.text) || ORDERED_RE.test(line.text) || CN_CHAPTER_RE.test(line.text)
905
+ }
906
+
907
+ /** Classify a non-heading line. */
908
+ function lineKind(line: Line, stats: LayoutStats): BlockKind {
909
+ if (line.mono && !stats.monoIsBody) return 'code'
910
+ if (BULLET_RE.test(line.text) && line.text.replace(BULLET_RE, '').trim() !== '') return 'bullet'
911
+ if (DASH_BULLET_RE.test(line.text)) return 'bullet'
912
+ if (ORDERED_RE.test(line.text)) return 'ordered'
913
+ if (PAREN_ITEM_RE.test(line.text)) return 'item'
914
+ return 'text'
915
+ }
916
+
917
+ /** Words seen in the document (lowercased), for de-hyphenation decisions. */
918
+ function vocabularyOf(lines: Line[]): Set<string> {
919
+ const words = new Set<string>()
920
+ for (const line of lines) {
921
+ for (const match of line.text.matchAll(/[A-Za-z]+(?:-[A-Za-z]+)*/g)) {
922
+ const word = match[0].toLowerCase()
923
+ words.add(word)
924
+ if (word.includes('-')) words.add(word.replace(/-/g, '\u0000'))
925
+ }
926
+ }
927
+ return words
928
+ }
929
+
930
+ /** Join two consecutive lines of one paragraph. */
931
+ function joinText(acc: string, next: string, vocabulary: Set<string>): string {
932
+ if (acc === '') return next
933
+ const hyphen = /([A-Za-z]+)[-\u2010\u00AD]$/.exec(acc)
934
+ const lead = /^([a-z]+)/.exec(next)
935
+ if (hyphen !== null && lead !== null && !/\s[-\u2010\u00AD]$/.test(acc)) {
936
+ const head = hyphen[1]!.toLowerCase()
937
+ const tail = lead[1]!
938
+ if (vocabulary.has(head + tail)) return acc.slice(0, -1) + next
939
+ if (vocabulary.has(`${head}\u0000${tail}`)) return `${acc.slice(0, -1)}-${next}`
940
+ return acc.slice(0, -1) + next
941
+ }
942
+ const last = acc[acc.length - 1]
943
+ const first = next[0]
944
+ if (isCjk(last) || isCjk(first)) {
945
+ // Latin word followed by CJK across a break: no space either, the
946
+ // original line break carried no space.
947
+ return acc + next
948
+ }
949
+ if (/[-/]$/.test(acc)) return acc + next
950
+ return `${acc} ${next}`
951
+ }
952
+
953
+ /** Escape what would otherwise read as markdown structure at a block start. */
954
+ function escapeLeading(text: string): string {
955
+ return text.replace(/^(#{1,6}\s|>|[-+*]\s|\d{1,9}[.)]\s)/, '\\$1')
956
+ }
957
+
958
+ /** Render a line with inline bold (when only part of it is bold). */
959
+ function renderLine(line: Line, inlineBold: boolean): string {
960
+ if (!inlineBold || !line.segments.some(s => s.bold) || line.segments.every(s => s.bold)) return line.text
961
+ let out = ''
962
+ for (const segment of line.segments) {
963
+ const text = segment.text
964
+ const core = text.trim()
965
+ if (segment.bold && core.replace(/[^\p{L}\p{N}]/gu, '').length >= 2 && !/^\*|\*$/.test(core)) {
966
+ const lead = text.slice(0, text.length - text.trimStart().length)
967
+ const trail = text.slice(text.trimEnd().length)
968
+ out += `${lead}**${core}**${trail}`
969
+ } else {
970
+ out += text
971
+ }
972
+ }
973
+ return out.replace(/\s+/g, ' ').trim()
974
+ }
975
+
976
+ /**
977
+ * Merge bold spans split only by whitespace or punctuation set in the
978
+ * regular face ("**一是…会议**,**推动会议** **精神**" → one span): the
979
+ * separator moves inside, and a line-break space between CJK spans goes.
980
+ */
981
+ function mergeBoldRuns(text: string): string {
982
+ return text.replace(/\*\*(\s*)([,、,;;::]?)(\s*)\*\*/g, (match: string, a: string, punct: string, b: string, offset: number, whole: string) => {
983
+ const before = whole[offset - 1]
984
+ const after = whole[offset + match.length]
985
+ const space = (a !== '' || b !== '') && !(isCjk(before) || isCjk(after) || isCjk(punct)) ? ' ' : ''
986
+ return punct === '' ? space : `${punct}${b !== '' && !isCjk(punct) ? ' ' : ''}`
987
+ })
988
+ }
989
+
990
+ /** Render a table block as GFM when its rows line up, else as plain rows. */
991
+ function renderTable(block: Block): string {
992
+ const rows = block.lines.map(line => line.cells.filter(cell => cell !== '').map(cell => cell.replace(/\|/g, '\\|')))
993
+ const width = Math.max(...rows.map(row => row.length))
994
+ const aligned = rows.filter(row => row.length === width).length
995
+ if (rows.length >= 2 && width >= 2 && aligned / rows.length >= 0.6) {
996
+ const pad = (row: string[]): string => `| ${[...row, ...Array<string>(width - row.length).fill('')].slice(0, width).join(' | ')} |`
997
+ return [pad(rows[0]!), `| ${Array<string>(width).fill('---').join(' | ')} |`, ...rows.slice(1).map(pad)].join('\n')
998
+ }
999
+ return block.lines.map(line => escapeLeading(line.cells.join(' '))).join(' \n')
1000
+ }
1001
+
1002
+ /** Whether a metadata title is worth printing as the document heading. */
1003
+ function plausibleTitle(title: string | undefined): title is string {
1004
+ if (title === undefined) return false
1005
+ const t = title.trim()
1006
+ if (t.length < 3 || t.length > 250) return false
1007
+ if (/^(?:microsoft\s+(?:word|powerpoint|excel)\b|untitled|slide\s*\d|document\d*$|title$|no title|无标题)/i.test(t)) return false
1008
+ if (/\.(?:docx?|pdf|pptx?|xlsx?|tex|dvi|indd|qxd|rtf|odt)$/i.test(t)) return false
1009
+ if (/^[\w-]+$/.test(t) && !/\s/.test(t) && /\d/.test(t)) return false // file-name-like ids
1010
+ return true
1011
+ }
1012
+
1013
+ /**
1014
+ * Lay a PDF's positioned text out as markdown (see the module docs for the
1015
+ * heuristics). Pure — no pdf.js — so the rules are unit-testable with
1016
+ * synthetic runs.
1017
+ */
1018
+ export function layoutPdfMarkdown(doc: PdfDocumentText): string {
1019
+ const pages = removeMarginNoise(doc.pages.map(buildLines))
1020
+ const all = pages.flat()
1021
+ if (all.length === 0) return plausibleTitle(doc.title) ? `# ${doc.title}\n` : ''
1022
+ const body = bodySizeOf(all)
1023
+ const totalChars = all.reduce((s, l) => s + l.chars, 0)
1024
+ const monoChars = all.reduce((s, l) => s + (l.mono ? l.chars : 0), 0)
1025
+ const monoIsBody = monoChars > totalChars * 0.5
1026
+
1027
+ // Heading sizes: larger-than-body sizes that only occur on short line runs.
1028
+ const sizeChars = new Map<number, number>()
1029
+ for (const line of all) {
1030
+ if (line.size < body * 1.12) continue
1031
+ const key = Math.round(line.size * 2) / 2
1032
+ sizeChars.set(key, (sizeChars.get(key) ?? 0) + line.chars)
1033
+ }
1034
+ let headingSizes = [...sizeChars.keys()].filter(size => (sizeChars.get(size) ?? 0) < Math.max(totalChars * 0.25, 400)).sort((a, b) => b - a)
1035
+ // The document title: the largest type on the first page, when it is
1036
+ // clearly display-sized and used for at most a few lines there.
1037
+ let titleSize: number | undefined
1038
+ const firstPage = doc.pages[0]?.number ?? 1
1039
+ const firstPageLines = all.filter(line => line.page === firstPage && /\p{L}{2}/u.test(line.text))
1040
+ const firstLargest = firstPageLines.reduce((m, line) => Math.max(m, Math.round(line.size * 2) / 2), 0)
1041
+ if (firstLargest >= body * 1.3) {
1042
+ const uses = firstPageLines.filter(line => Math.abs(Math.round(line.size * 2) / 2 - firstLargest) < 0.3)
1043
+ if (uses.length <= 3) {
1044
+ titleSize = firstLargest
1045
+ headingSizes = headingSizes.filter(size => size < firstLargest - 0.25)
1046
+ }
1047
+ }
1048
+ headingSizes = headingSizes.slice(0, 3)
1049
+ const outlineKeys = new Map<string, number>()
1050
+ for (const entry of doc.outline) {
1051
+ for (const key of [headingKey(entry.title), headingKey(stripNumbering(entry.title))]) {
1052
+ if (key.length >= 2 && !outlineKeys.has(key)) outlineKeys.set(key, entry.depth)
1053
+ }
1054
+ }
1055
+ const roots = doc.outline.filter(entry => entry.depth === 0)
1056
+ const titleKeys = new Set([doc.title ?? '', ...all.filter(line => titleSize !== undefined && Math.abs(line.size - titleSize) < 0.3).map(line => line.text)].map(headingKey))
1057
+ const outlineOffset = roots.length === 1 && titleKeys.has(headingKey(roots[0]!.title)) ? 1 : 0
1058
+ const stats: LayoutStats = { body, spacing: lineSpacingOf(pages, body), monoIsBody, headingSizes, ...(titleSize !== undefined ? { titleSize } : {}), outlineKeys, outlineOffset }
1059
+ const blocks = buildBlocks(pages, stats)
1060
+ const vocabulary = vocabularyOf(all)
1061
+
1062
+ const out: string[] = []
1063
+ const metaKey = headingKey(doc.title ?? '')
1064
+ const hasTitleHeading = blocks.slice(0, 12).some(block => block.kind === 'heading' && (block.level === 1 || headingKey(block.lines.map(l => l.text).join('')) === metaKey))
1065
+ if (!hasTitleHeading && plausibleTitle(doc.title)) out.push(`# ${doc.title.replace(/\s+/g, ' ').trim()}`)
1066
+ let lastPage = 0
1067
+ for (const block of blocks) {
1068
+ if (block.page !== lastPage) {
1069
+ out.push(`<!-- page ${String(block.page)} -->`)
1070
+ lastPage = block.page
1071
+ }
1072
+ switch (block.kind) {
1073
+ case 'heading': {
1074
+ const text = block.lines.map(l => l.text).reduce((acc, t) => joinText(acc, t, vocabulary), '')
1075
+ out.push(`${'#'.repeat(block.level ?? 2)} ${text}`)
1076
+ break
1077
+ }
1078
+ case 'code': {
1079
+ const left = Math.min(...block.lines.map(l => l.x0))
1080
+ // Monospace: one character's advance, measured on the block itself.
1081
+ const advances = block.lines.filter(l => l.text.length >= 4).map(l => (l.x1 - l.x0) / l.text.length).sort((p, q) => p - q)
1082
+ const advance = advances[Math.floor(advances.length / 2)] ?? block.lines[0]!.size * 0.6
1083
+ const body = block.lines.map((l) => {
1084
+ const indent = Math.max(0, Math.round((l.x0 - left) / advance))
1085
+ return ' '.repeat(Math.min(indent, 40)) + l.text
1086
+ }).join('\n')
1087
+ out.push(`\`\`\`\n${body.replace(/```/g, '`​``')}\n\`\`\``)
1088
+ break
1089
+ }
1090
+ case 'table':
1091
+ out.push(renderTable(block))
1092
+ break
1093
+ default: {
1094
+ let text = ''
1095
+ for (const line of block.lines) text = joinText(text, renderLine(line, true), vocabulary)
1096
+ text = mergeBoldRuns(text)
1097
+ if (block.kind === 'bullet') {
1098
+ text = text.replace(BULLET_RE, '').replace(DASH_BULLET_RE, '')
1099
+ // A bullet list stays one list: no blank line between its items.
1100
+ const prev = out[out.length - 1]
1101
+ if (prev?.startsWith('- ') === true) out[out.length - 1] = `${prev}\n- ${text}`
1102
+ else out.push(`- ${text}`)
1103
+ } else if (block.kind === 'ordered') {
1104
+ const prev = out[out.length - 1]
1105
+ if (prev !== undefined && /^\d{1,3}\.\s/.test(prev)) out[out.length - 1] = `${prev}\n${text.replace(/^(\d{1,3})\)/, '$1.')}`
1106
+ else out.push(text.replace(/^(\d{1,3})\)/, '$1.'))
1107
+ } else {
1108
+ out.push(escapeLeading(text))
1109
+ }
1110
+ }
1111
+ }
1112
+ }
1113
+ return `${out.join('\n\n')}\n`
1114
+ }