dsh-web-fetch-playwright 0.2.18 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/README.zh-CN.md +17 -1
- package/lib/client.js +18 -18
- package/lib/index.js +58203 -423
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +157 -0
- package/lib/types/index.d.ts +6 -2
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/pdf.d.ts +143 -0
- package/lib/types/provider.d.ts +9 -0
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/lib/types/types.d.ts +5 -0
- package/package.json +3 -2
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +639 -0
- package/src/index.ts +6 -2
- package/src/markdown.ts +136 -413
- package/src/pdf.ts +1114 -0
- package/src/provider.ts +117 -8
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +277 -0
- package/src/session-window.ts +8 -0
- package/src/trees.ts +43 -0
- package/src/types.ts +5 -0
package/src/pdf.ts
ADDED
|
@@ -0,0 +1,1114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PDF → markdown. A fetched URL that turns out to be a PDF (served inline as
|
|
3
|
+
* `application/pdf`, as a `Content-Disposition: attachment` download, or as
|
|
4
|
+
* an anonymous `application/octet-stream` whose bytes start with `%PDF-`) is
|
|
5
|
+
* routed here instead of failing with `WEB_UNSUPPORTED_CONTENT_TYPE`.
|
|
6
|
+
*
|
|
7
|
+
* Two layers, kept apart so the heuristics are testable without pdf.js:
|
|
8
|
+
*
|
|
9
|
+
* 1. {@link readPdfDocument} — the pdf.js boundary (through `unpdf`'s
|
|
10
|
+
* serverless build, loaded lazily so plugin start never pays for it).
|
|
11
|
+
* Per page it reads the positioned text runs (`getTextContent`) plus the
|
|
12
|
+
* real font names (`getOperatorList` + `fontExtraProperties`, the only way
|
|
13
|
+
* pdf.js exposes "NimbusRomNo9L-Medi" instead of a generic family), so the
|
|
14
|
+
* layout pass can see bold and monospace. Bounded by the fetch deadline,
|
|
15
|
+
* the caller's abort signal, a page cap, and a collected-character cap.
|
|
16
|
+
* 2. {@link layoutPdfMarkdown} — pure heuristics over those runs:
|
|
17
|
+
* - runs → lines (baseline grouping in content-stream order, x-sorted
|
|
18
|
+
* inside a line; rotated runs such as arXiv's side stamp are dropped);
|
|
19
|
+
* - running headers/footers (text repeated in the page margins across
|
|
20
|
+
* pages, digits normalized) and bare page numbers are removed;
|
|
21
|
+
* - the dominant (body) font size and line pitch are measured;
|
|
22
|
+
* - headings come from, strongest first: the PDF outline (bookmarks),
|
|
23
|
+
* section numbering ("3.1 Pre-training", "第二章", "一、"), larger font
|
|
24
|
+
* sizes (ranked into levels), and standalone bold lines;
|
|
25
|
+
* - lines are merged into paragraphs (vertical gap, first-line indent,
|
|
26
|
+
* short terminal line, font change; continuation across columns and
|
|
27
|
+
* pages when the sentence is unfinished), with dictionary-checked
|
|
28
|
+
* de-hyphenation and CJK-aware joining;
|
|
29
|
+
* - bullets / numbered items become markdown lists, monospace blocks
|
|
30
|
+
* become fenced code (unless the whole document is monospace), lines
|
|
31
|
+
* with wide column gaps become GFM tables when they line up;
|
|
32
|
+
* - `<!-- page N -->` markers keep page provenance for citations.
|
|
33
|
+
*
|
|
34
|
+
* Scanned PDFs without a text layer, encrypted PDFs, and broken files fail
|
|
35
|
+
* with dedicated, provider-specific error codes (OCR is out of scope).
|
|
36
|
+
*
|
|
37
|
+
* @module dsh-web-fetch-playwright/pdf
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
import { WebError } from '@deepseek-ai/dsh-web'
|
|
41
|
+
|
|
42
|
+
/** Largest PDF this provider downloads and parses (bytes). */
|
|
43
|
+
export const MAX_PDF_BYTES = 50 * 1024 * 1024
|
|
44
|
+
|
|
45
|
+
/** Most pages read from one document; later pages are reported as truncated. */
|
|
46
|
+
export const MAX_PDF_PAGES = 500
|
|
47
|
+
|
|
48
|
+
/** No text layer (a scanned / image-only PDF); OCR is not supported. */
|
|
49
|
+
export const WEB_PDF_NO_TEXT_CODE = 'WEB_PDF_NO_TEXT'
|
|
50
|
+
/** Password-protected PDF the parser cannot open. */
|
|
51
|
+
export const WEB_PDF_ENCRYPTED_CODE = 'WEB_PDF_ENCRYPTED'
|
|
52
|
+
/** Bytes claimed (or sniffed) to be a PDF that pdf.js cannot parse. */
|
|
53
|
+
export const WEB_PDF_INVALID_CODE = 'WEB_PDF_INVALID'
|
|
54
|
+
/** PDF larger than {@link MAX_PDF_BYTES}. */
|
|
55
|
+
export const WEB_PDF_TOO_LARGE_CODE = 'WEB_PDF_TOO_LARGE'
|
|
56
|
+
|
|
57
|
+
const PDF_MIME_TYPES = new Set(['application/pdf', 'application/x-pdf', 'application/acrobat', 'applications/vnd.pdf', 'text/pdf', 'text/x-pdf'])
|
|
58
|
+
|
|
59
|
+
/** Generic binary types a server may use for a PDF; their bytes get sniffed. */
|
|
60
|
+
const OPAQUE_BINARY_TYPES = new Set([
|
|
61
|
+
'application/octet-stream',
|
|
62
|
+
'binary/octet-stream',
|
|
63
|
+
'application/binary',
|
|
64
|
+
'application/download',
|
|
65
|
+
'application/x-download',
|
|
66
|
+
'application/force-download',
|
|
67
|
+
'application/unknown',
|
|
68
|
+
])
|
|
69
|
+
|
|
70
|
+
/** The bare media type of a `Content-Type` header value. */
|
|
71
|
+
function mediaType(contentType: string | undefined): string {
|
|
72
|
+
return (contentType ?? '').replace(/;.*$/s, '').trim().toLowerCase()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Whether a `Content-Type` names a PDF. */
|
|
76
|
+
export function isPdfContentType(contentType: string | undefined): boolean {
|
|
77
|
+
return PDF_MIME_TYPES.has(mediaType(contentType))
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Whether a response that is not otherwise decodable is worth downloading
|
|
82
|
+
* to sniff for PDF bytes: a PDF content type, a `.pdf` attachment filename,
|
|
83
|
+
* a `.pdf` URL path, or an opaque binary type (servers commonly hand PDFs out
|
|
84
|
+
* as `application/octet-stream`). The magic-byte check then decides.
|
|
85
|
+
*/
|
|
86
|
+
export function mayBePdf(headers: Record<string, string>, url: string): boolean {
|
|
87
|
+
if (isPdfContentType(headers['content-type'])) return true
|
|
88
|
+
const disposition = headers['content-disposition'] ?? ''
|
|
89
|
+
if (/filename\*?\s*=\s*(?:[\w-]+'[^']*')?"?[^";]*\.pdf\b/i.test(disposition)) return true
|
|
90
|
+
try {
|
|
91
|
+
if (/\.pdf$/i.test(new URL(url).pathname)) return true
|
|
92
|
+
} catch {
|
|
93
|
+
// not a URL — no path hint
|
|
94
|
+
}
|
|
95
|
+
return OPAQUE_BINARY_TYPES.has(mediaType(headers['content-type']))
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Whether the bytes are a PDF: `%PDF-` within the first 1024 bytes (the
|
|
100
|
+
* spec-tolerated leading-garbage window every mainstream reader accepts).
|
|
101
|
+
*/
|
|
102
|
+
export function isPdfBytes(bytes: Uint8Array): boolean {
|
|
103
|
+
const limit = Math.min(bytes.length - 5, 1024)
|
|
104
|
+
for (let i = 0; i <= limit; i++) {
|
|
105
|
+
if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true
|
|
106
|
+
}
|
|
107
|
+
return false
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// ---------------------------------------------------------------------------
|
|
111
|
+
// Positioned text model (the layout pass's input)
|
|
112
|
+
// ---------------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
/** One positioned, horizontal text run (pdf.js text item, normalized). */
|
|
115
|
+
export interface PdfTextRun {
|
|
116
|
+
text: string
|
|
117
|
+
/** Left edge in PDF user space. */
|
|
118
|
+
x: number
|
|
119
|
+
/** Baseline in PDF user space (origin bottom-left). */
|
|
120
|
+
y: number
|
|
121
|
+
width: number
|
|
122
|
+
/** Font size (vertical scale of the text matrix). */
|
|
123
|
+
size: number
|
|
124
|
+
bold: boolean
|
|
125
|
+
mono: boolean
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** One page's text runs in content-stream order. */
|
|
129
|
+
export interface PdfPageText {
|
|
130
|
+
/** 1-based page number. */
|
|
131
|
+
number: number
|
|
132
|
+
/** Page box bottom edge (user space). */
|
|
133
|
+
bottom: number
|
|
134
|
+
/** Page box height (user space). */
|
|
135
|
+
height: number
|
|
136
|
+
runs: PdfTextRun[]
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** One bookmark, flattened with its nesting depth (0 = top level). */
|
|
140
|
+
export interface PdfOutlineEntry {
|
|
141
|
+
title: string
|
|
142
|
+
depth: number
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Everything the layout pass consumes. */
|
|
146
|
+
export interface PdfDocumentText {
|
|
147
|
+
/** Document info / XMP title, when present. */
|
|
148
|
+
title?: string
|
|
149
|
+
outline: PdfOutlineEntry[]
|
|
150
|
+
pages: PdfPageText[]
|
|
151
|
+
totalPages: number
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** The finished extraction. */
|
|
155
|
+
export interface PdfExtraction {
|
|
156
|
+
markdown: string
|
|
157
|
+
totalPages: number
|
|
158
|
+
/** Pages actually read (fewer than `totalPages` when a limit cut in). */
|
|
159
|
+
pagesRead: number
|
|
160
|
+
/** True when a page, size, or time limit stopped the extraction early. */
|
|
161
|
+
truncated: boolean
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// ---------------------------------------------------------------------------
|
|
165
|
+
// pdf.js boundary
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
|
|
168
|
+
/** The structural pdf.js surface this module touches. */
|
|
169
|
+
interface PdfJsTextItem {
|
|
170
|
+
str?: string
|
|
171
|
+
transform?: number[]
|
|
172
|
+
width?: number
|
|
173
|
+
height?: number
|
|
174
|
+
fontName?: string
|
|
175
|
+
}
|
|
176
|
+
interface PdfJsPage {
|
|
177
|
+
view: number[]
|
|
178
|
+
getTextContent(): Promise<{ items: PdfJsTextItem[] }>
|
|
179
|
+
getOperatorList(): Promise<unknown>
|
|
180
|
+
commonObjs: { get(id: string): unknown }
|
|
181
|
+
cleanup(): unknown
|
|
182
|
+
}
|
|
183
|
+
interface PdfJsOutlineNode {
|
|
184
|
+
title?: string
|
|
185
|
+
items?: PdfJsOutlineNode[]
|
|
186
|
+
}
|
|
187
|
+
interface PdfJsDocument {
|
|
188
|
+
numPages: number
|
|
189
|
+
getPage(n: number): Promise<PdfJsPage>
|
|
190
|
+
getOutline(): Promise<PdfJsOutlineNode[] | null>
|
|
191
|
+
getMetadata(): Promise<{ info?: { Title?: unknown }; metadata?: { get(name: string): unknown } | null }>
|
|
192
|
+
loadingTask?: { destroy(): Promise<void> }
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Limits one extraction runs under. */
|
|
196
|
+
export interface PdfExtractOptions {
|
|
197
|
+
/** Abort → stop between pages and fail. */
|
|
198
|
+
signal?: AbortSignal
|
|
199
|
+
/** Wall-clock budget for reading pages (ms); pages left over are truncated. */
|
|
200
|
+
timeBudgetMs?: number
|
|
201
|
+
/** Stop reading pages once this many characters were collected. */
|
|
202
|
+
maxChars?: number
|
|
203
|
+
/** Page cap (defaults to {@link MAX_PDF_PAGES}). */
|
|
204
|
+
maxPages?: number
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Bold-looking PostScript font names (subset prefix already stripped). */
|
|
208
|
+
const BOLD_FONT_RE = /bold|black|heavy|semibold|demibold|demi\b|[-,]?medi(um)?\b|-B$|,B$|-SB$|CMBX|CMB\d|BX\d|\bHei\b|SimHei|Heiti|HTJ-|HT-|黑体/i
|
|
209
|
+
/** Monospace-looking font names. */
|
|
210
|
+
const MONO_FONT_RE = /mono|courier|consol|menlo|inconsolata|NimbusMonL|CMTT|SFTT|LMMono|SourceCodePro|FiraCode|DejaVuSansMono|LucidaConsole|Code/i
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Classify a pdf.js font object by its real (PostScript) name. pdf.js only
|
|
214
|
+
* exposes the name after the page's operator list loaded the font, and only
|
|
215
|
+
* with `fontExtraProperties`; anything unreadable classifies as regular.
|
|
216
|
+
*/
|
|
217
|
+
function classifyFont(page: PdfJsPage, loadedName: string | undefined, cache: Map<string, { bold: boolean; mono: boolean }>): { bold: boolean; mono: boolean } {
|
|
218
|
+
if (loadedName === undefined) return { bold: false, mono: false }
|
|
219
|
+
const cached = cache.get(loadedName)
|
|
220
|
+
if (cached !== undefined) return cached
|
|
221
|
+
let name = ''
|
|
222
|
+
let flagsBold = false
|
|
223
|
+
try {
|
|
224
|
+
const font = page.commonObjs.get(loadedName) as { name?: string; bold?: boolean; isMonospace?: boolean; black?: boolean } | undefined
|
|
225
|
+
name = (font?.name ?? '').replace(/^[A-Z]{6}\+/, '')
|
|
226
|
+
flagsBold = font?.bold === true || font?.black === true
|
|
227
|
+
} catch {
|
|
228
|
+
// font not resolved (op list skipped/failed) — regular
|
|
229
|
+
}
|
|
230
|
+
// Monospace by NAME only: CJK fonts routinely set the FixedPitch flag, and
|
|
231
|
+
// a Chinese paragraph is not a code block.
|
|
232
|
+
const result = { bold: flagsBold || BOLD_FONT_RE.test(name), mono: MONO_FONT_RE.test(name) }
|
|
233
|
+
cache.set(loadedName, result)
|
|
234
|
+
return result
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** Flatten the outline tree (bounded) into `{ title, depth }` entries. */
|
|
238
|
+
function flattenOutline(nodes: PdfJsOutlineNode[] | null, depth = 0, out: PdfOutlineEntry[] = []): PdfOutlineEntry[] {
|
|
239
|
+
if (nodes === null) return out
|
|
240
|
+
for (const node of nodes) {
|
|
241
|
+
if (out.length >= 2000) break
|
|
242
|
+
const title = (node.title ?? '').replace(/\s+/g, ' ').trim()
|
|
243
|
+
if (title !== '') out.push({ title, depth })
|
|
244
|
+
if (node.items !== undefined && node.items.length > 0 && depth < 8) flattenOutline(node.items, depth + 1, out)
|
|
245
|
+
}
|
|
246
|
+
return out
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** Map a pdf.js load failure onto the provider's PDF error codes. */
|
|
250
|
+
function translatePdfError(error: unknown): WebError {
|
|
251
|
+
if (error instanceof WebError) return error
|
|
252
|
+
const name = error instanceof Error ? error.name : ''
|
|
253
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
254
|
+
if (name === 'PasswordException' || /password/i.test(message)) {
|
|
255
|
+
return new WebError('the PDF is password-protected and cannot be read', WEB_PDF_ENCRYPTED_CODE, { cause: error })
|
|
256
|
+
}
|
|
257
|
+
return new WebError(`the PDF could not be parsed: ${message.split('\n')[0]!.slice(0, 200)}`, WEB_PDF_INVALID_CODE, { cause: error })
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Map Kangxi / CJK radical code points to the unified ideographs they look
|
|
262
|
+
* like. Some producers (Chromium printing Noto CJK among them) write a
|
|
263
|
+
* ToUnicode CMap that resolves 文 / 用 / 长 glyphs to U+2F42 ⽂ / U+2F64 ⽤
|
|
264
|
+
* / U+2EA7 ⻓ — visually identical, but they break search and reading.
|
|
265
|
+
* NFKC maps exactly these; it is applied to nothing else (fullwidth forms
|
|
266
|
+
* stay as the document wrote them).
|
|
267
|
+
*/
|
|
268
|
+
function normalizeRadicals(text: string): string {
|
|
269
|
+
return text.replace(/[\u2E80-\u2FDF]/g, ch => SIMPLIFIED_RADICALS[ch] ?? ch.normalize('NFKC'))
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* The C-simplified radicals (CJK Radicals Supplement) have no NFKC mapping;
|
|
274
|
+
* these are the ones that double as everyday simplified characters.
|
|
275
|
+
*/
|
|
276
|
+
const SIMPLIFIED_RADICALS: Record<string, string> = {
|
|
277
|
+
'\u2EC5': '见', '\u2EC6': '角', '\u2EC8': '讠', '\u2EC9': '贝', '\u2ECB': '车', '\u2ED0': '钅',
|
|
278
|
+
'\u2ED3': '长', '\u2ED4': '门', '\u2ED9': '韦', '\u2EDA': '页', '\u2EDB': '风', '\u2EDC': '飞',
|
|
279
|
+
'\u2EE0': '饣', '\u2EE2': '马', '\u2EE5': '鱼', '\u2EE6': '鸟', '\u2EE7': '卤', '\u2EE8': '麦',
|
|
280
|
+
'\u2EE9': '黄', '\u2EEA': '黾', '\u2EEC': '齐', '\u2EEE': '齿', '\u2EF0': '龙',
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** A run is horizontal when the text matrix has (almost) no rotation/skew. */
|
|
284
|
+
function isHorizontal(t: number[]): boolean {
|
|
285
|
+
const [a = 0, b = 0, c = 0, d = 0] = t
|
|
286
|
+
return Math.abs(b) <= Math.abs(a) * 0.1 + 1e-6 && Math.abs(c) <= Math.abs(d) * 0.1 + 1e-6 && a > 0 && d > 0
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Read a PDF's positioned text with pdf.js, page by page, inside the given
|
|
291
|
+
* limits. Text items become {@link PdfTextRun}s (horizontal only — rotated
|
|
292
|
+
* stamps and vertical margin text are dropped as layout noise).
|
|
293
|
+
*
|
|
294
|
+
* @returns the text model, plus whether a limit stopped reading early.
|
|
295
|
+
*/
|
|
296
|
+
export async function readPdfDocument(bytes: Uint8Array, options: PdfExtractOptions = {}): Promise<{ doc: PdfDocumentText; truncated: boolean }> {
|
|
297
|
+
const startedAt = Date.now()
|
|
298
|
+
const budget = options.timeBudgetMs ?? Number.POSITIVE_INFINITY
|
|
299
|
+
const maxChars = options.maxChars ?? Number.POSITIVE_INFINITY
|
|
300
|
+
const maxPages = options.maxPages ?? MAX_PDF_PAGES
|
|
301
|
+
const { getDocumentProxy } = await import('unpdf')
|
|
302
|
+
// pdf.js refuses Node `Buffer`s (and may detach what it is given): hand it
|
|
303
|
+
// a private plain `Uint8Array` copy.
|
|
304
|
+
const data = new Uint8Array(bytes.byteLength)
|
|
305
|
+
data.set(bytes)
|
|
306
|
+
let pdf: PdfJsDocument
|
|
307
|
+
try {
|
|
308
|
+
pdf = await getDocumentProxy(data, {
|
|
309
|
+
// Real font names (bold / monospace detection) after the operator list.
|
|
310
|
+
fontExtraProperties: true,
|
|
311
|
+
// (pdf.js 6 no longer compiles font programs with `new Function`, so
|
|
312
|
+
// there is no `isEvalSupported` switch left to turn off.)
|
|
313
|
+
verbosity: 0,
|
|
314
|
+
stopAtErrors: false,
|
|
315
|
+
}) as unknown as PdfJsDocument
|
|
316
|
+
} catch (error: unknown) {
|
|
317
|
+
throw translatePdfError(error)
|
|
318
|
+
}
|
|
319
|
+
try {
|
|
320
|
+
let title: string | undefined
|
|
321
|
+
try {
|
|
322
|
+
const meta = await pdf.getMetadata()
|
|
323
|
+
const xmpTitle = meta.metadata?.get('dc:title')
|
|
324
|
+
const infoTitle = meta.info?.Title
|
|
325
|
+
const candidate = typeof xmpTitle === 'string' && xmpTitle.trim() !== '' ? xmpTitle : typeof infoTitle === 'string' ? infoTitle : ''
|
|
326
|
+
if (candidate.trim() !== '') title = candidate.replace(/\s+/g, ' ').trim()
|
|
327
|
+
} catch {
|
|
328
|
+
// metadata is optional
|
|
329
|
+
}
|
|
330
|
+
let outline: PdfOutlineEntry[] = []
|
|
331
|
+
try {
|
|
332
|
+
outline = flattenOutline(await pdf.getOutline())
|
|
333
|
+
} catch {
|
|
334
|
+
// outline is optional
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const pages: PdfPageText[] = []
|
|
338
|
+
const fontCache = new Map<string, { bold: boolean; mono: boolean }>()
|
|
339
|
+
let chars = 0
|
|
340
|
+
let truncated = false
|
|
341
|
+
const pageCount = pdf.numPages
|
|
342
|
+
for (let n = 1; n <= pageCount; n++) {
|
|
343
|
+
if (options.signal?.aborted) throw options.signal.reason instanceof Error ? options.signal.reason : new Error('pdf extraction aborted')
|
|
344
|
+
if (n > maxPages || chars >= maxChars || Date.now() - startedAt >= budget) {
|
|
345
|
+
truncated = true
|
|
346
|
+
break
|
|
347
|
+
}
|
|
348
|
+
const page = await pdf.getPage(n)
|
|
349
|
+
const content = await page.getTextContent()
|
|
350
|
+
// Fonts resolve only through the operator list; skip it when time is
|
|
351
|
+
// short (the layout then simply sees no bold/monospace on this page).
|
|
352
|
+
if (Date.now() - startedAt < budget * 0.8) {
|
|
353
|
+
try { await page.getOperatorList() } catch { /* fonts stay unknown */ }
|
|
354
|
+
}
|
|
355
|
+
const [, viewBottom = 0, , viewTop = 0] = page.view
|
|
356
|
+
const runs: PdfTextRun[] = []
|
|
357
|
+
for (const item of content.items) {
|
|
358
|
+
if (typeof item.str !== 'string' || item.str === '' || item.transform === undefined) continue
|
|
359
|
+
if (!isHorizontal(item.transform)) continue
|
|
360
|
+
const t = item.transform
|
|
361
|
+
const size = Math.hypot(t[2] ?? 0, t[3] ?? 0) || item.height || 0
|
|
362
|
+
if (!(size > 0)) continue
|
|
363
|
+
const font = classifyFont(page, item.fontName, fontCache)
|
|
364
|
+
runs.push({ text: normalizeRadicals(item.str), x: t[4] ?? 0, y: t[5] ?? 0, width: item.width ?? 0, size, bold: font.bold, mono: font.mono })
|
|
365
|
+
chars += item.str.length
|
|
366
|
+
}
|
|
367
|
+
pages.push({ number: n, bottom: Math.min(viewBottom, viewTop), height: Math.abs(viewTop - viewBottom) || 792, runs })
|
|
368
|
+
page.cleanup()
|
|
369
|
+
}
|
|
370
|
+
return { doc: { ...(title !== undefined ? { title } : {}), outline, pages, totalPages: pageCount }, truncated }
|
|
371
|
+
} catch (error: unknown) {
|
|
372
|
+
if (options.signal?.aborted) throw error
|
|
373
|
+
throw translatePdfError(error)
|
|
374
|
+
} finally {
|
|
375
|
+
// Destroying the loading task tears down the (in-process) worker too.
|
|
376
|
+
void pdf.loadingTask?.destroy().catch(() => {})
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/**
|
|
381
|
+
* Extract a PDF as markdown: read it with pdf.js inside the limits, lay it
|
|
382
|
+
* out, and fail with {@link WEB_PDF_NO_TEXT_CODE} when there is no text layer
|
|
383
|
+
* to speak of (a scan). Sizes beyond {@link MAX_PDF_BYTES} are refused.
|
|
384
|
+
*/
|
|
385
|
+
export async function extractPdfMarkdown(bytes: Uint8Array, options: PdfExtractOptions = {}): Promise<PdfExtraction> {
|
|
386
|
+
if (bytes.length > MAX_PDF_BYTES) {
|
|
387
|
+
throw new WebError(`the PDF is ${String(Math.round(bytes.length / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`, WEB_PDF_TOO_LARGE_CODE)
|
|
388
|
+
}
|
|
389
|
+
const { doc, truncated } = await readPdfDocument(bytes, options)
|
|
390
|
+
const textChars = doc.pages.reduce((sum, page) => sum + page.runs.reduce((s, run) => s + run.text.replace(/\s+/g, '').length, 0), 0)
|
|
391
|
+
// A scan carries no (or a few stray) characters per page; a genuinely
|
|
392
|
+
// short born-digital document ("Dummy PDF file") still clears this.
|
|
393
|
+
if (doc.pages.length > 0 && textChars < Math.max(1, doc.pages.length * 5)) {
|
|
394
|
+
throw new WebError(
|
|
395
|
+
`the PDF has no extractable text layer (${String(doc.totalPages)} page(s), likely scanned images); OCR is not supported`,
|
|
396
|
+
WEB_PDF_NO_TEXT_CODE,
|
|
397
|
+
)
|
|
398
|
+
}
|
|
399
|
+
let markdown = layoutPdfMarkdown(doc)
|
|
400
|
+
if (truncated) {
|
|
401
|
+
markdown += `\n\n> [PDF extraction stopped after page ${String(doc.pages.length)} of ${String(doc.totalPages)} (size/time limit).]\n`
|
|
402
|
+
}
|
|
403
|
+
return { markdown, totalPages: doc.totalPages, pagesRead: doc.pages.length, truncated }
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
// ---------------------------------------------------------------------------
|
|
407
|
+
// Layout heuristics (pure)
|
|
408
|
+
// ---------------------------------------------------------------------------
|
|
409
|
+
|
|
410
|
+
/** CJK ideographs, kana, hangul, and fullwidth/CJK punctuation. */
|
|
411
|
+
const CJK_RE = /[⺀- -〿-ヿ-ㇿ㐀-䶿一-鿿가-豈--]/
|
|
412
|
+
|
|
413
|
+
function isCjk(ch: string | undefined): boolean {
|
|
414
|
+
return ch !== undefined && CJK_RE.test(ch)
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/** Letter-spaced CJK ("预 防 和 治 理") reads as one word. */
|
|
418
|
+
function collapseCjkSpacing(text: string): string {
|
|
419
|
+
return text.replace(new RegExp(`(?<=${CJK_RE.source}) +(?=${CJK_RE.source})`, 'g'), '')
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/** A styled fragment of a line (for inline bold). */
|
|
423
|
+
interface Segment {
|
|
424
|
+
text: string
|
|
425
|
+
bold: boolean
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/** One visual line. */
|
|
429
|
+
interface Line {
|
|
430
|
+
page: number
|
|
431
|
+
text: string
|
|
432
|
+
segments: Segment[]
|
|
433
|
+
x0: number
|
|
434
|
+
x1: number
|
|
435
|
+
y: number
|
|
436
|
+
size: number
|
|
437
|
+
bold: boolean
|
|
438
|
+
mono: boolean
|
|
439
|
+
/** Text cells separated by wide horizontal gaps (table-ish rows). */
|
|
440
|
+
cells: string[]
|
|
441
|
+
/** Left edge of each cell in {@link Line.cells}. */
|
|
442
|
+
cellX: number[]
|
|
443
|
+
/** Non-whitespace character count. */
|
|
444
|
+
chars: number
|
|
445
|
+
/** Position relative to the page box (0 = bottom, 1 = top). */
|
|
446
|
+
relY: number
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/** Group a page's runs into lines (content-stream order, baseline match). */
|
|
450
|
+
function buildLines(page: PdfPageText): Line[] {
|
|
451
|
+
const groups: PdfTextRun[][] = []
|
|
452
|
+
let current: PdfTextRun[] = []
|
|
453
|
+
let baseY = 0
|
|
454
|
+
let baseSize = 0
|
|
455
|
+
for (const run of page.runs) {
|
|
456
|
+
if (current.length > 0) {
|
|
457
|
+
const tolerance = Math.max(0.6 * Math.max(run.size, baseSize), 1)
|
|
458
|
+
if (Math.abs(run.y - baseY) <= tolerance) {
|
|
459
|
+
current.push(run)
|
|
460
|
+
if (run.text.trim() !== '' && run.size > baseSize) { baseSize = run.size; baseY = run.y }
|
|
461
|
+
continue
|
|
462
|
+
}
|
|
463
|
+
groups.push(current)
|
|
464
|
+
}
|
|
465
|
+
current = [run]
|
|
466
|
+
baseY = run.y
|
|
467
|
+
baseSize = run.size
|
|
468
|
+
}
|
|
469
|
+
if (current.length > 0) groups.push(current)
|
|
470
|
+
|
|
471
|
+
const lines: Line[] = []
|
|
472
|
+
for (const group of groups) {
|
|
473
|
+
const line = composeLine(group, page)
|
|
474
|
+
if (line !== undefined) lines.push(line)
|
|
475
|
+
}
|
|
476
|
+
return lines
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/** Join one baseline group's runs (x-sorted) into a {@link Line}. */
|
|
480
|
+
function composeLine(group: PdfTextRun[], page: PdfPageText): Line | undefined {
|
|
481
|
+
const runs = [...group].sort((a, b) => a.x - b.x)
|
|
482
|
+
const visible = runs.filter(run => run.text.trim() !== '')
|
|
483
|
+
if (visible.length === 0) return undefined
|
|
484
|
+
// Dominant size and baseline: the size carrying the most characters.
|
|
485
|
+
const weight = new Map<number, number>()
|
|
486
|
+
for (const run of visible) {
|
|
487
|
+
const key = Math.round(run.size * 10) / 10
|
|
488
|
+
weight.set(key, (weight.get(key) ?? 0) + run.text.trim().length)
|
|
489
|
+
}
|
|
490
|
+
let size = visible[0]!.size
|
|
491
|
+
let best = -1
|
|
492
|
+
for (const [key, w] of weight) if (w > best) { best = w; size = key }
|
|
493
|
+
const baseline = visible.find(run => Math.abs(run.size - size) < 0.11)?.y ?? visible[0]!.y
|
|
494
|
+
|
|
495
|
+
const segments: Segment[] = []
|
|
496
|
+
const cells: string[] = ['']
|
|
497
|
+
const cellX: number[] = []
|
|
498
|
+
let text = ''
|
|
499
|
+
let prevEnd: number | undefined
|
|
500
|
+
let boldChars = 0
|
|
501
|
+
let monoChars = 0
|
|
502
|
+
let chars = 0
|
|
503
|
+
for (const run of runs) {
|
|
504
|
+
const content = run.text
|
|
505
|
+
const trimmed = content.trim()
|
|
506
|
+
if (trimmed === '') {
|
|
507
|
+
// Whitespace runs only matter as separators; gaps are measured between
|
|
508
|
+
// visible runs.
|
|
509
|
+
continue
|
|
510
|
+
}
|
|
511
|
+
let sep = ''
|
|
512
|
+
if (prevEnd !== undefined && text !== '') {
|
|
513
|
+
const gap = run.x - prevEnd
|
|
514
|
+
const last = text[text.length - 1]
|
|
515
|
+
const first = trimmed[0]
|
|
516
|
+
if (gap > run.size * 1.5 && !(isCjk(last) && isCjk(first) && gap < run.size * 3)) {
|
|
517
|
+
cells.push('')
|
|
518
|
+
cellX.push(run.x)
|
|
519
|
+
sep = ' '
|
|
520
|
+
} else if (gap > run.size * 0.15 || /\s$/.test(content.slice(0, content.length - content.trimStart().length)) || /^\s/.test(content)) {
|
|
521
|
+
sep = isCjk(last) && isCjk(first) ? '' : ' '
|
|
522
|
+
}
|
|
523
|
+
if (/\s$/.test(text)) sep = ''
|
|
524
|
+
}
|
|
525
|
+
const piece = sep + trimmed
|
|
526
|
+
if (cellX.length === 0) cellX.push(run.x)
|
|
527
|
+
text += piece
|
|
528
|
+
cells[cells.length - 1] += (cells[cells.length - 1] === '' ? trimmed : piece)
|
|
529
|
+
const prev = segments[segments.length - 1]
|
|
530
|
+
if (prev !== undefined && prev.bold === run.bold) prev.text += piece
|
|
531
|
+
else segments.push({ text: piece, bold: run.bold })
|
|
532
|
+
prevEnd = Math.max(prevEnd ?? -Infinity, run.x + run.width)
|
|
533
|
+
const n = trimmed.replace(/\s+/g, '').length
|
|
534
|
+
chars += n
|
|
535
|
+
if (run.bold) boldChars += n
|
|
536
|
+
if (run.mono) monoChars += n
|
|
537
|
+
}
|
|
538
|
+
text = collapseCjkSpacing(text.replace(/\s+/g, ' ').trim())
|
|
539
|
+
if (text === '') return undefined
|
|
540
|
+
for (const segment of segments) segment.text = collapseCjkSpacing(segment.text)
|
|
541
|
+
const x0 = visible.reduce((m, run) => Math.min(m, run.x), Infinity)
|
|
542
|
+
const x1 = visible.reduce((m, run) => Math.max(m, run.x + run.width), -Infinity)
|
|
543
|
+
return {
|
|
544
|
+
page: page.number,
|
|
545
|
+
text,
|
|
546
|
+
segments,
|
|
547
|
+
x0,
|
|
548
|
+
x1,
|
|
549
|
+
y: baseline,
|
|
550
|
+
size,
|
|
551
|
+
bold: chars > 0 && boldChars / chars >= 0.8,
|
|
552
|
+
mono: chars > 0 && monoChars / chars >= 0.8,
|
|
553
|
+
cells: cells.map(cell => collapseCjkSpacing(cell.replace(/\s+/g, ' ').trim())),
|
|
554
|
+
cellX,
|
|
555
|
+
chars,
|
|
556
|
+
relY: page.height > 0 ? (baseline - page.bottom) / page.height : 0.5,
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** Bare page-number shapes found in page margins. */
|
|
561
|
+
const PAGE_NUMBER_RES = [
|
|
562
|
+
/^[-–—\s]*(?:page|p\.|pág\.|seite)?\s*\d{1,4}(?:\s*(?:\/|of|von|de)\s*\d{1,4})?[-–—\s]*$/i,
|
|
563
|
+
/^[-–—\s]*[ivxlcdm]{1,7}[-–—\s]*$/i,
|
|
564
|
+
/^第\s*\d{1,4}\s*页(?:\s*[,,//]?\s*共\s*\d{1,4}\s*页)?$/,
|
|
565
|
+
// "— 12 —" (Chinese official documents), including the OCR misreading of
|
|
566
|
+
// the dashes as 一 and fullwidth dash variants.
|
|
567
|
+
/^[-–—一-―·•]{1,2}\s*\d{1,4}\s*[-–—一-―·•]{1,2}$/,
|
|
568
|
+
]
|
|
569
|
+
|
|
570
|
+
const isMarginLine = (line: Line): boolean => line.relY > 0.9 || line.relY < 0.12
|
|
571
|
+
|
|
572
|
+
/** Drop running headers/footers and page numbers. */
|
|
573
|
+
function removeMarginNoise(pages: Line[][]): Line[][] {
|
|
574
|
+
const keyOf = (line: Line): string => line.text.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ').trim()
|
|
575
|
+
const pagesByKey = new Map<string, Set<number>>()
|
|
576
|
+
for (const lines of pages) {
|
|
577
|
+
for (const line of lines) {
|
|
578
|
+
if (!isMarginLine(line)) continue
|
|
579
|
+
const key = keyOf(line)
|
|
580
|
+
let set = pagesByKey.get(key)
|
|
581
|
+
if (set === undefined) { set = new Set(); pagesByKey.set(key, set) }
|
|
582
|
+
set.add(line.page)
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
const repeatThreshold = Math.max(3, Math.ceil(pages.length * 0.3))
|
|
586
|
+
return pages.map(lines => lines.filter((line) => {
|
|
587
|
+
if (!isMarginLine(line)) return true
|
|
588
|
+
const plain = line.text.normalize('NFKC')
|
|
589
|
+
if (PAGE_NUMBER_RES.some(re => re.test(plain))) return false
|
|
590
|
+
if (pages.length >= 3 && (pagesByKey.get(keyOf(line))?.size ?? 0) >= repeatThreshold) return false
|
|
591
|
+
return true
|
|
592
|
+
}))
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/** Character-weighted mode of the line font sizes (0.5pt buckets). */
|
|
596
|
+
function bodySizeOf(lines: Line[]): number {
|
|
597
|
+
const weight = new Map<number, number>()
|
|
598
|
+
for (const line of lines) {
|
|
599
|
+
const key = Math.round(line.size * 2) / 2
|
|
600
|
+
weight.set(key, (weight.get(key) ?? 0) + line.chars)
|
|
601
|
+
}
|
|
602
|
+
let body = 10
|
|
603
|
+
let best = -1
|
|
604
|
+
for (const [key, w] of weight) if (w > best) { best = w; body = key }
|
|
605
|
+
return body
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
/** Median baseline-to-baseline distance between consecutive body lines. */
|
|
609
|
+
function lineSpacingOf(pages: Line[][], body: number): number {
|
|
610
|
+
const deltas: number[] = []
|
|
611
|
+
for (const lines of pages) {
|
|
612
|
+
for (let i = 1; i < lines.length; i++) {
|
|
613
|
+
const a = lines[i - 1]!
|
|
614
|
+
const b = lines[i]!
|
|
615
|
+
if (Math.abs(a.size - body) > body * 0.1 || Math.abs(b.size - body) > body * 0.1) continue
|
|
616
|
+
const dy = a.y - b.y
|
|
617
|
+
if (dy > body * 0.6 && dy < body * 3) deltas.push(dy)
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
if (deltas.length === 0) return body * 1.2
|
|
621
|
+
deltas.sort((p, q) => p - q)
|
|
622
|
+
return deltas[Math.floor(deltas.length / 2)]!
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/** Normalization for comparing headings to outline titles. */
|
|
626
|
+
function headingKey(text: string): string {
|
|
627
|
+
return text.normalize('NFKC').toLowerCase().replace(/[\s\p{P}\p{S}]/gu, '')
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
/** Leading section number ("3", "3.1", "A.2") stripped for outline matching. */
|
|
631
|
+
function stripNumbering(text: string): string {
|
|
632
|
+
return text.replace(/^(?:\d+(?:\.\d+)*|[A-Z](?:\.\d+)+)\.?\s+/, '').replace(/^(?:第[一二三四五六七八九十百千零〇两\d]+[章节篇部]|[一二三四五六七八九十]+[、..]|[((][一二三四五六七八九十]+[))])\s*/, '')
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
const NUMBERED_HEADING_RE = /^(\d{1,2}(?:\.\d{1,2}){0,4}|[A-Z](?:\.\d{1,2}){1,4})\.?\s+(?=\S)/
|
|
636
|
+
const CN_CHAPTER_RE = /^第[一二三四五六七八九十百千零〇两\d]+[章篇部]/
|
|
637
|
+
const CN_SECTION_RE = /^第[一二三四五六七八九十百千零〇两\d]+节/
|
|
638
|
+
const CN_ENUM_RE = /^[一二三四五六七八九十]{1,3}[、.]/
|
|
639
|
+
const CN_PAREN_ENUM_RE = /^[((][一二三四五六七八九十]{1,3}[))]/
|
|
640
|
+
/** A table-of-contents entry: leader dots (or a wide gap) then a page reference. */
|
|
641
|
+
const TOC_ENTRY_RE = /(?:[.…·.]\s*){4,}[^.…·.]{0,16}?[((]?\s*[\d0-9ivxlc]+\s*[))]?$/i
|
|
642
|
+
const EXPLICIT_HEADING_RES = [NUMBERED_HEADING_RE, CN_CHAPTER_RE, CN_SECTION_RE, CN_ENUM_RE, CN_PAREN_ENUM_RE]
|
|
643
|
+
const TERMINAL_RE = /[.!?。!?…::;;"”’))\]】]$/
|
|
644
|
+
const SENTENCE_END_RE = /[.!?。!?…"”’]$/
|
|
645
|
+
|
|
646
|
+
const BULLET_RE = /^(?:[•◦▪▫●○■□►▸‣⁃∙·*❖➢✓✔]|[])\s*/
|
|
647
|
+
const DASH_BULLET_RE = /^[-–—]\s+(?=\S)/
|
|
648
|
+
const ORDERED_RE = /^(\d{1,3})[.)]\s+(?=\S)/
|
|
649
|
+
const PAREN_ITEM_RE = /^(?:[((](?:\d{1,3}|[a-zA-Z]|[ivx]{1,4})[))]|[a-z][.)]|[①-⑳])\s*(?=\S)/
|
|
650
|
+
|
|
651
|
+
/** Markdown kinds a line can open. */
|
|
652
|
+
type BlockKind = 'heading' | 'bullet' | 'ordered' | 'item' | 'code' | 'text' | 'table'
|
|
653
|
+
|
|
654
|
+
interface Block {
|
|
655
|
+
kind: BlockKind
|
|
656
|
+
page: number
|
|
657
|
+
level?: number
|
|
658
|
+
lines: Line[]
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
interface LayoutStats {
|
|
662
|
+
body: number
|
|
663
|
+
spacing: number
|
|
664
|
+
monoIsBody: boolean
|
|
665
|
+
/** Heading font sizes, largest first (each rank is one heading level). */
|
|
666
|
+
headingSizes: number[]
|
|
667
|
+
/** Whether a dedicated document-title size exists (rank 0 = h1). */
|
|
668
|
+
titleSize?: number
|
|
669
|
+
outlineKeys: Map<string, number>
|
|
670
|
+
/** 1 when the outline's single root is the document title (children → h2). */
|
|
671
|
+
outlineOffset: number
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/** Heading level placeholder: one below the most recent heading. */
|
|
675
|
+
const RELATIVE_HEADING = -1
|
|
676
|
+
|
|
677
|
+
/** Decide a line's heading level, if it is one. */
|
|
678
|
+
function headingLevel(line: Line, groupLines: Line[], stats: LayoutStats, standalone: boolean, styleRules: boolean): number | undefined {
|
|
679
|
+
const text = groupLines.map(l => l.text).join(' ')
|
|
680
|
+
const chars = groupLines.reduce((s, l) => s + l.chars, 0)
|
|
681
|
+
if (chars === 0 || chars > 200 || !/\p{L}/u.test(text)) return undefined
|
|
682
|
+
const words = text.split(/\s+/).length
|
|
683
|
+
const sentenceLike = /[,,;;]$/.test(text) || (/[.。]$/.test(text) && !/^\d+(\.\d+)*\.?$/.test(text.split(' ')[0] ?? ''))
|
|
684
|
+
// 1. The outline (bookmarks) — the author's own structure.
|
|
685
|
+
if (stats.titleSize !== undefined && Math.abs(line.size - stats.titleSize) < 0.3 && groupLines.length <= 3 && styleRules) return 1
|
|
686
|
+
const outlineDepth = stats.outlineKeys.get(headingKey(text)) ?? stats.outlineKeys.get(headingKey(stripNumbering(text)))
|
|
687
|
+
if (outlineDepth !== undefined && chars <= 160) return Math.min(6, Math.max(2, outlineDepth + 2 - stats.outlineOffset))
|
|
688
|
+
|
|
689
|
+
const bigger = line.size >= stats.body * 1.12
|
|
690
|
+
const emphasized = bigger || line.bold
|
|
691
|
+
// Table rows (wide column gaps) are never style-only headings.
|
|
692
|
+
const styleOk = styleRules && groupLines.every(l => l.cells.length < 2)
|
|
693
|
+
// 2. Section numbering.
|
|
694
|
+
const numbered = NUMBERED_HEADING_RE.exec(text)
|
|
695
|
+
if (numbered !== null && emphasized && !sentenceLike && words <= 16 && chars <= 120) {
|
|
696
|
+
const depth = numbered[1]!.split('.').length
|
|
697
|
+
return Math.min(6, depth + 1)
|
|
698
|
+
}
|
|
699
|
+
if (CN_CHAPTER_RE.test(text) && chars <= 60 && !sentenceLike) return 2
|
|
700
|
+
if (CN_SECTION_RE.test(text) && chars <= 60 && !sentenceLike) return 3
|
|
701
|
+
if (CN_ENUM_RE.test(text) && chars <= 40 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 2
|
|
702
|
+
if (CN_PAREN_ENUM_RE.test(text) && chars <= 30 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 3
|
|
703
|
+
if (!styleOk) return undefined
|
|
704
|
+
// 3. Larger type, ranked.
|
|
705
|
+
if (bigger && groupLines.length <= 3 && !sentenceLike) {
|
|
706
|
+
if (stats.titleSize !== undefined && Math.abs(line.size - stats.titleSize) < 0.3) return 1
|
|
707
|
+
const rank = stats.headingSizes.findIndex(size => Math.abs(size - Math.round(line.size * 2) / 2) < 0.3)
|
|
708
|
+
return Math.min(4, 2 + Math.max(0, rank))
|
|
709
|
+
}
|
|
710
|
+
// 4. Standalone bold line at body size: a child of the current section
|
|
711
|
+
// (resolved by the caller). A run-in heading — bold lead, regular text
|
|
712
|
+
// after it on the same line — is a paragraph, not a heading.
|
|
713
|
+
const lastSegment = groupLines[groupLines.length - 1]!.segments.at(-1)
|
|
714
|
+
if (line.bold && lastSegment?.bold === true && standalone && groupLines.length <= 2 && chars <= 100 && words <= 14 && !sentenceLike && !TERMINAL_RE.test(text.replace(/[))]$/, ''))) {
|
|
715
|
+
return RELATIVE_HEADING
|
|
716
|
+
}
|
|
717
|
+
return undefined
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/** Group lines into markdown blocks. */
|
|
721
|
+
function buildBlocks(pages: Line[][], stats: LayoutStats): Block[] {
|
|
722
|
+
const all = pages.flat()
|
|
723
|
+
const blocks: Block[] = []
|
|
724
|
+
// Column right edge for a line: the widest line on the page starting near it.
|
|
725
|
+
const rightEdge = new Map<Line, number>()
|
|
726
|
+
for (const lines of pages) {
|
|
727
|
+
for (const line of lines) {
|
|
728
|
+
let right = line.x1
|
|
729
|
+
for (const other of lines) {
|
|
730
|
+
if (Math.abs(other.x0 - line.x0) < line.size * 3 && Math.abs(other.size - line.size) < line.size * 0.15) right = Math.max(right, other.x1)
|
|
731
|
+
}
|
|
732
|
+
rightEdge.set(line, right)
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
const pitch = (line: Line): number => stats.spacing * (line.size / stats.body)
|
|
736
|
+
const gapBefore = (i: number): number => {
|
|
737
|
+
const prev = all[i - 1]
|
|
738
|
+
const line = all[i]!
|
|
739
|
+
if (prev === undefined || prev.page !== line.page) return Infinity
|
|
740
|
+
return prev.y - line.y
|
|
741
|
+
}
|
|
742
|
+
const isStandalone = (i: number, span: number): boolean => {
|
|
743
|
+
const before = gapBefore(i)
|
|
744
|
+
const after = i + span < all.length ? gapBefore(i + span) : Infinity
|
|
745
|
+
const first = all[i]!
|
|
746
|
+
const sepBefore = before === Infinity || before < 0 || before > pitch(first) * 1.3 || all[i - 1]!.size !== first.size || all[i - 1]!.bold !== first.bold
|
|
747
|
+
const next = all[i + span]
|
|
748
|
+
const sepAfter = after === Infinity || after < 0 || after > pitch(first) * 1.3 || next === undefined || Math.abs(next.size - first.size) > 0.3
|
|
749
|
+
const short = all[i + span - 1]!.x1 < (rightEdge.get(all[i + span - 1]!) ?? Infinity) - first.size * 2
|
|
750
|
+
return (sepBefore && (sepAfter || short))
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
// A lone monospace line wedged into running text (a URL or identifier
|
|
754
|
+
// the paragraph wrapped onto its own line) stays in the paragraph.
|
|
755
|
+
const isInlineMono = (index: number): boolean => {
|
|
756
|
+
const line = all[index]!
|
|
757
|
+
const prev = all[index - 1]
|
|
758
|
+
const next = all[index + 1]
|
|
759
|
+
if (prev === undefined || prev.mono || next?.mono === true && next.page === line.page && gapBefore(index + 1) > 0 && gapBefore(index + 1) < pitch(line) * 2.2) return false
|
|
760
|
+
const dy = gapBefore(index)
|
|
761
|
+
return dy > 0 && dy < pitch(line) * 1.45 && !TERMINAL_RE.test(prev.text) && line.chars <= 80
|
|
762
|
+
}
|
|
763
|
+
// Tables: runs of consecutive multi-cell lines whose columns line up.
|
|
764
|
+
// A justified line with wide word gaps has "cells" too, but no other row
|
|
765
|
+
// shares its column starts.
|
|
766
|
+
const tableEnd = new Map<number, number>()
|
|
767
|
+
const inTable = new Set<number>()
|
|
768
|
+
for (let start = 0; start < all.length;) {
|
|
769
|
+
if (all[start]!.cells.length < 2 || all[start]!.mono) { start++; continue }
|
|
770
|
+
let end = start + 1
|
|
771
|
+
while (end < all.length && all[end]!.cells.length >= 2 && !all[end]!.mono && all[end]!.page === all[start]!.page
|
|
772
|
+
&& gapBefore(end) > 0 && gapBefore(end) < pitch(all[end]!) * 2.5) end++
|
|
773
|
+
// Rows at either end that share no columns with the rest are not part
|
|
774
|
+
// of the table (a justified line just above or below it).
|
|
775
|
+
let first = start
|
|
776
|
+
let last = end
|
|
777
|
+
while (last - first >= 2 && !rowAligned(all[last - 1]!, all.slice(first, last - 1))) last--
|
|
778
|
+
while (last - first >= 2 && !rowAligned(all[first]!, all.slice(first + 1, last))) first++
|
|
779
|
+
if (last - first >= 2 && columnsAlign(all.slice(first, last))) {
|
|
780
|
+
tableEnd.set(first, last)
|
|
781
|
+
for (let k = first; k < last; k++) inTable.add(k)
|
|
782
|
+
}
|
|
783
|
+
start = end
|
|
784
|
+
}
|
|
785
|
+
let lastLevel = 1
|
|
786
|
+
let i = 0
|
|
787
|
+
while (i < all.length) {
|
|
788
|
+
const line = all[i]!
|
|
789
|
+
// Candidate heading group: consecutive lines of the same size/weight.
|
|
790
|
+
let span = 1
|
|
791
|
+
while (
|
|
792
|
+
span < 4 && i + span < all.length
|
|
793
|
+
&& all[i + span]!.page === line.page
|
|
794
|
+
&& Math.abs(all[i + span]!.size - line.size) < 0.3
|
|
795
|
+
&& all[i + span]!.bold === line.bold
|
|
796
|
+
&& gapBefore(i + span) > 0 && gapBefore(i + span) < pitch(line) * 1.6
|
|
797
|
+
&& !EXPLICIT_HEADING_RES.some(re => re.test(all[i + span]!.text))
|
|
798
|
+
) span++
|
|
799
|
+
let level: number | undefined
|
|
800
|
+
let used = 0
|
|
801
|
+
// Style rules (size / bold) judge the WHOLE same-style run — a slice of
|
|
802
|
+
// a longer run is just its first lines (a 3-line notice in larger type
|
|
803
|
+
// is not a heading plus a paragraph). Explicit signals (outline,
|
|
804
|
+
// numbering) may still claim the first line alone.
|
|
805
|
+
if (span <= 3) {
|
|
806
|
+
level = headingLevel(line, all.slice(i, i + span), stats, isStandalone(i, span), true)
|
|
807
|
+
if (level !== undefined) used = span
|
|
808
|
+
}
|
|
809
|
+
if (level === undefined && span > 1) {
|
|
810
|
+
level = headingLevel(line, [line], stats, isStandalone(i, 1), false)
|
|
811
|
+
if (level !== undefined) used = 1
|
|
812
|
+
}
|
|
813
|
+
if (level !== undefined && line.mono && !stats.monoIsBody) level = undefined
|
|
814
|
+
if (level === RELATIVE_HEADING) level = Math.min(6, Math.max(2, lastLevel + 1))
|
|
815
|
+
else if (level !== undefined) lastLevel = level
|
|
816
|
+
if (level !== undefined) {
|
|
817
|
+
blocks.push({ kind: 'heading', page: line.page, level, lines: all.slice(i, i + used) })
|
|
818
|
+
i += used
|
|
819
|
+
continue
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
const start = i
|
|
823
|
+
const kind = tableEnd.has(i) ? 'table' : lineKind(line, stats)
|
|
824
|
+
const block: Block = { kind: kind === 'code' && isInlineMono(i) ? 'text' : kind, page: line.page, lines: [line] }
|
|
825
|
+
i++
|
|
826
|
+
if (kind === 'table') {
|
|
827
|
+
const end = tableEnd.get(start) ?? start + 1
|
|
828
|
+
while (i < end) {
|
|
829
|
+
block.lines.push(all[i]!)
|
|
830
|
+
i++
|
|
831
|
+
}
|
|
832
|
+
blocks.push(block)
|
|
833
|
+
continue
|
|
834
|
+
}
|
|
835
|
+
if (kind === 'code') {
|
|
836
|
+
while (i < all.length && all[i]!.mono && gapBefore(i) > 0 && gapBefore(i) < pitch(all[i]!) * 2.2) {
|
|
837
|
+
block.lines.push(all[i]!)
|
|
838
|
+
i++
|
|
839
|
+
}
|
|
840
|
+
blocks.push(block)
|
|
841
|
+
continue
|
|
842
|
+
}
|
|
843
|
+
let blockRight = rightEdge.get(line) ?? line.x1
|
|
844
|
+
while (i < all.length) {
|
|
845
|
+
const next = all[i]!
|
|
846
|
+
const prev = block.lines[block.lines.length - 1]!
|
|
847
|
+
if (inTable.has(i)) break
|
|
848
|
+
const nextKind = lineKind(next, stats)
|
|
849
|
+
if (nextKind !== 'text' && !(nextKind === 'code' && isInlineMono(i))) break
|
|
850
|
+
if (Math.abs(next.size - prev.size) > Math.max(next.size, prev.size) * 0.12) break
|
|
851
|
+
const dy = next.page === prev.page ? prev.y - next.y : -Infinity
|
|
852
|
+
const sameFlow = dy > 0 && dy < pitch(next) * 1.45
|
|
853
|
+
if (!sameFlow) {
|
|
854
|
+
// A page or column change: continue only an unfinished sentence.
|
|
855
|
+
if (dy > 0) break // a vertical gap inside the column
|
|
856
|
+
if (TERMINAL_RE.test(prev.text)) break
|
|
857
|
+
if (headingStartLike(next)) break
|
|
858
|
+
if (next.page !== prev.page && block.kind !== 'text') break
|
|
859
|
+
} else {
|
|
860
|
+
// A table-of-contents entry is a line of its own.
|
|
861
|
+
if (TOC_ENTRY_RE.test(prev.text)) break
|
|
862
|
+
// First-line indent opens a paragraph: a full-measure line starting
|
|
863
|
+
// further right after a line that finished (centered lines are inset
|
|
864
|
+
// on both sides; a hanging-indent continuation follows an unfinished
|
|
865
|
+
// line).
|
|
866
|
+
if (next.x0 - prev.x0 > next.size * 0.8 && block.kind === 'text' && next.x1 >= blockRight - next.size * 2
|
|
867
|
+
&& (TERMINAL_RE.test(prev.text) || prev.x1 < blockRight - prev.size * 2)) break
|
|
868
|
+
// List items continue only on their hanging indent.
|
|
869
|
+
if (block.kind !== 'text' && next.x0 < block.lines[0]!.x0 + next.size * 0.3 && TERMINAL_RE.test(prev.text)) break
|
|
870
|
+
// A short line that ends a sentence closes its paragraph.
|
|
871
|
+
if (prev.x1 < blockRight - prev.size * 2.5 && SENTENCE_END_RE.test(prev.text)) break
|
|
872
|
+
if (next.bold !== prev.bold && next.bold && next.chars < 100 && SENTENCE_END_RE.test(prev.text)) break
|
|
873
|
+
}
|
|
874
|
+
block.lines.push(next)
|
|
875
|
+
blockRight = Math.max(blockRight, rightEdge.get(next) ?? next.x1)
|
|
876
|
+
i++
|
|
877
|
+
}
|
|
878
|
+
blocks.push(block)
|
|
879
|
+
}
|
|
880
|
+
return blocks
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
/**
|
|
884
|
+
* Whether table-candidate rows share column starts: at least 60% of the rows
|
|
885
|
+
* have two or more cell left edges within one em of another row's.
|
|
886
|
+
*/
|
|
887
|
+
function columnsAlign(rows: Line[]): boolean {
|
|
888
|
+
// Prose with a right-hand field label (forms: "Enter the amount . . . 3b $")
|
|
889
|
+
// has two "columns", the first of them sentences — not a table.
|
|
890
|
+
const firstCells = rows.map(row => row.cells.find(cell => cell !== '') ?? '').map(cell => cell.length).sort((a, b) => a - b)
|
|
891
|
+
const maxCells = Math.max(...rows.map(row => row.cells.filter(cell => cell !== '').length))
|
|
892
|
+
if (maxCells <= 2 && (firstCells[Math.floor(firstCells.length / 2)] ?? 0) > 50) return false
|
|
893
|
+
const aligned = rows.filter(row => rowAligned(row, rows.filter(other => other !== row))).length
|
|
894
|
+
return aligned / rows.length >= 0.6
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
/** Whether a row shares at least two column starts with some other row. */
|
|
898
|
+
function rowAligned(row: Line, others: Line[]): boolean {
|
|
899
|
+
return others.some(other => row.cellX.filter(x => other.cellX.some(o => Math.abs(o - x) < row.size)).length >= 2)
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
/** A line that looks like the start of a new structural element. */
|
|
903
|
+
function headingStartLike(line: Line): boolean {
|
|
904
|
+
return BULLET_RE.test(line.text) || ORDERED_RE.test(line.text) || CN_CHAPTER_RE.test(line.text)
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
/** Classify a non-heading line. */
|
|
908
|
+
function lineKind(line: Line, stats: LayoutStats): BlockKind {
|
|
909
|
+
if (line.mono && !stats.monoIsBody) return 'code'
|
|
910
|
+
if (BULLET_RE.test(line.text) && line.text.replace(BULLET_RE, '').trim() !== '') return 'bullet'
|
|
911
|
+
if (DASH_BULLET_RE.test(line.text)) return 'bullet'
|
|
912
|
+
if (ORDERED_RE.test(line.text)) return 'ordered'
|
|
913
|
+
if (PAREN_ITEM_RE.test(line.text)) return 'item'
|
|
914
|
+
return 'text'
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
/** Words seen in the document (lowercased), for de-hyphenation decisions. */
|
|
918
|
+
function vocabularyOf(lines: Line[]): Set<string> {
|
|
919
|
+
const words = new Set<string>()
|
|
920
|
+
for (const line of lines) {
|
|
921
|
+
for (const match of line.text.matchAll(/[A-Za-z]+(?:-[A-Za-z]+)*/g)) {
|
|
922
|
+
const word = match[0].toLowerCase()
|
|
923
|
+
words.add(word)
|
|
924
|
+
if (word.includes('-')) words.add(word.replace(/-/g, '\u0000'))
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
return words
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
/** Join two consecutive lines of one paragraph. */
|
|
931
|
+
function joinText(acc: string, next: string, vocabulary: Set<string>): string {
|
|
932
|
+
if (acc === '') return next
|
|
933
|
+
const hyphen = /([A-Za-z]+)[-\u2010\u00AD]$/.exec(acc)
|
|
934
|
+
const lead = /^([a-z]+)/.exec(next)
|
|
935
|
+
if (hyphen !== null && lead !== null && !/\s[-\u2010\u00AD]$/.test(acc)) {
|
|
936
|
+
const head = hyphen[1]!.toLowerCase()
|
|
937
|
+
const tail = lead[1]!
|
|
938
|
+
if (vocabulary.has(head + tail)) return acc.slice(0, -1) + next
|
|
939
|
+
if (vocabulary.has(`${head}\u0000${tail}`)) return `${acc.slice(0, -1)}-${next}`
|
|
940
|
+
return acc.slice(0, -1) + next
|
|
941
|
+
}
|
|
942
|
+
const last = acc[acc.length - 1]
|
|
943
|
+
const first = next[0]
|
|
944
|
+
if (isCjk(last) || isCjk(first)) {
|
|
945
|
+
// Latin word followed by CJK across a break: no space either, the
|
|
946
|
+
// original line break carried no space.
|
|
947
|
+
return acc + next
|
|
948
|
+
}
|
|
949
|
+
if (/[-/]$/.test(acc)) return acc + next
|
|
950
|
+
return `${acc} ${next}`
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
/** Escape what would otherwise read as markdown structure at a block start. */
|
|
954
|
+
function escapeLeading(text: string): string {
|
|
955
|
+
return text.replace(/^(#{1,6}\s|>|[-+*]\s|\d{1,9}[.)]\s)/, '\\$1')
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
/** Render a line with inline bold (when only part of it is bold). */
|
|
959
|
+
function renderLine(line: Line, inlineBold: boolean): string {
|
|
960
|
+
if (!inlineBold || !line.segments.some(s => s.bold) || line.segments.every(s => s.bold)) return line.text
|
|
961
|
+
let out = ''
|
|
962
|
+
for (const segment of line.segments) {
|
|
963
|
+
const text = segment.text
|
|
964
|
+
const core = text.trim()
|
|
965
|
+
if (segment.bold && core.replace(/[^\p{L}\p{N}]/gu, '').length >= 2 && !/^\*|\*$/.test(core)) {
|
|
966
|
+
const lead = text.slice(0, text.length - text.trimStart().length)
|
|
967
|
+
const trail = text.slice(text.trimEnd().length)
|
|
968
|
+
out += `${lead}**${core}**${trail}`
|
|
969
|
+
} else {
|
|
970
|
+
out += text
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
return out.replace(/\s+/g, ' ').trim()
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
/**
|
|
977
|
+
* Merge bold spans split only by whitespace or punctuation set in the
|
|
978
|
+
* regular face ("**一是…会议**,**推动会议** **精神**" → one span): the
|
|
979
|
+
* separator moves inside, and a line-break space between CJK spans goes.
|
|
980
|
+
*/
|
|
981
|
+
function mergeBoldRuns(text: string): string {
|
|
982
|
+
return text.replace(/\*\*(\s*)([,、,;;::]?)(\s*)\*\*/g, (match: string, a: string, punct: string, b: string, offset: number, whole: string) => {
|
|
983
|
+
const before = whole[offset - 1]
|
|
984
|
+
const after = whole[offset + match.length]
|
|
985
|
+
const space = (a !== '' || b !== '') && !(isCjk(before) || isCjk(after) || isCjk(punct)) ? ' ' : ''
|
|
986
|
+
return punct === '' ? space : `${punct}${b !== '' && !isCjk(punct) ? ' ' : ''}`
|
|
987
|
+
})
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
/** Render a table block as GFM when its rows line up, else as plain rows. */
|
|
991
|
+
function renderTable(block: Block): string {
|
|
992
|
+
const rows = block.lines.map(line => line.cells.filter(cell => cell !== '').map(cell => cell.replace(/\|/g, '\\|')))
|
|
993
|
+
const width = Math.max(...rows.map(row => row.length))
|
|
994
|
+
const aligned = rows.filter(row => row.length === width).length
|
|
995
|
+
if (rows.length >= 2 && width >= 2 && aligned / rows.length >= 0.6) {
|
|
996
|
+
const pad = (row: string[]): string => `| ${[...row, ...Array<string>(width - row.length).fill('')].slice(0, width).join(' | ')} |`
|
|
997
|
+
return [pad(rows[0]!), `| ${Array<string>(width).fill('---').join(' | ')} |`, ...rows.slice(1).map(pad)].join('\n')
|
|
998
|
+
}
|
|
999
|
+
return block.lines.map(line => escapeLeading(line.cells.join(' '))).join(' \n')
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/** Whether a metadata title is worth printing as the document heading. */
|
|
1003
|
+
function plausibleTitle(title: string | undefined): title is string {
|
|
1004
|
+
if (title === undefined) return false
|
|
1005
|
+
const t = title.trim()
|
|
1006
|
+
if (t.length < 3 || t.length > 250) return false
|
|
1007
|
+
if (/^(?:microsoft\s+(?:word|powerpoint|excel)\b|untitled|slide\s*\d|document\d*$|title$|no title|无标题)/i.test(t)) return false
|
|
1008
|
+
if (/\.(?:docx?|pdf|pptx?|xlsx?|tex|dvi|indd|qxd|rtf|odt)$/i.test(t)) return false
|
|
1009
|
+
if (/^[\w-]+$/.test(t) && !/\s/.test(t) && /\d/.test(t)) return false // file-name-like ids
|
|
1010
|
+
return true
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/**
|
|
1014
|
+
* Lay a PDF's positioned text out as markdown (see the module docs for the
|
|
1015
|
+
* heuristics). Pure — no pdf.js — so the rules are unit-testable with
|
|
1016
|
+
* synthetic runs.
|
|
1017
|
+
*/
|
|
1018
|
+
export function layoutPdfMarkdown(doc: PdfDocumentText): string {
|
|
1019
|
+
const pages = removeMarginNoise(doc.pages.map(buildLines))
|
|
1020
|
+
const all = pages.flat()
|
|
1021
|
+
if (all.length === 0) return plausibleTitle(doc.title) ? `# ${doc.title}\n` : ''
|
|
1022
|
+
const body = bodySizeOf(all)
|
|
1023
|
+
const totalChars = all.reduce((s, l) => s + l.chars, 0)
|
|
1024
|
+
const monoChars = all.reduce((s, l) => s + (l.mono ? l.chars : 0), 0)
|
|
1025
|
+
const monoIsBody = monoChars > totalChars * 0.5
|
|
1026
|
+
|
|
1027
|
+
// Heading sizes: larger-than-body sizes that only occur on short line runs.
|
|
1028
|
+
const sizeChars = new Map<number, number>()
|
|
1029
|
+
for (const line of all) {
|
|
1030
|
+
if (line.size < body * 1.12) continue
|
|
1031
|
+
const key = Math.round(line.size * 2) / 2
|
|
1032
|
+
sizeChars.set(key, (sizeChars.get(key) ?? 0) + line.chars)
|
|
1033
|
+
}
|
|
1034
|
+
let headingSizes = [...sizeChars.keys()].filter(size => (sizeChars.get(size) ?? 0) < Math.max(totalChars * 0.25, 400)).sort((a, b) => b - a)
|
|
1035
|
+
// The document title: the largest type on the first page, when it is
|
|
1036
|
+
// clearly display-sized and used for at most a few lines there.
|
|
1037
|
+
let titleSize: number | undefined
|
|
1038
|
+
const firstPage = doc.pages[0]?.number ?? 1
|
|
1039
|
+
const firstPageLines = all.filter(line => line.page === firstPage && /\p{L}{2}/u.test(line.text))
|
|
1040
|
+
const firstLargest = firstPageLines.reduce((m, line) => Math.max(m, Math.round(line.size * 2) / 2), 0)
|
|
1041
|
+
if (firstLargest >= body * 1.3) {
|
|
1042
|
+
const uses = firstPageLines.filter(line => Math.abs(Math.round(line.size * 2) / 2 - firstLargest) < 0.3)
|
|
1043
|
+
if (uses.length <= 3) {
|
|
1044
|
+
titleSize = firstLargest
|
|
1045
|
+
headingSizes = headingSizes.filter(size => size < firstLargest - 0.25)
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
headingSizes = headingSizes.slice(0, 3)
|
|
1049
|
+
const outlineKeys = new Map<string, number>()
|
|
1050
|
+
for (const entry of doc.outline) {
|
|
1051
|
+
for (const key of [headingKey(entry.title), headingKey(stripNumbering(entry.title))]) {
|
|
1052
|
+
if (key.length >= 2 && !outlineKeys.has(key)) outlineKeys.set(key, entry.depth)
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
const roots = doc.outline.filter(entry => entry.depth === 0)
|
|
1056
|
+
const titleKeys = new Set([doc.title ?? '', ...all.filter(line => titleSize !== undefined && Math.abs(line.size - titleSize) < 0.3).map(line => line.text)].map(headingKey))
|
|
1057
|
+
const outlineOffset = roots.length === 1 && titleKeys.has(headingKey(roots[0]!.title)) ? 1 : 0
|
|
1058
|
+
const stats: LayoutStats = { body, spacing: lineSpacingOf(pages, body), monoIsBody, headingSizes, ...(titleSize !== undefined ? { titleSize } : {}), outlineKeys, outlineOffset }
|
|
1059
|
+
const blocks = buildBlocks(pages, stats)
|
|
1060
|
+
const vocabulary = vocabularyOf(all)
|
|
1061
|
+
|
|
1062
|
+
const out: string[] = []
|
|
1063
|
+
const metaKey = headingKey(doc.title ?? '')
|
|
1064
|
+
const hasTitleHeading = blocks.slice(0, 12).some(block => block.kind === 'heading' && (block.level === 1 || headingKey(block.lines.map(l => l.text).join('')) === metaKey))
|
|
1065
|
+
if (!hasTitleHeading && plausibleTitle(doc.title)) out.push(`# ${doc.title.replace(/\s+/g, ' ').trim()}`)
|
|
1066
|
+
let lastPage = 0
|
|
1067
|
+
for (const block of blocks) {
|
|
1068
|
+
if (block.page !== lastPage) {
|
|
1069
|
+
out.push(`<!-- page ${String(block.page)} -->`)
|
|
1070
|
+
lastPage = block.page
|
|
1071
|
+
}
|
|
1072
|
+
switch (block.kind) {
|
|
1073
|
+
case 'heading': {
|
|
1074
|
+
const text = block.lines.map(l => l.text).reduce((acc, t) => joinText(acc, t, vocabulary), '')
|
|
1075
|
+
out.push(`${'#'.repeat(block.level ?? 2)} ${text}`)
|
|
1076
|
+
break
|
|
1077
|
+
}
|
|
1078
|
+
case 'code': {
|
|
1079
|
+
const left = Math.min(...block.lines.map(l => l.x0))
|
|
1080
|
+
// Monospace: one character's advance, measured on the block itself.
|
|
1081
|
+
const advances = block.lines.filter(l => l.text.length >= 4).map(l => (l.x1 - l.x0) / l.text.length).sort((p, q) => p - q)
|
|
1082
|
+
const advance = advances[Math.floor(advances.length / 2)] ?? block.lines[0]!.size * 0.6
|
|
1083
|
+
const body = block.lines.map((l) => {
|
|
1084
|
+
const indent = Math.max(0, Math.round((l.x0 - left) / advance))
|
|
1085
|
+
return ' '.repeat(Math.min(indent, 40)) + l.text
|
|
1086
|
+
}).join('\n')
|
|
1087
|
+
out.push(`\`\`\`\n${body.replace(/```/g, '```')}\n\`\`\``)
|
|
1088
|
+
break
|
|
1089
|
+
}
|
|
1090
|
+
case 'table':
|
|
1091
|
+
out.push(renderTable(block))
|
|
1092
|
+
break
|
|
1093
|
+
default: {
|
|
1094
|
+
let text = ''
|
|
1095
|
+
for (const line of block.lines) text = joinText(text, renderLine(line, true), vocabulary)
|
|
1096
|
+
text = mergeBoldRuns(text)
|
|
1097
|
+
if (block.kind === 'bullet') {
|
|
1098
|
+
text = text.replace(BULLET_RE, '').replace(DASH_BULLET_RE, '')
|
|
1099
|
+
// A bullet list stays one list: no blank line between its items.
|
|
1100
|
+
const prev = out[out.length - 1]
|
|
1101
|
+
if (prev?.startsWith('- ') === true) out[out.length - 1] = `${prev}\n- ${text}`
|
|
1102
|
+
else out.push(`- ${text}`)
|
|
1103
|
+
} else if (block.kind === 'ordered') {
|
|
1104
|
+
const prev = out[out.length - 1]
|
|
1105
|
+
if (prev !== undefined && /^\d{1,3}\.\s/.test(prev)) out[out.length - 1] = `${prev}\n${text.replace(/^(\d{1,3})\)/, '$1.')}`
|
|
1106
|
+
else out.push(text.replace(/^(\d{1,3})\)/, '$1.'))
|
|
1107
|
+
} else {
|
|
1108
|
+
out.push(escapeLeading(text))
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
return `${out.join('\n\n')}\n`
|
|
1114
|
+
}
|