dsh-web-fetch-playwright 0.2.18 → 0.2.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -25,6 +25,8 @@ export { deepCaptureWalk } from './autoscroll.ts';
25
25
  export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts';
26
26
  export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts';
27
27
  export { htmlToMarkdown } from './markdown.ts';
28
+ export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts';
29
+ export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts';
28
30
  export type { DenoiseMode, DenoiseResult } from './markdown.ts';
29
31
  /** Cordis plugin name used by loader diagnostics. */
30
32
  export declare const name = "dsh-web-fetch-playwright";
@@ -0,0 +1,143 @@
1
+ /**
2
+ * PDF → markdown. A fetched URL that turns out to be a PDF (served inline as
3
+ * `application/pdf`, as a `Content-Disposition: attachment` download, or as
4
+ * an anonymous `application/octet-stream` whose bytes start with `%PDF-`) is
5
+ * routed here instead of failing with `WEB_UNSUPPORTED_CONTENT_TYPE`.
6
+ *
7
+ * Two layers, kept apart so the heuristics are testable without pdf.js:
8
+ *
9
+ * 1. {@link readPdfDocument} — the pdf.js boundary (through `unpdf`'s
10
+ * serverless build, loaded lazily so plugin start never pays for it).
11
+ * Per page it reads the positioned text runs (`getTextContent`) plus the
12
+ * real font names (`getOperatorList` + `fontExtraProperties`, the only way
13
+ * pdf.js exposes "NimbusRomNo9L-Medi" instead of a generic family), so the
14
+ * layout pass can see bold and monospace. Bounded by the fetch deadline,
15
+ * the caller's abort signal, a page cap, and a collected-character cap.
16
+ * 2. {@link layoutPdfMarkdown} — pure heuristics over those runs:
17
+ * - runs → lines (baseline grouping in content-stream order, x-sorted
18
+ * inside a line; rotated runs such as arXiv's side stamp are dropped);
19
+ * - running headers/footers (text repeated in the page margins across
20
+ * pages, digits normalized) and bare page numbers are removed;
21
+ * - the dominant (body) font size and line pitch are measured;
22
+ * - headings come from, strongest first: the PDF outline (bookmarks),
23
+ * section numbering ("3.1 Pre-training", "第二章", "一、"), larger font
24
+ * sizes (ranked into levels), and standalone bold lines;
25
+ * - lines are merged into paragraphs (vertical gap, first-line indent,
26
+ * short terminal line, font change; continuation across columns and
27
+ * pages when the sentence is unfinished), with dictionary-checked
28
+ * de-hyphenation and CJK-aware joining;
29
+ * - bullets / numbered items become markdown lists, monospace blocks
30
+ * become fenced code (unless the whole document is monospace), lines
31
+ * with wide column gaps become GFM tables when they line up;
32
+ * - `<!-- page N -->` markers keep page provenance for citations.
33
+ *
34
+ * Scanned PDFs without a text layer, encrypted PDFs, and broken files fail
35
+ * with dedicated, provider-specific error codes (OCR is out of scope).
36
+ *
37
+ * @module dsh-web-fetch-playwright/pdf
38
+ */
39
+ /** Largest PDF this provider downloads and parses (bytes). */
40
+ export declare const MAX_PDF_BYTES: number;
41
+ /** Most pages read from one document; later pages are reported as truncated. */
42
+ export declare const MAX_PDF_PAGES = 500;
43
+ /** No text layer (a scanned / image-only PDF); OCR is not supported. */
44
+ export declare const WEB_PDF_NO_TEXT_CODE = "WEB_PDF_NO_TEXT";
45
+ /** Password-protected PDF the parser cannot open. */
46
+ export declare const WEB_PDF_ENCRYPTED_CODE = "WEB_PDF_ENCRYPTED";
47
+ /** Bytes claimed (or sniffed) to be a PDF that pdf.js cannot parse. */
48
+ export declare const WEB_PDF_INVALID_CODE = "WEB_PDF_INVALID";
49
+ /** PDF larger than {@link MAX_PDF_BYTES}. */
50
+ export declare const WEB_PDF_TOO_LARGE_CODE = "WEB_PDF_TOO_LARGE";
51
+ /** Whether a `Content-Type` names a PDF. */
52
+ export declare function isPdfContentType(contentType: string | undefined): boolean;
53
+ /**
54
+ * Whether a response that is not otherwise decodable is worth downloading
55
+ * to sniff for PDF bytes: a PDF content type, a `.pdf` attachment filename,
56
+ * a `.pdf` URL path, or an opaque binary type (servers commonly hand PDFs out
57
+ * as `application/octet-stream`). The magic-byte check then decides.
58
+ */
59
+ export declare function mayBePdf(headers: Record<string, string>, url: string): boolean;
60
+ /**
61
+ * Whether the bytes are a PDF: `%PDF-` within the first 1024 bytes (the
62
+ * spec-tolerated leading-garbage window every mainstream reader accepts).
63
+ */
64
+ export declare function isPdfBytes(bytes: Uint8Array): boolean;
65
+ /** One positioned, horizontal text run (pdf.js text item, normalized). */
66
+ export interface PdfTextRun {
67
+ text: string;
68
+ /** Left edge in PDF user space. */
69
+ x: number;
70
+ /** Baseline in PDF user space (origin bottom-left). */
71
+ y: number;
72
+ width: number;
73
+ /** Font size (vertical scale of the text matrix). */
74
+ size: number;
75
+ bold: boolean;
76
+ mono: boolean;
77
+ }
78
+ /** One page's text runs in content-stream order. */
79
+ export interface PdfPageText {
80
+ /** 1-based page number. */
81
+ number: number;
82
+ /** Page box bottom edge (user space). */
83
+ bottom: number;
84
+ /** Page box height (user space). */
85
+ height: number;
86
+ runs: PdfTextRun[];
87
+ }
88
+ /** One bookmark, flattened with its nesting depth (0 = top level). */
89
+ export interface PdfOutlineEntry {
90
+ title: string;
91
+ depth: number;
92
+ }
93
+ /** Everything the layout pass consumes. */
94
+ export interface PdfDocumentText {
95
+ /** Document info / XMP title, when present. */
96
+ title?: string;
97
+ outline: PdfOutlineEntry[];
98
+ pages: PdfPageText[];
99
+ totalPages: number;
100
+ }
101
+ /** The finished extraction. */
102
+ export interface PdfExtraction {
103
+ markdown: string;
104
+ totalPages: number;
105
+ /** Pages actually read (fewer than `totalPages` when a limit cut in). */
106
+ pagesRead: number;
107
+ /** True when a page, size, or time limit stopped the extraction early. */
108
+ truncated: boolean;
109
+ }
110
+ /** Limits one extraction runs under. */
111
+ export interface PdfExtractOptions {
112
+ /** Abort → stop between pages and fail. */
113
+ signal?: AbortSignal;
114
+ /** Wall-clock budget for reading pages (ms); pages left over are truncated. */
115
+ timeBudgetMs?: number;
116
+ /** Stop reading pages once this many characters were collected. */
117
+ maxChars?: number;
118
+ /** Page cap (defaults to {@link MAX_PDF_PAGES}). */
119
+ maxPages?: number;
120
+ }
121
+ /**
122
+ * Read a PDF's positioned text with pdf.js, page by page, inside the given
123
+ * limits. Text items become {@link PdfTextRun}s (horizontal only — rotated
124
+ * stamps and vertical margin text are dropped as layout noise).
125
+ *
126
+ * @returns the text model, plus whether a limit stopped reading early.
127
+ */
128
+ export declare function readPdfDocument(bytes: Uint8Array, options?: PdfExtractOptions): Promise<{
129
+ doc: PdfDocumentText;
130
+ truncated: boolean;
131
+ }>;
132
+ /**
133
+ * Extract a PDF as markdown: read it with pdf.js inside the limits, lay it
134
+ * out, and fail with {@link WEB_PDF_NO_TEXT_CODE} when there is no text layer
135
+ * to speak of (a scan). Sizes beyond {@link MAX_PDF_BYTES} are refused.
136
+ */
137
+ export declare function extractPdfMarkdown(bytes: Uint8Array, options?: PdfExtractOptions): Promise<PdfExtraction>;
138
+ /**
139
+ * Lay a PDF's positioned text out as markdown (see the module docs for the
140
+ * heuristics). Pure — no pdf.js — so the rules are unit-testable with
141
+ * synthetic runs.
142
+ */
143
+ export declare function layoutPdfMarkdown(doc: PdfDocumentText): string;
@@ -33,6 +33,15 @@
33
33
  * {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
34
34
  * a queue and fail fast (rather than hang until abort) when no slot frees.
35
35
  *
36
+ * PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
37
+ * handed to the download manager as an attachment, or served as an opaque
38
+ * `application/octet-stream` whose bytes start with `%PDF-` — is read as
39
+ * bytes (the navigation body, else a refetch through the page's API request
40
+ * context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
41
+ * plus layout heuristics: headings, paragraphs, lists, tables, page
42
+ * markers). Scans without a text layer, encrypted, oversized, and broken
43
+ * PDFs fail with the `WEB_PDF_*` codes.
44
+ *
36
45
  * Private-network and SSRF protection is not implemented (same stance as the
37
46
  * shipped HTTP provider); a page this provider can reach is whatever the
38
47
  * browser can reach. Profile mode additionally acts WITH the remote
@@ -12,6 +12,11 @@ export interface PlaywrightResponse {
12
12
  status(): number;
13
13
  headers(): Record<string, string>;
14
14
  text(): Promise<string>;
15
+ /**
16
+ * The raw body bytes — the PDF path reads binary bodies through it (a
17
+ * Node `Buffer` is a `Uint8Array`). Absent on minimal fakes.
18
+ */
19
+ body?(): Promise<Uint8Array>;
15
20
  /** This response's URL; absent on minimal fakes. */
16
21
  url?(): string;
17
22
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.18",
3
+ "version": "0.2.19",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -93,7 +93,8 @@
93
93
  "dompurify": "^3.2.4",
94
94
  "jsdom": "^26.0.0",
95
95
  "playwright-core": "^1.65.0",
96
- "turndown": "^7.2.0"
96
+ "turndown": "^7.2.0",
97
+ "unpdf": "^2.1.0"
97
98
  },
98
99
  "peerDependencies": {
99
100
  "@deepseek-ai/cordis": ">=4.0.1-rc.1 <5",
package/src/index.ts CHANGED
@@ -52,6 +52,8 @@ export { deepCaptureWalk } from './autoscroll.ts'
52
52
  export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts'
53
53
  export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
54
54
  export { htmlToMarkdown } from './markdown.ts'
55
+ export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
56
+ export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts'
55
57
  export type { DenoiseMode, DenoiseResult } from './markdown.ts'
56
58
 
57
59
  /** Cordis plugin name used by loader diagnostics. */