dsh-web-fetch-playwright 0.2.18 → 0.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/README.zh-CN.md +17 -1
- package/lib/client.js +18 -18
- package/lib/index.js +1004 -5
- package/lib/types/index.d.ts +2 -0
- package/lib/types/pdf.d.ts +143 -0
- package/lib/types/provider.d.ts +9 -0
- package/lib/types/types.d.ts +5 -0
- package/package.json +3 -2
- package/src/index.ts +2 -0
- package/src/pdf.ts +1114 -0
- package/src/provider.ts +117 -8
- package/src/session-window.ts +8 -0
- package/src/types.ts +5 -0
package/lib/types/index.d.ts
CHANGED
|
@@ -25,6 +25,8 @@ export { deepCaptureWalk } from './autoscroll.ts';
|
|
|
25
25
|
export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts';
|
|
26
26
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts';
|
|
27
27
|
export { htmlToMarkdown } from './markdown.ts';
|
|
28
|
+
export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts';
|
|
29
|
+
export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts';
|
|
28
30
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts';
|
|
29
31
|
/** Cordis plugin name used by loader diagnostics. */
|
|
30
32
|
export declare const name = "dsh-web-fetch-playwright";
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PDF → markdown. A fetched URL that turns out to be a PDF (served inline as
|
|
3
|
+
* `application/pdf`, as a `Content-Disposition: attachment` download, or as
|
|
4
|
+
* an anonymous `application/octet-stream` whose bytes start with `%PDF-`) is
|
|
5
|
+
* routed here instead of failing with `WEB_UNSUPPORTED_CONTENT_TYPE`.
|
|
6
|
+
*
|
|
7
|
+
* Two layers, kept apart so the heuristics are testable without pdf.js:
|
|
8
|
+
*
|
|
9
|
+
* 1. {@link readPdfDocument} — the pdf.js boundary (through `unpdf`'s
|
|
10
|
+
* serverless build, loaded lazily so plugin start never pays for it).
|
|
11
|
+
* Per page it reads the positioned text runs (`getTextContent`) plus the
|
|
12
|
+
* real font names (`getOperatorList` + `fontExtraProperties`, the only way
|
|
13
|
+
* pdf.js exposes "NimbusRomNo9L-Medi" instead of a generic family), so the
|
|
14
|
+
* layout pass can see bold and monospace. Bounded by the fetch deadline,
|
|
15
|
+
* the caller's abort signal, a page cap, and a collected-character cap.
|
|
16
|
+
* 2. {@link layoutPdfMarkdown} — pure heuristics over those runs:
|
|
17
|
+
* - runs → lines (baseline grouping in content-stream order, x-sorted
|
|
18
|
+
* inside a line; rotated runs such as arXiv's side stamp are dropped);
|
|
19
|
+
* - running headers/footers (text repeated in the page margins across
|
|
20
|
+
* pages, digits normalized) and bare page numbers are removed;
|
|
21
|
+
* - the dominant (body) font size and line pitch are measured;
|
|
22
|
+
* - headings come from, strongest first: the PDF outline (bookmarks),
|
|
23
|
+
* section numbering ("3.1 Pre-training", "第二章", "一、"), larger font
|
|
24
|
+
* sizes (ranked into levels), and standalone bold lines;
|
|
25
|
+
* - lines are merged into paragraphs (vertical gap, first-line indent,
|
|
26
|
+
* short terminal line, font change; continuation across columns and
|
|
27
|
+
* pages when the sentence is unfinished), with dictionary-checked
|
|
28
|
+
* de-hyphenation and CJK-aware joining;
|
|
29
|
+
* - bullets / numbered items become markdown lists, monospace blocks
|
|
30
|
+
* become fenced code (unless the whole document is monospace), lines
|
|
31
|
+
* with wide column gaps become GFM tables when they line up;
|
|
32
|
+
* - `<!-- page N -->` markers keep page provenance for citations.
|
|
33
|
+
*
|
|
34
|
+
* Scanned PDFs without a text layer, encrypted PDFs, and broken files fail
|
|
35
|
+
* with dedicated, provider-specific error codes (OCR is out of scope).
|
|
36
|
+
*
|
|
37
|
+
* @module dsh-web-fetch-playwright/pdf
|
|
38
|
+
*/
|
|
39
|
+
/** Largest PDF this provider downloads and parses (bytes). */
|
|
40
|
+
export declare const MAX_PDF_BYTES: number;
|
|
41
|
+
/** Most pages read from one document; later pages are reported as truncated. */
|
|
42
|
+
export declare const MAX_PDF_PAGES = 500;
|
|
43
|
+
/** No text layer (a scanned / image-only PDF); OCR is not supported. */
|
|
44
|
+
export declare const WEB_PDF_NO_TEXT_CODE = "WEB_PDF_NO_TEXT";
|
|
45
|
+
/** Password-protected PDF the parser cannot open. */
|
|
46
|
+
export declare const WEB_PDF_ENCRYPTED_CODE = "WEB_PDF_ENCRYPTED";
|
|
47
|
+
/** Bytes claimed (or sniffed) to be a PDF that pdf.js cannot parse. */
|
|
48
|
+
export declare const WEB_PDF_INVALID_CODE = "WEB_PDF_INVALID";
|
|
49
|
+
/** PDF larger than {@link MAX_PDF_BYTES}. */
|
|
50
|
+
export declare const WEB_PDF_TOO_LARGE_CODE = "WEB_PDF_TOO_LARGE";
|
|
51
|
+
/** Whether a `Content-Type` names a PDF. */
|
|
52
|
+
export declare function isPdfContentType(contentType: string | undefined): boolean;
|
|
53
|
+
/**
|
|
54
|
+
* Whether a response that is not otherwise decodable is worth downloading
|
|
55
|
+
* to sniff for PDF bytes: a PDF content type, a `.pdf` attachment filename,
|
|
56
|
+
* a `.pdf` URL path, or an opaque binary type (servers commonly hand PDFs out
|
|
57
|
+
* as `application/octet-stream`). The magic-byte check then decides.
|
|
58
|
+
*/
|
|
59
|
+
export declare function mayBePdf(headers: Record<string, string>, url: string): boolean;
|
|
60
|
+
/**
|
|
61
|
+
* Whether the bytes are a PDF: `%PDF-` within the first 1024 bytes (the
|
|
62
|
+
* spec-tolerated leading-garbage window every mainstream reader accepts).
|
|
63
|
+
*/
|
|
64
|
+
export declare function isPdfBytes(bytes: Uint8Array): boolean;
|
|
65
|
+
/** One positioned, horizontal text run (pdf.js text item, normalized). */
|
|
66
|
+
export interface PdfTextRun {
|
|
67
|
+
text: string;
|
|
68
|
+
/** Left edge in PDF user space. */
|
|
69
|
+
x: number;
|
|
70
|
+
/** Baseline in PDF user space (origin bottom-left). */
|
|
71
|
+
y: number;
|
|
72
|
+
width: number;
|
|
73
|
+
/** Font size (vertical scale of the text matrix). */
|
|
74
|
+
size: number;
|
|
75
|
+
bold: boolean;
|
|
76
|
+
mono: boolean;
|
|
77
|
+
}
|
|
78
|
+
/** One page's text runs in content-stream order. */
|
|
79
|
+
export interface PdfPageText {
|
|
80
|
+
/** 1-based page number. */
|
|
81
|
+
number: number;
|
|
82
|
+
/** Page box bottom edge (user space). */
|
|
83
|
+
bottom: number;
|
|
84
|
+
/** Page box height (user space). */
|
|
85
|
+
height: number;
|
|
86
|
+
runs: PdfTextRun[];
|
|
87
|
+
}
|
|
88
|
+
/** One bookmark, flattened with its nesting depth (0 = top level). */
|
|
89
|
+
export interface PdfOutlineEntry {
|
|
90
|
+
title: string;
|
|
91
|
+
depth: number;
|
|
92
|
+
}
|
|
93
|
+
/** Everything the layout pass consumes. */
|
|
94
|
+
export interface PdfDocumentText {
|
|
95
|
+
/** Document info / XMP title, when present. */
|
|
96
|
+
title?: string;
|
|
97
|
+
outline: PdfOutlineEntry[];
|
|
98
|
+
pages: PdfPageText[];
|
|
99
|
+
totalPages: number;
|
|
100
|
+
}
|
|
101
|
+
/** The finished extraction. */
|
|
102
|
+
export interface PdfExtraction {
|
|
103
|
+
markdown: string;
|
|
104
|
+
totalPages: number;
|
|
105
|
+
/** Pages actually read (fewer than `totalPages` when a limit cut in). */
|
|
106
|
+
pagesRead: number;
|
|
107
|
+
/** True when a page, size, or time limit stopped the extraction early. */
|
|
108
|
+
truncated: boolean;
|
|
109
|
+
}
|
|
110
|
+
/** Limits one extraction runs under. */
|
|
111
|
+
export interface PdfExtractOptions {
|
|
112
|
+
/** Abort → stop between pages and fail. */
|
|
113
|
+
signal?: AbortSignal;
|
|
114
|
+
/** Wall-clock budget for reading pages (ms); pages left over are truncated. */
|
|
115
|
+
timeBudgetMs?: number;
|
|
116
|
+
/** Stop reading pages once this many characters were collected. */
|
|
117
|
+
maxChars?: number;
|
|
118
|
+
/** Page cap (defaults to {@link MAX_PDF_PAGES}). */
|
|
119
|
+
maxPages?: number;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Read a PDF's positioned text with pdf.js, page by page, inside the given
|
|
123
|
+
* limits. Text items become {@link PdfTextRun}s (horizontal only — rotated
|
|
124
|
+
* stamps and vertical margin text are dropped as layout noise).
|
|
125
|
+
*
|
|
126
|
+
* @returns the text model, plus whether a limit stopped reading early.
|
|
127
|
+
*/
|
|
128
|
+
export declare function readPdfDocument(bytes: Uint8Array, options?: PdfExtractOptions): Promise<{
|
|
129
|
+
doc: PdfDocumentText;
|
|
130
|
+
truncated: boolean;
|
|
131
|
+
}>;
|
|
132
|
+
/**
|
|
133
|
+
* Extract a PDF as markdown: read it with pdf.js inside the limits, lay it
|
|
134
|
+
* out, and fail with {@link WEB_PDF_NO_TEXT_CODE} when there is no text layer
|
|
135
|
+
* to speak of (a scan). Sizes beyond {@link MAX_PDF_BYTES} are refused.
|
|
136
|
+
*/
|
|
137
|
+
export declare function extractPdfMarkdown(bytes: Uint8Array, options?: PdfExtractOptions): Promise<PdfExtraction>;
|
|
138
|
+
/**
|
|
139
|
+
* Lay a PDF's positioned text out as markdown (see the module docs for the
|
|
140
|
+
* heuristics). Pure — no pdf.js — so the rules are unit-testable with
|
|
141
|
+
* synthetic runs.
|
|
142
|
+
*/
|
|
143
|
+
export declare function layoutPdfMarkdown(doc: PdfDocumentText): string;
|
package/lib/types/provider.d.ts
CHANGED
|
@@ -33,6 +33,15 @@
|
|
|
33
33
|
* {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
|
|
34
34
|
* a queue and fail fast (rather than hang until abort) when no slot frees.
|
|
35
35
|
*
|
|
36
|
+
* PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
|
|
37
|
+
* handed to the download manager as an attachment, or served as an opaque
|
|
38
|
+
* `application/octet-stream` whose bytes start with `%PDF-` — is read as
|
|
39
|
+
* bytes (the navigation body, else a refetch through the page's API request
|
|
40
|
+
* context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
|
|
41
|
+
* plus layout heuristics: headings, paragraphs, lists, tables, page
|
|
42
|
+
* markers). Scans without a text layer, encrypted, oversized, and broken
|
|
43
|
+
* PDFs fail with the `WEB_PDF_*` codes.
|
|
44
|
+
*
|
|
36
45
|
* Private-network and SSRF protection is not implemented (same stance as the
|
|
37
46
|
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
38
47
|
* browser can reach. Profile mode additionally acts WITH the remote
|
package/lib/types/types.d.ts
CHANGED
|
@@ -12,6 +12,11 @@ export interface PlaywrightResponse {
|
|
|
12
12
|
status(): number;
|
|
13
13
|
headers(): Record<string, string>;
|
|
14
14
|
text(): Promise<string>;
|
|
15
|
+
/**
|
|
16
|
+
* The raw body bytes — the PDF path reads binary bodies through it (a
|
|
17
|
+
* Node `Buffer` is a `Uint8Array`). Absent on minimal fakes.
|
|
18
|
+
*/
|
|
19
|
+
body?(): Promise<Uint8Array>;
|
|
15
20
|
/** This response's URL; absent on minimal fakes. */
|
|
16
21
|
url?(): string;
|
|
17
22
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.19",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -93,7 +93,8 @@
|
|
|
93
93
|
"dompurify": "^3.2.4",
|
|
94
94
|
"jsdom": "^26.0.0",
|
|
95
95
|
"playwright-core": "^1.65.0",
|
|
96
|
-
"turndown": "^7.2.0"
|
|
96
|
+
"turndown": "^7.2.0",
|
|
97
|
+
"unpdf": "^2.1.0"
|
|
97
98
|
},
|
|
98
99
|
"peerDependencies": {
|
|
99
100
|
"@deepseek-ai/cordis": ">=4.0.1-rc.1 <5",
|
package/src/index.ts
CHANGED
|
@@ -52,6 +52,8 @@ export { deepCaptureWalk } from './autoscroll.ts'
|
|
|
52
52
|
export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts'
|
|
53
53
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
|
|
54
54
|
export { htmlToMarkdown } from './markdown.ts'
|
|
55
|
+
export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
|
|
56
|
+
export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts'
|
|
55
57
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts'
|
|
56
58
|
|
|
57
59
|
/** Cordis plugin name used by loader diagnostics. */
|