dsh-web-fetch-playwright 0.2.18 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/README.zh-CN.md +17 -1
- package/lib/client.js +18 -18
- package/lib/index.js +58203 -423
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +157 -0
- package/lib/types/index.d.ts +6 -2
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/pdf.d.ts +143 -0
- package/lib/types/provider.d.ts +9 -0
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/lib/types/types.d.ts +5 -0
- package/package.json +3 -2
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +639 -0
- package/src/index.ts +6 -2
- package/src/markdown.ts +136 -413
- package/src/pdf.ts +1114 -0
- package/src/provider.ts +117 -8
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +277 -0
- package/src/session-window.ts +8 -0
- package/src/trees.ts +43 -0
- package/src/types.ts +5 -0
package/src/provider.ts
CHANGED
|
@@ -33,6 +33,15 @@
|
|
|
33
33
|
* {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
|
|
34
34
|
* a queue and fail fast (rather than hang until abort) when no slot frees.
|
|
35
35
|
*
|
|
36
|
+
* PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
|
|
37
|
+
* handed to the download manager as an attachment, or served as an opaque
|
|
38
|
+
* `application/octet-stream` whose bytes start with `%PDF-` — is read as
|
|
39
|
+
* bytes (the navigation body, else a refetch through the page's API request
|
|
40
|
+
* context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
|
|
41
|
+
* plus layout heuristics: headings, paragraphs, lists, tables, page
|
|
42
|
+
* markers). Scans without a text layer, encrypted, oversized, and broken
|
|
43
|
+
* PDFs fail with the `WEB_PDF_*` codes.
|
|
44
|
+
*
|
|
36
45
|
* Private-network and SSRF protection is not implemented (same stance as the
|
|
37
46
|
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
38
47
|
* browser can reach. Profile mode additionally acts WITH the remote
|
|
@@ -87,6 +96,7 @@ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveCh
|
|
|
87
96
|
import type { ResolvedConfig } from './config.ts'
|
|
88
97
|
import { CdpConnectionPool } from './cdp-pool.ts'
|
|
89
98
|
import { htmlToMarkdown } from './markdown.ts'
|
|
99
|
+
import { extractPdfMarkdown, isPdfBytes, isPdfContentType, mayBePdf, MAX_PDF_BYTES, WEB_PDF_INVALID_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
|
|
90
100
|
import { STRIP_SCRIPTING_ELEMENTS } from './strip.ts'
|
|
91
101
|
import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
|
|
92
102
|
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
|
|
@@ -288,12 +298,16 @@ function validateFetchUrl(input: string): URL {
|
|
|
288
298
|
return url
|
|
289
299
|
}
|
|
290
300
|
|
|
291
|
-
/**
|
|
292
|
-
|
|
301
|
+
/**
|
|
302
|
+
* The decodable body kinds: the HTTP provider's classification plus `pdf`,
|
|
303
|
+
* which this provider extracts to markdown (see `src/pdf.ts`).
|
|
304
|
+
*/
|
|
305
|
+
type FetchableKind = 'html' | 'text' | 'pdf'
|
|
293
306
|
|
|
294
307
|
/** Classify a `Content-Type` into a decodable kind; undefined = unsupported. */
|
|
295
308
|
function classifyContentType(contentType: string | undefined): FetchableKind | undefined {
|
|
296
309
|
const mime = (contentType ?? '').replace(/;.*$/s, '').trim().toLowerCase()
|
|
310
|
+
if (isPdfContentType(mime)) return 'pdf'
|
|
297
311
|
if (mime === '' || mime === 'text/html' || mime === 'application/xhtml+xml') return 'html'
|
|
298
312
|
if (mime.startsWith('text/')) return 'text'
|
|
299
313
|
if (mime === 'application/json' || mime === 'application/xml' || mime.endsWith('+json') || mime.endsWith('+xml')) return 'text'
|
|
@@ -543,15 +557,24 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
543
557
|
// ran (the real page reloads in), else the response goto returned.
|
|
544
558
|
const finalResponse = (challengeWaitMs > 0 ? tracker.last() : undefined) ?? response
|
|
545
559
|
const kind = classifyContentType(finalResponse?.headers()['content-type'])
|
|
546
|
-
if (kind === undefined) {
|
|
547
|
-
throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
548
|
-
}
|
|
549
560
|
const finalUrl = page.url()
|
|
550
561
|
// An SPA-style clear swaps the document without navigating: no new
|
|
551
562
|
// response exists to report, so the cleared document reads as served.
|
|
552
563
|
const clearedWithoutNavigation = challengeEntryResponse !== null && finalResponse === challengeEntryResponse
|
|
553
564
|
const statusCode = finalResponse !== null && !clearedWithoutNavigation ? finalResponse.status() : 200
|
|
554
565
|
|
|
566
|
+
// A PDF the browser committed as a document (its built-in viewer — a
|
|
567
|
+
// headed or CDP-attached Chrome) or an opaque binary that may be one:
|
|
568
|
+
// read the bytes and extract them, never the viewer's wrapper DOM.
|
|
569
|
+
if (kind === 'pdf' || (kind === undefined && finalResponse !== null && mayBePdf(finalResponse.headers(), finalUrl))) {
|
|
570
|
+
const download = await downloadPdfBytes(page, finalResponse, finalResponse?.url?.() ?? finalUrl, deadline)
|
|
571
|
+
if (download.pdf) return await pdfResult(download.bytes, download.url, statusCode, deadline)
|
|
572
|
+
if (kind === 'pdf') throw new WebError('the response claims to be a PDF but its bytes are not one', WEB_PDF_INVALID_CODE)
|
|
573
|
+
}
|
|
574
|
+
if (kind === undefined) {
|
|
575
|
+
throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
576
|
+
}
|
|
577
|
+
|
|
555
578
|
// Non-HTML decodes straight from the response body; no denoise applies.
|
|
556
579
|
if (kind === 'text') {
|
|
557
580
|
const text = finalResponse !== null ? await finalResponse.text() : await page.content()
|
|
@@ -820,7 +843,7 @@ async function recoverUnrenderedStatus(
|
|
|
820
843
|
// class only fires on empty bodies, so '' is the honest fallback.
|
|
821
844
|
let content = ''
|
|
822
845
|
try { content = await response.text() } catch { content = '' }
|
|
823
|
-
if (kind
|
|
846
|
+
if (kind !== 'html' || content === '') {
|
|
824
847
|
// Empty HTML needs no pipeline — the empty string IS the body.
|
|
825
848
|
return capResult(finalUrl, status, { kind: 'text', content })
|
|
826
849
|
}
|
|
@@ -866,7 +889,9 @@ function isDownloadFailure(error: unknown, response: PlaywrightResponse | null):
|
|
|
866
889
|
* download decision — which works identically on the local and CDP
|
|
867
890
|
* backends because it never touches the remote artifact.
|
|
868
891
|
*
|
|
869
|
-
*
|
|
892
|
+
* A PDF download (declared `application/pdf`, or an opaque binary whose
|
|
893
|
+
* bytes sniff as `%PDF-`) is extracted to markdown (`src/pdf.ts`);
|
|
894
|
+
* undecodable downloads (a ZIP) still fail with
|
|
870
895
|
* `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
|
|
871
896
|
* actually served; a playwright build without a request context surfaces a
|
|
872
897
|
* clear "the browser will not render this" error instead of the raw
|
|
@@ -893,8 +918,14 @@ async function recoverDownloadResponse(
|
|
|
893
918
|
if (!isDownloadFailure(error, tracked)) return undefined
|
|
894
919
|
if (tracked !== null) {
|
|
895
920
|
// What the site served the navigation is the truth about the resource:
|
|
896
|
-
// an undecodable download (
|
|
921
|
+
// an undecodable download (ZIP/…) has no business being refetched — a
|
|
922
|
+
// PDF (declared, or an opaque binary that may be one) is the exception.
|
|
897
923
|
const trackedKind = classifyContentType(tracked.headers()['content-type'])
|
|
924
|
+
if (trackedKind === 'pdf' || (trackedKind === undefined && mayBePdf(tracked.headers(), tracked.url?.() ?? url.toString()))) {
|
|
925
|
+
const download = await downloadPdfBytes(page, null, tracked.url?.() ?? url.toString(), deadline, tracked.headers())
|
|
926
|
+
if (download.pdf) return await pdfResult(download.bytes, download.url, download.status, deadline)
|
|
927
|
+
throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download; not a PDF)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
928
|
+
}
|
|
898
929
|
if (trackedKind === undefined) {
|
|
899
930
|
throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
900
931
|
}
|
|
@@ -913,6 +944,13 @@ async function recoverDownloadResponse(
|
|
|
913
944
|
// answer falls back to what the navigation served.
|
|
914
945
|
const servedType = (headers['content-type'] ?? '').trim() !== '' ? headers['content-type'] : tracked?.headers()['content-type']
|
|
915
946
|
const kind = classifyContentType(servedType)
|
|
947
|
+
if (kind === 'pdf') {
|
|
948
|
+
// Untracked navigation, but the refetch says PDF: extract it.
|
|
949
|
+
assertPdfSize(headers)
|
|
950
|
+
const bytes = refetched.body !== undefined ? await refetched.body() : new Uint8Array()
|
|
951
|
+
if (!isPdfBytes(bytes)) throw new WebError('the response claims to be a PDF but its bytes are not one', WEB_PDF_INVALID_CODE)
|
|
952
|
+
return await pdfResult(bytes, refetched.url?.() ?? fetchUrl, refetched.status(), deadline)
|
|
953
|
+
}
|
|
916
954
|
if (kind === undefined) {
|
|
917
955
|
throw new WebError(`unsupported content type "${servedType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
918
956
|
}
|
|
@@ -927,6 +965,77 @@ async function recoverDownloadResponse(
|
|
|
927
965
|
return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
|
|
928
966
|
}
|
|
929
967
|
|
|
968
|
+
/** Time kept back from the PDF page walk for layout and result assembly. */
|
|
969
|
+
const PDF_FINISH_RESERVE_MS = 1_500
|
|
970
|
+
|
|
971
|
+
/** Refuse a PDF whose declared length is over the limit — before downloading it. */
|
|
972
|
+
function assertPdfSize(headers: Record<string, string> | undefined): void {
|
|
973
|
+
const declared = Number.parseInt(headers?.['content-length'] ?? '', 10)
|
|
974
|
+
if (Number.isFinite(declared) && declared > MAX_PDF_BYTES) {
|
|
975
|
+
throw new WebError(
|
|
976
|
+
`the PDF is ${String(Math.round(declared / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`,
|
|
977
|
+
WEB_PDF_TOO_LARGE_CODE,
|
|
978
|
+
)
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
/**
|
|
983
|
+
* Get a (possibly) PDF resource's bytes. The navigation response's own body
|
|
984
|
+
* is tried first — a PDF Chrome committed into its viewer still has it —
|
|
985
|
+
* then the page's API request context refetches the URL (same cookie jar,
|
|
986
|
+
* plain HTTP, no render-or-download decision; the only way to reach a body
|
|
987
|
+
* Chromium streamed to its download manager). The `%PDF-` magic decides.
|
|
988
|
+
*
|
|
989
|
+
* @param page - the fetch's tab (for its request context).
|
|
990
|
+
* @param response - the navigation response, when the browser committed one.
|
|
991
|
+
* @param url - the URL to refetch.
|
|
992
|
+
* @param deadline - the fetch budget.
|
|
993
|
+
* @param servedHeaders - the navigation's headers (size pre-check) when no
|
|
994
|
+
* response object is passed.
|
|
995
|
+
* @returns the bytes, where they came from, and whether they are a PDF.
|
|
996
|
+
*/
|
|
997
|
+
async function downloadPdfBytes(
|
|
998
|
+
page: PlaywrightPage,
|
|
999
|
+
response: PlaywrightResponse | null,
|
|
1000
|
+
url: string,
|
|
1001
|
+
deadline: Deadline,
|
|
1002
|
+
servedHeaders?: Record<string, string>,
|
|
1003
|
+
): Promise<{ pdf: boolean; bytes: Uint8Array; url: string; status: number }> {
|
|
1004
|
+
assertPdfSize(response?.headers() ?? servedHeaders)
|
|
1005
|
+
if (response?.body !== undefined) {
|
|
1006
|
+
try {
|
|
1007
|
+
const bytes = await response.body()
|
|
1008
|
+
if (isPdfBytes(bytes)) return { pdf: true, bytes, url: response.url?.() ?? url, status: response.status() }
|
|
1009
|
+
} catch {
|
|
1010
|
+
// the viewer / download manager owns the body — refetch below
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
if (page.request === undefined) {
|
|
1014
|
+
throw new WebError(
|
|
1015
|
+
'the response is a PDF the browser will not hand over, and this playwright build exposes no request context to refetch it',
|
|
1016
|
+
'WEB_PROVIDER_ERROR',
|
|
1017
|
+
)
|
|
1018
|
+
}
|
|
1019
|
+
const refetched = await page.request.get(url, { timeout: deadline.remainingMs(), signal: deadline.signal })
|
|
1020
|
+
assertPdfSize(refetched.headers())
|
|
1021
|
+
const bytes = refetched.body !== undefined ? await refetched.body() : new Uint8Array()
|
|
1022
|
+
return { pdf: isPdfBytes(bytes), bytes, url: refetched.url?.() ?? url, status: refetched.status() }
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
/**
|
|
1026
|
+
* Extract PDF bytes to a markdown text result inside the fetch budget.
|
|
1027
|
+
* A limit that stopped the page walk early flags the result truncated.
|
|
1028
|
+
*/
|
|
1029
|
+
async function pdfResult(bytes: Uint8Array, finalUrl: string, statusCode: number, deadline: Deadline): Promise<WebFetchResult> {
|
|
1030
|
+
const extraction = await extractPdfMarkdown(bytes, {
|
|
1031
|
+
signal: deadline.signal,
|
|
1032
|
+
timeBudgetMs: Math.max(1_000, deadline.remainingMs() - PDF_FINISH_RESERVE_MS),
|
|
1033
|
+
maxChars: MAX_BODY_CHARS,
|
|
1034
|
+
})
|
|
1035
|
+
const result = capResult(finalUrl, statusCode, { kind: 'text', content: extraction.markdown })
|
|
1036
|
+
return extraction.truncated && !result.truncated ? { ...result, truncated: true } : result
|
|
1037
|
+
}
|
|
1038
|
+
|
|
930
1039
|
/**
|
|
931
1040
|
* Probe the live document for challenge markers. Prefers the in-page probe
|
|
932
1041
|
* (`page.evaluate`) — the only thing that can see SPA-style clears — and
|