dsh-web-fetch-playwright 0.2.16 → 0.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -4
- package/README.zh-CN.md +47 -4
- package/lib/client.js +45 -20
- package/lib/index.js +1869 -75
- package/lib/types/autoscroll.d.ts +277 -0
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -3
- package/lib/types/index.d.ts +5 -1
- package/lib/types/markdown.d.ts +8 -5
- package/lib/types/pdf.d.ts +143 -0
- package/lib/types/provider.d.ts +34 -0
- package/lib/types/strip.d.ts +59 -0
- package/lib/types/types.d.ts +5 -0
- package/package.json +3 -2
- package/src/autoscroll.ts +653 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -3
- package/src/index.ts +7 -0
- package/src/markdown.ts +252 -6
- package/src/pdf.ts +1114 -0
- package/src/provider.ts +174 -76
- package/src/session-window.ts +8 -0
- package/src/strip.ts +72 -0
- package/src/types.ts +5 -0
package/src/provider.ts
CHANGED
|
@@ -33,6 +33,15 @@
|
|
|
33
33
|
* {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
|
|
34
34
|
* a queue and fail fast (rather than hang until abort) when no slot frees.
|
|
35
35
|
*
|
|
36
|
+
* PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
|
|
37
|
+
* handed to the download manager as an attachment, or served as an opaque
|
|
38
|
+
* `application/octet-stream` whose bytes start with `%PDF-` — is read as
|
|
39
|
+
* bytes (the navigation body, else a refetch through the page's API request
|
|
40
|
+
* context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
|
|
41
|
+
* plus layout heuristics: headings, paragraphs, lists, tables, page
|
|
42
|
+
* markers). Scans without a text layer, encrypted, oversized, and broken
|
|
43
|
+
* PDFs fail with the `WEB_PDF_*` codes.
|
|
44
|
+
*
|
|
36
45
|
* Private-network and SSRF protection is not implemented (same stance as the
|
|
37
46
|
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
38
47
|
* browser can reach. Profile mode additionally acts WITH the remote
|
|
@@ -50,6 +59,31 @@
|
|
|
50
59
|
* fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
|
|
51
60
|
* interstitial as content.
|
|
52
61
|
*
|
|
62
|
+
* Deep capture (viewport-virtualized feeds and app-shell documents): feed-
|
|
63
|
+
* shaped pages — Discourse topic streams, comment threads, timelines —
|
|
64
|
+
* render only the items near the scroll position; everything else is an
|
|
65
|
+
* empty, height-only placeholder that keeps the item's id but none of its
|
|
66
|
+
* content, so a single-instant capture hands back the skeleton while the
|
|
67
|
+
* bodies silently vanish (and the denoise pipeline's coverage gate cannot
|
|
68
|
+
* notice: the page's own visible text is the same windowed fragment as the
|
|
69
|
+
* extraction). App-shell document editors (Feishu docs) push the same
|
|
70
|
+
* pathology one level down: the body never scrolls, blocks are keyed by
|
|
71
|
+
* data-block-id with no id, and off-window blocks are REMOVED behind an
|
|
72
|
+
* anonymous height spacer — the fetch would return just the statically
|
|
73
|
+
* rendered first screen. Before serialization the fetch therefore runs one
|
|
74
|
+
* bounded in-page scroll walk — over the window or, when the document does
|
|
75
|
+
* not scroll, the page's dominant inner scroll container — stashing what
|
|
76
|
+
* each rendered window shows, terminating only AT the bottom with the
|
|
77
|
+
* height stable (that is what pages load-more sentinels; height stability
|
|
78
|
+
* elsewhere proves nothing), then rehydrating degraded keyed elements from
|
|
79
|
+
* their stashed snapshots and splicing removed keys back at their document
|
|
80
|
+
* position so the final DOM holds the union of every window in document
|
|
81
|
+
* order. The existing pipeline then runs unchanged. Bounded by
|
|
82
|
+
* `scrollBudgetMs` (default 20s; 0 restores the legacy single-instant
|
|
83
|
+
* behavior) and by the remaining fetch deadline; any walk failure silently
|
|
84
|
+
* degrades to the legacy capture (see `src/autoscroll.ts` for the measured
|
|
85
|
+
* linux.do and Feishu badcases).
|
|
86
|
+
*
|
|
53
87
|
* @module dsh-web-fetch-playwright/provider
|
|
54
88
|
*/
|
|
55
89
|
|
|
@@ -57,10 +91,13 @@ import { WebError } from '@deepseek-ai/dsh-web'
|
|
|
57
91
|
import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
|
|
58
92
|
import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
|
|
59
93
|
import type { ChallengeVerdict } from './challenge.ts'
|
|
60
|
-
import {
|
|
94
|
+
import { deepCaptureWalk, SCROLL_FINISH_RESERVE_MS } from './autoscroll.ts'
|
|
95
|
+
import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
|
|
61
96
|
import type { ResolvedConfig } from './config.ts'
|
|
62
97
|
import { CdpConnectionPool } from './cdp-pool.ts'
|
|
63
98
|
import { htmlToMarkdown } from './markdown.ts'
|
|
99
|
+
import { extractPdfMarkdown, isPdfBytes, isPdfContentType, mayBePdf, MAX_PDF_BYTES, WEB_PDF_INVALID_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
|
|
100
|
+
import { STRIP_SCRIPTING_ELEMENTS } from './strip.ts'
|
|
64
101
|
import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
|
|
65
102
|
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
|
|
66
103
|
|
|
@@ -261,12 +298,16 @@ function validateFetchUrl(input: string): URL {
|
|
|
261
298
|
return url
|
|
262
299
|
}
|
|
263
300
|
|
|
264
|
-
/**
|
|
265
|
-
|
|
301
|
+
/**
|
|
302
|
+
* The decodable body kinds: the HTTP provider's classification plus `pdf`,
|
|
303
|
+
* which this provider extracts to markdown (see `src/pdf.ts`).
|
|
304
|
+
*/
|
|
305
|
+
type FetchableKind = 'html' | 'text' | 'pdf'
|
|
266
306
|
|
|
267
307
|
/** Classify a `Content-Type` into a decodable kind; undefined = unsupported. */
|
|
268
308
|
function classifyContentType(contentType: string | undefined): FetchableKind | undefined {
|
|
269
309
|
const mime = (contentType ?? '').replace(/;.*$/s, '').trim().toLowerCase()
|
|
310
|
+
if (isPdfContentType(mime)) return 'pdf'
|
|
270
311
|
if (mime === '' || mime === 'text/html' || mime === 'application/xhtml+xml') return 'html'
|
|
271
312
|
if (mime.startsWith('text/')) return 'text'
|
|
272
313
|
if (mime === 'application/json' || mime === 'application/xml' || mime.endsWith('+json') || mime.endsWith('+xml')) return 'text'
|
|
@@ -516,15 +557,24 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
516
557
|
// ran (the real page reloads in), else the response goto returned.
|
|
517
558
|
const finalResponse = (challengeWaitMs > 0 ? tracker.last() : undefined) ?? response
|
|
518
559
|
const kind = classifyContentType(finalResponse?.headers()['content-type'])
|
|
519
|
-
if (kind === undefined) {
|
|
520
|
-
throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
521
|
-
}
|
|
522
560
|
const finalUrl = page.url()
|
|
523
561
|
// An SPA-style clear swaps the document without navigating: no new
|
|
524
562
|
// response exists to report, so the cleared document reads as served.
|
|
525
563
|
const clearedWithoutNavigation = challengeEntryResponse !== null && finalResponse === challengeEntryResponse
|
|
526
564
|
const statusCode = finalResponse !== null && !clearedWithoutNavigation ? finalResponse.status() : 200
|
|
527
565
|
|
|
566
|
+
// A PDF the browser committed as a document (its built-in viewer — a
|
|
567
|
+
// headed or CDP-attached Chrome) or an opaque binary that may be one:
|
|
568
|
+
// read the bytes and extract them, never the viewer's wrapper DOM.
|
|
569
|
+
if (kind === 'pdf' || (kind === undefined && finalResponse !== null && mayBePdf(finalResponse.headers(), finalUrl))) {
|
|
570
|
+
const download = await downloadPdfBytes(page, finalResponse, finalResponse?.url?.() ?? finalUrl, deadline)
|
|
571
|
+
if (download.pdf) return await pdfResult(download.bytes, download.url, statusCode, deadline)
|
|
572
|
+
if (kind === 'pdf') throw new WebError('the response claims to be a PDF but its bytes are not one', WEB_PDF_INVALID_CODE)
|
|
573
|
+
}
|
|
574
|
+
if (kind === undefined) {
|
|
575
|
+
throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
576
|
+
}
|
|
577
|
+
|
|
528
578
|
// Non-HTML decodes straight from the response body; no denoise applies.
|
|
529
579
|
if (kind === 'text') {
|
|
530
580
|
const text = finalResponse !== null ? await finalResponse.text() : await page.content()
|
|
@@ -535,14 +585,31 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
535
585
|
// what domcontentloaded already produced.
|
|
536
586
|
await page.waitForLoadState('networkidle', { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {})
|
|
537
587
|
|
|
538
|
-
//
|
|
539
|
-
//
|
|
540
|
-
//
|
|
541
|
-
//
|
|
542
|
-
//
|
|
543
|
-
|
|
588
|
+
// Deep capture (viewport-virtualized feeds and app-shell documents lose
|
|
589
|
+
// their off-window content at rest — see `src/autoscroll.ts`): a bounded
|
|
590
|
+
// walk stashes what each rendered window shows and, when it restored
|
|
591
|
+
// anything, hands back its own merged, already-stripped serialization.
|
|
592
|
+
// HTML documents only, inside the deadline with a decode reserve,
|
|
593
|
+
// silently degrading to the legacy capture; `scrollBudgetMs: 0` skips it.
|
|
594
|
+
const scrollBudgetMs = Math.min(
|
|
595
|
+
effectiveScrollBudgetMs(config),
|
|
596
|
+
deadline.remainingMs() - SCROLL_FINISH_RESERVE_MS,
|
|
597
|
+
)
|
|
598
|
+
const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : undefined
|
|
544
599
|
|
|
545
|
-
|
|
600
|
+
// Serialization safety for the live read: the document is about to be
|
|
601
|
+
// flattened into an HTML string and re-parsed by jsdom, and scripting
|
|
602
|
+
// elements are the one place that round-trip can break — see
|
|
603
|
+
// `src/strip.ts` for exactly what is removed, what is kept (JSON-LD,
|
|
604
|
+
// noscript), and why each removal cannot change what the pipeline
|
|
605
|
+
// extracts. (The walk's merged capture went through the same strip.)
|
|
606
|
+
let html: string
|
|
607
|
+
if (captured?.html !== undefined) {
|
|
608
|
+
html = captured.html
|
|
609
|
+
} else {
|
|
610
|
+
await stripScriptingElements(page)
|
|
611
|
+
html = await page.content()
|
|
612
|
+
}
|
|
546
613
|
if (!config.denoise) {
|
|
547
614
|
// The tool layer's own turndown renders raw HTML; the checkbox only
|
|
548
615
|
// governs the Readability/DOMPurify stage this provider owns.
|
|
@@ -776,7 +843,7 @@ async function recoverUnrenderedStatus(
|
|
|
776
843
|
// class only fires on empty bodies, so '' is the honest fallback.
|
|
777
844
|
let content = ''
|
|
778
845
|
try { content = await response.text() } catch { content = '' }
|
|
779
|
-
if (kind
|
|
846
|
+
if (kind !== 'html' || content === '') {
|
|
780
847
|
// Empty HTML needs no pipeline — the empty string IS the body.
|
|
781
848
|
return capResult(finalUrl, status, { kind: 'text', content })
|
|
782
849
|
}
|
|
@@ -822,7 +889,9 @@ function isDownloadFailure(error: unknown, response: PlaywrightResponse | null):
|
|
|
822
889
|
* download decision — which works identically on the local and CDP
|
|
823
890
|
* backends because it never touches the remote artifact.
|
|
824
891
|
*
|
|
825
|
-
*
|
|
892
|
+
* A PDF download (declared `application/pdf`, or an opaque binary whose
|
|
893
|
+
* bytes sniff as `%PDF-`) is extracted to markdown (`src/pdf.ts`);
|
|
894
|
+
* undecodable downloads (a ZIP) still fail with
|
|
826
895
|
* `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
|
|
827
896
|
* actually served; a playwright build without a request context surfaces a
|
|
828
897
|
* clear "the browser will not render this" error instead of the raw
|
|
@@ -849,8 +918,14 @@ async function recoverDownloadResponse(
|
|
|
849
918
|
if (!isDownloadFailure(error, tracked)) return undefined
|
|
850
919
|
if (tracked !== null) {
|
|
851
920
|
// What the site served the navigation is the truth about the resource:
|
|
852
|
-
// an undecodable download (
|
|
921
|
+
// an undecodable download (ZIP/…) has no business being refetched — a
|
|
922
|
+
// PDF (declared, or an opaque binary that may be one) is the exception.
|
|
853
923
|
const trackedKind = classifyContentType(tracked.headers()['content-type'])
|
|
924
|
+
if (trackedKind === 'pdf' || (trackedKind === undefined && mayBePdf(tracked.headers(), tracked.url?.() ?? url.toString()))) {
|
|
925
|
+
const download = await downloadPdfBytes(page, null, tracked.url?.() ?? url.toString(), deadline, tracked.headers())
|
|
926
|
+
if (download.pdf) return await pdfResult(download.bytes, download.url, download.status, deadline)
|
|
927
|
+
throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download; not a PDF)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
928
|
+
}
|
|
854
929
|
if (trackedKind === undefined) {
|
|
855
930
|
throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
856
931
|
}
|
|
@@ -869,6 +944,13 @@ async function recoverDownloadResponse(
|
|
|
869
944
|
// answer falls back to what the navigation served.
|
|
870
945
|
const servedType = (headers['content-type'] ?? '').trim() !== '' ? headers['content-type'] : tracked?.headers()['content-type']
|
|
871
946
|
const kind = classifyContentType(servedType)
|
|
947
|
+
if (kind === 'pdf') {
|
|
948
|
+
// Untracked navigation, but the refetch says PDF: extract it.
|
|
949
|
+
assertPdfSize(headers)
|
|
950
|
+
const bytes = refetched.body !== undefined ? await refetched.body() : new Uint8Array()
|
|
951
|
+
if (!isPdfBytes(bytes)) throw new WebError('the response claims to be a PDF but its bytes are not one', WEB_PDF_INVALID_CODE)
|
|
952
|
+
return await pdfResult(bytes, refetched.url?.() ?? fetchUrl, refetched.status(), deadline)
|
|
953
|
+
}
|
|
872
954
|
if (kind === undefined) {
|
|
873
955
|
throw new WebError(`unsupported content type "${servedType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
874
956
|
}
|
|
@@ -883,6 +965,77 @@ async function recoverDownloadResponse(
|
|
|
883
965
|
return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
|
|
884
966
|
}
|
|
885
967
|
|
|
968
|
+
/** Time kept back from the PDF page walk for layout and result assembly. */
|
|
969
|
+
const PDF_FINISH_RESERVE_MS = 1_500
|
|
970
|
+
|
|
971
|
+
/** Refuse a PDF whose declared length is over the limit — before downloading it. */
|
|
972
|
+
function assertPdfSize(headers: Record<string, string> | undefined): void {
|
|
973
|
+
const declared = Number.parseInt(headers?.['content-length'] ?? '', 10)
|
|
974
|
+
if (Number.isFinite(declared) && declared > MAX_PDF_BYTES) {
|
|
975
|
+
throw new WebError(
|
|
976
|
+
`the PDF is ${String(Math.round(declared / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`,
|
|
977
|
+
WEB_PDF_TOO_LARGE_CODE,
|
|
978
|
+
)
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
/**
|
|
983
|
+
* Get a (possibly) PDF resource's bytes. The navigation response's own body
|
|
984
|
+
* is tried first — a PDF Chrome committed into its viewer still has it —
|
|
985
|
+
* then the page's API request context refetches the URL (same cookie jar,
|
|
986
|
+
* plain HTTP, no render-or-download decision; the only way to reach a body
|
|
987
|
+
* Chromium streamed to its download manager). The `%PDF-` magic decides.
|
|
988
|
+
*
|
|
989
|
+
* @param page - the fetch's tab (for its request context).
|
|
990
|
+
* @param response - the navigation response, when the browser committed one.
|
|
991
|
+
* @param url - the URL to refetch.
|
|
992
|
+
* @param deadline - the fetch budget.
|
|
993
|
+
* @param servedHeaders - the navigation's headers (size pre-check) when no
|
|
994
|
+
* response object is passed.
|
|
995
|
+
* @returns the bytes, where they came from, and whether they are a PDF.
|
|
996
|
+
*/
|
|
997
|
+
async function downloadPdfBytes(
|
|
998
|
+
page: PlaywrightPage,
|
|
999
|
+
response: PlaywrightResponse | null,
|
|
1000
|
+
url: string,
|
|
1001
|
+
deadline: Deadline,
|
|
1002
|
+
servedHeaders?: Record<string, string>,
|
|
1003
|
+
): Promise<{ pdf: boolean; bytes: Uint8Array; url: string; status: number }> {
|
|
1004
|
+
assertPdfSize(response?.headers() ?? servedHeaders)
|
|
1005
|
+
if (response?.body !== undefined) {
|
|
1006
|
+
try {
|
|
1007
|
+
const bytes = await response.body()
|
|
1008
|
+
if (isPdfBytes(bytes)) return { pdf: true, bytes, url: response.url?.() ?? url, status: response.status() }
|
|
1009
|
+
} catch {
|
|
1010
|
+
// the viewer / download manager owns the body — refetch below
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
if (page.request === undefined) {
|
|
1014
|
+
throw new WebError(
|
|
1015
|
+
'the response is a PDF the browser will not hand over, and this playwright build exposes no request context to refetch it',
|
|
1016
|
+
'WEB_PROVIDER_ERROR',
|
|
1017
|
+
)
|
|
1018
|
+
}
|
|
1019
|
+
const refetched = await page.request.get(url, { timeout: deadline.remainingMs(), signal: deadline.signal })
|
|
1020
|
+
assertPdfSize(refetched.headers())
|
|
1021
|
+
const bytes = refetched.body !== undefined ? await refetched.body() : new Uint8Array()
|
|
1022
|
+
return { pdf: isPdfBytes(bytes), bytes, url: refetched.url?.() ?? url, status: refetched.status() }
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
/**
|
|
1026
|
+
* Extract PDF bytes to a markdown text result inside the fetch budget.
|
|
1027
|
+
* A limit that stopped the page walk early flags the result truncated.
|
|
1028
|
+
*/
|
|
1029
|
+
async function pdfResult(bytes: Uint8Array, finalUrl: string, statusCode: number, deadline: Deadline): Promise<WebFetchResult> {
|
|
1030
|
+
const extraction = await extractPdfMarkdown(bytes, {
|
|
1031
|
+
signal: deadline.signal,
|
|
1032
|
+
timeBudgetMs: Math.max(1_000, deadline.remainingMs() - PDF_FINISH_RESERVE_MS),
|
|
1033
|
+
maxChars: MAX_BODY_CHARS,
|
|
1034
|
+
})
|
|
1035
|
+
const result = capResult(finalUrl, statusCode, { kind: 'text', content: extraction.markdown })
|
|
1036
|
+
return extraction.truncated && !result.truncated ? { ...result, truncated: true } : result
|
|
1037
|
+
}
|
|
1038
|
+
|
|
886
1039
|
/**
|
|
887
1040
|
* Probe the live document for challenge markers. Prefers the in-page probe
|
|
888
1041
|
* (`page.evaluate`) — the only thing that can see SPA-style clears — and
|
|
@@ -907,68 +1060,13 @@ async function probeStillOnChallenge(page: PlaywrightPage): Promise<boolean> {
|
|
|
907
1060
|
}
|
|
908
1061
|
}
|
|
909
1062
|
|
|
910
|
-
/**
|
|
911
|
-
* In-page script that removes non-content scripting subtrees from the live
|
|
912
|
-
* document before `page.content()` reads it as an HTML string.
|
|
913
|
-
*
|
|
914
|
-
* WHAT goes, and why each removal is output-neutral versus the old pipeline:
|
|
915
|
-
*
|
|
916
|
-
* - `script` — three consumers ever saw script text before, and none loses
|
|
917
|
-
* it here: Readability's `parse()` deletes script/noscript nodes BEFORE
|
|
918
|
-
* extraction and scoring (so script text never reached the article or
|
|
919
|
-
* influenced the scorer); DOMPurify's default allowlist drops script; the
|
|
920
|
-
* tool layer's renderer removes SCRIPT wholesale on the raw path. What the
|
|
921
|
-
* removal buys is the reason this exists: a page's runtime JS may rewrite
|
|
922
|
-
* a data script's text with a literal `</script>` embedded — measured on
|
|
923
|
-
* techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
|
|
924
|
-
* Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
|
|
925
|
-
* turning the wire's 712 `\u003c` escapes back into raw `<` — and
|
|
926
|
-
* browsers serialize script text verbatim, so the flattened string ends
|
|
927
|
-
* that script at the FIRST embedded end tag and re-parses the remaining
|
|
928
|
-
* ~270KB of JSON as markup. The phantom elements that survive (unknown
|
|
929
|
-
* tags like `<id>`, anchors full of JSON text) are exactly what
|
|
930
|
-
* Readability's scorer reads as article mass, so the extracted "article"
|
|
931
|
-
* is CSS and i18n strings while the real body — still present in the DOM —
|
|
932
|
-
* loses. No string-level patch can undo this after serialization (the
|
|
933
|
-
* string is genuinely ambiguous); only the browser's own parse holds the
|
|
934
|
-
* document's true structure.
|
|
935
|
-
* - `script[type="application/ld+json"]` — KEPT, but defused: Readability
|
|
936
|
-
* extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
|
|
937
|
-
* "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
|
|
938
|
-
* scripts before removing them, so deleting them would regress the title
|
|
939
|
-
* on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
|
|
940
|
-
* valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
|
|
941
|
-
* same object) while making the serialization harmless: the tokenizer
|
|
942
|
-
* only terminates on a literal `</script`.
|
|
943
|
-
* - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
|
|
944
|
-
* real image out of lazy-load `<noscript><img …>` wrappers before
|
|
945
|
-
* `_removeScripts` deletes the noscript; pre-stripping it would silently
|
|
946
|
-
* drop those images from the markdown.
|
|
947
|
-
* - `template` — removed: its children parse into the inert `.content`
|
|
948
|
-
* fragment (invisible to Readability's traversal), DOMPurify's
|
|
949
|
-
* FORBID_TAGS drops it, and the tool layer's renderer removes it too.
|
|
950
|
-
* - `style` stays — its text cannot terminate a re-parse except at its own
|
|
951
|
-
* end tag, and the pipeline already drops it.
|
|
952
|
-
*/
|
|
953
|
-
const STRIP_SCRIPTING_ELEMENTS = `(() => {
|
|
954
|
-
for (const el of Array.from(document.querySelectorAll('script, template'))) {
|
|
955
|
-
if (el.tagName === 'SCRIPT' && (el.getAttribute('type') || '').trim().toLowerCase() === 'application/ld+json') {
|
|
956
|
-
// '<\\/' is still valid JSON, so the re-parsed textContent parses to
|
|
957
|
-
// the same object — while the serialized form can no longer close the
|
|
958
|
-
// script element early.
|
|
959
|
-
el.textContent = el.textContent.replace(/<\\//g, '<\\\\/')
|
|
960
|
-
continue
|
|
961
|
-
}
|
|
962
|
-
el.remove()
|
|
963
|
-
}
|
|
964
|
-
})()`
|
|
965
|
-
|
|
966
1063
|
/**
|
|
967
1064
|
* Remove non-content scripting subtrees from the live document before
|
|
968
|
-
* serialization
|
|
969
|
-
*
|
|
970
|
-
*
|
|
971
|
-
*
|
|
1065
|
+
* serialization (see `stripScriptingElementsIn` in `src/strip.ts` for exactly
|
|
1066
|
+
* what goes and why). Best-effort, mirroring {@link probeStillOnChallenge}: a backend
|
|
1067
|
+
* without `evaluate` (fakes, minimal engines) simply keeps the document
|
|
1068
|
+
* as-is — the fetch never fails here, and the pipeline degrades to whatever
|
|
1069
|
+
* the serialized string itself says.
|
|
972
1070
|
*/
|
|
973
1071
|
async function stripScriptingElements(page: PlaywrightPage): Promise<void> {
|
|
974
1072
|
if (typeof page.evaluate !== 'function') return
|
package/src/session-window.ts
CHANGED
|
@@ -18,6 +18,10 @@ interface TargetIdentity { targetId: string; browserContextId: string }
|
|
|
18
18
|
const ownership = new WeakMap<PlaywrightBrowser, Map<string, WeakRef<PlaywrightContext>>>()
|
|
19
19
|
// Never adopt an unconfirmed late tab after timing out, nor close sibling leases.
|
|
20
20
|
const unavailable = new WeakSet<PlaywrightContext>()
|
|
21
|
+
// Identities confirmed when each session tab was created. Re-identifying a busy
|
|
22
|
+
// sibling would attach to it, and detaching from a tab mid-navigation can stall
|
|
23
|
+
// until its response arrives; a target id and its context never change.
|
|
24
|
+
const identities = new WeakMap<PlaywrightPage, TargetIdentity>()
|
|
21
25
|
const CLEANUP_GRACE_MS = 100
|
|
22
26
|
|
|
23
27
|
function claim(browser: PlaywrightBrowser, context: PlaywrightContext, keys: string[]): void {
|
|
@@ -112,6 +116,8 @@ export async function createSessionPage(
|
|
|
112
116
|
return session
|
|
113
117
|
}
|
|
114
118
|
const identify = async (page: PlaywrightPage): Promise<TargetIdentity> => {
|
|
119
|
+
const known = identities.get(page)
|
|
120
|
+
if (known) return known
|
|
115
121
|
const session = await newSession(() => ctx.newCDPSession(page))
|
|
116
122
|
let identificationFailed = false
|
|
117
123
|
try {
|
|
@@ -164,6 +170,7 @@ export async function createSessionPage(
|
|
|
164
170
|
throw new Error('Chromium did not place the session tab in its dedicated window')
|
|
165
171
|
}
|
|
166
172
|
claim(browser, context, [`context:${actual.browserContextId}`, `window:${expectedWindow}`])
|
|
173
|
+
identities.set(created, actual)
|
|
167
174
|
succeeded = true
|
|
168
175
|
return created
|
|
169
176
|
}
|
|
@@ -209,6 +216,7 @@ export async function createSessionPage(
|
|
|
209
216
|
const explicitWindow = await windowOf(explicitTargetId)
|
|
210
217
|
if (explicitWindow === bootstrapWindow) throw new Error('Chromium ignored the dedicated session window request')
|
|
211
218
|
claim(browser, context, [`context:${identity.browserContextId}`, `window:${explicitWindow}`])
|
|
219
|
+
identities.set(created, { targetId: explicitTargetId, browserContextId: identity.browserContextId })
|
|
212
220
|
await deadline.run(() => bootstrap!.close())
|
|
213
221
|
bootstrap = undefined
|
|
214
222
|
succeeded = true
|
package/src/strip.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serialization safety for documents about to be flattened into an HTML
|
|
3
|
+
* string and re-parsed by jsdom: the one in-page cleanup both serialization
|
|
4
|
+
* paths share — the provider's live-document read and the deep-capture
|
|
5
|
+
* walk's merged clone.
|
|
6
|
+
*
|
|
7
|
+
* @module dsh-web-fetch-playwright/strip
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* In-page function that removes non-content scripting subtrees from a
|
|
12
|
+
* document before it is flattened into an HTML string — the live document
|
|
13
|
+
* before `page.content()`, or the deep-capture walk's merged clone before it
|
|
14
|
+
* serializes it itself (`src/autoscroll.ts`).
|
|
15
|
+
*
|
|
16
|
+
* It is evaluated BY SOURCE (`fn.toString()`), so it must stay
|
|
17
|
+
* self-contained: no imports, no module-level references.
|
|
18
|
+
*
|
|
19
|
+
* WHAT goes, and why each removal is output-neutral versus the old pipeline:
|
|
20
|
+
*
|
|
21
|
+
* - `script` — three consumers ever saw script text before, and none loses
|
|
22
|
+
* it here: Readability's `parse()` deletes script/noscript nodes BEFORE
|
|
23
|
+
* extraction and scoring (so script text never reached the article or
|
|
24
|
+
* influenced the scorer); DOMPurify's default allowlist drops script; the
|
|
25
|
+
* tool layer's renderer removes SCRIPT wholesale on the raw path. What the
|
|
26
|
+
* removal buys is the reason this exists: a page's runtime JS may rewrite
|
|
27
|
+
* a data script's text with a literal `</script>` embedded — measured on
|
|
28
|
+
* techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
|
|
29
|
+
* Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
|
|
30
|
+
* turning the wire's 712 `\u003c` escapes back into raw `<` — and
|
|
31
|
+
* browsers serialize script text verbatim, so the flattened string ends
|
|
32
|
+
* that script at the FIRST embedded end tag and re-parses the remaining
|
|
33
|
+
* ~270KB of JSON as markup. The phantom elements that survive (unknown
|
|
34
|
+
* tags like `<id>`, anchors full of JSON text) are exactly what
|
|
35
|
+
* Readability's scorer reads as article mass, so the extracted "article"
|
|
36
|
+
* is CSS and i18n strings while the real body — still present in the DOM —
|
|
37
|
+
* loses. No string-level patch can undo this after serialization (the
|
|
38
|
+
* string is genuinely ambiguous); only the browser's own parse holds the
|
|
39
|
+
* document's true structure.
|
|
40
|
+
* - `script[type="application/ld+json"]` — KEPT, but defused: Readability
|
|
41
|
+
* extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
|
|
42
|
+
* "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
|
|
43
|
+
* scripts before removing them, so deleting them would regress the title
|
|
44
|
+
* on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
|
|
45
|
+
* valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
|
|
46
|
+
* same object) while making the serialization harmless: the tokenizer
|
|
47
|
+
* only terminates on a literal `</script`.
|
|
48
|
+
* - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
|
|
49
|
+
* real image out of lazy-load `<noscript><img …>` wrappers before
|
|
50
|
+
* `_removeScripts` deletes the noscript; pre-stripping it would silently
|
|
51
|
+
* drop those images from the markdown.
|
|
52
|
+
* - `template` — removed: its children parse into the inert `.content`
|
|
53
|
+
* fragment (invisible to Readability's traversal), DOMPurify's
|
|
54
|
+
* FORBID_TAGS drops it, and the tool layer's renderer removes it too.
|
|
55
|
+
* - `style` stays — its text cannot terminate a re-parse except at its own
|
|
56
|
+
* end tag, and the pipeline already drops it.
|
|
57
|
+
*/
|
|
58
|
+
export function stripScriptingElementsIn(doc: Document): void {
|
|
59
|
+
for (const el of Array.from(doc.querySelectorAll('script, template'))) {
|
|
60
|
+
if (el.tagName === 'SCRIPT' && (el.getAttribute('type') ?? '').trim().toLowerCase() === 'application/ld+json') {
|
|
61
|
+
// '<\/' is still valid JSON, so the re-parsed textContent parses to the
|
|
62
|
+
// same object — while the serialized form can no longer close the
|
|
63
|
+
// script element early.
|
|
64
|
+
el.textContent = (el.textContent ?? '').replace(/<\//g, '<\\/')
|
|
65
|
+
continue
|
|
66
|
+
}
|
|
67
|
+
el.remove()
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** {@link stripScriptingElementsIn} over the live document, as an evaluate expression. */
|
|
72
|
+
export const STRIP_SCRIPTING_ELEMENTS = `(${stripScriptingElementsIn.toString()})(document)`
|
package/src/types.ts
CHANGED
|
@@ -13,6 +13,11 @@ export interface PlaywrightResponse {
|
|
|
13
13
|
status(): number
|
|
14
14
|
headers(): Record<string, string>
|
|
15
15
|
text(): Promise<string>
|
|
16
|
+
/**
|
|
17
|
+
* The raw body bytes — the PDF path reads binary bodies through it (a
|
|
18
|
+
* Node `Buffer` is a `Uint8Array`). Absent on minimal fakes.
|
|
19
|
+
*/
|
|
20
|
+
body?(): Promise<Uint8Array>
|
|
16
21
|
/** This response's URL; absent on minimal fakes. */
|
|
17
22
|
url?(): string
|
|
18
23
|
/**
|