dsh-web-fetch-playwright 0.2.16 → 0.2.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/provider.ts CHANGED
@@ -33,6 +33,15 @@
33
33
  * {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
34
34
  * a queue and fail fast (rather than hang until abort) when no slot frees.
35
35
  *
36
+ * PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
37
+ * handed to the download manager as an attachment, or served as an opaque
38
+ * `application/octet-stream` whose bytes start with `%PDF-` — is read as
39
+ * bytes (the navigation body, else a refetch through the page's API request
40
+ * context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
41
+ * plus layout heuristics: headings, paragraphs, lists, tables, page
42
+ * markers). Scans without a text layer, encrypted, oversized, and broken
43
+ * PDFs fail with the `WEB_PDF_*` codes.
44
+ *
36
45
  * Private-network and SSRF protection is not implemented (same stance as the
37
46
  * shipped HTTP provider); a page this provider can reach is whatever the
38
47
  * browser can reach. Profile mode additionally acts WITH the remote
@@ -50,6 +59,31 @@
50
59
  * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
51
60
  * interstitial as content.
52
61
  *
62
+ * Deep capture (viewport-virtualized feeds and app-shell documents): feed-
63
+ * shaped pages — Discourse topic streams, comment threads, timelines —
64
+ * render only the items near the scroll position; everything else is an
65
+ * empty, height-only placeholder that keeps the item's id but none of its
66
+ * content, so a single-instant capture hands back the skeleton while the
67
+ * bodies silently vanish (and the denoise pipeline's coverage gate cannot
68
+ * notice: the page's own visible text is the same windowed fragment as the
69
+ * extraction). App-shell document editors (Feishu docs) push the same
70
+ * pathology one level down: the body never scrolls, blocks are keyed by
71
+ * data-block-id with no id, and off-window blocks are REMOVED behind an
72
+ * anonymous height spacer — the fetch would return just the statically
73
+ * rendered first screen. Before serialization the fetch therefore runs one
74
+ * bounded in-page scroll walk — over the window or, when the document does
75
+ * not scroll, the page's dominant inner scroll container — stashing what
76
+ * each rendered window shows, terminating only AT the bottom with the
77
+ * height stable (that is what pages load-more sentinels; height stability
78
+ * elsewhere proves nothing), then rehydrating degraded keyed elements from
79
+ * their stashed snapshots and splicing removed keys back at their document
80
+ * position so the final DOM holds the union of every window in document
81
+ * order. The existing pipeline then runs unchanged. Bounded by
82
+ * `scrollBudgetMs` (default 20s; 0 restores the legacy single-instant
83
+ * behavior) and by the remaining fetch deadline; any walk failure silently
84
+ * degrades to the legacy capture (see `src/autoscroll.ts` for the measured
85
+ * linux.do and Feishu badcases).
86
+ *
53
87
  * @module dsh-web-fetch-playwright/provider
54
88
  */
55
89
 
@@ -57,10 +91,13 @@ import { WebError } from '@deepseek-ai/dsh-web'
57
91
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
58
92
  import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
59
93
  import type { ChallengeVerdict } from './challenge.ts'
60
- import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
94
+ import { deepCaptureWalk, SCROLL_FINISH_RESERVE_MS } from './autoscroll.ts'
95
+ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
61
96
  import type { ResolvedConfig } from './config.ts'
62
97
  import { CdpConnectionPool } from './cdp-pool.ts'
63
98
  import { htmlToMarkdown } from './markdown.ts'
99
+ import { extractPdfMarkdown, isPdfBytes, isPdfContentType, mayBePdf, MAX_PDF_BYTES, WEB_PDF_INVALID_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
100
+ import { STRIP_SCRIPTING_ELEMENTS } from './strip.ts'
64
101
  import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
65
102
  import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
66
103
 
@@ -261,12 +298,16 @@ function validateFetchUrl(input: string): URL {
261
298
  return url
262
299
  }
263
300
 
264
- /** The decodable body kinds, mirroring the HTTP provider's classification. */
265
- type FetchableKind = 'html' | 'text'
301
+ /**
302
+ * The decodable body kinds: the HTTP provider's classification plus `pdf`,
303
+ * which this provider extracts to markdown (see `src/pdf.ts`).
304
+ */
305
+ type FetchableKind = 'html' | 'text' | 'pdf'
266
306
 
267
307
  /** Classify a `Content-Type` into a decodable kind; undefined = unsupported. */
268
308
  function classifyContentType(contentType: string | undefined): FetchableKind | undefined {
269
309
  const mime = (contentType ?? '').replace(/;.*$/s, '').trim().toLowerCase()
310
+ if (isPdfContentType(mime)) return 'pdf'
270
311
  if (mime === '' || mime === 'text/html' || mime === 'application/xhtml+xml') return 'html'
271
312
  if (mime.startsWith('text/')) return 'text'
272
313
  if (mime === 'application/json' || mime === 'application/xml' || mime.endsWith('+json') || mime.endsWith('+xml')) return 'text'
@@ -516,15 +557,24 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
516
557
  // ran (the real page reloads in), else the response goto returned.
517
558
  const finalResponse = (challengeWaitMs > 0 ? tracker.last() : undefined) ?? response
518
559
  const kind = classifyContentType(finalResponse?.headers()['content-type'])
519
- if (kind === undefined) {
520
- throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
521
- }
522
560
  const finalUrl = page.url()
523
561
  // An SPA-style clear swaps the document without navigating: no new
524
562
  // response exists to report, so the cleared document reads as served.
525
563
  const clearedWithoutNavigation = challengeEntryResponse !== null && finalResponse === challengeEntryResponse
526
564
  const statusCode = finalResponse !== null && !clearedWithoutNavigation ? finalResponse.status() : 200
527
565
 
566
+ // A PDF the browser committed as a document (its built-in viewer — a
567
+ // headed or CDP-attached Chrome) or an opaque binary that may be one:
568
+ // read the bytes and extract them, never the viewer's wrapper DOM.
569
+ if (kind === 'pdf' || (kind === undefined && finalResponse !== null && mayBePdf(finalResponse.headers(), finalUrl))) {
570
+ const download = await downloadPdfBytes(page, finalResponse, finalResponse?.url?.() ?? finalUrl, deadline)
571
+ if (download.pdf) return await pdfResult(download.bytes, download.url, statusCode, deadline)
572
+ if (kind === 'pdf') throw new WebError('the response claims to be a PDF but its bytes are not one', WEB_PDF_INVALID_CODE)
573
+ }
574
+ if (kind === undefined) {
575
+ throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
576
+ }
577
+
528
578
  // Non-HTML decodes straight from the response body; no denoise applies.
529
579
  if (kind === 'text') {
530
580
  const text = finalResponse !== null ? await finalResponse.text() : await page.content()
@@ -535,14 +585,31 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
535
585
  // what domcontentloaded already produced.
536
586
  await page.waitForLoadState('networkidle', { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {})
537
587
 
538
- // Serialization safety: the document is about to be flattened into an
539
- // HTML string and re-parsed by jsdom, and scripting elements are the one
540
- // place that round-trip can break — see {@link STRIP_SCRIPTING_ELEMENTS}
541
- // for exactly what is removed, what is kept (JSON-LD, noscript), and why
542
- // each removal cannot change what the pipeline extracts.
543
- await stripScriptingElements(page)
588
+ // Deep capture (viewport-virtualized feeds and app-shell documents lose
589
+ // their off-window content at rest — see `src/autoscroll.ts`): a bounded
590
+ // walk stashes what each rendered window shows and, when it restored
591
+ // anything, hands back its own merged, already-stripped serialization.
592
+ // HTML documents only, inside the deadline with a decode reserve,
593
+ // silently degrading to the legacy capture; `scrollBudgetMs: 0` skips it.
594
+ const scrollBudgetMs = Math.min(
595
+ effectiveScrollBudgetMs(config),
596
+ deadline.remainingMs() - SCROLL_FINISH_RESERVE_MS,
597
+ )
598
+ const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : undefined
544
599
 
545
- const html = await page.content()
600
+ // Serialization safety for the live read: the document is about to be
601
+ // flattened into an HTML string and re-parsed by jsdom, and scripting
602
+ // elements are the one place that round-trip can break — see
603
+ // `src/strip.ts` for exactly what is removed, what is kept (JSON-LD,
604
+ // noscript), and why each removal cannot change what the pipeline
605
+ // extracts. (The walk's merged capture went through the same strip.)
606
+ let html: string
607
+ if (captured?.html !== undefined) {
608
+ html = captured.html
609
+ } else {
610
+ await stripScriptingElements(page)
611
+ html = await page.content()
612
+ }
546
613
  if (!config.denoise) {
547
614
  // The tool layer's own turndown renders raw HTML; the checkbox only
548
615
  // governs the Readability/DOMPurify stage this provider owns.
@@ -776,7 +843,7 @@ async function recoverUnrenderedStatus(
776
843
  // class only fires on empty bodies, so '' is the honest fallback.
777
844
  let content = ''
778
845
  try { content = await response.text() } catch { content = '' }
779
- if (kind === 'text' || content === '') {
846
+ if (kind !== 'html' || content === '') {
780
847
  // Empty HTML needs no pipeline — the empty string IS the body.
781
848
  return capResult(finalUrl, status, { kind: 'text', content })
782
849
  }
@@ -822,7 +889,9 @@ function isDownloadFailure(error: unknown, response: PlaywrightResponse | null):
822
889
  * download decision — which works identically on the local and CDP
823
890
  * backends because it never touches the remote artifact.
824
891
  *
825
- * Undecodable downloads (a PDF, a ZIP) still fail with
892
+ * A PDF download (declared `application/pdf`, or an opaque binary whose
893
+ * bytes sniff as `%PDF-`) is extracted to markdown (`src/pdf.ts`);
894
+ * undecodable downloads (a ZIP) still fail with
826
895
  * `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
827
896
  * actually served; a playwright build without a request context surfaces a
828
897
  * clear "the browser will not render this" error instead of the raw
@@ -849,8 +918,14 @@ async function recoverDownloadResponse(
849
918
  if (!isDownloadFailure(error, tracked)) return undefined
850
919
  if (tracked !== null) {
851
920
  // What the site served the navigation is the truth about the resource:
852
- // an undecodable download (PDF/ZIP/…) has no business being refetched.
921
+ // an undecodable download (ZIP/…) has no business being refetched — a
922
+ // PDF (declared, or an opaque binary that may be one) is the exception.
853
923
  const trackedKind = classifyContentType(tracked.headers()['content-type'])
924
+ if (trackedKind === 'pdf' || (trackedKind === undefined && mayBePdf(tracked.headers(), tracked.url?.() ?? url.toString()))) {
925
+ const download = await downloadPdfBytes(page, null, tracked.url?.() ?? url.toString(), deadline, tracked.headers())
926
+ if (download.pdf) return await pdfResult(download.bytes, download.url, download.status, deadline)
927
+ throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download; not a PDF)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
928
+ }
854
929
  if (trackedKind === undefined) {
855
930
  throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
856
931
  }
@@ -869,6 +944,13 @@ async function recoverDownloadResponse(
869
944
  // answer falls back to what the navigation served.
870
945
  const servedType = (headers['content-type'] ?? '').trim() !== '' ? headers['content-type'] : tracked?.headers()['content-type']
871
946
  const kind = classifyContentType(servedType)
947
+ if (kind === 'pdf') {
948
+ // Untracked navigation, but the refetch says PDF: extract it.
949
+ assertPdfSize(headers)
950
+ const bytes = refetched.body !== undefined ? await refetched.body() : new Uint8Array()
951
+ if (!isPdfBytes(bytes)) throw new WebError('the response claims to be a PDF but its bytes are not one', WEB_PDF_INVALID_CODE)
952
+ return await pdfResult(bytes, refetched.url?.() ?? fetchUrl, refetched.status(), deadline)
953
+ }
872
954
  if (kind === undefined) {
873
955
  throw new WebError(`unsupported content type "${servedType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
874
956
  }
@@ -883,6 +965,77 @@ async function recoverDownloadResponse(
883
965
  return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
884
966
  }
885
967
 
968
+ /** Time kept back from the PDF page walk for layout and result assembly. */
969
+ const PDF_FINISH_RESERVE_MS = 1_500
970
+
971
+ /** Refuse a PDF whose declared length is over the limit — before downloading it. */
972
+ function assertPdfSize(headers: Record<string, string> | undefined): void {
973
+ const declared = Number.parseInt(headers?.['content-length'] ?? '', 10)
974
+ if (Number.isFinite(declared) && declared > MAX_PDF_BYTES) {
975
+ throw new WebError(
976
+ `the PDF is ${String(Math.round(declared / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`,
977
+ WEB_PDF_TOO_LARGE_CODE,
978
+ )
979
+ }
980
+ }
981
+
982
+ /**
983
+ * Get a (possibly) PDF resource's bytes. The navigation response's own body
984
+ * is tried first — a PDF Chrome committed into its viewer still has it —
985
+ * then the page's API request context refetches the URL (same cookie jar,
986
+ * plain HTTP, no render-or-download decision; the only way to reach a body
987
+ * Chromium streamed to its download manager). The `%PDF-` magic decides.
988
+ *
989
+ * @param page - the fetch's tab (for its request context).
990
+ * @param response - the navigation response, when the browser committed one.
991
+ * @param url - the URL to refetch.
992
+ * @param deadline - the fetch budget.
993
+ * @param servedHeaders - the navigation's headers (size pre-check) when no
994
+ * response object is passed.
995
+ * @returns the bytes, where they came from, and whether they are a PDF.
996
+ */
997
+ async function downloadPdfBytes(
998
+ page: PlaywrightPage,
999
+ response: PlaywrightResponse | null,
1000
+ url: string,
1001
+ deadline: Deadline,
1002
+ servedHeaders?: Record<string, string>,
1003
+ ): Promise<{ pdf: boolean; bytes: Uint8Array; url: string; status: number }> {
1004
+ assertPdfSize(response?.headers() ?? servedHeaders)
1005
+ if (response?.body !== undefined) {
1006
+ try {
1007
+ const bytes = await response.body()
1008
+ if (isPdfBytes(bytes)) return { pdf: true, bytes, url: response.url?.() ?? url, status: response.status() }
1009
+ } catch {
1010
+ // the viewer / download manager owns the body — refetch below
1011
+ }
1012
+ }
1013
+ if (page.request === undefined) {
1014
+ throw new WebError(
1015
+ 'the response is a PDF the browser will not hand over, and this playwright build exposes no request context to refetch it',
1016
+ 'WEB_PROVIDER_ERROR',
1017
+ )
1018
+ }
1019
+ const refetched = await page.request.get(url, { timeout: deadline.remainingMs(), signal: deadline.signal })
1020
+ assertPdfSize(refetched.headers())
1021
+ const bytes = refetched.body !== undefined ? await refetched.body() : new Uint8Array()
1022
+ return { pdf: isPdfBytes(bytes), bytes, url: refetched.url?.() ?? url, status: refetched.status() }
1023
+ }
1024
+
1025
+ /**
1026
+ * Extract PDF bytes to a markdown text result inside the fetch budget.
1027
+ * A limit that stopped the page walk early flags the result truncated.
1028
+ */
1029
+ async function pdfResult(bytes: Uint8Array, finalUrl: string, statusCode: number, deadline: Deadline): Promise<WebFetchResult> {
1030
+ const extraction = await extractPdfMarkdown(bytes, {
1031
+ signal: deadline.signal,
1032
+ timeBudgetMs: Math.max(1_000, deadline.remainingMs() - PDF_FINISH_RESERVE_MS),
1033
+ maxChars: MAX_BODY_CHARS,
1034
+ })
1035
+ const result = capResult(finalUrl, statusCode, { kind: 'text', content: extraction.markdown })
1036
+ return extraction.truncated && !result.truncated ? { ...result, truncated: true } : result
1037
+ }
1038
+
886
1039
  /**
887
1040
  * Probe the live document for challenge markers. Prefers the in-page probe
888
1041
  * (`page.evaluate`) — the only thing that can see SPA-style clears — and
@@ -907,68 +1060,13 @@ async function probeStillOnChallenge(page: PlaywrightPage): Promise<boolean> {
907
1060
  }
908
1061
  }
909
1062
 
910
- /**
911
- * In-page script that removes non-content scripting subtrees from the live
912
- * document before `page.content()` reads it as an HTML string.
913
- *
914
- * WHAT goes, and why each removal is output-neutral versus the old pipeline:
915
- *
916
- * - `script` — three consumers ever saw script text before, and none loses
917
- * it here: Readability's `parse()` deletes script/noscript nodes BEFORE
918
- * extraction and scoring (so script text never reached the article or
919
- * influenced the scorer); DOMPurify's default allowlist drops script; the
920
- * tool layer's renderer removes SCRIPT wholesale on the raw path. What the
921
- * removal buys is the reason this exists: a page's runtime JS may rewrite
922
- * a data script's text with a literal `</script>` embedded — measured on
923
- * techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
924
- * Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
925
- * turning the wire's 712 `\u003c` escapes back into raw `<` — and
926
- * browsers serialize script text verbatim, so the flattened string ends
927
- * that script at the FIRST embedded end tag and re-parses the remaining
928
- * ~270KB of JSON as markup. The phantom elements that survive (unknown
929
- * tags like `<id>`, anchors full of JSON text) are exactly what
930
- * Readability's scorer reads as article mass, so the extracted "article"
931
- * is CSS and i18n strings while the real body — still present in the DOM —
932
- * loses. No string-level patch can undo this after serialization (the
933
- * string is genuinely ambiguous); only the browser's own parse holds the
934
- * document's true structure.
935
- * - `script[type="application/ld+json"]` — KEPT, but defused: Readability
936
- * extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
937
- * "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
938
- * scripts before removing them, so deleting them would regress the title
939
- * on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
940
- * valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
941
- * same object) while making the serialization harmless: the tokenizer
942
- * only terminates on a literal `</script`.
943
- * - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
944
- * real image out of lazy-load `<noscript><img …>` wrappers before
945
- * `_removeScripts` deletes the noscript; pre-stripping it would silently
946
- * drop those images from the markdown.
947
- * - `template` — removed: its children parse into the inert `.content`
948
- * fragment (invisible to Readability's traversal), DOMPurify's
949
- * FORBID_TAGS drops it, and the tool layer's renderer removes it too.
950
- * - `style` stays — its text cannot terminate a re-parse except at its own
951
- * end tag, and the pipeline already drops it.
952
- */
953
- const STRIP_SCRIPTING_ELEMENTS = `(() => {
954
- for (const el of Array.from(document.querySelectorAll('script, template'))) {
955
- if (el.tagName === 'SCRIPT' && (el.getAttribute('type') || '').trim().toLowerCase() === 'application/ld+json') {
956
- // '<\\/' is still valid JSON, so the re-parsed textContent parses to
957
- // the same object — while the serialized form can no longer close the
958
- // script element early.
959
- el.textContent = el.textContent.replace(/<\\//g, '<\\\\/')
960
- continue
961
- }
962
- el.remove()
963
- }
964
- })()`
965
-
966
1063
  /**
967
1064
  * Remove non-content scripting subtrees from the live document before
968
- * serialization. Best-effort, mirroring {@link probeStillOnChallenge}: a
969
- * backend without `evaluate` (fakes, minimal engines) simply keeps the
970
- * document as-is — the fetch never fails here, and the pipeline degrades to
971
- * whatever the serialized string itself says.
1065
+ * serialization (see `stripScriptingElementsIn` in `src/strip.ts` for exactly
1066
+ * what goes and why). Best-effort, mirroring {@link probeStillOnChallenge}: a backend
1067
+ * without `evaluate` (fakes, minimal engines) simply keeps the document
1068
+ * as-is — the fetch never fails here, and the pipeline degrades to whatever
1069
+ * the serialized string itself says.
972
1070
  */
973
1071
  async function stripScriptingElements(page: PlaywrightPage): Promise<void> {
974
1072
  if (typeof page.evaluate !== 'function') return
@@ -18,6 +18,10 @@ interface TargetIdentity { targetId: string; browserContextId: string }
18
18
  const ownership = new WeakMap<PlaywrightBrowser, Map<string, WeakRef<PlaywrightContext>>>()
19
19
  // Never adopt an unconfirmed late tab after timing out, nor close sibling leases.
20
20
  const unavailable = new WeakSet<PlaywrightContext>()
21
+ // Identities confirmed when each session tab was created. Re-identifying a busy
22
+ // sibling would attach to it, and detaching from a tab mid-navigation can stall
23
+ // until its response arrives; a target id and its context never change.
24
+ const identities = new WeakMap<PlaywrightPage, TargetIdentity>()
21
25
  const CLEANUP_GRACE_MS = 100
22
26
 
23
27
  function claim(browser: PlaywrightBrowser, context: PlaywrightContext, keys: string[]): void {
@@ -112,6 +116,8 @@ export async function createSessionPage(
112
116
  return session
113
117
  }
114
118
  const identify = async (page: PlaywrightPage): Promise<TargetIdentity> => {
119
+ const known = identities.get(page)
120
+ if (known) return known
115
121
  const session = await newSession(() => ctx.newCDPSession(page))
116
122
  let identificationFailed = false
117
123
  try {
@@ -164,6 +170,7 @@ export async function createSessionPage(
164
170
  throw new Error('Chromium did not place the session tab in its dedicated window')
165
171
  }
166
172
  claim(browser, context, [`context:${actual.browserContextId}`, `window:${expectedWindow}`])
173
+ identities.set(created, actual)
167
174
  succeeded = true
168
175
  return created
169
176
  }
@@ -209,6 +216,7 @@ export async function createSessionPage(
209
216
  const explicitWindow = await windowOf(explicitTargetId)
210
217
  if (explicitWindow === bootstrapWindow) throw new Error('Chromium ignored the dedicated session window request')
211
218
  claim(browser, context, [`context:${identity.browserContextId}`, `window:${explicitWindow}`])
219
+ identities.set(created, { targetId: explicitTargetId, browserContextId: identity.browserContextId })
212
220
  await deadline.run(() => bootstrap!.close())
213
221
  bootstrap = undefined
214
222
  succeeded = true
package/src/strip.ts ADDED
@@ -0,0 +1,72 @@
1
+ /**
2
+ * Serialization safety for documents about to be flattened into an HTML
3
+ * string and re-parsed by jsdom: the one in-page cleanup both serialization
4
+ * paths share — the provider's live-document read and the deep-capture
5
+ * walk's merged clone.
6
+ *
7
+ * @module dsh-web-fetch-playwright/strip
8
+ */
9
+
10
+ /**
11
+ * In-page function that removes non-content scripting subtrees from a
12
+ * document before it is flattened into an HTML string — the live document
13
+ * before `page.content()`, or the deep-capture walk's merged clone before it
14
+ * serializes it itself (`src/autoscroll.ts`).
15
+ *
16
+ * It is evaluated BY SOURCE (`fn.toString()`), so it must stay
17
+ * self-contained: no imports, no module-level references.
18
+ *
19
+ * WHAT goes, and why each removal is output-neutral versus the old pipeline:
20
+ *
21
+ * - `script` — three consumers ever saw script text before, and none loses
22
+ * it here: Readability's `parse()` deletes script/noscript nodes BEFORE
23
+ * extraction and scoring (so script text never reached the article or
24
+ * influenced the scorer); DOMPurify's default allowlist drops script; the
25
+ * tool layer's renderer removes SCRIPT wholesale on the raw path. What the
26
+ * removal buys is the reason this exists: a page's runtime JS may rewrite
27
+ * a data script's text with a literal `</script>` embedded — measured on
28
+ * techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
29
+ * Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
30
+ * turning the wire's 712 `\u003c` escapes back into raw `<` — and
31
+ * browsers serialize script text verbatim, so the flattened string ends
32
+ * that script at the FIRST embedded end tag and re-parses the remaining
33
+ * ~270KB of JSON as markup. The phantom elements that survive (unknown
34
+ * tags like `<id>`, anchors full of JSON text) are exactly what
35
+ * Readability's scorer reads as article mass, so the extracted "article"
36
+ * is CSS and i18n strings while the real body — still present in the DOM —
37
+ * loses. No string-level patch can undo this after serialization (the
38
+ * string is genuinely ambiguous); only the browser's own parse holds the
39
+ * document's true structure.
40
+ * - `script[type="application/ld+json"]` — KEPT, but defused: Readability
41
+ * extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
42
+ * "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
43
+ * scripts before removing them, so deleting them would regress the title
44
+ * on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
45
+ * valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
46
+ * same object) while making the serialization harmless: the tokenizer
47
+ * only terminates on a literal `</script`.
48
+ * - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
49
+ * real image out of lazy-load `<noscript><img …>` wrappers before
50
+ * `_removeScripts` deletes the noscript; pre-stripping it would silently
51
+ * drop those images from the markdown.
52
+ * - `template` — removed: its children parse into the inert `.content`
53
+ * fragment (invisible to Readability's traversal), DOMPurify's
54
+ * FORBID_TAGS drops it, and the tool layer's renderer removes it too.
55
+ * - `style` stays — its text cannot terminate a re-parse except at its own
56
+ * end tag, and the pipeline already drops it.
57
+ */
58
+ export function stripScriptingElementsIn(doc: Document): void {
59
+ for (const el of Array.from(doc.querySelectorAll('script, template'))) {
60
+ if (el.tagName === 'SCRIPT' && (el.getAttribute('type') ?? '').trim().toLowerCase() === 'application/ld+json') {
61
+ // '<\/' is still valid JSON, so the re-parsed textContent parses to the
62
+ // same object — while the serialized form can no longer close the
63
+ // script element early.
64
+ el.textContent = (el.textContent ?? '').replace(/<\//g, '<\\/')
65
+ continue
66
+ }
67
+ el.remove()
68
+ }
69
+ }
70
+
71
+ /** {@link stripScriptingElementsIn} over the live document, as an evaluate expression. */
72
+ export const STRIP_SCRIPTING_ELEMENTS = `(${stripScriptingElementsIn.toString()})(document)`
package/src/types.ts CHANGED
@@ -13,6 +13,11 @@ export interface PlaywrightResponse {
13
13
  status(): number
14
14
  headers(): Record<string, string>
15
15
  text(): Promise<string>
16
+ /**
17
+ * The raw body bytes — the PDF path reads binary bodies through it (a
18
+ * Node `Buffer` is a `Uint8Array`). Absent on minimal fakes.
19
+ */
20
+ body?(): Promise<Uint8Array>
16
21
  /** This response's URL; absent on minimal fakes. */
17
22
  url?(): string
18
23
  /**