dsh-web-fetch-playwright 0.2.18 → 0.2.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/provider.ts CHANGED
@@ -33,6 +33,15 @@
33
33
  * {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
34
34
  * a queue and fail fast (rather than hang until abort) when no slot frees.
35
35
  *
36
+ * PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
37
+ * handed to the download manager as an attachment, or served as an opaque
38
+ * `application/octet-stream` whose bytes start with `%PDF-` — is read as
39
+ * bytes (the navigation body, else a refetch through the page's API request
40
+ * context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
41
+ * plus layout heuristics: headings, paragraphs, lists, tables, page
42
+ * markers). Scans without a text layer, encrypted, oversized, and broken
43
+ * PDFs fail with the `WEB_PDF_*` codes.
44
+ *
36
45
  * Private-network and SSRF protection is not implemented (same stance as the
37
46
  * shipped HTTP provider); a page this provider can reach is whatever the
38
47
  * browser can reach. Profile mode additionally acts WITH the remote
@@ -87,6 +96,7 @@ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveCh
87
96
  import type { ResolvedConfig } from './config.ts'
88
97
  import { CdpConnectionPool } from './cdp-pool.ts'
89
98
  import { htmlToMarkdown } from './markdown.ts'
99
+ import { extractPdfMarkdown, isPdfBytes, isPdfContentType, mayBePdf, MAX_PDF_BYTES, WEB_PDF_INVALID_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
90
100
  import { STRIP_SCRIPTING_ELEMENTS } from './strip.ts'
91
101
  import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
92
102
  import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
@@ -288,12 +298,16 @@ function validateFetchUrl(input: string): URL {
288
298
  return url
289
299
  }
290
300
 
291
- /** The decodable body kinds, mirroring the HTTP provider's classification. */
292
- type FetchableKind = 'html' | 'text'
301
+ /**
302
+ * The decodable body kinds: the HTTP provider's classification plus `pdf`,
303
+ * which this provider extracts to markdown (see `src/pdf.ts`).
304
+ */
305
+ type FetchableKind = 'html' | 'text' | 'pdf'
293
306
 
294
307
  /** Classify a `Content-Type` into a decodable kind; undefined = unsupported. */
295
308
  function classifyContentType(contentType: string | undefined): FetchableKind | undefined {
296
309
  const mime = (contentType ?? '').replace(/;.*$/s, '').trim().toLowerCase()
310
+ if (isPdfContentType(mime)) return 'pdf'
297
311
  if (mime === '' || mime === 'text/html' || mime === 'application/xhtml+xml') return 'html'
298
312
  if (mime.startsWith('text/')) return 'text'
299
313
  if (mime === 'application/json' || mime === 'application/xml' || mime.endsWith('+json') || mime.endsWith('+xml')) return 'text'
@@ -543,15 +557,24 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
543
557
  // ran (the real page reloads in), else the response goto returned.
544
558
  const finalResponse = (challengeWaitMs > 0 ? tracker.last() : undefined) ?? response
545
559
  const kind = classifyContentType(finalResponse?.headers()['content-type'])
546
- if (kind === undefined) {
547
- throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
548
- }
549
560
  const finalUrl = page.url()
550
561
  // An SPA-style clear swaps the document without navigating: no new
551
562
  // response exists to report, so the cleared document reads as served.
552
563
  const clearedWithoutNavigation = challengeEntryResponse !== null && finalResponse === challengeEntryResponse
553
564
  const statusCode = finalResponse !== null && !clearedWithoutNavigation ? finalResponse.status() : 200
554
565
 
566
+ // A PDF the browser committed as a document (its built-in viewer — a
567
+ // headed or CDP-attached Chrome) or an opaque binary that may be one:
568
+ // read the bytes and extract them, never the viewer's wrapper DOM.
569
+ if (kind === 'pdf' || (kind === undefined && finalResponse !== null && mayBePdf(finalResponse.headers(), finalUrl))) {
570
+ const download = await downloadPdfBytes(page, finalResponse, finalResponse?.url?.() ?? finalUrl, deadline)
571
+ if (download.pdf) return await pdfResult(download.bytes, download.url, statusCode, deadline)
572
+ if (kind === 'pdf') throw new WebError('the response claims to be a PDF but its bytes are not one', WEB_PDF_INVALID_CODE)
573
+ }
574
+ if (kind === undefined) {
575
+ throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
576
+ }
577
+
555
578
  // Non-HTML decodes straight from the response body; no denoise applies.
556
579
  if (kind === 'text') {
557
580
  const text = finalResponse !== null ? await finalResponse.text() : await page.content()
@@ -820,7 +843,7 @@ async function recoverUnrenderedStatus(
820
843
  // class only fires on empty bodies, so '' is the honest fallback.
821
844
  let content = ''
822
845
  try { content = await response.text() } catch { content = '' }
823
- if (kind === 'text' || content === '') {
846
+ if (kind !== 'html' || content === '') {
824
847
  // Empty HTML needs no pipeline — the empty string IS the body.
825
848
  return capResult(finalUrl, status, { kind: 'text', content })
826
849
  }
@@ -866,7 +889,9 @@ function isDownloadFailure(error: unknown, response: PlaywrightResponse | null):
866
889
  * download decision — which works identically on the local and CDP
867
890
  * backends because it never touches the remote artifact.
868
891
  *
869
- * Undecodable downloads (a PDF, a ZIP) still fail with
892
+ * A PDF download (declared `application/pdf`, or an opaque binary whose
893
+ * bytes sniff as `%PDF-`) is extracted to markdown (`src/pdf.ts`);
894
+ * undecodable downloads (a ZIP) still fail with
870
895
  * `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
871
896
  * actually served; a playwright build without a request context surfaces a
872
897
  * clear "the browser will not render this" error instead of the raw
@@ -893,8 +918,14 @@ async function recoverDownloadResponse(
893
918
  if (!isDownloadFailure(error, tracked)) return undefined
894
919
  if (tracked !== null) {
895
920
  // What the site served the navigation is the truth about the resource:
896
- // an undecodable download (PDF/ZIP/…) has no business being refetched.
921
+ // an undecodable download (ZIP/…) has no business being refetched — a
922
+ // PDF (declared, or an opaque binary that may be one) is the exception.
897
923
  const trackedKind = classifyContentType(tracked.headers()['content-type'])
924
+ if (trackedKind === 'pdf' || (trackedKind === undefined && mayBePdf(tracked.headers(), tracked.url?.() ?? url.toString()))) {
925
+ const download = await downloadPdfBytes(page, null, tracked.url?.() ?? url.toString(), deadline, tracked.headers())
926
+ if (download.pdf) return await pdfResult(download.bytes, download.url, download.status, deadline)
927
+ throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download; not a PDF)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
928
+ }
898
929
  if (trackedKind === undefined) {
899
930
  throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
900
931
  }
@@ -913,6 +944,13 @@ async function recoverDownloadResponse(
913
944
  // answer falls back to what the navigation served.
914
945
  const servedType = (headers['content-type'] ?? '').trim() !== '' ? headers['content-type'] : tracked?.headers()['content-type']
915
946
  const kind = classifyContentType(servedType)
947
+ if (kind === 'pdf') {
948
+ // Untracked navigation, but the refetch says PDF: extract it.
949
+ assertPdfSize(headers)
950
+ const bytes = refetched.body !== undefined ? await refetched.body() : new Uint8Array()
951
+ if (!isPdfBytes(bytes)) throw new WebError('the response claims to be a PDF but its bytes are not one', WEB_PDF_INVALID_CODE)
952
+ return await pdfResult(bytes, refetched.url?.() ?? fetchUrl, refetched.status(), deadline)
953
+ }
916
954
  if (kind === undefined) {
917
955
  throw new WebError(`unsupported content type "${servedType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
918
956
  }
@@ -927,6 +965,77 @@ async function recoverDownloadResponse(
927
965
  return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
928
966
  }
929
967
 
968
+ /** Time kept back from the PDF page walk for layout and result assembly. */
969
+ const PDF_FINISH_RESERVE_MS = 1_500
970
+
971
+ /** Refuse a PDF whose declared length is over the limit — before downloading it. */
972
+ function assertPdfSize(headers: Record<string, string> | undefined): void {
973
+ const declared = Number.parseInt(headers?.['content-length'] ?? '', 10)
974
+ if (Number.isFinite(declared) && declared > MAX_PDF_BYTES) {
975
+ throw new WebError(
976
+ `the PDF is ${String(Math.round(declared / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`,
977
+ WEB_PDF_TOO_LARGE_CODE,
978
+ )
979
+ }
980
+ }
981
+
982
+ /**
983
+ * Get a (possibly) PDF resource's bytes. The navigation response's own body
984
+ * is tried first — a PDF Chrome committed into its viewer still has it —
985
+ * then the page's API request context refetches the URL (same cookie jar,
986
+ * plain HTTP, no render-or-download decision; the only way to reach a body
987
+ * Chromium streamed to its download manager). The `%PDF-` magic decides.
988
+ *
989
+ * @param page - the fetch's tab (for its request context).
990
+ * @param response - the navigation response, when the browser committed one.
991
+ * @param url - the URL to refetch.
992
+ * @param deadline - the fetch budget.
993
+ * @param servedHeaders - the navigation's headers (size pre-check) when no
994
+ * response object is passed.
995
+ * @returns the bytes, where they came from, and whether they are a PDF.
996
+ */
997
+ async function downloadPdfBytes(
998
+ page: PlaywrightPage,
999
+ response: PlaywrightResponse | null,
1000
+ url: string,
1001
+ deadline: Deadline,
1002
+ servedHeaders?: Record<string, string>,
1003
+ ): Promise<{ pdf: boolean; bytes: Uint8Array; url: string; status: number }> {
1004
+ assertPdfSize(response?.headers() ?? servedHeaders)
1005
+ if (response?.body !== undefined) {
1006
+ try {
1007
+ const bytes = await response.body()
1008
+ if (isPdfBytes(bytes)) return { pdf: true, bytes, url: response.url?.() ?? url, status: response.status() }
1009
+ } catch {
1010
+ // the viewer / download manager owns the body — refetch below
1011
+ }
1012
+ }
1013
+ if (page.request === undefined) {
1014
+ throw new WebError(
1015
+ 'the response is a PDF the browser will not hand over, and this playwright build exposes no request context to refetch it',
1016
+ 'WEB_PROVIDER_ERROR',
1017
+ )
1018
+ }
1019
+ const refetched = await page.request.get(url, { timeout: deadline.remainingMs(), signal: deadline.signal })
1020
+ assertPdfSize(refetched.headers())
1021
+ const bytes = refetched.body !== undefined ? await refetched.body() : new Uint8Array()
1022
+ return { pdf: isPdfBytes(bytes), bytes, url: refetched.url?.() ?? url, status: refetched.status() }
1023
+ }
1024
+
1025
+ /**
1026
+ * Extract PDF bytes to a markdown text result inside the fetch budget.
1027
+ * A limit that stopped the page walk early flags the result truncated.
1028
+ */
1029
+ async function pdfResult(bytes: Uint8Array, finalUrl: string, statusCode: number, deadline: Deadline): Promise<WebFetchResult> {
1030
+ const extraction = await extractPdfMarkdown(bytes, {
1031
+ signal: deadline.signal,
1032
+ timeBudgetMs: Math.max(1_000, deadline.remainingMs() - PDF_FINISH_RESERVE_MS),
1033
+ maxChars: MAX_BODY_CHARS,
1034
+ })
1035
+ const result = capResult(finalUrl, statusCode, { kind: 'text', content: extraction.markdown })
1036
+ return extraction.truncated && !result.truncated ? { ...result, truncated: true } : result
1037
+ }
1038
+
930
1039
  /**
931
1040
  * Probe the live document for challenge markers. Prefers the in-page probe
932
1041
  * (`page.evaluate`) — the only thing that can see SPA-style clears — and