@browserless/pdf 13.6.8 → 13.6.9

package/README.md

@@ -73,7 +73,6 @@ The package applies these optimized defaults:
 | `scale` | `number` | `0.65` | Scale of the webpage rendering (0.1 - 2) |
 | `printBackground` | `boolean` | `true` | Print background graphics |
 | `waitUntil` | `string` | `'auto'` | When to consider navigation done |
-| `waitForDom` | `number` | `0` | DOM stability window in ms (idle is `waitForDom / 10`, `0` disables DOM wait) |
 | `format` | `string` | `'Letter'` | Paper format (A4, Letter, etc.) |
 | `landscape` | `boolean` | `false` | Paper orientation |
 | `width` | `string \| number` | — | Paper width (overrides format) |
@@ -112,10 +111,9 @@ Supported units: `px`, `in`, `cm`, `mm`
 When `waitUntil: 'auto'` (the default), the package:
 
 1. Navigates to the page
-2. Optionally waits for DOM stability (`waitForDom`, default `0` = disabled)
-3. Takes a low-quality screenshot to check for white/blank content
-4. If the page appears blank, retries until content is detected
-5. Generates the PDF once content is confirmed
+2. Waits for paint/readiness signals (`waitForReady` + `isPageReady`)
+3. Hydrates overflow content when ready (`prepareFullDocument`)
+4. Generates the PDF
 
 This ensures dynamic content (JavaScript-rendered pages) is fully loaded before PDF generation.
 

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/pdf",
   "description": "Convert websites to high-quality PDFs with customizable margins, background printing, and optimized scaling.",
   "homepage": "https://browserless.js.org/#/?id=pdfurl-options",
-  "version": "13.6.8",
+  "version": "13.6.9",
   "main": "src",
   "author": {
     "email": "hello@microlink.io",
@@ -30,8 +30,8 @@
     "website-to-pdf"
   ],
   "dependencies": {
-    "@browserless/goto": "13.6.8",
-    "@browserless/screenshot": "13.6.8",
+    "@browserless/goto": "13.6.9",
+    "@browserless/screenshot": "13.6.9",
     "@kikobeats/time-span": "~1.0.12",
     "debug-logfmt": "~1.4.10",
     "p-reflect": "~2.1.0",
@@ -47,5 +47,5 @@
     "test": "exit 0"
   },
   "license": "MIT",
-  "gitHead": "1967ae39bd38666f20239ecc1befd484ed8349e9"
+  "gitHead": "c625834b3b37e036432a0756d67eccc149861ad0"
 }

package/src/index.js

@@ -8,31 +8,23 @@ const pReflect = require('p-reflect')
 const {
   captureWithNavigationRetry,
   isWhiteScreenshot,
-  waitForDomStability,
   waitForReady,
-  resolveWaitForDom,
+  prepareFullDocument,
+  expandOverflow,
+  checkPageReady,
+  tryHydrateScroll,
   SCREENSHOT_DEFAULT_OPTS
 } = require('@browserless/screenshot')
 
 const PDF_DEFAULT_OPTS = {
-  waitForDom: SCREENSHOT_DEFAULT_OPTS.waitForDom,
   margin: '0.35cm',
   scale: 0.65,
   printBackground: true,
-  waitUntil: 'auto'
+  waitUntil: 'auto',
+  isPageReady: SCREENSHOT_DEFAULT_OPTS.isPageReady
 }
 
-// Share of the phase's load allowance the readiness gate may consume in `auto`
-// mode. Pages observed settling in 0.6-3.3s (a hydrating document is the slow
-// end), so a quarter of the allowance — ~3.9s at the default request budget —
-// covers them with margin while keeping the cap well short of the render's
-// share. The gate returns as soon as the page is quiet, so this bounds only a
-// page that never settles.
 const READY_BUDGET_RATIO = 0.25
-
-// Minimum visible characters for the text fast path. Matches the counting cap
-// in `waitForReady`'s snapshot, which stops walking text nodes once reached —
-// raising this above the cap would make the text fast path unreachable.
 const TEXT_PAINTED_MIN = 200
 
 const getMargin = unit => {
@@ -46,140 +38,127 @@ const getMargin = unit => {
   }
 }
 
+const isPaintedContent = ({ painted = 0, text = 0, fonts = true } = {}) =>
+  painted > 0 || (text >= TEXT_PAINTED_MIN && fonts)
+
 module.exports = ({ goto, ...gotoOpts } = {}) => {
   goto = goto || createGoto(gotoOpts)
 
-  // Render an already-prepared page to a PDF buffer. Split out from the load so
-  // a single load can be reused across page-range chunks (microlink-api's
-  // parallel renderer) without re-navigating.
-  const render = (page, opts = {}) => {
+  const render = async (page, opts = {}) => {
     const {
       margin = PDF_DEFAULT_OPTS.margin,
       scale = PDF_DEFAULT_OPTS.scale,
       printBackground = PDF_DEFAULT_OPTS.printBackground,
-      waitUntil,
-      waitForDom,
       ...rest
     } = opts
+
+    await pReflect(expandOverflow(page))
+
     return captureWithNavigationRetry(
-      () => page.pdf({ ...rest, margin: getMargin(margin), printBackground, scale }),
+      () =>
+        page.pdf({
+          ...rest,
+          margin: getMargin(margin),
+          printBackground,
+          scale
+        }),
       { page, goto, timeout: goto.timeouts.action(rest.timeout) }
     )
   }
 
-  // Navigate `page` to `url` and wait until it is ready to print: DOM stability
-  // plus, in `auto` mode, a navigation-tolerant readiness gate. Only a page that
-  // settles still-blank falls back to the (expensive, navigation-fragile)
-  // screenshot poll.
   const prepare = async (page, url, opts = {}) => {
     const {
-      margin,
-      scale,
-      printBackground,
       waitUntil = PDF_DEFAULT_OPTS.waitUntil,
-      waitForDom = PDF_DEFAULT_OPTS.waitForDom,
+      isPageReady = PDF_DEFAULT_OPTS.isPageReady,
       ...rest
     } = opts
-    const waitForDomOpts = resolveWaitForDom(waitForDom)
-
-    const waitForDomStabilityResult = async page => {
-      if (!waitForDomOpts) return
-
-      const result = await pReflect(page.evaluate(waitForDomStability, waitForDomOpts))
-      debug(
-        'waitForDomStability',
-        result.isRejected
-          ? { ...waitForDomOpts, error: result.reason.message || result.reason }
-          : { ...waitForDomOpts, ...result.value }
-      )
-    }
 
     if (waitUntil !== 'auto') {
       await goto(page, { ...rest, url, waitUntil })
-      await waitForDomStabilityResult(page)
+      await prepareFullDocument(page, { goto, timeout: rest.timeout })
       return
     }
 
-    // Surfaced to the caller: a page whose readiness could not be confirmed
-    // (`timedOut`) is a poor one to keep rendering on, so a caller reusing this
-    // load across page-ranges can choose a fresh context instead.
     let readiness
+    let isReady = false
+    let didHydrateScroll = false
 
     await goto(page, { ...rest, url, waitUntil, waitUntilAuto })
+
+    if (isReady) {
+      const prep = await prepareFullDocument(page, {
+        goto,
+        timeout: rest.timeout,
+        scrolled: didHydrateScroll
+      })
+      readiness = { ...readiness, ...prep }
+    }
+
     return readiness
 
-    async function waitUntilAuto (page, { timeout: autoTimeout } = {}) {
-      await waitForDomStabilityResult(page)
-      const timeout = goto.timeouts.action(rest.timeout)
+    async function waitUntilAuto (page, { response, timeout: autoTimeout } = {}) {
+      const timeout = autoTimeout ?? goto.timeouts.action(rest.timeout)
+      let didHydrateAttempt = false
 
-      // The readiness gate waits for a page to settle — page-load work, not a
-      // small action. Budgeting it from `timeouts.action` (timeout/11) gave it
-      // ~1.2s while a hydrating document needs 2-3s, so it timed out on every
-      // tall page and the zero-capture fast path never fired. Budget it from
-      // the load allowance goto actually assigned to this phase; the gate
-      // returns as soon as the page is quiet, so this is a cap, not a cost.
       const readyTime = timeSpan()
       const ready = await waitForReady(page, {
-        timeout: Math.round((autoTimeout ?? timeout) * READY_BUDGET_RATIO)
+        timeout: Math.round(timeout * READY_BUDGET_RATIO)
       })
       readiness = ready
       debug('ready', { ...ready, duration: readyTime() })
 
-      // The blank-page poll keeps its own action budget, measured from here so
-      // a slow gate cannot starve it.
       const elapsed = timeSpan()
+      const pollTimeout = Math.max(0, timeout - readyTime())
 
-      // Fast path: the page settled with real painted content in a document
-      // taller than the viewport — a visibly rendered image (not a tracking
-      // pixel), or enough visible text with webfonts loaded (a pending
-      // `font-display: block` font renders text invisible, exactly when a
-      // capture would be white) — and that content is not `covered` by an
-      // opaque viewport-filling layer (a fixed white loading overlay passes
-      // every other DOM signal while a capture stays white): skip the
-      // screenshot poll. Height and viewport come from the same in-page
-      // snapshot (`page.viewport()` is null under `defaultViewport: null`),
-      // and an unknown viewport skips the fast path rather than dropping the
-      // taller-than-viewport guard. A gate that timed out never settled, so
-      // don't trust its partial snapshot: fall through to the blank check.
-      const painted = ready.painted > 0 || (ready.text >= TEXT_PAINTED_MIN && ready.fonts)
-      if (
+      isReady =
         !ready.timedOut &&
-        painted &&
+        isPaintedContent(ready) &&
         !ready.covered &&
         ready.viewport > 0 &&
         ready.height > ready.viewport
-      ) {
-        return
+
+      if (isReady) {
+        isReady = await checkPageReady(page, { isPageReady, response, isWhite: false })
+        if (!isReady) debug('ready:isPageReady', { rejected: true })
+      }
+
+      if (!isReady) {
+        let retry = -1
+        do {
+          ++retry
+          const screenshotTime = timeSpan()
+          const screenshot = await captureWithNavigationRetry(
+            () =>
+              page.screenshot({
+                optimizeForSpeed: true,
+                type: 'jpeg',
+                quality: 30
+              }),
+            { page, goto, timeout: Math.max(0, pollTimeout - elapsed()) }
+          )
+          const isWhite = await isWhiteScreenshot(screenshot)
+          isReady = await checkPageReady(page, { isPageReady, response, screenshot, isWhite })
+
+          const remaining = pollTimeout - elapsed()
+          if (!isReady && !didHydrateAttempt && !isWhite) {
+            didHydrateAttempt = true
+            const { hydrated, info } = await tryHydrateScroll(page, remaining)
+            didHydrateScroll = hydrated
+            debug('ready:hydrateScroll', { remaining, hydrated, ...info })
+          }
+
+          if (!isReady) await goto.waitUntilAuto(page, { timeout: rest.timeout })
+          debug('retry', {
+            waitUntil,
+            isReady,
+            isWhite,
+            retry,
+            duration: screenshotTime()
+          })
+        } while (!isReady && elapsed() < pollTimeout)
       }
 
-      // Otherwise fall back to the screenshot poll — re-wait while the first
-      // paint is still blank — to keep the blank-SPA protection. The page has
-      // already settled, so the first capture won't race a navigation, and a
-      // non-blank page exits after that single shot without an extra re-wait.
-      let isWhite = false
-      let retry = -1
-      do {
-        ++retry
-        const screenshotTime = timeSpan()
-        const screenshot = await captureWithNavigationRetry(
-          () =>
-            page.screenshot({
-              ...rest,
-              optimizeForSpeed: true,
-              type: 'jpeg',
-              quality: 30
-            }),
-          // The retry loop keeps its own clock, so hand it only what is left
-          // of the shared budget — a fresh full `timeout` here would let one
-          // navigation-racing capture double the worst-case prepare time.
-          { page, goto, timeout: Math.max(0, timeout - elapsed()) }
-        )
-        isWhite = await isWhiteScreenshot(screenshot)
-        if (isWhite) await goto.waitUntilAuto(page, { timeout: rest.timeout })
-        debug('retry', { waitUntil, isWhite, retry, duration: screenshotTime() })
-      } while (isWhite && elapsed() < timeout)
-
-      debug({ waitUntil, isWhite, timeout, duration: require('pretty-ms')(elapsed()) })
+      debug({ waitUntil, isReady, timeout: pollTimeout, duration: require('pretty-ms')(elapsed()) })
     }
   }