@browserless/screenshot 10.10.1 → 10.10.3

package/README.md

@@ -69,6 +69,8 @@ const buffer = await browserless.screenshot('https://example.com', {
 | `element` | `string` | — | CSS selector for element screenshot |
 | `codeScheme` | `string` | `'atom-dark'` | Prism.js theme for code highlighting |
 | `waitUntil` | `string` | `'auto'` | When to consider navigation done |
+| `waitForDom` | `number` | `0` | DOM stability window in ms (idle is `waitForDom / 10`, `0` disables DOM wait) |
+| `isPageReady` | `function` | `({ isWhite }) => !isWhite` | Custom readiness predicate for retry loop |
 | `overlay` | `object` | `{}` | Browser overlay options |
 
 All [Puppeteer page.screenshot() options](https://pptr.dev/api/puppeteer.screenshotoptions) are supported.
@@ -231,8 +233,7 @@ This is a **core functionality package** for screenshot capture:
 | Package | Purpose |
 |---------|---------|
 | `@browserless/goto` | Page navigation with ad blocking |
-| `sharp` | Image composition for overlays |
-| `jimp` | White/blank screenshot detection |
+| `sharp` | Image composition for overlays; White/blank screenshot detection |
 | `prism-themes` | Syntax highlighting themes |
 | `svg-gradient` | Generate gradient backgrounds |
 | `got` | Fetch remote background images |

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/screenshot",
   "description": "Capture high-quality screenshots of websites with overlay support, device emulation, and automated image optimization.",
   "homepage": "https://browserless.js.org/#/?id=screenshoturl-options",
-  "version": "10.10.1",
+  "version": "10.10.3",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -32,14 +32,13 @@
     "jpeg"
   ],
   "dependencies": {
-    "@browserless/goto": "^10.10.1",
+    "@browserless/goto": "^10.10.2",
     "@kikobeats/content-type": "~1.0.3",
     "@kikobeats/time-span": "~1.0.11",
     "debug-logfmt": "~1.4.7",
     "got": "~11.8.6",
     "is-html-content": "~1.0.0",
     "is-url-http": "~2.3.13",
-    "jimp": "~1.6.0",
     "lodash": "~4.17.23",
     "map-values-deep": "~1.0.2",
     "mime": "~3.0.0",
@@ -72,5 +71,5 @@
     "timeout": "2m",
     "workerThreads": false
   },
-  "gitHead": "32f6e72bcb489a83ac9659520a3961aeb97c47b7"
+  "gitHead": "4d2b893e8215a91d830f4471f51c567a53cd6bbb"
 }

package/src/index.js

@@ -8,6 +8,21 @@ const isWhiteScreenshot = require('./is-white-screenshot')
 const waitForPrism = require('./pretty')
 const timeSpan = require('./time-span')
 const overlay = require('./overlay')
+const { waitForDomStability, resolveWaitForDom, DEFAULT_WAIT_FOR_DOM } = require('./wait-for-dom')
+
+const createElapsed = () => {
+  const start = Date.now()
+  return () => Date.now() - start
+}
+
+const getPageSnapshot = page =>
+  page.evaluate(() => ({
+    title: document.title || '',
+    bodyText: document.body ? document.body.innerText || '' : '',
+    url: window.location.href || ''
+  }))
+
+const defaultIsPageReady = ({ isWhite }) => !isWhite
 
 const getBoundingClientRect = element => {
   const { top, left, height, width, x, y } = element.getBoundingClientRect()
@@ -32,37 +47,6 @@ const waitForImagesOnViewport = page =>
     )
   )
 
-const waitForDomStability = ({ idle, timeout } = {}) =>
-  new Promise(resolve => {
-    if (!document.body) return resolve({ status: 'no-body' })
-
-    let lastChange = performance.now()
-    const observer = new window.MutationObserver(() => {
-      lastChange = performance.now()
-    })
-    observer.observe(document.body, {
-      childList: true,
-      subtree: true,
-      attributes: false,
-      characterData: false
-    })
-
-    const deadline = performance.now() + timeout
-
-    ;(function check () {
-      const now = performance.now()
-      if (now - lastChange >= idle) {
-        observer.disconnect()
-        return resolve({ status: 'idle' })
-      }
-      if (now >= deadline) {
-        observer.disconnect()
-        return resolve({ status: 'timeout' })
-      }
-      window.requestAnimationFrame(check)
-    })()
-  })
-
 const scrollFullPageToLoadContent = async (page, timeout) => {
   const debug = require('debug-logfmt')('browserless:goto')
 
@@ -115,13 +99,21 @@ module.exports = ({ goto, ...gotoOpts }) => {
   return function screenshot (page) {
     return async (
       url,
-      { codeScheme = 'atom-dark', overlay: overlayOpts = {}, waitUntil = 'auto', ...opts } = {}
+      {
+        codeScheme = 'atom-dark',
+        overlay: overlayOpts = {},
+        waitUntil = 'auto',
+        waitForDom = DEFAULT_WAIT_FOR_DOM,
+        isPageReady = defaultIsPageReady,
+        ...opts
+      } = {}
     ) => {
       let screenshot
       let response
 
       const beforeScreenshot = async (page, response, { element, fullPage = false } = {}) => {
         const timeout = goto.timeouts.action(opts.timeout)
+        const waitForDomOpts = resolveWaitForDom(waitForDom)
 
         let screenshotOpts = {}
         const tasks = [
@@ -135,6 +127,13 @@ module.exports = ({ goto, ...gotoOpts }) => {
           }
         ]
 
+        if (waitForDomOpts) {
+          tasks.push({
+            fn: () => page.evaluate(waitForDomStability, waitForDomOpts),
+            debug: 'beforeScreenshot:waitForDomStability'
+          })
+        }
+
         if (codeScheme && response) {
           tasks.push({
             fn: () => waitForPrism(page, response, { codeScheme, ...opts }),
@@ -170,38 +169,80 @@ module.exports = ({ goto, ...gotoOpts }) => {
       }
 
       const takeScreenshot = async opts => {
-        screenshot = await page.screenshot(opts)
-        const isWhite = await isWhiteScreenshot(screenshot)
-        if (isWhite) {
-          await goto.waitUntilAuto(page, opts)
+        const timeout = goto.timeouts.action(opts.timeout)
+        const elapsed = createElapsed()
+        let retry = 0
+        let isWhite = false
+        let isReady = false
+
+        do {
           screenshot = await page.screenshot(opts)
-        }
-        return { isWhite }
+          isWhite = await isWhiteScreenshot(screenshot)
+          const snapshotResult = await pReflect(getPageSnapshot(page))
+          const pageSnapshot = snapshotResult.isRejected ? {} : snapshotResult.value
+          const pageReadyResult = await pReflect(
+            opts.isPageReady({
+              page,
+              response: opts.response,
+              screenshot,
+              isWhite,
+              isWhiteScreenshot,
+              ...pageSnapshot
+            })
+          )
+          isReady = !pageReadyResult.isRejected && !!pageReadyResult.value
+
+          if (isReady || elapsed() >= timeout) break
+
+          retry += 1
+          await goto.waitUntilAuto(page, { timeout })
+        } while (!isReady)
+
+        return { isWhite, isReady, retry }
       }
 
-      page.on('dialog', dialog => pReflect(dialog.dismiss()))
+      const onDialog = dialog => pReflect(dialog.dismiss())
+      page.on('dialog', onDialog)
 
-      const timeScreenshot = timeSpan()
+      try {
+        const timeScreenshot = timeSpan()
 
-      if (waitUntil !== 'auto') {
-        ;({ response } = await goto(page, { ...opts, url, waitUntil }))
-        const screenshotOpts = await beforeScreenshot(page, response, opts)
-        screenshot = await page.screenshot({ ...opts, ...screenshotOpts })
-        debug('screenshot', { waitUntil, duration: timeScreenshot() })
-      } else {
-        ;({ response } = await goto(page, { ...opts, url, waitUntil, waitUntilAuto }))
-        async function waitUntilAuto (page, { response }) {
+        if (waitUntil !== 'auto') {
+          ;({ response } = await goto(page, { ...opts, url, waitUntil }))
           const screenshotOpts = await beforeScreenshot(page, response, opts)
-          const { isWhite } = await takeScreenshot({ ...opts, ...screenshotOpts })
-          debug('screenshot', { waitUntil, isWhite, duration: timeScreenshot() })
+          screenshot = await page.screenshot({ ...opts, ...screenshotOpts })
+          debug('screenshot', { waitUntil, duration: timeScreenshot() })
+        } else {
+          ;({ response } = await goto(page, { ...opts, url, waitUntil, waitUntilAuto }))
+          async function waitUntilAuto (page, { response }) {
+            const screenshotOpts = await beforeScreenshot(page, response, opts)
+            const { isWhite, isReady, retry } = await takeScreenshot({
+              ...opts,
+              ...screenshotOpts,
+              isPageReady,
+              response
+            })
+            debug('screenshot', {
+              waitUntil,
+              isReady,
+              isWhite,
+              retry,
+              duration: timeScreenshot()
+            })
+          }
         }
-      }
 
-      return Object.keys(overlayOpts).length === 0
-        ? screenshot
-        : overlay(screenshot, { ...opts, ...overlayOpts, viewport: page.viewport() })
+        return Object.keys(overlayOpts).length === 0
+          ? screenshot
+          : overlay(screenshot, { ...opts, ...overlayOpts, viewport: page.viewport() })
+      } finally {
+        page.off('dialog', onDialog)
+      }
     }
   }
 }
 
 module.exports.isWhiteScreenshot = isWhiteScreenshot
+module.exports.waitForDomStability = waitForDomStability
+module.exports.resolveWaitForDom = resolveWaitForDom
+module.exports.DEFAULT_WAIT_FOR_DOM = DEFAULT_WAIT_FOR_DOM

package/src/is-white-screenshot.js

@@ -1,32 +1,67 @@
 'use strict'
 
-const { Jimp } = require('jimp')
+const sharp = require('sharp')
 
-module.exports = async uint8array => {
-  try {
-    const image = await Jimp.fromBuffer(Buffer.from(uint8array))
-    const firstPixel = image.getPixelColor(0, 0)
-    const height = image.bitmap.height
-    const width = image.bitmap.width
-
-    // For 2D grid sampling, calculate stepSize to achieve approximately the target sample percentage.
-    // When sampling every 'stepSize' pixels in both dimensions, actual samples = (height/stepSize) * (width/stepSize).
-    // To achieve samplePercentage, we need: (h*w)/(stepSize²) ≈ samplePercentage*(h*w)
-    // Therefore: stepSize ≈ sqrt(1 / samplePercentage)
-    const samplePercentage = 0.25 // Sample ~25% of the image
-    const stepSize = Math.max(1, Math.ceil(Math.sqrt(1 / samplePercentage)))
-
-    for (let i = 0; i < height; i += stepSize) {
-      for (let j = 0; j < width; j += stepSize) {
-        if (firstPixel !== image.getPixelColor(j, i)) return false
+const SAMPLE_PERCENTAGE = 0.25
+const SAMPLE_STEP_SIZE = Math.max(1, Math.ceil(Math.sqrt(1 / SAMPLE_PERCENTAGE)))
+const WHITE_PIXEL_THRESHOLD = 245
+const WHITE_COLOR_VARIANCE_TOLERANCE = 8
+
+const blendOnWhite = (channel, alpha) => (channel * alpha + 255 * (255 - alpha)) / 255
+
+const isWhiteSampledImage = (data, { width, height, channels }) => {
+  if (!width || !height || !channels || data.length < channels) return false
+
+  let minR = 255
+  let minG = 255
+  let minB = 255
+  let maxR = 0
+  let maxG = 0
+  let maxB = 0
+
+  for (let y = 0; y < height; y += SAMPLE_STEP_SIZE) {
+    const rowOffset = y * width * channels
+
+    for (let x = 0; x < width; x += SAMPLE_STEP_SIZE) {
+      const pixelOffset = rowOffset + x * channels
+      const a = channels > 3 ? data[pixelOffset + 3] : 255
+      const r = blendOnWhite(data[pixelOffset], a)
+      const g = blendOnWhite(data[pixelOffset + 1], a)
+      const b = blendOnWhite(data[pixelOffset + 2], a)
+
+      if (r < WHITE_PIXEL_THRESHOLD || g < WHITE_PIXEL_THRESHOLD || b < WHITE_PIXEL_THRESHOLD) {
+        return false
       }
-    }
 
-    return true
-  } catch (error) {
-    if (error.message.includes('maxMemoryUsageInMB')) {
-      return false
+      if (r < minR) minR = r
+      if (g < minG) minG = g
+      if (b < minB) minB = b
+      if (r > maxR) maxR = r
+      if (g > maxG) maxG = g
+      if (b > maxB) maxB = b
     }
-    throw error
   }
+
+  return (
+    maxR - minR <= WHITE_COLOR_VARIANCE_TOLERANCE &&
+    maxG - minG <= WHITE_COLOR_VARIANCE_TOLERANCE &&
+    maxB - minB <= WHITE_COLOR_VARIANCE_TOLERANCE
+  )
 }
+
+module.exports = async uint8array => {
+  const input = Buffer.isBuffer(uint8array) ? uint8array : Buffer.from(uint8array)
+  const { data, info } = await sharp(input)
+    .ensureAlpha()
+    .raw()
+    .toBuffer({ resolveWithObject: true })
+
+  return isWhiteSampledImage(data, info)
+}
+
+module.exports.SAMPLE_PERCENTAGE = SAMPLE_PERCENTAGE
+module.exports.SAMPLE_STEP_SIZE = SAMPLE_STEP_SIZE
+module.exports.WHITE_PIXEL_THRESHOLD = WHITE_PIXEL_THRESHOLD
+module.exports.WHITE_COLOR_VARIANCE_TOLERANCE = WHITE_COLOR_VARIANCE_TOLERANCE
+module.exports.blendOnWhite = blendOnWhite
+module.exports.isWhiteSampledImage = isWhiteSampledImage

package/src/wait-for-dom.js

@@ -0,0 +1,53 @@
+'use strict'
+
+const DEFAULT_WAIT_FOR_DOM = 0
+const WAIT_FOR_DOM_IDLE_RATIO = 10
+
+const resolveWaitForDom = waitForDom => {
+  const timeout = Number.isFinite(waitForDom) && waitForDom >= 0 ? waitForDom : DEFAULT_WAIT_FOR_DOM
+
+  if (timeout === 0) return undefined
+
+  return {
+    timeout,
+    idle: timeout / WAIT_FOR_DOM_IDLE_RATIO
+  }
+}
+
+const waitForDomStability = ({ idle, timeout } = {}) =>
+  new Promise(resolve => {
+    if (!document.body) return resolve({ status: 'no-body' })
+
+    let lastChange = performance.now()
+    const observer = new window.MutationObserver(() => {
+      lastChange = performance.now()
+    })
+    observer.observe(document.body, {
+      childList: true,
+      subtree: true,
+      attributes: false,
+      characterData: false
+    })
+
+    const deadline = performance.now() + timeout
+
+    ;(function check () {
+      const now = performance.now()
+      if (now - lastChange >= idle) {
+        observer.disconnect()
+        return resolve({ status: 'idle' })
+      }
+      if (now >= deadline) {
+        observer.disconnect()
+        return resolve({ status: 'timeout' })
+      }
+      window.requestAnimationFrame(check)
+    })()
+  })
+
+module.exports = {
+  DEFAULT_WAIT_FOR_DOM,
+  WAIT_FOR_DOM_IDLE_RATIO,
+  resolveWaitForDom,
+  waitForDomStability
+}