@browserless.io/browserless 2.34.0 → 2.36.0

package/extensions/ublocklite/rulesets/strictblock/vie-1.json

@@ -16,6 +16,7 @@
    "adtima-static.zascdn.me",
    "adtimaserver.vn",
    "ae.carnelbawrel.com",
+   "afzamltca.com",
    "aiactiv.io",
    "aj1047.online",
    "aj2393.online",
@@ -26,7 +27,6 @@
    "bidgear-syndication.com",
    "cdn.adop.asia",
    "cdn.adxcontent.com",
-   "cdn.dtadnetwork.com",
    "cdn.tpmedia.online",
    "cdnv.tpmedia.online",
    "click.advertnative.com",
@@ -34,25 +34,28 @@
    "contineljs.com",
    "cpt.geniee.jp",
    "cv.bentsdiorism.com",
+   "deemsunways.com",
    "delivery.yomedia.vn",
    "distinctrobin.com",
+   "ds.recrampwiped.com",
    "fv.togataurnfuls.com",
    "fx.gearmanswings.com",
    "genmonet.com",
    "gullyending.com",
-   "i9zk5f8szj4x8saj5wcks8acja5sk1dz8stjs2g8isdcn6ejxn8zsr5xjs5h.com",
    "iads.staticscdn.net",
    "iads.vision",
    "img.yeuketqua.net",
    "immaculate-suspect.com",
    "js.aiservice.vn",
    "jsc.yodimedia.com",
+   "lepidpomps.com",
+   "ligheechoagool.com",
    "media.yomedia.vn",
    "midgetmaying.com",
    "mneydirec.com",
    "mt.deimosfestuca.com",
-   "nb.passuplums.com",
    "ny.foonerne.com",
+   "ob.joyrodetoment.com",
    "oursty.com",
    "platform.adlinknetwork.vn",
    "player.gliacloud.com",
@@ -66,17 +69,13 @@
    "ss.yomedia.vn",
    "static.adconnect.vn",
    "static.dable.io",
-   "static.eclick.vn",
    "static.philacct.com",
-   "syphonseity.com",
    "tags.fly-ads.net",
    "tags.native-ad.net",
-   "urgerrebasis.com",
    "ut.uranininnage.com",
    "v9banners.com",
    "vads.net.vn",
-   "wr.brandonketting.com",
-   "zr.mezcalumlaut.com"
+   "wr.brandonketting.com"
   ],
   "resourceTypes": [
    "main_frame"

package/extensions/ublocklite/rulesets/urlskip/ublock-filters.json

@@ -221,6 +221,12 @@
    "-base64"
   ]
  },
+ {
+  "re": "^[a-z-]+://(?:[^/?#]+\\.)?link\\.onlinekosh\\.com(?:[^%.0-9a-z_-]|$)",
+  "steps": [
+   "?url"
+  ]
+ },
  {
   "re": "^[a-z-]+://(?:[^/?#]+\\.)?rd\\.remove\\.bg/cl0/http",
   "steps": [
@@ -1068,6 +1074,24 @@
    "?url"
   ]
  },
+ {
+  "re": "^[a-z-]+://(?:[^/?#]+\\.)?hb\\.afl\\.rakuten\\.co\\.jp/hgc/\\S*?(?:[^%.0-9a-z_-]|$)pc=http",
+  "steps": [
+   "?pc"
+  ]
+ },
+ {
+  "re": "^[a-z-]+://(?:[^/?#]+\\.)?atmedia\\.link/product\\?url=http",
+  "steps": [
+   "?url"
+  ]
+ },
+ {
+  "re": "^[a-z-]+://(?:[^/?#]+\\.)?f\\.gdeslon\\.ru/cf/\\S*?(?:[^%.0-9a-z_-]|$)goto=http",
+  "steps": [
+   "?goto"
+  ]
+ },
  {
   "re": "/vp/player/to/\\?u=http",
   "steps": [
@@ -1086,6 +1110,12 @@
    "?url"
   ]
  },
+ {
+  "re": "^[a-z-]+://(?:[^/?#]+\\.)?donanimhaber\\.com/externallinkredirect\\?\\S*?(?:[^%.0-9a-z_-]|$)url=http",
+  "steps": [
+   "?url"
+  ]
+ },
  {
   "re": "^[a-z-]+://(?:[^/?#]+\\.)?email\\.stripe\\.com/cl0/",
   "steps": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.34.0",
+  "version": "2.36.0",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -31,6 +31,7 @@
     "prepack": "npm run build:dev",
     "prettier": "prettier '{src,functions,scripts,bin,external,.github}/**/*.{js,ts,json,yml,yaml,md}' --log-level error --write",
     "test": "npm run build:ts && cross-env DEBUG=quiet mocha",
+    "coverage": "npm run build:ts && c8 --reporter=text --reporter=html --reporter=lcov cross-env DEBUG=quiet mocha",
     "start": "env-cmd -f .env node build"
   },
   "files": [
@@ -51,25 +52,25 @@
     "@hapi/bourne": "^3.0.0",
     "@hapi/hoek": "^11.0.4",
     "debug": "^4.4.1",
-    "del": "^8.0.0",
+    "del": "^8.0.1",
     "file-type": "^21.0.0",
     "get-port": "^7.1.0",
     "gradient-string": "^3.0.0",
     "http-proxy": "^1.18.1",
-    "joi": "^18.0.0",
-    "lighthouse": "^12.8.1",
+    "joi": "^18.0.1",
+    "lighthouse": "^12.8.2",
     "micromatch": "^4.0.8",
-    "playwright-1.50": "npm:playwright-core@1.50.1",
     "playwright-1.51": "npm:playwright-core@1.51.1",
     "playwright-1.52": "npm:playwright-core@1.52.0",
     "playwright-1.53": "npm:playwright-core@1.53.2",
-    "playwright-core": "1.54.2",
-    "puppeteer-core": "24.16.1",
+    "playwright-1.54": "npm:playwright-core@1.54.2",
+    "playwright-core": "1.55.0",
+    "puppeteer-core": "24.22.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "queue": "^7.0.0",
-    "systeminformation": "^5.27.7",
-    "tar-fs": "^3.1.0"
+    "systeminformation": "^5.27.10",
+    "tar-fs": "^3.1.1"
   },
   "optionalDependencies": {
     "@types/chai": "^5.2.2",
@@ -78,21 +79,22 @@
     "@types/http-proxy": "^1.17.16",
     "@types/micromatch": "^4.0.9",
     "@types/mocha": "^10.0.10",
-    "@types/node": "^24.2.1",
+    "@types/node": "^24.5.2",
     "@types/sinon": "^17.0.4",
-    "@typescript-eslint/eslint-plugin": "^8.39.1",
-    "@typescript-eslint/parser": "^8.38.0",
+    "@typescript-eslint/eslint-plugin": "^8.44.0",
+    "@typescript-eslint/parser": "^8.42.0",
     "assert": "^2.0.0",
-    "chai": "^5.2.1",
+    "c8": "^10.1.3",
+    "chai": "^6.0.1",
     "cross-env": "^10.0.0",
-    "env-cmd": "^10.1.0",
-    "esbuild": "^0.25.8",
+    "env-cmd": "^11.0.0",
+    "esbuild": "^0.25.10",
     "esbuild-plugin-polyfill-node": "^0.3.0",
-    "eslint": "^9.33.0",
+    "eslint": "^9.35.0",
     "extract-zip": "^2.0.1",
     "gunzip-maybe": "^1.4.2",
-    "marked": "^16.1.2",
-    "mocha": "^11.7.1",
+    "marked": "^16.3.0",
+    "mocha": "^11.7.2",
     "move-file": "^3.1.0",
     "prettier": "^3.6.2",
     "sinon": "^21.0.0",
@@ -102,11 +104,11 @@
   },
   "playwrightVersions": {
     "default": "playwright-core",
-    "1.54": "playwright-core",
+    "1.55": "playwright-core",
+    "1.54": "playwright-1.54",
     "1.53": "playwright-1.53",
     "1.52": "playwright-1.52",
-    "1.51": "playwright-1.51",
-    "1.50": "playwright-1.50"
+    "1.51": "playwright-1.51"
   },
   "engines": {
     "node": ">= 24 < 25"

package/scripts/build-open-api.js

@@ -46,6 +46,17 @@ const sortSwaggerRequiredAlpha = (prop, otherProp) => {
   return Number(otherProp.required) - Number(prop.required);
 };
 
+const cleanupAbsolutePaths = (swaggerJSON) => {
+  const jsonString = JSON.stringify(swaggerJSON);
+  
+  const cleanedString = jsonString.replace(
+    /"\$ref":\s*"#\/definitions\/import\([^)]+\)\.([^"]+)"/g,
+    '"$ref": "#/definitions/$1"'
+  );
+  
+  return JSON.parse(cleanedString);
+};
+
 const buildOpenAPI = async (
   externalHTTPRoutes = [],
   externalWebSocketRoutes = [],
@@ -298,7 +309,10 @@ const buildOpenAPI = async (
     JSON.stringify(swaggerJSON, null, '  '),
   );
   swaggerJSON.info.description = readme + `\n# Changelog\n` + changelog;
-  await fs.writeFile(swaggerJSONPath, JSON.stringify(swaggerJSON, null, '  '));
+  
+  const cleanedSwaggerJSON = cleanupAbsolutePaths(swaggerJSON);
+  
+  await fs.writeFile(swaggerJSONPath, JSON.stringify(cleanedSwaggerJSON, null, '  '));
 };
 
 export default buildOpenAPI;

package/scripts/install-adblock.js

@@ -2,11 +2,11 @@
 /* global fetch, console, process */
 'use strict';
 
-import { createWriteStream, existsSync, mkdirSync } from 'fs';
+import { createWriteStream, existsSync, mkdirSync, readdirSync, statSync } from 'fs';
 import path, { join } from 'path';
 import { Readable } from 'stream';
 import { deleteAsync } from 'del';
-import { moveFile } from 'move-file';
+import { cp } from 'fs/promises';
 import os from 'os';
 import unzip from 'extract-zip';
 
@@ -19,7 +19,6 @@ import unzip from 'extract-zip';
   }
 
   const zipFile = tmpDir + '/ublock.zip';
-  const tmpUblockLitePath = path.join(tmpDir);
   const extensionsDir = join(process.cwd(), 'extensions');
   const uBlockLiteDir = join(extensionsDir, 'ublocklite');
 
@@ -45,7 +44,29 @@ import unzip from 'extract-zip';
 
   await downloadUrlToDirectory(json.assets[0].browser_download_url, zipFile);
   await unzip(zipFile, { dir: tmpDir });
-  await moveFile(join(tmpUblockLitePath), join(extensionsDir, 'ublocklite'));
+
+  const findExtensionDir = (dir) => {
+    const items = readdirSync(dir);
+    if (items.includes('manifest.json')) {
+      return dir;
+    }
+    for (const item of items) {
+      if (item === 'ublock.zip') continue;
+      const itemPath = join(dir, item);
+      if (existsSync(itemPath) && statSync(itemPath).isDirectory()) {
+        const foundDir = findExtensionDir(itemPath);
+        if (foundDir) return foundDir;
+      }
+    }
+    return null;
+  };
+
+  const extensionSourceDir = findExtensionDir(tmpDir);
+  if (!extensionSourceDir) {
+    throw new Error('Could not find uBlock Lite extension directory with manifest.json');
+  }
+
+  await cp(extensionSourceDir, join(extensionsDir, 'ublocklite'), { recursive: true });
   await deleteAsync(zipFile, { force: true }).catch((err) => {
     console.warn('Could not delete temporary download file: ' + err.message);
   });

package/scripts/start.sh

@@ -15,7 +15,7 @@ trap _kill_procs SIGTERM SIGINT
 
 if [ -z "$DISPLAY" ]
 then
-  Xvfb :99 -screen 0 1024x768x16 -nolisten tcp -nolisten unix &
+  Xvfb :99 -screen 0 1024x768x16 -nolisten tcp -nolisten unix >/dev/null 2>&1 &
   xvfb=$!
   export DISPLAY=:99
 fi

package/scripts/test.sh

@@ -2,7 +2,7 @@
 set -e
 
 # Install all dependencies for tests
-npm i --production=false
+npm clean-install
 
 # Setup Env Variables
 export DEBUG=-*
@@ -11,7 +11,7 @@ export NODE_OPTIONS="--loader ts-node/esm"
 
 if [ -z "$DISPLAY" ]
 then
-  Xvfb :99 -screen 0 1024x768x16 -nolisten tcp -nolisten unix &
+  Xvfb :99 -screen 0 1024x768x16 -nolisten tcp -nolisten unix >/dev/null 2>&1 &
   xvfb=$!
   export DISPLAY=:99
 fi

package/src/limiter.ts

@@ -207,7 +207,7 @@ export class Limiter extends q {
           const queueLimit = this.queued;
           return rej(
             new TooManyRequests(
-              `Concurrency limit of ${concurrencyLimit} and queue limit of ${queueLimit} reached. Possible causes: 1) Your token has reached maximum capacity, 2) Your token may not have access to this version, 3) Your requests are coming too quickly.`,
+              `Your plan allows ${concurrencyLimit} concurrent sessions and ${queueLimit} queued requests, but both limits have been reached. Possible causes: 1) Your plan has reached maximum capacity, 2) Your token may not have access to this version, 3) Your requests are coming too quickly.`,
             ),
           );
         }

package/src/routes/chromium/tests/scrape.spec.ts

@@ -414,4 +414,52 @@ describe('/chromium/scrape API', function () {
       expect(res.status).to.equal(200);
     });
   });
+
+  it('handles bestAttempt with failing selectors', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      bestAttempt: true,
+      elements: [
+        {
+          selector: 'a',
+        },
+        {
+          selector: 'nonexistent-element',
+          timeout: 1000,
+        },
+        {
+          selector: 'h1',
+        },
+      ],
+      url: 'https://one.one.one.one',
+    };
+
+    await fetch('http://localhost:3000/chromium/scrape?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then(async (res) => {
+      expect(res.status).to.equal(200);
+      expect(res.headers.get('content-type')).to.equal(
+        'application/json; charset=UTF-8',
+      );
+
+      const json = await res.json();
+      expect(json.data).to.be.an('array');
+      expect(json.data).to.have.length(3);
+
+      const aResults = json.data.find((item: any) => item.selector === 'a');
+      const nonexistentResults = json.data.find((item: any) => item.selector === 'nonexistent-element');
+      const h1Results = json.data.find((item: any) => item.selector === 'h1');
+
+      expect(aResults.results).to.be.an('array');
+      expect(nonexistentResults.results).to.be.an('array').that.is.empty;
+      expect(h1Results.results).to.be.an('array');
+    });
+  });
 });

package/src/sdk-utils.ts

@@ -118,7 +118,7 @@ export const prompt = async (question: string) => {
 export const installDependencies = async (
   workingDirectory: string,
 ): Promise<void> => {
-  await waitForCommand('npm install', workingDirectory);
+  await waitForCommand('npm clean-install', workingDirectory);
   await installBrowsers(workingDirectory);
 };
 

package/src/shared/scrape.http.ts

@@ -157,7 +157,7 @@ export interface ResponseSchema {
   } | null;
 }
 
-const scrape = async (elements: ScrapeElementSelector[]) => {
+const scrape = async (elements: ScrapeElementSelector[], bestAttempt = false) => {
   const wait = (selector: string, timeout = 30000) => {
     return new Promise<void>((resolve, reject) => {
       const timeoutId = setTimeout(() => {
@@ -175,9 +175,21 @@ const scrape = async (elements: ScrapeElementSelector[]) => {
     });
   };
 
-  await Promise.all(
-    elements.map(({ selector, timeout }) => wait(selector, timeout)),
-  );
+  if (bestAttempt) {
+    const results = await Promise.allSettled(
+      elements.map(({ selector, timeout }) => wait(selector, timeout)),
+    );
+
+    results.forEach((result, index) => {
+      if (result.status === 'rejected') {
+        console.warn(`Selector "${elements[index].selector}" failed: ${result.reason.message}`);
+      }
+    });
+  } else {
+    await Promise.all(
+      elements.map(({ selector, timeout }) => wait(selector, timeout)),
+    );
+  }
 
   return elements.map(({ selector }) => {
     const $els = [...document.querySelectorAll(selector)] as HTMLElement[];
@@ -410,7 +422,7 @@ export default class ChromiumScrapePostRoute extends BrowserHTTPRoute {
       }
     }
 
-    const data = await page.evaluate(scrape, elements).catch((e) => {
+    const data = await page.evaluate(scrape, elements, bestAttempt).catch((e) => {
       if (e.message.includes('Timed out')) {
         throw new Timeout(e);
       }