capskip 1.3.0 → 1.5.0

package/examples/datadome.js

@@ -0,0 +1,142 @@
+'use strict';
+
+/**
+ * Pass a DataDome challenge and fetch the page with the cookie.
+ *
+ * DataDome answers a request it doubts with a block page (status 403, sometimes
+ * 401) instead of the site. That page carries a `var dd={...}` object and loads
+ * the challenge in an iframe from geo.captcha-delivery.com: a slider under
+ * /captcha/, or a few-second device check under /interstitial/. CapSkip passes
+ * either and returns the `datadome` cookie.
+ *
+ * The cookie belongs to one IP address and one User-Agent, so:
+ *
+ *   * solve through the proxy your requests use (or none, if they leave from
+ *     the machine running CapSkip), and
+ *   * send the User-Agent CapSkip reports back with every request.
+ *
+ * When you send no proxy and the CapSkip app has its own DataDome proxy list,
+ * it solves through one of those and the result names it as `proxy`
+ * (host:port): the cookie then only works through that proxy.
+ *
+ * Requests go out with Node's built-in fetch. Sending them through a proxy
+ * needs undici's ProxyAgent (`npm install undici`); without it, the example
+ * prints what to do instead of re-fetching through the proxy.
+ *
+ * Environment:
+ *   TARGET_URL         a page behind DataDome (default: https://www.example.com/)
+ *   DATADOME_PROXY     optional, e.g. http://login:password@203.0.113.10:8080
+ *   CAPSKIP_API_KEY / CAPSKIP_HOST / CAPSKIP_PORT
+ */
+
+const { CapSkip } = require('capskip');
+
+const solver = new CapSkip({
+  apiKey: process.env.CAPSKIP_API_KEY || 'capskip',
+  host: process.env.CAPSKIP_HOST || '127.0.0.1',
+  port: Number(process.env.CAPSKIP_PORT || 8080),
+});
+
+const PAGE_URL = process.env.TARGET_URL || 'https://www.example.com/';
+const PROXY = process.env.DATADOME_PROXY; // scheme://[login:password@]host:port, or unset
+
+let ProxyAgent;
+try {
+  ({ ProxyAgent } = require('undici'));
+} catch (error) {
+  ProxyAgent = undefined;
+}
+
+/** The challenge iframe address, as the block page's own script builds it. */
+function captchaUrlFromBlockPage(html, pageUrl) {
+  const iframe = html.match(/<iframe[^>]+src="(https:\/\/[^"]*captcha-delivery\.com\/[^"]+)"/);
+  if (iframe) {
+    return iframe[1]; // HTML-escaped &amp; is fine, CapSkip decodes it
+  }
+  const match = html.match(/var dd=(\{[^}]*\})/);
+  if (!match) {
+    return null;
+  }
+  const dd = JSON.parse(match[1].replace(/'/g, '"'));
+  const path = dd.rt === 'i' ? 'interstitial' : 'captcha';
+  const query = new URLSearchParams({ initialCid: dd.cid, hash: dd.hsh, cid: dd.cookie });
+  if (dd.t) {
+    query.set('t', dd.t);
+  }
+  query.set('referer', pageUrl);
+  query.set('s', String(dd.s));
+  for (const key of ['e', 'b']) {
+    if (dd[key]) {
+      query.set(key, String(dd[key]));
+    }
+  }
+  query.set('dm', 'cd');
+  return `https://${dd.host || 'geo.captcha-delivery.com'}/${path}/?${query}`;
+}
+
+/** fetch, through `proxyUrl` when one is given and undici is installed. */
+function get(url, { proxyUrl, headers } = {}) {
+  const options = { headers, redirect: 'follow' };
+  if (proxyUrl) {
+    options.dispatcher = new ProxyAgent(proxyUrl);
+  }
+  return fetch(url, options);
+}
+
+async function main() {
+  if (PROXY && !ProxyAgent) {
+    console.log('DATADOME_PROXY is set: run `npm install undici` so the requests use it.');
+    return;
+  }
+
+  // 1. Request the page. A 403/401 with a dd object is DataDome's block page.
+  const response = await get(PAGE_URL, { proxyUrl: PROXY });
+  const html = await response.text();
+  const captchaUrl = captchaUrlFromBlockPage(html, PAGE_URL);
+  if (![401, 403].includes(response.status) || !captchaUrl) {
+    console.log(`No DataDome block page at ${PAGE_URL} (HTTP ${response.status}).`);
+    return;
+  }
+  console.log('Block page:', `${captchaUrl.slice(0, 90)}...`);
+
+  // 2. Solve it. 2Captcha's argument order: datadome(captcha_url, pageurl,
+  //    userAgent, proxy), with userAgent and proxy as options. No userAgent:
+  //    CapSkip uses its own browser's and reports it. With one, it must be
+  //    Chrome for Windows with the same major version as that browser.
+  const options = {};
+  if (PROXY) {
+    const scheme = PROXY.includes('://') ? PROXY.split('://', 1)[0] : 'http';
+    options.proxy = { type: scheme.toUpperCase(), uri: PROXY.split('://').pop() };
+  }
+  const result = await solver.dataDome(captchaUrl, PAGE_URL, options);
+
+  console.log('Captcha ID:', result.captchaId);
+  console.log('Cookie:    ', `${result.cookie.slice(0, 60)}...`);
+  console.log('User-Agent:', result.userAgent);
+  if (result.proxy) {
+    console.log('Proxy:     ', result.proxy, '(from the CapSkip app DataDome list; the cookie only works through it)');
+  }
+
+  // 3. Repeat the request with the cookie, the same User-Agent and the same IP.
+  let proxyUrl = PROXY;
+  if (result.proxy && !PROXY) {
+    // Named without its login: add yours if that list entry has one.
+    if (!ProxyAgent) {
+      console.log(`Send the cookie through http://${result.proxy}: run \`npm install undici\` to do it here.`);
+      return;
+    }
+    proxyUrl = `http://${result.proxy}`;
+  }
+  // `cookie` is the datadome=... pair, ready for the Cookie header; `code`
+  // keeps the attributes (Domain, Max-Age...) for a cookie jar.
+  const again = await get(PAGE_URL, {
+    proxyUrl,
+    headers: { 'User-Agent': result.userAgent, Cookie: result.cookie },
+  });
+  console.log('With the cookie: HTTP', again.status);
+}
+
+main().catch((error) => {
+  console.error(`${error.name}: ${error.message}`);
+  process.exitCode = 1;
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "capskip",
-  "version": "1.3.0",
-  "description": "Captcha solver for Node.js: solve reCAPTCHA v2/v3, Cloudflare Turnstile, GeeTest and image captchas with CapSkip, a local unlimited captcha solver with no per-solve fees.",
+  "version": "1.5.0",
+  "description": "Captcha solver for Node.js: solve reCAPTCHA v2/v3, Cloudflare Turnstile, GeeTest, AWS WAF, DataDome and image captchas with CapSkip, a local unlimited captcha solver with no per-solve fees.",
   "main": "src/index.js",
   "types": "types/index.d.ts",
   "exports": {
@@ -39,6 +39,10 @@
     "cloudflare-turnstile",
     "geetest",
     "geetest-solver",
+    "aws-waf",
+    "aws-waf-captcha",
+    "datadome",
+    "datadome-solver",
     "image-captcha",
     "ocr",
     "2captcha-alternative",

package/src/apiParams.js

@@ -46,6 +46,42 @@ const FRIENDLY_CAPTCHA_SUBMIT = new Set([
   'api_server', 'useragent', 'json', 'proxy', 'proxytype',
 ]);
 
+const AMAZON_WAF_SUBMIT = new Set([
+  'method', 'sitekey', 'pageurl', 'iv', 'context', 'challenge_script',
+  'captcha_script', 'jsapiScript', 'json', 'proxy', 'proxytype',
+]);
+
+// The script parameters of an AWS WAF task. CapSkip POSTs the solution to the
+// host these name, so it refuses any that is not a genuine AWS WAF one.
+const AMAZON_WAF_SCRIPTS = ['challenge_script', 'captcha_script', 'jsapiScript'];
+
+// The server's own test: one or more [A-Za-z0-9_-] path segments, then one of
+// the three AWS WAF script files.
+const AMAZON_WAF_SCRIPT_PATH = /^(\/[A-Za-z0-9_-]+)+\/(challenge|captcha|jsapi)\.js$/;
+
+const DATADOME_SUBMIT = new Set([
+  'method', 'captcha_url', 'pageurl', 'useragent', 'json', 'proxy', 'proxytype',
+]);
+
+// Every DataDome challenge is served from this domain or a subdomain of it
+// (geo.captcha-delivery.com in practice). CapSkip refuses any other host.
+const DATADOME_CHALLENGE_DOMAIN = 'captcha-delivery.com';
+
+// The two challenge pages: /captcha/ draws the slider, /interstitial/ runs the
+// device check. Any other path renders nothing to solve.
+const DATADOME_CHALLENGE_PATHS = ['/captcha', '/interstitial'];
+
+// A Chrome version as Chrome writes it in its User-Agent: four dotted numbers.
+const CHROME_VERSION = /\bChrome\/(\d+)\.\d+\.\d+\.\d+\b/;
+
+// Chrome-derived browsers and non-desktop builds. A User-Agent carrying any of
+// these is not stock Windows Chrome, so a cookie earned under it would be
+// refused on replay; the server answers ERROR_WRONG_USERAGENT for each.
+const NOT_STOCK_WINDOWS_CHROME = [
+  'Edg/', 'EdgA/', 'OPR/', 'YaBrowser/', 'SamsungBrowser/', 'Vivaldi/',
+  'HeadlessChrome/', 'CriOS/', 'Mobile', 'Android',
+];
+
 // CapSkip solves the puzzle family only. `avatar` is a different challenge
 // behind a different endpoint; the server refuses it at submit time rather than
 // answering it with a puzzle answer, which would bill for a solve the target
@@ -81,6 +117,16 @@ const PARAM_ALIASES = {
   moduleScript: 'module_script',
   nomoduleScript: 'nomodule_script',
   noModuleScript: 'nomodule_script',
+  // AWS WAF: the API documents challenge_script / captcha_script in snake case
+  // and jsapiScript in camel case, as 2Captcha does. Accept the other spelling
+  // of each, which is what createTask-style callers arrive with.
+  challengeScript: 'challenge_script',
+  captchaScript: 'captcha_script',
+  jsapi_script: 'jsapiScript',
+  // DataDome: 2Captcha and the API document captcha_url; createTask-style
+  // callers arrive with captchaUrl.
+  captchaUrl: 'captcha_url',
+  captchaURL: 'captcha_url',
 };
 
 function has(obj, key) {
@@ -353,6 +399,255 @@ function validateFriendlyCaptchaSubmit(params) {
   }
 }
 
+/**
+ * True for a script URL CapSkip will accept on an AWS WAF task.
+ *
+ * Mirrors the server: https, a host under .awswaf.com, no credentials or
+ * explicit port, and a path ending in challenge.js, captcha.js or jsapi.js.
+ * The server only applies the check once the task is queued, so failing here
+ * saves a submit that could never solve.
+ */
+function isAmazonWafScriptUrl(url) {
+  let parsed;
+  try {
+    parsed = new URL(String(url).trim());
+  } catch (error) {
+    return false;
+  }
+  // WHATWG URL already lowercases the scheme and host, and drops a :443 on an
+  // https URL, so `port` is empty for both the default and an explicit 443.
+  return parsed.protocol === 'https:'
+    && parsed.hostname.toLowerCase().endsWith('.awswaf.com')
+    && parsed.username === ''
+    && parsed.password === ''
+    && (parsed.port === '' || parsed.port === '443')
+    && AMAZON_WAF_SCRIPT_PATH.test(parsed.pathname);
+}
+
+/**
+ * Drop unset optionals, and the empty ones too: an AWS WAF optional left as ''
+ * means "not sent" to the server, exactly like an absent one, so it is dropped
+ * rather than tripping the iv/context pairing.
+ */
+function normalizeAmazonWafSubmit(params) {
+  const out = {};
+  for (const [key, value] of Object.entries(dropUnset(params))) {
+    if (value !== '') {
+      out[key] = value;
+    }
+  }
+  return out;
+}
+
+function validateAmazonWafSubmit(params) {
+  // Without sitekey the server answers ERROR_BAD_PARAMETERS and without
+  // pageurl ERROR_PAGEURL. Fail locally so a missing value costs no round-trip.
+  for (const key of ['sitekey', 'pageurl']) {
+    if (!params[key]) {
+      throw new ValidationException(`'${key}' is required for Amazon WAF.`);
+    }
+  }
+
+  const pageurl = String(params.pageurl).trim();
+  if (!/^https?:\/\//i.test(pageurl)) {
+    throw new ValidationException(
+      `'pageurl' must be a full http or https URL for Amazon WAF, got '${pageurl}'.`,
+    );
+  }
+
+  // iv and context come off the same challenge page and only work as a pair;
+  // the server refuses one without the other with ERROR_BAD_PARAMETERS.
+  if (Boolean(params.iv) !== Boolean(params.context)) {
+    throw new ValidationException(
+      "Amazon WAF 'iv' and 'context' go together: send both (a challenge "
+      + 'page) or neither (a widget page, or let CapSkip read them from pageurl).',
+    );
+  }
+
+  for (const key of AMAZON_WAF_SCRIPTS) {
+    const value = params[key];
+    if (value && !isAmazonWafScriptUrl(value)) {
+      throw new ValidationException(
+        `'${key}' must be the https URL of an AWS WAF script on an awswaf.com `
+        + `host (challenge.js, captcha.js or jsapi.js), got '${value}'. CapSkip `
+        + 'refuses any other script, because it is not an AWS WAF integration.',
+      );
+    }
+  }
+
+  const unknown = unknownKeys(params, AMAZON_WAF_SUBMIT);
+  if (unknown.length > 0) {
+    throw new ValidationException(
+      `Unsupported parameters for Amazon WAF: ${reprList(unknown)}.`,
+    );
+  }
+}
+
+const HTML_NAMED_ENTITIES = {
+  amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ',
+};
+
+/**
+ * Decode HTML character references, as the server does before it parses a
+ * DataDome captcha_url: an iframe src copied out of the page source carries
+ * &amp; between its query parameters.
+ */
+function htmlUnescape(text) {
+  return text.replace(/&(#[0-9]+|#[xX][0-9a-fA-F]+|[A-Za-z]+);/g, (match, ref) => {
+    if (ref[0] === '#') {
+      const code = ref[1] === 'x' || ref[1] === 'X'
+        ? parseInt(ref.slice(2), 16)
+        : parseInt(ref.slice(1), 10);
+      try {
+        return String.fromCodePoint(code);
+      } catch (error) {
+        return match;
+      }
+    }
+    const named = HTML_NAMED_ENTITIES[ref.toLowerCase()];
+    return named === undefined ? match : named;
+  });
+}
+
+function isDataDomeHost(host) {
+  const lowered = String(host || '').toLowerCase();
+  return lowered === DATADOME_CHALLENGE_DOMAIN
+    || lowered.endsWith(`.${DATADOME_CHALLENGE_DOMAIN}`);
+}
+
+/**
+ * The query of a DataDome challenge URL as a URLSearchParams, or null if it is
+ * not one.
+ *
+ * Mirrors the server: an https URL on captcha-delivery.com (or a subdomain),
+ * path /captcha/ or /interstitial/, carrying initialCid, hash and cid. An
+ * iframe src copied HTML-escaped (&amp;) and a scheme-relative //geo... URL
+ * are both read the way the server reads them.
+ */
+function parseDataDomeCaptchaUrl(url) {
+  let text = htmlUnescape(String(url === undefined || url === null ? '' : url).trim());
+  if (text.startsWith('//')) {
+    text = `https:${text}`;
+  }
+  let parsed;
+  try {
+    parsed = new URL(text);
+  } catch (error) {
+    return null;
+  }
+  if (parsed.protocol !== 'https:' || !isDataDomeHost(parsed.hostname)) {
+    return null;
+  }
+  const pathname = parsed.pathname.replace(/\/+$/, '').toLowerCase();
+  if (!DATADOME_CHALLENGE_PATHS.includes(pathname)) {
+    return null;
+  }
+  const query = parsed.searchParams;
+  if (!['initialCid', 'hash', 'cid'].every((key) => query.get(key))) {
+    return null;
+  }
+  return query;
+}
+
+/**
+ * True for a stock Chrome-for-Windows User-Agent, the only kind CapSkip takes.
+ *
+ * DataDome ties the cookie to the browser that earned it, and CapSkip solves
+ * in Chrome on Windows, so a cookie for any other User-Agent would be refused
+ * on replay. The server also requires the same Chrome major version as its
+ * own browser; that part can only be checked there.
+ */
+function isDataDomeUserAgent(userAgent) {
+  const ua = String(userAgent === undefined || userAgent === null ? '' : userAgent);
+  return ua.includes('Windows NT ')
+    && CHROME_VERSION.test(ua)
+    && !NOT_STOCK_WINDOWS_CHROME.some((token) => ua.includes(token));
+}
+
+/**
+ * Drop unset optionals, and the empty ones too: a DataDome optional left as ''
+ * means "not sent" to the server, exactly like an absent one.
+ */
+function normalizeDataDomeSubmit(params) {
+  const out = {};
+  for (const [key, value] of Object.entries(dropUnset(params))) {
+    if (value !== '') {
+      out[key] = value;
+    }
+  }
+  return out;
+}
+
+/** An absolute http(s) URL with a host that is not DataDome's own. */
+function isDataDomePageUrl(pageurl) {
+  let parsed;
+  try {
+    parsed = new URL(pageurl);
+  } catch (error) {
+    return false;
+  }
+  return (parsed.protocol === 'http:' || parsed.protocol === 'https:')
+    && parsed.hostname !== ''
+    && !isDataDomeHost(parsed.hostname);
+}
+
+function validateDataDomeSubmit(params) {
+  // Without captcha_url the server answers ERROR_BAD_PARAMETERS and without
+  // pageurl ERROR_PAGEURL. Fail locally so a missing value costs no round-trip.
+  for (const key of ['captcha_url', 'pageurl']) {
+    if (!params[key]) {
+      throw new ValidationException(`'${key}' is required for DataDome.`);
+    }
+  }
+
+  const query = parseDataDomeCaptchaUrl(params.captcha_url);
+  if (query === null) {
+    throw new ValidationException(
+      "'captcha_url' must be the https address of a DataDome challenge on "
+      + 'captcha-delivery.com, under /captcha/ or /interstitial/ and carrying '
+      + `initialCid, hash and cid, got '${params.captcha_url}'. Take it from `
+      + "the block page's iframe src, or build it from the page's `var dd` object.",
+    );
+  }
+  // t=bv is DataDome saying the address that fetched the block page is
+  // banned. Nothing sent from it can pass, and the server refuses the task
+  // with ERROR_PROXY_BANNED, so say so before the round-trip.
+  if (query.get('t') === 'bv') {
+    throw new ValidationException(
+      'This DataDome block page bans the IP address that requested it '
+      + '(t=bv), so no solve from that address can pass. Change the address '
+      + '(proxy), request the page again and submit the new captcha_url.',
+    );
+  }
+
+  const pageurl = String(params.pageurl).trim();
+  if (!isDataDomePageUrl(pageurl)) {
+    throw new ValidationException(
+      "'pageurl' must be the full http or https URL of the page that "
+      + `returned the DataDome block page, got '${pageurl}'.`,
+    );
+  }
+
+  const userAgent = params.useragent;
+  if (userAgent !== undefined && userAgent !== null && userAgent !== ''
+      && !isDataDomeUserAgent(userAgent)) {
+    throw new ValidationException(
+      "DataDome 'userAgent' must be a stock Chrome for Windows User-Agent "
+      + 'with the same major version as the browser CapSkip solves with, got '
+      + `'${userAgent}'. The cookie only works with the User-Agent that `
+      + "earned it. Leave it out and CapSkip uses its own browser's, which "
+      + 'the result reports as `userAgent`.',
+    );
+  }
+
+  const unknown = unknownKeys(params, DATADOME_SUBMIT);
+  if (unknown.length > 0) {
+    throw new ValidationException(
+      `Unsupported parameters for DataDome: ${reprList(unknown)}.`,
+    );
+  }
+}
+
 function validateProxyType(params) {
   const proxytype = params.proxytype;
   if (proxytype === undefined || proxytype === null || proxytype === '') {
@@ -390,6 +685,12 @@ function prepareSubmitParams(params, captchaType, version = 'v2') {
   } else if (captchaType === 'friendly_captcha') {
     prepared = dropUnset(prepared);
     validateFriendlyCaptchaSubmit(prepared);
+  } else if (captchaType === 'amazon_waf') {
+    prepared = normalizeAmazonWafSubmit(prepared);
+    validateAmazonWafSubmit(prepared);
+  } else if (captchaType === 'datadome') {
+    prepared = normalizeDataDomeSubmit(prepared);
+    validateDataDomeSubmit(prepared);
   }
 
   // Skipped for 'normal', which rejects proxy outright with a clearer message.
@@ -410,6 +711,11 @@ module.exports = {
   CAPY_SUBMIT,
   CAPTCHAFOX_SUBMIT,
   FRIENDLY_CAPTCHA_SUBMIT,
+  AMAZON_WAF_SUBMIT,
+  AMAZON_WAF_SCRIPTS,
+  DATADOME_SUBMIT,
+  DATADOME_CHALLENGE_DOMAIN,
+  DATADOME_CHALLENGE_PATHS,
   CAPY_VERSIONS,
   FRIENDLY_CAPTCHA_VERSIONS,
   PROXY_TYPES,
@@ -426,6 +732,14 @@ module.exports = {
   validateCapySubmit,
   validateCaptchaFoxSubmit,
   validateFriendlyCaptchaSubmit,
+  isAmazonWafScriptUrl,
+  normalizeAmazonWafSubmit,
+  validateAmazonWafSubmit,
+  htmlUnescape,
+  parseDataDomeCaptchaUrl,
+  isDataDomeUserAgent,
+  normalizeDataDomeSubmit,
+  validateDataDomeSubmit,
   validateProxyType,
   prepareSubmitParams,
 };

package/src/index.js

@@ -27,5 +27,5 @@ module.exports = {
   NetworkException,
   ApiException,
   TimeoutException,
-  version: '1.1.0',
+  version: '1.5.0',
 };