@prosopo/provider 3.15.0 → 4.7.1

package/dist/cjs/tasks/spam/checkTrafficFilter.cjs

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const types = require("@prosopo/types");
+const checkTrafficFilter = (ipInfo, trafficFilter) => {
+  if (!ipInfo || !ipInfo.isValid) {
+    return { isBlocked: false };
+  }
+  if (trafficFilter.blockVpn && ipInfo.isVPN) {
+    return { isBlocked: true, reason: types.ResultReason.VPN_BLOCKED };
+  }
+  if (trafficFilter.blockProxy && ipInfo.isProxy) {
+    return { isBlocked: true, reason: types.ResultReason.PROXY_BLOCKED };
+  }
+  if (trafficFilter.blockTor && ipInfo.isTor) {
+    return { isBlocked: true, reason: types.ResultReason.TOR_BLOCKED };
+  }
+  if ((trafficFilter.blockAbuser ?? true) && ipInfo.isAbuser) {
+    const threshold = trafficFilter.abuserScoreThreshold ?? types.trafficFilterAbuserScoreThresholdDefault;
+    const maxScore = Math.max(
+      ipInfo.abuserScore ?? 0,
+      ipInfo.companyAbuserScore ?? 0
+    );
+    if (maxScore >= threshold) {
+      return { isBlocked: true, reason: types.ResultReason.ABUSER_BLOCKED };
+    }
+  }
+  if (trafficFilter.blockDatacenter && ipInfo.isDatacenter) {
+    return { isBlocked: true, reason: types.ResultReason.DATACENTER_BLOCKED };
+  }
+  if (trafficFilter.blockMobile && ipInfo.isMobile) {
+    return { isBlocked: true, reason: types.ResultReason.MOBILE_BLOCKED };
+  }
+  if (trafficFilter.blockSatellite && ipInfo.isSatellite) {
+    return { isBlocked: true, reason: types.ResultReason.SATELLITE_BLOCKED };
+  }
+  if (trafficFilter.blockCrawler && ipInfo.isCrawler) {
+    return { isBlocked: true, reason: types.ResultReason.CRAWLER_BLOCKED };
+  }
+  return { isBlocked: false };
+};
+exports.checkTrafficFilter = checkTrafficFilter;

package/dist/cjs/tasks/spam/evaluateEmailSpamRules.cjs

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const types = require("@prosopo/types");
+const GMAIL_DOMAINS = /* @__PURE__ */ new Set(["gmail.com", "googlemail.com"]);
+const DEFAULT_EMAIL_SPAM_PATTERNS = [
+  // More than 3 dots in the local part of any address (very common
+  // evasion against systems that don't normalise gmail addresses).
+  {
+    name: "many-dots-local-part",
+    pattern: /^[^@]*\.[^@]*\.[^@]*\.[^@]*\.[^@]*@/
+  },
+  // Random alphanumeric suffix tags on gmail (e.g. +vkd38uoukd5).
+  {
+    name: "gmail-random-plus-tag",
+    pattern: /^[^@+]+\+[a-z0-9]{6,}@(gmail|googlemail)\.com$/i
+  }
+];
+const splitEmail = (email) => {
+  const trimmed = email.trim().toLowerCase();
+  const at = trimmed.lastIndexOf("@");
+  if (at <= 0 || at === trimmed.length - 1) {
+    return null;
+  }
+  return { local: trimmed.slice(0, at), domain: trimmed.slice(at + 1) };
+};
+const normaliseGmailAddress = (email) => {
+  const parts = splitEmail(email);
+  if (!parts) return email.trim().toLowerCase();
+  if (!GMAIL_DOMAINS.has(parts.domain)) {
+    return `${parts.local}@${parts.domain}`;
+  }
+  const stripped = parts.local.replace(/\./g, "").split("+")[0] ?? "";
+  return `${stripped}@gmail.com`;
+};
+const countDots = (s) => {
+  let n = 0;
+  for (const c of s) if (c === ".") n += 1;
+  return n;
+};
+const regexCache = /* @__PURE__ */ new Map();
+const getCompiledRegex = (raw) => {
+  const cached = regexCache.get(raw);
+  if (cached !== void 0) {
+    return cached;
+  }
+  let compiled;
+  try {
+    compiled = new RegExp(raw, "i");
+  } catch {
+    compiled = null;
+  }
+  regexCache.set(raw, compiled);
+  return compiled;
+};
+const evaluateEmailSpamRules = (email, rules) => {
+  if (!rules || !rules.enabled || !email) {
+    return { isSpam: false };
+  }
+  const parts = splitEmail(email);
+  if (!parts) {
+    return { isSpam: true, reason: types.ResultReason.EMAIL_INVALID };
+  }
+  if (typeof rules.maxLocalPartDots === "number" && countDots(parts.local) > rules.maxLocalPartDots) {
+    return { isSpam: true, reason: types.ResultReason.EMAIL_TOO_MANY_DOTS };
+  }
+  const target = rules.normaliseGmail ? normaliseGmailAddress(`${parts.local}@${parts.domain}`) : `${parts.local}@${parts.domain}`;
+  if (rules.useDefaultPatterns) {
+    for (const { name, pattern } of DEFAULT_EMAIL_SPAM_PATTERNS) {
+      if (pattern.test(`${parts.local}@${parts.domain}`)) {
+        return {
+          isSpam: true,
+          reason: types.ResultReason.EMAIL_MATCHED_DEFAULT_PATTERN,
+          pattern: name
+        };
+      }
+    }
+  }
+  for (const raw of rules.customRegexBlocklist ?? []) {
+    const regex = getCompiledRegex(raw);
+    if (regex?.test(target)) {
+      return {
+        isSpam: true,
+        reason: types.ResultReason.EMAIL_MATCHED_CUSTOM_PATTERN,
+        pattern: raw
+      };
+    }
+  }
+  return { isSpam: false };
+};
+exports.DEFAULT_EMAIL_SPAM_PATTERNS = DEFAULT_EMAIL_SPAM_PATTERNS;
+exports.evaluateEmailSpamRules = evaluateEmailSpamRules;
+exports.normaliseGmailAddress = normaliseGmailAddress;

package/dist/cjs/tasks/spam/updateSpamEmailDomains.cjs

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const fs = require("node:fs");
+const types = require("@prosopo/types");
+const util = require("@prosopo/util");
+const node = require("@prosopo/util/node");
+const CACHE_FILE_PREFIX = "spam-email-domains";
+const CACHE_FILE_TYPE = ".txt";
+const updateSpamEmailDomains = async (db, logger, spamEmailDomainsUrls, cacheDir = "./spam-cache") => {
+  const taskID = await db.createScheduledTaskStatus(
+    types.ScheduledTaskNames.UpdateSpamEmailDomains,
+    types.ScheduledTaskStatus.Running
+  );
+  try {
+    const domainsSet = /* @__PURE__ */ new Set();
+    for (const [index, url] of spamEmailDomainsUrls.entries()) {
+      const filePath = await node.cacheFile(
+        cacheDir,
+        url,
+        logger,
+        `${CACHE_FILE_PREFIX}-${index.toString()}-`,
+        CACHE_FILE_TYPE
+      );
+      const text = fs.readFileSync(filePath, "utf-8");
+      const domains = text.trim().split("\n");
+      for (const domain of domains) {
+        const trimmedDomain = domain.trim().toLowerCase();
+        if (trimmedDomain && !trimmedDomain.startsWith("#") && !trimmedDomain.startsWith("//") && util.validateDomain(trimmedDomain)) {
+          domainsSet.add(trimmedDomain);
+        }
+      }
+    }
+    const ops = Array.from(domainsSet).map((domain) => ({
+      filter: { domain: domain.trim() },
+      update: { domain: domain.trim() }
+    }));
+    await db.bulkUpdateSpamEmailDomains(ops, true);
+    await db.updateScheduledTaskStatus(taskID, types.ScheduledTaskStatus.Completed, {
+      data: {
+        domainsProcessed: domainsSet.size
+      }
+    });
+    logger.info(() => ({
+      msg: "Successfully updated spam email domains.",
+      data: { domainsProcessed: domainsSet.size }
+    }));
+  } catch (error) {
+    logger.error(() => ({
+      msg: "Error updating spam email domains.",
+      error
+    }));
+    await db.updateScheduledTaskStatus(taskID, types.ScheduledTaskStatus.Failed, {
+      error: error instanceof Error ? error.message : String(error)
+    });
+    throw error;
+  }
+};
+exports.updateSpamEmailDomains = updateSpamEmailDomains;

package/dist/cjs/tasks/tasks.cjs

@@ -1,28 +1,57 @@
 "use strict";
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const common = require("@prosopo/common");
+const database = require("@prosopo/database");
+const logger = require("@prosopo/logger");
+const usageCounters = require("../util/usageCounters.cjs");
 const clientTasks = require("./client/clientTasks.cjs");
 const datasetTasks = require("./dataset/datasetTasks.cjs");
+const decisionMachineRunner = require("./decisionMachine/decisionMachineRunner.cjs");
 const frictionlessTasks = require("./frictionless/frictionlessTasks.cjs");
 const imgCaptchaTasks = require("./imgCaptcha/imgCaptchaTasks.cjs");
 const powTasks = require("./powCaptcha/powTasks.cjs");
+const puzzleTasks = require("./puzzleCaptcha/puzzleTasks.cjs");
+let globalWriteQueue = null;
+let flushStarted = false;
 class Tasks {
-  constructor(env, logger) {
+  constructor(env, logger$1) {
     this.config = env.config;
     this.db = env.getDb();
     this.captchaConfig = env.config.captchas;
-    this.logger = logger || common.getLogger(common.parseLogLevel(env.config.logLevel), "Tasks");
+    this.logger = logger$1 || logger.getLogger(logger.parseLogLevel(env.config.logLevel), "Tasks");
     if (!env.pair) {
       throw new common.ProsopoEnvError("DEVELOPER.MISSING_PROVIDER_PAIR", {
         context: { failedFuncName: "Tasks.constructor" }
       });
     }
     this.pair = env.pair;
+    if (this.config.mongoCaptchaUri && this.db instanceof database.ProviderDatabase && !this.db.hasCentralDbStreamer()) {
+      const streamer = new database.CentralDbStreamer(
+        this.config.mongoCaptchaUri,
+        this.logger
+      );
+      this.db.setCentralDbStreamer(streamer);
+      this.logger.info(() => ({
+        msg: "Central DB streamer initialized for real-time captcha data streaming"
+      }));
+    }
+    this.writeQueue = this.initWriteQueue();
+    this.decisionMachineRunner = new decisionMachineRunner.DecisionMachineRunner(this.db);
+    this.usageCounters = this.initUsageCounters();
     this.powCaptchaManager = new powTasks.PowCaptchaManager(
       this.db,
       this.pair,
       this.config,
-      this.logger
+      this.logger,
+      this.writeQueue,
+      this.usageCounters
+    );
+    this.puzzleCaptchaManager = new puzzleTasks.PuzzleCaptchaManager(
+      this.db,
+      this.pair,
+      this.config,
+      this.logger,
+      this.usageCounters
     );
     this.datasetManager = new datasetTasks.DatasetManager(
       this.config,
@@ -34,7 +63,9 @@ class Tasks {
       this.db,
       this.pair,
       this.config,
-      this.logger
+      this.logger,
+      this.writeQueue,
+      this.usageCounters
     );
     this.clientTaskManager = new clientTasks.ClientTaskManager(
       this.config,
@@ -45,17 +76,84 @@ class Tasks {
       this.db,
       this.pair,
       this.config,
-      this.logger
+      this.logger,
+      this.writeQueue,
+      this.decisionMachineRunner,
+      this.usageCounters
     );
   }
-  setLogger(logger) {
-    this.logger = logger;
-    this.powCaptchaManager.logger = logger;
-    this.datasetManager.logger = logger;
-    this.imgCaptchaManager.logger = logger;
-    this.clientTaskManager.logger = logger;
-    this.frictionlessManager.logger = logger;
-    this.db.logger = logger;
+  initUsageCounters() {
+    if (!(this.db instanceof database.ProviderDatabase)) return null;
+    try {
+      return new usageCounters.UsageCounters(this.db.getRedisConnection(), this.logger);
+    } catch {
+      return null;
+    }
+  }
+  initWriteQueue() {
+    if (globalWriteQueue) {
+      return globalWriteQueue;
+    }
+    if (!(this.db instanceof database.ProviderDatabase)) {
+      return null;
+    }
+    try {
+      const redisConnection = this.db.getRedisConnection();
+      globalWriteQueue = new database.RedisWriteQueue(redisConnection, this.logger);
+      if (!flushStarted) {
+        flushStarted = true;
+        const db = this.db;
+        const logger2 = this.logger;
+        globalWriteQueue.startPeriodicFlush(async (queue) => {
+          await Tasks.flushWriteQueue(queue, db, logger2);
+        }, 1e4);
+      }
+      return globalWriteQueue;
+    } catch {
+      this.logger.debug(() => ({
+        msg: "Redis not available for write queue - falling back to direct writes"
+      }));
+      return null;
+    }
+  }
+  /**
+   * Flush queued session records from Redis to MongoDB in bulk.
+   * Called periodically by the write queue timer.
+   *
+   * Note: PoW challenge and pending image commitment records are written
+   * directly to MongoDB (not queued) because verification endpoints need
+   * to read them immediately after creation. Session records are safe to
+   * queue because they are not read in the immediate request path.
+   */
+  static async flushWriteQueue(queue, db, logger2) {
+    const sessionRecords = await queue.drainSessionRecords();
+    if (sessionRecords.length > 0) {
+      logger2.info(() => ({
+        msg: `Flushing ${sessionRecords.length} queued session records to MongoDB`
+      }));
+      for (const { record } of sessionRecords) {
+        try {
+          await db.storeSessionRecord(
+            record
+          );
+        } catch (error) {
+          logger2.error(() => ({
+            msg: "Failed to flush queued session record",
+            err: error
+          }));
+        }
+      }
+    }
+  }
+  setLogger(logger2) {
+    this.logger = logger2;
+    this.powCaptchaManager.logger = logger2;
+    this.puzzleCaptchaManager.logger = logger2;
+    this.datasetManager.logger = logger2;
+    this.imgCaptchaManager.logger = logger2;
+    this.clientTaskManager.logger = logger2;
+    this.frictionlessManager.logger = logger2;
+    this.db.logger = logger2;
   }
 }
 exports.Tasks = Tasks;

package/dist/cjs/util/usageCounters.cjs

@@ -0,0 +1,201 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const types = require("@prosopo/types");
+const INCR_WITH_TTL_LUA = `local count = redis.call('INCR', KEYS[1])
+if count == 1 then
+  redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1]))
+end
+return count`;
+class UsageCounters {
+  constructor(connection, logger) {
+    this.scriptShaPromise = null;
+    this.connection = connection;
+    this.logger = logger;
+  }
+  async getClient() {
+    if (!this.connection.isReady()) {
+      return null;
+    }
+    try {
+      return await this.connection.getClient();
+    } catch {
+      return null;
+    }
+  }
+  async getScriptSha(client) {
+    if (!this.scriptShaPromise) {
+      this.scriptShaPromise = client.scriptLoad(INCR_WITH_TTL_LUA).then(
+        (sha) => sha,
+        (error) => {
+          this.scriptShaPromise = null;
+          throw error;
+        }
+      );
+    }
+    return this.scriptShaPromise;
+  }
+  /**
+   * Atomically increment a counter. Sets TTL only on the first write within
+   * the window. Returns the post-increment value, or null on Redis failure.
+   */
+  async incr(dappAccount, spec, value) {
+    const client = await this.getClient();
+    if (!client) {
+      return null;
+    }
+    const key = types.encodeCounterKey(dappAccount, spec, value);
+    const ttlSeconds = types.COUNTER_WINDOW_SECONDS[spec.window];
+    try {
+      const sha = await this.getScriptSha(client);
+      const result = await client.evalSha(sha, {
+        keys: [key],
+        arguments: [String(ttlSeconds)]
+      });
+      return Number(result);
+    } catch (error) {
+      this.logger.warn(() => ({
+        msg: "Counter incr failed",
+        err: error,
+        data: { key }
+      }));
+      return null;
+    }
+  }
+  /**
+   * Fire-and-forget increment. Errors are logged but do not propagate.
+   */
+  incrAsync(dappAccount, spec, value) {
+    this.incr(dappAccount, spec, value).catch((error) => {
+      this.logger.error(() => ({
+        msg: "Counter incrAsync rejected",
+        err: error,
+        data: { dappAccount, spec, value }
+      }));
+    });
+  }
+  /**
+   * Fire-and-forget increment of multiple counters in parallel.
+   */
+  incrManyAsync(dappAccount, increments) {
+    for (const { spec, value } of increments) {
+      this.incrAsync(dappAccount, spec, value);
+    }
+  }
+  /**
+   * Delete every counter key matching the given dapp (or all counters when
+   * dappAccount is omitted). Intended for admin manual-testing flows — not
+   * for the hot path. Uses SCAN + DEL in chunks; returns the number of keys
+   * deleted. Returns null on Redis failure.
+   */
+  async clearAll(dappAccount) {
+    const client = await this.getClient();
+    if (!client) {
+      return null;
+    }
+    const pattern = dappAccount ? `cnt:${dappAccount}:*` : "cnt:*";
+    let deleted = 0;
+    try {
+      const batchSize = 500;
+      let batch = [];
+      for await (const key of client.scanIterator({
+        MATCH: pattern,
+        COUNT: 200
+      })) {
+        if (Array.isArray(key)) {
+          for (const k of key) batch.push(k);
+        } else {
+          batch.push(key);
+        }
+        if (batch.length >= batchSize) {
+          deleted += Number(await client.del(batch));
+          batch = [];
+        }
+      }
+      if (batch.length > 0) {
+        deleted += Number(await client.del(batch));
+      }
+      return deleted;
+    } catch (error) {
+      this.logger.warn(() => ({
+        msg: "Counter clearAll failed",
+        err: error,
+        data: { pattern, deleted }
+      }));
+      return null;
+    }
+  }
+  /**
+   * Bulk read counters via MGET. Missing keys map to 0. Returns a record
+   * keyed by encoded counter key. Returns null on Redis failure so callers
+   * can fall back.
+   */
+  async batchGet(dappAccount, reads) {
+    if (reads.length === 0) {
+      return {};
+    }
+    const client = await this.getClient();
+    if (!client) {
+      return null;
+    }
+    const keys = reads.map(
+      (r) => types.encodeCounterKey(dappAccount, r.spec, r.value)
+    );
+    try {
+      const values = await client.mGet(keys);
+      const out = {};
+      for (let i = 0; i < keys.length; i++) {
+        const raw = values[i];
+        const key = keys[i];
+        if (key === void 0) continue;
+        out[key] = raw === null || raw === void 0 ? 0 : Number(raw);
+      }
+      return out;
+    } catch (error) {
+      this.logger.warn(() => ({
+        msg: "Counter batchGet failed",
+        err: error,
+        data: { keyCount: keys.length }
+      }));
+      return null;
+    }
+  }
+}
+const buildAllWindowIncrements = (kind, captchaType, ip, userAccount) => {
+  const out = [];
+  for (const window of [
+    "1m",
+    "10m",
+    "1h",
+    "3h",
+    "6h",
+    "24h"
+  ]) {
+    out.push(
+      { spec: { kind, captchaType, dimension: "ip", window }, value: ip },
+      {
+        spec: { kind, captchaType, dimension: "userAccount", window },
+        value: userAccount
+      }
+    );
+    if (captchaType !== "any") {
+      out.push(
+        {
+          spec: { kind, captchaType: "any", dimension: "ip", window },
+          value: ip
+        },
+        {
+          spec: {
+            kind,
+            captchaType: "any",
+            dimension: "userAccount",
+            window
+          },
+          value: userAccount
+        }
+      );
+    }
+  }
+  return out;
+};
+exports.UsageCounters = UsageCounters;
+exports.buildAllWindowIncrements = buildAllWindowIncrements;

package/dist/cjs/util.cjs

@@ -162,7 +162,6 @@ const evaluateIpValidationRules = (comparison, rules, logger) => {
       message: `IP addresses are ${distanceKm.toFixed(2)}km apart (>${effectiveRules.distanceThresholdKm}km limit)`
     });
   }
-  console.log(JSON.stringify(effectiveRules, null, 2));
   const ip2AbuseScore = comparison.comparison.ip2Details?.abuserScore;
   if (ip2AbuseScore !== void 0 && ip2AbuseScore > effectiveRules.abuseScoreThreshold) {
     conditions.push({
@@ -225,7 +224,7 @@ const evaluateIpValidationRules = (comparison, rules, logger) => {
     shouldFlag
   };
 };
-const deepValidateIpAddress = async (ip, challengeIpAddress, logger, apiKey, apiUrl, ipValidationRules) => {
+const deepValidateIpAddress = async (ip, challengeIpAddress, logger, ipInfoService, ipValidationRules) => {
   const standardValidation = validateIpAddress(ip, challengeIpAddress, logger);
   if (!standardValidation.isValid) {
     if (standardValidation.errorMessage?.includes("Invalid IP address")) {
@@ -249,7 +248,7 @@ const deepValidateIpAddress = async (ip, challengeIpAddress, logger, apiKey, api
   }
   try {
     const challengeIpString = challengeIpAddress.address;
-    const comparison = await ipComparison.compareIPs(challengeIpString, ip, apiKey, apiUrl);
+    const comparison = await ipComparison.compareIPs(challengeIpString, ip, ipInfoService);
     if ("error" in comparison) {
       logger.error(() => ({
         msg: "Failed to get IP distance comparison",
@@ -260,7 +259,7 @@ const deepValidateIpAddress = async (ip, challengeIpAddress, logger, apiKey, api
         }
       }));
       return {
-        isValid: false,
+        isValid: true,
         errorMessage: "Could not determine IP distance"
       };
     }

package/dist/cjs/utils/devicePlatform.cjs

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const APPLE_UA_REGEX = /iPhone|iPad|iPod|Macintosh|Mac OS X/i;
+const MOBILE_UA_REGEX = /Mobile|Android|iPhone|iPad|iPod|Opera Mini|IEMobile/i;
+const derivePlatform = (userAgent, webView, ipInfo) => ({
+  isApple: APPLE_UA_REGEX.test(userAgent),
+  isWebView: webView,
+  isMobile: typeof ipInfo?.isMobile === "boolean" ? ipInfo.isMobile : MOBILE_UA_REGEX.test(userAgent)
+});
+exports.derivePlatform = derivePlatform;