@prosopo/provider 3.2.5 → 3.12.3

package/dist/api/captcha.js

@@ -4,8 +4,11 @@ import { parseCaptchaAssets } from "@prosopo/datasets";
 import { ClientApiPaths, CaptchaRequestBody, CaptchaType, ApiParams, CaptchaSolutionBody, GetPowCaptchaChallengeRequestBody, SubmitPowCaptchaSolutionBody, GetFrictionlessCaptchaChallengeRequestBody } from "@prosopo/types";
 import { getIPAddress, flatten } from "@prosopo/util";
 import express from "express";
+import { getCompositeIpAddress } from "../compositeIpAddress.js";
 import { FrictionlessManager } from "../tasks/frictionless/frictionlessTasks.js";
+import { timestampDecayFunction } from "../tasks/frictionless/frictionlessTasksUtils.js";
 import { Tasks } from "../tasks/tasks.js";
+import { hashUserAgent } from "../utils/hashUserAgent.js";
 import { getRequestUserScope } from "./blacklistRequestInspector.js";
 import { validateSiteKey, validateAddr } from "./validateAddress.js";
 const DEFAULT_FRICTIONLESS_THRESHOLD = 0.5;
@@ -63,11 +66,13 @@ function prosopoRouter(env) {
           dapp,
           userScope
         ))[0];
-        const { valid, reason, frictionlessTokenId } = await tasks.imgCaptchaManager.isValidRequest(
+        const { valid, reason, frictionlessTokenId, solvedImagesCount } = await tasks.imgCaptchaManager.isValidRequest(
           clientRecord,
           CaptchaType.image,
+          env,
           sessionId,
-          userAccessPolicy
+          userAccessPolicy,
+          req.ip
         );
         if (!valid) {
           return next(
@@ -84,7 +89,7 @@ function prosopoRouter(env) {
         }
         const captchaConfig = {
           solved: {
-            count: userAccessPolicy?.solvedImagesCount || env.config.captchas.solved.count
+            count: solvedImagesCount || userAccessPolicy?.solvedImagesCount || env.config.captchas.solved.count
           },
           unsolved: {
             count: userAccessPolicy?.unsolvedImagesCount || env.config.captchas.unsolved.count
@@ -115,12 +120,23 @@ function prosopoRouter(env) {
             }
           }
         };
+        req.logger.info(() => ({
+          msg: "Image captcha challenge issued",
+          data: {
+            captchaType: CaptchaType.image,
+            requestHash: taskData.requestHash,
+            solvedImagesCount: captchaConfig.solved.count,
+            user,
+            dapp,
+            sessionId
+          }
+        }));
         return res.json(captchaResponse);
       } catch (err) {
         req.logger.error(() => ({
           err,
           data: req.params,
-          msg: "Error in PoW captcha solution submission"
+          msg: "Error in image captcha challenge request"
         }));
         return next(
           new ProsopoApiError("API.BAD_REQUEST", {
@@ -175,7 +191,7 @@ function prosopoRouter(env) {
           parsed[ApiParams.signature].user.timestamp,
           Number.parseInt(parsed[ApiParams.timestamp]),
           parsed[ApiParams.signature].provider.requestHash,
-          getIPAddress(req.ip || "").bigInt(),
+          getIPAddress(req.ip || ""),
           flatten(req.headers),
           req.ja4
         );
@@ -190,7 +206,7 @@ function prosopoRouter(env) {
         req.logger.error(() => ({
           err,
           body: req.body,
-          msg: "Error in PoW captcha solution submission"
+          msg: "Error in image captcha solution submission"
         }));
         return next(
           new ProsopoApiError("API.BAD_REQUEST", {
@@ -246,11 +262,13 @@ function prosopoRouter(env) {
         dapp,
         userScope
       ))[0];
-      const { valid, reason, frictionlessTokenId } = await tasks.powCaptchaManager.isValidRequest(
+      const { valid, reason, frictionlessTokenId, powDifficulty } = await tasks.powCaptchaManager.isValidRequest(
         clientSettings,
         CaptchaType.pow,
+        env,
         sessionId,
-        userAccessPolicy
+        userAccessPolicy,
+        req.ip
       );
       if (!valid) {
         return next(
@@ -280,11 +298,12 @@ function prosopoRouter(env) {
           })
         );
       }
+      const difficulty = powDifficulty || userAccessPolicy?.powDifficulty || clientSettings?.settings?.powDifficulty;
       const challenge = await tasks.powCaptchaManager.getPowCaptchaChallenge(
         user,
         dapp,
         origin,
-        userAccessPolicy?.powDifficulty || clientSettings?.settings?.powDifficulty
+        difficulty
       );
       await tasks.db.storePowCaptchaRecord(
         challenge.challenge,
@@ -295,7 +314,7 @@ function prosopoRouter(env) {
         },
         challenge.difficulty,
         challenge.providerSignature,
-        getIPAddress(req.ip || "").bigInt(),
+        getCompositeIpAddress(req.ip || ""),
         flatten(req.headers),
         req.ja4,
         frictionlessTokenId
@@ -311,12 +330,23 @@ function prosopoRouter(env) {
           }
         }
       };
+      req.logger.info(() => ({
+        msg: "PoW captcha challenge issued",
+        data: {
+          captchaType: CaptchaType.pow,
+          challenge: challenge.challenge,
+          difficulty: challenge.difficulty,
+          user,
+          dapp,
+          session: sessionId
+        }
+      }));
       return res.json(getPowCaptchaResponse);
     } catch (err) {
       req.logger.error(() => ({
         err,
         body: req.body,
-        msg: "Error in PoW captcha solution submission"
+        msg: "Error in PoW captcha challenge request"
       }));
       return next(
         new ProsopoApiError("API.BAD_REQUEST", {
@@ -348,15 +378,7 @@ function prosopoRouter(env) {
           })
         );
       }
-      const {
-        challenge,
-        difficulty,
-        signature,
-        nonce,
-        verifiedTimeout,
-        dapp,
-        user
-      } = parsed;
+      const { challenge, signature, nonce, verifiedTimeout, dapp, user } = parsed;
       validateSiteKey(dapp);
       validateAddr(user);
       try {
@@ -372,7 +394,6 @@ function prosopoRouter(env) {
         }
         const verified = await tasks.powCaptchaManager.verifyPowCaptchaSolution(
           challenge,
-          difficulty,
           signature.provider.challenge,
           nonce,
           verifiedTimeout,
@@ -414,17 +435,39 @@ function prosopoRouter(env) {
             token: existingToken,
             msg: "Token has already been used"
           }));
-          return res.json(
-            await tasks.frictionlessManager.sendImageCaptcha(
-              existingToken._id
-            )
+          return next(
+            new ProsopoApiError("API.BAD_REQUEST", {
+              context: {
+                code: 400,
+                siteKey: dapp,
+                user
+              },
+              i18n: req.i18n,
+              logger: req.logger
+            })
           );
         }
         const lScore = tasks.frictionlessManager.checkLangRules(
           req.headers["accept-language"] || ""
         );
-        const { baseBotScore, timestamp } = await tasks.frictionlessManager.decryptPayload(token);
-        const botScore = baseBotScore + lScore;
+        const {
+          baseBotScore,
+          timestamp,
+          providerSelectEntropy,
+          userId,
+          userAgent
+        } = await tasks.frictionlessManager.decryptPayload(token);
+        req.logger.debug(() => ({
+          msg: "Decrypted payload",
+          data: {
+            baseBotScore,
+            timestamp,
+            providerSelectEntropy,
+            userId,
+            userAgent
+          }
+        }));
+        let botScore = baseBotScore + lScore;
         const clientRecord = await tasks.db.getClientRecord(dapp);
         if (!clientRecord) {
           return next(
@@ -437,7 +480,8 @@ function prosopoRouter(env) {
         }
         const { valid, reason } = await tasks.frictionlessManager.isValidRequest(
           clientRecord,
-          CaptchaType.frictionless
+          CaptchaType.frictionless,
+          env
         );
         if (!valid) {
           return next(
@@ -460,7 +504,9 @@ function prosopoRouter(env) {
           scoreComponents: {
             baseScore: baseBotScore,
             ...lScore && { lScore }
-          }
+          },
+          providerSelectEntropy,
+          ipAddress: getCompositeIpAddress(req.ip || "")
         });
         const userScope = getRequestUserScope(
           flatten(req.headers),
@@ -473,6 +519,28 @@ function prosopoRouter(env) {
           dapp,
           userScope
         ))[0];
+        const headersUserAgent = req.headers["user-agent"];
+        const hashedHeadersUserAgent = headersUserAgent ? hashUserAgent(headersUserAgent) : "";
+        const headersProsopoUser = req.headers["prosopo-user"];
+        if (hashedHeadersUserAgent !== userAgent || headersProsopoUser !== userId) {
+          req.logger.info(() => ({
+            msg: "User agent or user id does not match",
+            data: {
+              headersUserAgent,
+              hashedHeadersUserAgent,
+              userAgent,
+              // This is the hashed user agent from the token
+              headersProsopoUser,
+              userId
+            }
+          }));
+          return res.json(
+            await tasks.frictionlessManager.sendImageCaptcha(
+              tokenId,
+              timestampDecayFunction(timestamp)
+            )
+          );
+        }
         if (userAccessPolicy) {
           await tasks.frictionlessManager.scoreIncreaseAccessPolicy(
             userAccessPolicy,
@@ -482,7 +550,10 @@ function prosopoRouter(env) {
           );
           if (userAccessPolicy.captchaType === CaptchaType.image) {
             return res.json(
-              await tasks.frictionlessManager.sendImageCaptcha(tokenId)
+              await tasks.frictionlessManager.sendImageCaptcha(
+                tokenId,
+                userAccessPolicy.solvedImagesCount
+              )
             );
           }
           if (userAccessPolicy.captchaType === CaptchaType.pow) {
@@ -499,12 +570,26 @@ function prosopoRouter(env) {
             tokenId
           );
           return res.json(
-            await tasks.frictionlessManager.sendImageCaptcha(tokenId)
+            await tasks.frictionlessManager.sendImageCaptcha(
+              tokenId,
+              timestampDecayFunction(timestamp)
+            )
+          );
+        }
+        const hostVerified = await tasks.frictionlessManager.hostVerified(
+          providerSelectEntropy
+        );
+        if (!hostVerified.verified) {
+          botScore = await tasks.frictionlessManager.scoreIncreaseUnverifiedHost(
+            hostVerified.domain,
+            baseBotScore,
+            botScore,
+            tokenId
           );
         }
         if (Number(botScore) > botThreshold) {
           req.logger.info(() => ({
-            message: "Bot score is greater than threshold",
+            msg: "Bot score is greater than threshold",
             data: {
               botScore,
               botThreshold,
@@ -512,7 +597,10 @@ function prosopoRouter(env) {
             }
           }));
           return res.json(
-            await tasks.frictionlessManager.sendImageCaptcha(tokenId)
+            await tasks.frictionlessManager.sendImageCaptcha(
+              tokenId,
+              env.config.captchas.solved.count
+            )
           );
         }
         return res.json(

package/dist/api/domainMiddleware.js

@@ -8,26 +8,26 @@ const domainMiddleware = (env) => {
   const tasks = new Tasks(env);
   return async (req, res, next) => {
     try {
-      const dapp = req.headers["prosopo-site-key"];
-      if (!dapp)
+      const siteKey = req.headers["prosopo-site-key"];
+      if (!siteKey)
         throw siteKeyNotRegisteredError(
           req.i18n,
           "No sitekey provided",
           req.logger
         );
       try {
-        validateAddress(dapp, false, 42);
+        validateAddress(siteKey, false, 42);
       } catch (err) {
-        throw invalidSiteKeyError(req.i18n, dapp, req.logger);
+        throw invalidSiteKeyError(req.i18n, siteKey, req.logger);
       }
-      const clientSettings = await tasks.db.getClientRecord(dapp);
+      const clientSettings = await tasks.db.getClientRecord(siteKey);
       if (!clientSettings)
-        throw siteKeyNotRegisteredError(req.i18n, dapp, req.logger);
+        throw siteKeyNotRegisteredError(req.i18n, siteKey, req.logger);
       const allowedDomains = clientSettings.settings?.domains;
       if (!allowedDomains)
         throw siteKeyInvalidDomainError(
           req.i18n,
-          dapp,
+          siteKey,
           req.hostname,
           req.logger
         );
@@ -35,7 +35,7 @@ const domainMiddleware = (env) => {
       if (!origin)
         throw unauthorizedOriginError(req.i18n, void 0, req.logger);
       for (const domain of allowedDomains) {
-        if (tasks.clientTaskManager.isSubdomainOrExactMatch(origin, domain)) {
+        if (tasks.clientTaskManager.domainPatternMatcher(origin, domain)) {
           next();
           return;
         }

package/dist/api/headerCheckMiddleware.js

@@ -17,6 +17,10 @@ const headerCheckMiddleware = (env) => {
       validateAddr(user, void 0, req.logger);
       req.user = user;
       req.siteKey = siteKey;
+      req.logger = req.logger.with({
+        user,
+        siteKey
+      });
       next();
     } catch (err) {
       return handleErrors(err, req, res, next);

package/dist/api/ignoreMiddleware.js

@@ -1,6 +1,9 @@
-import { ApiPrefix } from "@prosopo/types";
+import { PublicApiPaths, ApiPrefix } from "@prosopo/types";
 function ignoreMiddleware() {
   return (req, res, next) => {
+    if (req.originalUrl.indexOf(PublicApiPaths.Healthz) !== -1) {
+      return next();
+    }
     if (req.originalUrl.indexOf(ApiPrefix) === -1) {
       res.statusCode = 404;
       res.send("Not Found");

package/dist/api/ja4Middleware.js

@@ -1,9 +1,8 @@
-import { createHash } from "node:crypto";
 import { Readable } from "node:stream";
 import { handleErrors } from "@prosopo/api-express-router";
 import { getLogger } from "@prosopo/common";
 import { randomAsHex } from "@prosopo/util-crypto";
-import { readTlsClientHello } from "read-tls-client-hello";
+import { readTlsClientHello, calculateJa4FromHelloData } from "read-tls-client-hello";
 const DEFAULT_JA4 = "ja4";
 const getJA4 = async (headers, logger) => {
   logger = logger || getLogger("info", import.meta.url);
@@ -15,7 +14,6 @@ const getJA4 = async (headers, logger) => {
   try {
     const xTlsClientHello = (headers["x-tls-clienthello"] || "").toString();
     const xTlsVersion = (headers["x-tls-version"] || "").toString().toLowerCase();
-    const xTlsServerName = (headers["x-tls-server-name"] || "").toString();
     const clientHelloBuffer = Buffer.from(xTlsClientHello, "base64");
     logger.debug(() => ({
       msg: "ClientHello First Bytes:",
@@ -31,32 +29,13 @@ const getJA4 = async (headers, logger) => {
       msg: "Headers TLS Version:",
       data: { xTlsVersion }
     }));
-    const tlsVersion = xTlsVersion.replace(/(tls)|\./g, "");
     const readableStream = new Readable({
       read() {
         this.push(clientHelloBuffer);
       }
     });
     const clientHello = await readTlsClientHello(readableStream);
-    const { alpnProtocols } = clientHello;
-    const [_tlsVersion, cipherSuites, extensions] = clientHello.fingerprintData;
-    const transport = "t";
-    const sniIndicator = xTlsServerName ? "d" : "i";
-    const validCipherSuites = cipherSuites.filter(
-      (cs) => (cs & 3855) !== 2570
-    );
-    const cipherCount = validCipherSuites.length;
-    const validExtensions = extensions.filter(
-      (ext) => (ext & 3855) !== 2570
-    );
-    const extensionCount = validExtensions.length;
-    const alpn = alpnProtocols?.length ? alpnProtocols[0] : "";
-    const alpnLabel = alpn ? `${alpn[0]}${alpn[alpn.length - 1]}` : "00";
-    const sortedCiphers = validCipherSuites.map((cs) => cs.toString(16).padStart(4, "0")).sort().join(",");
-    const cipherHash = createHash("sha256").update(sortedCiphers).digest("hex").slice(0, 12);
-    const decimalString = extensions.sort((a, b) => a - b).map((ext) => ext.toString(10)).join("-");
-    const extensionHash = createHash("sha256").update(decimalString).digest("hex").slice(0, 12);
-    const ja4PlusFingerprint = `${transport}${tlsVersion}${sniIndicator}${cipherCount}${extensionCount}${alpnLabel}_${cipherHash}_${extensionHash}`;
+    const ja4PlusFingerprint = calculateJa4FromHelloData(clientHello);
     return { ja4PlusFingerprint };
   } catch (e) {
     logger.error(() => ({
@@ -72,6 +51,9 @@ const ja4Middleware = (env) => {
       req.logger.debug(() => ({ data: { url: req.url } }));
       const ja4 = await getJA4(req.headers, req.logger);
       req.ja4 = ja4.ja4PlusFingerprint || "";
+      req.logger = req.logger.with({
+        ja4: req.ja4
+      });
       next();
     } catch (err) {
       return handleErrors(err, req, res, next);

package/dist/api/public.js

@@ -3,16 +3,39 @@ import { ProsopoApiError } from "@prosopo/common";
 import { PublicApiPaths } from "@prosopo/types";
 import { version } from "@prosopo/util";
 import express from "express";
-function publicRouter() {
+function publicRouter(env) {
   const router = express.Router();
   router.get(PublicApiPaths.Healthz, (req, res) => {
     res.status(200).send("OK");
   });
   router.get(PublicApiPaths.GetProviderDetails, async (req, res, next) => {
     try {
-      return res.json({ version, ...{ message: "Provider online" } });
+      const db = env.getDb();
+      const redisConnection = db.getRedisConnection();
+      const redisAccessRulesConnection = db.getRedisAccessRulesConnection();
+      const response = {
+        version,
+        message: "Provider online",
+        redis: [
+          {
+            actor: "General",
+            isReady: redisConnection.isReady(),
+            awaitingTimeSeconds: Math.ceil(
+              redisConnection.getAwaitingTimeMs() / 1e3
+            )
+          },
+          {
+            actor: "UAP",
+            isReady: redisAccessRulesConnection.isReady(),
+            awaitingTimeSeconds: Math.ceil(
+              redisAccessRulesConnection.getAwaitingTimeMs() / 1e3
+            )
+          }
+        ]
+      };
+      return res.json(response);
     } catch (err) {
-      req.logger.error(() => ({
+      env.logger.error(() => ({
         err,
         data: { reqParams: req.params },
         msg: "Error getting provider details"

package/dist/api/verify.js

@@ -22,7 +22,7 @@ function prosopoVerifyRouter(env) {
           })
         );
       }
-      const { dappSignature, token, ip } = parsed;
+      const { dappSignature, token, ip, maxVerifiedTime } = parsed;
       try {
         const { user, dapp, timestamp, commitmentId } = decodeProcaptchaOutput(token);
         validateAddress(dapp, false, 42);
@@ -43,7 +43,8 @@ function prosopoVerifyRouter(env) {
           user,
           dapp,
           commitmentId,
-          parsed.maxVerifiedTime,
+          env,
+          maxVerifiedTime,
           ip
         );
         req.logger.debug(() => ({ data: { response } }));
@@ -111,6 +112,7 @@ function prosopoVerifyRouter(env) {
           dapp,
           challenge,
           verifiedTimeout,
+          env,
           ip
         );
         const verificationResponse = tasks.powCaptchaManager.getVerificationResponse(

package/dist/cjs/api/admin/apiRemoveDetectorKeyEndpoint.cjs

@@ -10,10 +10,13 @@ class ApiRemoveDetectorKeyEndpoint {
   async processRequest(args, logger) {
     logger = logger || common.getLogger("info", module);
     try {
-      const { detectorKey } = args;
+      const { detectorKey, expirationInSeconds } = args;
       logger = logger || common.getLogger("info", module);
       logger.info(() => ({ msg: "Removing detector key" }));
-      await this.clientTaskManager.removeDetectorKey(detectorKey);
+      await this.clientTaskManager.removeDetectorKey(
+        detectorKey,
+        expirationInSeconds
+      );
       return {
         status: apiRoute.ApiEndpointResponseStatus.SUCCESS
       };
@@ -26,7 +29,7 @@ class ApiRemoveDetectorKeyEndpoint {
     }
   }
   getRequestArgsSchema() {
-    return types.UpdateDetectorKeyBody;
+    return types.RemoveDetectorKeyBodySpec;
   }
 }
 exports.ApiRemoveDetectorKeyEndpoint = ApiRemoveDetectorKeyEndpoint;

package/dist/cjs/api/blacklistRequestInspector.cjs

@@ -10,13 +10,12 @@ const getRequestUserScope = (requestHeaders, ja4, ip, user) => {
     ...ja4 && { ja4Hash: ja4 },
     ...userAgent && { userAgent },
     ...ip && { ip }
+    // TODO more things with headers
   };
 };
-const getPrioritisedAccessRule = async (userAccessRulesStorage, userScope, clientId) => {
-  const userScopeKeys = Object.keys(userScope).filter(
-    (key) => userScope[key] !== void 0
-  );
-  const prioritisedUserScopes = util.uniqueSubsets(userScopeKeys).map(
+const getPrioritisedUserScopes = (userScope) => {
+  const userScopeKeys = Object.keys(userScope);
+  return util.uniqueSubsets(userScopeKeys).map(
     (subset) => subset.reduce(
       (acc, key) => {
         acc[key] = userScope[key];
@@ -24,22 +23,29 @@ const getPrioritisedAccessRule = async (userAccessRulesStorage, userScope, clien
       },
       {}
     )
-  ).filter((us) => Object.keys(us).length > 0).filter((us) => Object.values(us).some((value) => value !== void 0));
+  );
+};
+const getPrioritisedAccessRule = async (userAccessRulesStorage, userScope, clientId) => {
+  const prioritisedUserScopes = getPrioritisedUserScopes(userScope);
   const policyPromises = [];
-  for (const clientOrUndefined of [clientId, void 0]) {
+  const clientLoop = clientId ? [clientId, void 0] : [void 0];
+  for (const clientOrUndefined of clientLoop) {
     for (const scope of prioritisedUserScopes) {
-      policyPromises.push(
-        userAccessRulesStorage.findRules({
-          ...clientOrUndefined && {
-            policyScope: {
-              clientId: clientOrUndefined
-            }
-          },
-          policyScopeMatch: userAccessPolicy.ScopeMatch.Exact,
-          userScope: userAccessPolicy.userScopeInputSchema.parse(scope),
-          userScopeMatch: userAccessPolicy.ScopeMatch.Exact
-        })
-      );
+      if (Object.values(scope).every((value) => value === void 0)) {
+        continue;
+      }
+      const parsedUserScope = userAccessPolicy.userScopeInputSchema.parse(scope);
+      const filter = {
+        ...clientOrUndefined && {
+          policyScope: {
+            clientId: clientOrUndefined
+          }
+        },
+        policyScopeMatch: userAccessPolicy.ScopeMatch.Exact,
+        userScope: parsedUserScope,
+        userScopeMatch: userAccessPolicy.ScopeMatch.Exact
+      };
+      policyPromises.push(userAccessRulesStorage.findRules(filter, true, true));
     }
   }
   return (await Promise.all(policyPromises)).flat();