@prosopo/database 3.5.6 → 3.13.7

package/dist/databases/provider.js

@@ -1,24 +1,28 @@
 import { isHex } from "@polkadot/util/is";
 import { ProsopoDBError } from "@prosopo/common";
 import { connectToRedis, setupRedisIndex } from "@prosopo/redis-client";
-import { DatasetWithIdsAndTreeSchema, StoredStatusNames, CaptchaStatus, ApiParams, CaptchaStates, ContextType } from "@prosopo/types";
-import { CaptchaRecordSchema, PoWCaptchaRecordSchema, DatasetRecordSchema, SolutionRecordSchema, UserCommitmentRecordSchema, UserSolutionRecordSchema, PendingRecordSchema, ScheduledTaskRecordSchema, ClientRecordSchema, SessionRecordSchema, DetectorRecordSchema, ClientContextEntropyRecordSchema, UserCommitmentSchema, ScheduledTaskSchema } from "@prosopo/types-database";
+import { DatasetWithIdsAndTreeSchema, UserCommitmentSchema, CaptchaStatus, StoredStatusNames, CaptchaStates, ContextType } from "@prosopo/types";
+import { CaptchaRecordSchema, PoWCaptchaRecordSchema, PuzzleCaptchaRecordSchema, DatasetRecordSchema, SolutionRecordSchema, UserCommitmentRecordSchema, UserSolutionRecordSchema, ScheduledTaskRecordSchema, ClientRecordSchema, SessionRecordSchema, DetectorRecordSchema, DecisionMachineArtifactRecordSchema, ClientContextEntropyRecordSchema, SpamEmailDomainRecordSchema, ScheduledTaskSchema } from "@prosopo/types-database";
 import { accessRulesRedisIndex, createRedisAccessRulesStorage } from "@prosopo/user-access-policy/redis";
+import { buildDomainSuffixCandidates } from "@prosopo/util";
 import { MongoDatabase } from "../base/mongo.js";
 const TWENTY_FOUR_HOURS_IN_MS = 24 * 60 * 60 * 1e3;
+const MAX_DOMAIN_SUFFIX_CANDIDATES = 5;
 var TableNames = /* @__PURE__ */ ((TableNames2) => {
   TableNames2["captcha"] = "captcha";
   TableNames2["dataset"] = "dataset";
   TableNames2["solution"] = "solution";
   TableNames2["commitment"] = "commitment";
   TableNames2["usersolution"] = "usersolution";
-  TableNames2["pending"] = "pending";
   TableNames2["scheduler"] = "scheduler";
   TableNames2["powcaptcha"] = "powcaptcha";
+  TableNames2["puzzlecaptcha"] = "puzzlecaptcha";
   TableNames2["client"] = "client";
   TableNames2["session"] = "session";
   TableNames2["detector"] = "detector";
+  TableNames2["decisionMachine"] = "decisionMachine";
   TableNames2["clientContextEntropy"] = "clientContextEntropy";
+  TableNames2["spamEmailDomain"] = "spamEmailDomain";
   return TableNames2;
 })(TableNames || {});
 const PROVIDER_TABLES = [
@@ -32,6 +36,11 @@ const PROVIDER_TABLES = [
     modelName: "PowCaptcha",
     schema: PoWCaptchaRecordSchema
   },
+  {
+    collectionName: "puzzlecaptcha",
+    modelName: "PuzzleCaptcha",
+    schema: PuzzleCaptchaRecordSchema
+  },
   {
     collectionName: "dataset",
     modelName: "Dataset",
@@ -52,11 +61,6 @@ const PROVIDER_TABLES = [
     modelName: "UserSolution",
     schema: UserSolutionRecordSchema
   },
-  {
-    collectionName: "pending",
-    modelName: "Pending",
-    schema: PendingRecordSchema
-  },
   {
     collectionName: "scheduler",
     modelName: "Scheduler",
@@ -77,10 +81,20 @@ const PROVIDER_TABLES = [
     modelName: "Detector",
     schema: DetectorRecordSchema
   },
+  {
+    collectionName: "decisionMachine",
+    modelName: "DecisionMachine",
+    schema: DecisionMachineArtifactRecordSchema
+  },
   {
     collectionName: "clientContextEntropy",
     modelName: "ClientContextEntropy",
     schema: ClientContextEntropyRecordSchema
+  },
+  {
+    collectionName: "spamEmailDomain",
+    modelName: "SpamEmailDomain",
+    schema: SpamEmailDomainRecordSchema
   }
 ];
 class ProviderDatabase extends MongoDatabase {
@@ -99,6 +113,12 @@ class ProviderDatabase extends MongoDatabase {
     this.redisConnection = null;
     this.userAccessRulesStorage = null;
   }
+  setCentralDbStreamer(streamer) {
+    this.centralStreamer = streamer;
+  }
+  hasCentralDbStreamer() {
+    return this.centralStreamer !== void 0;
+  }
   async connect() {
     await super.connect();
     this.loadTables();
@@ -380,7 +400,12 @@ class ProviderDatabase extends MongoDatabase {
    */
   async getCaptchaById(captchaId) {
     const filter = { captchaId: { $in: captchaId } };
-    const cursor = this.tables?.captcha.find(filter).lean();
+    const cursor = this.tables?.captcha.find(filter, {
+      _id: 0,
+      captchaId: 1,
+      datasetId: 1,
+      items: 1
+    }).lean();
     const docs = await cursor;
     if (docs?.length) {
       return docs.map(({ _id, ...keepAttrs }) => keepAttrs);
@@ -476,6 +501,11 @@ class ProviderDatabase extends MongoDatabase {
         }
       }));
       await this.tables?.usersolution.bulkWrite(ops);
+      this.centralStreamer?.streamImageRecord(
+        commitmentRecord,
+        (ts) => this.tables.commitment.updateOne({ id: commit.id }, { $set: { storedAtTimestamp: ts } }).then(() => {
+        })
+      );
     }
   }
   /**
@@ -492,9 +522,10 @@ class ProviderDatabase extends MongoDatabase {
    * @param userSubmitted
    * @param storedStatus
    * @param userSignature
+   * @param ipInfo full ipinfo payload from the ipInfoMiddleware
    * @returns {Promise<void>} A promise that resolves when the record is added.
    */
-  async storePowCaptchaRecord(challenge, components, difficulty, providerSignature, ipAddress, headers, ja4, sessionId, serverChecked = false, userSubmitted = false, storedStatus = StoredStatusNames.notStored, userSignature) {
+  async storePowCaptchaRecord(challenge, components, difficulty, providerSignature, ipAddress, headers, ja4, sessionId, serverChecked = false, userSubmitted = false, userSignature, ipInfo) {
     const tables = this.getTables();
     const powCaptchaRecord = {
       challenge,
@@ -511,7 +542,9 @@ class ProviderDatabase extends MongoDatabase {
       providerSignature,
       userSignature,
       lastUpdatedTimestamp: /* @__PURE__ */ new Date(),
-      sessionId
+      pendingStage: true,
+      sessionId,
+      ipInfo
     };
     try {
       await tables.powcaptcha.create(powCaptchaRecord);
@@ -520,10 +553,27 @@ class ProviderDatabase extends MongoDatabase {
           challenge,
           userSubmitted,
           serverChecked,
-          storedStatus
+          countryCode: ipInfo?.isValid ? ipInfo.countryCode : void 0
         },
         msg: "PowCaptcha record added successfully"
       }));
+      this.centralStreamer?.streamPowRecord(
+        powCaptchaRecord,
+        (ts) => (
+          // Guard with `lastUpdatedTimestamp: { $lte: ts }` so a concurrent
+          // update (newer lastUpdatedTimestamp) doesn't get its pendingStage
+          // flag cleared by this older stage completion. Mismatched docs
+          // leave pendingStage set so the next sweep picks them up.
+          this.tables.powcaptcha.updateOne(
+            { challenge, lastUpdatedTimestamp: { $lte: ts } },
+            {
+              $set: { storedAtTimestamp: ts },
+              $unset: { pendingStage: 1 }
+            }
+          ).then(() => {
+          })
+        )
+      );
     } catch (error) {
       const err = new ProsopoDBError("DATABASE.CAPTCHA_UPDATE_FAILED", {
         context: {
@@ -531,7 +581,7 @@ class ProviderDatabase extends MongoDatabase {
           challenge,
           userSubmitted,
           serverChecked,
-          storedStatus
+          ipInfo
         },
         logger: this.logger
       });
@@ -556,7 +606,24 @@ class ProviderDatabase extends MongoDatabase {
     }
     try {
       const filter = { challenge };
-      const record = await this.tables.powcaptcha.findOne(filter).lean();
+      const record = await this.tables.powcaptcha.findOne(filter, {
+        challenge: 1,
+        userAccount: 1,
+        dappAccount: 1,
+        requestedAtTimestamp: 1,
+        ipAddress: 1,
+        headers: 1,
+        ja4: 1,
+        result: 1,
+        difficulty: 1,
+        sessionId: 1,
+        ipInfo: 1,
+        deviceCapability: 1,
+        behavioralDataPacked: 1,
+        serverChecked: 1,
+        userSubmitted: 1,
+        coords: 1
+      }).lean();
       if (record) {
         this.logger.info(() => ({
           data: { challenge },
@@ -590,7 +657,7 @@ class ProviderDatabase extends MongoDatabase {
    * @param userSignature
    * @returns {Promise<void>} A promise that resolves when the record is updated.
    */
-  async updatePowCaptchaRecordResult(challenge, result, serverChecked = false, userSubmitted = false, userSignature) {
+  async updatePowCaptchaRecordResult(challenge, result, serverChecked = false, userSubmitted = false, userSignature, coords) {
     const tables = this.getTables();
     const timestamp = /* @__PURE__ */ new Date();
     const update = {
@@ -598,7 +665,9 @@ class ProviderDatabase extends MongoDatabase {
       serverChecked,
       userSubmitted,
       userSignature,
-      lastUpdatedTimestamp: timestamp
+      lastUpdatedTimestamp: timestamp,
+      pendingStage: true,
+      ...coords && { coords }
     };
     try {
       const updateResult = await tables.powcaptcha.updateOne(
@@ -628,6 +697,17 @@ class ProviderDatabase extends MongoDatabase {
         },
         msg: "PowCaptcha record updated successfully"
       }));
+      this.centralStreamer?.streamPowUpdate(
+        () => this.getPowCaptchaRecordByChallenge(challenge),
+        (ts) => this.tables.powcaptcha.updateOne(
+          { challenge, lastUpdatedTimestamp: { $lte: ts } },
+          {
+            $set: { storedAtTimestamp: ts },
+            $unset: { pendingStage: 1 }
+          }
+        ).then(() => {
+        })
+      );
     } catch (error) {
       const err = new ProsopoDBError("DATABASE.CAPTCHA_UPDATE_FAILED", {
         context: {
@@ -648,7 +728,205 @@ class ProviderDatabase extends MongoDatabase {
     const tables = this.getTables();
     await tables.powcaptcha.updateOne(
       { challenge },
-      { $set: updates },
+      { $set: { ...updates, pendingStage: true } },
+      { upsert: false }
+    );
+    this.centralStreamer?.streamPowUpdate(
+      () => this.getPowCaptchaRecordByChallenge(challenge),
+      (ts) => this.tables.powcaptcha.updateOne(
+        { challenge, lastUpdatedTimestamp: { $lte: ts } },
+        {
+          $set: { storedAtTimestamp: ts },
+          $unset: { pendingStage: 1 }
+        }
+      ).then(() => {
+      })
+    );
+  }
+  async storePuzzleCaptchaRecord(challenge, components, targetX, targetY, originX, originY, tolerance, providerSignature, ipAddress, headers, ja4, sessionId, ipInfo) {
+    const tables = this.getTables();
+    const puzzleCaptchaRecord = {
+      challenge,
+      userAccount: components.userAccount,
+      dappAccount: components.dappAccount,
+      requestedAtTimestamp: new Date(components.requestedAtTimestamp),
+      ipAddress,
+      headers,
+      ja4,
+      result: { status: CaptchaStatus.pending },
+      userSubmitted: false,
+      serverChecked: false,
+      targetX,
+      targetY,
+      originX,
+      originY,
+      tolerance,
+      providerSignature,
+      lastUpdatedTimestamp: /* @__PURE__ */ new Date(),
+      pendingStage: true,
+      sessionId,
+      ipInfo
+    };
+    try {
+      await tables.puzzlecaptcha.create(puzzleCaptchaRecord);
+      this.logger.info(() => ({
+        data: {
+          challenge,
+          countryCode: ipInfo?.isValid ? ipInfo.countryCode : void 0
+        },
+        msg: "PuzzleCaptcha record added successfully"
+      }));
+    } catch (error) {
+      const err = new ProsopoDBError("DATABASE.CAPTCHA_UPDATE_FAILED", {
+        context: {
+          error,
+          challenge,
+          ipInfo
+        },
+        logger: this.logger
+      });
+      this.logger.error(() => ({
+        err: error,
+        msg: "Failed to add PuzzleCaptcha record"
+      }));
+      throw err;
+    }
+  }
+  /**
+   * @description Retrieves a Puzzle Captcha record by its challenge string.
+   * @param {string} challenge The challenge string to search for.
+   * @returns {Promise<PuzzleCaptchaRecord | null>} A promise that resolves with the found record or null if not found.
+   */
+  async getPuzzleCaptchaRecordByChallenge(challenge) {
+    if (!this.tables) {
+      throw new ProsopoDBError("DATABASE.DATABASE_UNDEFINED", {
+        context: {
+          failedFuncName: this.getPuzzleCaptchaRecordByChallenge.name
+        },
+        logger: this.logger
+      });
+    }
+    try {
+      const filter = { challenge };
+      const record = await this.tables.puzzlecaptcha.findOne(filter, {
+        challenge: 1,
+        userAccount: 1,
+        dappAccount: 1,
+        requestedAtTimestamp: 1,
+        ipAddress: 1,
+        headers: 1,
+        ja4: 1,
+        result: 1,
+        targetX: 1,
+        targetY: 1,
+        originX: 1,
+        originY: 1,
+        tolerance: 1,
+        puzzleEvents: 1,
+        sessionId: 1,
+        ipInfo: 1,
+        deviceCapability: 1,
+        behavioralDataPacked: 1,
+        serverChecked: 1,
+        userSubmitted: 1,
+        coords: 1
+      }).lean();
+      if (record) {
+        this.logger.info(() => ({
+          data: { challenge },
+          msg: "PuzzleCaptcha record retrieved successfully"
+        }));
+        return record;
+      }
+      this.logger.info(() => ({
+        data: { challenge },
+        msg: "No PuzzleCaptcha record found"
+      }));
+      return null;
+    } catch (error) {
+      const err = new ProsopoDBError("DATABASE.CAPTCHA_GET_FAILED", {
+        context: { error, challenge },
+        logger: this.logger
+      });
+      this.logger.error(() => ({
+        err,
+        msg: "Failed to retrieve PuzzleCaptcha record"
+      }));
+      throw err;
+    }
+  }
+  /**
+   * @description Updates a Puzzle Captcha record result in the database.
+   * @param {string} challenge The challenge string of the captcha to be updated.
+   * @param result
+   * @param serverChecked
+   * @param userSubmitted
+   * @param userSignature
+   * @param coords
+   * @param lastUpdatedTimestamp
+   * @returns {Promise<void>} A promise that resolves when the record is updated.
+   */
+  async updatePuzzleCaptchaRecordResult(challenge, result, serverChecked = false, userSubmitted = false, userSignature, coords, lastUpdatedTimestamp) {
+    const tables = this.getTables();
+    const timestamp = lastUpdatedTimestamp ?? /* @__PURE__ */ new Date();
+    const update = {
+      result,
+      serverChecked,
+      userSubmitted,
+      userSignature,
+      lastUpdatedTimestamp: timestamp,
+      pendingStage: true,
+      ...coords && { coords }
+    };
+    try {
+      const updateResult = await tables.puzzlecaptcha.updateOne(
+        { challenge },
+        {
+          $set: update
+        }
+      );
+      if (updateResult.matchedCount === 0) {
+        const err = new ProsopoDBError("DATABASE.CAPTCHA_GET_FAILED", {
+          context: {
+            challenge,
+            ...update
+          },
+          logger: this.logger
+        });
+        this.logger.info(() => ({
+          err,
+          msg: "No PuzzleCaptcha record found to update"
+        }));
+        throw err;
+      }
+      this.logger.info(() => ({
+        data: {
+          challenge,
+          ...update
+        },
+        msg: "PuzzleCaptcha record updated successfully"
+      }));
+    } catch (error) {
+      const err = new ProsopoDBError("DATABASE.CAPTCHA_UPDATE_FAILED", {
+        context: {
+          error,
+          challenge,
+          ...update
+        },
+        logger: this.logger
+      });
+      this.logger.error(() => ({
+        err,
+        msg: "Failed to update PuzzleCaptcha record"
+      }));
+      throw err;
+    }
+  }
+  async updatePuzzleCaptchaRecord(challenge, updates) {
+    const tables = this.getTables();
+    await tables.puzzlecaptcha.updateOne(
+      { challenge },
+      { $set: { ...updates, pendingStage: true } },
       { upsert: false }
     );
   }
@@ -660,44 +938,38 @@ class ProviderDatabase extends MongoDatabase {
     return docs || [];
   }
   /** @description Get Dapp User captcha commitments from the commitments table that have not been counted towards the
-   * client's total
+   * client's total.
+   *
+   * Served by the `pendingStage_partial` index. Records have
+   * `pendingStage: true` set on insert and on every mutation (see
+   * `updateDappUserCommitment`, `markDappUserCommitmentsChecked`,
+   * `approveDappUserCommitment`, `disapproveDappUserCommitment`,
+   * `storePendingImageCommitment`). `markDappUserCommitmentsStored` clears
+   * the flag after a successful stage, guarded by `lastUpdatedTimestamp`
+   * so an in-flight update isn't lost.
    */
   async getUnstoredDappUserCommitments(limit = 1e3, skip = 0) {
-    const filterNoStoredTimestamp = { storedAtTimestamp: { $exists: false } };
-    const docs = await this.tables?.commitment.aggregate([
-      {
-        $match: {
-          $or: [
-            filterNoStoredTimestamp,
-            {
-              $expr: {
-                $lt: ["$storedAtTimestamp", "$lastUpdatedTimestamp"]
-              }
-            }
-          ]
-        }
-      },
-      {
-        $sort: { _id: 1 }
-      },
-      {
-        $skip: skip
-      },
-      {
-        $limit: limit
-      }
-    ]);
+    const docs = await this.tables?.commitment.find({ pendingStage: true }).sort({ _id: 1 }).skip(skip).limit(limit).lean();
     return docs || [];
   }
-  /** @description Mark a list of captcha commits as stored
+  /** @description Mark a list of captcha commits as stored.
+   *
+   * `asOfTimestamp` defaults to "now" but the sweep should pass the time
+   * at which it fetched the batch. The lastUpdatedTimestamp guard prevents
+   * us from clearing pendingStage on a record that was mutated between
+   * fetch and mark-stored — those records will leave pendingStage set so
+   * the next sweep picks them up.
    */
-  async markDappUserCommitmentsStored(commitmentIds) {
-    const updateDoc = {
-      storedAtTimestamp: /* @__PURE__ */ new Date()
-    };
+  async markDappUserCommitmentsStored(commitmentIds, asOfTimestamp = /* @__PURE__ */ new Date()) {
     await this.tables?.commitment.updateMany(
-      { id: { $in: commitmentIds } },
-      { $set: updateDoc },
+      {
+        id: { $in: commitmentIds },
+        lastUpdatedTimestamp: { $lte: asOfTimestamp }
+      },
+      {
+        $set: { storedAtTimestamp: /* @__PURE__ */ new Date() },
+        $unset: { pendingStage: 1 }
+      },
       { upsert: false }
     );
   }
@@ -706,7 +978,8 @@ class ProviderDatabase extends MongoDatabase {
   async markDappUserCommitmentsChecked(commitmentIds) {
     const updateDoc = {
       [StoredStatusNames.serverChecked]: true,
-      lastUpdatedTimestamp: /* @__PURE__ */ new Date()
+      lastUpdatedTimestamp: /* @__PURE__ */ new Date(),
+      pendingStage: true
     };
     await this.tables?.commitment.updateMany(
       { id: { $in: commitmentIds } },
@@ -718,7 +991,11 @@ class ProviderDatabase extends MongoDatabase {
    */
   async updateDappUserCommitment(commitmentId, updates) {
     const filter = { id: commitmentId };
-    await this.tables?.commitment.updateOne(filter, updates);
+    await this.tables?.commitment.updateOne(filter, {
+      ...updates,
+      lastUpdatedAtTimestamp: /* @__PURE__ */ new Date(),
+      pendingStage: true
+    });
   }
   /**
    * @description Get Dapp User PoW captcha commitments that have not been counted towards the client's total
@@ -727,54 +1004,25 @@ class ProviderDatabase extends MongoDatabase {
    * @returns {Promise<PoWCaptchaRecord[]>} Array of PoW captcha records
    */
   async getUnstoredDappUserPoWCommitments(limit = 1e3, skip = 0) {
-    const filterNoStoredTimestamp = { storedAtTimestamp: { $exists: false } };
-    const docs = await this.tables?.powcaptcha.aggregate([
-      {
-        $match: {
-          $or: [
-            filterNoStoredTimestamp,
-            {
-              $expr: {
-                $lt: [
-                  {
-                    $convert: {
-                      input: "$storedAtTimestamp",
-                      to: "date"
-                    }
-                  },
-                  {
-                    $convert: {
-                      input: "$lastUpdatedTimestamp",
-                      to: "date"
-                    }
-                  }
-                ]
-              }
-            }
-          ]
-        }
-      },
-      {
-        $sort: { _id: 1 }
-      },
-      {
-        $skip: skip
-      },
-      {
-        $limit: limit
-      }
-    ]);
+    const docs = await this.tables?.powcaptcha.find({ pendingStage: true }).sort({ _id: 1 }).skip(skip).limit(limit).lean();
     return docs || [];
   }
-  /** @description Mark a list of PoW captcha commits as stored
+  /** @description Mark a list of PoW captcha commits as stored.
+   *
+   * `asOfTimestamp` defaults to "now" but the sweep should pass the time
+   * at which it fetched the batch. See markDappUserCommitmentsStored for
+   * the guard rationale.
    */
-  async markDappUserPoWCommitmentsStored(challenges) {
-    const updateDoc = {
-      storedAtTimestamp: /* @__PURE__ */ new Date()
-    };
+  async markDappUserPoWCommitmentsStored(challenges, asOfTimestamp = /* @__PURE__ */ new Date()) {
     await this.tables?.powcaptcha.updateMany(
-      { challenge: { $in: challenges } },
-      { $set: updateDoc },
+      {
+        challenge: { $in: challenges },
+        lastUpdatedTimestamp: { $lte: asOfTimestamp }
+      },
+      {
+        $set: { storedAtTimestamp: /* @__PURE__ */ new Date() },
+        $unset: { pendingStage: 1 }
+      },
       { upsert: false }
     );
   }
@@ -783,7 +1031,8 @@ class ProviderDatabase extends MongoDatabase {
   async markDappUserPoWCommitmentsChecked(challenges) {
     const updateDoc = {
       [StoredStatusNames.serverChecked]: true,
-      lastUpdatedTimestamp: /* @__PURE__ */ new Date()
+      lastUpdatedTimestamp: /* @__PURE__ */ new Date(),
+      pendingStage: true
     };
     await this.tables?.powcaptcha.updateMany(
       { challenge: { $in: challenges } },
@@ -801,7 +1050,26 @@ class ProviderDatabase extends MongoDatabase {
       this.logger.debug(() => ({
         data: { action: "storing", sessionRecord }
       }));
-      await this.tables.session.create(sessionRecord);
+      const recordWithFlag = {
+        ...sessionRecord,
+        lastUpdatedTimestamp: sessionRecord.lastUpdatedTimestamp ?? sessionRecord.createdAt,
+        pendingStage: true
+      };
+      await this.tables.session.create(recordWithFlag);
+      this.centralStreamer?.streamSessionRecord(
+        recordWithFlag,
+        (ts) => this.tables.session.updateOne(
+          {
+            sessionId: sessionRecord.sessionId,
+            lastUpdatedTimestamp: { $lte: ts }
+          },
+          {
+            $set: { storedAtTimestamp: ts },
+            $unset: { pendingStage: 1 }
+          }
+        ).then(() => {
+        })
+      );
     } catch (err) {
       throw new ProsopoDBError("DATABASE.SESSION_STORE_FAILED", {
         context: { error: err, sessionId: sessionRecord.sessionId },
@@ -814,7 +1082,14 @@ class ProviderDatabase extends MongoDatabase {
    */
   async getSessionRecordBySessionId(sessionId) {
     const filter = { sessionId };
-    const doc = await this.tables.session.findOne(filter).lean();
+    const doc = await this.tables.session.findOne(filter, {
+      sessionId: 1,
+      ipInfo: 1,
+      scoreComponents: 1,
+      webView: 1,
+      reason: 1,
+      decryptedHeadHash: 1
+    }).lean();
     return doc || void 0;
   }
   /**
@@ -841,7 +1116,8 @@ class ProviderDatabase extends MongoDatabase {
     try {
       const session = await this.tables.session.findOneAndUpdate(filter, {
         deleted: true,
-        lastUpdatedTimestamp: /* @__PURE__ */ new Date()
+        lastUpdatedTimestamp: /* @__PURE__ */ new Date(),
+        pendingStage: true
       }).lean();
       return session || void 0;
     } catch (err) {
@@ -851,6 +1127,74 @@ class ProviderDatabase extends MongoDatabase {
       });
     }
   }
+  /**
+   * Update a session record by sessionId. Pure Mongo write — callers in
+   * the provider package are responsible for refreshing the Redis cache
+   * via `RedisWriteQueue.patchCachedSession`, matching the existing
+   * caller-side caching pattern (see `frictionlessTasks.createSession`).
+   */
+  async updateSessionRecord(sessionId, updates, streamToCentral) {
+    try {
+      await this.tables.session.updateOne(
+        { sessionId },
+        {
+          $set: {
+            ...updates,
+            lastUpdatedTimestamp: /* @__PURE__ */ new Date(),
+            pendingStage: true
+          }
+        }
+      );
+      if (streamToCentral && this.centralStreamer) {
+        const streamer = this.centralStreamer;
+        const markStored = (ts) => this.tables.session.updateOne(
+          { sessionId, lastUpdatedTimestamp: { $lte: ts } },
+          {
+            $set: { storedAtTimestamp: ts },
+            $unset: { pendingStage: 1 }
+          }
+        ).then(() => {
+        });
+        this.tables.session.findOne({ sessionId }).lean().then((record) => {
+          if (record) {
+            streamer.streamSessionRecord(record, markStored);
+          }
+        }).catch(() => {
+        });
+      }
+    } catch (err) {
+      throw new ProsopoDBError("DATABASE.SESSION_GET_FAILED", {
+        context: { error: err, sessionId },
+        logger: this.logger
+      });
+    }
+  }
+  /**
+   * Atomically record SIMD CPU fingerprint readings on the session — only
+   * if they aren't already present. Uses a Mongo aggregation-pipeline
+   * update so `simdReadings` and `simdReadingsStage` are set together,
+   * first-hop-wins. Pure Mongo write — cache refresh is the caller's
+   * responsibility via `RedisWriteQueue.patchCachedSimdReadingsIfAbsent`.
+   */
+  async recordSessionSimdReadingsIfAbsent(sessionId, readings, stage) {
+    try {
+      await this.tables.session.updateOne({ sessionId }, [
+        {
+          $set: {
+            simdReadings: { $ifNull: ["$simdReadings", readings] },
+            simdReadingsStage: { $ifNull: ["$simdReadingsStage", stage] },
+            lastUpdatedTimestamp: /* @__PURE__ */ new Date(),
+            pendingStage: true
+          }
+        }
+      ]);
+    } catch (err) {
+      throw new ProsopoDBError("DATABASE.SESSION_GET_FAILED", {
+        context: { error: err, sessionId, stage },
+        logger: this.logger
+      });
+    }
+  }
   /**
    * Get an active session by user IP hash
    * @param userSitekeyIpHash The hash of user, IP and sitekey combination
@@ -875,64 +1219,43 @@ class ProviderDatabase extends MongoDatabase {
     }
   }
   /** Get unstored session records
-   * @description Get session records that have not been stored yet
+   * @description Get session records that have not been stored yet.
+   *
+   * Served by the `pendingStage_partial` index — see
+   * `getUnstoredDappUserCommitments` for the lifecycle of the flag.
+   * `checkAndRemoveSession` also flips the flag so consumed sessions
+   * propagate to the central DB via the next sweep.
    * @param limit
    * @param skip
    */
   getUnstoredSessionRecords(limit = 1e3, skip = 0) {
-    const filterNoStoredTimestamp = { storedAtTimestamp: { $exists: false } };
-    return this.tables?.session.aggregate([
-      {
-        $match: {
-          $or: [
-            filterNoStoredTimestamp,
-            {
-              $expr: {
-                $lt: [
-                  {
-                    $convert: {
-                      input: "$storedAtTimestamp",
-                      to: "date"
-                    }
-                  },
-                  {
-                    $convert: {
-                      input: "$lastUpdatedTimestamp",
-                      to: "date"
-                    }
-                  }
-                ]
-              }
-            }
-          ]
-        }
-      },
+    return Promise.resolve(this.tables?.session).then(
+      (tbl) => tbl?.find({ pendingStage: true }).sort({ _id: 1 }).skip(skip).limit(limit).lean()
+    ).then((docs) => docs || []);
+  }
+  /** Mark a list of session records as stored.
+   *
+   * `asOfTimestamp` defaults to "now" but the sweep should pass the time
+   * at which it fetched the batch. See markDappUserCommitmentsStored for
+   * the guard rationale.
+   */
+  async markSessionRecordsStored(sessionIds, asOfTimestamp = /* @__PURE__ */ new Date()) {
+    await this.tables?.session.updateMany(
       {
-        $sort: { _id: 1 }
+        sessionId: { $in: sessionIds },
+        lastUpdatedTimestamp: { $lte: asOfTimestamp }
       },
       {
-        $skip: skip
+        $set: { storedAtTimestamp: /* @__PURE__ */ new Date() },
+        $unset: { pendingStage: 1 }
       },
-      {
-        $limit: limit
-      }
-    ]).then((docs) => docs || []);
-  }
-  /** Mark a list of session records as stored */
-  async markSessionRecordsStored(sessionIds) {
-    const updateDoc = {
-      storedAtTimestamp: /* @__PURE__ */ new Date()
-    };
-    await this.tables?.session.updateMany(
-      { sessionId: { $in: sessionIds } },
-      { $set: updateDoc },
       { upsert: false }
     );
   }
   /**
    * @description Store a Dapp User's pending record
    */
-  async storePendingImageCommitment(userAccount, requestHash, salt, deadlineTimestamp, requestedAtTimestamp, ipAddress, threshold, sessionId) {
+  async storePendingImageCommitment(userAccount, requestHash, salt, deadlineTimestamp, requestedAtTimestamp, ipAddress, threshold, sessionId, ipInfo) {
     if (!isHex(requestHash)) {
       throw new ProsopoDBError("DATABASE.INVALID_HASH", {
         context: {
@@ -942,24 +1265,47 @@ class ProviderDatabase extends MongoDatabase {
       });
     }
     const pendingRecord = {
-      accountId: userAccount,
+      userAccount,
       pending: true,
       salt,
       requestHash,
       deadlineTimestamp,
-      requestedAtTimestamp: new Date(requestedAtTimestamp),
+      requestedAtTimestamp,
       ipAddress,
       sessionId,
-      threshold
+      threshold,
+      ipInfo,
+      // Deliberately NOT setting pendingStage here. Placeholder
+      // records have id: "" until the user submits a solution; if we
+      // flag them, the sweep picks them up via the partial index but
+      // then `markDappUserCommitmentsStored` runs
+      // `{ id: { $in: ["", ...] } }` which collapses to a single ""
+      // bound and scans every empty-id row via the `id_-1` index —
+      // turning a cheap sweep into a fresh cache evictor. The real
+      // commitment only gets `pendingStage: true` once `id` is
+      // populated by approve/disapprove, at which point staging it is
+      // meaningful.
+      // Placeholder fields required by schema but not needed for pending state
+      dappAccount: "",
+      providerAccount: "",
+      datasetId: "",
+      id: "",
+      // id is populated by the user's solution record when the user submits a solution, so we can leave it blank here
+      result: { status: CaptchaStatus.pending },
+      headers: {},
+      ja4: "",
+      userSignature: "",
+      userSubmitted: false,
+      serverChecked: false
     };
-    await this.tables?.pending.updateOne(
+    await this.tables?.commitment.updateOne(
       { requestHash },
       { $set: pendingRecord },
       { upsert: true }
     );
   }
   /**
-   * @description Get a Dapp user's pending record
+   * @description Get a user's pending record
    */
   async getPendingImageCommitment(requestHash) {
     if (!isHex(requestHash)) {
@@ -970,12 +1316,22 @@ class ProviderDatabase extends MongoDatabase {
         }
       });
     }
-    const filter = {
-      [ApiParams.requestHash]: requestHash
-    };
-    const doc = await this.tables?.pending.findOne(filter).lean();
+    const doc = await this.tables?.commitment.findOne({
+      requestHash,
+      pending: true
+    }).lean();
     if (doc) {
-      return doc;
+      return {
+        dappAccount: doc.dappAccount,
+        pending: doc.pending,
+        salt: doc.salt,
+        requestHash: doc.requestHash,
+        deadlineTimestamp: doc.deadlineTimestamp,
+        requestedAtTimestamp: doc.requestedAtTimestamp,
+        ipAddress: doc.ipAddress,
+        sessionId: doc.sessionId,
+        threshold: doc.threshold
+      };
     }
     throw new ProsopoDBError("DATABASE.PENDING_RECORD_NOT_FOUND", {
       context: {
@@ -996,17 +1352,13 @@ class ProviderDatabase extends MongoDatabase {
         }
       });
     }
-    const filter = {
-      [ApiParams.requestHash]: requestHash
-    };
-    await this.tables?.pending.updateOne(
-      filter,
+    await this.tables?.commitment.updateOne(
+      { requestHash },
       {
         $set: {
-          [CaptchaStatus.pending]: false
+          pending: false
         }
-      },
-      { upsert: true }
+      }
     );
   }
   /**
@@ -1107,7 +1459,7 @@ class ProviderDatabase extends MongoDatabase {
     const filter = {
       commitmentId
     };
-    const project = { projection: { _id: 0 } };
+    const project = { _id: 0 };
     const cursor = this.tables?.usersolution?.findOne(filter, project).lean();
     const doc = await cursor;
     if (doc) {
@@ -1123,7 +1475,18 @@ class ProviderDatabase extends MongoDatabase {
    */
   async getDappUserCommitmentById(commitmentId) {
     const filter = { id: commitmentId };
-    const commitmentCursor = this.tables?.commitment?.findOne(filter).lean();
+    const commitmentCursor = this.tables?.commitment?.findOne(filter, {
+      id: 1,
+      result: 1,
+      serverChecked: 1,
+      requestedAtTimestamp: 1,
+      ipAddress: 1,
+      sessionId: 1,
+      userAccount: 1,
+      dappAccount: 1,
+      headers: 1,
+      ipInfo: 1
+    }).lean();
     const doc = await commitmentCursor;
     return doc ? doc : void 0;
   }
@@ -1137,7 +1500,10 @@ class ProviderDatabase extends MongoDatabase {
       userAccount,
       dappAccount
     };
-    const project = { _id: 0 };
+    const project = {
+      _id: 0,
+      result: 1
+    };
     const sort = { sort: { _id: -1 } };
     const docs = await this.tables?.commitment?.find(filter, project, sort).lean();
     return docs ? docs : [];
@@ -1153,10 +1519,22 @@ class ProviderDatabase extends MongoDatabase {
       const updateDoc = {
         result,
         lastUpdatedTimestamp: /* @__PURE__ */ new Date(),
+        pendingStage: true,
         ...coords ? { coords } : {}
       };
       const filter = { id: commitmentId };
       await this.tables?.commitment?.findOneAndUpdate(filter, { $set: updateDoc }, { upsert: false }).lean();
+      this.centralStreamer?.streamImageUpdate(
+        () => this.tables.commitment.findOne({ id: commitmentId }).lean(),
+        (ts) => this.tables.commitment.updateOne(
+          { id: commitmentId, lastUpdatedTimestamp: { $lte: ts } },
+          {
+            $set: { storedAtTimestamp: ts },
+            $unset: { pendingStage: 1 }
+          }
+        ).then(() => {
+        })
+      );
     } catch (err) {
       throw new ProsopoDBError("DATABASE.SOLUTION_APPROVE_FAILED", {
         context: { error: err, commitmentId }
@@ -1174,10 +1552,22 @@ class ProviderDatabase extends MongoDatabase {
       const updateDoc = {
         result: { status: CaptchaStatus.disapproved, reason },
         lastUpdatedTimestamp: /* @__PURE__ */ new Date(),
+        pendingStage: true,
         ...coords ? { coords } : {}
       };
       const filter = { id: commitmentId };
       await this.tables?.commitment?.findOneAndUpdate(filter, { $set: updateDoc }, { upsert: false }).lean();
+      this.centralStreamer?.streamImageUpdate(
+        () => this.tables.commitment.findOne({ id: commitmentId }).lean(),
+        (ts) => this.tables.commitment.updateOne(
+          { id: commitmentId, lastUpdatedTimestamp: { $lte: ts } },
+          {
+            $set: { storedAtTimestamp: ts },
+            $unset: { pendingStage: 1 }
+          }
+        ).then(() => {
+        })
+      );
     } catch (err) {
       throw new ProsopoDBError("DATABASE.SOLUTION_APPROVE_FAILED", {
         context: { error: err, commitmentId }
@@ -1313,6 +1703,17 @@ class ProviderDatabase extends MongoDatabase {
     }
     await this.tables?.client.bulkWrite(ops);
   }
+  /**
+   * @description Remove client records by account
+   */
+  async removeClientRecords(accounts) {
+    if (!accounts || accounts.length === 0) {
+      return;
+    }
+    await this.tables?.client.deleteMany({
+      account: { $in: accounts }
+    });
+  }
   /**
    * @description Get all client records
    */
@@ -1325,7 +1726,11 @@ class ProviderDatabase extends MongoDatabase {
    */
   async getClientRecord(account) {
     const filter = { account };
-    const doc = await this.tables?.client.findOne(filter).lean();
+    const doc = await this.tables?.client.findOne(filter, {
+      account: 1,
+      settings: 1,
+      tier: 1
+    }).lean();
     return doc ? doc : void 0;
   }
   /**
@@ -1363,6 +1768,97 @@ class ProviderDatabase extends MongoDatabase {
     ).sort({ createdAt: -1 }).lean();
     return (keyRecords || []).map((record) => record.detectorKey);
   }
+  /**
+   * Stores a decision machine artifact with a unique scope identifier.
+   *
+   * The combination of scope + dappAccount uniquely identifies a single artifact:
+   * - Global scope: One artifact per provider (dappAccount is null)
+   * - Dapp scope: One artifact per dapp account (dappAccount is specified)
+   *
+   * @param artifact - The decision machine artifact to store
+   */
+  async upsertDecisionMachineArtifact(artifact) {
+    const now = /* @__PURE__ */ new Date();
+    const dappAccount = artifact.dappAccount ?? null;
+    const filter = {
+      scope: artifact.scope,
+      dappAccount
+    };
+    await this.tables?.decisionMachine.updateOne(
+      filter,
+      {
+        $set: {
+          scope: artifact.scope,
+          dappAccount,
+          runtime: artifact.runtime,
+          language: artifact.language,
+          source: artifact.source,
+          name: artifact.name,
+          version: artifact.version,
+          updatedAt: now
+        },
+        $setOnInsert: {
+          createdAt: now
+        }
+      },
+      { upsert: true }
+    );
+  }
+  /**
+   * Retrieves a single decision machine artifact by scope and optional dapp account.
+   *
+   * @param scope - The scope level (Global or Dapp)
+   * @param dappAccount - Required for Dapp scope, unused for Global scope
+   * @returns The matching artifact, or undefined if not found
+   */
+  async getDecisionMachineArtifact(scope, dappAccount) {
+    const filter = {
+      scope,
+      dappAccount: dappAccount ?? null
+    };
+    const doc = await this.tables?.decisionMachine.findOne(filter).lean();
+    return doc ?? void 0;
+  }
+  /**
+   * Retrieves all decision machine artifacts.
+   *
+   * @returns Array of all decision machine artifacts
+   */
+  async getAllDecisionMachineArtifacts() {
+    const docs = await this.tables?.decisionMachine.find({}).lean();
+    return docs ?? [];
+  }
+  /**
+   * Retrieves a single decision machine artifact by MongoDB ObjectId.
+   *
+   * @param id - The MongoDB ObjectId as a string
+   * @returns The matching artifact, or undefined if not found
+   */
+  async getDecisionMachineArtifactById(id) {
+    const doc = await this.tables?.decisionMachine.findById(id).lean();
+    return doc ?? void 0;
+  }
+  /**
+   * Removes a decision machine artifact by MongoDB ObjectId.
+   *
+   * @param id - The MongoDB ObjectId as a string
+   * @returns true if deleted, false if not found
+   */
+  async removeDecisionMachineArtifact(id) {
+    const result = await this.tables?.decisionMachine.deleteOne({
+      _id: id
+    });
+    return (result?.deletedCount ?? 0) > 0;
+  }
+  /**
+   * Removes all decision machine artifacts.
+   *
+   * @returns The number of artifacts deleted
+   */
+  async removeAllDecisionMachineArtifacts() {
+    const result = await this.tables?.decisionMachine.deleteMany({});
+    return result?.deletedCount ?? 0;
+  }
   /**
    * @description set client context-specific entropy
    */
@@ -1405,7 +1901,7 @@ class ProviderDatabase extends MongoDatabase {
       },
       {
         $lookup: {
-          from: "session",
+          from: "sessions",
           localField: "sessionId",
           foreignField: "sessionId",
           as: "sessionData"
@@ -1458,6 +1954,36 @@ class ProviderDatabase extends MongoDatabase {
       })
     )).filter((headHash) => headHash !== void 0);
   }
+  async getSpamEmailDomain(domain) {
+    if (!this.tables?.spamEmailDomain) {
+      throw new ProsopoDBError("DATABASE.DATABASE_IMPORT_ERROR", {
+        context: { failedFuncName: this.getSpamEmailDomain.name }
+      });
+    }
+    const suffixCandidates = buildDomainSuffixCandidates(domain).slice(
+      0,
+      MAX_DOMAIN_SUFFIX_CANDIDATES
+    );
+    const query = suffixCandidates.length > 0 ? { domain: { $in: suffixCandidates } } : { domain };
+    return await this.tables.spamEmailDomain.findOne(query).exec();
+  }
+  async bulkUpdateSpamEmailDomains(domains, upsert) {
+    if (!this.tables?.spamEmailDomain) {
+      throw new ProsopoDBError("DATABASE.DATABASE_IMPORT_ERROR", {
+        context: { failedFuncName: this.bulkUpdateSpamEmailDomains.name }
+      });
+    }
+    const bulkOps = domains.map((op) => ({
+      updateOne: {
+        filter: op.filter,
+        update: { $set: op.update },
+        upsert
+      }
+    }));
+    if (bulkOps.length > 0) {
+      await this.tables.spamEmailDomain.bulkWrite(bulkOps);
+    }
+  }
 }
 export {
   ProviderDatabase