@prosopo/database 4.2.10 → 4.4.0

package/dist/tsc/databases/provider.js

@@ -0,0 +1,2054 @@
+import { isHex } from "@polkadot/util/is";
+import { ProsopoError } from "@prosopo/common";
+import { connectToRedis, setupRedisIndex, } from "@prosopo/redis-client";
+import { CaptchaStates, CaptchaStatus, CaptchaType, DatasetWithIdsAndTreeSchema, DecisionMachineKind, StoredStatusNames, UserCommitmentSchema, isBlockingCaptchaResult, } from "@prosopo/types";
+import { AudioCaptchaRecordSchema, CaptchaRecordSchema, ClientRecordSchema, DatasetRecordSchema, DecisionMachineArtifactRecordSchema, IconOrderCaptchaRecordSchema, PoWCaptchaRecordSchema, PuzzleCaptchaRecordSchema, SESSION_PROJECTION, ScheduledTaskRecordSchema, ScheduledTaskSchema, SessionRecordSchema, SolutionRecordSchema, SpamEmailDomainRecordSchema, UserCommitmentRecordSchema, UserSolutionRecordSchema, } from "@prosopo/types-database";
+import { accessRulesRedisIndex, createRedisAccessRulesStorage, } from "@prosopo/user-access-policy/redis";
+import { assertCoordsSafe, buildDomainSuffixCandidates } from "@prosopo/util";
+import { MongoDatabase } from "../base/mongo.js";
+import { clientRecordCache } from "./clientRecordCache.js";
+const TWENTY_FOUR_HOURS_IN_MS = 24 * 60 * 60 * 1000;
+const MAX_DOMAIN_SUFFIX_CANDIDATES = 5;
+var TableNames;
+(function (TableNames) {
+    TableNames["captcha"] = "captcha";
+    TableNames["dataset"] = "dataset";
+    TableNames["solution"] = "solution";
+    TableNames["commitment"] = "commitment";
+    TableNames["usersolution"] = "usersolution";
+    TableNames["scheduler"] = "scheduler";
+    TableNames["powcaptcha"] = "powcaptcha";
+    TableNames["puzzlecaptcha"] = "puzzlecaptcha";
+    TableNames["audiocaptcha"] = "audiocaptcha";
+    TableNames["iconordercaptcha"] = "iconordercaptcha";
+    TableNames["client"] = "client";
+    TableNames["session"] = "session";
+    TableNames["detector"] = "detector";
+    TableNames["decisionMachine"] = "decisionMachine";
+    TableNames["spamEmailDomain"] = "spamEmailDomain";
+})(TableNames || (TableNames = {}));
+const PROVIDER_TABLES = [
+    {
+        collectionName: TableNames.captcha,
+        modelName: "Captcha",
+        schema: CaptchaRecordSchema,
+    },
+    {
+        collectionName: TableNames.powcaptcha,
+        modelName: "PowCaptcha",
+        schema: PoWCaptchaRecordSchema,
+    },
+    {
+        collectionName: TableNames.puzzlecaptcha,
+        modelName: "PuzzleCaptcha",
+        schema: PuzzleCaptchaRecordSchema,
+    },
+    {
+        collectionName: TableNames.audiocaptcha,
+        modelName: "AudioCaptcha",
+        schema: AudioCaptchaRecordSchema,
+    },
+    {
+        collectionName: TableNames.iconordercaptcha,
+        modelName: "IconOrderCaptcha",
+        schema: IconOrderCaptchaRecordSchema,
+    },
+    {
+        collectionName: TableNames.dataset,
+        modelName: "Dataset",
+        schema: DatasetRecordSchema,
+    },
+    {
+        collectionName: TableNames.solution,
+        modelName: "Solution",
+        schema: SolutionRecordSchema,
+    },
+    {
+        collectionName: TableNames.commitment,
+        modelName: "UserCommitment",
+        schema: UserCommitmentRecordSchema,
+    },
+    {
+        collectionName: TableNames.usersolution,
+        modelName: "UserSolution",
+        schema: UserSolutionRecordSchema,
+    },
+    {
+        collectionName: TableNames.scheduler,
+        modelName: "Scheduler",
+        schema: ScheduledTaskRecordSchema,
+    },
+    {
+        collectionName: TableNames.client,
+        modelName: "Client",
+        schema: ClientRecordSchema,
+    },
+    {
+        collectionName: TableNames.session,
+        modelName: "Session",
+        schema: SessionRecordSchema,
+    },
+    {
+        collectionName: TableNames.decisionMachine,
+        modelName: "DecisionMachine",
+        schema: DecisionMachineArtifactRecordSchema,
+    },
+    {
+        collectionName: TableNames.spamEmailDomain,
+        modelName: "SpamEmailDomain",
+        schema: SpamEmailDomainRecordSchema,
+    },
+];
+export class ProviderDatabase extends MongoDatabase {
+    constructor(options) {
+        super(options.mongo.url, options.mongo.dbname, options.mongo.authSource, options.logger);
+        this.options = options;
+        this.tables = {};
+        this.indexesEnsured = false;
+        this.tables = {};
+        this.redisAccessRulesConnection = null;
+        this.redisConnection = null;
+        this.userAccessRulesStorage = null;
+    }
+    setCentralDbStreamer(streamer) {
+        this.centralStreamer = streamer;
+    }
+    hasCentralDbStreamer() {
+        return this.centralStreamer !== undefined;
+    }
+    async connect() {
+        await super.connect();
+        this.loadTables();
+        await this.setupRedis();
+    }
+    async setupRedis() {
+        this.redisConnection = connectToRedis({
+            url: this.options.redis?.url,
+            password: this.options.redis?.password,
+            logger: this.logger,
+        });
+        this.redisAccessRulesConnection = setupRedisIndex(this.redisConnection, {
+            ...accessRulesRedisIndex,
+            name: this.options.redis?.indexName || accessRulesRedisIndex.name,
+        }, this.logger);
+        this.userAccessRulesStorage = createRedisAccessRulesStorage(this.redisAccessRulesConnection, this.logger);
+    }
+    loadTables() {
+        const tables = {};
+        PROVIDER_TABLES.map(({ collectionName, modelName, schema }) => {
+            if (this.connection) {
+                tables[collectionName] = this.connection.model(modelName, schema);
+            }
+        });
+        this.tables = tables;
+    }
+    getTables() {
+        if (!this.tables) {
+            throw new ProsopoError({
+                code: "DB_UNAVAILABLE",
+                errorSite: "DBSE-5",
+                message: "Tables undefined",
+                context: { failedFuncName: this.getTables.name },
+            });
+        }
+        return this.tables;
+    }
+    getRedisAccessRulesConnection() {
+        if (null === this.redisAccessRulesConnection) {
+            throw new ProsopoError({
+                code: "DB_UNAVAILABLE",
+                errorSite: "DBSE-6",
+                message: "Redis access rules connection undefined",
+            });
+        }
+        return this.redisAccessRulesConnection;
+    }
+    getRedisConnection() {
+        if (null === this.redisConnection) {
+            throw new ProsopoError({
+                code: "DB_UNAVAILABLE",
+                errorSite: "DBSE-7",
+                message: "Redis access rules connection undefined",
+            });
+        }
+        return this.redisConnection;
+    }
+    async ensureIndexes() {
+        const indexPromises = [];
+        if (!this.indexesEnsured) {
+            PROVIDER_TABLES.map(({ collectionName }) => {
+                indexPromises.push(new Promise((resolve) => {
+                    if (this.connected) {
+                        this.tables[collectionName].collection.dropIndexes().then(() => {
+                            this.tables[collectionName]
+                                .ensureIndexes()
+                                .then(() => {
+                                this.logger.info(() => ({
+                                    msg: `Indexes ensured for collection ${collectionName}`,
+                                }));
+                                resolve();
+                            })
+                                .catch((err) => {
+                                this.logger.warn(() => ({
+                                    err,
+                                    msg: `Error creating indexes for collection ${collectionName}`,
+                                }));
+                                resolve();
+                            });
+                        });
+                    }
+                    else {
+                        this.logger.info(() => ({
+                            msg: `Skipping index creation for collection ${collectionName} as not connected`,
+                        }));
+                        resolve();
+                    }
+                }));
+            });
+        }
+        await Promise.all(indexPromises);
+        this.indexesEnsured = true;
+    }
+    getUserAccessRulesStorage() {
+        if (null === this.userAccessRulesStorage) {
+            throw new ProsopoError({
+                code: "DB_UNAVAILABLE",
+                errorSite: "DBSE-8",
+                message: "User access rules storage undefined",
+            });
+        }
+        return this.userAccessRulesStorage;
+    }
+    async storeDataset(dataset) {
+        try {
+            this.logger.debug(() => ({
+                data: { datasetId: dataset.datasetId },
+                msg: "Storing dataset in database",
+            }));
+            const parsedDataset = DatasetWithIdsAndTreeSchema.parse(dataset);
+            const datasetDoc = {
+                datasetId: parsedDataset.datasetId,
+                datasetContentId: parsedDataset.datasetContentId,
+                format: parsedDataset.format,
+                contentTree: parsedDataset.contentTree,
+                solutionTree: parsedDataset.solutionTree,
+            };
+            const filter = {
+                datasetId: parsedDataset.datasetId,
+            };
+            await this.tables.dataset?.updateOne(filter, { $set: datasetDoc }, { upsert: true });
+            const captchaDocs = parsedDataset.captchas.map(({ solution, ...captcha }, index) => ({
+                ...captcha,
+                datasetId: parsedDataset.datasetId,
+                datasetContentId: parsedDataset.datasetContentId,
+                index,
+                solved: !!solution?.length,
+            }));
+            this.logger.debug(() => ({
+                msg: "Inserting captcha records",
+            }));
+            if (captchaDocs.length) {
+                await this.tables?.captcha.bulkWrite(captchaDocs.map((captchaDoc) => ({
+                    updateOne: {
+                        filter: { captchaId: captchaDoc.captchaId },
+                        update: {
+                            $set: captchaDoc,
+                            $setOnInsert: { randomKey: Math.random() },
+                        },
+                        upsert: true,
+                    },
+                })));
+            }
+            const captchaSolutionDocs = parsedDataset.captchas
+                .filter(({ solution }) => solution?.length)
+                .map((captcha) => ({
+                captchaId: captcha.captchaId,
+                captchaContentId: captcha.captchaContentId,
+                solution: captcha.solution,
+                salt: captcha.salt,
+                datasetId: parsedDataset.datasetId,
+                datasetContentId: parsedDataset.datasetContentId,
+            }));
+            this.logger.debug(() => ({
+                msg: "Inserting solution records",
+            }));
+            if (captchaSolutionDocs.length) {
+                await this.tables?.solution.bulkWrite(captchaSolutionDocs.map((captchaSolutionDoc) => ({
+                    updateOne: {
+                        filter: { captchaId: captchaSolutionDoc.captchaId },
+                        update: { $set: captchaSolutionDoc },
+                        upsert: true,
+                    },
+                })));
+            }
+            this.logger.debug(() => ({
+                msg: "Dataset stored in database",
+            }));
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-9",
+                message: "Data set load failed",
+                cause: err,
+                context: { failedFuncName: this.storeDataset.name },
+            });
+        }
+    }
+    async getSolutions(datasetId) {
+        const filter = { datasetId };
+        const docs = await this.tables?.solution
+            .find(filter)
+            .lean();
+        return docs ? docs : [];
+    }
+    async getSolutionByCaptchaId(captchaId) {
+        const filter = { captchaId };
+        const doc = await this.tables?.solution
+            .findOne(filter)
+            .lean();
+        return doc || null;
+    }
+    async getDataset(datasetId) {
+        const filter = { datasetId };
+        const datasetDoc = await this.tables?.dataset.findOne(filter).lean();
+        if (datasetDoc) {
+            const { datasetContentId, format, contentTree, solutionTree } = datasetDoc;
+            const captchas = (await this.tables?.captcha.find(filter).lean()) || [];
+            const solutions = (await this.tables?.solution.find(filter).lean()) ||
+                [];
+            const solutionsKeyed = {};
+            for (const solution of solutions) {
+                solutionsKeyed[solution.captchaId] = solution;
+            }
+            return {
+                datasetId,
+                datasetContentId,
+                format,
+                contentTree: contentTree || [],
+                solutionTree: solutionTree || [],
+                captchas: captchas.map((captchaDoc) => {
+                    const { captchaId, captchaContentId, items, target, salt, solved } = captchaDoc;
+                    const solution = solutionsKeyed[captchaId];
+                    return {
+                        captchaId,
+                        captchaContentId,
+                        solved: !!solved,
+                        salt,
+                        items,
+                        target,
+                        solution: solved && solution ? solution.solution : [],
+                    };
+                }),
+            };
+        }
+        throw new ProsopoError({
+            code: "DB_QUERY_FAILED",
+            errorSite: "DBSE-10",
+            message: "Failed to get dataset",
+            context: { failedFuncName: this.getDataset.name, datasetId },
+        });
+    }
+    async getRandomCaptcha(solved, datasetId, size) {
+        if (!isHex(datasetId)) {
+            throw new ProsopoError({
+                code: "INVALID_DATA",
+                errorSite: "DBSE-11",
+                message: "Invalid hash",
+                context: { failedFuncName: this.getRandomCaptcha.name, datasetId },
+            });
+        }
+        const sampleSize = size ? Math.abs(Math.trunc(size)) : 1;
+        const projection = {
+            _id: 0,
+            datasetId: 1,
+            datasetContentId: 1,
+            captchaId: 1,
+            captchaContentId: 1,
+            items: 1,
+            target: 1,
+        };
+        const baseFilter = { datasetId: datasetId, solved };
+        const pivot = Math.random();
+        const head = (await this.tables?.captcha
+            .find({ ...baseFilter, randomKey: { $gte: pivot } }, projection)
+            .sort({ randomKey: 1 })
+            .limit(sampleSize)
+            .lean()) ?? [];
+        let docs = head;
+        if (docs.length < sampleSize) {
+            const tail = (await this.tables?.captcha
+                .find({ ...baseFilter, randomKey: { $lt: pivot } }, projection)
+                .sort({ randomKey: 1 })
+                .limit(sampleSize - docs.length)
+                .lean()) ?? [];
+            docs = docs.concat(tail);
+        }
+        if (docs.length === 0) {
+            const fallback = (await this.tables?.captcha
+                .aggregate([
+                { $match: baseFilter },
+                { $sample: { size: sampleSize } },
+                { $project: projection },
+            ])
+                .exec()) ?? [];
+            if (fallback.length > 0) {
+                return fallback.map(({ _id, ...keep }) => keep);
+            }
+        }
+        if (docs.length > 0) {
+            return docs;
+        }
+        throw new ProsopoError({
+            code: "DB_QUERY_FAILED",
+            errorSite: "DBSE-12",
+            message: "Failed to get captcha",
+            context: {
+                failedFuncName: this.getRandomCaptcha.name,
+                solved,
+                datasetId,
+                size,
+            },
+        });
+    }
+    async getCaptchaById(captchaId) {
+        const filter = { captchaId: { $in: captchaId } };
+        const cursor = this.tables?.captcha
+            .find(filter, {
+            _id: 0,
+            captchaId: 1,
+            datasetId: 1,
+            items: 1,
+        })
+            .lean();
+        const docs = await cursor;
+        if (docs?.length) {
+            return docs.map(({ _id, ...keepAttrs }) => keepAttrs);
+        }
+        throw new ProsopoError({
+            code: "DB_QUERY_FAILED",
+            errorSite: "DBSE-13",
+            message: "Failed to get captcha",
+            context: { failedFuncName: this.getCaptchaById.name, captchaId },
+        });
+    }
+    async updateCaptcha(captcha, datasetId) {
+        if (!isHex(datasetId)) {
+            throw new ProsopoError({
+                code: "INVALID_DATA",
+                errorSite: "DBSE-14",
+                message: "Invalid hash",
+                context: { failedFuncName: this.updateCaptcha.name, datasetId },
+            });
+        }
+        try {
+            const filter = { datasetId };
+            await this.tables?.captcha.updateOne(filter, { $set: captcha }, { upsert: false });
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-15",
+                message: "Failed to update captcha",
+                cause: err,
+                context: { failedFuncName: this.getDatasetDetails.name },
+            });
+        }
+    }
+    async removeCaptchas(captchaIds) {
+        const filter = { captchaId: { $in: captchaIds } };
+        await this.tables?.captcha.deleteMany(filter);
+    }
+    async getMostRecentDatasetId() {
+        const dataset = await this.tables?.dataset
+            .findOne()
+            .sort({ _id: -1 })
+            .lean();
+        const datasetId = dataset?.datasetId;
+        return typeof datasetId === "string" ? datasetId : undefined;
+    }
+    async getDatasetDetails(datasetId) {
+        if (!isHex(datasetId)) {
+            throw new ProsopoError({
+                code: "INVALID_DATA",
+                errorSite: "DBSE-16",
+                message: "Invalid hash",
+                context: { failedFuncName: this.getDatasetDetails.name, datasetId },
+            });
+        }
+        const filter = { datasetId };
+        const doc = await this.tables?.dataset
+            .findOne(filter)
+            .lean();
+        if (doc) {
+            return doc;
+        }
+        throw new ProsopoError({
+            code: "RECORD_NOT_FOUND",
+            errorSite: "DBSE-17",
+            message: "Failed to get dataset",
+            context: { failedFuncName: this.getDatasetDetails.name, datasetId },
+        });
+    }
+    async storeUserImageCaptchaSolution(captchas, commit) {
+        const commitmentRecord = UserCommitmentSchema.parse({
+            ...commit,
+            lastUpdatedTimestamp: new Date(),
+        });
+        if (captchas.length) {
+            const filter = {
+                id: commit.id,
+            };
+            await this.tables?.commitment.updateOne(filter, { $set: { ...commitmentRecord } }, { upsert: true });
+            const ops = captchas.map((captcha) => ({
+                updateOne: {
+                    filter: {
+                        commitmentId: commit.id,
+                        captchaId: captcha.captchaId,
+                    },
+                    update: {
+                        $set: {
+                            captchaId: captcha.captchaId,
+                            captchaContentId: captcha.captchaContentId,
+                            salt: captcha.salt,
+                            solution: captcha.solution,
+                            commitmentId: commit.id,
+                            processed: false,
+                        },
+                    },
+                    upsert: true,
+                },
+            }));
+            await this.tables?.usersolution.bulkWrite(ops);
+            this.centralStreamer?.streamImageRecord(commitmentRecord, (ts) => this.tables.commitment
+                .updateOne({ id: commit.id }, { $set: { storedAtTimestamp: ts } })
+                .then(() => { }));
+        }
+    }
+    async storePowCaptchaRecord(challenge, components, difficulty, providerSignature, ipAddress, headers, ja4, sessionId, serverChecked = false, userSubmitted = false, userSignature, ipInfo, fingerprintSummary) {
+        const tables = this.getTables();
+        const powCaptchaRecord = {
+            challenge,
+            userAccount: components.userAccount,
+            dappAccount: components.dappAccount,
+            requestedAtTimestamp: new Date(components.requestedAtTimestamp),
+            ipAddress,
+            headers,
+            ja4,
+            result: { status: CaptchaStatus.pending },
+            userSubmitted,
+            serverChecked,
+            difficulty,
+            providerSignature,
+            userSignature,
+            lastUpdatedTimestamp: new Date(),
+            pendingStage: true,
+            sessionId,
+            ipInfo,
+            ...(fingerprintSummary && { fingerprintSummary }),
+        };
+        try {
+            await tables.powcaptcha.create(powCaptchaRecord);
+            this.logger.info(() => ({
+                data: {
+                    challenge,
+                    userSubmitted,
+                    serverChecked,
+                    countryCode: ipInfo?.isValid ? ipInfo.countryCode : undefined,
+                },
+                msg: "PowCaptcha record added successfully",
+            }));
+            this.centralStreamer?.streamPowRecord(powCaptchaRecord, (ts) => this.tables.powcaptcha
+                .updateOne({ challenge, lastUpdatedTimestamp: { $lte: ts } }, {
+                $set: { storedAtTimestamp: ts },
+                $unset: { pendingStage: 1 },
+            })
+                .then(() => { }));
+        }
+        catch (error) {
+            const err = new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-18",
+                message: "Failed to update captcha",
+                cause: error,
+                context: { challenge, userSubmitted, serverChecked, ipInfo },
+            });
+            this.logger.error(() => ({
+                err: error,
+                msg: "Failed to add PowCaptcha record",
+            }));
+            throw err;
+        }
+    }
+    async getPowCaptchaRecordByChallenge(challenge) {
+        if (!this.tables) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-19",
+                message: "Database client is not connected",
+                context: { failedFuncName: this.getPowCaptchaRecordByChallenge.name },
+            });
+        }
+        try {
+            const filter = { challenge };
+            const record = await this.tables.powcaptcha
+                .findOne(filter, {
+                challenge: 1,
+                userAccount: 1,
+                dappAccount: 1,
+                requestedAtTimestamp: 1,
+                submittedAtTimestamp: 1,
+                verifiedAtTimestamp: 1,
+                failedAtTimestamp: 1,
+                ipAddress: 1,
+                headers: 1,
+                ja4: 1,
+                result: 1,
+                difficulty: 1,
+                sessionId: 1,
+                ipInfo: 1,
+                deviceCapability: 1,
+                behavioralDataPacked: 1,
+                serverChecked: 1,
+                userSubmitted: 1,
+                coords: 1,
+                clientMetaData: 1,
+            })
+                .lean();
+            if (record) {
+                this.logger.info(() => ({
+                    data: { challenge },
+                    msg: "PowCaptcha record retrieved successfully",
+                }));
+                return record;
+            }
+            this.logger.info(() => ({
+                data: { challenge },
+                msg: "No PowCaptcha record found",
+            }));
+            return null;
+        }
+        catch (error) {
+            const err = new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-20",
+                message: "Failed to get captcha",
+                cause: error,
+                context: { challenge },
+            });
+            this.logger.error(() => ({
+                err: err,
+                msg: "Failed to retrieve PowCaptcha record",
+            }));
+            throw err;
+        }
+    }
+    async updatePowCaptchaRecordResult(challenge, result, serverChecked = false, userSubmitted = false, userSignature, coords) {
+        const tables = this.getTables();
+        const timestamp = new Date();
+        const isDisapproved = result.status === CaptchaStatus.disapproved;
+        assertCoordsSafe(coords, "coords");
+        const setStage = {
+            result,
+            serverChecked,
+            userSubmitted,
+            userSignature,
+            lastUpdatedTimestamp: timestamp,
+            pendingStage: true,
+            blocked: isBlockingCaptchaResult(CaptchaType.pow, result),
+            ...(coords && { coords }),
+        };
+        if (userSubmitted) {
+            setStage.submittedAtTimestamp = {
+                $ifNull: ["$submittedAtTimestamp", timestamp],
+            };
+        }
+        if (isDisapproved) {
+            setStage.failedAtTimestamp = {
+                $ifNull: ["$failedAtTimestamp", timestamp],
+            };
+        }
+        try {
+            const updateResult = await tables.powcaptcha.updateOne({ challenge }, [{ $set: setStage }], { updatePipeline: true });
+            if (updateResult.matchedCount === 0) {
+                const err = new ProsopoError({
+                    code: "DB_QUERY_FAILED",
+                    errorSite: "DBSE-21",
+                    message: "Failed to get captcha",
+                    context: { challenge, ...setStage },
+                });
+                this.logger.info(() => ({
+                    err: err,
+                    msg: "No PowCaptcha record found to update",
+                }));
+                throw err;
+            }
+            this.logger.info(() => ({
+                data: {
+                    challenge,
+                    ...setStage,
+                },
+                msg: "PowCaptcha record updated successfully",
+            }));
+            this.centralStreamer?.streamPowUpdate(() => this.getPowCaptchaRecordByChallenge(challenge), (ts) => this.tables.powcaptcha
+                .updateOne({ challenge, lastUpdatedTimestamp: { $lte: ts } }, {
+                $set: { storedAtTimestamp: ts },
+                $unset: { pendingStage: 1 },
+            })
+                .then(() => { }));
+        }
+        catch (error) {
+            const err = new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-22",
+                message: "Failed to update captcha",
+                cause: error,
+                context: { challenge, ...setStage },
+            });
+            this.logger.error(() => ({
+                err: err,
+                msg: "Failed to update PowCaptcha record",
+            }));
+            throw err;
+        }
+    }
+    async updatePowCaptchaRecord(challenge, updates) {
+        const tables = this.getTables();
+        await tables.powcaptcha.updateOne({ challenge }, { $set: { ...updates, pendingStage: true } }, { upsert: false });
+        this.centralStreamer?.streamPowUpdate(() => this.getPowCaptchaRecordByChallenge(challenge), (ts) => this.tables.powcaptcha
+            .updateOne({ challenge, lastUpdatedTimestamp: { $lte: ts } }, {
+            $set: { storedAtTimestamp: ts },
+            $unset: { pendingStage: 1 },
+        })
+            .then(() => { }));
+    }
+    interactiveCollection(captchaType) {
+        switch (captchaType) {
+            case CaptchaType.puzzle:
+                return {
+                    table: TableNames.puzzlecaptcha,
+                    label: "PuzzleCaptcha",
+                    captchaType,
+                    answerProjection: {
+                        targetX: 1,
+                        targetY: 1,
+                        originX: 1,
+                        originY: 1,
+                        tolerance: 1,
+                        puzzleEvents: 1,
+                    },
+                    streamRecord: (record, markStored) => this.centralStreamer?.streamPuzzleRecord(record, markStored),
+                    streamUpdate: (challenge, markStored) => this.centralStreamer?.streamPuzzleUpdate(() => this.getPuzzleCaptchaRecordByChallenge(challenge), markStored),
+                };
+            case CaptchaType.iconOrder:
+                return {
+                    table: TableNames.iconordercaptcha,
+                    label: "IconOrderCaptcha",
+                    captchaType,
+                    answerProjection: {
+                        targets: 1,
+                        tolerance: 1,
+                        clicks: 1,
+                        iconOrderEvents: 1,
+                    },
+                    streamRecord: (record, markStored) => this.centralStreamer?.streamIconOrderRecord(record, markStored),
+                    streamUpdate: (challenge, markStored) => this.centralStreamer?.streamIconOrderUpdate(() => this.getIconOrderCaptchaRecordByChallenge(challenge), markStored),
+                };
+            case CaptchaType.audio:
+                return {
+                    table: TableNames.audiocaptcha,
+                    label: "AudioCaptcha",
+                    captchaType,
+                    answerProjection: {
+                        answer: 1,
+                        submittedAnswer: 1,
+                        replays: 1,
+                        audioEvents: 1,
+                    },
+                    streamRecord: (record, markStored) => this.centralStreamer?.streamAudioRecord(record, markStored),
+                    streamUpdate: (challenge, markStored) => this.centralStreamer?.streamAudioUpdate(() => this.getAudioCaptchaRecordByChallenge(challenge), markStored),
+                };
+        }
+    }
+    markInteractiveRecordStored(collection, challenge) {
+        return (ts) => this.tables[collection.table]
+            .updateOne({ challenge, lastUpdatedTimestamp: { $lte: ts } }, {
+            $set: { storedAtTimestamp: ts },
+            $unset: { pendingStage: 1 },
+        })
+            .then(() => { });
+    }
+    streamInteractiveUpdate(collection, challenge) {
+        collection.streamUpdate(challenge, this.markInteractiveRecordStored(collection, challenge));
+    }
+    async storeInteractiveCaptchaRecord(collection, record) {
+        const tables = this.getTables();
+        const { challenge, ipInfo } = record;
+        try {
+            await tables[collection.table].create(record);
+            this.logger.info(() => ({
+                data: {
+                    challenge,
+                    countryCode: ipInfo?.isValid ? ipInfo.countryCode : undefined,
+                },
+                msg: `${collection.label} record added successfully`,
+            }));
+            collection.streamRecord(record, this.markInteractiveRecordStored(collection, challenge));
+        }
+        catch (error) {
+            const err = new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-23",
+                message: "Failed to update captcha",
+                cause: error,
+                context: { challenge, ipInfo },
+            });
+            this.logger.error(() => ({
+                err: error,
+                msg: `Failed to add ${collection.label} record`,
+            }));
+            throw err;
+        }
+    }
+    pendingInteractiveRecord(challenge, components, providerSignature, ipAddress, headers, ja4, sessionId, ipInfo) {
+        return {
+            challenge,
+            userAccount: components.userAccount,
+            dappAccount: components.dappAccount,
+            requestedAtTimestamp: new Date(components.requestedAtTimestamp),
+            ipAddress,
+            headers,
+            ja4,
+            result: { status: CaptchaStatus.pending },
+            userSubmitted: false,
+            serverChecked: false,
+            providerSignature,
+            lastUpdatedTimestamp: new Date(),
+            pendingStage: true,
+            sessionId,
+            ipInfo,
+        };
+    }
+    async storePuzzleCaptchaRecord(challenge, components, targetX, targetY, originX, originY, tolerance, providerSignature, ipAddress, headers, ja4, sessionId, ipInfo, fingerprintSummary) {
+        await this.storeInteractiveCaptchaRecord(this.interactiveCollection(CaptchaType.puzzle), {
+            ...this.pendingInteractiveRecord(challenge, components, providerSignature, ipAddress, headers, ja4, sessionId, ipInfo),
+            tolerance,
+            targetX,
+            targetY,
+            originX,
+            originY,
+            ...(fingerprintSummary && { fingerprintSummary }),
+        });
+    }
+    async storeAudioCaptchaRecord(challenge, components, answer, providerSignature, ipAddress, headers, ja4, sessionId, ipInfo) {
+        await this.storeInteractiveCaptchaRecord(this.interactiveCollection(CaptchaType.audio), {
+            ...this.pendingInteractiveRecord(challenge, components, providerSignature, ipAddress, headers, ja4, sessionId, ipInfo),
+            answer,
+        });
+    }
+    async storeIconOrderCaptchaRecord(challenge, components, targets, tolerance, providerSignature, ipAddress, headers, ja4, sessionId, ipInfo) {
+        await this.storeInteractiveCaptchaRecord(this.interactiveCollection(CaptchaType.iconOrder), {
+            ...this.pendingInteractiveRecord(challenge, components, providerSignature, ipAddress, headers, ja4, sessionId, ipInfo),
+            targets,
+            tolerance,
+        });
+    }
+    async getInteractiveCaptchaRecordByChallenge(collection, challenge, failedFuncName) {
+        if (!this.tables) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-24",
+                message: "Database client is not connected",
+                context: { failedFuncName },
+            });
+        }
+        try {
+            const record = await this.tables[collection.table]
+                .findOne({ challenge }, {
+                challenge: 1,
+                userAccount: 1,
+                dappAccount: 1,
+                requestedAtTimestamp: 1,
+                submittedAtTimestamp: 1,
+                ipAddress: 1,
+                headers: 1,
+                ja4: 1,
+                result: 1,
+                sessionId: 1,
+                ipInfo: 1,
+                deviceCapability: 1,
+                behavioralDataPacked: 1,
+                serverChecked: 1,
+                userSubmitted: 1,
+                coords: 1,
+                clientMetaData: 1,
+                ...collection.answerProjection,
+            })
+                .lean();
+            if (record) {
+                this.logger.info(() => ({
+                    data: { challenge },
+                    msg: `${collection.label} record retrieved successfully`,
+                }));
+                return record;
+            }
+            this.logger.info(() => ({
+                data: { challenge },
+                msg: `No ${collection.label} record found`,
+            }));
+            return null;
+        }
+        catch (error) {
+            const err = new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-25",
+                message: "Failed to get captcha",
+                cause: error,
+                context: { challenge },
+            });
+            this.logger.error(() => ({
+                err: err,
+                msg: `Failed to retrieve ${collection.label} record`,
+            }));
+            throw err;
+        }
+    }
+    async getPuzzleCaptchaRecordByChallenge(challenge) {
+        return this.getInteractiveCaptchaRecordByChallenge(this.interactiveCollection(CaptchaType.puzzle), challenge, this.getPuzzleCaptchaRecordByChallenge.name);
+    }
+    async getIconOrderCaptchaRecordByChallenge(challenge) {
+        return this.getInteractiveCaptchaRecordByChallenge(this.interactiveCollection(CaptchaType.iconOrder), challenge, this.getIconOrderCaptchaRecordByChallenge.name);
+    }
+    async getAudioCaptchaRecordByChallenge(challenge) {
+        return this.getInteractiveCaptchaRecordByChallenge(this.interactiveCollection(CaptchaType.audio), challenge, this.getAudioCaptchaRecordByChallenge.name);
+    }
+    async claimInteractiveCaptchaSubmission(collection, challenge) {
+        const tables = this.getTables();
+        try {
+            const claim = await tables[collection.table].updateOne({ challenge, userSubmitted: { $ne: true } }, { $set: { userSubmitted: true, submittedAtTimestamp: new Date() } });
+            const won = claim.modifiedCount === 1;
+            this.logger.info(() => ({
+                data: { challenge, won },
+                msg: won
+                    ? `${collection.label} submission claimed`
+                    : `${collection.label} submission already claimed`,
+            }));
+            return won;
+        }
+        catch (error) {
+            const err = new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-26",
+                message: "Failed to update captcha",
+                cause: error,
+                context: { challenge },
+            });
+            this.logger.error(() => ({
+                err: err,
+                msg: `Failed to claim ${collection.label} submission`,
+            }));
+            throw err;
+        }
+    }
+    async claimIconOrderCaptchaSubmission(challenge) {
+        return this.claimInteractiveCaptchaSubmission(this.interactiveCollection(CaptchaType.iconOrder), challenge);
+    }
+    async claimAudioCaptchaSubmission(challenge) {
+        return this.claimInteractiveCaptchaSubmission(this.interactiveCollection(CaptchaType.audio), challenge);
+    }
+    async updateInteractiveCaptchaRecordResult(collection, challenge, result, serverChecked, userSubmitted, userSignature, coords, lastUpdatedTimestamp) {
+        const tables = this.getTables();
+        const timestamp = lastUpdatedTimestamp ?? new Date();
+        const isDisapproved = result.status === CaptchaStatus.disapproved;
+        assertCoordsSafe(coords, "coords");
+        const setStage = {
+            result,
+            serverChecked,
+            userSubmitted,
+            userSignature,
+            lastUpdatedTimestamp: timestamp,
+            pendingStage: true,
+            blocked: isBlockingCaptchaResult(collection.captchaType, result),
+            ...(coords && { coords }),
+            ...(userSubmitted && { submittedAtTimestamp: timestamp }),
+            ...(isDisapproved && { failedAtTimestamp: timestamp }),
+        };
+        try {
+            const updateResult = await tables[collection.table].updateOne({ challenge }, { $set: setStage });
+            if (updateResult.matchedCount === 0) {
+                const err = new ProsopoError({
+                    code: "DB_QUERY_FAILED",
+                    errorSite: "DBSE-27",
+                    message: "Failed to get captcha",
+                    context: { challenge, ...setStage },
+                });
+                this.logger.info(() => ({
+                    err: err,
+                    msg: `No ${collection.label} record found to update`,
+                }));
+                throw err;
+            }
+            this.logger.info(() => ({
+                data: {
+                    challenge,
+                    ...setStage,
+                },
+                msg: `${collection.label} record updated successfully`,
+            }));
+            this.streamInteractiveUpdate(collection, challenge);
+        }
+        catch (error) {
+            const err = new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-28",
+                message: "Failed to update captcha",
+                cause: error,
+                context: { challenge, ...setStage },
+            });
+            this.logger.error(() => ({
+                err: err,
+                msg: `Failed to update ${collection.label} record`,
+            }));
+            throw err;
+        }
+    }
+    async updatePuzzleCaptchaRecordResult(challenge, result, serverChecked = false, userSubmitted = false, userSignature, coords, lastUpdatedTimestamp) {
+        await this.updateInteractiveCaptchaRecordResult(this.interactiveCollection(CaptchaType.puzzle), challenge, result, serverChecked, userSubmitted, userSignature, coords, lastUpdatedTimestamp);
+    }
+    async updateIconOrderCaptchaRecordResult(challenge, result, serverChecked = false, userSubmitted = false, userSignature, coords, lastUpdatedTimestamp) {
+        await this.updateInteractiveCaptchaRecordResult(this.interactiveCollection(CaptchaType.iconOrder), challenge, result, serverChecked, userSubmitted, userSignature, coords, lastUpdatedTimestamp);
+    }
+    async updateAudioCaptchaRecordResult(challenge, result, serverChecked = false, userSubmitted = false, userSignature, coords, lastUpdatedTimestamp) {
+        await this.updateInteractiveCaptchaRecordResult(this.interactiveCollection(CaptchaType.audio), challenge, result, serverChecked, userSubmitted, userSignature, coords, lastUpdatedTimestamp);
+    }
+    async updateInteractiveCaptchaRecord(collection, challenge, updates) {
+        const tables = this.getTables();
+        const timestamp = new Date();
+        const baseSet = {
+            ...updates,
+            pendingStage: true,
+            ...(updates.serverChecked === true && {
+                verifiedAtTimestamp: timestamp,
+            }),
+            ...(updates.userSubmitted === true && {
+                submittedAtTimestamp: timestamp,
+            }),
+            ...(updates.result?.status === CaptchaStatus.disapproved && {
+                failedAtTimestamp: timestamp,
+            }),
+        };
+        await tables[collection.table].updateOne({ challenge }, { $set: baseSet });
+        this.streamInteractiveUpdate(collection, challenge);
+    }
+    async updatePuzzleCaptchaRecord(challenge, updates) {
+        await this.updateInteractiveCaptchaRecord(this.interactiveCollection(CaptchaType.puzzle), challenge, updates);
+    }
+    async updateIconOrderCaptchaRecord(challenge, updates) {
+        await this.updateInteractiveCaptchaRecord(this.interactiveCollection(CaptchaType.iconOrder), challenge, updates);
+    }
+    async updateAudioCaptchaRecord(challenge, updates) {
+        await this.updateInteractiveCaptchaRecord(this.interactiveCollection(CaptchaType.audio), challenge, updates);
+    }
+    async markInteractiveCaptchaRecordChecked(collection, challenge) {
+        const tables = this.getTables();
+        const timestamp = new Date();
+        const result = await tables[collection.table].updateOne({ challenge, serverChecked: { $ne: true } }, {
+            $set: {
+                serverChecked: true,
+                lastUpdatedTimestamp: timestamp,
+                verifiedAtTimestamp: timestamp,
+                pendingStage: true,
+            },
+        });
+        if (result.modifiedCount === 0) {
+            return false;
+        }
+        this.streamInteractiveUpdate(collection, challenge);
+        return true;
+    }
+    async markIconOrderCaptchaRecordChecked(challenge) {
+        return this.markInteractiveCaptchaRecordChecked(this.interactiveCollection(CaptchaType.iconOrder), challenge);
+    }
+    async markAudioCaptchaRecordChecked(challenge) {
+        return this.markInteractiveCaptchaRecordChecked(this.interactiveCollection(CaptchaType.audio), challenge);
+    }
+    async markPuzzleCaptchaRecordChecked(challenge) {
+        return this.markInteractiveCaptchaRecordChecked(this.interactiveCollection(CaptchaType.puzzle), challenge);
+    }
+    async getCheckedDappUserCommitments() {
+        const filter = { [StoredStatusNames.serverChecked]: true };
+        const docs = await this.tables?.commitment
+            .find(filter)
+            .lean();
+        return docs || [];
+    }
+    async getUnstoredDappUserCommitments(limit = 1000, afterId) {
+        const filter = {
+            pendingStage: true,
+        };
+        if (afterId !== undefined) {
+            filter._id = { $gt: afterId };
+        }
+        const docs = await this.tables?.commitment
+            .find(filter)
+            .hint("pendingStage_partial")
+            .sort({ _id: 1 })
+            .limit(limit)
+            .lean();
+        return docs || [];
+    }
+    async markDappUserCommitmentsStored(commitmentIds, asOfTimestamp = new Date()) {
+        await this.tables?.commitment.updateMany({
+            id: { $in: commitmentIds },
+            lastUpdatedTimestamp: { $lte: asOfTimestamp },
+        }, {
+            $set: { storedAtTimestamp: new Date() },
+            $unset: { pendingStage: 1 },
+        }, { upsert: false });
+    }
+    async markDappUserCommitmentsChecked(commitmentIds) {
+        const timestamp = new Date();
+        const result = await this.getTables().commitment.updateMany({ id: { $in: commitmentIds }, serverChecked: { $ne: true } }, [
+            {
+                $set: {
+                    serverChecked: true,
+                    lastUpdatedTimestamp: timestamp,
+                    pendingStage: true,
+                    verifiedAtTimestamp: {
+                        $ifNull: ["$verifiedAtTimestamp", timestamp],
+                    },
+                },
+            },
+        ], { updatePipeline: true });
+        return result.modifiedCount;
+    }
+    async updateDappUserCommitment(commitmentId, updates) {
+        const filter = { id: commitmentId };
+        const timestamp = new Date();
+        const baseSet = {
+            ...updates,
+            lastUpdatedTimestamp: timestamp,
+            pendingStage: true,
+        };
+        const pipelineExprs = {};
+        if (updates.userSubmitted === true) {
+            pipelineExprs.submittedAtTimestamp = {
+                $ifNull: ["$submittedAtTimestamp", timestamp],
+            };
+        }
+        if (updates.serverChecked === true) {
+            pipelineExprs.verifiedAtTimestamp = {
+                $ifNull: ["$verifiedAtTimestamp", timestamp],
+            };
+        }
+        if (updates.result?.status === CaptchaStatus.disapproved) {
+            pipelineExprs.failedAtTimestamp = {
+                $ifNull: ["$failedAtTimestamp", timestamp],
+            };
+        }
+        if (Object.keys(pipelineExprs).length === 0) {
+            await this.tables?.commitment.updateOne(filter, { $set: baseSet });
+            return;
+        }
+        await this.tables?.commitment.updateOne(filter, [{ $set: { ...baseSet, ...pipelineExprs } }], { updatePipeline: true });
+    }
+    async countCommitmentsByNormalisedEmail(dappAccount, emailNormalised) {
+        if (!emailNormalised)
+            return 0;
+        const tables = this.getTables();
+        const filter = {
+            dappAccount,
+            serverChecked: true,
+            "metadata.emailNormalised": emailNormalised,
+        };
+        const counts = await Promise.all([
+            tables.commitment.countDocuments(filter),
+            tables.powcaptcha.countDocuments(filter),
+            tables.puzzlecaptcha.countDocuments(filter),
+            tables.iconordercaptcha.countDocuments(filter),
+            tables.audiocaptcha.countDocuments(filter),
+        ]);
+        return counts.reduce((total, count) => total + count, 0);
+    }
+    async getUnstoredDappUserPoWCommitments(limit = 1000, afterId) {
+        const filter = {
+            pendingStage: true,
+        };
+        if (afterId !== undefined) {
+            filter._id = { $gt: afterId };
+        }
+        const docs = await this.tables?.powcaptcha
+            .find(filter)
+            .hint("pendingStage_partial")
+            .sort({ _id: 1 })
+            .limit(limit)
+            .lean();
+        return docs || [];
+    }
+    async markDappUserPoWCommitmentsStored(challenges, asOfTimestamp = new Date()) {
+        await this.tables?.powcaptcha.updateMany({
+            challenge: { $in: challenges },
+            lastUpdatedTimestamp: { $lte: asOfTimestamp },
+        }, {
+            $set: { storedAtTimestamp: new Date() },
+            $unset: { pendingStage: 1 },
+        }, { upsert: false });
+    }
+    async markDappUserPoWCommitmentsChecked(challenges) {
+        const timestamp = new Date();
+        const result = await this.getTables().powcaptcha.updateMany({ challenge: { $in: challenges }, serverChecked: { $ne: true } }, [
+            {
+                $set: {
+                    serverChecked: true,
+                    lastUpdatedTimestamp: timestamp,
+                    pendingStage: true,
+                    verifiedAtTimestamp: {
+                        $ifNull: ["$verifiedAtTimestamp", timestamp],
+                    },
+                },
+            },
+        ], { upsert: false, updatePipeline: true });
+        return result.modifiedCount;
+    }
+    async storeSessionRecord(sessionRecord) {
+        try {
+            this.logger.debug(() => ({
+                data: { action: "storing", sessionRecord },
+            }));
+            const recordWithFlag = {
+                ...sessionRecord,
+                lastUpdatedTimestamp: sessionRecord.lastUpdatedTimestamp ?? sessionRecord.createdAt,
+                pendingStage: true,
+            };
+            await this.tables.session.create(recordWithFlag);
+            this.centralStreamer?.streamSessionRecord(recordWithFlag, (ts) => this.tables.session
+                .updateOne({
+                sessionId: sessionRecord.sessionId,
+                lastUpdatedTimestamp: { $lte: ts },
+            }, {
+                $set: { storedAtTimestamp: ts },
+                $unset: { pendingStage: 1 },
+            })
+                .then(() => { }));
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-29",
+                message: "Failed to store session",
+                cause: err,
+                context: { sessionId: sessionRecord.sessionId },
+            });
+        }
+    }
+    async storeBlockedSession(record) {
+        try {
+            const stored = {
+                ...record,
+                blocked: true,
+                deleted: true,
+                storedAtTimestamp: record.storedAtTimestamp ?? record.createdAt,
+                lastUpdatedTimestamp: record.lastUpdatedTimestamp ?? record.createdAt,
+            };
+            await this.tables.session.create(stored);
+            this.centralStreamer?.streamSessionRecord(stored, (ts) => this.tables.session
+                .updateOne({
+                sessionId: stored.sessionId,
+                lastUpdatedTimestamp: { $lte: ts },
+            }, { $set: { storedAtTimestamp: ts } })
+                .then(() => { }));
+        }
+        catch (err) {
+            this.logger.warn(() => ({
+                err,
+                msg: "Failed to persist blocked session record",
+                data: { sessionId: record.sessionId, siteKey: record.siteKey },
+            }));
+        }
+    }
+    async getSessionRecordBySessionId(sessionId) {
+        const filter = { sessionId };
+        const doc = await this.tables.session
+            .findOne(filter, SESSION_PROJECTION)
+            .lean();
+        return doc || undefined;
+    }
+    async getSessionRecordByToken(token) {
+        const filter = { token };
+        const record = await this.tables.session.findOne(filter).lean();
+        return record || undefined;
+    }
+    async checkAndRemoveSession(sessionId) {
+        this.logger.debug(() => ({
+            data: { action: "checking and removing", sessionId },
+        }));
+        const filter = {
+            sessionId,
+            deleted: { $exists: false },
+        };
+        try {
+            const session = await this.tables.session
+                .findOneAndUpdate(filter, {
+                deleted: true,
+                lastUpdatedTimestamp: new Date(),
+                pendingStage: true,
+            })
+                .lean();
+            return session || undefined;
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-30",
+                message: "Failed to check and remove session",
+                cause: err,
+                context: { sessionId },
+            });
+        }
+    }
+    async claimSessionServerCheck(sessionId) {
+        try {
+            const result = await this.tables.session.updateOne({ sessionId, serverChecked: { $ne: true } }, {
+                $set: {
+                    serverChecked: true,
+                    lastUpdatedTimestamp: new Date(),
+                    pendingStage: true,
+                },
+            });
+            return result.modifiedCount > 0;
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-31",
+                message: "Failed to get session",
+                cause: err,
+                context: { sessionId },
+            });
+        }
+    }
+    async updateSessionRecord(sessionId, updates, streamToCentral) {
+        try {
+            await this.tables.session.updateOne({ sessionId }, {
+                $set: {
+                    ...updates,
+                    lastUpdatedTimestamp: new Date(),
+                    pendingStage: true,
+                },
+            });
+            if (streamToCentral && this.centralStreamer) {
+                const streamer = this.centralStreamer;
+                const markStored = (ts) => this.tables.session
+                    .updateOne({ sessionId, lastUpdatedTimestamp: { $lte: ts } }, {
+                    $set: { storedAtTimestamp: ts },
+                    $unset: { pendingStage: 1 },
+                })
+                    .then(() => { });
+                this.tables.session
+                    .findOne({ sessionId })
+                    .lean()
+                    .then((record) => {
+                    if (record) {
+                        streamer.streamSessionRecord(record, markStored);
+                    }
+                })
+                    .catch(() => { });
+            }
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-32",
+                message: "Failed to get session",
+                cause: err,
+                context: { sessionId },
+            });
+        }
+    }
+    async recordSessionSimdReadingsIfAbsent(sessionId, readings, stage) {
+        try {
+            await this.tables.session.updateOne({ sessionId }, [
+                {
+                    $set: {
+                        simdReadings: { $ifNull: ["$simdReadings", readings] },
+                        simdReadingsStage: { $ifNull: ["$simdReadingsStage", stage] },
+                        lastUpdatedTimestamp: new Date(),
+                        pendingStage: true,
+                    },
+                },
+            ], { updatePipeline: true });
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-33",
+                message: "Failed to get session",
+                cause: err,
+                context: { sessionId, stage },
+            });
+        }
+    }
+    async mergeSessionDnsEvent(sessionId, fields, receivedAtIfAbsent) {
+        const setStage = {
+            "dnsEvent.receivedAt": {
+                $ifNull: ["$dnsEvent.receivedAt", receivedAtIfAbsent],
+            },
+            lastUpdatedTimestamp: new Date(),
+            pendingStage: true,
+        };
+        if (fields.resolverIp !== undefined) {
+            setStage["dnsEvent.resolverIp"] = fields.resolverIp;
+        }
+        if (fields.peerIp !== undefined) {
+            setStage["dnsEvent.peerIp"] = fields.peerIp;
+        }
+        if (fields.pathValid !== undefined) {
+            setStage["dnsEvent.pathValid"] = fields.pathValid;
+        }
+        try {
+            const result = await this.tables.session.updateOne({ sessionId }, [{ $set: setStage }], { updatePipeline: true });
+            return result.matchedCount > 0;
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-34",
+                message: "Failed to get session",
+                cause: err,
+                context: { sessionId },
+            });
+        }
+    }
+    async getSessionByuserSitekeyIpHash(userSitekeyIpHash) {
+        this.logger.debug(() => ({
+            data: { action: "getting session by user IP hash", userSitekeyIpHash },
+        }));
+        const filter = {
+            userSitekeyIpHash,
+            deleted: { $exists: false },
+        };
+        try {
+            const session = await this.tables.session
+                .findOne(filter)
+                .lean();
+            return session || undefined;
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-35",
+                message: "Failed to get session",
+                cause: err,
+                context: { userSitekeyIpHash },
+            });
+        }
+    }
+    getUnstoredSessionRecords(limit = 1000, afterId) {
+        const filter = {
+            pendingStage: true,
+        };
+        if (afterId !== undefined) {
+            filter._id = { $gt: afterId };
+        }
+        return Promise.resolve(this.tables?.session)
+            .then((tbl) => tbl
+            ?.find(filter)
+            .hint("pendingStage_partial")
+            .sort({ _id: 1 })
+            .limit(limit)
+            .lean())
+            .then((docs) => docs || []);
+    }
+    async markSessionRecordsStored(sessionIds, asOfTimestamp = new Date()) {
+        await this.tables?.session.updateMany({
+            sessionId: { $in: sessionIds },
+            lastUpdatedTimestamp: { $lte: asOfTimestamp },
+        }, {
+            $set: { storedAtTimestamp: new Date() },
+            $unset: { pendingStage: 1 },
+        }, { upsert: false });
+    }
+    async storePendingImageCommitment(userAccount, requestHash, salt, deadlineTimestamp, requestedAtTimestamp, ipAddress, threshold, sessionId, ipInfo) {
+        if (!isHex(requestHash)) {
+            throw new ProsopoError({
+                code: "INVALID_DATA",
+                errorSite: "DBSE-36",
+                message: "Invalid hash",
+                context: {
+                    failedFuncName: this.storePendingImageCommitment.name,
+                    requestHash,
+                },
+            });
+        }
+        const pendingRecord = {
+            userAccount,
+            pending: true,
+            salt,
+            requestHash,
+            deadlineTimestamp,
+            requestedAtTimestamp,
+            ipAddress,
+            sessionId,
+            threshold,
+            ipInfo,
+            dappAccount: "",
+            providerAccount: "",
+            datasetId: "",
+            id: "",
+            result: { status: CaptchaStatus.pending },
+            headers: {},
+            ja4: "",
+            userSignature: "",
+            userSubmitted: false,
+            serverChecked: false,
+        };
+        await this.tables?.commitment.updateOne({ requestHash: requestHash }, { $set: pendingRecord }, { upsert: true });
+    }
+    async getPendingImageCommitment(requestHash) {
+        if (!isHex(requestHash)) {
+            throw new ProsopoError({
+                code: "INVALID_DATA",
+                errorSite: "DBSE-37",
+                message: "Invalid hash",
+                context: {
+                    failedFuncName: this.getPendingImageCommitment.name,
+                    requestHash,
+                },
+            });
+        }
+        const doc = await this.tables?.commitment
+            .findOne({
+            requestHash: requestHash,
+            pending: true,
+        })
+            .lean();
+        if (doc) {
+            return {
+                dappAccount: doc.dappAccount,
+                pending: doc.pending,
+                salt: doc.salt,
+                requestHash: doc.requestHash,
+                deadlineTimestamp: doc.deadlineTimestamp,
+                requestedAtTimestamp: doc.requestedAtTimestamp,
+                ipAddress: doc.ipAddress,
+                sessionId: doc.sessionId,
+                threshold: doc.threshold,
+            };
+        }
+        throw new ProsopoError({
+            code: "RECORD_NOT_FOUND",
+            errorSite: "DBSE-38",
+            message: "No pending record found",
+            context: {
+                failedFuncName: this.getPendingImageCommitment.name,
+                requestHash,
+            },
+        });
+    }
+    async updatePendingImageCommitmentStatus(requestHash) {
+        if (!isHex(requestHash)) {
+            throw new ProsopoError({
+                code: "INVALID_DATA",
+                errorSite: "DBSE-39",
+                message: "Invalid hash",
+                context: {
+                    failedFuncName: this.updatePendingImageCommitmentStatus.name,
+                    requestHash,
+                },
+            });
+        }
+        const result = await this.getTables().commitment.updateOne({ requestHash: requestHash, pending: true }, {
+            $set: {
+                pending: false,
+            },
+        });
+        return result.modifiedCount > 0;
+    }
+    async getAllCaptchasByDatasetId(datasetId, state) {
+        const filter = {
+            datasetId,
+            solved: state === CaptchaStates.Solved,
+        };
+        const cursor = this.tables?.captcha
+            .find(filter)
+            .lean();
+        const docs = await cursor;
+        if (docs) {
+            return docs.map(({ _id, ...keepAttrs }) => keepAttrs);
+        }
+        throw new ProsopoError({
+            code: "DB_QUERY_FAILED",
+            errorSite: "DBSE-40",
+            message: "Failed to get captcha",
+        });
+    }
+    async getAllDappUserSolutions(captchaId) {
+        const filter = {
+            captchaId: { $in: captchaId },
+        };
+        const cursor = this.tables?.usersolution
+            ?.find(filter)
+            .lean();
+        const docs = await cursor;
+        if (docs) {
+            return docs.map(({ _id, ...keepAttrs }) => keepAttrs);
+        }
+        throw new ProsopoError({
+            code: "DB_QUERY_FAILED",
+            errorSite: "DBSE-41",
+            message: "Failed to get solution",
+        });
+    }
+    async getDatasetIdWithSolvedCaptchasOfSizeN(solvedCaptchaCount) {
+        const cursor = this.tables?.solution.aggregate([
+            {
+                $match: {},
+            },
+            {
+                $group: {
+                    _id: "$datasetId",
+                    count: { $sum: 1 },
+                },
+            },
+            {
+                $match: {
+                    count: { $gte: solvedCaptchaCount },
+                },
+            },
+            {
+                $sample: { size: 1 },
+            },
+        ]);
+        const docs = await cursor;
+        if (docs?.length) {
+            return docs[0]._id;
+        }
+        throw new ProsopoError({
+            code: "DATASET_INVALID",
+            errorSite: "DBSE-42",
+            message: "No datasets found with required number of solutions",
+        });
+    }
+    async getRandomSolvedCaptchasFromSingleDataset(datasetId, size) {
+        if (!isHex(datasetId)) {
+            throw new ProsopoError({
+                code: "INVALID_DATA",
+                errorSite: "DBSE-43",
+                message: "Invalid hash",
+                context: {
+                    failedFuncName: this.getRandomSolvedCaptchasFromSingleDataset.name,
+                    datasetId,
+                },
+            });
+        }
+        const sampleSize = size ? Math.abs(Math.trunc(size)) : 1;
+        const cursor = this.tables?.solution.aggregate([
+            { $match: { datasetId } },
+            { $sample: { size: sampleSize } },
+            {
+                $project: {
+                    captchaId: 1,
+                    captchaContentId: 1,
+                    solution: 1,
+                },
+            },
+        ]);
+        const docs = await cursor;
+        if (docs?.length) {
+            return docs;
+        }
+        throw new ProsopoError({
+            code: "DB_QUERY_FAILED",
+            errorSite: "DBSE-44",
+            message: "Failed to get solution",
+            context: {
+                failedFuncName: this.getRandomSolvedCaptchasFromSingleDataset.name,
+                datasetId,
+                size,
+            },
+        });
+    }
+    async getDappUserSolutionById(commitmentId) {
+        const filter = {
+            commitmentId: commitmentId,
+        };
+        const project = { _id: 0 };
+        const cursor = this.tables?.usersolution?.findOne(filter, project).lean();
+        const doc = await cursor;
+        if (doc) {
+            return doc;
+        }
+        throw new ProsopoError({
+            code: "DB_QUERY_FAILED",
+            errorSite: "DBSE-45",
+            message: "Failed to get solution",
+            context: { failedFuncName: this.getCaptchaById.name, commitmentId },
+        });
+    }
+    async getDappUserCommitmentById(commitmentId) {
+        const filter = { id: commitmentId };
+        const commitmentCursor = this.tables?.commitment
+            ?.findOne(filter, ProviderDatabase.DAPP_USER_COMMITMENT_PROJECTION)
+            .lean();
+        const doc = await commitmentCursor;
+        return doc ? doc : undefined;
+    }
+    async getDappUserCommitmentByAccount(userAccount, dappAccount) {
+        const filter = {
+            userAccount,
+            dappAccount,
+        };
+        const sort = { sort: { _id: -1 } };
+        const docs = await this.tables?.commitment
+            ?.find(filter, ProviderDatabase.DAPP_USER_COMMITMENT_PROJECTION, sort)
+            .lean();
+        return docs ? docs : [];
+    }
+    async approveDappUserCommitment(commitmentId, coords) {
+        try {
+            const result = { status: CaptchaStatus.approved };
+            const updateDoc = {
+                result,
+                lastUpdatedTimestamp: new Date(),
+                pendingStage: true,
+                blocked: isBlockingCaptchaResult(CaptchaType.image, result),
+                ...(coords ? { coords } : {}),
+            };
+            const filter = { id: commitmentId };
+            await this.tables?.commitment
+                ?.findOneAndUpdate(filter, { $set: updateDoc }, { upsert: false })
+                .lean();
+            this.centralStreamer?.streamImageUpdate(() => this.tables.commitment
+                .findOne({ id: commitmentId })
+                .lean(), (ts) => this.tables.commitment
+                .updateOne({ id: commitmentId, lastUpdatedTimestamp: { $lte: ts } }, {
+                $set: { storedAtTimestamp: ts },
+                $unset: { pendingStage: 1 },
+            })
+                .then(() => { }));
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-46",
+                message: "Failed to approve solution",
+                cause: err,
+                context: { commitmentId },
+            });
+        }
+    }
+    async disapproveDappUserCommitment(commitmentId, reason, coords) {
+        try {
+            const result = {
+                status: CaptchaStatus.disapproved,
+                reason,
+            };
+            const updateDoc = {
+                result,
+                lastUpdatedTimestamp: new Date(),
+                pendingStage: true,
+                blocked: isBlockingCaptchaResult(CaptchaType.image, result),
+                ...(coords ? { coords } : {}),
+            };
+            const filter = { id: commitmentId };
+            await this.tables?.commitment
+                ?.findOneAndUpdate(filter, { $set: updateDoc }, { upsert: false })
+                .lean();
+            this.centralStreamer?.streamImageUpdate(() => this.tables.commitment
+                .findOne({ id: commitmentId })
+                .lean(), (ts) => this.tables.commitment
+                .updateOne({ id: commitmentId, lastUpdatedTimestamp: { $lte: ts } }, {
+                $set: { storedAtTimestamp: ts },
+                $unset: { pendingStage: 1 },
+            })
+                .then(() => { }));
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-47",
+                message: "Failed to approve solution",
+                cause: err,
+                context: { commitmentId },
+            });
+        }
+    }
+    async flagProcessedDappUserSolutions(captchaIds) {
+        try {
+            await this.tables?.usersolution
+                ?.updateMany({ captchaId: { $in: captchaIds } }, { $set: { processed: true } }, { upsert: false })
+                .lean();
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-48",
+                message: "Failed to flag solution as processed",
+                cause: err,
+                context: { captchaIds },
+            });
+        }
+    }
+    async flagProcessedDappUserCommitments(commitmentIds) {
+        try {
+            const distinctCommitmentIds = [...new Set(commitmentIds)];
+            await this.tables?.commitment
+                ?.updateMany({ id: { $in: distinctCommitmentIds } }, { $set: { processed: true } }, { upsert: false })
+                .lean();
+        }
+        catch (err) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-49",
+                message: "Failed to flag commitment as processed",
+                cause: err,
+                context: { commitmentIds },
+            });
+        }
+    }
+    async getScheduledTaskStatus(taskId, status) {
+        const filter = {
+            _id: taskId,
+            status: status,
+        };
+        const cursor = await this.tables?.scheduler
+            ?.findOne(filter)
+            .lean();
+        return cursor ? cursor : undefined;
+    }
+    async getLastScheduledTaskStatus(task, status) {
+        const filter = { processName: task };
+        if (status) {
+            filter.status = status;
+        }
+        const sort = {
+            datetime: -1,
+        };
+        const cursor = await this.tables?.scheduler
+            ?.findOne(filter)
+            .sort(sort)
+            .limit(1)
+            .lean();
+        return cursor ? cursor : undefined;
+    }
+    async createScheduledTaskStatus(taskName, status) {
+        const now = new Date();
+        const doc = ScheduledTaskSchema.parse({
+            processName: taskName,
+            datetime: now,
+            status,
+        });
+        const taskRecord = await this.tables?.scheduler.create(doc);
+        return taskRecord._id;
+    }
+    async updateScheduledTaskStatus(taskId, status, result) {
+        const update = {
+            status,
+            updated: new Date(),
+            ...(result && { result }),
+        };
+        const filter = { _id: taskId };
+        await this.tables?.scheduler.updateOne(filter, { $set: update }, {
+            upsert: false,
+        });
+    }
+    async cleanupScheduledTaskStatus(status) {
+        const filter = {
+            status,
+        };
+        await this.tables?.scheduler.deleteMany(filter);
+    }
+    async updateClientRecords(clientRecords) {
+        const usable = clientRecords.filter((record) => !!record.account);
+        const skipped = clientRecords.length - usable.length;
+        if (skipped > 0) {
+            await this.logger.error(() => ({
+                msg: "Refusing to upsert client records with no account",
+                data: { skipped, received: clientRecords.length },
+            }));
+        }
+        const ops = usable.map((record) => {
+            const clientRecord = {
+                account: record.account,
+                settings: record.settings,
+                tier: record.tier,
+            };
+            const filter = {
+                account: record.account,
+            };
+            return {
+                updateOne: {
+                    filter,
+                    update: {
+                        $set: clientRecord,
+                    },
+                    upsert: true,
+                },
+            };
+        });
+        if (!ops || ops.length === 0) {
+            await this.logger.debug(() => ({ msg: "No client updates to apply" }));
+            return;
+        }
+        try {
+            await this.tables?.client.bulkWrite(ops);
+        }
+        finally {
+            clientRecordCache.clear();
+        }
+    }
+    async removeClientRecords(accounts) {
+        if (!accounts || accounts.length === 0) {
+            return;
+        }
+        try {
+            await this.tables?.client.deleteMany({
+                account: { $in: accounts },
+            });
+        }
+        finally {
+            clientRecordCache.clear();
+        }
+    }
+    async getAllClientRecords() {
+        const docs = await this.tables?.client.find().lean();
+        return docs || [];
+    }
+    async getClientRecord(account) {
+        const cacheKey = `${this.url}|${account}`;
+        const cached = clientRecordCache.get(cacheKey);
+        if (cached)
+            return cached;
+        const filter = { account };
+        const doc = await this.tables?.client
+            .findOne(filter, {
+            account: 1,
+            settings: 1,
+            tier: 1,
+        })
+            .lean();
+        if (!doc)
+            return undefined;
+        clientRecordCache.set(cacheKey, doc);
+        return doc;
+    }
+    async upsertDecisionMachineArtifact(artifact) {
+        const now = new Date();
+        const dappAccount = artifact.dappAccount ?? null;
+        const kind = artifact.kind ?? DecisionMachineKind.Routing;
+        const filter = {
+            scope: artifact.scope,
+            dappAccount,
+            kind,
+        };
+        await this.tables?.decisionMachine.updateOne(filter, {
+            $set: {
+                scope: artifact.scope,
+                dappAccount,
+                kind,
+                runtime: artifact.runtime,
+                language: artifact.language,
+                source: artifact.source,
+                name: artifact.name,
+                version: artifact.version,
+                captchaType: artifact.captchaType,
+                updatedAt: now,
+            },
+            $setOnInsert: {
+                createdAt: now,
+            },
+        }, { upsert: true });
+    }
+    async getDecisionMachineArtifact(scope, dappAccount, kind) {
+        const filter = {
+            scope,
+            dappAccount: dappAccount ?? null,
+        };
+        if (kind !== undefined) {
+            filter.kind = kind;
+        }
+        const doc = await this.tables?.decisionMachine
+            .findOne(filter)
+            .lean();
+        return doc ?? undefined;
+    }
+    async getAllDecisionMachineArtifacts() {
+        const docs = await this.tables?.decisionMachine
+            .find({})
+            .lean();
+        return docs ?? [];
+    }
+    async getDecisionMachineArtifactById(id) {
+        const doc = await this.tables?.decisionMachine
+            .findById(id)
+            .lean();
+        return doc ?? undefined;
+    }
+    async removeDecisionMachineArtifact(id) {
+        const result = await this.tables?.decisionMachine.deleteOne({
+            _id: id,
+        });
+        return (result?.deletedCount ?? 0) > 0;
+    }
+    async removeAllDecisionMachineArtifacts() {
+        const result = await this.tables?.decisionMachine.deleteMany({});
+        return result?.deletedCount ?? 0;
+    }
+    async getSpamEmailDomain(domain) {
+        if (!this.tables?.spamEmailDomain) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-50",
+                message: "Failed to import database engine",
+                context: { failedFuncName: this.getSpamEmailDomain.name },
+            });
+        }
+        const suffixCandidates = buildDomainSuffixCandidates(domain).slice(0, MAX_DOMAIN_SUFFIX_CANDIDATES);
+        const query = suffixCandidates.length > 0
+            ? { domain: { $in: suffixCandidates } }
+            : { domain };
+        return await this.tables.spamEmailDomain
+            .findOne(query)
+            .exec();
+    }
+    async bulkUpdateSpamEmailDomains(domains, upsert) {
+        if (!this.tables?.spamEmailDomain) {
+            throw new ProsopoError({
+                code: "DB_QUERY_FAILED",
+                errorSite: "DBSE-51",
+                message: "Failed to import database engine",
+                context: { failedFuncName: this.bulkUpdateSpamEmailDomains.name },
+            });
+        }
+        const bulkOps = domains.map((op) => ({
+            updateOne: {
+                filter: op.filter,
+                update: { $set: op.update },
+                upsert,
+            },
+        }));
+        if (bulkOps.length > 0) {
+            await this.tables.spamEmailDomain.bulkWrite(bulkOps);
+        }
+    }
+}
+ProviderDatabase.DAPP_USER_COMMITMENT_PROJECTION = {
+    id: 1,
+    result: 1,
+    serverChecked: 1,
+    requestedAtTimestamp: 1,
+    submittedAtTimestamp: 1,
+    verifiedAtTimestamp: 1,
+    failedAtTimestamp: 1,
+    ipAddress: 1,
+    sessionId: 1,
+    userAccount: 1,
+    dappAccount: 1,
+    headers: 1,
+    ipInfo: 1,
+    behavioralDataPacked: 1,
+    deviceCapability: 1,
+    coords: 1,
+    clientMetaData: 1,
+};
+//# sourceMappingURL=provider.js.map