@prosopo/database 4.5.1 → 4.6.1

package/.turbo/turbo-build$colon$cjs.log

@@ -34,21 +34,22 @@ ViteCommonJSConfig: .
 vite v8.1.6 building ssr environment for production...
 Bundle build started
 
-✓ 13 modules transformed.
+✓ 14 modules transformed.
 rendering chunks...
 computing gzip size...
-dist/cjs/base/index.cjs                    0.22 kB │ gzip:  0.10 kB
+dist/cjs/base/index.cjs                    0.43 kB │ gzip:  0.16 kB
 dist/cjs/base/mongoMemory.cjs              0.66 kB │ gzip:  0.32 kB
 dist/cjs/_virtual/_rolldown/runtime.cjs    1.00 kB │ gzip:  0.50 kB
 dist/cjs/databases/index.cjs               1.14 kB │ gzip:  0.32 kB
 dist/cjs/databases/clientRecordCache.cjs   1.45 kB │ gzip:  0.69 kB
-dist/cjs/index.cjs                         1.66 kB │ gzip:  0.49 kB
+dist/cjs/index.cjs                         1.87 kB │ gzip:  0.54 kB
 dist/cjs/databases/client.cjs              1.96 kB │ gzip:  0.82 kB
 dist/cjs/mongooseOptions.cjs               3.74 kB │ gzip:  1.56 kB
-dist/cjs/base/mongo.cjs                    4.29 kB │ gzip:  1.42 kB
+dist/cjs/base/mongo.cjs                    4.47 kB │ gzip:  1.49 kB
+dist/cjs/base/reconcileIndexes.cjs         5.97 kB │ gzip:  1.98 kB
 dist/cjs/databases/centralDbStreamer.cjs   6.40 kB │ gzip:  1.52 kB
-dist/cjs/databases/captcha.cjs             8.23 kB │ gzip:  2.38 kB
+dist/cjs/databases/captcha.cjs             8.39 kB │ gzip:  2.40 kB
 dist/cjs/redisCache.cjs                   14.30 kB │ gzip:  4.45 kB
-dist/cjs/databases/provider.cjs           71.01 kB │ gzip: 13.57 kB
+dist/cjs/databases/provider.cjs           72.31 kB │ gzip: 13.98 kB
 
-✓ built in 103ms
+✓ built in 125ms

package/.turbo/turbo-build$colon$esm.log

@@ -34,21 +34,22 @@ ViteEsmConfig: .
 vite v8.1.6 building ssr environment for production...
 Bundle build started
 
-✓ 13 modules transformed.
+✓ 14 modules transformed.
 rendering chunks...
 computing gzip size...
 dist/esm/_virtual/_rolldown/runtime.js    0.04 kB │ gzip:  0.06 kB
-dist/esm/base/index.js                    0.14 kB │ gzip:  0.08 kB
+dist/esm/base/index.js                    0.25 kB │ gzip:  0.12 kB
 dist/esm/base/mongoMemory.js              0.60 kB │ gzip:  0.30 kB
 dist/esm/databases/index.js               0.77 kB │ gzip:  0.27 kB
-dist/esm/index.js                         1.07 kB │ gzip:  0.38 kB
+dist/esm/index.js                         1.19 kB │ gzip:  0.42 kB
 dist/esm/databases/clientRecordCache.js   1.40 kB │ gzip:  0.68 kB
 dist/esm/databases/client.js              1.85 kB │ gzip:  0.80 kB
 dist/esm/mongooseOptions.js               3.58 kB │ gzip:  1.53 kB
-dist/esm/base/mongo.js                    4.04 kB │ gzip:  1.33 kB
+dist/esm/base/mongo.js                    4.22 kB │ gzip:  1.40 kB
+dist/esm/base/reconcileIndexes.js         5.93 kB │ gzip:  1.97 kB
 dist/esm/databases/centralDbStreamer.js   6.33 kB │ gzip:  1.51 kB
-dist/esm/databases/captcha.js             8.17 kB │ gzip:  2.38 kB
+dist/esm/databases/captcha.js             8.29 kB │ gzip:  2.40 kB
 dist/esm/redisCache.js                   14.25 kB │ gzip:  4.43 kB
-dist/esm/databases/provider.js           69.27 kB │ gzip: 13.58 kB
+dist/esm/databases/provider.js           70.50 kB │ gzip: 13.97 kB
 
-✓ built in 96ms
+✓ built in 121ms

package/.turbo/turbo-build$colon$tsc.log

@@ -1,5 +1,5 @@
 $ tsc --build --verbose
-3:55:59 PM - Projects in this build: 
+4:57:07 PM - Projects in this build: 
     * ../../dev/config/tsconfig.json
     * ../util/tsconfig.json
     * ../locale/tsconfig.json
@@ -14,31 +14,31 @@ $ tsc --build --verbose
     * ../types-database/tsconfig.json
     * tsconfig.json
 
-3:55:59 PM - Project '../../dev/config/tsconfig.json' is up to date because newest input '../../dev/config/src/webpack/webpack.config.ts' is older than output '../../dev/config/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../../dev/config/tsconfig.json' is up to date because newest input '../../dev/config/src/webpack/webpack.config.ts' is older than output '../../dev/config/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../util/tsconfig.json' is up to date because newest input '../util/src/url.ts' is older than output '../util/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../util/tsconfig.json' is up to date because newest input '../util/src/url.ts' is older than output '../util/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../locale/tsconfig.json' is up to date because newest input '../locale/src/translationKey.ts' is older than output '../locale/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../locale/tsconfig.json' is up to date because newest input '../locale/src/translationKey.ts' is older than output '../locale/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../util-crypto/tsconfig.json' is up to date because newest input '../util-crypto/src/types.ts' is older than output '../util-crypto/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../util-crypto/tsconfig.json' is up to date because newest input '../util-crypto/src/types.ts' is older than output '../util-crypto/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../types/tsconfig.json' is up to date because newest input '../types/src/types.test-d.ts' is older than output '../types/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../types/tsconfig.json' is up to date because newest input '../types/src/types.test-d.ts' is older than output '../types/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../common/tsconfig.json' is up to date because newest input '../common/src/tests/utils/objectUtils.unit.test.ts' is older than output '../common/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../common/tsconfig.json' is up to date because newest input '../common/src/tests/errorSites.unit.test.ts' is older than output '../common/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../logger/tsconfig.json' is up to date because newest input '../logger/src/index.ts' is older than output '../logger/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../logger/tsconfig.json' is up to date because newest input '../logger/src/logger.ts' is older than output '../logger/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../api-route/tsconfig.json' is up to date because newest input '../api-route/src/apiRoutes.ts' is older than output '../api-route/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../api-route/tsconfig.json' is up to date because newest input '../api-route/src/apiRoutes.ts' is older than output '../api-route/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../redis-client/tsconfig.json' is up to date because newest input '../redis-client/src/index.ts' is older than output '../redis-client/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../redis-client/tsconfig.json' is up to date because newest input '../redis-client/src/tests/redisIndex.integration.test.ts' is older than output '../redis-client/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../api/tsconfig.json' is up to date because newest input '../api/src/index.ts' is older than output '../api/dist/tsc/tsconfig.tsbuildinfo'
+4:57:07 PM - Project '../api/tsconfig.json' is up to date because newest input '../api/src/index.ts' is older than output '../api/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../user-access-policy/tsconfig.json' is up to date because newest input '../user-access-policy/src/ruleRecord.ts' is older than output '../user-access-policy/dist/tsc/tsconfig.tsbuildinfo'
+4:57:08 PM - Project '../user-access-policy/tsconfig.json' is up to date because newest input '../user-access-policy/src/transformRule.ts' is older than output '../user-access-policy/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project '../types-database/tsconfig.json' is up to date because newest input '../types-database/src/types/captcha.ts' is older than output '../types-database/dist/tsc/tsconfig.tsbuildinfo'
+4:57:08 PM - Project '../types-database/tsconfig.json' is up to date because newest input '../types-database/src/types/captcha.ts' is older than output '../types-database/dist/tsc/tsconfig.tsbuildinfo'
 
-3:55:59 PM - Project 'tsconfig.json' is out of date because output file 'dist/tsc/tsconfig.tsbuildinfo' does not exist
+4:57:08 PM - Project 'tsconfig.json' is out of date because output file 'dist/tsc/tsconfig.tsbuildinfo' does not exist
 
-3:55:59 PM - Building project '/home/runner/work/captcha/captcha/packages/database/tsconfig.json'...
+4:57:08 PM - Building project '/home/runner/work/captcha/captcha/packages/database/tsconfig.json'...
 

package/CHANGELOG.md

@@ -1,5 +1,49 @@
 # @prosopo/database
 
+## 4.6.1
+### Patch Changes
+
+- 007f43c: A puzzle served after a wrong answer is now judged on the device the failed solve showed, and an escalation's solve readings are no longer dropped.
+  
+  - After a wrong puzzle answer, the widget names the failed challenge in a new `retryOf` field on the next `/frictionless` request. `BotDetectionFunction` and `ProviderApi.getFrictionlessCaptcha` take it as a new trailing argument.
+  - The provider carries the failed solve's device readings onto the new session only when its own record of that puzzle shows the same client: same site, user account and exact IP address, identical user agent and TLS fingerprint, and a solve that was submitted and failed. The readings come from that solve's own behavioural payload, and each failed puzzle can be followed once (`claimPuzzleCaptchaRetry`). Anything else is ignored.
+  - `createSession` now persists `solveDevice`. It was missing from the session's field list, so a PoW escalation's solve readings never reached the challenge they were meant to size.
+- ec54f21: The scheduled spam email domain sync now only writes domains that are not already stored. It used to upsert the whole list (about 225k domains) on every run, which rewrote every row and held Mongo write locks for around 8 seconds per batch on every provider at the same time, slowing requests on the hour. It now reads the stored domains with an index-only scan and upserts just the new ones.
+  
+  `bulkUpdateSpamEmailDomains` on the provider database is replaced by `insertMissingSpamEmailDomains(domains)`, which returns how many domains were added. The task records that count as `domainsAdded` alongside `domainsProcessed`. Domains that drop off the source lists are still kept, as before.
+- Updated dependencies [b80b627]
+- Updated dependencies [007f43c]
+- Updated dependencies [ec54f21]
+  - @prosopo/types@5.25.0
+  - @prosopo/types-database@5.17.0
+  - @prosopo/common@3.3.2
+  - @prosopo/user-access-policy@3.15.17
+
+## 4.6.0
+### Minor Changes
+
+- c12d199: Add `reconcileIndexes`, which brings a collection's indexes in line with its schema. Mongoose's own `ensureIndexes`/autoIndex cannot change an existing index: when a declared index's key or options change (for example `pendingStage_partial` gaining `_id`, or a field becoming unique) it logs an error and leaves the old index in place, so the new one never appears. `reconcileIndexes` drops and rebuilds only those clashing indexes, puts the old one back if the rebuild fails (for example a new unique index meeting duplicates), can drop named obsolete indexes, can keep extra indexes declared outside the schema, and has a dry-run mode.
+  
+  The provider's `ensureIndexes` now uses it on its own database (dropping indexes the schema no longer declares). The central captcha database uses it too, but keeps indexes it does not know about, since other services add their own there.
+
+### Patch Changes
+
+- f31105a: Mongo connection URLs in logs now hide the username and password properly. The old redaction replaced the first `word:word` it found, so a password containing a `-` (which every password `generateSelfHostedEnv.ts` creates can) was logged in full or in part, with the host and port hidden instead. URLs without credentials no longer lose their host and port either.
+- Updated dependencies [91ef450]
+- Updated dependencies [7c58be4]
+- Updated dependencies [cdea91f]
+- Updated dependencies [f679760]
+- Updated dependencies [cf9a1bb]
+- Updated dependencies [cdea91f]
+- Updated dependencies [ec4ed10]
+  - @prosopo/types@5.24.0
+  - @prosopo/logger@2.1.8
+  - @prosopo/user-access-policy@3.15.16
+  - @prosopo/types-database@5.16.2
+  - @prosopo/util@3.5.0
+  - @prosopo/common@3.3.2
+  - @prosopo/redis-client@1.0.45
+
 ## 4.5.1
 ### Patch Changes
 

package/dist/cjs/base/index.cjs

@@ -1,4 +1,7 @@
 const require_mongo = require("./mongo.cjs");
 const require_mongoMemory = require("./mongoMemory.cjs");
+const require_reconcileIndexes = require("./reconcileIndexes.cjs");
 exports.MongoDatabase = require_mongo.MongoDatabase;
 exports.MongoMemoryDatabase = require_mongoMemory.MongoMemoryDatabase;
+exports.defaultIndexName = require_reconcileIndexes.defaultIndexName;
+exports.reconcileIndexes = require_reconcileIndexes.reconcileIndexes;

package/dist/cjs/base/mongo.cjs

@@ -8,6 +8,10 @@ mongoose = require_runtime.__toESM(mongoose, 1);
 //#region src/base/mongo.ts
 mongoose.default.set("strictQuery", false);
 var DEFAULT_ENDPOINT = "mongodb://127.0.0.1:27017";
+var redactCredentials = (url) => {
+	const credentials = url.username || url.password ? "<Credentials>@" : "";
+	return `${url.protocol}//${credentials}${url.host}${url.pathname}${url.search}`;
+};
 /**
 * Returns a generic Mongo database layer
 * @param {string} url          The database endpoint
@@ -23,7 +27,7 @@ var MongoDatabase = class {
 		if (dbname) parsedUrl.pathname = dbname;
 		if (authSource) parsedUrl.searchParams.set("authSource", authSource);
 		this._url = parsedUrl.toString();
-		this.safeURL = this.url.replace(/\w+:\w+/, "<Credentials>");
+		this.safeURL = redactCredentials(parsedUrl);
 		this.dbname = dbname || parsedUrl.pathname.replace("/", "");
 		this.logger = (logger || (0, _prosopo_logger.getLogger)("info", "database:mongo")).with({ mongoUrl: this.safeURL });
 	}

package/dist/cjs/base/reconcileIndexes.cjs

@@ -0,0 +1,162 @@
+//#region src/base/reconcileIndexes.ts
+var defaultIndexName = (key) => Object.entries(key).map(([field, direction]) => `${field}_${direction}`).join("_");
+var toDriverKey = (key) => Object.fromEntries(Object.entries(key).map(([field, direction]) => [field, direction === "ascending" || direction === "asc" ? 1 : direction === "descending" || direction === "desc" ? -1 : direction]));
+var keyOf = (index) => index.key instanceof Map ? Object.fromEntries(index.key) : index.key;
+var sameKey = (a, b) => {
+	const left = Object.entries(a);
+	const right = Object.entries(b);
+	return left.length === right.length && left.every(([field, direction], i) => right[i]?.[0] === field && String(right[i]?.[1]) === String(direction));
+};
+var listIndexesOrEmpty = async (model) => {
+	try {
+		return await model.listIndexes();
+	} catch (err) {
+		if (err.codeName === "NamespaceNotFound") return [];
+		throw err;
+	}
+};
+var toDriverOptions = ({ name, options: { unique, ...options } }) => ({
+	...options,
+	...unique !== void 0 && { unique: Array.isArray(unique) ? unique[0] : unique },
+	name
+});
+var errorMessage = (err) => err instanceof Error ? err.message : String(err);
+var specOf = (index) => {
+	const { key, v: _v, ns: _ns, ...options } = index;
+	return {
+		key,
+		options
+	};
+};
+var BRIDGE_SUFFIX = "_reconcile_bridge";
+var BRIDGE_FIELD = "__reconcileBridge";
+var bridgeOf = (index) => {
+	const { options } = specOf(index);
+	const { expireAfterSeconds: _ttl, ...bridgeOptions } = options;
+	return {
+		key: {
+			...index.key,
+			[BRIDGE_FIELD]: 1
+		},
+		options: {
+			...bridgeOptions,
+			name: `${index.name}${BRIDGE_SUFFIX}`
+		}
+	};
+};
+var dropIfPresent = async (model, name) => {
+	try {
+		await model.collection.dropIndex(name);
+	} catch (err) {
+		if (err.codeName !== "IndexNotFound") throw err;
+	}
+};
+/**
+* Brings a collection's indexes in line with its schema without the window of
+* a drop-everything-then-rebuild. Indexes that already match are left alone;
+* an existing index that clashes with a declared one (same name or same key,
+* different key or options) is rebuilt, because `createIndex` refuses to change
+* an index in place and mongoose's autoIndex only logs that refusal. A bridge
+* index is built before the old one is dropped, so no query loses its index
+* during the rebuild. If a rebuild fails (e.g. a new unique index meets
+* duplicates) the index it replaced is restored, so a failed run never leaves
+* a collection with less than it started with.
+*/
+var reconcileIndexes = async (model, options = {}) => {
+	const result = {
+		collection: model.collection.collectionName,
+		dropped: [],
+		created: [],
+		restored: [],
+		failed: []
+	};
+	const existing = await listIndexesOrEmpty(model);
+	const diff = await model.diffIndexes({ indexOptionsToCreate: true });
+	const toCreate = diff.toCreate;
+	const undeclared = diff.toDrop;
+	const wanted = toCreate.map(([key, indexOptions]) => ({
+		key,
+		options: indexOptions,
+		name: indexOptions.name ?? defaultIndexName(key)
+	}));
+	const extra = options.extra ?? [];
+	const isExtra = (index) => extra.some((e) => sameKey(keyOf(e), index.key));
+	const missingExtra = extra.filter((e) => !existing.some((index) => sameKey(keyOf(e), index.key)) && !wanted.some((w) => sameKey(keyOf(e), w.key)));
+	const obsolete = new Set(options.obsolete ?? []);
+	const clashes = (index, w) => w.name === index.name || sameKey(w.key, index.key);
+	const toDrop = existing.filter((index) => index.name !== void 0 && undeclared.includes(index.name) && !isExtra(index) && (options.dropUndeclared || obsolete.has(index.name) || wanted.some((w) => clashes(index, w))));
+	const extraName = (e) => e.name ?? defaultIndexName(keyOf(e));
+	if (options.dryRun) {
+		result.dropped = toDrop.map((index) => index.name ?? "");
+		result.created = [...wanted.map((w) => w.name), ...missingExtra.map(extraName)];
+		return result;
+	}
+	for (const w of wanted) {
+		const replaced = toDrop.filter((index) => clashes(index, w) && !result.dropped.includes(index.name ?? ""));
+		const bridges = [];
+		try {
+			for (const index of replaced) {
+				const { key, options: bridgeOptions } = bridgeOf(index);
+				await model.collection.createIndex(key, bridgeOptions);
+				bridges.push(bridgeOptions.name ?? "");
+			}
+		} catch (err) {
+			result.failed.push({
+				name: w.name,
+				error: `bridge failed: ${errorMessage(err)}`
+			});
+			for (const bridge of bridges) await dropIfPresent(model, bridge);
+			continue;
+		}
+		for (const index of replaced) {
+			const name = index.name ?? "";
+			await dropIfPresent(model, name);
+			result.dropped.push(name);
+		}
+		let restoreFailed = false;
+		try {
+			await model.collection.createIndex(toDriverKey(w.key), toDriverOptions(w));
+			result.created.push(w.name);
+		} catch (err) {
+			result.failed.push({
+				name: w.name,
+				error: errorMessage(err)
+			});
+			for (const index of replaced) {
+				const { key, options: replacedOptions } = specOf(index);
+				try {
+					await model.collection.createIndex(key, replacedOptions);
+					result.restored.push(index.name ?? "");
+				} catch (restoreErr) {
+					restoreFailed = true;
+					result.failed.push({
+						name: index.name ?? "",
+						error: `restore failed: ${errorMessage(restoreErr)}`
+					});
+				}
+			}
+		}
+		if (!restoreFailed) for (const bridge of bridges) await dropIfPresent(model, bridge);
+	}
+	for (const e of missingExtra) try {
+		await model.collection.createIndexes([e]);
+		result.created.push(extraName(e));
+	} catch (err) {
+		result.failed.push({
+			name: extraName(e),
+			error: errorMessage(err)
+		});
+	}
+	for (const index of toDrop) {
+		const name = index.name ?? "";
+		if (!result.dropped.includes(name) && !wanted.some((w) => clashes(index, w))) {
+			await dropIfPresent(model, name);
+			result.dropped.push(name);
+		}
+	}
+	for (const index of existing) if (index.name?.endsWith(BRIDGE_SUFFIX)) await dropIfPresent(model, index.name);
+	return result;
+};
+//#endregion
+exports.defaultIndexName = defaultIndexName;
+exports.reconcileIndexes = reconcileIndexes;

package/dist/cjs/databases/captcha.cjs

@@ -1,4 +1,5 @@
 const require_mongo = require("../base/mongo.cjs");
+const require_reconcileIndexes = require("../base/reconcileIndexes.cjs");
 let _prosopo_common = require("@prosopo/common");
 let _prosopo_logger = require("@prosopo/logger");
 let _prosopo_types_database = require("@prosopo/types-database");
@@ -60,27 +61,25 @@ var CaptchaDatabase = class CaptchaDatabase extends require_mongo.MongoDatabase
 		return this.tables;
 	}
 	async ensureIndexes() {
-		const indexPromises = [];
-		if (!this.indexesEnsured) CAPTCHA_TABLES.map(({ collectionName }) => {
-			indexPromises.push(new Promise((resolve) => {
-				if (this.connected) this.tables[collectionName].collection.dropIndexes().then(() => {
-					this.tables[collectionName].ensureIndexes().then(() => {
-						resolve();
-					}).catch((err) => {
-						this.logger.warn(() => ({
-							err,
-							msg: `Error creating indexes for collection ${collectionName}`
-						}));
-						resolve();
-					});
-				});
-				else {
-					this.logger.info(() => ({ msg: `Skipping index creation for collection ${collectionName} as not connected` }));
-					resolve();
-				}
-			}));
-		});
-		await Promise.all(indexPromises);
+		if (!this.indexesEnsured && this.connected) await Promise.all(CAPTCHA_TABLES.map(async ({ collectionName }) => {
+			try {
+				const result = await require_reconcileIndexes.reconcileIndexes(this.tables[collectionName], { dropUndeclared: false });
+				if (result.failed.length > 0) this.logger.warn(() => ({
+					msg: `Error creating indexes for collection ${collectionName}`,
+					data: result
+				}));
+				else this.logger.info(() => ({
+					msg: `Indexes ensured for collection ${collectionName}`,
+					data: result
+				}));
+			} catch (err) {
+				this.logger.warn(() => ({
+					err,
+					msg: `Error creating indexes for collection ${collectionName}`
+				}));
+			}
+		}));
+		else if (!this.connected) this.logger.info(() => ({ msg: "Skipping index creation as not connected" }));
 		this.indexesEnsured = true;
 	}
 	/**

package/dist/cjs/databases/provider.cjs

@@ -1,4 +1,5 @@
 const require_mongo = require("../base/mongo.cjs");
+const require_reconcileIndexes = require("../base/reconcileIndexes.cjs");
 const require_clientRecordCache = require("./clientRecordCache.cjs");
 let _prosopo_common = require("@prosopo/common");
 let _prosopo_types = require("@prosopo/types");
@@ -147,28 +148,25 @@ var ProviderDatabase = class ProviderDatabase extends require_mongo.MongoDatabas
 		return this.redisConnection;
 	}
 	async ensureIndexes() {
-		const indexPromises = [];
-		if (!this.indexesEnsured) PROVIDER_TABLES.map(({ collectionName }) => {
-			indexPromises.push(new Promise((resolve) => {
-				if (this.connected) this.tables[collectionName].collection.dropIndexes().then(() => {
-					this.tables[collectionName].ensureIndexes().then(() => {
-						this.logger.info(() => ({ msg: `Indexes ensured for collection ${collectionName}` }));
-						resolve();
-					}).catch((err) => {
-						this.logger.warn(() => ({
-							err,
-							msg: `Error creating indexes for collection ${collectionName}`
-						}));
-						resolve();
-					});
-				});
-				else {
-					this.logger.info(() => ({ msg: `Skipping index creation for collection ${collectionName} as not connected` }));
-					resolve();
-				}
-			}));
-		});
-		await Promise.all(indexPromises);
+		if (!this.indexesEnsured && this.connected) await Promise.all(PROVIDER_TABLES.map(async ({ collectionName }) => {
+			try {
+				const result = await require_reconcileIndexes.reconcileIndexes(this.tables[collectionName], { dropUndeclared: true });
+				if (result.failed.length > 0) this.logger.warn(() => ({
+					msg: `Error creating indexes for collection ${collectionName}`,
+					data: result
+				}));
+				else this.logger.info(() => ({
+					msg: `Indexes ensured for collection ${collectionName}`,
+					data: result
+				}));
+			} catch (err) {
+				this.logger.warn(() => ({
+					err,
+					msg: `Error creating indexes for collection ${collectionName}`
+				}));
+			}
+		}));
+		else if (!this.connected) this.logger.info(() => ({ msg: "Skipping index creation as not connected" }));
 		this.indexesEnsured = true;
 	}
 	getUserAccessRulesStorage() {
@@ -1054,6 +1052,19 @@ var ProviderDatabase = class ProviderDatabase extends require_mongo.MongoDatabas
 	async updatePuzzleCaptchaRecord(challenge, updates) {
 		await this.updateInteractiveCaptchaRecord(this.interactiveCollection(_prosopo_types.CaptchaType.puzzle), challenge, updates);
 	}
+	/**
+	* Take a failed puzzle for one retry. Succeeds once per puzzle, and only
+	* for a puzzle that was submitted and disapproved.
+	* @returns true when this call took it
+	*/
+	async claimPuzzleCaptchaRetry(challenge) {
+		return (await this.getTables()[this.interactiveCollection(_prosopo_types.CaptchaType.puzzle).table].updateOne({
+			challenge,
+			userSubmitted: true,
+			"result.status": _prosopo_types.CaptchaStatus.disapproved,
+			retried: { $ne: true }
+		}, { $set: { retried: true } })).modifiedCount === 1;
+	}
 	async updateIconOrderCaptchaRecord(challenge, updates) {
 		await this.updateInteractiveCaptchaRecord(this.interactiveCollection(_prosopo_types.CaptchaType.iconOrder), challenge, updates);
 	}
@@ -2087,19 +2098,35 @@ var ProviderDatabase = class ProviderDatabase extends require_mongo.MongoDatabas
 		const query = suffixCandidates.length > 0 ? { domain: { $in: suffixCandidates } } : { domain };
 		return await this.tables.spamEmailDomain.findOne(query).exec();
 	}
-	async bulkUpdateSpamEmailDomains(domains, upsert) {
+	/**
+	* Store any of `domains` not already in the collection and return how many
+	* were added. The source lists hold ~225k domains and barely change between
+	* runs, so upserting all of them rewrote every row and held write locks for
+	* ~8s per batch on each provider at the same moment. Reading the existing
+	* set first is an index-only scan, and only the difference is written.
+	*/
+	async insertMissingSpamEmailDomains(domains) {
 		if (!this.tables?.spamEmailDomain) throw new _prosopo_common.ProsopoError({
 			code: "DB_QUERY_FAILED",
 			errorSite: "DBSE-51",
 			message: "Failed to import database engine",
-			context: { failedFuncName: this.bulkUpdateSpamEmailDomains.name }
+			context: { failedFuncName: this.insertMissingSpamEmailDomains.name }
 		});
-		const bulkOps = domains.map((op) => ({ updateOne: {
-			filter: op.filter,
-			update: { $set: op.update },
-			upsert
+		const missing = new Set(domains);
+		if (missing.size === 0) return 0;
+		const existing = this.tables.spamEmailDomain.find({}, {
+			domain: 1,
+			_id: 0
+		}).hint({ domain: 1 }).lean().cursor();
+		for await (const record of existing) missing.delete(record.domain);
+		if (missing.size === 0) return 0;
+		const bulkOps = Array.from(missing, (domain) => ({ updateOne: {
+			filter: { domain },
+			update: { $set: { domain } },
+			upsert: true
 		} }));
-		if (bulkOps.length > 0) await this.tables.spamEmailDomain.bulkWrite(bulkOps);
+		await this.tables.spamEmailDomain.bulkWrite(bulkOps, { ordered: false });
+		return missing.size;
 	}
 };
 //#endregion

package/dist/cjs/index.cjs

@@ -2,6 +2,7 @@ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const require_mongooseOptions = require("./mongooseOptions.cjs");
 const require_mongo = require("./base/mongo.cjs");
 const require_mongoMemory = require("./base/mongoMemory.cjs");
+const require_reconcileIndexes = require("./base/reconcileIndexes.cjs");
 const require_captcha = require("./databases/captcha.cjs");
 const require_client = require("./databases/client.cjs");
 const require_provider = require("./databases/provider.cjs");
@@ -23,5 +24,7 @@ exports.MongoDatabase = require_mongo.MongoDatabase;
 exports.MongoMemoryDatabase = require_mongoMemory.MongoMemoryDatabase;
 exports.ProviderDatabase = require_provider.ProviderDatabase;
 exports.RedisWriteQueue = require_redisCache.RedisWriteQueue;
+exports.defaultIndexName = require_reconcileIndexes.defaultIndexName;
 exports.getMongoCompressors = require_mongooseOptions.getMongoCompressors;
 exports.getMongoConnectionOptions = require_mongooseOptions.getMongoConnectionOptions;
+exports.reconcileIndexes = require_reconcileIndexes.reconcileIndexes;

package/dist/esm/base/index.js

@@ -1,3 +1,4 @@
 import { MongoDatabase } from "./mongo.js";
 import { MongoMemoryDatabase } from "./mongoMemory.js";
-export { MongoDatabase, MongoMemoryDatabase };
+import { defaultIndexName, reconcileIndexes } from "./reconcileIndexes.js";
+export { MongoDatabase, MongoMemoryDatabase, defaultIndexName, reconcileIndexes };

package/dist/esm/base/mongo.js

@@ -6,6 +6,10 @@ import mongoose from "mongoose";
 //#region src/base/mongo.ts
 mongoose.set("strictQuery", false);
 var DEFAULT_ENDPOINT = "mongodb://127.0.0.1:27017";
+var redactCredentials = (url) => {
+	const credentials = url.username || url.password ? "<Credentials>@" : "";
+	return `${url.protocol}//${credentials}${url.host}${url.pathname}${url.search}`;
+};
 /**
 * Returns a generic Mongo database layer
 * @param {string} url          The database endpoint
@@ -21,7 +25,7 @@ var MongoDatabase = class {
 		if (dbname) parsedUrl.pathname = dbname;
 		if (authSource) parsedUrl.searchParams.set("authSource", authSource);
 		this._url = parsedUrl.toString();
-		this.safeURL = this.url.replace(/\w+:\w+/, "<Credentials>");
+		this.safeURL = redactCredentials(parsedUrl);
 		this.dbname = dbname || parsedUrl.pathname.replace("/", "");
 		this.logger = (logger || getLogger("info", "database:mongo")).with({ mongoUrl: this.safeURL });
 	}