@prosopo/user-access-policy 3.8.1 → 3.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@prosopo/user-access-policy",
-	"version": "3.8.1",
+	"version": "3.9.1",
 	"type": "module",
 	"engines": {
 		"node": "^24",
@@ -43,12 +43,12 @@
 		"test": "npm run test:unit && npm run test:integration"
 	},
 	"dependencies": {
-		"@prosopo/api": "3.4.10",
+		"@prosopo/api": "3.4.11",
 		"@prosopo/api-route": "2.6.46",
 		"@prosopo/common": "3.1.38",
 		"@prosopo/logger": "1.0.2",
 		"@prosopo/redis-client": "1.0.23",
-		"@prosopo/types": "4.4.0",
+		"@prosopo/types": "4.4.1",
 		"@prosopo/util": "3.2.15",
 		"@redis/search": "5.0.0",
 		"cidr-calc": "1.0.4",

package/src/api/read/fetchRules.ts

@@ -36,9 +36,17 @@ export type FetchRulesResponse = {
 	ruleEntries: AccessRuleEntry[];
 };
 
-export const fetchRulesResponse = z.object({
+// Explicit annotation with `unknown` input position rather than the
+// strict identity form because `ruleEntryInput.rule` transitively uses
+// `z.preprocess` on `deferToVerify`. Required for portable declaration
+// emit; the `AllKeys<...>` constraint still catches missing fields.
+export const fetchRulesResponse: ZodType<
+	FetchRulesResponse,
+	z.ZodTypeDef,
+	unknown
+> = z.object({
 	ruleEntries: ruleEntryInput.array(),
-} satisfies AllKeys<FetchRulesResponse>) satisfies ZodType<FetchRulesResponse>;
+} satisfies AllKeys<FetchRulesResponse>);
 
 export type FetchRulesEndpointResponse = ApiEndpointResponse & {
 	data?: FetchRulesResponse;

package/src/api/write/insertRules.ts

@@ -19,7 +19,7 @@ import {
 } from "@prosopo/api-route";
 import type { AllKeys } from "@prosopo/common";
 import { LogLevel, type Logger } from "@prosopo/logger";
-import { type ZodType, z } from "zod";
+import { type ZodType, type ZodTypeDef, z } from "zod";
 import type {
 	AccessPolicy,
 	AccessRule,
@@ -56,7 +56,9 @@ type ParsedInsertRulesGroup = InsertRulesGroup & {
 
 type ParsedInsertRuleGroups = ParsedInsertRulesGroup[];
 
-type InsertRulesSchema = ZodType<InsertRulesGroup[]>;
+// Input position widened to `unknown` because `accessPolicyInput` uses
+// `z.preprocess` on `deferToVerify` for Redis string round-tripping.
+type InsertRulesSchema = ZodType<InsertRulesGroup[], ZodTypeDef, unknown>;
 
 export class InsertRulesEndpoint implements ApiEndpoint<InsertRulesSchema> {
 	public constructor(

package/src/mongoose/mongooseRuleSchema.ts

@@ -56,6 +56,7 @@ const accessPolicySchema: SchemaDefinition<AccessPolicy> = {
 	powDifficulty: { type: Number, required: false },
 	unsolvedImagesCount: { type: Number, required: false },
 	frictionlessScore: { type: Number, required: false },
+	deferToVerify: { type: Boolean, required: false },
 } satisfies AllKeys<AccessPolicy>;
 
 export const accessRuleMongooseSchema: SchemaDefinition<AccessRuleRecord> = {

package/src/redis/reader/redisAggregate.ts

@@ -29,6 +29,7 @@ export const aggregateRedisKeys = async (
 	query: string,
 	logger: Logger,
 	batchHandler?: (keys: string[]) => Promise<void>,
+	maxKeys?: number,
 ): Promise<string[]> => {
 	const keyField = "__key";
 
@@ -38,8 +39,13 @@ export const aggregateRedisKeys = async (
 	});
 
 	const foundKeys: string[] = [];
+	let stopRequested = false;
 
 	const addRecordKeys = async (records: object[]) => {
+		if (stopRequested) {
+			return;
+		}
+
 		const parsedRecords = parseRedisRecords(records, recordSchema, logger);
 
 		const recordKeys = parsedRecords.map((record) => record[keyField]);
@@ -47,6 +53,24 @@ export const aggregateRedisKeys = async (
 		if (batchHandler) {
 			await batchHandler(recordKeys);
 		} else {
+			if (
+				maxKeys !== undefined &&
+				foundKeys.length + recordKeys.length > maxKeys
+			) {
+				const remaining = Math.max(0, maxKeys - foundKeys.length);
+				foundKeys.push(...recordKeys.slice(0, remaining));
+				stopRequested = true;
+
+				logger.warn(() => ({
+					msg: "Redis aggregation candidate cap hit; truncating result set. This can suppress less-frequent rules and should be investigated.",
+					data: {
+						maxKeys,
+						query,
+					},
+				}));
+				return;
+			}
+
 			foundKeys.push(...recordKeys);
 
 			logger.debug(() => ({
@@ -68,6 +92,7 @@ export const aggregateRedisKeys = async (
 			LOAD: `@${keyField}`,
 		},
 		addRecordKeys,
+		() => stopRequested,
 	);
 
 	return foundKeys;
@@ -78,6 +103,7 @@ const executeAggregation = async (
 	query: string,
 	aggregateOptions: FtAggregateWithCursorOptions,
 	handleBatch: (records: object[]) => Promise<void>,
+	shouldStop?: () => boolean,
 ): Promise<void> => {
 	const initialReply = await client.ft.aggregateWithCursor(
 		ACCESS_RULES_REDIS_INDEX_NAME,
@@ -89,7 +115,7 @@ const executeAggregation = async (
 
 	let cursor = initialReply.cursor;
 
-	while (0 !== cursor) {
+	while (0 !== cursor && !shouldStop?.()) {
 		const batchReply = await client.ft.cursorRead(
 			ACCESS_RULES_REDIS_INDEX_NAME,
 			cursor,

package/src/redis/reader/redisRulesReader.ts

@@ -16,20 +16,14 @@ import * as util from "node:util";
 import { chunkIntoBatches, executeBatchesSequentially } from "@prosopo/common";
 import type { Logger } from "@prosopo/logger";
 import type { RedisClientType } from "redis";
-import {
-	REDIS_QUERY_DIALECT,
-	getRulesRedisQuery,
-} from "#policy/redis/reader/redisRulesQuery.js";
+import { getRulesRedisQuery } from "#policy/redis/reader/redisRulesQuery.js";
 import {
 	REDIS_BATCH_SIZE,
 	fetchRedisHashRecords,
 	getMissingRedisKeys,
 	parseRedisRecords,
 } from "#policy/redis/redisClient.js";
-import {
-	ACCESS_RULES_REDIS_INDEX_NAME,
-	ACCESS_RULE_REDIS_KEY_PREFIX,
-} from "#policy/redis/redisRuleIndex.js";
+import { ACCESS_RULE_REDIS_KEY_PREFIX } from "#policy/redis/redisRuleIndex.js";
 import type { AccessRule } from "#policy/rule.js";
 import { accessRuleInput } from "#policy/ruleInput/ruleInput.js";
 import type {
@@ -39,6 +33,15 @@ import type {
 } from "#policy/rulesStorage.js";
 import { aggregateRedisKeys } from "./redisAggregate.js";
 
+// Verify-time lookup safety cap. The greedy `userScopeMatch` OR-query can match
+// thousands of candidates on bot-attack-scale accounts; we still need to bring
+// every candidate back so JS-side specificity ranking can pick the right rule,
+// but a hard cap prevents a pathological match (e.g. an attacker crafting a
+// userScope that hits the entire account's rule index) from spiralling the
+// per-request cost. 10× the page size leaves comfortable headroom over the
+// observed worst case (~2k candidates).
+export const FIND_RULES_MAX_CANDIDATES = REDIS_BATCH_SIZE * 10;
+
 export class RedisRulesReader implements AccessRulesReader {
 	constructor(
 		private readonly client: RedisClientType,
@@ -85,46 +88,45 @@ export class RedisRulesReader implements AccessRulesReader {
 		}
 
 		try {
-			// Use searchNoContent to get document keys only, then fetch data separately.
-			// This avoids a bug in @redis/search where ft.search crashes with
-			// "Cannot read properties of null (reading 'length')" when a document
-			// is deleted/expired between the index scan and data retrieval.
-			const searchReply = await this.client.ft.searchNoContent(
-				ACCESS_RULES_REDIS_INDEX_NAME,
+			// FT.AGGREGATE WITHCURSOR (via aggregateRedisKeys) instead of
+			// FT.SEARCH. FT.SEARCH's LIMIT caps the candidate set at
+			// REDIS_BATCH_SIZE (1000), silently dropping anything past that.
+			// Under the greedy `userScopeMatch` mode (#2657), the OR-of-fields
+			// query for a popular ja4 hash returns thousands of candidates —
+			// matches sitting past offset 1000 (block rules emitted by
+			// less-frequent detectors like SUDDEN_VOLUME_INCREASE) were lost,
+			// so verify-time ranking only saw the high-volume rules and the
+			// most-specific block rule for the request never reached the
+			// JS-side specificity sort.
+			const ruleKeys = await aggregateRedisKeys(
+				this.client,
 				query,
-				{
-					DIALECT: REDIS_QUERY_DIALECT,
-					// FT.search doesn't support "unlimited" selects
-					LIMIT: {
-						from: 0,
-						size: REDIS_BATCH_SIZE,
-					},
-				},
+				this.logger,
+				undefined,
+				FIND_RULES_MAX_CANDIDATES,
 			);
 
-			if (searchReply.total > 0) {
-				this.logger.debug(() => ({
-					msg: "Executed search query",
-					data: {
-						inspect: util.inspect(
-							{
-								filter: filter,
-								searchReply: searchReply,
-								query: query,
-							},
-							{ depth: null },
-						),
-					},
-				}));
-			}
-
-			if (searchReply.documents.length === 0) {
+			if (ruleKeys.length === 0) {
 				return [];
 			}
 
+			this.logger.debug(() => ({
+				msg: "Executed search query",
+				data: {
+					inspect: util.inspect(
+						{
+							filter: filter,
+							foundCount: ruleKeys.length,
+							query: query,
+						},
+						{ depth: null },
+					),
+				},
+			}));
+
 			const { records } = await fetchRedisHashRecords(
 				this.client,
-				searchReply.documents,
+				ruleKeys,
 				this.logger,
 			);
 

package/src/redis/redisClient.ts

@@ -14,7 +14,7 @@
 
 import type { Logger } from "@prosopo/logger";
 import type { RedisClientType } from "redis";
-import { type ZodType, z } from "zod";
+import { type ZodType, type ZodTypeDef, z } from "zod";
 
 export const REDIS_BATCH_SIZE = 1_000;
 
@@ -67,9 +67,14 @@ export const fetchRedisHashRecords = async (
 	};
 };
 
+// `recordSchema` is intentionally typed with `unknown` as the input
+// position. Some schemas use `z.preprocess` (e.g. accessPolicyInput's
+// deferToVerify boolean coercion from Redis strings), which widens the
+// zod input type. The strict `ZodType<T, ZodTypeDef, T>` form rejects
+// those at the call site even though they parse correctly at runtime.
 export const parseRedisRecords = <T>(
 	records: unknown[],
-	recordSchema: ZodType<T>,
+	recordSchema: ZodType<T, ZodTypeDef, unknown>,
 	logger: Logger,
 ): T[] =>
 	records.flatMap((record) => {

package/src/rule.ts

@@ -27,6 +27,18 @@ export type AccessPolicy = {
 	powDifficulty?: number;
 	unsolvedImagesCount?: number;
 	frictionlessScore?: number;
+	// When true, a Block policy does NOT fire at the request-time
+	// blockMiddleware (so the user does not see a 401 on the captcha
+	// challenge endpoint) — it fires at the verify step instead, marking
+	// the commitment ACCESS_POLICY_BLOCK / disapproved. The verify
+	// response returns `{verified:false}` to the dApp's server while the
+	// user-facing widget completes normally. Mirrors the existing
+	// coords-rule deferral pattern: the middleware blanks coords out of
+	// the userScope, so coords rules can only ever be matched in the
+	// verify path; `deferToVerify` is the explicit form for non-coords
+	// signals (ja4, headersHash, etc.) when the operator wants the
+	// attacker to pay the captcha-solving cost before being rejected.
+	deferToVerify?: boolean;
 };
 
 export type PolicyScope = {

package/src/ruleInput/policyInput.ts

@@ -21,6 +21,11 @@ import {
 	type PolicyScope,
 } from "#policy/rule.js";
 
+// `satisfies ZodType<AccessPolicy>` is intentionally omitted: the
+// `deferToVerify` preprocess widens the schema's input type to `unknown`
+// (preprocess accepts anything), which fails the
+// `ZodType<T, ZodTypeDef, T>` identity check. The `AllKeys<AccessPolicy>`
+// constraint still catches missing-field regressions.
 export const accessPolicyInput = z.object({
 	type: z.nativeEnum(AccessPolicyType),
 	captchaType: CaptchaTypeSchema.optional(),
@@ -35,7 +40,13 @@ export const accessPolicyInput = z.object({
 	unsolvedImagesCount: z.coerce.number().optional(),
 	// used to increase the user's score
 	frictionlessScore: z.coerce.number().optional(),
-} satisfies AllKeys<AccessPolicy>) satisfies ZodType<AccessPolicy>;
+	// Skip the request-time block middleware and only fire at verify.
+	// Redis stores booleans as strings — preprocess so "true"/"false"
+	// round-trip to the JS boolean the matcher expects.
+	deferToVerify: z
+		.preprocess((v) => (typeof v === "string" ? v === "true" : v), z.boolean())
+		.optional(),
+} satisfies AllKeys<AccessPolicy>);
 
 // Sanitize block policies by removing captchaType and solvedImagesCount
 export const sanitizeAccessPolicy = (policy: AccessPolicy): AccessPolicy => {

package/src/ruleInput/ruleInput.ts

@@ -48,20 +48,25 @@ const ruleGroupInput = z
 		return ruleGroup;
 	});
 
-export const accessRuleInput: ZodType<AccessRule> = z
+// Explicit `ZodType<…, ZodTypeDef, unknown>` annotation rather than the
+// strict-identity form because `accessPolicyInput.shape.deferToVerify`
+// uses `z.preprocess` which widens the input position to `unknown`. The
+// relaxed annotation is portable for declaration emit; the `transform`
+// pins the OUTPUT to AccessRule.
+export const accessRuleInput: ZodType<AccessRule, z.ZodTypeDef, unknown> = z
 	.object({
 		...accessPolicyInput.shape,
 		...policyScopeInput.shape,
 	})
 	.and(userScopeInput)
 	.and(ruleGroupInput)
-	// transform is used for type safety only - plain "satisfies ZodType<x>" doesn't work after ".and()"
 	.transform((ruleInput: AccessRuleInput): AccessRule => ruleInput);
 
-export const ruleEntryInput = z.object({
-	rule: accessRuleInput,
-	expiresUnixTimestamp: z.coerce.number().optional(),
-} satisfies AllKeys<AccessRuleEntry>) satisfies ZodType<AccessRuleEntry>;
+export const ruleEntryInput: ZodType<AccessRuleEntry, z.ZodTypeDef, unknown> =
+	z.object({
+		rule: accessRuleInput,
+		expiresUnixTimestamp: z.coerce.number().optional(),
+	} satisfies AllKeys<AccessRuleEntry>);
 
 export type AccessRulesFilterInput = AccessRulesFilter & {
 	userScope?: UserScopeInput;

package/src/ruleInput/userScopeInput.ts

@@ -15,7 +15,7 @@
 import crypto from "node:crypto";
 import type { AllKeys } from "@prosopo/common";
 import { getIPAddress } from "@prosopo/util";
-import { Address4 } from "ip-address";
+import { Address4, Address6 } from "ip-address";
 import { type ZodType, z } from "zod";
 import type { UserAttributes, UserIp, UserScope } from "#policy/rule.js";
 import type { UserAttributesRecord, UserIpRecord } from "#policy/ruleRecord.js";
@@ -77,14 +77,14 @@ const userIpInput = z
 
 		// Assuming ipMask is already validated to be a string in CIDR format
 		if ("string" === typeof ipMask) {
-			// Create an Address4 object from the CIDR string.
-			// Address4 automatically understands CIDR notation and represents the entire network range.
-			const ipObject = new Address4(ipMask);
+			// Try IPv4 CIDR first (e.g., 192.168.1.0/24); fall back to IPv6
+			// CIDR (e.g., 2001:db8::/32). Both Address4 and Address6 understand
+			// CIDR notation and expose start/end addresses for the network.
+			const ipObject = Address4.isValid(ipMask)
+				? new Address4(ipMask)
+				: new Address6(ipMask);
 
-			// The minimum IP in the CIDR range is the start address of the network.
 			numericUserIp.numericIpMaskMin = ipObject.startAddress().bigInt();
-
-			// The maximum IP in the CIDR range is the end address of the network.
 			numericUserIp.numericIpMaskMax = ipObject.endAddress().bigInt();
 		}
 

package/src/tests/redis/redisRulesStorage.integration.test.ts

@@ -1115,6 +1115,58 @@ describe("redisAccessRulesStorage", () => {
 			expect(foundAccessRules).toEqual([]);
 		});
 
+		test("returns all matches when the candidate set exceeds the FT.SEARCH page size", async () => {
+			// Regression: under the production traffic profile of a high-volume
+			// bot attack, the greedy `@field:{X} | @field:{Y}` query returns
+			// thousands of candidate rules sharing the dominant ja4 fingerprint.
+			// FT.SEARCH's LIMIT (1000) silently truncated the candidate set,
+			// dropping less-frequent block rules — they never reached the
+			// JS-side specificity sort, so verify let the bot through.
+			// FT.AGGREGATE WITHCURSOR paginates the result and returns all of
+			// them. This test inserts > 1000 rules so the old code would
+			// truncate; the target block rule must still come back.
+			const clientId = getUniqueString();
+			const popularJa4 = `t13d1516h2_${getUniqueString()}`;
+
+			const targetBlockRule: AccessRule = {
+				type: AccessPolicyType.Block,
+				clientId: clientId,
+				ja4Hash: popularJa4,
+				coords: "[[[867,60]]]",
+			};
+
+			// 1500 noise rules sharing the popular ja4 but with distinct coords.
+			// 1500 > REDIS_BATCH_SIZE (1000) — guarantees the targetBlockRule's
+			// FT index position has at least a 1/3 chance of sitting past the
+			// truncation boundary on any given run. With pagination, it must
+			// always come back regardless of indexed order.
+			const noiseRules: AccessRule[] = Array.from({ length: 1500 }, (_, i) => ({
+				type: AccessPolicyType.Restrict,
+				clientId: clientId,
+				ja4Hash: popularJa4,
+				coords: `[[[${i % 1024},${60 + (i % 32)}]]]`,
+				description: `noise-${i}`,
+			}));
+
+			await insertRules([targetBlockRule, ...noiseRules]);
+
+			const indexRecordsCount = await getIndexRecordsCount(indexName);
+			expect(indexRecordsCount).toBe(1501);
+
+			const found = await accessRulesReader.findRules({
+				policyScope: { clientId: clientId },
+				policyScopeMatch: FilterScopeMatch.Greedy,
+				userScope: {
+					ja4Hash: popularJa4,
+					coords: "[[[867,60]]]",
+				},
+				userScopeMatch: FilterScopeMatch.Greedy,
+			});
+
+			expect(found.length).toBe(1501);
+			expect(found).toContainEqual(targetBlockRule);
+		}, 60_000);
+
 		test("finds rules with matchingFieldsOnly when only userId is set and all IP fields are missing", async () => {
 			// This is the exact scenario from the production error where the query
 			// contained duplicate ismissing(@numericIpMaskMin) ismissing(@numericIpMaskMax)

package/src/tests/transformRule.unit.test.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 import { CaptchaType } from "@prosopo/types";
-import { Address4 } from "ip-address";
+import { Address4, Address6 } from "ip-address";
 import { describe, expect, it } from "vitest";
 import { AccessPolicyType, type AccessRule } from "#policy/rule.js";
 import type { AccessRuleRecord } from "#policy/ruleRecord.js";
@@ -106,6 +106,7 @@ describe("transformRule", () => {
 		powDifficulty: 1,
 		unsolvedImagesCount: 1,
 		frictionlessScore: 1,
+		deferToVerify: false,
 		headersHash: "headersHash",
 		ja4Hash: "js4Hash",
 		clientId: "client",
@@ -186,6 +187,32 @@ describe("transformRule", () => {
 			} as unknown as AccessRule),
 		).toThrow();
 	});
+
+	it("should round-trip an IPv6 address and CIDR mask", () => {
+		const ipv6 = "2001:db8::1";
+		const ipv6Cidr = "2001:db8::/32";
+
+		const expectedNumericIp = new Address6(ipv6).bigInt();
+		const expectedMaskMin = new Address6(ipv6Cidr).startAddress().bigInt();
+		const expectedMaskMax = new Address6(ipv6Cidr).endAddress().bigInt();
+
+		const ruleRecord: AccessRuleRecord = {
+			type: AccessPolicyType.Restrict,
+			ip: ipv6,
+			ipMask: ipv6Cidr,
+		};
+
+		const accessRule = transformAccessRuleRecordIntoRule(ruleRecord);
+
+		expect(accessRule.numericIp).toEqual(expectedNumericIp);
+		expect(accessRule.numericIpMaskMin).toEqual(expectedMaskMin);
+		expect(accessRule.numericIpMaskMax).toEqual(expectedMaskMax);
+
+		const reconstructed = transformAccessRuleIntoRecord(accessRule);
+
+		expect(reconstructed.ip).toEqual(new Address6(ipv6).correctForm());
+		expect(reconstructed.ipMask).toEqual(ipv6Cidr);
+	});
 });
 
 describe("getCidrFromNumericIpRange", () => {
@@ -267,4 +294,44 @@ describe("getCidrFromNumericIpRange", () => {
 
 		expect(cird).toEqual(cirdExample.cidr);
 	});
+
+	type Ipv6CidrExample = {
+		cidr: string;
+		startIp: string;
+		endIp: string;
+		description: string;
+	};
+
+	const ipv6CirdsSet: Ipv6CidrExample[] = [
+		{
+			cidr: "2001:db8::/32",
+			startIp: "2001:db8::",
+			endIp: "2001:db8:ffff:ffff:ffff:ffff:ffff:ffff",
+			description: "/32 IPv6 network",
+		},
+		{
+			cidr: "2001:db8::/64",
+			startIp: "2001:db8::",
+			endIp: "2001:db8::ffff:ffff:ffff:ffff",
+			description: "/64 IPv6 subnet",
+		},
+		{
+			cidr: "fe80::/10",
+			startIp: "fe80::",
+			endIp: "febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
+			description: "/10 IPv6 link-local",
+		},
+	];
+
+	it.each(ipv6CirdsSet)(
+		"should convert $description to $cidr",
+		(cirdExample) => {
+			const cidr = getCidrFromNumericIpRange(
+				new Address6(cirdExample.startIp).bigInt(),
+				new Address6(cirdExample.endIp).bigInt(),
+			);
+
+			expect(cidr).toEqual(cirdExample.cidr);
+		},
+	);
 });

package/src/transformRule.ts

@@ -14,7 +14,7 @@
 
 import crypto from "node:crypto";
 import { IpAddress, IpRange } from "cidr-calc";
-import { Address4 } from "ip-address";
+import { Address4, Address6 } from "ip-address";
 import { z } from "zod";
 import type { AccessRule } from "./rule.js";
 import {
@@ -27,6 +27,9 @@ import type { AccessRuleRecord } from "./ruleRecord.js";
 
 const RULE_HASH_ALGORITHM = "md5";
 
+// IPv4 numeric values occupy 0..2^32-1. Anything above is treated as IPv6.
+const MAX_IPV4_NUMERIC = (1n << 32n) - 1n;
+
 export const makeAccessRuleHash = (rule: AccessRule): string => {
 	// 1. exclude "undefined" values to ensure hash consistency
 	const valueProperties = Object.entries(rule).filter(
@@ -111,7 +114,9 @@ const hashObject = (
 		.digest("hex");
 
 const getStringIpFromNumeric = (numericIp: bigint): string =>
-	Address4.fromInteger(Number(numericIp)).address;
+	numericIp > MAX_IPV4_NUMERIC
+		? Address6.fromBigInt(numericIp).correctForm()
+		: Address4.fromInteger(Number(numericIp)).address;
 
 export const getCidrFromNumericIpRange = (
 	startIp: bigint,