@prosopo/user-access-policy 3.12.27 → 3.12.29

package/src/redis/reader/redisRulesSplitQuery.ts

@@ -12,11 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import {
-	AccessPolicyType,
-	GLOBAL_CLIENT_SCOPE_SENTINEL,
-	type UserScope,
-} from "#policy/rule.js";
+import { getBlockPoolClause } from "#policy/redis/reader/redisRulesQuery.js";
+import { GLOBAL_CLIENT_SCOPE_SENTINEL, type UserScope } from "#policy/rule.js";
 
 // Escapes special characters in Redis TAG queries. Mirrors the escape
 // function in redisRulesQuery.ts — kept local to avoid coupling the two
@@ -113,6 +110,9 @@ type SubQuery = {
  *
  * `blockOnly` narrows the candidate set to rules with `type:{block}` —
  * used by the hard-block middleware which never needs Restrict rules.
+ * `includeDeferred` widens that pool back out to
+ * `(@type:{block} | @deferToVerify:{true})` for the verify-time
+ * hard-block lookup, where a deferred Restrict is a valid block.
  * Every other caller passes false so both Block and Restrict rules are
  * fetched; the JS-side ranker picks the right one via specificity +
  * severity. Merging the block and restrict paths behind one split-query
@@ -124,14 +124,51 @@ type SubQuery = {
 export const buildScopedRulesSubQueries = (
 	userScope: UserScope,
 	clientId: string | undefined,
-	options: { blockOnly?: boolean } = {},
+	options: { blockOnly?: boolean; includeDeferred?: boolean } = {},
 ): SubQuery[] => {
-	const typeClause = options.blockOnly
-		? `@type:{${AccessPolicyType.Block}} `
-		: "";
 	const scopeClause = buildScopeClause(clientId);
-	const prefix = `${typeClause}${scopeClause}`;
 
+	// `includeDeferred` emits the probe set twice — once for plain hard
+	// blocks, once for deferred rules — rather than widening one type
+	// clause to `(@type:{block} | @deferToVerify:{true})`.
+	//
+	// Widening merges both populations into a single probe, so they share
+	// one SPLIT_MAX_CANDIDATES_PER_SUB budget and a dense deferred cohort
+	// can truncate hard blocks out of the candidate set. Measured on a
+	// scope with 400 block + 400 deferred rules on one ja4Hash: the merged
+	// probe returns 800 against a cap of 500, so 300 candidates are
+	// dropped with no ordering guarantee about which. That is precisely
+	// the crowding `blockOnly` was introduced to prevent.
+	//
+	// Two disjoint probe sets each get their own budget, so a hard block
+	// is never displaced by a deferred rule. The pools are disjoint by
+	// construction (the block probe excludes deferred), so the union
+	// double-counts nothing.
+	// Probe `kind` is only namespaced when there are genuinely two pools;
+	// single-pool callers keep the original unprefixed labels.
+	const typePools: Array<{ tag: string; clause: string }> = options.blockOnly
+		? options.includeDeferred === true
+			? [
+					{ tag: "block", clause: `${getBlockPoolClause(false)} ` },
+					{ tag: "deferred", clause: "@deferToVerify:{true} " },
+				]
+			: [{ tag: "", clause: `${getBlockPoolClause(false)} ` }]
+		: [{ tag: "", clause: "" }];
+
+	const subQueries: SubQuery[] = [];
+	for (const pool of typePools) {
+		const prefix = `${pool.clause}${scopeClause}`;
+		const label = pool.tag === "" ? "" : `${pool.tag}:`;
+		subQueries.push(...buildProbes(userScope, prefix, label));
+	}
+	return subQueries;
+};
+
+const buildProbes = (
+	userScope: UserScope,
+	prefix: string,
+	label: string,
+): SubQuery[] => {
 	const subQueries: SubQuery[] = [];
 
 	// One probe per populated scalar user-scope field. Each uses that
@@ -143,7 +180,7 @@ export const buildScopedRulesSubQueries = (
 			continue;
 		}
 		subQueries.push({
-			kind: `field:${field}`,
+			kind: `${label}field:${field}`,
 			query: `${prefix} ${clause}`,
 		});
 	}
@@ -158,11 +195,11 @@ export const buildScopedRulesSubQueries = (
 	const requestIp = userScope.numericIp;
 	if (requestIp !== undefined) {
 		subQueries.push({
-			kind: "ip:exact",
+			kind: `${label}ip:exact`,
 			query: `${prefix} @numericIp:[${requestIp} ${requestIp}]`,
 		});
 		subQueries.push({
-			kind: "ip:mask",
+			kind: `${label}ip:mask`,
 			query: `${prefix} @numericIpMaskMin:[-inf ${requestIp}] @numericIpMaskMax:[${requestIp} +inf]`,
 		});
 	}
@@ -175,7 +212,7 @@ export const buildScopedRulesSubQueries = (
 		(field) => `ismissing(@${field})`,
 	).join(" ");
 	subQueries.push({
-		kind: "no-user-scope",
+		kind: `${label}no-user-scope`,
 		query: `${prefix} ${noScopeIsmissing}`,
 	});
 

package/src/redis/redisRuleIndex.ts

@@ -73,6 +73,13 @@ export const accessRuleRedisSchema: RediSearchSchema = {
 	// Restrict / routing-Block populations push the hard-block rules out
 	// of the top-N candidate set and the lookup silently misses them.
 	type: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
+	// Indexed so `checkForHardBlock` can widen its Block-only pool to
+	// `(@type:{block} | @deferToVerify:{true})`. Deferred rules are
+	// skipped at request time and enforced at verify, so a deferred
+	// Restrict is a legitimate hard block — without this field in the
+	// index it can't be selected server-side and never fires.
+	// Stored by `getRedisRuleValue` as the string "true"/"false".
+	deferToVerify: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
 } satisfies Keys<AccessRule>;
 
 export const ACCESS_RULES_REDIS_INDEX_NAME = "index:user-access-rules";

package/src/ruleInput/ruleInput.ts

@@ -98,6 +98,7 @@ export const accessRulesFilterInput = z.object({
 		.default(FilterScopeMatch.Exact),
 	groupId: z.string().optional(),
 	blockOnly: z.boolean().optional(),
+	includeDeferred: z.boolean().optional(),
 } satisfies AllKeys<AccessRulesFilterInput>);
 
 export const getAccessRuleFiltersFromInput = (

package/src/rulesStorage.ts

@@ -42,6 +42,15 @@ export type AccessRulesFilter = {
 	 * and the lookup misses them.
 	 */
 	blockOnly?: boolean;
+	/**
+	 * Widen a `blockOnly` pool to also admit deferred rules of any type.
+	 * A `deferToVerify` rule is skipped at request time and enforced at
+	 * verify, so it can legitimately be a Restrict — `blockOnly` alone
+	 * would filter it out of the verify-time lookup and the rule would
+	 * never fire. Only `checkForHardBlock` sets this; the request-time
+	 * middleware keeps the narrow Block-only pool.
+	 */
+	includeDeferred?: boolean;
 };
 
 export type AccessRuleEntry = {

package/src/tests/blockingRuleShapes.unit.test.ts

@@ -0,0 +1,244 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Characterisation tests for the two blocking rule shapes as they pass
+// through the write-side zod types:
+//
+//   1. plain Block            — rejects at request time
+//   2. Block + deferToVerify  — passes request time, rejects at verify
+//
+// These pin down what each shape may carry, and what survives the write
+// path into storage. Written against the schemas the rule-sync API
+// actually validates with, so a change in either direction fails here
+// first rather than as a 400 in production.
+
+import type { Logger } from "@prosopo/logger";
+import { CaptchaType } from "@prosopo/types";
+import { describe, expect, it, vi } from "vitest";
+import { InsertRulesEndpoint } from "#policy/api/write/insertRules.js";
+import { AccessPolicyType } from "#policy/rule.js";
+import {
+	accessPolicyInput,
+	sanitizeAccessPolicy,
+} from "#policy/ruleInput/policyInput.js";
+import type { AccessRulesWriter } from "#policy/rulesStorage.js";
+
+const makeMockLogger = (): Logger =>
+	({
+		trace: vi.fn(),
+		debug: vi.fn(),
+		info: vi.fn(),
+		warn: vi.fn(),
+		error: vi.fn(),
+		fatal: vi.fn(),
+		log: vi.fn(),
+		setLogLevel: vi.fn(),
+		getLogLevel: vi.fn().mockReturnValue("info"),
+		with: vi.fn().mockReturnThis(),
+		getScope: vi.fn().mockReturnValue("test"),
+		getPretty: vi.fn().mockReturnValue(false),
+		setPretty: vi.fn(),
+		getPrintStack: vi.fn().mockReturnValue(false),
+		setPrintStack: vi.fn(),
+		getFormat: vi.fn().mockReturnValue("json"),
+		setFormat: vi.fn(),
+	}) satisfies Logger;
+
+const makeMockWriter = (): AccessRulesWriter =>
+	({
+		insertRules: vi.fn().mockResolvedValue(["id1"]),
+	}) as unknown as AccessRulesWriter;
+
+const getInsertSchema = () =>
+	new InsertRulesEndpoint(
+		makeMockWriter(),
+		makeMockLogger(),
+	).getRequestArgsSchema();
+
+describe("blocking rule shapes: plain Block", () => {
+	it("accepts a bare Block", () => {
+		const result = accessPolicyInput.safeParse({
+			type: AccessPolicyType.Block,
+			description: "bare block",
+		});
+		expect(result.success).toBe(true);
+	});
+
+	it("rejects captchaType and solvedImagesCount together", () => {
+		const result = accessPolicyInput.safeParse({
+			type: AccessPolicyType.Block,
+			captchaType: CaptchaType.image,
+			solvedImagesCount: 2,
+		});
+		expect(result.success).toBe(false);
+		if (!result.success) {
+			expect(result.error.issues.map((i) => i.path.at(-1)).sort()).toEqual([
+				"captchaType",
+				"solvedImagesCount",
+			]);
+		}
+	});
+
+	it("strips both fields on the way into storage", () => {
+		expect(
+			sanitizeAccessPolicy({
+				type: AccessPolicyType.Block,
+				captchaType: CaptchaType.image,
+				solvedImagesCount: 2,
+				description: "legacy block",
+			}),
+		).toEqual({
+			type: AccessPolicyType.Block,
+			description: "legacy block",
+		});
+	});
+});
+
+describe("blocking rule shapes: Block + deferToVerify", () => {
+	it("accepts the flag on its own, and preserves it", () => {
+		const result = accessPolicyInput.safeParse({
+			type: AccessPolicyType.Block,
+			deferToVerify: true,
+			description: "deferred block",
+		});
+		expect(result.success).toBe(true);
+		if (result.success) {
+			expect(result.data.deferToVerify).toBe(true);
+		}
+	});
+
+	// The shape that 400s the rule-sync push. The refinement has no
+	// deferToVerify carve-out, so a deferred Block is held to the same
+	// rule as a plain one.
+	it("rejects captchaType and solvedImagesCount despite the flag", () => {
+		const result = accessPolicyInput.safeParse({
+			type: AccessPolicyType.Block,
+			deferToVerify: true,
+			captchaType: CaptchaType.image,
+			solvedImagesCount: 2,
+		});
+		expect(result.success).toBe(false);
+		if (!result.success) {
+			expect(result.error.issues.map((i) => i.path.at(-1)).sort()).toEqual([
+				"captchaType",
+				"solvedImagesCount",
+			]);
+		}
+	});
+
+	// Even were the refinement to admit them, the sanitiser drops them
+	// before storage — the flag does not exempt a Block here either.
+	it("strips both fields on the way into storage", () => {
+		expect(
+			sanitizeAccessPolicy({
+				type: AccessPolicyType.Block,
+				deferToVerify: true,
+				captchaType: CaptchaType.image,
+				solvedImagesCount: 2,
+				description: "deferred block",
+			}),
+		).toEqual({
+			type: AccessPolicyType.Block,
+			deferToVerify: true,
+			description: "deferred block",
+		});
+	});
+
+	it("round-trips the flag from its stored string form", () => {
+		const result = accessPolicyInput.safeParse({
+			type: AccessPolicyType.Block,
+			deferToVerify: "true",
+		});
+		expect(result.success).toBe(true);
+		if (result.success) {
+			expect(result.data.deferToVerify).toBe(true);
+		}
+	});
+});
+
+describe("blocking rule shapes: Restrict keeps its tuning fields", () => {
+	it("accepts captchaType and solvedImagesCount, with or without the flag", () => {
+		for (const deferToVerify of [undefined, true]) {
+			const result = accessPolicyInput.safeParse({
+				type: AccessPolicyType.Restrict,
+				captchaType: CaptchaType.image,
+				solvedImagesCount: 2,
+				...(deferToVerify === undefined ? {} : { deferToVerify }),
+			});
+			expect(result.success).toBe(true);
+		}
+	});
+
+	it("survives the sanitiser intact", () => {
+		expect(
+			sanitizeAccessPolicy({
+				type: AccessPolicyType.Restrict,
+				deferToVerify: true,
+				captchaType: CaptchaType.image,
+				solvedImagesCount: 2,
+			}),
+		).toEqual({
+			type: AccessPolicyType.Restrict,
+			deferToVerify: true,
+			captchaType: CaptchaType.image,
+			solvedImagesCount: 2,
+		});
+	});
+});
+
+// The insert endpoint validates the batch as one `z.array`, so validity
+// is all-or-nothing across the whole push.
+describe("blocking rule shapes: batch validation", () => {
+	const validGroup = {
+		accessPolicy: {
+			type: AccessPolicyType.Block,
+			deferToVerify: true,
+			description: "deferred block",
+		},
+		userScopes: [{ ip: "1.1.1.1" }],
+	};
+
+	const invalidGroup = {
+		accessPolicy: {
+			type: AccessPolicyType.Block,
+			deferToVerify: true,
+			captchaType: CaptchaType.image,
+			solvedImagesCount: 2,
+			description: "deferred block with tuning fields",
+		},
+		userScopes: [{ ip: "2.2.2.2" }],
+	};
+
+	it("accepts a batch of well-formed groups", () => {
+		const result = getInsertSchema().safeParse([validGroup, validGroup]);
+		expect(result.success).toBe(true);
+	});
+
+	it("rejects the entire batch when a single group is malformed", () => {
+		const batch = [
+			...Array.from({ length: 20 }, () => validGroup),
+			invalidGroup,
+			...Array.from({ length: 20 }, () => validGroup),
+		];
+
+		const result = getInsertSchema().safeParse(batch);
+
+		expect(result.success).toBe(false);
+		if (!result.success) {
+			// Every issue points at the one bad group; the other 40 are
+			// well-formed and are still rejected along with it.
+			expect(result.error.issues.map((i) => i.path[0])).toEqual([20, 20]);
+		}
+	});
+});

package/src/tests/redis/deferredPoolClause.benchmark.ts

@@ -0,0 +1,249 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Standalone benchmark for the hard-block candidate-pool clauses.
+//
+// Compares the previous single-field clause against the two new
+// two-field clauses on the shapes they actually run in:
+//
+//   baseline   @type:{block}                                (old, both callers)
+//   middleware @type:{block} -@deferToVerify:{true}         (new, request time)
+//   verify     (@type:{block} | @deferToVerify:{true})      (new, verify)
+//
+// Also runs the MISGROUPED union as a control. `|` binds looser than the
+// implicit AND, so `(@a)|(@b) scope ip` lets the left operand escape the
+// scope/IP filters — it matches every Block rule in the index. That
+// misgrouping is both wrong and ~7x slower, and it is invisible on a
+// toy dataset; it only shows up at population scale, which is why this
+// benchmark exists.
+//
+// Population mirrors redisRulesReaderLoad.benchmark.integration.test.ts
+// (the 17k bulk-ban incident shape) plus a deferred-rule cohort.
+//
+// Run: npx tsx <this file>   (needs redis-stack on localhost:6379)
+
+import { createClient } from "redis";
+
+const IP_RULE_COUNT = 17_000;
+const CIDR_RULE_COUNT = 1_300;
+const MIXED_RULE_COUNT = 1_000;
+// Deferred rules as a share of the population. Production currently has
+// ~380 deferred against ~4k live rules; 2_000 here is a deliberate
+// over-weight so the union has real work to merge.
+const DEFERRED_RULE_COUNT = 2_000;
+
+const WARMUP = 50;
+const ITERATIONS = 500;
+
+const INDEX = "bench:idx";
+const PREFIX = "bench:";
+
+type Clause = { name: string; clause: string };
+
+const CLAUSES: Clause[] = [
+	{ name: "baseline   @type:{block}", clause: "@type:{block}" },
+	{
+		name: "middleware @type:{block} -@deferToVerify:{true}",
+		clause: "@type:{block} -@deferToVerify:{true}",
+	},
+	{
+		name: "verify  (@type:{block} | @deferToVerify:{true})",
+		clause: "(@type:{block} | @deferToVerify:{true})",
+	},
+	{
+		name: "MISGROUPED (@type:{block})|(@deferToVerify:{true})",
+		clause: "(@type:{block})|(@deferToVerify:{true})",
+	},
+];
+
+const ipv4 = (i: number): bigint => (10n << 24n) + BigInt(i);
+
+const ipv4CidrRange = (i: number): { min: bigint; max: bigint } => {
+	const octet = i % 256;
+	const base = (192n << 24n) + (168n << 16n) + (BigInt(octet) << 8n);
+	return { min: base, max: base + 255n };
+};
+
+const percentile = (sorted: number[], p: number): number => {
+	const idx = Math.min(
+		sorted.length - 1,
+		Math.floor((p / 100) * sorted.length),
+	);
+	return sorted[idx] ?? 0;
+};
+
+const main = async (): Promise<void> => {
+	const client = createClient({
+		url: "redis://localhost:6379",
+		password: "root",
+	});
+	await client.connect();
+
+	await client.flushAll();
+
+	// Real index schema shape: type + deferToVerify as TAG INDEXMISSING,
+	// IP fields NUMERIC INDEXMISSING.
+	await client.sendCommand([
+		"FT.CREATE",
+		INDEX,
+		"ON",
+		"HASH",
+		"PREFIX",
+		"1",
+		PREFIX,
+		"SCHEMA",
+		"type",
+		"TAG",
+		"INDEXMISSING",
+		"deferToVerify",
+		"TAG",
+		"INDEXMISSING",
+		"clientId",
+		"TAG",
+		"INDEXMISSING",
+		"numericIp",
+		"NUMERIC",
+		"INDEXMISSING",
+		"numericIpMaskMin",
+		"NUMERIC",
+		"INDEXMISSING",
+		"numericIpMaskMax",
+		"NUMERIC",
+		"INDEXMISSING",
+	]);
+
+	let n = 0;
+	const write = async (fields: Record<string, string>): Promise<void> => {
+		await client.hSet(`${PREFIX}${n++}`, fields);
+	};
+
+	for (let i = 0; i < IP_RULE_COUNT; i++) {
+		await write({
+			type: "block",
+			description: `bulk-ban-ip-${i}`,
+			numericIp: ipv4(i).toString(),
+			clientId: "global",
+		});
+	}
+	for (let i = 0; i < CIDR_RULE_COUNT; i++) {
+		const { min, max } = ipv4CidrRange(i);
+		await write({
+			type: "block",
+			description: `bulk-ban-cidr-${i}`,
+			numericIpMaskMin: min.toString(),
+			numericIpMaskMax: max.toString(),
+			clientId: "global",
+		});
+	}
+	for (let i = 0; i < MIXED_RULE_COUNT; i++) {
+		await write({
+			type: i % 2 === 0 ? "restrict" : "block",
+			description: `mixed-${i}`,
+			numericIp: ipv4(IP_RULE_COUNT + i).toString(),
+			clientId: `client-${i % 20}`,
+		});
+	}
+	// Deferred cohort — the shape this change is about. Restrict, so the
+	// baseline clause would miss them entirely.
+	for (let i = 0; i < DEFERRED_RULE_COUNT; i++) {
+		await write({
+			type: "restrict",
+			description: `deferred-${i}`,
+			deferToVerify: "true",
+			numericIp: ipv4(IP_RULE_COUNT + MIXED_RULE_COUNT + i).toString(),
+			clientId: `client-${i % 20}`,
+		});
+	}
+
+	const total = n;
+	// Let RediSearch finish indexing.
+	for (;;) {
+		const info = (await client.sendCommand(["FT.INFO", INDEX])) as unknown[];
+		const idx = info.findIndex((v) => String(v) === "indexing");
+		if (idx === -1 || String(info[idx + 1]) === "0") break;
+		await new Promise((r) => setTimeout(r, 200));
+	}
+
+	console.log(`population: ${total} rules`);
+	console.log(
+		`  ${IP_RULE_COUNT} ip-block, ${CIDR_RULE_COUNT} cidr-block, ${MIXED_RULE_COUNT} mixed, ${DEFERRED_RULE_COUNT} deferred-restrict\n`,
+	);
+
+	// The two probe shapes that dominate the split query: an exact-IP hit
+	// (the common ban case) and the CIDR mask-range probe (the expensive
+	// one — it walks two numeric ranges).
+	const hitIp = ipv4(1234).toString();
+	const cidrIp = ((192n << 24n) + (168n << 16n) + (7n << 8n) + 9n).toString();
+	const scope = "( @clientId:{client-3} | @clientId:{global} )";
+
+	const probes: Array<{ label: string; suffix: string }> = [
+		{ label: "ip:exact", suffix: `${scope} @numericIp:[${hitIp} ${hitIp}]` },
+		{
+			label: "ip:mask ",
+			suffix: `${scope} @numericIpMaskMin:[-inf ${cidrIp}] @numericIpMaskMax:[${cidrIp} +inf]`,
+		},
+	];
+
+	for (const probe of probes) {
+		console.log(`--- probe: ${probe.label} ---`);
+		for (const { name, clause } of CLAUSES) {
+			const query = `${clause} ${probe.suffix}`;
+
+			for (let i = 0; i < WARMUP; i++) {
+				await client.sendCommand([
+					"FT.SEARCH",
+					INDEX,
+					query,
+					"NOCONTENT",
+					"DIALECT",
+					"2",
+				]);
+			}
+
+			const samples: number[] = [];
+			let hits = 0;
+			for (let i = 0; i < ITERATIONS; i++) {
+				const t0 = process.hrtime.bigint();
+				const res = (await client.sendCommand([
+					"FT.SEARCH",
+					INDEX,
+					query,
+					"NOCONTENT",
+					"DIALECT",
+					"2",
+				])) as unknown[];
+				const t1 = process.hrtime.bigint();
+				samples.push(Number(t1 - t0) / 1e6);
+				hits = Number(res[0]);
+			}
+			samples.sort((a, b) => a - b);
+			const mean = samples.reduce((s, v) => s + v, 0) / samples.length;
+			console.log(
+				`  ${name.padEnd(52)} hits=${String(hits).padStart(5)}  ` +
+					`mean=${mean.toFixed(3)}ms  p50=${percentile(samples, 50).toFixed(3)}ms  ` +
+					`p95=${percentile(samples, 95).toFixed(3)}ms  p99=${percentile(samples, 99).toFixed(3)}ms`,
+			);
+		}
+		console.log();
+	}
+
+	await client.sendCommand(["FT.DROPINDEX", INDEX]);
+	await client.flushAll();
+	await client.quit();
+};
+
+main().catch((e: unknown) => {
+	console.error(e);
+	process.exit(1);
+});