@prosopo/user-access-policy 3.14.13 → 3.14.15

package/src/redis/reader/redisRulesSplitQuery.ts

@@ -1,234 +0,0 @@
-// Copyright 2021-2026 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { getBlockPoolClause } from "#policy/redis/reader/redisRulesQuery.js";
-import { GLOBAL_CLIENT_SCOPE_SENTINEL, type UserScope } from "#policy/rule.js";
-
-// Escapes special characters in Redis TAG queries. Mirrors the escape
-// function in redisRulesQuery.ts — kept local to avoid coupling the two
-// query builders since they'll diverge in shape.
-const escapeTagValue = (value: string): string =>
-	value.replace(/([,.<>{}\[\]"':;!@#$%^&*()\-+=~|/\\])/g, "\\$1");
-
-// Fields where TAG queries need escaping (they may hold JSON etc.).
-const FIELDS_REQUIRING_ESCAPE: ReadonlySet<keyof UserScope> = new Set([
-	"coords",
-]);
-
-// Fields indexed as NUMERIC — require range syntax `@x:[N N]` not TAG.
-const NUMERIC_FIELDS: ReadonlySet<keyof UserScope> = new Set(["asn"]);
-
-// Enumerating the scalar (non-IP) user-scope fields keeps the "no
-// user-scope constraint" fall-through query stable when userScopeSchema
-// grows a new field — the enumeration below drives ismissing() clauses.
-const SCALAR_USER_SCOPE_FIELDS: ReadonlyArray<keyof UserScope> = [
-	"userId",
-	"ja4Hash",
-	"headersHash",
-	"userAgentHash",
-	"headHash",
-	"coords",
-	"countryCode",
-	"asn",
-	"os",
-	"browser",
-];
-
-const ALL_USER_SCOPE_FIELDS: ReadonlyArray<keyof UserScope> = [
-	...SCALAR_USER_SCOPE_FIELDS,
-	"numericIp",
-	"numericIpMaskMin",
-	"numericIpMaskMax",
-];
-
-// Global rules written under the new sentinel are found via
-// `@clientId:{global}`; the ismissing() fallback covers legacy rules
-// that predate the sentinel stamp (rehash-all migrates them). This
-// disjunction still hits the clientId posting list rather than walking
-// the full rule set the way the old single-query path did.
-const GLOBAL_SCOPE_INNER = `@clientId:{${GLOBAL_CLIENT_SCOPE_SENTINEL}} | ismissing(@clientId)`;
-
-const buildScopeClause = (clientId: string | undefined): string => {
-	if (clientId === undefined) {
-		return `( ${GLOBAL_SCOPE_INNER} )`;
-	}
-	return `( @clientId:{${clientId}} | ${GLOBAL_SCOPE_INNER} )`;
-};
-
-const buildFieldClause = (
-	field: keyof UserScope,
-	value: unknown,
-): string | null => {
-	if (value === undefined) {
-		return null;
-	}
-	const stringValue = String(value);
-	if (NUMERIC_FIELDS.has(field)) {
-		return `@${field}:[${stringValue} ${stringValue}]`;
-	}
-	const queryValue = FIELDS_REQUIRING_ESCAPE.has(field)
-		? escapeTagValue(stringValue)
-		: stringValue;
-	return `@${field}:{${queryValue}}`;
-};
-
-type SubQuery = {
-	// A short label used for logging + metrics. Not part of the query
-	// wire format.
-	kind: string;
-	query: string;
-};
-
-/**
- * Build the set of FT sub-queries that together cover every rule that
- * could apply to a request.
- *
- * The single-query design this replaces produced one FT.AGGREGATE whose
- * candidate set was the entire scope's rule pool — the APPLY / SORTBY
- * pipeline then walked every rule per request, which degrades sharply
- * once rule populations reach the 10k+ range. The split approach fires
- * one probe per populated request field. Each probe uses the field's
- * own index posting list, so a 17k IP-ban population collapses to 1
- * result (exact IP) plus a handful (CIDR ranges containing the IP)
- * instead of forcing a full-set scan.
- *
- * A single fall-through probe covers rules with no user-scope constraint
- * (client-wide restriction); those are rare so the ismissing()-heavy
- * query is cheap in practice.
- *
- * Rank + strict-match filtering happens in JS via `rankCandidateRules`
- * after the union — the FT layer is used purely as a candidate fetcher.
- *
- * `blockOnly` narrows the candidate set to rules with `type:{block}` —
- * used by the hard-block middleware which never needs Restrict rules.
- * `includeDeferred` widens that pool back out to
- * `(@type:{block} | @deferToVerify:{true})` for the verify-time
- * hard-block lookup, where a deferred Restrict is a valid block.
- * Every other caller passes false so both Block and Restrict rules are
- * fetched; the JS-side ranker picks the right one via specificity +
- * severity. Merging the block and restrict paths behind one split-query
- * fixed the Twickets IP-rule regression: the previous FT.AGGREGATE ranker
- * used for non-block lookups was truncating specific-IP restrict rules
- * out of the top-N when many higher-specificity irrelevant rules matched
- * the greedy `ismissing()` disjunction.
- */
-export const buildScopedRulesSubQueries = (
-	userScope: UserScope,
-	clientId: string | undefined,
-	options: { blockOnly?: boolean; includeDeferred?: boolean } = {},
-): SubQuery[] => {
-	const scopeClause = buildScopeClause(clientId);
-
-	// `includeDeferred` emits the probe set twice — once for plain hard
-	// blocks, once for deferred rules — rather than widening one type
-	// clause to `(@type:{block} | @deferToVerify:{true})`.
-	//
-	// Widening merges both populations into a single probe, so they share
-	// one SPLIT_MAX_CANDIDATES_PER_SUB budget and a dense deferred cohort
-	// can truncate hard blocks out of the candidate set. Measured on a
-	// scope with 400 block + 400 deferred rules on one ja4Hash: the merged
-	// probe returns 800 against a cap of 500, so 300 candidates are
-	// dropped with no ordering guarantee about which. That is precisely
-	// the crowding `blockOnly` was introduced to prevent.
-	//
-	// Two disjoint probe sets each get their own budget, so a hard block
-	// is never displaced by a deferred rule. The pools are disjoint by
-	// construction (the block probe excludes deferred), so the union
-	// double-counts nothing.
-	// Probe `kind` is only namespaced when there are genuinely two pools;
-	// single-pool callers keep the original unprefixed labels.
-	const typePools: Array<{ tag: string; clause: string }> = options.blockOnly
-		? options.includeDeferred === true
-			? [
-					{ tag: "block", clause: `${getBlockPoolClause(false)} ` },
-					{ tag: "deferred", clause: "@deferToVerify:{true} " },
-				]
-			: [{ tag: "", clause: `${getBlockPoolClause(false)} ` }]
-		: [{ tag: "", clause: "" }];
-
-	const subQueries: SubQuery[] = [];
-	for (const pool of typePools) {
-		const prefix = `${pool.clause}${scopeClause}`;
-		const label = pool.tag === "" ? "" : `${pool.tag}:`;
-		subQueries.push(...buildProbes(userScope, prefix, label));
-	}
-	return subQueries;
-};
-
-const buildProbes = (
-	userScope: UserScope,
-	prefix: string,
-	label: string,
-): SubQuery[] => {
-	const subQueries: SubQuery[] = [];
-
-	// One probe per populated scalar user-scope field. Each uses that
-	// field's posting list rather than intersecting ismissing() across
-	// the full set.
-	for (const field of SCALAR_USER_SCOPE_FIELDS) {
-		const clause = buildFieldClause(field, userScope[field]);
-		if (clause === null) {
-			continue;
-		}
-		subQueries.push({
-			kind: `${label}field:${field}`,
-			query: `${prefix} ${clause}`,
-		});
-	}
-
-	// IP: two disjoint probes so each hits a single index. The exact-IP
-	// probe catches individual-IP bans (the dominant shape in bulk-ban
-	// scripts). The mask-range probe catches CIDR aggregates containing
-	// the request IP. Splitting them means neither has to walk a mixed
-	// posting list — the mask-range probe was the slower of the two
-	// under load, so isolating it lets Redis skip it entirely for hits
-	// on the fast path.
-	const requestIp = userScope.numericIp;
-	if (requestIp !== undefined) {
-		subQueries.push({
-			kind: `${label}ip:exact`,
-			query: `${prefix} @numericIp:[${requestIp} ${requestIp}]`,
-		});
-		subQueries.push({
-			kind: `${label}ip:mask`,
-			query: `${prefix} @numericIpMaskMin:[-inf ${requestIp}] @numericIpMaskMax:[${requestIp} +inf]`,
-		});
-	}
-
-	// Fall-through: rules that constrain nothing on the user scope
-	// ("block everything for this client / everyone"). Rare in
-	// practice — a handful per tenant at most — so the ismissing()
-	// intersection is cheap despite touching every field.
-	const noScopeIsmissing = ALL_USER_SCOPE_FIELDS.map(
-		(field) => `ismissing(@${field})`,
-	).join(" ");
-	subQueries.push({
-		kind: `${label}no-user-scope`,
-		query: `${prefix} ${noScopeIsmissing}`,
-	});
-
-	return subQueries;
-};
-
-/**
- * Backwards-compatible alias for the previous block-only builder. New
- * callers should use `buildScopedRulesSubQueries` directly.
- *
- * @deprecated use `buildScopedRulesSubQueries(..., { blockOnly: true })`
- */
-export const buildScopedBlockSubQueries = (
-	userScope: UserScope,
-	clientId: string | undefined,
-): SubQuery[] =>
-	buildScopedRulesSubQueries(userScope, clientId, { blockOnly: true });

package/src/redis/redisClient.ts

@@ -1,125 +0,0 @@
-// Copyright 2021-2026 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import type { Logger } from "@prosopo/logger";
-import type { RedisClientType } from "redis";
-import { type ZodType, type ZodTypeDef, z } from "zod";
-
-export const REDIS_BATCH_SIZE = 1_000;
-
-export const getMissingRedisKeys = async (
-	client: RedisClientType,
-	keys: string[],
-): Promise<string[]> => {
-	const queries = client.multi();
-
-	keys.map((key) => {
-		queries.exists(key);
-	});
-
-	const records: unknown[] = await queries.exec();
-
-	const missingKeys: string[] = [];
-
-	records.map((exists, recordIndex) => {
-		if ("0" === String(exists)) {
-			const key = keys[recordIndex];
-
-			if (key) {
-				missingKeys.push(key);
-			}
-		}
-	});
-
-	return missingKeys;
-};
-
-export const fetchRedisHashRecords = async (
-	client: RedisClientType,
-	keys: string[],
-	logger: Logger,
-): Promise<{ records: object[]; expirations: (number | undefined)[] }> => {
-	const rulesPipe = client.multi();
-	const expirationPipe = client.multi();
-
-	for (const key of keys) {
-		rulesPipe.hGetAll(key);
-		expirationPipe.expireTime(key);
-	}
-
-	const records = (await rulesPipe.exec()) as object[];
-	const expirationRecords = (await expirationPipe.exec()) as unknown[];
-
-	return {
-		records: records,
-		expirations: parseExpirationRecords(expirationRecords, logger),
-	};
-};
-
-// `recordSchema` is intentionally typed with `unknown` as the input
-// position. Some schemas use `z.preprocess` (e.g. accessPolicyInput's
-// deferToVerify boolean coercion from Redis strings), which widens the
-// zod input type. The strict `ZodType<T, ZodTypeDef, T>` form rejects
-// those at the call site even though they parse correctly at runtime.
-export const parseRedisRecords = <T>(
-	records: unknown[],
-	recordSchema: ZodType<T, ZodTypeDef, unknown>,
-	logger: Logger,
-): T[] =>
-	records.flatMap((record) => {
-		const parseResult = recordSchema.safeParse(record);
-
-		if (parseResult.success) {
-			return [parseResult.data];
-		}
-
-		logger.error(() => ({
-			msg: "Failed to parse Redis record",
-			data: { record, error: parseResult.error },
-		}));
-
-		return [];
-	});
-
-const expirationRecordSchema = z.coerce.number();
-// Redis returns -1 when expiration is not set
-const UNSET_EXPIRATION_VALUE = -1;
-
-const parseExpirationRecords = <T>(
-	records: unknown[],
-	logger: Logger,
-): (number | undefined)[] =>
-	records.flatMap((record) => {
-		const parseResult = expirationRecordSchema.safeParse(record);
-
-		if (parseResult.success) {
-			const expiration =
-				UNSET_EXPIRATION_VALUE === parseResult.data
-					? undefined
-					: parseResult.data;
-
-			return [expiration];
-		}
-
-		logger.error(() => ({
-			msg: "Failed to parse Redis expiration record",
-			data: {
-				record,
-				error: parseResult.error,
-			},
-		}));
-
-		// ensure consistent output length
-		return [undefined];
-	});

package/src/redis/redisRuleIndex.ts

@@ -1,121 +0,0 @@
-// Copyright 2021-2026 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import type { AllKeys, Keys } from "@prosopo/common";
-import type { RedisIndex } from "@prosopo/redis-client";
-import { type RediSearchSchema, SCHEMA_FIELD_TYPE } from "@redis/search";
-import type {
-	AccessRule,
-	PolicyScope,
-	UserAttributes,
-	UserIp,
-	UserScope,
-} from "#policy/rule.js";
-import { makeAccessRuleHash } from "#policy/transformRule.js";
-
-export const userIpRedisSchema: RediSearchSchema = {
-	numericIpMaskMin: { type: SCHEMA_FIELD_TYPE.NUMERIC, INDEXMISSING: true },
-	numericIpMaskMax: { type: SCHEMA_FIELD_TYPE.NUMERIC, INDEXMISSING: true },
-	numericIp: { type: SCHEMA_FIELD_TYPE.NUMERIC, INDEXMISSING: true },
-} satisfies AllKeys<UserIp>;
-
-export const userAttributesRedisSchema: RediSearchSchema = {
-	userId: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	ja4Hash: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	headersHash: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	userAgentHash: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	headHash: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	// Use pipe separator for coords since JSON strings contain commas
-	coords: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true, SEPARATOR: "|" },
-	countryCode: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	asn: { type: SCHEMA_FIELD_TYPE.NUMERIC, INDEXMISSING: true },
-	os: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	browser: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	// Header-restriction fields. NONE of these are queried on the hot path:
-	// the split-query builder probes neither `headerMatch` nor the
-	// name/value/operator triple, and a header rule populates none of the
-	// fields that builder does probe, so it is returned by the `no-user-scope`
-	// fall-through probe (the same route `os` rules take) and the concrete
-	// condition is evaluated in code by `accessRuleHeaderMatches`. They are
-	// indexed to satisfy the schema exhaustiveness check, and `headerMatch`
-	// additionally gates candidacy JS-side in `ruleApplies` and scores one
-	// specificity point in the reader's SPECIFICITY_EXPR.
-	headerMatch: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	headerName: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	headerValue: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	headerOperator: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	webBotAuthAgent: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-} satisfies AllKeys<UserAttributes>;
-
-export const userScopeRedisSchema: RediSearchSchema = {
-	...userAttributesRedisSchema,
-	...userIpRedisSchema,
-} satisfies Keys<UserScope>;
-
-export const policyScopeRedisSchema: RediSearchSchema = {
-	clientId: {
-		type: SCHEMA_FIELD_TYPE.TAG,
-		INDEXMISSING: true,
-	},
-} satisfies AllKeys<PolicyScope>;
-
-/**
- * Note on the field type decision
- *
- * TAG is designed for the exact value matching
- * TEXT is designed for the word-based and pattern matching
- *
- * For our goal TAG fits perfectly and, more performant
- */
-export const accessRuleRedisSchema: RediSearchSchema = {
-	...policyScopeRedisSchema,
-	...userScopeRedisSchema,
-	groupId: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	// Indexed so request-time callers that only care about Block policies
-	// (checkForHardBlock, blockMiddleware) can pre-filter via @type:{block}
-	// before the SERVER_SIDE_RANK_TOP_N cap kicks in. Without this, dense
-	// Restrict / routing-Block populations push the hard-block rules out
-	// of the top-N candidate set and the lookup silently misses them.
-	type: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-	// Indexed so `checkForHardBlock` can widen its Block-only pool to
-	// `(@type:{block} | @deferToVerify:{true})`. Deferred rules are
-	// skipped at request time and enforced at verify, so a deferred
-	// Restrict is a legitimate hard block — without this field in the
-	// index it can't be selected server-side and never fires.
-	// Stored by `getRedisRuleValue` as the string "true"/"false".
-	deferToVerify: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
-} satisfies Keys<AccessRule>;
-
-export const ACCESS_RULES_REDIS_INDEX_NAME = "index:user-access-rules";
-
-// names take space, so we use an acronym instead of the long-tailed one
-export const ACCESS_RULE_REDIS_KEY_PREFIX = "uar:";
-
-// Re-exported from #policy/rule.js — that module is types-only so both
-// the parse side (policyInput.ts) and the write/index side (this file
-// and redisRulesWriter.ts) can import the sentinel without pulling
-// transformRule into a cycle.
-export { GLOBAL_CLIENT_SCOPE_SENTINEL } from "#policy/rule.js";
-
-export const accessRulesRedisIndex: RedisIndex = {
-	name: ACCESS_RULES_REDIS_INDEX_NAME,
-	schema: accessRuleRedisSchema,
-	options: {
-		ON: "HASH" as const,
-		PREFIX: [ACCESS_RULE_REDIS_KEY_PREFIX],
-	},
-};
-
-export const getAccessRuleRedisKey = (rule: AccessRule): string =>
-	ACCESS_RULE_REDIS_KEY_PREFIX + makeAccessRuleHash(rule);

package/src/redis/redisRulesStorage.ts

@@ -1,68 +0,0 @@
-// Copyright 2021-2026 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import type { Logger } from "@prosopo/logger";
-import type { RedisConnection } from "@prosopo/redis-client";
-import type {
-	AccessRulesReader,
-	AccessRulesStorage,
-	AccessRulesWriter,
-} from "#policy/rulesStorage.js";
-import {
-	DummyRedisRulesReader,
-	RedisRulesReader,
-} from "./reader/redisRulesReader.js";
-import { DummyRedisRulesWriter, RedisRulesWriter } from "./redisRulesWriter.js";
-
-export const createRedisAccessRulesStorage = (
-	connection: RedisConnection,
-	logger: Logger,
-): AccessRulesStorage => {
-	const storage: AccessRulesStorage = composeStorage(
-		new DummyRedisRulesReader(logger),
-		new DummyRedisRulesWriter(logger),
-	);
-
-	connection.getClient().then((client) => {
-		const realStorage = composeStorage(
-			new RedisRulesReader(client, logger),
-			new RedisRulesWriter(client, logger),
-		);
-
-		// use assigning instead of var overwriting to keep the object reference
-		Object.assign(storage, realStorage);
-
-		logger.info(() => ({
-			msg: "RedisAccessRules storage got a ready Redis client",
-		}));
-	});
-
-	return storage;
-};
-
-const composeStorage = (
-	reader: AccessRulesReader,
-	writer: AccessRulesWriter,
-): AccessRulesStorage => ({
-	// reader
-	fetchRules: reader.fetchRules.bind(reader),
-	getMissingRuleIds: reader.getMissingRuleIds.bind(reader),
-	findRules: reader.findRules.bind(reader),
-	findRuleIds: reader.findRuleIds.bind(reader),
-	fetchAllRuleIds: reader.fetchAllRuleIds.bind(reader),
-	// writer
-	insertRules: writer.insertRules.bind(writer),
-	deleteRules: writer.deleteRules.bind(writer),
-	deleteAllRules: writer.deleteAllRules.bind(writer),
-});