@prosopo/user-access-policy 3.5.32 → 3.7.11

package/src/redis/reader/redisRulesReader.ts

@@ -0,0 +1,318 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as util from "node:util";
+import { chunkIntoBatches, executeBatchesSequentially } from "@prosopo/common";
+import type { Logger } from "@prosopo/logger";
+import type { RedisClientType } from "redis";
+import {
+	REDIS_QUERY_DIALECT,
+	getRulesRedisQuery,
+} from "#policy/redis/reader/redisRulesQuery.js";
+import {
+	REDIS_BATCH_SIZE,
+	fetchRedisHashRecords,
+	getMissingRedisKeys,
+	parseRedisRecords,
+} from "#policy/redis/redisClient.js";
+import {
+	ACCESS_RULES_REDIS_INDEX_NAME,
+	ACCESS_RULE_REDIS_KEY_PREFIX,
+} from "#policy/redis/redisRuleIndex.js";
+import type { AccessRule } from "#policy/rule.js";
+import { accessRuleInput } from "#policy/ruleInput/ruleInput.js";
+import type {
+	AccessRuleEntry,
+	AccessRulesFilter,
+	AccessRulesReader,
+} from "#policy/rulesStorage.js";
+import { aggregateRedisKeys } from "./redisAggregate.js";
+
+export class RedisRulesReader implements AccessRulesReader {
+	constructor(
+		private readonly client: RedisClientType,
+		private readonly logger: Logger,
+	) {}
+
+	async getMissingRuleIds(ruleIds: string[]): Promise<string[]> {
+		const ruleKeys = this.getRuleKeys(ruleIds);
+		const keyBatches = chunkIntoBatches(ruleKeys, REDIS_BATCH_SIZE);
+
+		const missingKeyBatches = await executeBatchesSequentially(
+			keyBatches,
+			async (keysBatch) => getMissingRedisKeys(this.client, keysBatch),
+		);
+
+		return missingKeyBatches
+			.flat()
+			.map((ruleKey) => ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length));
+	}
+
+	async fetchRules(ruleIds: string[]): Promise<AccessRuleEntry[]> {
+		const ruleKeys = this.getRuleKeys(ruleIds);
+
+		const keyBatches = chunkIntoBatches(ruleKeys, REDIS_BATCH_SIZE);
+
+		const entryBatches = await executeBatchesSequentially(
+			keyBatches,
+			(keysBatch) => this.fetchRuleEntries(keysBatch),
+		);
+
+		return entryBatches.flat();
+	}
+
+	async findRules(
+		filter: AccessRulesFilter,
+		matchingFieldsOnly = false,
+		skipEmptyUserScopes = true,
+	): Promise<AccessRule[]> {
+		const query = getRulesRedisQuery(filter, matchingFieldsOnly);
+
+		if (skipEmptyUserScopes && query === "ismissing(@clientId)") {
+			// We don't want to accidentally return all rules when the filter is empty
+			return [];
+		}
+
+		try {
+			// Use searchNoContent to get document keys only, then fetch data separately.
+			// This avoids a bug in @redis/search where ft.search crashes with
+			// "Cannot read properties of null (reading 'length')" when a document
+			// is deleted/expired between the index scan and data retrieval.
+			const searchReply = await this.client.ft.searchNoContent(
+				ACCESS_RULES_REDIS_INDEX_NAME,
+				query,
+				{
+					DIALECT: REDIS_QUERY_DIALECT,
+					// FT.search doesn't support "unlimited" selects
+					LIMIT: {
+						from: 0,
+						size: REDIS_BATCH_SIZE,
+					},
+				},
+			);
+
+			if (searchReply.total > 0) {
+				this.logger.debug(() => ({
+					msg: "Executed search query",
+					data: {
+						inspect: util.inspect(
+							{
+								filter: filter,
+								searchReply: searchReply,
+								query: query,
+							},
+							{ depth: null },
+						),
+					},
+				}));
+			}
+
+			if (searchReply.documents.length === 0) {
+				return [];
+			}
+
+			const { records } = await fetchRedisHashRecords(
+				this.client,
+				searchReply.documents,
+				this.logger,
+			);
+
+			const nonEmptyRecords = records.filter(
+				(record) => Object.keys(record).length > 0,
+			);
+
+			return parseRedisRecords(nonEmptyRecords, accessRuleInput, this.logger);
+		} catch (e) {
+			this.logger.error(() => ({
+				err: e,
+				data: {
+					inspect: util.inspect(
+						{
+							query: query,
+							filter: filter,
+						},
+						{
+							depth: null,
+						},
+					),
+				},
+				msg: "failed to execute search query",
+			}));
+
+			return [];
+		}
+	}
+
+	async findRuleIds(
+		filter: AccessRulesFilter,
+		matchingFieldsOnly = false,
+	): Promise<string[]> {
+		const query = getRulesRedisQuery(filter, matchingFieldsOnly);
+
+		let ruleIds: string[] = [];
+
+		try {
+			// aggregation is used instead ft.search to overcome the limitation on search results number
+			const ruleKeys = await aggregateRedisKeys(
+				this.client,
+				query,
+				this.logger,
+			);
+
+			ruleIds = ruleKeys.map((ruleKey) =>
+				ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length),
+			);
+		} catch (e) {
+			this.logger.error(() => ({
+				err: e,
+				data: {
+					inspect: util.inspect(
+						{
+							query: query,
+							filter: filter,
+						},
+						{
+							depth: null,
+						},
+					),
+				},
+				msg: "Failed to execute search query for rule IDs",
+			}));
+
+			return [];
+		}
+
+		this.logger.debug(() => ({
+			msg: "Executed search query for rule IDs",
+			data: {
+				query: query,
+				foundCount: ruleIds.length,
+				foundIds: ruleIds,
+			},
+		}));
+
+		return ruleIds;
+	}
+
+	async fetchAllRuleIds(
+		batchHandler: (ruleIds: string[]) => Promise<void>,
+	): Promise<void> {
+		const keysBatchHandler = async (keys: string[]) => {
+			const ids = keys.map((ruleKey) =>
+				ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length),
+			);
+
+			await batchHandler(ids);
+		};
+
+		await aggregateRedisKeys(this.client, "*", this.logger, keysBatchHandler);
+	}
+
+	protected async fetchRuleEntries(keys: string[]): Promise<AccessRuleEntry[]> {
+		const { records, expirations } = await fetchRedisHashRecords(
+			this.client,
+			keys,
+			this.logger,
+		);
+		const entries: AccessRuleEntry[] = [];
+
+		for (const [index, ruleData] of records.entries()) {
+			const isRulePresent = Object.keys(ruleData).length > 0;
+
+			if (isRulePresent) {
+				const rule = parseRedisRecords(
+					[ruleData],
+					accessRuleInput,
+					this.logger,
+				)[0];
+
+				if (rule) {
+					entries.push({
+						rule: rule,
+						expiresUnixTimestamp: expirations[index],
+					});
+				}
+			}
+		}
+
+		return entries;
+	}
+
+	protected getRuleKeys(ruleIds: string[]): string[] {
+		return ruleIds.map((id) => `${ACCESS_RULE_REDIS_KEY_PREFIX}${id}`);
+	}
+}
+
+export class DummyRedisRulesReader implements AccessRulesReader {
+	constructor(private readonly logger: Logger) {}
+
+	async getMissingRuleIds(ruleIds: string[]): Promise<string[]> {
+		this.logger.info(() => ({
+			msg: "Dummy getMissingRuleIds() has no effect (redis is not ready)",
+			data: {
+				ruleIds,
+			},
+		}));
+
+		return [];
+	}
+
+	async fetchRules(ruleIds: string[]): Promise<AccessRuleEntry[]> {
+		this.logger.info(() => ({
+			msg: "Dummy fetchRule() has no effect (redis is not ready)",
+			data: {
+				ruleIds,
+			},
+		}));
+
+		return [];
+	}
+
+	async findRules(
+		filter: AccessRulesFilter,
+		matchingFieldsOnly = false,
+		skipEmptyUserScopes = true,
+	): Promise<AccessRule[]> {
+		this.logger.info(() => ({
+			msg: "Dummy findRules() has no effect (redis is not ready)",
+			data: {
+				filter,
+			},
+		}));
+
+		return [];
+	}
+
+	async findRuleIds(
+		filter: AccessRulesFilter,
+		matchingFieldsOnly = false,
+	): Promise<string[]> {
+		this.logger.info(() => ({
+			msg: "Dummy findRuleIds() has no effect (redis is not ready)",
+			data: {
+				filter,
+			},
+		}));
+
+		return [];
+	}
+
+	async fetchAllRuleIds(
+		batchHandler: (ruleIds: string[]) => Promise<void>,
+	): Promise<void> {
+		this.logger.info(() => ({
+			msg: "Dummy fetchAllRuleIds() has no effect (redis is not ready)",
+		}));
+	}
+}

package/src/redis/redisClient.ts

@@ -0,0 +1,120 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import type { Logger } from "@prosopo/logger";
+import type { RedisClientType } from "redis";
+import { type ZodType, z } from "zod";
+
+export const REDIS_BATCH_SIZE = 1_000;
+
+export const getMissingRedisKeys = async (
+	client: RedisClientType,
+	keys: string[],
+): Promise<string[]> => {
+	const queries = client.multi();
+
+	keys.map((key) => {
+		queries.exists(key);
+	});
+
+	const records: unknown[] = await queries.exec();
+
+	const missingKeys: string[] = [];
+
+	records.map((exists, recordIndex) => {
+		if ("0" === String(exists)) {
+			const key = keys[recordIndex];
+
+			if (key) {
+				missingKeys.push(key);
+			}
+		}
+	});
+
+	return missingKeys;
+};
+
+export const fetchRedisHashRecords = async (
+	client: RedisClientType,
+	keys: string[],
+	logger: Logger,
+): Promise<{ records: object[]; expirations: (number | undefined)[] }> => {
+	const rulesPipe = client.multi();
+	const expirationPipe = client.multi();
+
+	for (const key of keys) {
+		rulesPipe.hGetAll(key);
+		expirationPipe.expireTime(key);
+	}
+
+	const records = (await rulesPipe.exec()) as object[];
+	const expirationRecords = (await expirationPipe.exec()) as unknown[];
+
+	return {
+		records: records,
+		expirations: parseExpirationRecords(expirationRecords, logger),
+	};
+};
+
+export const parseRedisRecords = <T>(
+	records: unknown[],
+	recordSchema: ZodType<T>,
+	logger: Logger,
+): T[] =>
+	records.flatMap((record) => {
+		const parseResult = recordSchema.safeParse(record);
+
+		if (parseResult.success) {
+			return [parseResult.data];
+		}
+
+		logger.error(() => ({
+			msg: "Failed to parse Redis record",
+			data: { record, error: parseResult.error },
+		}));
+
+		return [];
+	});
+
+const expirationRecordSchema = z.coerce.number();
+// Redis returns -1 when expiration is not set
+const UNSET_EXPIRATION_VALUE = -1;
+
+const parseExpirationRecords = <T>(
+	records: unknown[],
+	logger: Logger,
+): (number | undefined)[] =>
+	records.flatMap((record) => {
+		const parseResult = expirationRecordSchema.safeParse(record);
+
+		if (parseResult.success) {
+			const expiration =
+				UNSET_EXPIRATION_VALUE === parseResult.data
+					? undefined
+					: parseResult.data;
+
+			return [expiration];
+		}
+
+		logger.error(() => ({
+			msg: "Failed to parse Redis expiration record",
+			data: {
+				record,
+				error: parseResult.error,
+			},
+		}));
+
+		// ensure consistent output length
+		return [undefined];
+	});

package/src/redis/redisRuleIndex.ts

@@ -0,0 +1,85 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import type { AllKeys, Keys } from "@prosopo/common";
+import type { RedisIndex } from "@prosopo/redis-client";
+import { type RediSearchSchema, SCHEMA_FIELD_TYPE } from "@redis/search";
+import type {
+	AccessRule,
+	PolicyScope,
+	UserAttributes,
+	UserIp,
+	UserScope,
+} from "#policy/rule.js";
+import { makeAccessRuleHash } from "#policy/transformRule.js";
+
+export const userIpRedisSchema: RediSearchSchema = {
+	numericIpMaskMin: { type: SCHEMA_FIELD_TYPE.NUMERIC, INDEXMISSING: true },
+	numericIpMaskMax: { type: SCHEMA_FIELD_TYPE.NUMERIC, INDEXMISSING: true },
+	numericIp: { type: SCHEMA_FIELD_TYPE.NUMERIC, INDEXMISSING: true },
+} satisfies AllKeys<UserIp>;
+
+export const userAttributesRedisSchema: RediSearchSchema = {
+	userId: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
+	ja4Hash: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
+	headersHash: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
+	userAgentHash: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
+	headHash: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
+	// Use pipe separator for coords since JSON strings contain commas
+	coords: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true, SEPARATOR: "|" },
+	countryCode: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
+} satisfies AllKeys<UserAttributes>;
+
+export const userScopeRedisSchema: RediSearchSchema = {
+	...userAttributesRedisSchema,
+	...userIpRedisSchema,
+} satisfies Keys<UserScope>;
+
+export const policyScopeRedisSchema: RediSearchSchema = {
+	clientId: {
+		type: SCHEMA_FIELD_TYPE.TAG,
+		INDEXMISSING: true,
+	},
+} satisfies AllKeys<PolicyScope>;
+
+/**
+ * Note on the field type decision
+ *
+ * TAG is designed for the exact value matching
+ * TEXT is designed for the word-based and pattern matching
+ *
+ * For our goal TAG fits perfectly and, more performant
+ */
+export const accessRuleRedisSchema: RediSearchSchema = {
+	...policyScopeRedisSchema,
+	...userScopeRedisSchema,
+	groupId: { type: SCHEMA_FIELD_TYPE.TAG, INDEXMISSING: true },
+} satisfies Keys<AccessRule>;
+
+export const ACCESS_RULES_REDIS_INDEX_NAME = "index:user-access-rules";
+
+// names take space, so we use an acronym instead of the long-tailed one
+export const ACCESS_RULE_REDIS_KEY_PREFIX = "uar:";
+
+export const accessRulesRedisIndex: RedisIndex = {
+	name: ACCESS_RULES_REDIS_INDEX_NAME,
+	schema: accessRuleRedisSchema,
+	options: {
+		ON: "HASH" as const,
+		PREFIX: [ACCESS_RULE_REDIS_KEY_PREFIX],
+	},
+};
+
+export const getAccessRuleRedisKey = (rule: AccessRule): string =>
+	ACCESS_RULE_REDIS_KEY_PREFIX + makeAccessRuleHash(rule);

package/src/redis/redisRulesStorage.ts

@@ -0,0 +1,68 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import type { Logger } from "@prosopo/logger";
+import type { RedisConnection } from "@prosopo/redis-client";
+import type {
+	AccessRulesReader,
+	AccessRulesStorage,
+	AccessRulesWriter,
+} from "#policy/rulesStorage.js";
+import {
+	DummyRedisRulesReader,
+	RedisRulesReader,
+} from "./reader/redisRulesReader.js";
+import { DummyRedisRulesWriter, RedisRulesWriter } from "./redisRulesWriter.js";
+
+export const createRedisAccessRulesStorage = (
+	connection: RedisConnection,
+	logger: Logger,
+): AccessRulesStorage => {
+	const storage: AccessRulesStorage = composeStorage(
+		new DummyRedisRulesReader(logger),
+		new DummyRedisRulesWriter(logger),
+	);
+
+	connection.getClient().then((client) => {
+		const realStorage = composeStorage(
+			new RedisRulesReader(client, logger),
+			new RedisRulesWriter(client, logger),
+		);
+
+		// use assigning instead of var overwriting to keep the object reference
+		Object.assign(storage, realStorage);
+
+		logger.info(() => ({
+			msg: "RedisAccessRules storage got a ready Redis client",
+		}));
+	});
+
+	return storage;
+};
+
+const composeStorage = (
+	reader: AccessRulesReader,
+	writer: AccessRulesWriter,
+): AccessRulesStorage => ({
+	// reader
+	fetchRules: reader.fetchRules.bind(reader),
+	getMissingRuleIds: reader.getMissingRuleIds.bind(reader),
+	findRules: reader.findRules.bind(reader),
+	findRuleIds: reader.findRuleIds.bind(reader),
+	fetchAllRuleIds: reader.fetchAllRuleIds.bind(reader),
+	// writer
+	insertRules: writer.insertRules.bind(writer),
+	deleteRules: writer.deleteRules.bind(writer),
+	deleteAllRules: writer.deleteAllRules.bind(writer),
+});