@prosopo/user-access-policy 3.11.3 → 3.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build$colon$cjs.log +38 -37
- package/.turbo/turbo-build$colon$tsc.log +14 -14
- package/.turbo/turbo-build.log +39 -38
- package/CHANGELOG.md +30 -0
- package/dist/api/delete/deleteRules.d.ts +2 -2
- package/dist/api/delete/deleteRules.d.ts.map +1 -1
- package/dist/api/delete/deleteRules.js.map +1 -1
- package/dist/api/read/findRuleIds.d.ts +1 -1
- package/dist/api/read/findRuleIds.d.ts.map +1 -1
- package/dist/api/read/findRuleIds.js.map +1 -1
- package/dist/cjs/redis/reader/redisRulesQuery.cjs +3 -2
- package/dist/cjs/redis/reader/redisRulesReader.cjs +101 -1
- package/dist/cjs/redis/reader/redisRulesSplitQuery.cjs +78 -0
- package/dist/cjs/redis/redisRuleIndex.cjs +3 -1
- package/dist/cjs/redis/redisRulesWriter.cjs +17 -6
- package/dist/cjs/rule.cjs +2 -0
- package/dist/cjs/ruleInput/policyInput.cjs +9 -1
- package/dist/redis/reader/redisRulesQuery.d.ts.map +1 -1
- package/dist/redis/reader/redisRulesQuery.js +4 -3
- package/dist/redis/reader/redisRulesQuery.js.map +1 -1
- package/dist/redis/reader/redisRulesReader.d.ts +1 -0
- package/dist/redis/reader/redisRulesReader.d.ts.map +1 -1
- package/dist/redis/reader/redisRulesReader.js +101 -1
- package/dist/redis/reader/redisRulesReader.js.map +1 -1
- package/dist/redis/reader/redisRulesSplitQuery.d.ts +8 -0
- package/dist/redis/reader/redisRulesSplitQuery.d.ts.map +1 -0
- package/dist/redis/reader/redisRulesSplitQuery.js +78 -0
- package/dist/redis/reader/redisRulesSplitQuery.js.map +1 -0
- package/dist/redis/redisRuleIndex.d.ts +1 -0
- package/dist/redis/redisRuleIndex.d.ts.map +1 -1
- package/dist/redis/redisRuleIndex.js +2 -0
- package/dist/redis/redisRuleIndex.js.map +1 -1
- package/dist/redis/redisRulesWriter.d.ts +1 -1
- package/dist/redis/redisRulesWriter.d.ts.map +1 -1
- package/dist/redis/redisRulesWriter.js +14 -3
- package/dist/redis/redisRulesWriter.js.map +1 -1
- package/dist/rule.d.ts +1 -0
- package/dist/rule.d.ts.map +1 -1
- package/dist/rule.js +3 -1
- package/dist/rule.js.map +1 -1
- package/dist/ruleInput/policyInput.d.ts +2 -2
- package/dist/ruleInput/policyInput.d.ts.map +1 -1
- package/dist/ruleInput/policyInput.js +10 -2
- package/dist/ruleInput/policyInput.js.map +1 -1
- package/dist/ruleInput/ruleInput.d.ts +6 -6
- package/dist/ruleInput/ruleInput.d.ts.map +1 -1
- package/dist/ruleInput/ruleInput.js.map +1 -1
- package/dist/tests/redis/reader/redisRulesSplitQuery.unit.test.d.ts +2 -0
- package/dist/tests/redis/reader/redisRulesSplitQuery.unit.test.d.ts.map +1 -0
- package/dist/tests/redis/reader/redisRulesSplitQuery.unit.test.js +79 -0
- package/dist/tests/redis/reader/redisRulesSplitQuery.unit.test.js.map +1 -0
- package/dist/tests/redis/redisRulesReaderLoad.benchmark.integration.test.d.ts +2 -0
- package/dist/tests/redis/redisRulesReaderLoad.benchmark.integration.test.d.ts.map +1 -0
- package/dist/tests/redis/redisRulesReaderLoad.benchmark.integration.test.js +288 -0
- package/dist/tests/redis/redisRulesReaderLoad.benchmark.integration.test.js.map +1 -0
- package/package.json +3 -3
- package/src/api/delete/deleteRules.ts +7 -1
- package/src/api/read/findRuleIds.ts +3 -1
- package/src/redis/reader/redisRulesQuery.ts +12 -2
- package/src/redis/reader/redisRulesReader.ts +145 -7
- package/src/redis/reader/redisRulesSplitQuery.ts +170 -0
- package/src/redis/redisRuleIndex.ts +6 -0
- package/src/redis/redisRulesWriter.ts +19 -5
- package/src/rule.ts +12 -0
- package/src/ruleInput/policyInput.ts +17 -3
- package/src/ruleInput/ruleInput.ts +7 -1
- package/src/tests/redis/reader/redisRulesSplitQuery.unit.test.ts +132 -0
- package/src/tests/redis/redisRulesReaderLoad.benchmark.integration.test.ts +516 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
REDIS_QUERY_DIALECT,
|
|
21
21
|
getRulesRedisQuery,
|
|
22
22
|
} from "#policy/redis/reader/redisRulesQuery.js";
|
|
23
|
+
import { buildScopedBlockSubQueries } from "#policy/redis/reader/redisRulesSplitQuery.js";
|
|
23
24
|
import {
|
|
24
25
|
REDIS_BATCH_SIZE,
|
|
25
26
|
fetchRedisHashRecords,
|
|
@@ -63,6 +64,16 @@ export const SERVER_SIDE_RANK_TOP_N = 20;
|
|
|
63
64
|
// per-request path.
|
|
64
65
|
const GREEDY_MAX_CANDIDATES = REDIS_BATCH_SIZE * 10;
|
|
65
66
|
|
|
67
|
+
// Per-sub-query cap on the hot split-query path. Each sub-query probes a
|
|
68
|
+
// single discriminating index (a specific numericIp value, a specific
|
|
69
|
+
// ja4Hash value, etc.) so the natural cardinality is tiny — typically
|
|
70
|
+
// 1–5. The cap only fires on pathological rule-set shapes (e.g. many
|
|
71
|
+
// duplicate-value rules against one attribute) and prevents a bad shape
|
|
72
|
+
// from tanking the request. It intentionally lets the union grow past
|
|
73
|
+
// SERVER_SIDE_RANK_TOP_N because JS-side rankCandidateRules picks the
|
|
74
|
+
// winner and the extra candidates are cheap to HGETALL.
|
|
75
|
+
const SPLIT_MAX_CANDIDATES_PER_SUB = 500;
|
|
76
|
+
|
|
66
77
|
// Fields that contribute one specificity point each. Mirrors
|
|
67
78
|
// SCALAR_USER_SCOPE_FIELDS + clientId + ip-constraint in
|
|
68
79
|
// blacklistRequestInspector.ruleSpecificity. numericIp and the
|
|
@@ -166,16 +177,38 @@ export class RedisRulesReader implements AccessRulesReader {
|
|
|
166
177
|
): Promise<AccessRule[]> {
|
|
167
178
|
const query = getRulesRedisQuery(filter, matchingFieldsOnly);
|
|
168
179
|
|
|
169
|
-
if (
|
|
170
|
-
|
|
180
|
+
if (
|
|
181
|
+
skipEmptyUserScopes &&
|
|
182
|
+
// Sentinel query shape emitted by getPolicyScopeQuery when the
|
|
183
|
+
// caller passed neither a userScope nor a clientId — i.e. an
|
|
184
|
+
// empty filter that would otherwise resolve to "all global
|
|
185
|
+
// rules". Guard so an accidentally-empty filter can't return
|
|
186
|
+
// the full global rule set. The exact literal must stay in
|
|
187
|
+
// sync with GLOBAL_MATCH_CLAUSE_INNER over there.
|
|
188
|
+
query === "( @clientId:{global} | ismissing(@clientId) )"
|
|
189
|
+
) {
|
|
171
190
|
return [];
|
|
172
191
|
}
|
|
173
192
|
|
|
174
|
-
// Hot path: strict-match
|
|
175
|
-
//
|
|
176
|
-
//
|
|
177
|
-
//
|
|
178
|
-
//
|
|
193
|
+
// Hot path (checkForHardBlock / blockMiddleware): the strict-match
|
|
194
|
+
// caller passes blockOnly + a userScope, and every field returned
|
|
195
|
+
// gets JS-side re-ranked by rankCandidateRules anyway. Route it
|
|
196
|
+
// through the split-query path — one FT probe per populated
|
|
197
|
+
// request field, each using its own posting list — instead of the
|
|
198
|
+
// single wide FT.AGGREGATE whose APPLY step scaled linearly in
|
|
199
|
+
// the tenant's total rule count and tipped over under large
|
|
200
|
+
// bulk-ban populations.
|
|
201
|
+
if (
|
|
202
|
+
matchingFieldsOnly &&
|
|
203
|
+
filter.blockOnly &&
|
|
204
|
+
filter.userScope !== undefined
|
|
205
|
+
) {
|
|
206
|
+
return this.findBlockRulesSplit(filter);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Non-hot-path matchingFieldsOnly (e.g. Restrict-only lookups):
|
|
210
|
+
// keep the pre-existing server-side-ranked single query. Not on
|
|
211
|
+
// the hot path so the APPLY overhead is acceptable.
|
|
179
212
|
if (matchingFieldsOnly) {
|
|
180
213
|
return this.findRulesRanked(filter, query);
|
|
181
214
|
}
|
|
@@ -186,6 +219,111 @@ export class RedisRulesReader implements AccessRulesReader {
|
|
|
186
219
|
return this.findRulesGreedy(filter, query);
|
|
187
220
|
}
|
|
188
221
|
|
|
222
|
+
// Union candidate keys from each split sub-query, de-dupe, HGETALL,
|
|
223
|
+
// parse, hand back to `getPrioritisedAccessRule` for JS-side ranking.
|
|
224
|
+
// Ranking stays in JS because rankCandidateRules already encodes the
|
|
225
|
+
// strict "every populated rule field matches request" semantics —
|
|
226
|
+
// FT can't express that shape cheaply and the previous server-side
|
|
227
|
+
// APPLY / SORTBY imposed the linear-per-rule cost we're eliminating.
|
|
228
|
+
private async findBlockRulesSplit(
|
|
229
|
+
filter: AccessRulesFilter,
|
|
230
|
+
): Promise<AccessRule[]> {
|
|
231
|
+
const userScope = filter.userScope;
|
|
232
|
+
if (userScope === undefined) {
|
|
233
|
+
return [];
|
|
234
|
+
}
|
|
235
|
+
const clientId = filter.policyScope?.clientId;
|
|
236
|
+
|
|
237
|
+
const subQueries = buildScopedBlockSubQueries(userScope, clientId);
|
|
238
|
+
|
|
239
|
+
try {
|
|
240
|
+
const keyLists = await Promise.all(
|
|
241
|
+
subQueries.map((sub) =>
|
|
242
|
+
aggregateRedisKeys(
|
|
243
|
+
this.client,
|
|
244
|
+
sub.query,
|
|
245
|
+
this.logger,
|
|
246
|
+
undefined,
|
|
247
|
+
// Per-sub-query cap. Each probe hits a discriminating
|
|
248
|
+
// index (e.g. numericIp posting list), so the natural
|
|
249
|
+
// cardinality is tiny — this cap only fires on
|
|
250
|
+
// pathological rule-set shapes and mirrors
|
|
251
|
+
// SPLIT_MAX_CANDIDATES_PER_SUB below.
|
|
252
|
+
SPLIT_MAX_CANDIDATES_PER_SUB,
|
|
253
|
+
).catch((err) => {
|
|
254
|
+
// One sub-query failing (e.g. a transient RediSearch
|
|
255
|
+
// error) shouldn't drop the whole lookup — the other
|
|
256
|
+
// probes may still find the applicable rule. Log and
|
|
257
|
+
// continue with an empty result for this probe.
|
|
258
|
+
this.logger.error(() => ({
|
|
259
|
+
err,
|
|
260
|
+
data: {
|
|
261
|
+
inspect: util.inspect(
|
|
262
|
+
{ subKind: sub.kind, query: sub.query },
|
|
263
|
+
{ depth: null },
|
|
264
|
+
),
|
|
265
|
+
},
|
|
266
|
+
msg: "failed to execute split block sub-query",
|
|
267
|
+
}));
|
|
268
|
+
return [];
|
|
269
|
+
}),
|
|
270
|
+
),
|
|
271
|
+
);
|
|
272
|
+
|
|
273
|
+
const uniqueKeys = new Set<string>();
|
|
274
|
+
for (const list of keyLists) {
|
|
275
|
+
for (const key of list) {
|
|
276
|
+
uniqueKeys.add(key);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
if (uniqueKeys.size === 0) {
|
|
281
|
+
return [];
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
const ruleKeys = [...uniqueKeys];
|
|
285
|
+
|
|
286
|
+
this.logger.debug(() => ({
|
|
287
|
+
msg: "Executed split block sub-queries",
|
|
288
|
+
data: {
|
|
289
|
+
inspect: util.inspect(
|
|
290
|
+
{
|
|
291
|
+
subQueryCount: subQueries.length,
|
|
292
|
+
uniqueCandidates: ruleKeys.length,
|
|
293
|
+
perSubQueryCounts: keyLists.map((l, i) => ({
|
|
294
|
+
kind: subQueries[i]?.kind,
|
|
295
|
+
found: l.length,
|
|
296
|
+
})),
|
|
297
|
+
filter,
|
|
298
|
+
},
|
|
299
|
+
{ depth: null },
|
|
300
|
+
),
|
|
301
|
+
},
|
|
302
|
+
}));
|
|
303
|
+
|
|
304
|
+
const { records } = await fetchRedisHashRecords(
|
|
305
|
+
this.client,
|
|
306
|
+
ruleKeys,
|
|
307
|
+
this.logger,
|
|
308
|
+
);
|
|
309
|
+
|
|
310
|
+
const nonEmptyRecords = records.filter(
|
|
311
|
+
(record) => Object.keys(record).length > 0,
|
|
312
|
+
);
|
|
313
|
+
|
|
314
|
+
return parseRedisRecords(nonEmptyRecords, accessRuleInput, this.logger);
|
|
315
|
+
} catch (e) {
|
|
316
|
+
this.logger.error(() => ({
|
|
317
|
+
err: e,
|
|
318
|
+
data: {
|
|
319
|
+
inspect: util.inspect({ filter }, { depth: null }),
|
|
320
|
+
},
|
|
321
|
+
msg: "failed to execute split block query set",
|
|
322
|
+
}));
|
|
323
|
+
return [];
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
189
327
|
private async findRulesRanked(
|
|
190
328
|
filter: AccessRulesFilter,
|
|
191
329
|
query: string,
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
// Copyright 2021-2026 Prosopo (UK) Ltd.
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
|
|
15
|
+
import {
|
|
16
|
+
AccessPolicyType,
|
|
17
|
+
GLOBAL_CLIENT_SCOPE_SENTINEL,
|
|
18
|
+
type UserScope,
|
|
19
|
+
} from "#policy/rule.js";
|
|
20
|
+
|
|
21
|
+
// Escapes special characters in Redis TAG queries. Mirrors the escape
|
|
22
|
+
// function in redisRulesQuery.ts — kept local to avoid coupling the two
|
|
23
|
+
// query builders since they'll diverge in shape.
|
|
24
|
+
const escapeTagValue = (value: string): string =>
|
|
25
|
+
value.replace(/([,.<>{}\[\]"':;!@#$%^&*()\-+=~|/\\])/g, "\\$1");
|
|
26
|
+
|
|
27
|
+
// Fields where TAG queries need escaping (they may hold JSON etc.).
|
|
28
|
+
const FIELDS_REQUIRING_ESCAPE: ReadonlySet<keyof UserScope> = new Set([
|
|
29
|
+
"coords",
|
|
30
|
+
]);
|
|
31
|
+
|
|
32
|
+
// Fields indexed as NUMERIC — require range syntax `@x:[N N]` not TAG.
|
|
33
|
+
const NUMERIC_FIELDS: ReadonlySet<keyof UserScope> = new Set(["asn"]);
|
|
34
|
+
|
|
35
|
+
// Enumerating the scalar (non-IP) user-scope fields keeps the "no
|
|
36
|
+
// user-scope constraint" fall-through query stable when userScopeSchema
|
|
37
|
+
// grows a new field — the enumeration below drives ismissing() clauses.
|
|
38
|
+
const SCALAR_USER_SCOPE_FIELDS: ReadonlyArray<keyof UserScope> = [
|
|
39
|
+
"userId",
|
|
40
|
+
"ja4Hash",
|
|
41
|
+
"headersHash",
|
|
42
|
+
"userAgentHash",
|
|
43
|
+
"headHash",
|
|
44
|
+
"coords",
|
|
45
|
+
"countryCode",
|
|
46
|
+
"asn",
|
|
47
|
+
];
|
|
48
|
+
|
|
49
|
+
const ALL_USER_SCOPE_FIELDS: ReadonlyArray<keyof UserScope> = [
|
|
50
|
+
...SCALAR_USER_SCOPE_FIELDS,
|
|
51
|
+
"numericIp",
|
|
52
|
+
"numericIpMaskMin",
|
|
53
|
+
"numericIpMaskMax",
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
// Global rules written under the new sentinel are found via
|
|
57
|
+
// `@clientId:{global}`; the ismissing() fallback covers legacy rules
|
|
58
|
+
// that predate the sentinel stamp (rehash-all migrates them). This
|
|
59
|
+
// disjunction still hits the clientId posting list rather than walking
|
|
60
|
+
// the full rule set the way the old single-query path did.
|
|
61
|
+
const GLOBAL_SCOPE_INNER = `@clientId:{${GLOBAL_CLIENT_SCOPE_SENTINEL}} | ismissing(@clientId)`;
|
|
62
|
+
|
|
63
|
+
const buildScopeClause = (clientId: string | undefined): string => {
|
|
64
|
+
if (clientId === undefined) {
|
|
65
|
+
return `( ${GLOBAL_SCOPE_INNER} )`;
|
|
66
|
+
}
|
|
67
|
+
return `( @clientId:{${clientId}} | ${GLOBAL_SCOPE_INNER} )`;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
const buildFieldClause = (
|
|
71
|
+
field: keyof UserScope,
|
|
72
|
+
value: unknown,
|
|
73
|
+
): string | null => {
|
|
74
|
+
if (value === undefined) {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
const stringValue = String(value);
|
|
78
|
+
if (NUMERIC_FIELDS.has(field)) {
|
|
79
|
+
return `@${field}:[${stringValue} ${stringValue}]`;
|
|
80
|
+
}
|
|
81
|
+
const queryValue = FIELDS_REQUIRING_ESCAPE.has(field)
|
|
82
|
+
? escapeTagValue(stringValue)
|
|
83
|
+
: stringValue;
|
|
84
|
+
return `@${field}:{${queryValue}}`;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
type SubQuery = {
|
|
88
|
+
// A short label used for logging + metrics. Not part of the query
|
|
89
|
+
// wire format.
|
|
90
|
+
kind: string;
|
|
91
|
+
query: string;
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Build the set of FT sub-queries that together cover every rule that
|
|
96
|
+
* could apply to a request for hard-block resolution.
|
|
97
|
+
*
|
|
98
|
+
* The single-query design this replaces produced one FT.AGGREGATE whose
|
|
99
|
+
* candidate set was the entire scope's rule pool — the APPLY / SORTBY
|
|
100
|
+
* pipeline then walked every rule per request, which degrades sharply
|
|
101
|
+
* once rule populations reach the 10k+ range. The split approach fires
|
|
102
|
+
* one probe per populated request field. Each probe uses the field's
|
|
103
|
+
* own index posting list, so a 17k IP-ban population collapses to 1
|
|
104
|
+
* result (exact IP) plus a handful (CIDR ranges containing the IP)
|
|
105
|
+
* instead of forcing a full-set scan.
|
|
106
|
+
*
|
|
107
|
+
* A single fall-through probe covers rules with no user-scope constraint
|
|
108
|
+
* (client-wide block); those are rare so the ismissing()-heavy query is
|
|
109
|
+
* cheap in practice.
|
|
110
|
+
*
|
|
111
|
+
* Rank + strict-match filtering happens in JS via `rankCandidateRules`
|
|
112
|
+
* after the union — the FT layer is used purely as a candidate fetcher.
|
|
113
|
+
*/
|
|
114
|
+
export const buildScopedBlockSubQueries = (
|
|
115
|
+
userScope: UserScope,
|
|
116
|
+
clientId: string | undefined,
|
|
117
|
+
): SubQuery[] => {
|
|
118
|
+
const typeClause = `@type:{${AccessPolicyType.Block}}`;
|
|
119
|
+
const scopeClause = buildScopeClause(clientId);
|
|
120
|
+
const prefix = `${typeClause} ${scopeClause}`;
|
|
121
|
+
|
|
122
|
+
const subQueries: SubQuery[] = [];
|
|
123
|
+
|
|
124
|
+
// One probe per populated scalar user-scope field. Each uses that
|
|
125
|
+
// field's posting list rather than intersecting ismissing() across
|
|
126
|
+
// the full set.
|
|
127
|
+
for (const field of SCALAR_USER_SCOPE_FIELDS) {
|
|
128
|
+
const clause = buildFieldClause(field, userScope[field]);
|
|
129
|
+
if (clause === null) {
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
subQueries.push({
|
|
133
|
+
kind: `field:${field}`,
|
|
134
|
+
query: `${prefix} ${clause}`,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// IP: two disjoint probes so each hits a single index. The exact-IP
|
|
139
|
+
// probe catches individual-IP bans (the dominant shape in bulk-ban
|
|
140
|
+
// scripts). The mask-range probe catches CIDR aggregates containing
|
|
141
|
+
// the request IP. Splitting them means neither has to walk a mixed
|
|
142
|
+
// posting list — the mask-range probe was the slower of the two
|
|
143
|
+
// under load, so isolating it lets Redis skip it entirely for hits
|
|
144
|
+
// on the fast path.
|
|
145
|
+
const requestIp = userScope.numericIp;
|
|
146
|
+
if (requestIp !== undefined) {
|
|
147
|
+
subQueries.push({
|
|
148
|
+
kind: "ip:exact",
|
|
149
|
+
query: `${prefix} @numericIp:[${requestIp} ${requestIp}]`,
|
|
150
|
+
});
|
|
151
|
+
subQueries.push({
|
|
152
|
+
kind: "ip:mask",
|
|
153
|
+
query: `${prefix} @numericIpMaskMin:[-inf ${requestIp}] @numericIpMaskMax:[${requestIp} +inf]`,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Fall-through: rules that constrain nothing on the user scope
|
|
158
|
+
// ("block everything for this client / everyone"). Rare in
|
|
159
|
+
// practice — a handful per tenant at most — so the ismissing()
|
|
160
|
+
// intersection is cheap despite touching every field.
|
|
161
|
+
const noScopeIsmissing = ALL_USER_SCOPE_FIELDS.map(
|
|
162
|
+
(field) => `ismissing(@${field})`,
|
|
163
|
+
).join(" ");
|
|
164
|
+
subQueries.push({
|
|
165
|
+
kind: "no-user-scope",
|
|
166
|
+
query: `${prefix} ${noScopeIsmissing}`,
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
return subQueries;
|
|
170
|
+
};
|
|
@@ -80,6 +80,12 @@ export const ACCESS_RULES_REDIS_INDEX_NAME = "index:user-access-rules";
|
|
|
80
80
|
// names take space, so we use an acronym instead of the long-tailed one
|
|
81
81
|
export const ACCESS_RULE_REDIS_KEY_PREFIX = "uar:";
|
|
82
82
|
|
|
83
|
+
// Re-exported from #policy/rule.js — that module is types-only so both
|
|
84
|
+
// the parse side (policyInput.ts) and the write/index side (this file
|
|
85
|
+
// and redisRulesWriter.ts) can import the sentinel without pulling
|
|
86
|
+
// transformRule into a cycle.
|
|
87
|
+
export { GLOBAL_CLIENT_SCOPE_SENTINEL } from "#policy/rule.js";
|
|
88
|
+
|
|
83
89
|
export const accessRulesRedisIndex: RedisIndex = {
|
|
84
90
|
name: ACCESS_RULES_REDIS_INDEX_NAME,
|
|
85
91
|
schema: accessRuleRedisSchema,
|
|
@@ -16,7 +16,7 @@ import { chunkIntoBatches, executeBatchesSequentially } from "@prosopo/common";
|
|
|
16
16
|
import type { Logger } from "@prosopo/logger";
|
|
17
17
|
import type { RedisClientType } from "redis";
|
|
18
18
|
import { REDIS_BATCH_SIZE } from "#policy/redis/redisClient.js";
|
|
19
|
-
import type
|
|
19
|
+
import { type AccessRule, GLOBAL_CLIENT_SCOPE_SENTINEL } from "#policy/rule.js";
|
|
20
20
|
import type {
|
|
21
21
|
AccessRuleEntry,
|
|
22
22
|
AccessRulesWriter,
|
|
@@ -120,10 +120,24 @@ export class RedisRulesWriter implements AccessRulesWriter {
|
|
|
120
120
|
}
|
|
121
121
|
}
|
|
122
122
|
|
|
123
|
-
export const getRedisRuleValue = (rule: AccessRule): Record<string, string> =>
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
123
|
+
export const getRedisRuleValue = (rule: AccessRule): Record<string, string> => {
|
|
124
|
+
const record: Record<string, string> = {};
|
|
125
|
+
for (const [key, value] of Object.entries(rule)) {
|
|
126
|
+
if (value === undefined) {
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
record[key] = String(value);
|
|
130
|
+
}
|
|
131
|
+
// Global rules used to be stored with no clientId field at all and looked
|
|
132
|
+
// up via `ismissing(@clientId)`. Stamp the sentinel so the read path can
|
|
133
|
+
// probe `@clientId:{global}` instead — a posting-list intersection is
|
|
134
|
+
// orders of magnitude cheaper than ismissing over a large rule set.
|
|
135
|
+
// Rules with a real clientId are untouched.
|
|
136
|
+
if (record.clientId === undefined) {
|
|
137
|
+
record.clientId = GLOBAL_CLIENT_SCOPE_SENTINEL;
|
|
138
|
+
}
|
|
139
|
+
return record;
|
|
140
|
+
};
|
|
127
141
|
|
|
128
142
|
export class DummyRedisRulesWriter implements AccessRulesWriter {
|
|
129
143
|
constructor(private readonly logger: Logger) {}
|
package/src/rule.ts
CHANGED
|
@@ -18,6 +18,18 @@ export enum AccessPolicyType {
|
|
|
18
18
|
Restrict = "restrict",
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
+
// Sentinel stamped on the Redis `clientId` field for rules that would
|
|
22
|
+
// otherwise store undefined (i.e. global rules). Lets the read path
|
|
23
|
+
// probe `@clientId:{global}` via the posting-list index instead of the
|
|
24
|
+
// expensive `ismissing(@clientId)` set-difference walk that degrades
|
|
25
|
+
// sharply once the global rule set grows into the 10k+ range. The
|
|
26
|
+
// reader converts it back to `undefined` at parse time so downstream
|
|
27
|
+
// code sees the original AccessRule shape. Kept in this types-only
|
|
28
|
+
// module so both the writer/index side and the parser/input side can
|
|
29
|
+
// import it without pulling in a circular dependency through
|
|
30
|
+
// transformRule.
|
|
31
|
+
export const GLOBAL_CLIENT_SCOPE_SENTINEL = "global";
|
|
32
|
+
|
|
21
33
|
export type AccessPolicy = {
|
|
22
34
|
type: AccessPolicyType;
|
|
23
35
|
captchaType?: CaptchaType;
|
|
@@ -14,10 +14,11 @@
|
|
|
14
14
|
|
|
15
15
|
import type { AllKeys } from "@prosopo/common";
|
|
16
16
|
import { CaptchaTypeSchema } from "@prosopo/types";
|
|
17
|
-
import {
|
|
17
|
+
import { z } from "zod";
|
|
18
18
|
import {
|
|
19
19
|
type AccessPolicy,
|
|
20
20
|
AccessPolicyType,
|
|
21
|
+
GLOBAL_CLIENT_SCOPE_SENTINEL,
|
|
21
22
|
type PolicyScope,
|
|
22
23
|
} from "#policy/rule.js";
|
|
23
24
|
|
|
@@ -57,6 +58,19 @@ export const sanitizeAccessPolicy = (policy: AccessPolicy): AccessPolicy => {
|
|
|
57
58
|
return policy;
|
|
58
59
|
};
|
|
59
60
|
|
|
61
|
+
// `satisfies ZodType<PolicyScope>` is intentionally omitted (matches
|
|
62
|
+
// accessPolicyInput above): the `preprocess` wrapper widens the schema's
|
|
63
|
+
// input type to `unknown`, which fails the `ZodType<T, ZodTypeDef, T>`
|
|
64
|
+
// identity check. The `AllKeys<PolicyScope>` constraint still catches
|
|
65
|
+
// missing-field regressions.
|
|
60
66
|
export const policyScopeInput = z.object({
|
|
61
|
-
|
|
62
|
-
}
|
|
67
|
+
// `getRedisRuleValue` stamps a sentinel string on global rules so the
|
|
68
|
+
// read-time query can probe `@clientId:{global}` cheaply. Undo the
|
|
69
|
+
// stamp here so consumers (rankCandidateRules, response payloads,
|
|
70
|
+
// tests) continue to see `undefined` for global rules — the mongoose
|
|
71
|
+
// side and the API input side never emit the sentinel.
|
|
72
|
+
clientId: z.preprocess(
|
|
73
|
+
(v) => (v === GLOBAL_CLIENT_SCOPE_SENTINEL ? undefined : v),
|
|
74
|
+
z.coerce.string().optional(),
|
|
75
|
+
),
|
|
76
|
+
} satisfies AllKeys<PolicyScope>);
|
|
@@ -73,6 +73,12 @@ export type AccessRulesFilterInput = AccessRulesFilter & {
|
|
|
73
73
|
policyScopes?: PolicyScope[];
|
|
74
74
|
};
|
|
75
75
|
|
|
76
|
+
// `satisfies ZodType<AccessRulesFilterInput>` is intentionally omitted:
|
|
77
|
+
// `policyScopeInput.clientId` uses `z.preprocess` to unwrap the Redis
|
|
78
|
+
// `global` sentinel, which widens the input type to `unknown`. The
|
|
79
|
+
// output type is still `AccessRulesFilterInput` (Zod's `_output`); the
|
|
80
|
+
// downstream `DeleteRulesSchema` / `FindRulesSchema` use the relaxed
|
|
81
|
+
// `ZodType<T, ZodTypeDef, unknown>` form for the same reason.
|
|
76
82
|
export const accessRulesFilterInput = z.object({
|
|
77
83
|
policyScope: policyScopeInput.optional(),
|
|
78
84
|
policyScopes: z.array(policyScopeInput).optional(),
|
|
@@ -85,7 +91,7 @@ export const accessRulesFilterInput = z.object({
|
|
|
85
91
|
.default(FilterScopeMatch.Exact),
|
|
86
92
|
groupId: z.string().optional(),
|
|
87
93
|
blockOnly: z.boolean().optional(),
|
|
88
|
-
} satisfies AllKeys<AccessRulesFilterInput>)
|
|
94
|
+
} satisfies AllKeys<AccessRulesFilterInput>);
|
|
89
95
|
|
|
90
96
|
export const getAccessRuleFiltersFromInput = (
|
|
91
97
|
filterInput: AccessRulesFilterInput,
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// Copyright 2021-2026 Prosopo (UK) Ltd.
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
|
|
15
|
+
import { describe, expect, it } from "vitest";
|
|
16
|
+
import { buildScopedBlockSubQueries } from "#policy/redis/reader/redisRulesSplitQuery.js";
|
|
17
|
+
|
|
18
|
+
describe("buildScopedBlockSubQueries", () => {
|
|
19
|
+
it("emits one sub-query per populated user-scope field", () => {
|
|
20
|
+
const subs = buildScopedBlockSubQueries(
|
|
21
|
+
{
|
|
22
|
+
ja4Hash: "abc",
|
|
23
|
+
userAgentHash: "def",
|
|
24
|
+
},
|
|
25
|
+
"client-A",
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
const kinds = subs.map((s) => s.kind).sort();
|
|
29
|
+
// Two field probes + one fall-through for "no user-scope
|
|
30
|
+
// constraint" rules. No IP sub-queries because request has no IP.
|
|
31
|
+
expect(kinds).toEqual([
|
|
32
|
+
"field:ja4Hash",
|
|
33
|
+
"field:userAgentHash",
|
|
34
|
+
"no-user-scope",
|
|
35
|
+
]);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it("emits both ip-exact and ip-mask sub-queries when request has an IP", () => {
|
|
39
|
+
const subs = buildScopedBlockSubQueries(
|
|
40
|
+
{
|
|
41
|
+
numericIp: 3232235777n, // 192.168.1.1
|
|
42
|
+
},
|
|
43
|
+
"client-A",
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
const kinds = subs.map((s) => s.kind);
|
|
47
|
+
expect(kinds).toContain("ip:exact");
|
|
48
|
+
expect(kinds).toContain("ip:mask");
|
|
49
|
+
|
|
50
|
+
const ipExact = subs.find((s) => s.kind === "ip:exact");
|
|
51
|
+
expect(ipExact?.query).toContain("@numericIp:[3232235777 3232235777]");
|
|
52
|
+
|
|
53
|
+
const ipMask = subs.find((s) => s.kind === "ip:mask");
|
|
54
|
+
expect(ipMask?.query).toContain("@numericIpMaskMin:[-inf 3232235777]");
|
|
55
|
+
expect(ipMask?.query).toContain("@numericIpMaskMax:[3232235777 +inf]");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("scopes every sub-query with @type:{block} and @clientId probe", () => {
|
|
59
|
+
const subs = buildScopedBlockSubQueries({ ja4Hash: "abc" }, "client-A");
|
|
60
|
+
|
|
61
|
+
for (const sub of subs) {
|
|
62
|
+
expect(sub.query).toContain("@type:{block}");
|
|
63
|
+
// Every sub-query must include the client-or-global scope
|
|
64
|
+
// probe. `@clientId:{client-A}` matches client-scoped rules;
|
|
65
|
+
// `@clientId:{global}` matches new-format global rules;
|
|
66
|
+
// `ismissing(@clientId)` matches legacy pre-sentinel rules.
|
|
67
|
+
expect(sub.query).toMatch(/@clientId:\{client-A\}/);
|
|
68
|
+
expect(sub.query).toMatch(/@clientId:\{global\}/);
|
|
69
|
+
expect(sub.query).toMatch(/ismissing\(@clientId\)/);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("skips the client-specific probe when no clientId is passed", () => {
|
|
74
|
+
const subs = buildScopedBlockSubQueries({ ja4Hash: "abc" }, undefined);
|
|
75
|
+
|
|
76
|
+
for (const sub of subs) {
|
|
77
|
+
// No client-scoped probe — request is scoped to global rules
|
|
78
|
+
// only. `global` sentinel and `ismissing()` fallback remain.
|
|
79
|
+
expect(sub.query).toMatch(/@clientId:\{global\}/);
|
|
80
|
+
expect(sub.query).toMatch(/ismissing\(@clientId\)/);
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it("emits a no-user-scope fall-through query with ismissing() over every user-scope field", () => {
|
|
85
|
+
const subs = buildScopedBlockSubQueries({ ja4Hash: "abc" }, "client-A");
|
|
86
|
+
|
|
87
|
+
const fallThrough = subs.find((s) => s.kind === "no-user-scope");
|
|
88
|
+
expect(fallThrough).toBeDefined();
|
|
89
|
+
// Every user-scope field must be constrained as missing so the
|
|
90
|
+
// probe returns only rules that are truly client-wide blocks
|
|
91
|
+
// (no user-scope constraint at all).
|
|
92
|
+
for (const field of [
|
|
93
|
+
"userId",
|
|
94
|
+
"ja4Hash",
|
|
95
|
+
"headersHash",
|
|
96
|
+
"userAgentHash",
|
|
97
|
+
"headHash",
|
|
98
|
+
"coords",
|
|
99
|
+
"countryCode",
|
|
100
|
+
"asn",
|
|
101
|
+
"numericIp",
|
|
102
|
+
"numericIpMaskMin",
|
|
103
|
+
"numericIpMaskMax",
|
|
104
|
+
]) {
|
|
105
|
+
expect(fallThrough?.query).toContain(`ismissing(@${field})`);
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it("uses NUMERIC range syntax for asn, not TAG syntax", () => {
|
|
110
|
+
const subs = buildScopedBlockSubQueries({ asn: 205016 }, "client-A");
|
|
111
|
+
|
|
112
|
+
const asnSub = subs.find((s) => s.kind === "field:asn");
|
|
113
|
+
expect(asnSub).toBeDefined();
|
|
114
|
+
// asn is indexed as NUMERIC; TAG syntax (@asn:{...}) would
|
|
115
|
+
// silently fail to match. Must use range syntax.
|
|
116
|
+
expect(asnSub?.query).toContain("@asn:[205016 205016]");
|
|
117
|
+
expect(asnSub?.query).not.toContain("@asn:{");
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("escapes JSON special characters in coords tag queries", () => {
|
|
121
|
+
const subs = buildScopedBlockSubQueries(
|
|
122
|
+
{ coords: "[[[100,200]]]" },
|
|
123
|
+
"client-A",
|
|
124
|
+
);
|
|
125
|
+
|
|
126
|
+
const coordsSub = subs.find((s) => s.kind === "field:coords");
|
|
127
|
+
expect(coordsSub).toBeDefined();
|
|
128
|
+
// Coords hold JSON with brackets and commas — every one must be
|
|
129
|
+
// backslash-escaped or the TAG query silently returns no matches.
|
|
130
|
+
expect(coordsSub?.query).toContain("@coords:{\\[\\[\\[100\\,200\\]\\]\\]}");
|
|
131
|
+
});
|
|
132
|
+
});
|