@prosopo/user-access-policy 3.14.14 → 3.14.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build$colon$cjs.log +2 -5
- package/.turbo/turbo-build$colon$tsc.log +14 -17
- package/.turbo/turbo-build.log +3 -9
- package/CHANGELOG.md +18 -0
- package/LICENSE +201 -0
- package/package.json +83 -81
- package/src/.export.ts +0 -65
- package/src/api/.export.ts +0 -25
- package/src/api/accessRulesApiClient.ts +0 -13
- package/src/api/delete/.export.ts +0 -18
- package/src/api/delete/deleteAllRules.ts +0 -47
- package/src/api/delete/deleteRuleGroups.ts +0 -96
- package/src/api/delete/deleteRules.ts +0 -87
- package/src/api/read/.export.ts +0 -25
- package/src/api/read/fetchRules.ts +0 -96
- package/src/api/read/findRuleIds.ts +0 -97
- package/src/api/read/getMissingIds.ts +0 -81
- package/src/api/ruleApiRoutes.ts +0 -146
- package/src/api/rulesApiClient.ts +0 -154
- package/src/api/write/.export.ts +0 -15
- package/src/api/write/insertRules.ts +0 -185
- package/src/api/write/rehashRules.ts +0 -85
- package/src/classifyBrowser.ts +0 -77
- package/src/classifyOs.ts +0 -74
- package/src/headerMatch.ts +0 -180
- package/src/matchedRule.ts +0 -99
- package/src/mongoose/.export.ts +0 -15
- package/src/mongoose/mongooseRuleSchema.ts +0 -75
- package/src/redis/.export.ts +0 -17
- package/src/redis/reader/redisAggregate.ts +0 -157
- package/src/redis/reader/redisRulesQuery.ts +0 -268
- package/src/redis/reader/redisRulesReader.ts +0 -738
- package/src/redis/reader/redisRulesSplitQuery.ts +0 -234
- package/src/redis/redisClient.ts +0 -125
- package/src/redis/redisRuleIndex.ts +0 -121
- package/src/redis/redisRulesStorage.ts +0 -68
- package/src/redis/redisRulesWriter.ts +0 -172
- package/src/rule.ts +0 -122
- package/src/ruleInput/.export.ts +0 -19
- package/src/ruleInput/policyInput.ts +0 -120
- package/src/ruleInput/ruleInput.ts +0 -123
- package/src/ruleInput/userScopeInput.ts +0 -142
- package/src/ruleRecord.ts +0 -77
- package/src/rulesStorage.ts +0 -90
- package/src/tests/blockingRuleShapes.unit.test.ts +0 -244
- package/src/tests/classifyBrowser.unit.test.ts +0 -138
- package/src/tests/classifyOs.unit.test.ts +0 -97
- package/src/tests/headerMatch.unit.test.ts +0 -350
- package/src/tests/insertRulesEndpoint.unit.test.ts +0 -89
- package/src/tests/matchedRule.unit.test.ts +0 -184
- package/src/tests/policyInput.unit.test.ts +0 -218
- package/src/tests/redis/deferredPoolClause.benchmark.ts +0 -249
- package/src/tests/redis/reader/redisAggregate.unit.test.ts +0 -121
- package/src/tests/redis/reader/redisRulesQuery.unit.test.ts +0 -349
- package/src/tests/redis/reader/redisRulesSplitQuery.unit.test.ts +0 -297
- package/src/tests/redis/redisRulesReaderLoad.benchmark.integration.test.ts +0 -520
- package/src/tests/redis/redisRulesReaderRank.benchmark.integration.test.ts +0 -350
- package/src/tests/redis/redisRulesStorage.integration.test.ts +0 -1811
- package/src/tests/testLogger.ts +0 -38
- package/src/tests/transformRule.unit.test.ts +0 -345
- package/src/transformRule.ts +0 -133
- package/tsconfig.cjs.json +0 -41
- package/tsconfig.json +0 -47
- package/tsconfig.tsbuildinfo +0 -1
- package/tsconfig.types.json +0 -9
|
@@ -1,738 +0,0 @@
|
|
|
1
|
-
// Copyright 2021-2026 Prosopo (UK) Ltd.
|
|
2
|
-
//
|
|
3
|
-
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
// you may not use this file except in compliance with the License.
|
|
5
|
-
// You may obtain a copy of the License at
|
|
6
|
-
//
|
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
//
|
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
// See the License for the specific language governing permissions and
|
|
13
|
-
// limitations under the License.
|
|
14
|
-
|
|
15
|
-
import * as util from "node:util";
|
|
16
|
-
import { chunkIntoBatches, executeBatchesSequentially } from "@prosopo/common";
|
|
17
|
-
import type { Logger } from "@prosopo/logger";
|
|
18
|
-
import type { RedisClientType } from "redis";
|
|
19
|
-
import {
|
|
20
|
-
REDIS_QUERY_DIALECT,
|
|
21
|
-
getRulesRedisQuery,
|
|
22
|
-
} from "#policy/redis/reader/redisRulesQuery.js";
|
|
23
|
-
import { buildScopedRulesSubQueries } from "#policy/redis/reader/redisRulesSplitQuery.js";
|
|
24
|
-
import {
|
|
25
|
-
REDIS_BATCH_SIZE,
|
|
26
|
-
fetchRedisHashRecords,
|
|
27
|
-
getMissingRedisKeys,
|
|
28
|
-
parseRedisRecords,
|
|
29
|
-
} from "#policy/redis/redisClient.js";
|
|
30
|
-
import {
|
|
31
|
-
ACCESS_RULES_REDIS_INDEX_NAME,
|
|
32
|
-
ACCESS_RULE_REDIS_KEY_PREFIX,
|
|
33
|
-
} from "#policy/redis/redisRuleIndex.js";
|
|
34
|
-
import { AccessPolicyType, type AccessRule } from "#policy/rule.js";
|
|
35
|
-
import { accessRuleInput } from "#policy/ruleInput/ruleInput.js";
|
|
36
|
-
import type {
|
|
37
|
-
AccessRuleEntry,
|
|
38
|
-
AccessRulesFilter,
|
|
39
|
-
AccessRulesReader,
|
|
40
|
-
} from "#policy/rulesStorage.js";
|
|
41
|
-
import {
|
|
42
|
-
aggregateRedisKeys,
|
|
43
|
-
searchRedisKeysBounded,
|
|
44
|
-
} from "./redisAggregate.js";
|
|
45
|
-
|
|
46
|
-
// Server-side specificity ranking config.
|
|
47
|
-
//
|
|
48
|
-
// Strict-match findRules (matchingFieldsOnly=true) used to pull every
|
|
49
|
-
// matching rule's hash into Node and sort in JS. With the greedy regression
|
|
50
|
-
// in #2689 this hauled ~1190 hashes per request and pegged provider CPU.
|
|
51
|
-
//
|
|
52
|
-
// The new path lets RediSearch compute specificity itself: one
|
|
53
|
-
// FT.AGGREGATE that filters via the strict AND-of-disjunctions query,
|
|
54
|
-
// counts populated rule fields with APPLY+exists(), sorts by that
|
|
55
|
-
// derived score DESC, and returns only the top-N candidates. Node
|
|
56
|
-
// receives at most TOP_N small records, no HGETALL fanout, no JS rank.
|
|
57
|
-
//
|
|
58
|
-
// TOP_N >> 1 leaves headroom for the deferToVerify filter that runs
|
|
59
|
-
// post-aggregate (a Block rule with deferToVerify is skipped in the
|
|
60
|
-
// blockMiddleware path, so we need a few alternates ready). 20 covers
|
|
61
|
-
// the worst observed concentration of deferToVerify rules per scope by
|
|
62
|
-
// comfortable margin while keeping payload tiny.
|
|
63
|
-
export const SERVER_SIDE_RANK_TOP_N = 20;
|
|
64
|
-
|
|
65
|
-
// Safety cap for the greedy (admin/internal) path. Generous because
|
|
66
|
-
// admin tooling depends on the full result set; not on the hot
|
|
67
|
-
// per-request path.
|
|
68
|
-
const GREEDY_MAX_CANDIDATES = REDIS_BATCH_SIZE * 10;
|
|
69
|
-
|
|
70
|
-
// Per-sub-query cap on the hot split-query path. Each sub-query probes a
|
|
71
|
-
// single discriminating index (a specific numericIp value, a specific
|
|
72
|
-
// ja4Hash value, etc.) so the natural cardinality is tiny — typically
|
|
73
|
-
// 1–5. The cap only fires on pathological rule-set shapes (e.g. many
|
|
74
|
-
// duplicate-value rules against one attribute) and prevents a bad shape
|
|
75
|
-
// from tanking the request. It intentionally lets the union grow past
|
|
76
|
-
// SERVER_SIDE_RANK_TOP_N because JS-side rankCandidateRules picks the
|
|
77
|
-
// winner and the extra candidates are cheap to HGETALL.
|
|
78
|
-
const SPLIT_MAX_CANDIDATES_PER_SUB = 500;
|
|
79
|
-
|
|
80
|
-
// Fields that contribute one specificity point each. Mirrors
|
|
81
|
-
// SCALAR_USER_SCOPE_FIELDS + clientId + ip-constraint in
|
|
82
|
-
// blacklistRequestInspector.ruleSpecificity. numericIp and the
|
|
83
|
-
// numericIpMaskMin range are mutually exclusive on a rule (see
|
|
84
|
-
// ruleHasIpConstraint), so `exists(@numericIp) + exists(@numericIpMaskMin)`
|
|
85
|
-
// contributes 1 in practice — never 2 — when applied to writer-produced
|
|
86
|
-
// rules.
|
|
87
|
-
const SPECIFICITY_EXPR = [
|
|
88
|
-
"exists(@clientId)",
|
|
89
|
-
"exists(@userId)",
|
|
90
|
-
"exists(@ja4Hash)",
|
|
91
|
-
"exists(@headersHash)",
|
|
92
|
-
"exists(@userAgentHash)",
|
|
93
|
-
"exists(@headHash)",
|
|
94
|
-
"exists(@coords)",
|
|
95
|
-
"exists(@countryCode)",
|
|
96
|
-
"exists(@asn)",
|
|
97
|
-
"exists(@os)",
|
|
98
|
-
"exists(@browser)",
|
|
99
|
-
// A header rule is one logical dimension: only the sentinel counts toward
|
|
100
|
-
// specificity, not the name/value/operator triple it also carries.
|
|
101
|
-
"exists(@headerMatch)",
|
|
102
|
-
"exists(@numericIp)",
|
|
103
|
-
"exists(@numericIpMaskMin)",
|
|
104
|
-
].join(" + ");
|
|
105
|
-
|
|
106
|
-
// Block outranks Restrict on equal specificity (defensive: a request
|
|
107
|
-
// that would match a Block rule must never be downgraded). Note the
|
|
108
|
-
// string literal is lowercase because AccessPolicyType.Block = "block"
|
|
109
|
-
// and getRedisRuleValue serialises via String().
|
|
110
|
-
const SEVERITY_EXPR = `(@type == "${AccessPolicyType.Block}")`;
|
|
111
|
-
|
|
112
|
-
// Final rank: specificity weighted, severity as tiebreaker on equal
|
|
113
|
-
// specificity. spec*2 + sev fits both into a single sortable float.
|
|
114
|
-
const RANK_EXPR = "(@_spec * 2) + @_sev";
|
|
115
|
-
|
|
116
|
-
// Every field referenced by SPECIFICITY_EXPR/SEVERITY_EXPR, plus the
|
|
117
|
-
// fields needed to reconstruct an AccessRule downstream. APPLY can only
|
|
118
|
-
// see fields explicitly loaded into the aggregate pipeline, so this
|
|
119
|
-
// list must include every rule attribute the parser cares about.
|
|
120
|
-
const RULE_LOAD_FIELDS = [
|
|
121
|
-
"@__key",
|
|
122
|
-
"@type",
|
|
123
|
-
"@captchaType",
|
|
124
|
-
"@description",
|
|
125
|
-
"@solvedImagesCount",
|
|
126
|
-
"@imageThreshold",
|
|
127
|
-
"@powDifficulty",
|
|
128
|
-
"@unsolvedImagesCount",
|
|
129
|
-
"@frictionlessScore",
|
|
130
|
-
"@deferToVerify",
|
|
131
|
-
"@messageKey",
|
|
132
|
-
"@clientId",
|
|
133
|
-
"@groupId",
|
|
134
|
-
"@userId",
|
|
135
|
-
"@ja4Hash",
|
|
136
|
-
"@headersHash",
|
|
137
|
-
"@userAgentHash",
|
|
138
|
-
"@headHash",
|
|
139
|
-
"@coords",
|
|
140
|
-
"@countryCode",
|
|
141
|
-
"@asn",
|
|
142
|
-
"@os",
|
|
143
|
-
"@browser",
|
|
144
|
-
"@headerMatch",
|
|
145
|
-
"@headerName",
|
|
146
|
-
"@headerValue",
|
|
147
|
-
"@headerOperator",
|
|
148
|
-
"@numericIp",
|
|
149
|
-
"@numericIpMaskMin",
|
|
150
|
-
"@numericIpMaskMax",
|
|
151
|
-
] as const;
|
|
152
|
-
|
|
153
|
-
// Populated-field count for reader-level rank (mirrors the JS ranker's
|
|
154
|
-
// specificity semantics without any request-scope dependence — the reader
|
|
155
|
-
// doesn't have the request scope). Enough to preserve the sort the old
|
|
156
|
-
// FT.AGGREGATE ranker imposed on findRules callers.
|
|
157
|
-
const readerSpecificity = (rule: AccessRule): number => {
|
|
158
|
-
let score = 0;
|
|
159
|
-
if (rule.clientId !== undefined) score++;
|
|
160
|
-
if (rule.userId !== undefined) score++;
|
|
161
|
-
if (rule.ja4Hash !== undefined) score++;
|
|
162
|
-
if (rule.headersHash !== undefined) score++;
|
|
163
|
-
if (rule.userAgentHash !== undefined) score++;
|
|
164
|
-
if (rule.headHash !== undefined) score++;
|
|
165
|
-
if (rule.coords !== undefined) score++;
|
|
166
|
-
if (rule.countryCode !== undefined) score++;
|
|
167
|
-
if (rule.asn !== undefined) score++;
|
|
168
|
-
if (rule.os !== undefined) score++;
|
|
169
|
-
if (rule.browser !== undefined) score++;
|
|
170
|
-
if (rule.numericIp !== undefined) score++;
|
|
171
|
-
if (rule.numericIpMaskMin !== undefined) score++;
|
|
172
|
-
return score;
|
|
173
|
-
};
|
|
174
|
-
|
|
175
|
-
// Sort by (specificity desc, block-first). Matches the ranking the old
|
|
176
|
-
// FT.AGGREGATE `RANK_EXPR = (_spec * 2) + _sev` imposed server-side, so
|
|
177
|
-
// callers that consume the reader directly see the same ordering.
|
|
178
|
-
const sortByReaderRank = (rules: AccessRule[]): AccessRule[] =>
|
|
179
|
-
[...rules].sort((a, b) => {
|
|
180
|
-
const specDelta = readerSpecificity(b) - readerSpecificity(a);
|
|
181
|
-
if (specDelta !== 0) return specDelta;
|
|
182
|
-
const aBlock = a.type === AccessPolicyType.Block ? 1 : 0;
|
|
183
|
-
const bBlock = b.type === AccessPolicyType.Block ? 1 : 0;
|
|
184
|
-
return bBlock - aBlock;
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
export class RedisRulesReader implements AccessRulesReader {
|
|
188
|
-
constructor(
|
|
189
|
-
private readonly client: RedisClientType,
|
|
190
|
-
private readonly logger: Logger,
|
|
191
|
-
) {}
|
|
192
|
-
|
|
193
|
-
async getMissingRuleIds(ruleIds: string[]): Promise<string[]> {
|
|
194
|
-
const ruleKeys = this.getRuleKeys(ruleIds);
|
|
195
|
-
const keyBatches = chunkIntoBatches(ruleKeys, REDIS_BATCH_SIZE);
|
|
196
|
-
|
|
197
|
-
const missingKeyBatches = await executeBatchesSequentially(
|
|
198
|
-
keyBatches,
|
|
199
|
-
async (keysBatch) => getMissingRedisKeys(this.client, keysBatch),
|
|
200
|
-
);
|
|
201
|
-
|
|
202
|
-
return missingKeyBatches
|
|
203
|
-
.flat()
|
|
204
|
-
.map((ruleKey) => ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length));
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
async fetchRules(ruleIds: string[]): Promise<AccessRuleEntry[]> {
|
|
208
|
-
const ruleKeys = this.getRuleKeys(ruleIds);
|
|
209
|
-
|
|
210
|
-
const keyBatches = chunkIntoBatches(ruleKeys, REDIS_BATCH_SIZE);
|
|
211
|
-
|
|
212
|
-
const entryBatches = await executeBatchesSequentially(
|
|
213
|
-
keyBatches,
|
|
214
|
-
(keysBatch) => this.fetchRuleEntries(keysBatch),
|
|
215
|
-
);
|
|
216
|
-
|
|
217
|
-
return entryBatches.flat();
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
async findRules(
|
|
221
|
-
filter: AccessRulesFilter,
|
|
222
|
-
matchingFieldsOnly = false,
|
|
223
|
-
skipEmptyUserScopes = true,
|
|
224
|
-
): Promise<AccessRule[]> {
|
|
225
|
-
const query = getRulesRedisQuery(filter, matchingFieldsOnly);
|
|
226
|
-
|
|
227
|
-
if (
|
|
228
|
-
skipEmptyUserScopes &&
|
|
229
|
-
// Sentinel query shape emitted by getPolicyScopeQuery when the
|
|
230
|
-
// caller passed neither a userScope nor a clientId — i.e. an
|
|
231
|
-
// empty filter that would otherwise resolve to "all global
|
|
232
|
-
// rules". Guard so an accidentally-empty filter can't return
|
|
233
|
-
// the full global rule set. The exact literal must stay in
|
|
234
|
-
// sync with GLOBAL_MATCH_CLAUSE_INNER over there.
|
|
235
|
-
query === "( @clientId:{global} | ismissing(@clientId) )"
|
|
236
|
-
) {
|
|
237
|
-
return [];
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
// Hot path (all strict-match lookups): route every
|
|
241
|
-
// matchingFieldsOnly caller through the split-query path — one FT
|
|
242
|
-
// probe per populated request field, each using its own posting
|
|
243
|
-
// list — instead of the single wide FT.AGGREGATE. `blockOnly`
|
|
244
|
-
// narrows the sub-queries to `@type:{block}` for the hard-block
|
|
245
|
-
// middleware; every other caller (frictionless access-policy
|
|
246
|
-
// lookup, verify-time hard-block re-check) fetches both Block
|
|
247
|
-
// and Restrict rules and lets `rankCandidateRules` pick.
|
|
248
|
-
//
|
|
249
|
-
// The previous FT.AGGREGATE ranker used for the non-block path
|
|
250
|
-
// silently dropped specific-IP Restrict rules from the top-N
|
|
251
|
-
// when many higher-specificity irrelevant rules matched the
|
|
252
|
-
// greedy `ismissing(@field)` disjunction — the union-of-probes
|
|
253
|
-
// shape here can't hit that failure mode because each probe
|
|
254
|
-
// only returns rules where the probed field is actually
|
|
255
|
-
// populated with the request's value.
|
|
256
|
-
if (matchingFieldsOnly && filter.userScope !== undefined) {
|
|
257
|
-
return this.findRulesSplit(filter);
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
// Admin / internal callers (greedy mode): keep the FT.SEARCH +
|
|
261
|
-
// HGETALL fanout path. Cap at REDIS_BATCH_SIZE; truncation past
|
|
262
|
-
// that point is the known liability tracked in #2689.
|
|
263
|
-
return this.findRulesGreedy(filter, query);
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
// Union candidate keys from each split sub-query, de-dupe, HGETALL,
|
|
267
|
-
// parse, hand back to `getPrioritisedAccessRule` for JS-side ranking.
|
|
268
|
-
// Ranking stays in JS because rankCandidateRules already encodes the
|
|
269
|
-
// strict "every populated rule field matches request" semantics —
|
|
270
|
-
// FT can't express that shape cheaply and the previous server-side
|
|
271
|
-
// APPLY / SORTBY imposed the linear-per-rule cost we're eliminating.
|
|
272
|
-
//
|
|
273
|
-
// Handles both blockOnly (hard-block middleware) and non-blockOnly
|
|
274
|
-
// (frictionless access-policy lookup) callers via the `blockOnly`
|
|
275
|
-
// flag on the sub-query builder.
|
|
276
|
-
private async findRulesSplit(
|
|
277
|
-
filter: AccessRulesFilter,
|
|
278
|
-
): Promise<AccessRule[]> {
|
|
279
|
-
const userScope = filter.userScope;
|
|
280
|
-
if (userScope === undefined) {
|
|
281
|
-
return [];
|
|
282
|
-
}
|
|
283
|
-
const clientId = filter.policyScope?.clientId;
|
|
284
|
-
|
|
285
|
-
const subQueries = buildScopedRulesSubQueries(userScope, clientId, {
|
|
286
|
-
...(filter.blockOnly === true && { blockOnly: true }),
|
|
287
|
-
...(filter.includeDeferred === true && { includeDeferred: true }),
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
try {
|
|
291
|
-
const keyLists = await Promise.all(
|
|
292
|
-
subQueries.map((sub) =>
|
|
293
|
-
// FT.SEARCH NOCONTENT (via searchRedisKeysBounded) rather
|
|
294
|
-
// than FT.AGGREGATE with LOAD @__key. The aggregate reply
|
|
295
|
-
// path in `@redis/client` 5.x calls transformTuplesReply
|
|
296
|
-
// on every result row; when the RediSearch coordinator
|
|
297
|
-
// returns a null tuple (observed continuously on
|
|
298
|
-
// field:ja4Hash probes in prod), that throws
|
|
299
|
-
// `Cannot read properties of null (reading 'length')` and
|
|
300
|
-
// the whole sub-query resolves empty via the .catch below.
|
|
301
|
-
// NOCONTENT returns a flat `[total, key1, key2, ...]`
|
|
302
|
-
// array so there is no nested tuple that can be null.
|
|
303
|
-
searchRedisKeysBounded(
|
|
304
|
-
this.client,
|
|
305
|
-
sub.query,
|
|
306
|
-
// Per-sub-query cap. Each probe hits a discriminating
|
|
307
|
-
// index (e.g. numericIp posting list), so the natural
|
|
308
|
-
// cardinality is tiny — this cap only fires on
|
|
309
|
-
// pathological rule-set shapes and mirrors
|
|
310
|
-
// SPLIT_MAX_CANDIDATES_PER_SUB below.
|
|
311
|
-
SPLIT_MAX_CANDIDATES_PER_SUB,
|
|
312
|
-
).catch((err) => {
|
|
313
|
-
// One sub-query failing (e.g. a transient RediSearch
|
|
314
|
-
// error) shouldn't drop the whole lookup — the other
|
|
315
|
-
// probes may still find the applicable rule. Log and
|
|
316
|
-
// continue with an empty result for this probe.
|
|
317
|
-
this.logger.error(() => ({
|
|
318
|
-
err,
|
|
319
|
-
data: {
|
|
320
|
-
inspect: util.inspect(
|
|
321
|
-
{ subKind: sub.kind, query: sub.query },
|
|
322
|
-
{ depth: null },
|
|
323
|
-
),
|
|
324
|
-
},
|
|
325
|
-
msg: "failed to execute split rule sub-query",
|
|
326
|
-
}));
|
|
327
|
-
return [];
|
|
328
|
-
}),
|
|
329
|
-
),
|
|
330
|
-
);
|
|
331
|
-
|
|
332
|
-
const uniqueKeys = new Set<string>();
|
|
333
|
-
for (const list of keyLists) {
|
|
334
|
-
for (const key of list) {
|
|
335
|
-
uniqueKeys.add(key);
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
if (uniqueKeys.size === 0) {
|
|
340
|
-
return [];
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
const ruleKeys = [...uniqueKeys];
|
|
344
|
-
|
|
345
|
-
this.logger.debug(() => ({
|
|
346
|
-
msg: "Executed split rule sub-queries",
|
|
347
|
-
data: {
|
|
348
|
-
inspect: util.inspect(
|
|
349
|
-
{
|
|
350
|
-
subQueryCount: subQueries.length,
|
|
351
|
-
uniqueCandidates: ruleKeys.length,
|
|
352
|
-
perSubQueryCounts: keyLists.map((l, i) => ({
|
|
353
|
-
kind: subQueries[i]?.kind,
|
|
354
|
-
found: l.length,
|
|
355
|
-
})),
|
|
356
|
-
filter,
|
|
357
|
-
},
|
|
358
|
-
{ depth: null },
|
|
359
|
-
),
|
|
360
|
-
},
|
|
361
|
-
}));
|
|
362
|
-
|
|
363
|
-
const { records } = await fetchRedisHashRecords(
|
|
364
|
-
this.client,
|
|
365
|
-
ruleKeys,
|
|
366
|
-
this.logger,
|
|
367
|
-
);
|
|
368
|
-
|
|
369
|
-
const nonEmptyRecords = records.filter(
|
|
370
|
-
(record) => Object.keys(record).length > 0,
|
|
371
|
-
);
|
|
372
|
-
|
|
373
|
-
const parsedRules = parseRedisRecords(
|
|
374
|
-
nonEmptyRecords,
|
|
375
|
-
accessRuleInput,
|
|
376
|
-
this.logger,
|
|
377
|
-
);
|
|
378
|
-
|
|
379
|
-
// Preserve the reader-level (specificity desc, block-first)
|
|
380
|
-
// ordering the old FT.AGGREGATE ranker gave. Split sub-queries
|
|
381
|
-
// return keys per-probe (i.e. in insertion order), so without
|
|
382
|
-
// this sort the caller sees candidates in a shape the previous
|
|
383
|
-
// implementation never returned. Downstream callers
|
|
384
|
-
// (`rankCandidateRules` in the provider) do a full re-sort, so
|
|
385
|
-
// this is purely a defence against silent behaviour changes
|
|
386
|
-
// for anyone else that consumes the reader directly.
|
|
387
|
-
return sortByReaderRank(parsedRules);
|
|
388
|
-
} catch (e) {
|
|
389
|
-
this.logger.error(() => ({
|
|
390
|
-
err: e,
|
|
391
|
-
data: {
|
|
392
|
-
inspect: util.inspect({ filter }, { depth: null }),
|
|
393
|
-
},
|
|
394
|
-
msg: "failed to execute split rule query set",
|
|
395
|
-
}));
|
|
396
|
-
return [];
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
private async findRulesRanked(
|
|
401
|
-
filter: AccessRulesFilter,
|
|
402
|
-
query: string,
|
|
403
|
-
): Promise<AccessRule[]> {
|
|
404
|
-
try {
|
|
405
|
-
const reply = await this.client.ft.aggregate(
|
|
406
|
-
ACCESS_RULES_REDIS_INDEX_NAME,
|
|
407
|
-
query,
|
|
408
|
-
{
|
|
409
|
-
DIALECT: REDIS_QUERY_DIALECT,
|
|
410
|
-
LOAD: [...RULE_LOAD_FIELDS],
|
|
411
|
-
STEPS: [
|
|
412
|
-
// Drop candidates whose hash is missing `type` before
|
|
413
|
-
// any APPLY runs. SEVERITY_EXPR dereferences `@type`
|
|
414
|
-
// directly, and RediSearch throws
|
|
415
|
-
// "Could not find the value for a parameter name,
|
|
416
|
-
// consider using EXISTS if applicable for type"
|
|
417
|
-
// when the LOAD pulls an undefined @type, aborting
|
|
418
|
-
// the whole aggregate. The undefined @type can come
|
|
419
|
-
// from a partial-write race in the writer or a
|
|
420
|
-
// stale RediSearch index entry pointing at a hash
|
|
421
|
-
// whose `type` field has been removed; in both
|
|
422
|
-
// cases the doc is malformed and not a valid
|
|
423
|
-
// AccessRule, so dropping it is correct.
|
|
424
|
-
{ type: "FILTER", expression: "exists(@type)" },
|
|
425
|
-
{ type: "APPLY", expression: SPECIFICITY_EXPR, AS: "_spec" },
|
|
426
|
-
{ type: "APPLY", expression: SEVERITY_EXPR, AS: "_sev" },
|
|
427
|
-
{ type: "APPLY", expression: RANK_EXPR, AS: "_rank" },
|
|
428
|
-
{
|
|
429
|
-
type: "SORTBY",
|
|
430
|
-
BY: [{ BY: "@_rank", DIRECTION: "DESC" }],
|
|
431
|
-
// SORTBY MAX would let RediSearch keep only the
|
|
432
|
-
// top-N in a bounded heap rather than fully
|
|
433
|
-
// sorting, but the @redis/search client
|
|
434
|
-
// serialises MAX *before* DESC and RediSearch
|
|
435
|
-
// rejects that order ("MISSING ASC or DESC
|
|
436
|
-
// after sort field (MAX)"). LIMIT below still
|
|
437
|
-
// trims; SORTBY does a full sort. Acceptable
|
|
438
|
-
// because the strict-match filter already
|
|
439
|
-
// keeps the candidate set small in practice.
|
|
440
|
-
},
|
|
441
|
-
{ type: "LIMIT", from: 0, size: SERVER_SIDE_RANK_TOP_N },
|
|
442
|
-
],
|
|
443
|
-
},
|
|
444
|
-
);
|
|
445
|
-
|
|
446
|
-
if (reply.results.length === 0) {
|
|
447
|
-
return [];
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
this.logger.debug(() => ({
|
|
451
|
-
msg: "Executed ranked search query",
|
|
452
|
-
data: {
|
|
453
|
-
inspect: util.inspect(
|
|
454
|
-
{
|
|
455
|
-
filter: filter,
|
|
456
|
-
foundCount: reply.results.length,
|
|
457
|
-
query: query,
|
|
458
|
-
},
|
|
459
|
-
{ depth: null },
|
|
460
|
-
),
|
|
461
|
-
},
|
|
462
|
-
}));
|
|
463
|
-
|
|
464
|
-
// FT.AGGREGATE LOAD reads NUMERIC fields from the index (an
|
|
465
|
-
// 8-byte double), not from the underlying hash, so any
|
|
466
|
-
// numericIp/numericIpMaskMin/numericIpMaskMax larger than
|
|
467
|
-
// Number.MAX_SAFE_INTEGER (e.g. every IPv6 rule) round-trips
|
|
468
|
-
// as scientific notation — `5.59112965392e+37` — and
|
|
469
|
-
// `z.coerce.bigint()` throws
|
|
470
|
-
// "Cannot convert 5.59112965392e+37 to a BigInt"
|
|
471
|
-
// which aborts the whole aggregate via the catch below and
|
|
472
|
-
// returns no rules. The hash itself stores the full
|
|
473
|
-
// 38-digit string verbatim, so the aggregate is used purely
|
|
474
|
-
// as a ranker (it gives us the top-N keys, sorted by spec /
|
|
475
|
-
// severity); the real field values come back via HGETALL
|
|
476
|
-
// over those keys, identical to the greedy path. Same
|
|
477
|
-
// race-safety as the greedy path: HGETALL on a key that's
|
|
478
|
-
// been deleted between FT.AGGREGATE and HGETALL returns an
|
|
479
|
-
// empty hash, which is dropped by the non-empty filter.
|
|
480
|
-
const ruleKeys: string[] = [];
|
|
481
|
-
for (const result of reply.results) {
|
|
482
|
-
const key = (result as { __key?: unknown }).__key;
|
|
483
|
-
if (typeof key === "string") {
|
|
484
|
-
ruleKeys.push(key);
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
if (ruleKeys.length === 0) {
|
|
489
|
-
return [];
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
const { records } = await fetchRedisHashRecords(
|
|
493
|
-
this.client,
|
|
494
|
-
ruleKeys,
|
|
495
|
-
this.logger,
|
|
496
|
-
);
|
|
497
|
-
|
|
498
|
-
const nonEmptyRecords = records.filter(
|
|
499
|
-
(record) => Object.keys(record).length > 0,
|
|
500
|
-
);
|
|
501
|
-
|
|
502
|
-
return parseRedisRecords(nonEmptyRecords, accessRuleInput, this.logger);
|
|
503
|
-
} catch (e) {
|
|
504
|
-
this.logger.error(() => ({
|
|
505
|
-
err: e,
|
|
506
|
-
data: {
|
|
507
|
-
inspect: util.inspect({ query, filter }, { depth: null }),
|
|
508
|
-
},
|
|
509
|
-
msg: "failed to execute ranked search query",
|
|
510
|
-
}));
|
|
511
|
-
|
|
512
|
-
return [];
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
|
-
|
|
516
|
-
// Used only by admin / internal callers (matchingFieldsOnly=false).
|
|
517
|
-
// Not on the per-request blockMiddleware path any more, so the CPU
|
|
518
|
-
// cost of the HGETALL fanout is acceptable. Same shape as #2689's
|
|
519
|
-
// aggregate-with-cursor: returns every candidate up to
|
|
520
|
-
// GREEDY_MAX_CANDIDATES so we don't silently drop matches that
|
|
521
|
-
// admin tools depend on (e.g. ASN-wide rule listings).
|
|
522
|
-
private async findRulesGreedy(
|
|
523
|
-
filter: AccessRulesFilter,
|
|
524
|
-
query: string,
|
|
525
|
-
): Promise<AccessRule[]> {
|
|
526
|
-
try {
|
|
527
|
-
const ruleKeys = await aggregateRedisKeys(
|
|
528
|
-
this.client,
|
|
529
|
-
query,
|
|
530
|
-
this.logger,
|
|
531
|
-
undefined,
|
|
532
|
-
GREEDY_MAX_CANDIDATES,
|
|
533
|
-
);
|
|
534
|
-
|
|
535
|
-
if (ruleKeys.length === 0) {
|
|
536
|
-
return [];
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
this.logger.debug(() => ({
|
|
540
|
-
msg: "Executed greedy search query",
|
|
541
|
-
data: {
|
|
542
|
-
inspect: util.inspect(
|
|
543
|
-
{
|
|
544
|
-
filter: filter,
|
|
545
|
-
foundCount: ruleKeys.length,
|
|
546
|
-
query: query,
|
|
547
|
-
},
|
|
548
|
-
{ depth: null },
|
|
549
|
-
),
|
|
550
|
-
},
|
|
551
|
-
}));
|
|
552
|
-
|
|
553
|
-
const { records } = await fetchRedisHashRecords(
|
|
554
|
-
this.client,
|
|
555
|
-
ruleKeys,
|
|
556
|
-
this.logger,
|
|
557
|
-
);
|
|
558
|
-
|
|
559
|
-
const nonEmptyRecords = records.filter(
|
|
560
|
-
(record) => Object.keys(record).length > 0,
|
|
561
|
-
);
|
|
562
|
-
|
|
563
|
-
return parseRedisRecords(nonEmptyRecords, accessRuleInput, this.logger);
|
|
564
|
-
} catch (e) {
|
|
565
|
-
this.logger.error(() => ({
|
|
566
|
-
err: e,
|
|
567
|
-
data: {
|
|
568
|
-
inspect: util.inspect({ query, filter }, { depth: null }),
|
|
569
|
-
},
|
|
570
|
-
msg: "failed to execute greedy search query",
|
|
571
|
-
}));
|
|
572
|
-
|
|
573
|
-
return [];
|
|
574
|
-
}
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
async findRuleIds(
|
|
578
|
-
filter: AccessRulesFilter,
|
|
579
|
-
matchingFieldsOnly = false,
|
|
580
|
-
): Promise<string[]> {
|
|
581
|
-
const query = getRulesRedisQuery(filter, matchingFieldsOnly);
|
|
582
|
-
|
|
583
|
-
let ruleIds: string[] = [];
|
|
584
|
-
|
|
585
|
-
try {
|
|
586
|
-
// aggregation is used instead ft.search to overcome the limitation on search results number
|
|
587
|
-
const ruleKeys = await aggregateRedisKeys(
|
|
588
|
-
this.client,
|
|
589
|
-
query,
|
|
590
|
-
this.logger,
|
|
591
|
-
);
|
|
592
|
-
|
|
593
|
-
ruleIds = ruleKeys.map((ruleKey) =>
|
|
594
|
-
ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length),
|
|
595
|
-
);
|
|
596
|
-
} catch (e) {
|
|
597
|
-
this.logger.error(() => ({
|
|
598
|
-
err: e,
|
|
599
|
-
data: {
|
|
600
|
-
inspect: util.inspect(
|
|
601
|
-
{
|
|
602
|
-
query: query,
|
|
603
|
-
filter: filter,
|
|
604
|
-
},
|
|
605
|
-
{
|
|
606
|
-
depth: null,
|
|
607
|
-
},
|
|
608
|
-
),
|
|
609
|
-
},
|
|
610
|
-
msg: "Failed to execute search query for rule IDs",
|
|
611
|
-
}));
|
|
612
|
-
|
|
613
|
-
return [];
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
this.logger.debug(() => ({
|
|
617
|
-
msg: "Executed search query for rule IDs",
|
|
618
|
-
data: {
|
|
619
|
-
query: query,
|
|
620
|
-
foundCount: ruleIds.length,
|
|
621
|
-
foundIds: ruleIds,
|
|
622
|
-
},
|
|
623
|
-
}));
|
|
624
|
-
|
|
625
|
-
return ruleIds;
|
|
626
|
-
}
|
|
627
|
-
|
|
628
|
-
async fetchAllRuleIds(
|
|
629
|
-
batchHandler: (ruleIds: string[]) => Promise<void>,
|
|
630
|
-
): Promise<void> {
|
|
631
|
-
const keysBatchHandler = async (keys: string[]) => {
|
|
632
|
-
const ids = keys.map((ruleKey) =>
|
|
633
|
-
ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length),
|
|
634
|
-
);
|
|
635
|
-
|
|
636
|
-
await batchHandler(ids);
|
|
637
|
-
};
|
|
638
|
-
|
|
639
|
-
await aggregateRedisKeys(this.client, "*", this.logger, keysBatchHandler);
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
protected async fetchRuleEntries(keys: string[]): Promise<AccessRuleEntry[]> {
|
|
643
|
-
const { records, expirations } = await fetchRedisHashRecords(
|
|
644
|
-
this.client,
|
|
645
|
-
keys,
|
|
646
|
-
this.logger,
|
|
647
|
-
);
|
|
648
|
-
const entries: AccessRuleEntry[] = [];
|
|
649
|
-
|
|
650
|
-
for (const [index, ruleData] of records.entries()) {
|
|
651
|
-
const isRulePresent = Object.keys(ruleData).length > 0;
|
|
652
|
-
|
|
653
|
-
if (isRulePresent) {
|
|
654
|
-
const rule = parseRedisRecords(
|
|
655
|
-
[ruleData],
|
|
656
|
-
accessRuleInput,
|
|
657
|
-
this.logger,
|
|
658
|
-
)[0];
|
|
659
|
-
|
|
660
|
-
if (rule) {
|
|
661
|
-
entries.push({
|
|
662
|
-
rule: rule,
|
|
663
|
-
expiresUnixTimestamp: expirations[index],
|
|
664
|
-
});
|
|
665
|
-
}
|
|
666
|
-
}
|
|
667
|
-
}
|
|
668
|
-
|
|
669
|
-
return entries;
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
protected getRuleKeys(ruleIds: string[]): string[] {
|
|
673
|
-
return ruleIds.map((id) => `${ACCESS_RULE_REDIS_KEY_PREFIX}${id}`);
|
|
674
|
-
}
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
export class DummyRedisRulesReader implements AccessRulesReader {
|
|
678
|
-
constructor(private readonly logger: Logger) {}
|
|
679
|
-
|
|
680
|
-
async getMissingRuleIds(ruleIds: string[]): Promise<string[]> {
|
|
681
|
-
this.logger.info(() => ({
|
|
682
|
-
msg: "Dummy getMissingRuleIds() has no effect (redis is not ready)",
|
|
683
|
-
data: {
|
|
684
|
-
ruleIds,
|
|
685
|
-
},
|
|
686
|
-
}));
|
|
687
|
-
|
|
688
|
-
return [];
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
async fetchRules(ruleIds: string[]): Promise<AccessRuleEntry[]> {
|
|
692
|
-
this.logger.info(() => ({
|
|
693
|
-
msg: "Dummy fetchRule() has no effect (redis is not ready)",
|
|
694
|
-
data: {
|
|
695
|
-
ruleIds,
|
|
696
|
-
},
|
|
697
|
-
}));
|
|
698
|
-
|
|
699
|
-
return [];
|
|
700
|
-
}
|
|
701
|
-
|
|
702
|
-
async findRules(
|
|
703
|
-
filter: AccessRulesFilter,
|
|
704
|
-
matchingFieldsOnly = false,
|
|
705
|
-
skipEmptyUserScopes = true,
|
|
706
|
-
): Promise<AccessRule[]> {
|
|
707
|
-
this.logger.info(() => ({
|
|
708
|
-
msg: "Dummy findRules() has no effect (redis is not ready)",
|
|
709
|
-
data: {
|
|
710
|
-
filter,
|
|
711
|
-
},
|
|
712
|
-
}));
|
|
713
|
-
|
|
714
|
-
return [];
|
|
715
|
-
}
|
|
716
|
-
|
|
717
|
-
async findRuleIds(
|
|
718
|
-
filter: AccessRulesFilter,
|
|
719
|
-
matchingFieldsOnly = false,
|
|
720
|
-
): Promise<string[]> {
|
|
721
|
-
this.logger.info(() => ({
|
|
722
|
-
msg: "Dummy findRuleIds() has no effect (redis is not ready)",
|
|
723
|
-
data: {
|
|
724
|
-
filter,
|
|
725
|
-
},
|
|
726
|
-
}));
|
|
727
|
-
|
|
728
|
-
return [];
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
async fetchAllRuleIds(
|
|
732
|
-
batchHandler: (ruleIds: string[]) => Promise<void>,
|
|
733
|
-
): Promise<void> {
|
|
734
|
-
this.logger.info(() => ({
|
|
735
|
-
msg: "Dummy fetchAllRuleIds() has no effect (redis is not ready)",
|
|
736
|
-
}));
|
|
737
|
-
}
|
|
738
|
-
}
|