@prosopo/user-access-policy 3.14.14 → 3.14.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/.turbo/turbo-build$colon$cjs.log +3 -6
  2. package/.turbo/turbo-build$colon$tsc.log +14 -17
  3. package/.turbo/turbo-build.log +4 -10
  4. package/CHANGELOG.md +37 -0
  5. package/LICENSE +201 -0
  6. package/dist/cjs/ruleInput/policyInput.cjs +1 -1
  7. package/dist/ruleInput/policyInput.d.ts +2 -2
  8. package/dist/ruleInput/policyInput.d.ts.map +1 -1
  9. package/dist/ruleInput/policyInput.js +2 -2
  10. package/dist/ruleInput/policyInput.js.map +1 -1
  11. package/package.json +83 -81
  12. package/src/.export.ts +0 -65
  13. package/src/api/.export.ts +0 -25
  14. package/src/api/accessRulesApiClient.ts +0 -13
  15. package/src/api/delete/.export.ts +0 -18
  16. package/src/api/delete/deleteAllRules.ts +0 -47
  17. package/src/api/delete/deleteRuleGroups.ts +0 -96
  18. package/src/api/delete/deleteRules.ts +0 -87
  19. package/src/api/read/.export.ts +0 -25
  20. package/src/api/read/fetchRules.ts +0 -96
  21. package/src/api/read/findRuleIds.ts +0 -97
  22. package/src/api/read/getMissingIds.ts +0 -81
  23. package/src/api/ruleApiRoutes.ts +0 -146
  24. package/src/api/rulesApiClient.ts +0 -154
  25. package/src/api/write/.export.ts +0 -15
  26. package/src/api/write/insertRules.ts +0 -185
  27. package/src/api/write/rehashRules.ts +0 -85
  28. package/src/classifyBrowser.ts +0 -77
  29. package/src/classifyOs.ts +0 -74
  30. package/src/headerMatch.ts +0 -180
  31. package/src/matchedRule.ts +0 -99
  32. package/src/mongoose/.export.ts +0 -15
  33. package/src/mongoose/mongooseRuleSchema.ts +0 -75
  34. package/src/redis/.export.ts +0 -17
  35. package/src/redis/reader/redisAggregate.ts +0 -157
  36. package/src/redis/reader/redisRulesQuery.ts +0 -268
  37. package/src/redis/reader/redisRulesReader.ts +0 -738
  38. package/src/redis/reader/redisRulesSplitQuery.ts +0 -234
  39. package/src/redis/redisClient.ts +0 -125
  40. package/src/redis/redisRuleIndex.ts +0 -121
  41. package/src/redis/redisRulesStorage.ts +0 -68
  42. package/src/redis/redisRulesWriter.ts +0 -172
  43. package/src/rule.ts +0 -122
  44. package/src/ruleInput/.export.ts +0 -19
  45. package/src/ruleInput/policyInput.ts +0 -120
  46. package/src/ruleInput/ruleInput.ts +0 -123
  47. package/src/ruleInput/userScopeInput.ts +0 -142
  48. package/src/ruleRecord.ts +0 -77
  49. package/src/rulesStorage.ts +0 -90
  50. package/src/tests/blockingRuleShapes.unit.test.ts +0 -244
  51. package/src/tests/classifyBrowser.unit.test.ts +0 -138
  52. package/src/tests/classifyOs.unit.test.ts +0 -97
  53. package/src/tests/headerMatch.unit.test.ts +0 -350
  54. package/src/tests/insertRulesEndpoint.unit.test.ts +0 -89
  55. package/src/tests/matchedRule.unit.test.ts +0 -184
  56. package/src/tests/policyInput.unit.test.ts +0 -218
  57. package/src/tests/redis/deferredPoolClause.benchmark.ts +0 -249
  58. package/src/tests/redis/reader/redisAggregate.unit.test.ts +0 -121
  59. package/src/tests/redis/reader/redisRulesQuery.unit.test.ts +0 -349
  60. package/src/tests/redis/reader/redisRulesSplitQuery.unit.test.ts +0 -297
  61. package/src/tests/redis/redisRulesReaderLoad.benchmark.integration.test.ts +0 -520
  62. package/src/tests/redis/redisRulesReaderRank.benchmark.integration.test.ts +0 -350
  63. package/src/tests/redis/redisRulesStorage.integration.test.ts +0 -1811
  64. package/src/tests/testLogger.ts +0 -38
  65. package/src/tests/transformRule.unit.test.ts +0 -345
  66. package/src/transformRule.ts +0 -133
  67. package/tsconfig.cjs.json +0 -41
  68. package/tsconfig.json +0 -47
  69. package/tsconfig.tsbuildinfo +0 -1
  70. package/tsconfig.types.json +0 -9
@@ -1,738 +0,0 @@
1
- // Copyright 2021-2026 Prosopo (UK) Ltd.
2
- //
3
- // Licensed under the Apache License, Version 2.0 (the "License");
4
- // you may not use this file except in compliance with the License.
5
- // You may obtain a copy of the License at
6
- //
7
- // http://www.apache.org/licenses/LICENSE-2.0
8
- //
9
- // Unless required by applicable law or agreed to in writing, software
10
- // distributed under the License is distributed on an "AS IS" BASIS,
11
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- // See the License for the specific language governing permissions and
13
- // limitations under the License.
14
-
15
- import * as util from "node:util";
16
- import { chunkIntoBatches, executeBatchesSequentially } from "@prosopo/common";
17
- import type { Logger } from "@prosopo/logger";
18
- import type { RedisClientType } from "redis";
19
- import {
20
- REDIS_QUERY_DIALECT,
21
- getRulesRedisQuery,
22
- } from "#policy/redis/reader/redisRulesQuery.js";
23
- import { buildScopedRulesSubQueries } from "#policy/redis/reader/redisRulesSplitQuery.js";
24
- import {
25
- REDIS_BATCH_SIZE,
26
- fetchRedisHashRecords,
27
- getMissingRedisKeys,
28
- parseRedisRecords,
29
- } from "#policy/redis/redisClient.js";
30
- import {
31
- ACCESS_RULES_REDIS_INDEX_NAME,
32
- ACCESS_RULE_REDIS_KEY_PREFIX,
33
- } from "#policy/redis/redisRuleIndex.js";
34
- import { AccessPolicyType, type AccessRule } from "#policy/rule.js";
35
- import { accessRuleInput } from "#policy/ruleInput/ruleInput.js";
36
- import type {
37
- AccessRuleEntry,
38
- AccessRulesFilter,
39
- AccessRulesReader,
40
- } from "#policy/rulesStorage.js";
41
- import {
42
- aggregateRedisKeys,
43
- searchRedisKeysBounded,
44
- } from "./redisAggregate.js";
45
-
46
- // Server-side specificity ranking config.
47
- //
48
- // Strict-match findRules (matchingFieldsOnly=true) used to pull every
49
- // matching rule's hash into Node and sort in JS. With the greedy regression
50
- // in #2689 this hauled ~1190 hashes per request and pegged provider CPU.
51
- //
52
- // The new path lets RediSearch compute specificity itself: one
53
- // FT.AGGREGATE that filters via the strict AND-of-disjunctions query,
54
- // counts populated rule fields with APPLY+exists(), sorts by that
55
- // derived score DESC, and returns only the top-N candidates. Node
56
- // receives at most TOP_N small records, no HGETALL fanout, no JS rank.
57
- //
58
- // TOP_N >> 1 leaves headroom for the deferToVerify filter that runs
59
- // post-aggregate (a Block rule with deferToVerify is skipped in the
60
- // blockMiddleware path, so we need a few alternates ready). 20 covers
61
- // the worst observed concentration of deferToVerify rules per scope by
62
- // comfortable margin while keeping payload tiny.
63
- export const SERVER_SIDE_RANK_TOP_N = 20;
64
-
65
- // Safety cap for the greedy (admin/internal) path. Generous because
66
- // admin tooling depends on the full result set; not on the hot
67
- // per-request path.
68
- const GREEDY_MAX_CANDIDATES = REDIS_BATCH_SIZE * 10;
69
-
70
- // Per-sub-query cap on the hot split-query path. Each sub-query probes a
71
- // single discriminating index (a specific numericIp value, a specific
72
- // ja4Hash value, etc.) so the natural cardinality is tiny — typically
73
- // 1–5. The cap only fires on pathological rule-set shapes (e.g. many
74
- // duplicate-value rules against one attribute) and prevents a bad shape
75
- // from tanking the request. It intentionally lets the union grow past
76
- // SERVER_SIDE_RANK_TOP_N because JS-side rankCandidateRules picks the
77
- // winner and the extra candidates are cheap to HGETALL.
78
- const SPLIT_MAX_CANDIDATES_PER_SUB = 500;
79
-
80
- // Fields that contribute one specificity point each. Mirrors
81
- // SCALAR_USER_SCOPE_FIELDS + clientId + ip-constraint in
82
- // blacklistRequestInspector.ruleSpecificity. numericIp and the
83
- // numericIpMaskMin range are mutually exclusive on a rule (see
84
- // ruleHasIpConstraint), so `exists(@numericIp) + exists(@numericIpMaskMin)`
85
- // contributes 1 in practice — never 2 — when applied to writer-produced
86
- // rules.
87
- const SPECIFICITY_EXPR = [
88
- "exists(@clientId)",
89
- "exists(@userId)",
90
- "exists(@ja4Hash)",
91
- "exists(@headersHash)",
92
- "exists(@userAgentHash)",
93
- "exists(@headHash)",
94
- "exists(@coords)",
95
- "exists(@countryCode)",
96
- "exists(@asn)",
97
- "exists(@os)",
98
- "exists(@browser)",
99
- // A header rule is one logical dimension: only the sentinel counts toward
100
- // specificity, not the name/value/operator triple it also carries.
101
- "exists(@headerMatch)",
102
- "exists(@numericIp)",
103
- "exists(@numericIpMaskMin)",
104
- ].join(" + ");
105
-
106
- // Block outranks Restrict on equal specificity (defensive: a request
107
- // that would match a Block rule must never be downgraded). Note the
108
- // string literal is lowercase because AccessPolicyType.Block = "block"
109
- // and getRedisRuleValue serialises via String().
110
- const SEVERITY_EXPR = `(@type == "${AccessPolicyType.Block}")`;
111
-
112
- // Final rank: specificity weighted, severity as tiebreaker on equal
113
- // specificity. spec*2 + sev fits both into a single sortable float.
114
- const RANK_EXPR = "(@_spec * 2) + @_sev";
115
-
116
- // Every field referenced by SPECIFICITY_EXPR/SEVERITY_EXPR, plus the
117
- // fields needed to reconstruct an AccessRule downstream. APPLY can only
118
- // see fields explicitly loaded into the aggregate pipeline, so this
119
- // list must include every rule attribute the parser cares about.
120
- const RULE_LOAD_FIELDS = [
121
- "@__key",
122
- "@type",
123
- "@captchaType",
124
- "@description",
125
- "@solvedImagesCount",
126
- "@imageThreshold",
127
- "@powDifficulty",
128
- "@unsolvedImagesCount",
129
- "@frictionlessScore",
130
- "@deferToVerify",
131
- "@messageKey",
132
- "@clientId",
133
- "@groupId",
134
- "@userId",
135
- "@ja4Hash",
136
- "@headersHash",
137
- "@userAgentHash",
138
- "@headHash",
139
- "@coords",
140
- "@countryCode",
141
- "@asn",
142
- "@os",
143
- "@browser",
144
- "@headerMatch",
145
- "@headerName",
146
- "@headerValue",
147
- "@headerOperator",
148
- "@numericIp",
149
- "@numericIpMaskMin",
150
- "@numericIpMaskMax",
151
- ] as const;
152
-
153
- // Populated-field count for reader-level rank (mirrors the JS ranker's
154
- // specificity semantics without any request-scope dependence — the reader
155
- // doesn't have the request scope). Enough to preserve the sort the old
156
- // FT.AGGREGATE ranker imposed on findRules callers.
157
- const readerSpecificity = (rule: AccessRule): number => {
158
- let score = 0;
159
- if (rule.clientId !== undefined) score++;
160
- if (rule.userId !== undefined) score++;
161
- if (rule.ja4Hash !== undefined) score++;
162
- if (rule.headersHash !== undefined) score++;
163
- if (rule.userAgentHash !== undefined) score++;
164
- if (rule.headHash !== undefined) score++;
165
- if (rule.coords !== undefined) score++;
166
- if (rule.countryCode !== undefined) score++;
167
- if (rule.asn !== undefined) score++;
168
- if (rule.os !== undefined) score++;
169
- if (rule.browser !== undefined) score++;
170
- if (rule.numericIp !== undefined) score++;
171
- if (rule.numericIpMaskMin !== undefined) score++;
172
- return score;
173
- };
174
-
175
- // Sort by (specificity desc, block-first). Matches the ranking the old
176
- // FT.AGGREGATE `RANK_EXPR = (_spec * 2) + _sev` imposed server-side, so
177
- // callers that consume the reader directly see the same ordering.
178
- const sortByReaderRank = (rules: AccessRule[]): AccessRule[] =>
179
- [...rules].sort((a, b) => {
180
- const specDelta = readerSpecificity(b) - readerSpecificity(a);
181
- if (specDelta !== 0) return specDelta;
182
- const aBlock = a.type === AccessPolicyType.Block ? 1 : 0;
183
- const bBlock = b.type === AccessPolicyType.Block ? 1 : 0;
184
- return bBlock - aBlock;
185
- });
186
-
187
- export class RedisRulesReader implements AccessRulesReader {
188
- constructor(
189
- private readonly client: RedisClientType,
190
- private readonly logger: Logger,
191
- ) {}
192
-
193
- async getMissingRuleIds(ruleIds: string[]): Promise<string[]> {
194
- const ruleKeys = this.getRuleKeys(ruleIds);
195
- const keyBatches = chunkIntoBatches(ruleKeys, REDIS_BATCH_SIZE);
196
-
197
- const missingKeyBatches = await executeBatchesSequentially(
198
- keyBatches,
199
- async (keysBatch) => getMissingRedisKeys(this.client, keysBatch),
200
- );
201
-
202
- return missingKeyBatches
203
- .flat()
204
- .map((ruleKey) => ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length));
205
- }
206
-
207
- async fetchRules(ruleIds: string[]): Promise<AccessRuleEntry[]> {
208
- const ruleKeys = this.getRuleKeys(ruleIds);
209
-
210
- const keyBatches = chunkIntoBatches(ruleKeys, REDIS_BATCH_SIZE);
211
-
212
- const entryBatches = await executeBatchesSequentially(
213
- keyBatches,
214
- (keysBatch) => this.fetchRuleEntries(keysBatch),
215
- );
216
-
217
- return entryBatches.flat();
218
- }
219
-
220
- async findRules(
221
- filter: AccessRulesFilter,
222
- matchingFieldsOnly = false,
223
- skipEmptyUserScopes = true,
224
- ): Promise<AccessRule[]> {
225
- const query = getRulesRedisQuery(filter, matchingFieldsOnly);
226
-
227
- if (
228
- skipEmptyUserScopes &&
229
- // Sentinel query shape emitted by getPolicyScopeQuery when the
230
- // caller passed neither a userScope nor a clientId — i.e. an
231
- // empty filter that would otherwise resolve to "all global
232
- // rules". Guard so an accidentally-empty filter can't return
233
- // the full global rule set. The exact literal must stay in
234
- // sync with GLOBAL_MATCH_CLAUSE_INNER over there.
235
- query === "( @clientId:{global} | ismissing(@clientId) )"
236
- ) {
237
- return [];
238
- }
239
-
240
- // Hot path (all strict-match lookups): route every
241
- // matchingFieldsOnly caller through the split-query path — one FT
242
- // probe per populated request field, each using its own posting
243
- // list — instead of the single wide FT.AGGREGATE. `blockOnly`
244
- // narrows the sub-queries to `@type:{block}` for the hard-block
245
- // middleware; every other caller (frictionless access-policy
246
- // lookup, verify-time hard-block re-check) fetches both Block
247
- // and Restrict rules and lets `rankCandidateRules` pick.
248
- //
249
- // The previous FT.AGGREGATE ranker used for the non-block path
250
- // silently dropped specific-IP Restrict rules from the top-N
251
- // when many higher-specificity irrelevant rules matched the
252
- // greedy `ismissing(@field)` disjunction — the union-of-probes
253
- // shape here can't hit that failure mode because each probe
254
- // only returns rules where the probed field is actually
255
- // populated with the request's value.
256
- if (matchingFieldsOnly && filter.userScope !== undefined) {
257
- return this.findRulesSplit(filter);
258
- }
259
-
260
- // Admin / internal callers (greedy mode): keep the FT.SEARCH +
261
- // HGETALL fanout path. Cap at REDIS_BATCH_SIZE; truncation past
262
- // that point is the known liability tracked in #2689.
263
- return this.findRulesGreedy(filter, query);
264
- }
265
-
266
- // Union candidate keys from each split sub-query, de-dupe, HGETALL,
267
- // parse, hand back to `getPrioritisedAccessRule` for JS-side ranking.
268
- // Ranking stays in JS because rankCandidateRules already encodes the
269
- // strict "every populated rule field matches request" semantics —
270
- // FT can't express that shape cheaply and the previous server-side
271
- // APPLY / SORTBY imposed the linear-per-rule cost we're eliminating.
272
- //
273
- // Handles both blockOnly (hard-block middleware) and non-blockOnly
274
- // (frictionless access-policy lookup) callers via the `blockOnly`
275
- // flag on the sub-query builder.
276
- private async findRulesSplit(
277
- filter: AccessRulesFilter,
278
- ): Promise<AccessRule[]> {
279
- const userScope = filter.userScope;
280
- if (userScope === undefined) {
281
- return [];
282
- }
283
- const clientId = filter.policyScope?.clientId;
284
-
285
- const subQueries = buildScopedRulesSubQueries(userScope, clientId, {
286
- ...(filter.blockOnly === true && { blockOnly: true }),
287
- ...(filter.includeDeferred === true && { includeDeferred: true }),
288
- });
289
-
290
- try {
291
- const keyLists = await Promise.all(
292
- subQueries.map((sub) =>
293
- // FT.SEARCH NOCONTENT (via searchRedisKeysBounded) rather
294
- // than FT.AGGREGATE with LOAD @__key. The aggregate reply
295
- // path in `@redis/client` 5.x calls transformTuplesReply
296
- // on every result row; when the RediSearch coordinator
297
- // returns a null tuple (observed continuously on
298
- // field:ja4Hash probes in prod), that throws
299
- // `Cannot read properties of null (reading 'length')` and
300
- // the whole sub-query resolves empty via the .catch below.
301
- // NOCONTENT returns a flat `[total, key1, key2, ...]`
302
- // array so there is no nested tuple that can be null.
303
- searchRedisKeysBounded(
304
- this.client,
305
- sub.query,
306
- // Per-sub-query cap. Each probe hits a discriminating
307
- // index (e.g. numericIp posting list), so the natural
308
- // cardinality is tiny — this cap only fires on
309
- // pathological rule-set shapes and mirrors
310
- // SPLIT_MAX_CANDIDATES_PER_SUB below.
311
- SPLIT_MAX_CANDIDATES_PER_SUB,
312
- ).catch((err) => {
313
- // One sub-query failing (e.g. a transient RediSearch
314
- // error) shouldn't drop the whole lookup — the other
315
- // probes may still find the applicable rule. Log and
316
- // continue with an empty result for this probe.
317
- this.logger.error(() => ({
318
- err,
319
- data: {
320
- inspect: util.inspect(
321
- { subKind: sub.kind, query: sub.query },
322
- { depth: null },
323
- ),
324
- },
325
- msg: "failed to execute split rule sub-query",
326
- }));
327
- return [];
328
- }),
329
- ),
330
- );
331
-
332
- const uniqueKeys = new Set<string>();
333
- for (const list of keyLists) {
334
- for (const key of list) {
335
- uniqueKeys.add(key);
336
- }
337
- }
338
-
339
- if (uniqueKeys.size === 0) {
340
- return [];
341
- }
342
-
343
- const ruleKeys = [...uniqueKeys];
344
-
345
- this.logger.debug(() => ({
346
- msg: "Executed split rule sub-queries",
347
- data: {
348
- inspect: util.inspect(
349
- {
350
- subQueryCount: subQueries.length,
351
- uniqueCandidates: ruleKeys.length,
352
- perSubQueryCounts: keyLists.map((l, i) => ({
353
- kind: subQueries[i]?.kind,
354
- found: l.length,
355
- })),
356
- filter,
357
- },
358
- { depth: null },
359
- ),
360
- },
361
- }));
362
-
363
- const { records } = await fetchRedisHashRecords(
364
- this.client,
365
- ruleKeys,
366
- this.logger,
367
- );
368
-
369
- const nonEmptyRecords = records.filter(
370
- (record) => Object.keys(record).length > 0,
371
- );
372
-
373
- const parsedRules = parseRedisRecords(
374
- nonEmptyRecords,
375
- accessRuleInput,
376
- this.logger,
377
- );
378
-
379
- // Preserve the reader-level (specificity desc, block-first)
380
- // ordering the old FT.AGGREGATE ranker gave. Split sub-queries
381
- // return keys per-probe (i.e. in insertion order), so without
382
- // this sort the caller sees candidates in a shape the previous
383
- // implementation never returned. Downstream callers
384
- // (`rankCandidateRules` in the provider) do a full re-sort, so
385
- // this is purely a defence against silent behaviour changes
386
- // for anyone else that consumes the reader directly.
387
- return sortByReaderRank(parsedRules);
388
- } catch (e) {
389
- this.logger.error(() => ({
390
- err: e,
391
- data: {
392
- inspect: util.inspect({ filter }, { depth: null }),
393
- },
394
- msg: "failed to execute split rule query set",
395
- }));
396
- return [];
397
- }
398
- }
399
-
400
- private async findRulesRanked(
401
- filter: AccessRulesFilter,
402
- query: string,
403
- ): Promise<AccessRule[]> {
404
- try {
405
- const reply = await this.client.ft.aggregate(
406
- ACCESS_RULES_REDIS_INDEX_NAME,
407
- query,
408
- {
409
- DIALECT: REDIS_QUERY_DIALECT,
410
- LOAD: [...RULE_LOAD_FIELDS],
411
- STEPS: [
412
- // Drop candidates whose hash is missing `type` before
413
- // any APPLY runs. SEVERITY_EXPR dereferences `@type`
414
- // directly, and RediSearch throws
415
- // "Could not find the value for a parameter name,
416
- // consider using EXISTS if applicable for type"
417
- // when the LOAD pulls an undefined @type, aborting
418
- // the whole aggregate. The undefined @type can come
419
- // from a partial-write race in the writer or a
420
- // stale RediSearch index entry pointing at a hash
421
- // whose `type` field has been removed; in both
422
- // cases the doc is malformed and not a valid
423
- // AccessRule, so dropping it is correct.
424
- { type: "FILTER", expression: "exists(@type)" },
425
- { type: "APPLY", expression: SPECIFICITY_EXPR, AS: "_spec" },
426
- { type: "APPLY", expression: SEVERITY_EXPR, AS: "_sev" },
427
- { type: "APPLY", expression: RANK_EXPR, AS: "_rank" },
428
- {
429
- type: "SORTBY",
430
- BY: [{ BY: "@_rank", DIRECTION: "DESC" }],
431
- // SORTBY MAX would let RediSearch keep only the
432
- // top-N in a bounded heap rather than fully
433
- // sorting, but the @redis/search client
434
- // serialises MAX *before* DESC and RediSearch
435
- // rejects that order ("MISSING ASC or DESC
436
- // after sort field (MAX)"). LIMIT below still
437
- // trims; SORTBY does a full sort. Acceptable
438
- // because the strict-match filter already
439
- // keeps the candidate set small in practice.
440
- },
441
- { type: "LIMIT", from: 0, size: SERVER_SIDE_RANK_TOP_N },
442
- ],
443
- },
444
- );
445
-
446
- if (reply.results.length === 0) {
447
- return [];
448
- }
449
-
450
- this.logger.debug(() => ({
451
- msg: "Executed ranked search query",
452
- data: {
453
- inspect: util.inspect(
454
- {
455
- filter: filter,
456
- foundCount: reply.results.length,
457
- query: query,
458
- },
459
- { depth: null },
460
- ),
461
- },
462
- }));
463
-
464
- // FT.AGGREGATE LOAD reads NUMERIC fields from the index (an
465
- // 8-byte double), not from the underlying hash, so any
466
- // numericIp/numericIpMaskMin/numericIpMaskMax larger than
467
- // Number.MAX_SAFE_INTEGER (e.g. every IPv6 rule) round-trips
468
- // as scientific notation — `5.59112965392e+37` — and
469
- // `z.coerce.bigint()` throws
470
- // "Cannot convert 5.59112965392e+37 to a BigInt"
471
- // which aborts the whole aggregate via the catch below and
472
- // returns no rules. The hash itself stores the full
473
- // 38-digit string verbatim, so the aggregate is used purely
474
- // as a ranker (it gives us the top-N keys, sorted by spec /
475
- // severity); the real field values come back via HGETALL
476
- // over those keys, identical to the greedy path. Same
477
- // race-safety as the greedy path: HGETALL on a key that's
478
- // been deleted between FT.AGGREGATE and HGETALL returns an
479
- // empty hash, which is dropped by the non-empty filter.
480
- const ruleKeys: string[] = [];
481
- for (const result of reply.results) {
482
- const key = (result as { __key?: unknown }).__key;
483
- if (typeof key === "string") {
484
- ruleKeys.push(key);
485
- }
486
- }
487
-
488
- if (ruleKeys.length === 0) {
489
- return [];
490
- }
491
-
492
- const { records } = await fetchRedisHashRecords(
493
- this.client,
494
- ruleKeys,
495
- this.logger,
496
- );
497
-
498
- const nonEmptyRecords = records.filter(
499
- (record) => Object.keys(record).length > 0,
500
- );
501
-
502
- return parseRedisRecords(nonEmptyRecords, accessRuleInput, this.logger);
503
- } catch (e) {
504
- this.logger.error(() => ({
505
- err: e,
506
- data: {
507
- inspect: util.inspect({ query, filter }, { depth: null }),
508
- },
509
- msg: "failed to execute ranked search query",
510
- }));
511
-
512
- return [];
513
- }
514
- }
515
-
516
- // Used only by admin / internal callers (matchingFieldsOnly=false).
517
- // Not on the per-request blockMiddleware path any more, so the CPU
518
- // cost of the HGETALL fanout is acceptable. Same shape as #2689's
519
- // aggregate-with-cursor: returns every candidate up to
520
- // GREEDY_MAX_CANDIDATES so we don't silently drop matches that
521
- // admin tools depend on (e.g. ASN-wide rule listings).
522
- private async findRulesGreedy(
523
- filter: AccessRulesFilter,
524
- query: string,
525
- ): Promise<AccessRule[]> {
526
- try {
527
- const ruleKeys = await aggregateRedisKeys(
528
- this.client,
529
- query,
530
- this.logger,
531
- undefined,
532
- GREEDY_MAX_CANDIDATES,
533
- );
534
-
535
- if (ruleKeys.length === 0) {
536
- return [];
537
- }
538
-
539
- this.logger.debug(() => ({
540
- msg: "Executed greedy search query",
541
- data: {
542
- inspect: util.inspect(
543
- {
544
- filter: filter,
545
- foundCount: ruleKeys.length,
546
- query: query,
547
- },
548
- { depth: null },
549
- ),
550
- },
551
- }));
552
-
553
- const { records } = await fetchRedisHashRecords(
554
- this.client,
555
- ruleKeys,
556
- this.logger,
557
- );
558
-
559
- const nonEmptyRecords = records.filter(
560
- (record) => Object.keys(record).length > 0,
561
- );
562
-
563
- return parseRedisRecords(nonEmptyRecords, accessRuleInput, this.logger);
564
- } catch (e) {
565
- this.logger.error(() => ({
566
- err: e,
567
- data: {
568
- inspect: util.inspect({ query, filter }, { depth: null }),
569
- },
570
- msg: "failed to execute greedy search query",
571
- }));
572
-
573
- return [];
574
- }
575
- }
576
-
577
- async findRuleIds(
578
- filter: AccessRulesFilter,
579
- matchingFieldsOnly = false,
580
- ): Promise<string[]> {
581
- const query = getRulesRedisQuery(filter, matchingFieldsOnly);
582
-
583
- let ruleIds: string[] = [];
584
-
585
- try {
586
- // aggregation is used instead ft.search to overcome the limitation on search results number
587
- const ruleKeys = await aggregateRedisKeys(
588
- this.client,
589
- query,
590
- this.logger,
591
- );
592
-
593
- ruleIds = ruleKeys.map((ruleKey) =>
594
- ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length),
595
- );
596
- } catch (e) {
597
- this.logger.error(() => ({
598
- err: e,
599
- data: {
600
- inspect: util.inspect(
601
- {
602
- query: query,
603
- filter: filter,
604
- },
605
- {
606
- depth: null,
607
- },
608
- ),
609
- },
610
- msg: "Failed to execute search query for rule IDs",
611
- }));
612
-
613
- return [];
614
- }
615
-
616
- this.logger.debug(() => ({
617
- msg: "Executed search query for rule IDs",
618
- data: {
619
- query: query,
620
- foundCount: ruleIds.length,
621
- foundIds: ruleIds,
622
- },
623
- }));
624
-
625
- return ruleIds;
626
- }
627
-
628
- async fetchAllRuleIds(
629
- batchHandler: (ruleIds: string[]) => Promise<void>,
630
- ): Promise<void> {
631
- const keysBatchHandler = async (keys: string[]) => {
632
- const ids = keys.map((ruleKey) =>
633
- ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length),
634
- );
635
-
636
- await batchHandler(ids);
637
- };
638
-
639
- await aggregateRedisKeys(this.client, "*", this.logger, keysBatchHandler);
640
- }
641
-
642
- protected async fetchRuleEntries(keys: string[]): Promise<AccessRuleEntry[]> {
643
- const { records, expirations } = await fetchRedisHashRecords(
644
- this.client,
645
- keys,
646
- this.logger,
647
- );
648
- const entries: AccessRuleEntry[] = [];
649
-
650
- for (const [index, ruleData] of records.entries()) {
651
- const isRulePresent = Object.keys(ruleData).length > 0;
652
-
653
- if (isRulePresent) {
654
- const rule = parseRedisRecords(
655
- [ruleData],
656
- accessRuleInput,
657
- this.logger,
658
- )[0];
659
-
660
- if (rule) {
661
- entries.push({
662
- rule: rule,
663
- expiresUnixTimestamp: expirations[index],
664
- });
665
- }
666
- }
667
- }
668
-
669
- return entries;
670
- }
671
-
672
- protected getRuleKeys(ruleIds: string[]): string[] {
673
- return ruleIds.map((id) => `${ACCESS_RULE_REDIS_KEY_PREFIX}${id}`);
674
- }
675
- }
676
-
677
- export class DummyRedisRulesReader implements AccessRulesReader {
678
- constructor(private readonly logger: Logger) {}
679
-
680
- async getMissingRuleIds(ruleIds: string[]): Promise<string[]> {
681
- this.logger.info(() => ({
682
- msg: "Dummy getMissingRuleIds() has no effect (redis is not ready)",
683
- data: {
684
- ruleIds,
685
- },
686
- }));
687
-
688
- return [];
689
- }
690
-
691
- async fetchRules(ruleIds: string[]): Promise<AccessRuleEntry[]> {
692
- this.logger.info(() => ({
693
- msg: "Dummy fetchRule() has no effect (redis is not ready)",
694
- data: {
695
- ruleIds,
696
- },
697
- }));
698
-
699
- return [];
700
- }
701
-
702
- async findRules(
703
- filter: AccessRulesFilter,
704
- matchingFieldsOnly = false,
705
- skipEmptyUserScopes = true,
706
- ): Promise<AccessRule[]> {
707
- this.logger.info(() => ({
708
- msg: "Dummy findRules() has no effect (redis is not ready)",
709
- data: {
710
- filter,
711
- },
712
- }));
713
-
714
- return [];
715
- }
716
-
717
- async findRuleIds(
718
- filter: AccessRulesFilter,
719
- matchingFieldsOnly = false,
720
- ): Promise<string[]> {
721
- this.logger.info(() => ({
722
- msg: "Dummy findRuleIds() has no effect (redis is not ready)",
723
- data: {
724
- filter,
725
- },
726
- }));
727
-
728
- return [];
729
- }
730
-
731
- async fetchAllRuleIds(
732
- batchHandler: (ruleIds: string[]) => Promise<void>,
733
- ): Promise<void> {
734
- this.logger.info(() => ({
735
- msg: "Dummy fetchAllRuleIds() has no effect (redis is not ready)",
736
- }));
737
- }
738
- }