@prosopo/user-access-policy 3.12.8 → 3.12.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build$colon$cjs.log +45 -41
- package/.turbo/turbo-build$colon$tsc.log +14 -14
- package/.turbo/turbo-build.log +46 -42
- package/CHANGELOG.md +24 -0
- package/dist/.export.js +7 -21
- package/dist/_virtual/_rolldown/runtime.js +3 -0
- package/dist/api/.export.js +2 -9
- package/dist/api/accessRulesApiClient.js +1 -1
- package/dist/api/delete/.export.js +2 -1
- package/dist/api/delete/deleteAllRules.js +22 -24
- package/dist/api/delete/deleteRuleGroups.js +39 -51
- package/dist/api/delete/deleteRules.js +37 -41
- package/dist/api/read/.export.js +2 -1
- package/dist/api/read/fetchRules.js +34 -42
- package/dist/api/read/findRuleIds.js +35 -48
- package/dist/api/read/getMissingIds.js +36 -40
- package/dist/api/ruleApiRoutes.js +62 -120
- package/dist/api/rulesApiClient.js +51 -87
- package/dist/api/write/.export.js +2 -1
- package/dist/api/write/insertRules.js +84 -102
- package/dist/api/write/rehashRules.js +48 -56
- package/dist/cjs/.export.cjs +21 -23
- package/dist/cjs/_virtual/_rolldown/runtime.cjs +23 -0
- package/dist/cjs/api/.export.cjs +6 -10
- package/dist/cjs/api/accessRulesApiClient.cjs +0 -0
- package/dist/cjs/api/delete/.export.cjs +0 -1
- package/dist/cjs/api/delete/deleteAllRules.cjs +22 -24
- package/dist/cjs/api/delete/deleteRuleGroups.cjs +39 -51
- package/dist/cjs/api/delete/deleteRules.cjs +41 -42
- package/dist/cjs/api/read/.export.cjs +0 -1
- package/dist/cjs/api/read/fetchRules.cjs +34 -41
- package/dist/cjs/api/read/findRuleIds.cjs +37 -48
- package/dist/cjs/api/read/getMissingIds.cjs +36 -39
- package/dist/cjs/api/ruleApiRoutes.cjs +69 -125
- package/dist/cjs/api/rulesApiClient.cjs +54 -90
- package/dist/cjs/api/write/.export.cjs +0 -1
- package/dist/cjs/api/write/insertRules.cjs +85 -103
- package/dist/cjs/api/write/rehashRules.cjs +48 -56
- package/dist/cjs/classifyOs.cjs +46 -26
- package/dist/cjs/mongoose/.export.cjs +2 -3
- package/dist/cjs/mongoose/mongooseRuleSchema.cjs +97 -36
- package/dist/cjs/redis/.export.cjs +4 -5
- package/dist/cjs/redis/reader/redisAggregate.cjs +59 -84
- package/dist/cjs/redis/reader/redisRulesQuery.cjs +74 -116
- package/dist/cjs/redis/reader/redisRulesReader.cjs +322 -503
- package/dist/cjs/redis/reader/redisRulesSplitQuery.cjs +94 -72
- package/dist/cjs/redis/redisClient.cjs +53 -58
- package/dist/cjs/redis/redisRuleIndex.cjs +93 -49
- package/dist/cjs/redis/redisRulesStorage.cjs +21 -30
- package/dist/cjs/redis/redisRulesWriter.cjs +86 -112
- package/dist/cjs/rule.cjs +8 -7
- package/dist/cjs/ruleInput/.export.cjs +7 -8
- package/dist/cjs/ruleInput/policyInput.cjs +35 -58
- package/dist/cjs/ruleInput/ruleInput.cjs +37 -42
- package/dist/cjs/ruleInput/userScopeInput.cjs +47 -55
- package/dist/cjs/ruleRecord.cjs +15 -22
- package/dist/cjs/rulesStorage.cjs +7 -6
- package/dist/cjs/transformRule.cjs +36 -70
- package/dist/classifyOs.js +48 -29
- package/dist/mongoose/.export.js +2 -3
- package/dist/mongoose/mongooseRuleSchema.js +99 -38
- package/dist/redis/.export.js +3 -5
- package/dist/redis/reader/redisAggregate.js +60 -86
- package/dist/redis/reader/redisRulesQuery.js +75 -118
- package/dist/redis/reader/redisRulesReader.js +312 -487
- package/dist/redis/reader/redisRulesSplitQuery.js +96 -75
- package/dist/redis/redisClient.js +54 -62
- package/dist/redis/redisRuleIndex.js +93 -58
- package/dist/redis/redisRulesStorage.js +21 -30
- package/dist/redis/redisRulesWriter.js +82 -113
- package/dist/rule.js +10 -10
- package/dist/ruleInput/.export.js +3 -7
- package/dist/ruleInput/policyInput.js +35 -61
- package/dist/ruleInput/ruleInput.js +36 -44
- package/dist/ruleInput/userScopeInput.js +44 -55
- package/dist/ruleRecord.js +17 -27
- package/dist/rulesStorage.js +9 -8
- package/dist/tests/redis/redisRulesStorage.integration.test.js +1 -3
- package/dist/tests/redis/redisRulesStorage.integration.test.js.map +1 -1
- package/dist/transformRule.js +34 -73
- package/entries.ts +25 -6
- package/package.json +13 -12
- package/src/tests/redis/redisRulesStorage.integration.test.ts +191 -195
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,494 +1,319 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { chunkIntoBatches, executeBatchesSequentially } from "@prosopo/common";
|
|
3
|
-
import { getRulesRedisQuery, REDIS_QUERY_DIALECT } from "./redisRulesQuery.js";
|
|
4
|
-
import { buildScopedRulesSubQueries } from "./redisRulesSplitQuery.js";
|
|
5
|
-
import { REDIS_BATCH_SIZE, getMissingRedisKeys, fetchRedisHashRecords, parseRedisRecords } from "../redisClient.js";
|
|
6
|
-
import { ACCESS_RULE_REDIS_KEY_PREFIX, ACCESS_RULES_REDIS_INDEX_NAME } from "../redisRuleIndex.js";
|
|
1
|
+
import "../../_virtual/_rolldown/runtime.js";
|
|
7
2
|
import { AccessPolicyType } from "../../rule.js";
|
|
8
3
|
import { accessRuleInput } from "../../ruleInput/ruleInput.js";
|
|
9
|
-
import {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
4
|
+
import { getRulesRedisQuery } from "./redisRulesQuery.js";
|
|
5
|
+
import { buildScopedRulesSubQueries } from "./redisRulesSplitQuery.js";
|
|
6
|
+
import { REDIS_BATCH_SIZE, fetchRedisHashRecords, getMissingRedisKeys, parseRedisRecords } from "../redisClient.js";
|
|
7
|
+
import { ACCESS_RULES_REDIS_INDEX_NAME, ACCESS_RULE_REDIS_KEY_PREFIX } from "../redisRuleIndex.js";
|
|
8
|
+
import { aggregateRedisKeys, searchRedisKeysBounded } from "./redisAggregate.js";
|
|
9
|
+
import { chunkIntoBatches, executeBatchesSequentially } from "@prosopo/common";
|
|
10
|
+
import * as util from "node:util";
|
|
11
|
+
//#region src/redis/reader/redisRulesReader.ts
|
|
12
|
+
var SERVER_SIDE_RANK_TOP_N = 20;
|
|
13
|
+
var GREEDY_MAX_CANDIDATES = REDIS_BATCH_SIZE * 10;
|
|
14
|
+
var SPLIT_MAX_CANDIDATES_PER_SUB = 500;
|
|
15
|
+
var SPECIFICITY_EXPR = [
|
|
16
|
+
"exists(@clientId)",
|
|
17
|
+
"exists(@userId)",
|
|
18
|
+
"exists(@ja4Hash)",
|
|
19
|
+
"exists(@headersHash)",
|
|
20
|
+
"exists(@userAgentHash)",
|
|
21
|
+
"exists(@headHash)",
|
|
22
|
+
"exists(@coords)",
|
|
23
|
+
"exists(@countryCode)",
|
|
24
|
+
"exists(@asn)",
|
|
25
|
+
"exists(@os)",
|
|
26
|
+
"exists(@numericIp)",
|
|
27
|
+
"exists(@numericIpMaskMin)"
|
|
26
28
|
].join(" + ");
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
29
|
+
var SEVERITY_EXPR = `(@type == "${AccessPolicyType.Block}")`;
|
|
30
|
+
var RANK_EXPR = "(@_spec * 2) + @_sev";
|
|
31
|
+
var RULE_LOAD_FIELDS = [
|
|
32
|
+
"@__key",
|
|
33
|
+
"@type",
|
|
34
|
+
"@captchaType",
|
|
35
|
+
"@description",
|
|
36
|
+
"@solvedImagesCount",
|
|
37
|
+
"@imageThreshold",
|
|
38
|
+
"@powDifficulty",
|
|
39
|
+
"@unsolvedImagesCount",
|
|
40
|
+
"@frictionlessScore",
|
|
41
|
+
"@deferToVerify",
|
|
42
|
+
"@clientId",
|
|
43
|
+
"@groupId",
|
|
44
|
+
"@userId",
|
|
45
|
+
"@ja4Hash",
|
|
46
|
+
"@headersHash",
|
|
47
|
+
"@userAgentHash",
|
|
48
|
+
"@headHash",
|
|
49
|
+
"@coords",
|
|
50
|
+
"@countryCode",
|
|
51
|
+
"@asn",
|
|
52
|
+
"@os",
|
|
53
|
+
"@numericIp",
|
|
54
|
+
"@numericIpMaskMin",
|
|
55
|
+
"@numericIpMaskMax"
|
|
54
56
|
];
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
57
|
+
var readerSpecificity = (rule) => {
|
|
58
|
+
let score = 0;
|
|
59
|
+
if (rule.clientId !== void 0) score++;
|
|
60
|
+
if (rule.userId !== void 0) score++;
|
|
61
|
+
if (rule.ja4Hash !== void 0) score++;
|
|
62
|
+
if (rule.headersHash !== void 0) score++;
|
|
63
|
+
if (rule.userAgentHash !== void 0) score++;
|
|
64
|
+
if (rule.headHash !== void 0) score++;
|
|
65
|
+
if (rule.coords !== void 0) score++;
|
|
66
|
+
if (rule.countryCode !== void 0) score++;
|
|
67
|
+
if (rule.asn !== void 0) score++;
|
|
68
|
+
if (rule.os !== void 0) score++;
|
|
69
|
+
if (rule.numericIp !== void 0) score++;
|
|
70
|
+
if (rule.numericIpMaskMin !== void 0) score++;
|
|
71
|
+
return score;
|
|
70
72
|
};
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
return bBlock - aBlock;
|
|
73
|
+
var sortByReaderRank = (rules) => [...rules].sort((a, b) => {
|
|
74
|
+
const specDelta = readerSpecificity(b) - readerSpecificity(a);
|
|
75
|
+
if (specDelta !== 0) return specDelta;
|
|
76
|
+
const aBlock = a.type === AccessPolicyType.Block ? 1 : 0;
|
|
77
|
+
return (b.type === AccessPolicyType.Block ? 1 : 0) - aBlock;
|
|
77
78
|
});
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
// Not on the per-request blockMiddleware path any more, so the CPU
|
|
317
|
-
// cost of the HGETALL fanout is acceptable. Same shape as #2689's
|
|
318
|
-
// aggregate-with-cursor: returns every candidate up to
|
|
319
|
-
// GREEDY_MAX_CANDIDATES so we don't silently drop matches that
|
|
320
|
-
// admin tools depend on (e.g. ASN-wide rule listings).
|
|
321
|
-
async findRulesGreedy(filter, query) {
|
|
322
|
-
try {
|
|
323
|
-
const ruleKeys = await aggregateRedisKeys(
|
|
324
|
-
this.client,
|
|
325
|
-
query,
|
|
326
|
-
this.logger,
|
|
327
|
-
void 0,
|
|
328
|
-
GREEDY_MAX_CANDIDATES
|
|
329
|
-
);
|
|
330
|
-
if (ruleKeys.length === 0) {
|
|
331
|
-
return [];
|
|
332
|
-
}
|
|
333
|
-
this.logger.debug(() => ({
|
|
334
|
-
msg: "Executed greedy search query",
|
|
335
|
-
data: {
|
|
336
|
-
inspect: util.inspect(
|
|
337
|
-
{
|
|
338
|
-
filter,
|
|
339
|
-
foundCount: ruleKeys.length,
|
|
340
|
-
query
|
|
341
|
-
},
|
|
342
|
-
{ depth: null }
|
|
343
|
-
)
|
|
344
|
-
}
|
|
345
|
-
}));
|
|
346
|
-
const { records } = await fetchRedisHashRecords(
|
|
347
|
-
this.client,
|
|
348
|
-
ruleKeys,
|
|
349
|
-
this.logger
|
|
350
|
-
);
|
|
351
|
-
const nonEmptyRecords = records.filter(
|
|
352
|
-
(record) => Object.keys(record).length > 0
|
|
353
|
-
);
|
|
354
|
-
return parseRedisRecords(nonEmptyRecords, accessRuleInput, this.logger);
|
|
355
|
-
} catch (e) {
|
|
356
|
-
this.logger.error(() => ({
|
|
357
|
-
err: e,
|
|
358
|
-
data: {
|
|
359
|
-
inspect: util.inspect({ query, filter }, { depth: null })
|
|
360
|
-
},
|
|
361
|
-
msg: "failed to execute greedy search query"
|
|
362
|
-
}));
|
|
363
|
-
return [];
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
async findRuleIds(filter, matchingFieldsOnly = false) {
|
|
367
|
-
const query = getRulesRedisQuery(filter, matchingFieldsOnly);
|
|
368
|
-
let ruleIds = [];
|
|
369
|
-
try {
|
|
370
|
-
const ruleKeys = await aggregateRedisKeys(
|
|
371
|
-
this.client,
|
|
372
|
-
query,
|
|
373
|
-
this.logger
|
|
374
|
-
);
|
|
375
|
-
ruleIds = ruleKeys.map(
|
|
376
|
-
(ruleKey) => ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length)
|
|
377
|
-
);
|
|
378
|
-
} catch (e) {
|
|
379
|
-
this.logger.error(() => ({
|
|
380
|
-
err: e,
|
|
381
|
-
data: {
|
|
382
|
-
inspect: util.inspect(
|
|
383
|
-
{
|
|
384
|
-
query,
|
|
385
|
-
filter
|
|
386
|
-
},
|
|
387
|
-
{
|
|
388
|
-
depth: null
|
|
389
|
-
}
|
|
390
|
-
)
|
|
391
|
-
},
|
|
392
|
-
msg: "Failed to execute search query for rule IDs"
|
|
393
|
-
}));
|
|
394
|
-
return [];
|
|
395
|
-
}
|
|
396
|
-
this.logger.debug(() => ({
|
|
397
|
-
msg: "Executed search query for rule IDs",
|
|
398
|
-
data: {
|
|
399
|
-
query,
|
|
400
|
-
foundCount: ruleIds.length,
|
|
401
|
-
foundIds: ruleIds
|
|
402
|
-
}
|
|
403
|
-
}));
|
|
404
|
-
return ruleIds;
|
|
405
|
-
}
|
|
406
|
-
async fetchAllRuleIds(batchHandler) {
|
|
407
|
-
const keysBatchHandler = async (keys) => {
|
|
408
|
-
const ids = keys.map(
|
|
409
|
-
(ruleKey) => ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length)
|
|
410
|
-
);
|
|
411
|
-
await batchHandler(ids);
|
|
412
|
-
};
|
|
413
|
-
await aggregateRedisKeys(this.client, "*", this.logger, keysBatchHandler);
|
|
414
|
-
}
|
|
415
|
-
async fetchRuleEntries(keys) {
|
|
416
|
-
const { records, expirations } = await fetchRedisHashRecords(
|
|
417
|
-
this.client,
|
|
418
|
-
keys,
|
|
419
|
-
this.logger
|
|
420
|
-
);
|
|
421
|
-
const entries = [];
|
|
422
|
-
for (const [index, ruleData] of records.entries()) {
|
|
423
|
-
const isRulePresent = Object.keys(ruleData).length > 0;
|
|
424
|
-
if (isRulePresent) {
|
|
425
|
-
const rule = parseRedisRecords(
|
|
426
|
-
[ruleData],
|
|
427
|
-
accessRuleInput,
|
|
428
|
-
this.logger
|
|
429
|
-
)[0];
|
|
430
|
-
if (rule) {
|
|
431
|
-
entries.push({
|
|
432
|
-
rule,
|
|
433
|
-
expiresUnixTimestamp: expirations[index]
|
|
434
|
-
});
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
return entries;
|
|
439
|
-
}
|
|
440
|
-
getRuleKeys(ruleIds) {
|
|
441
|
-
return ruleIds.map((id) => `${ACCESS_RULE_REDIS_KEY_PREFIX}${id}`);
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
class DummyRedisRulesReader {
|
|
445
|
-
constructor(logger) {
|
|
446
|
-
this.logger = logger;
|
|
447
|
-
}
|
|
448
|
-
async getMissingRuleIds(ruleIds) {
|
|
449
|
-
this.logger.info(() => ({
|
|
450
|
-
msg: "Dummy getMissingRuleIds() has no effect (redis is not ready)",
|
|
451
|
-
data: {
|
|
452
|
-
ruleIds
|
|
453
|
-
}
|
|
454
|
-
}));
|
|
455
|
-
return [];
|
|
456
|
-
}
|
|
457
|
-
async fetchRules(ruleIds) {
|
|
458
|
-
this.logger.info(() => ({
|
|
459
|
-
msg: "Dummy fetchRule() has no effect (redis is not ready)",
|
|
460
|
-
data: {
|
|
461
|
-
ruleIds
|
|
462
|
-
}
|
|
463
|
-
}));
|
|
464
|
-
return [];
|
|
465
|
-
}
|
|
466
|
-
async findRules(filter, matchingFieldsOnly = false, skipEmptyUserScopes = true) {
|
|
467
|
-
this.logger.info(() => ({
|
|
468
|
-
msg: "Dummy findRules() has no effect (redis is not ready)",
|
|
469
|
-
data: {
|
|
470
|
-
filter
|
|
471
|
-
}
|
|
472
|
-
}));
|
|
473
|
-
return [];
|
|
474
|
-
}
|
|
475
|
-
async findRuleIds(filter, matchingFieldsOnly = false) {
|
|
476
|
-
this.logger.info(() => ({
|
|
477
|
-
msg: "Dummy findRuleIds() has no effect (redis is not ready)",
|
|
478
|
-
data: {
|
|
479
|
-
filter
|
|
480
|
-
}
|
|
481
|
-
}));
|
|
482
|
-
return [];
|
|
483
|
-
}
|
|
484
|
-
async fetchAllRuleIds(batchHandler) {
|
|
485
|
-
this.logger.info(() => ({
|
|
486
|
-
msg: "Dummy fetchAllRuleIds() has no effect (redis is not ready)"
|
|
487
|
-
}));
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
export {
|
|
491
|
-
DummyRedisRulesReader,
|
|
492
|
-
RedisRulesReader,
|
|
493
|
-
SERVER_SIDE_RANK_TOP_N
|
|
79
|
+
var RedisRulesReader = class {
|
|
80
|
+
constructor(client, logger) {
|
|
81
|
+
this.client = client;
|
|
82
|
+
this.logger = logger;
|
|
83
|
+
}
|
|
84
|
+
async getMissingRuleIds(ruleIds) {
|
|
85
|
+
return (await executeBatchesSequentially(chunkIntoBatches(this.getRuleKeys(ruleIds), REDIS_BATCH_SIZE), async (keysBatch) => getMissingRedisKeys(this.client, keysBatch))).flat().map((ruleKey) => ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length));
|
|
86
|
+
}
|
|
87
|
+
async fetchRules(ruleIds) {
|
|
88
|
+
return (await executeBatchesSequentially(chunkIntoBatches(this.getRuleKeys(ruleIds), REDIS_BATCH_SIZE), (keysBatch) => this.fetchRuleEntries(keysBatch))).flat();
|
|
89
|
+
}
|
|
90
|
+
async findRules(filter, matchingFieldsOnly = false, skipEmptyUserScopes = true) {
|
|
91
|
+
const query = getRulesRedisQuery(filter, matchingFieldsOnly);
|
|
92
|
+
if (skipEmptyUserScopes && query === "( @clientId:{global} | ismissing(@clientId) )") return [];
|
|
93
|
+
if (matchingFieldsOnly && filter.userScope !== void 0) return this.findRulesSplit(filter);
|
|
94
|
+
return this.findRulesGreedy(filter, query);
|
|
95
|
+
}
|
|
96
|
+
async findRulesSplit(filter) {
|
|
97
|
+
const userScope = filter.userScope;
|
|
98
|
+
if (userScope === void 0) return [];
|
|
99
|
+
const clientId = filter.policyScope?.clientId;
|
|
100
|
+
const subQueries = buildScopedRulesSubQueries(userScope, clientId, { ...filter.blockOnly === true && { blockOnly: true } });
|
|
101
|
+
try {
|
|
102
|
+
const keyLists = await Promise.all(subQueries.map((sub) => searchRedisKeysBounded(this.client, sub.query, SPLIT_MAX_CANDIDATES_PER_SUB).catch((err) => {
|
|
103
|
+
this.logger.error(() => ({
|
|
104
|
+
err,
|
|
105
|
+
data: { inspect: util.inspect({
|
|
106
|
+
subKind: sub.kind,
|
|
107
|
+
query: sub.query
|
|
108
|
+
}, { depth: null }) },
|
|
109
|
+
msg: "failed to execute split rule sub-query"
|
|
110
|
+
}));
|
|
111
|
+
return [];
|
|
112
|
+
})));
|
|
113
|
+
const uniqueKeys = /* @__PURE__ */ new Set();
|
|
114
|
+
for (const list of keyLists) for (const key of list) uniqueKeys.add(key);
|
|
115
|
+
if (uniqueKeys.size === 0) return [];
|
|
116
|
+
const ruleKeys = [...uniqueKeys];
|
|
117
|
+
this.logger.debug(() => ({
|
|
118
|
+
msg: "Executed split rule sub-queries",
|
|
119
|
+
data: { inspect: util.inspect({
|
|
120
|
+
subQueryCount: subQueries.length,
|
|
121
|
+
uniqueCandidates: ruleKeys.length,
|
|
122
|
+
perSubQueryCounts: keyLists.map((l, i) => ({
|
|
123
|
+
kind: subQueries[i]?.kind,
|
|
124
|
+
found: l.length
|
|
125
|
+
})),
|
|
126
|
+
filter
|
|
127
|
+
}, { depth: null }) }
|
|
128
|
+
}));
|
|
129
|
+
const { records } = await fetchRedisHashRecords(this.client, ruleKeys, this.logger);
|
|
130
|
+
return sortByReaderRank(parseRedisRecords(records.filter((record) => Object.keys(record).length > 0), accessRuleInput, this.logger));
|
|
131
|
+
} catch (e) {
|
|
132
|
+
this.logger.error(() => ({
|
|
133
|
+
err: e,
|
|
134
|
+
data: { inspect: util.inspect({ filter }, { depth: null }) },
|
|
135
|
+
msg: "failed to execute split rule query set"
|
|
136
|
+
}));
|
|
137
|
+
return [];
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
async findRulesRanked(filter, query) {
|
|
141
|
+
try {
|
|
142
|
+
const reply = await this.client.ft.aggregate(ACCESS_RULES_REDIS_INDEX_NAME, query, {
|
|
143
|
+
DIALECT: 2,
|
|
144
|
+
LOAD: [...RULE_LOAD_FIELDS],
|
|
145
|
+
STEPS: [
|
|
146
|
+
{
|
|
147
|
+
type: "FILTER",
|
|
148
|
+
expression: "exists(@type)"
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
type: "APPLY",
|
|
152
|
+
expression: SPECIFICITY_EXPR,
|
|
153
|
+
AS: "_spec"
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
type: "APPLY",
|
|
157
|
+
expression: SEVERITY_EXPR,
|
|
158
|
+
AS: "_sev"
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
type: "APPLY",
|
|
162
|
+
expression: RANK_EXPR,
|
|
163
|
+
AS: "_rank"
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
type: "SORTBY",
|
|
167
|
+
BY: [{
|
|
168
|
+
BY: "@_rank",
|
|
169
|
+
DIRECTION: "DESC"
|
|
170
|
+
}]
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
type: "LIMIT",
|
|
174
|
+
from: 0,
|
|
175
|
+
size: 20
|
|
176
|
+
}
|
|
177
|
+
]
|
|
178
|
+
});
|
|
179
|
+
if (reply.results.length === 0) return [];
|
|
180
|
+
this.logger.debug(() => ({
|
|
181
|
+
msg: "Executed ranked search query",
|
|
182
|
+
data: { inspect: util.inspect({
|
|
183
|
+
filter,
|
|
184
|
+
foundCount: reply.results.length,
|
|
185
|
+
query
|
|
186
|
+
}, { depth: null }) }
|
|
187
|
+
}));
|
|
188
|
+
const ruleKeys = [];
|
|
189
|
+
for (const result of reply.results) {
|
|
190
|
+
const key = result.__key;
|
|
191
|
+
if (typeof key === "string") ruleKeys.push(key);
|
|
192
|
+
}
|
|
193
|
+
if (ruleKeys.length === 0) return [];
|
|
194
|
+
const { records } = await fetchRedisHashRecords(this.client, ruleKeys, this.logger);
|
|
195
|
+
return parseRedisRecords(records.filter((record) => Object.keys(record).length > 0), accessRuleInput, this.logger);
|
|
196
|
+
} catch (e) {
|
|
197
|
+
this.logger.error(() => ({
|
|
198
|
+
err: e,
|
|
199
|
+
data: { inspect: util.inspect({
|
|
200
|
+
query,
|
|
201
|
+
filter
|
|
202
|
+
}, { depth: null }) },
|
|
203
|
+
msg: "failed to execute ranked search query"
|
|
204
|
+
}));
|
|
205
|
+
return [];
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
async findRulesGreedy(filter, query) {
|
|
209
|
+
try {
|
|
210
|
+
const ruleKeys = await aggregateRedisKeys(this.client, query, this.logger, void 0, GREEDY_MAX_CANDIDATES);
|
|
211
|
+
if (ruleKeys.length === 0) return [];
|
|
212
|
+
this.logger.debug(() => ({
|
|
213
|
+
msg: "Executed greedy search query",
|
|
214
|
+
data: { inspect: util.inspect({
|
|
215
|
+
filter,
|
|
216
|
+
foundCount: ruleKeys.length,
|
|
217
|
+
query
|
|
218
|
+
}, { depth: null }) }
|
|
219
|
+
}));
|
|
220
|
+
const { records } = await fetchRedisHashRecords(this.client, ruleKeys, this.logger);
|
|
221
|
+
return parseRedisRecords(records.filter((record) => Object.keys(record).length > 0), accessRuleInput, this.logger);
|
|
222
|
+
} catch (e) {
|
|
223
|
+
this.logger.error(() => ({
|
|
224
|
+
err: e,
|
|
225
|
+
data: { inspect: util.inspect({
|
|
226
|
+
query,
|
|
227
|
+
filter
|
|
228
|
+
}, { depth: null }) },
|
|
229
|
+
msg: "failed to execute greedy search query"
|
|
230
|
+
}));
|
|
231
|
+
return [];
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
async findRuleIds(filter, matchingFieldsOnly = false) {
|
|
235
|
+
const query = getRulesRedisQuery(filter, matchingFieldsOnly);
|
|
236
|
+
let ruleIds = [];
|
|
237
|
+
try {
|
|
238
|
+
ruleIds = (await aggregateRedisKeys(this.client, query, this.logger)).map((ruleKey) => ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length));
|
|
239
|
+
} catch (e) {
|
|
240
|
+
this.logger.error(() => ({
|
|
241
|
+
err: e,
|
|
242
|
+
data: { inspect: util.inspect({
|
|
243
|
+
query,
|
|
244
|
+
filter
|
|
245
|
+
}, { depth: null }) },
|
|
246
|
+
msg: "Failed to execute search query for rule IDs"
|
|
247
|
+
}));
|
|
248
|
+
return [];
|
|
249
|
+
}
|
|
250
|
+
this.logger.debug(() => ({
|
|
251
|
+
msg: "Executed search query for rule IDs",
|
|
252
|
+
data: {
|
|
253
|
+
query,
|
|
254
|
+
foundCount: ruleIds.length,
|
|
255
|
+
foundIds: ruleIds
|
|
256
|
+
}
|
|
257
|
+
}));
|
|
258
|
+
return ruleIds;
|
|
259
|
+
}
|
|
260
|
+
async fetchAllRuleIds(batchHandler) {
|
|
261
|
+
const keysBatchHandler = async (keys) => {
|
|
262
|
+
await batchHandler(keys.map((ruleKey) => ruleKey.slice(ACCESS_RULE_REDIS_KEY_PREFIX.length)));
|
|
263
|
+
};
|
|
264
|
+
await aggregateRedisKeys(this.client, "*", this.logger, keysBatchHandler);
|
|
265
|
+
}
|
|
266
|
+
async fetchRuleEntries(keys) {
|
|
267
|
+
const { records, expirations } = await fetchRedisHashRecords(this.client, keys, this.logger);
|
|
268
|
+
const entries = [];
|
|
269
|
+
for (const [index, ruleData] of records.entries()) if (Object.keys(ruleData).length > 0) {
|
|
270
|
+
const rule = parseRedisRecords([ruleData], accessRuleInput, this.logger)[0];
|
|
271
|
+
if (rule) entries.push({
|
|
272
|
+
rule,
|
|
273
|
+
expiresUnixTimestamp: expirations[index]
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
return entries;
|
|
277
|
+
}
|
|
278
|
+
getRuleKeys(ruleIds) {
|
|
279
|
+
return ruleIds.map((id) => `${ACCESS_RULE_REDIS_KEY_PREFIX}${id}`);
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
var DummyRedisRulesReader = class {
|
|
283
|
+
constructor(logger) {
|
|
284
|
+
this.logger = logger;
|
|
285
|
+
}
|
|
286
|
+
async getMissingRuleIds(ruleIds) {
|
|
287
|
+
this.logger.info(() => ({
|
|
288
|
+
msg: "Dummy getMissingRuleIds() has no effect (redis is not ready)",
|
|
289
|
+
data: { ruleIds }
|
|
290
|
+
}));
|
|
291
|
+
return [];
|
|
292
|
+
}
|
|
293
|
+
async fetchRules(ruleIds) {
|
|
294
|
+
this.logger.info(() => ({
|
|
295
|
+
msg: "Dummy fetchRule() has no effect (redis is not ready)",
|
|
296
|
+
data: { ruleIds }
|
|
297
|
+
}));
|
|
298
|
+
return [];
|
|
299
|
+
}
|
|
300
|
+
async findRules(filter, matchingFieldsOnly = false, skipEmptyUserScopes = true) {
|
|
301
|
+
this.logger.info(() => ({
|
|
302
|
+
msg: "Dummy findRules() has no effect (redis is not ready)",
|
|
303
|
+
data: { filter }
|
|
304
|
+
}));
|
|
305
|
+
return [];
|
|
306
|
+
}
|
|
307
|
+
async findRuleIds(filter, matchingFieldsOnly = false) {
|
|
308
|
+
this.logger.info(() => ({
|
|
309
|
+
msg: "Dummy findRuleIds() has no effect (redis is not ready)",
|
|
310
|
+
data: { filter }
|
|
311
|
+
}));
|
|
312
|
+
return [];
|
|
313
|
+
}
|
|
314
|
+
async fetchAllRuleIds(batchHandler) {
|
|
315
|
+
this.logger.info(() => ({ msg: "Dummy fetchAllRuleIds() has no effect (redis is not ready)" }));
|
|
316
|
+
}
|
|
494
317
|
};
|
|
318
|
+
//#endregion
|
|
319
|
+
export { DummyRedisRulesReader, RedisRulesReader, SERVER_SIDE_RANK_TOP_N };
|