@prosopo/user-access-policy 3.12.8 → 3.12.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build$colon$cjs.log +45 -41
- package/.turbo/turbo-build$colon$tsc.log +14 -14
- package/.turbo/turbo-build.log +46 -42
- package/CHANGELOG.md +18 -0
- package/dist/.export.js +7 -21
- package/dist/_virtual/_rolldown/runtime.js +3 -0
- package/dist/api/.export.js +2 -9
- package/dist/api/accessRulesApiClient.js +1 -1
- package/dist/api/delete/.export.js +2 -1
- package/dist/api/delete/deleteAllRules.js +22 -24
- package/dist/api/delete/deleteRuleGroups.js +39 -51
- package/dist/api/delete/deleteRules.js +37 -41
- package/dist/api/read/.export.js +2 -1
- package/dist/api/read/fetchRules.js +34 -42
- package/dist/api/read/findRuleIds.js +35 -48
- package/dist/api/read/getMissingIds.js +36 -40
- package/dist/api/ruleApiRoutes.js +62 -120
- package/dist/api/rulesApiClient.js +51 -87
- package/dist/api/write/.export.js +2 -1
- package/dist/api/write/insertRules.js +84 -102
- package/dist/api/write/rehashRules.js +48 -56
- package/dist/cjs/.export.cjs +21 -23
- package/dist/cjs/_virtual/_rolldown/runtime.cjs +23 -0
- package/dist/cjs/api/.export.cjs +6 -10
- package/dist/cjs/api/accessRulesApiClient.cjs +0 -0
- package/dist/cjs/api/delete/.export.cjs +0 -1
- package/dist/cjs/api/delete/deleteAllRules.cjs +22 -24
- package/dist/cjs/api/delete/deleteRuleGroups.cjs +39 -51
- package/dist/cjs/api/delete/deleteRules.cjs +41 -42
- package/dist/cjs/api/read/.export.cjs +0 -1
- package/dist/cjs/api/read/fetchRules.cjs +34 -41
- package/dist/cjs/api/read/findRuleIds.cjs +37 -48
- package/dist/cjs/api/read/getMissingIds.cjs +36 -39
- package/dist/cjs/api/ruleApiRoutes.cjs +69 -125
- package/dist/cjs/api/rulesApiClient.cjs +54 -90
- package/dist/cjs/api/write/.export.cjs +0 -1
- package/dist/cjs/api/write/insertRules.cjs +85 -103
- package/dist/cjs/api/write/rehashRules.cjs +48 -56
- package/dist/cjs/classifyOs.cjs +46 -26
- package/dist/cjs/mongoose/.export.cjs +2 -3
- package/dist/cjs/mongoose/mongooseRuleSchema.cjs +97 -36
- package/dist/cjs/redis/.export.cjs +4 -5
- package/dist/cjs/redis/reader/redisAggregate.cjs +59 -84
- package/dist/cjs/redis/reader/redisRulesQuery.cjs +74 -116
- package/dist/cjs/redis/reader/redisRulesReader.cjs +322 -503
- package/dist/cjs/redis/reader/redisRulesSplitQuery.cjs +94 -72
- package/dist/cjs/redis/redisClient.cjs +53 -58
- package/dist/cjs/redis/redisRuleIndex.cjs +93 -49
- package/dist/cjs/redis/redisRulesStorage.cjs +21 -30
- package/dist/cjs/redis/redisRulesWriter.cjs +86 -112
- package/dist/cjs/rule.cjs +8 -7
- package/dist/cjs/ruleInput/.export.cjs +7 -8
- package/dist/cjs/ruleInput/policyInput.cjs +35 -58
- package/dist/cjs/ruleInput/ruleInput.cjs +37 -42
- package/dist/cjs/ruleInput/userScopeInput.cjs +47 -55
- package/dist/cjs/ruleRecord.cjs +15 -22
- package/dist/cjs/rulesStorage.cjs +7 -6
- package/dist/cjs/transformRule.cjs +36 -70
- package/dist/classifyOs.js +48 -29
- package/dist/mongoose/.export.js +2 -3
- package/dist/mongoose/mongooseRuleSchema.js +99 -38
- package/dist/redis/.export.js +3 -5
- package/dist/redis/reader/redisAggregate.js +60 -86
- package/dist/redis/reader/redisRulesQuery.js +75 -118
- package/dist/redis/reader/redisRulesReader.js +312 -487
- package/dist/redis/reader/redisRulesSplitQuery.js +96 -75
- package/dist/redis/redisClient.js +54 -62
- package/dist/redis/redisRuleIndex.js +93 -58
- package/dist/redis/redisRulesStorage.js +21 -30
- package/dist/redis/redisRulesWriter.js +82 -113
- package/dist/rule.js +10 -10
- package/dist/ruleInput/.export.js +3 -7
- package/dist/ruleInput/policyInput.js +35 -61
- package/dist/ruleInput/ruleInput.js +36 -44
- package/dist/ruleInput/userScopeInput.js +44 -55
- package/dist/ruleRecord.js +17 -27
- package/dist/rulesStorage.js +9 -8
- package/dist/tests/redis/redisRulesStorage.integration.test.js +1 -3
- package/dist/tests/redis/redisRulesStorage.integration.test.js.map +1 -1
- package/dist/transformRule.js +34 -73
- package/entries.ts +25 -6
- package/package.json +13 -12
- package/src/tests/redis/redisRulesStorage.integration.test.ts +191 -195
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,77 +1,43 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
3
|
-
const
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
return hashObject(objectToHash, RULE_HASH_ALGORITHM);
|
|
2
|
+
const require_runtime = require("./_virtual/_rolldown/runtime.cjs");
|
|
3
|
+
const require_ruleInput_policyInput = require("./ruleInput/policyInput.cjs");
|
|
4
|
+
const require_ruleInput_userScopeInput = require("./ruleInput/userScopeInput.cjs");
|
|
5
|
+
const require_ruleInput_ruleInput = require("./ruleInput/ruleInput.cjs");
|
|
6
|
+
let zod = require("zod");
|
|
7
|
+
let node_crypto = require("node:crypto");
|
|
8
|
+
node_crypto = require_runtime.__toESM(node_crypto, 1);
|
|
9
|
+
let ip_address = require("ip-address");
|
|
10
|
+
let cidr_calc = require("cidr-calc");
|
|
11
|
+
//#region src/transformRule.ts
|
|
12
|
+
var RULE_HASH_ALGORITHM = "md5";
|
|
13
|
+
var MAX_IPV4_NUMERIC = (1n << 32n) - 1n;
|
|
14
|
+
var makeAccessRuleHash = (rule) => {
|
|
15
|
+
const orderedProperties = Object.entries(rule).filter(([key, value]) => "undefined" !== typeof value).sort();
|
|
16
|
+
return hashObject(Object.fromEntries(orderedProperties), RULE_HASH_ALGORITHM);
|
|
19
17
|
};
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
numericIpMaskMax,
|
|
36
|
-
userAgentHash,
|
|
37
|
-
...rule
|
|
38
|
-
} = ruleInput2;
|
|
39
|
-
const record = rule;
|
|
40
|
-
if ("string" === typeof groupId) {
|
|
41
|
-
record.ruleGroupId = groupId;
|
|
42
|
-
}
|
|
43
|
-
if ("string" === typeof userAgentHash) {
|
|
44
|
-
record.userAgent = userAgentHash;
|
|
45
|
-
}
|
|
46
|
-
if ("bigint" === typeof numericIp) {
|
|
47
|
-
record.ip = getStringIpFromNumeric(numericIp);
|
|
48
|
-
}
|
|
49
|
-
if ("bigint" === typeof numericIpMaskMin && "bigint" === typeof numericIpMaskMax) {
|
|
50
|
-
record.ipMask = getCidrFromNumericIpRange(
|
|
51
|
-
numericIpMaskMin,
|
|
52
|
-
numericIpMaskMax
|
|
53
|
-
);
|
|
54
|
-
}
|
|
55
|
-
return record;
|
|
18
|
+
var transformAccessRuleRecordIntoRule = (ruleRecord) => require_ruleInput_ruleInput.accessRuleInput.parse(ruleRecord);
|
|
19
|
+
var transformAccessRuleIntoRecord = (rule) => accessRuleToRecordScheme.parse(rule);
|
|
20
|
+
var accessRuleToRecordScheme = zod.z.object({
|
|
21
|
+
...require_ruleInput_policyInput.accessPolicyInputShape.shape,
|
|
22
|
+
...require_ruleInput_policyInput.policyScopeInput.shape,
|
|
23
|
+
...require_ruleInput_userScopeInput.userScopeSchema.shape,
|
|
24
|
+
groupId: zod.z.coerce.string().optional()
|
|
25
|
+
}).transform((ruleInput) => {
|
|
26
|
+
const { groupId, numericIp, numericIpMaskMin, numericIpMaskMax, userAgentHash, ...rule } = ruleInput;
|
|
27
|
+
const record = rule;
|
|
28
|
+
if ("string" === typeof groupId) record.ruleGroupId = groupId;
|
|
29
|
+
if ("string" === typeof userAgentHash) record.userAgent = userAgentHash;
|
|
30
|
+
if ("bigint" === typeof numericIp) record.ip = getStringIpFromNumeric(numericIp);
|
|
31
|
+
if ("bigint" === typeof numericIpMaskMin && "bigint" === typeof numericIpMaskMax) record.ipMask = getCidrFromNumericIpRange(numericIpMaskMin, numericIpMaskMax);
|
|
32
|
+
return record;
|
|
56
33
|
});
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
"bigint" === typeof value ? value.toString() : value
|
|
63
|
-
)
|
|
64
|
-
)
|
|
65
|
-
).digest("hex");
|
|
66
|
-
const getStringIpFromNumeric = (numericIp) => numericIp > MAX_IPV4_NUMERIC ? ipAddress.Address6.fromBigInt(numericIp).correctForm() : ipAddress.Address4.fromInteger(Number(numericIp)).address;
|
|
67
|
-
const getCidrFromNumericIpRange = (startIp, endIp) => {
|
|
68
|
-
const ipRange = new cidrCalc.IpRange(
|
|
69
|
-
cidrCalc.IpAddress.of(getStringIpFromNumeric(startIp)),
|
|
70
|
-
cidrCalc.IpAddress.of(getStringIpFromNumeric(endIp))
|
|
71
|
-
);
|
|
72
|
-
const cidr = ipRange.toCidrs()[0];
|
|
73
|
-
return cidr ? `${cidr.prefix.toString()}/${cidr.prefixLen}` : void 0;
|
|
34
|
+
var hashObject = (object, algorithm) => node_crypto.default.createHash(algorithm).update(JSON.stringify(object, (key, value) => "bigint" === typeof value ? value.toString() : value)).digest("hex");
|
|
35
|
+
var getStringIpFromNumeric = (numericIp) => numericIp > MAX_IPV4_NUMERIC ? ip_address.Address6.fromBigInt(numericIp).correctForm() : ip_address.Address4.fromInteger(Number(numericIp)).address;
|
|
36
|
+
var getCidrFromNumericIpRange = (startIp, endIp) => {
|
|
37
|
+
const cidr = new cidr_calc.IpRange(cidr_calc.IpAddress.of(getStringIpFromNumeric(startIp)), cidr_calc.IpAddress.of(getStringIpFromNumeric(endIp))).toCidrs()[0];
|
|
38
|
+
return cidr ? `${cidr.prefix.toString()}/${cidr.prefixLen}` : void 0;
|
|
74
39
|
};
|
|
40
|
+
//#endregion
|
|
75
41
|
exports.getCidrFromNumericIpRange = getCidrFromNumericIpRange;
|
|
76
42
|
exports.makeAccessRuleHash = makeAccessRuleHash;
|
|
77
43
|
exports.transformAccessRuleIntoRecord = transformAccessRuleIntoRecord;
|
package/dist/classifyOs.js
CHANGED
|
@@ -1,31 +1,50 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
import "./_virtual/_rolldown/runtime.js";
|
|
2
|
+
//#region src/classifyOs.ts
|
|
3
|
+
/**
|
|
4
|
+
* The operating systems the classifier can resolve a request to. Kept
|
|
5
|
+
* deliberately in step with the OS vocabulary in
|
|
6
|
+
* `@prosopo/decision-machines`' `uaClassify` (so a routing-rule `osNameIn` and
|
|
7
|
+
* an access-rule `os` agree on the same names), but defined here because the
|
|
8
|
+
* provider / user-access-policy request path cannot depend on the
|
|
9
|
+
* decision-machines package.
|
|
10
|
+
*
|
|
11
|
+
* `unknown` is the catch-all when no signature matches. It is a first-class
|
|
12
|
+
* value so an allow-list ("only let Windows through") can block unrecognised
|
|
13
|
+
* User-Agents instead of silently passing them.
|
|
14
|
+
*/
|
|
15
|
+
var OS_NAMES = [
|
|
16
|
+
"windows",
|
|
17
|
+
"macos",
|
|
18
|
+
"ios",
|
|
19
|
+
"android",
|
|
20
|
+
"linux",
|
|
21
|
+
"unknown"
|
|
8
22
|
];
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Classify the operating system from a raw User-Agent string. Pure — no
|
|
25
|
+
* navigator/window/DOM dependence — so it is safe to call on the provider's
|
|
26
|
+
* server-side request path.
|
|
27
|
+
*
|
|
28
|
+
* Detection order matches `uaClassify.classifyUserAgent`: iOS (iphone/ipad/
|
|
29
|
+
* ipod) is tested before the generic mac signature, and Android before Linux,
|
|
30
|
+
* because those User-Agents contain the more general token too. Detection is
|
|
31
|
+
* conservative — anything unrecognised returns `unknown` rather than guessing,
|
|
32
|
+
* so an `os === "ios"` rule never fires on a UA we don't understand.
|
|
33
|
+
*
|
|
34
|
+
* The classification is driven off the full User-Agent rather than the
|
|
35
|
+
* `sec-ch-ua-platform` client hint on purpose: a client can simply omit client
|
|
36
|
+
* hints, but stripping the User-Agent breaks far more, so the UA is the more
|
|
37
|
+
* reliable signal for an OS-restriction rule that is meant to be hard to
|
|
38
|
+
* bypass.
|
|
39
|
+
*/
|
|
40
|
+
var classifyOs = (userAgent) => {
|
|
41
|
+
const ua = (userAgent || "").toLowerCase();
|
|
42
|
+
if (/iphone|ipad|ipod/.test(ua)) return "ios";
|
|
43
|
+
if (/android/.test(ua)) return "android";
|
|
44
|
+
if (/windows/.test(ua)) return "windows";
|
|
45
|
+
if (/macintosh|mac os/.test(ua)) return "macos";
|
|
46
|
+
if (/linux/.test(ua)) return "linux";
|
|
47
|
+
return "unknown";
|
|
31
48
|
};
|
|
49
|
+
//#endregion
|
|
50
|
+
export { OS_NAMES, classifyOs };
|
package/dist/mongoose/.export.js
CHANGED
|
@@ -1,42 +1,103 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
1
|
+
import "../_virtual/_rolldown/runtime.js";
|
|
2
|
+
//#region src/mongoose/mongooseRuleSchema.ts
|
|
3
|
+
var userAttributesSchema = {
|
|
4
|
+
userId: {
|
|
5
|
+
type: String,
|
|
6
|
+
required: false
|
|
7
|
+
},
|
|
8
|
+
ja4Hash: {
|
|
9
|
+
type: String,
|
|
10
|
+
required: false
|
|
11
|
+
},
|
|
12
|
+
userAgent: {
|
|
13
|
+
type: String,
|
|
14
|
+
required: false
|
|
15
|
+
},
|
|
16
|
+
headersHash: {
|
|
17
|
+
type: String,
|
|
18
|
+
required: false
|
|
19
|
+
},
|
|
20
|
+
headHash: {
|
|
21
|
+
type: String,
|
|
22
|
+
required: false
|
|
23
|
+
},
|
|
24
|
+
coords: {
|
|
25
|
+
type: String,
|
|
26
|
+
required: false
|
|
27
|
+
},
|
|
28
|
+
countryCode: {
|
|
29
|
+
type: String,
|
|
30
|
+
required: false
|
|
31
|
+
},
|
|
32
|
+
asn: {
|
|
33
|
+
type: Number,
|
|
34
|
+
required: false
|
|
35
|
+
},
|
|
36
|
+
os: {
|
|
37
|
+
type: String,
|
|
38
|
+
required: false
|
|
39
|
+
}
|
|
11
40
|
};
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
41
|
+
var userIpSchema = {
|
|
42
|
+
ip: {
|
|
43
|
+
type: String,
|
|
44
|
+
required: false
|
|
45
|
+
},
|
|
46
|
+
ipMask: {
|
|
47
|
+
type: String,
|
|
48
|
+
required: false
|
|
49
|
+
}
|
|
15
50
|
};
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
51
|
+
var userScopeSchema = {
|
|
52
|
+
...userAttributesSchema,
|
|
53
|
+
...userIpSchema
|
|
19
54
|
};
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
55
|
+
var accessRuleMongooseSchema = {
|
|
56
|
+
type: {
|
|
57
|
+
type: String,
|
|
58
|
+
required: true
|
|
59
|
+
},
|
|
60
|
+
captchaType: {
|
|
61
|
+
type: String,
|
|
62
|
+
required: false
|
|
63
|
+
},
|
|
64
|
+
description: {
|
|
65
|
+
type: String,
|
|
66
|
+
required: false
|
|
67
|
+
},
|
|
68
|
+
solvedImagesCount: {
|
|
69
|
+
type: Number,
|
|
70
|
+
required: false
|
|
71
|
+
},
|
|
72
|
+
imageThreshold: {
|
|
73
|
+
type: Number,
|
|
74
|
+
required: false
|
|
75
|
+
},
|
|
76
|
+
powDifficulty: {
|
|
77
|
+
type: Number,
|
|
78
|
+
required: false
|
|
79
|
+
},
|
|
80
|
+
unsolvedImagesCount: {
|
|
81
|
+
type: Number,
|
|
82
|
+
required: false
|
|
83
|
+
},
|
|
84
|
+
frictionlessScore: {
|
|
85
|
+
type: Number,
|
|
86
|
+
required: false
|
|
87
|
+
},
|
|
88
|
+
deferToVerify: {
|
|
89
|
+
type: Boolean,
|
|
90
|
+
required: false
|
|
91
|
+
},
|
|
92
|
+
clientId: {
|
|
93
|
+
type: String,
|
|
94
|
+
required: false
|
|
95
|
+
},
|
|
96
|
+
...userScopeSchema,
|
|
97
|
+
ruleGroupId: {
|
|
98
|
+
type: String,
|
|
99
|
+
required: false
|
|
100
|
+
}
|
|
42
101
|
};
|
|
102
|
+
//#endregion
|
|
103
|
+
export { accessRuleMongooseSchema };
|
package/dist/redis/.export.js
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import "../_virtual/_rolldown/runtime.js";
|
|
2
2
|
import { accessRulesRedisIndex } from "./redisRuleIndex.js";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
createRedisAccessRulesStorage
|
|
6
|
-
};
|
|
3
|
+
import { createRedisAccessRulesStorage } from "./redisRulesStorage.js";
|
|
4
|
+
export { accessRulesRedisIndex, createRedisAccessRulesStorage };
|
|
@@ -1,90 +1,64 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
1
|
+
import "../../_virtual/_rolldown/runtime.js";
|
|
2
|
+
import "./redisRulesQuery.js";
|
|
3
|
+
import { REDIS_BATCH_SIZE, parseRedisRecords } from "../redisClient.js";
|
|
4
4
|
import { ACCESS_RULES_REDIS_INDEX_NAME } from "../redisRuleIndex.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
};
|
|
16
|
-
const aggregateRedisKeys = async (client, query, logger, batchHandler, maxKeys) => {
|
|
17
|
-
const keyField = "__key";
|
|
18
|
-
const recordSchema = z.object({
|
|
19
|
-
// it's a reserved name for the record key
|
|
20
|
-
[keyField]: z.string()
|
|
21
|
-
});
|
|
22
|
-
const foundKeys = [];
|
|
23
|
-
let stopRequested = false;
|
|
24
|
-
const addRecordKeys = async (records) => {
|
|
25
|
-
if (stopRequested) {
|
|
26
|
-
return;
|
|
27
|
-
}
|
|
28
|
-
const parsedRecords = parseRedisRecords(records, recordSchema, logger);
|
|
29
|
-
const recordKeys = parsedRecords.map((record) => record[keyField]);
|
|
30
|
-
if (batchHandler) {
|
|
31
|
-
await batchHandler(recordKeys);
|
|
32
|
-
} else {
|
|
33
|
-
if (maxKeys !== void 0 && foundKeys.length + recordKeys.length > maxKeys) {
|
|
34
|
-
const remaining = Math.max(0, maxKeys - foundKeys.length);
|
|
35
|
-
foundKeys.push(...recordKeys.slice(0, remaining));
|
|
36
|
-
stopRequested = true;
|
|
37
|
-
logger.warn(() => ({
|
|
38
|
-
msg: "Redis aggregation candidate cap hit; truncating result set. This can suppress less-frequent rules and should be investigated.",
|
|
39
|
-
data: {
|
|
40
|
-
maxKeys,
|
|
41
|
-
query
|
|
42
|
-
}
|
|
43
|
-
}));
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
foundKeys.push(...recordKeys);
|
|
47
|
-
logger.debug(() => ({
|
|
48
|
-
msg: "Processed aggregation batch",
|
|
49
|
-
data: {
|
|
50
|
-
size: recordKeys.length
|
|
51
|
-
}
|
|
52
|
-
}));
|
|
53
|
-
}
|
|
54
|
-
};
|
|
55
|
-
await executeAggregation(
|
|
56
|
-
client,
|
|
57
|
-
query,
|
|
58
|
-
{
|
|
59
|
-
// #2 is a required option when the 'ismissing()' function is in the query body
|
|
60
|
-
DIALECT: REDIS_QUERY_DIALECT,
|
|
61
|
-
COUNT: REDIS_BATCH_SIZE,
|
|
62
|
-
LOAD: `@${keyField}`
|
|
63
|
-
},
|
|
64
|
-
addRecordKeys,
|
|
65
|
-
() => stopRequested
|
|
66
|
-
);
|
|
67
|
-
return foundKeys;
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
//#region src/redis/reader/redisAggregate.ts
|
|
7
|
+
var searchRedisKeysBounded = async (client, query, maxKeys) => {
|
|
8
|
+
return (await client.ft.searchNoContent(ACCESS_RULES_REDIS_INDEX_NAME, query, {
|
|
9
|
+
DIALECT: 2,
|
|
10
|
+
LIMIT: {
|
|
11
|
+
from: 0,
|
|
12
|
+
size: maxKeys
|
|
13
|
+
}
|
|
14
|
+
})).documents;
|
|
68
15
|
};
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
16
|
+
var aggregateRedisKeys = async (client, query, logger, batchHandler, maxKeys) => {
|
|
17
|
+
const keyField = "__key";
|
|
18
|
+
const recordSchema = z.object({ [keyField]: z.string() });
|
|
19
|
+
const foundKeys = [];
|
|
20
|
+
let stopRequested = false;
|
|
21
|
+
const addRecordKeys = async (records) => {
|
|
22
|
+
if (stopRequested) return;
|
|
23
|
+
const recordKeys = parseRedisRecords(records, recordSchema, logger).map((record) => record[keyField]);
|
|
24
|
+
if (batchHandler) await batchHandler(recordKeys);
|
|
25
|
+
else {
|
|
26
|
+
if (maxKeys !== void 0 && foundKeys.length + recordKeys.length > maxKeys) {
|
|
27
|
+
const remaining = Math.max(0, maxKeys - foundKeys.length);
|
|
28
|
+
foundKeys.push(...recordKeys.slice(0, remaining));
|
|
29
|
+
stopRequested = true;
|
|
30
|
+
logger.warn(() => ({
|
|
31
|
+
msg: "Redis aggregation candidate cap hit; truncating result set. This can suppress less-frequent rules and should be investigated.",
|
|
32
|
+
data: {
|
|
33
|
+
maxKeys,
|
|
34
|
+
query
|
|
35
|
+
}
|
|
36
|
+
}));
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
foundKeys.push(...recordKeys);
|
|
40
|
+
logger.debug(() => ({
|
|
41
|
+
msg: "Processed aggregation batch",
|
|
42
|
+
data: { size: recordKeys.length }
|
|
43
|
+
}));
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
await executeAggregation(client, query, {
|
|
47
|
+
DIALECT: 2,
|
|
48
|
+
COUNT: REDIS_BATCH_SIZE,
|
|
49
|
+
LOAD: `@${keyField}`
|
|
50
|
+
}, addRecordKeys, () => stopRequested);
|
|
51
|
+
return foundKeys;
|
|
86
52
|
};
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
53
|
+
var executeAggregation = async (client, query, aggregateOptions, handleBatch, shouldStop) => {
|
|
54
|
+
const initialReply = await client.ft.aggregateWithCursor(ACCESS_RULES_REDIS_INDEX_NAME, query, aggregateOptions);
|
|
55
|
+
await handleBatch(initialReply.results);
|
|
56
|
+
let cursor = initialReply.cursor;
|
|
57
|
+
while (0 !== cursor && !shouldStop?.()) {
|
|
58
|
+
const batchReply = await client.ft.cursorRead(ACCESS_RULES_REDIS_INDEX_NAME, cursor, { COUNT: aggregateOptions.COUNT });
|
|
59
|
+
await handleBatch(batchReply.results);
|
|
60
|
+
cursor = batchReply.cursor;
|
|
61
|
+
}
|
|
90
62
|
};
|
|
63
|
+
//#endregion
|
|
64
|
+
export { aggregateRedisKeys, searchRedisKeysBounded };
|