@prosopo/user-access-policy 3.9.0 → 3.10.0

package/.turbo/turbo-build$colon$cjs.log

@@ -1,5 +1,5 @@
 
-> @prosopo/user-access-policy@3.9.0 build:cjs
+> @prosopo/user-access-policy@3.10.0 build:cjs
 > NODE_ENV=${NODE_ENV:-development}; vite build --config vite.cjs.config.ts --mode $NODE_ENV
 
 ViteCommonJSConfig: .
@@ -35,37 +35,37 @@ transforming...
 Build end
 ✓ 33 modules transformed.
 rendering chunks...
-dist/cjs/api/delete/.export.cjs             0.01 kB
-dist/cjs/api/read/.export.cjs               0.01 kB
-dist/cjs/api/write/.export.cjs              0.01 kB
-dist/cjs/mongoose/.export.cjs               0.23 kB
-dist/cjs/rulesStorage.cjs                   0.34 kB
-dist/cjs/rule.cjs                           0.34 kB
-dist/cjs/redis/.export.cjs                  0.36 kB
-dist/cjs/ruleInput/.export.cjs              0.46 kB
-dist/cjs/api/.export.cjs                    0.57 kB
-dist/cjs/api/delete/deleteAllRules.cjs      0.77 kB
-dist/cjs/ruleRecord.cjs                     0.83 kB
-dist/cjs/api/read/getMissingIds.cjs         1.15 kB
-dist/cjs/api/read/fetchRules.cjs            1.22 kB
-dist/cjs/.export.cjs                        1.24 kB
-dist/cjs/redis/redisRulesStorage.cjs        1.35 kB
-dist/cjs/mongoose/mongooseRuleSchema.cjs    1.48 kB
-dist/cjs/api/delete/deleteRules.cjs         1.51 kB
-dist/cjs/api/delete/deleteRuleGroups.cjs    1.57 kB
-dist/cjs/api/write/rehashRules.cjs          1.64 kB
-dist/cjs/api/read/findRuleIds.cjs           1.65 kB
-dist/cjs/ruleInput/policyInput.cjs          1.70 kB
-dist/cjs/redis/reader/redisAggregate.cjs    1.95 kB
-dist/cjs/ruleInput/ruleInput.cjs            2.07 kB
-dist/cjs/redis/redisClient.cjs              2.09 kB
-dist/cjs/ruleInput/userScopeInput.cjs       2.45 kB
-dist/cjs/redis/redisRuleIndex.cjs           2.46 kB
-dist/cjs/api/rulesApiClient.cjs             2.60 kB
-dist/cjs/transformRule.cjs                  3.05 kB
-dist/cjs/api/write/insertRules.cjs          3.38 kB
-dist/cjs/redis/redisRulesWriter.cjs         3.44 kB
-dist/cjs/redis/reader/redisRulesQuery.cjs   4.34 kB
-dist/cjs/api/ruleApiRoutes.cjs              4.55 kB
-dist/cjs/redis/reader/redisRulesReader.cjs  7.01 kB
-✓ built in 451ms
+dist/cjs/api/delete/.export.cjs              0.01 kB
+dist/cjs/api/read/.export.cjs                0.01 kB
+dist/cjs/api/write/.export.cjs               0.01 kB
+dist/cjs/mongoose/.export.cjs                0.23 kB
+dist/cjs/rulesStorage.cjs                    0.34 kB
+dist/cjs/rule.cjs                            0.34 kB
+dist/cjs/redis/.export.cjs                   0.36 kB
+dist/cjs/ruleInput/.export.cjs               0.46 kB
+dist/cjs/api/.export.cjs                     0.57 kB
+dist/cjs/api/delete/deleteAllRules.cjs       0.77 kB
+dist/cjs/ruleRecord.cjs                      0.83 kB
+dist/cjs/api/read/getMissingIds.cjs          1.15 kB
+dist/cjs/api/read/fetchRules.cjs             1.22 kB
+dist/cjs/.export.cjs                         1.24 kB
+dist/cjs/redis/redisRulesStorage.cjs         1.35 kB
+dist/cjs/mongoose/mongooseRuleSchema.cjs     1.48 kB
+dist/cjs/api/delete/deleteRules.cjs          1.51 kB
+dist/cjs/api/delete/deleteRuleGroups.cjs     1.57 kB
+dist/cjs/api/write/rehashRules.cjs           1.64 kB
+dist/cjs/api/read/findRuleIds.cjs            1.65 kB
+dist/cjs/ruleInput/policyInput.cjs           1.70 kB
+dist/cjs/ruleInput/ruleInput.cjs             2.07 kB
+dist/cjs/redis/redisClient.cjs               2.09 kB
+dist/cjs/ruleInput/userScopeInput.cjs        2.45 kB
+dist/cjs/redis/redisRuleIndex.cjs            2.46 kB
+dist/cjs/api/rulesApiClient.cjs              2.60 kB
+dist/cjs/redis/reader/redisAggregate.cjs     2.61 kB
+dist/cjs/transformRule.cjs                   3.05 kB
+dist/cjs/api/write/insertRules.cjs           3.38 kB
+dist/cjs/redis/redisRulesWriter.cjs          3.44 kB
+dist/cjs/redis/reader/redisRulesQuery.cjs    4.34 kB
+dist/cjs/api/ruleApiRoutes.cjs               4.55 kB
+dist/cjs/redis/reader/redisRulesReader.cjs  10.28 kB
+✓ built in 465ms

package/.turbo/turbo-build$colon$tsc.log

@@ -1,8 +1,8 @@
 
-> @prosopo/user-access-policy@3.9.0 build:tsc
+> @prosopo/user-access-policy@3.10.0 build:tsc
 > tsc --build --verbose
 
-11:34:14 AM - Projects in this build: 
+2:03:13 PM - Projects in this build: 
     * ../../dev/config/tsconfig.json
     * ../locale/tsconfig.json
     * ../util/tsconfig.json
@@ -15,27 +15,27 @@
     * ../api/tsconfig.json
     * tsconfig.json
 
-11:34:14 AM - Project '../../dev/config/tsconfig.json' is up to date because newest input '../../dev/config/src/webpack/webpack.config.ts' is older than output '../../dev/config/tsconfig.tsbuildinfo'
+2:03:13 PM - Project '../../dev/config/tsconfig.json' is up to date because newest input '../../dev/config/src/webpack/webpack.config.ts' is older than output '../../dev/config/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project '../locale/tsconfig.json' is up to date because newest input '../locale/src/translationKey.ts' is older than output '../locale/tsconfig.tsbuildinfo'
+2:03:13 PM - Project '../locale/tsconfig.json' is up to date because newest input '../locale/src/util.ts' is older than output '../locale/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project '../util/tsconfig.json' is up to date because newest input '../util/src/url.ts' is older than output '../util/tsconfig.tsbuildinfo'
+2:03:13 PM - Project '../util/tsconfig.json' is up to date because newest input '../util/src/url.ts' is older than output '../util/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project '../logger/tsconfig.json' is up to date because newest input '../logger/src/index.ts' is older than output '../logger/tsconfig.tsbuildinfo'
+2:03:13 PM - Project '../logger/tsconfig.json' is up to date because newest input '../logger/src/index.ts' is older than output '../logger/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project '../util-crypto/tsconfig.json' is up to date because newest input '../util-crypto/src/types.ts' is older than output '../util-crypto/tsconfig.tsbuildinfo'
+2:03:13 PM - Project '../util-crypto/tsconfig.json' is up to date because newest input '../util-crypto/src/types.ts' is older than output '../util-crypto/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project '../types/tsconfig.json' is up to date because newest input '../types/src/procaptcha/api.ts' is older than output '../types/tsconfig.tsbuildinfo'
+2:03:14 PM - Project '../types/tsconfig.json' is up to date because newest input '../types/src/provider/api.ts' is older than output '../types/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project '../common/tsconfig.json' is up to date because newest input '../common/src/tests/utils/batches.unit.test.ts' is older than output '../common/tsconfig.tsbuildinfo'
+2:03:14 PM - Project '../common/tsconfig.json' is up to date because newest input '../common/src/index.ts' is older than output '../common/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project '../api-route/tsconfig.json' is up to date because newest input '../api-route/src/apiRoutes.ts' is older than output '../api-route/tsconfig.tsbuildinfo'
+2:03:14 PM - Project '../api-route/tsconfig.json' is up to date because newest input '../api-route/src/apiRoutes.ts' is older than output '../api-route/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project '../redis-client/tsconfig.json' is up to date because newest input '../redis-client/src/tests/testRedisConnection.ts' is older than output '../redis-client/tsconfig.tsbuildinfo'
+2:03:14 PM - Project '../redis-client/tsconfig.json' is up to date because newest input '../redis-client/src/index.ts' is older than output '../redis-client/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project '../api/tsconfig.json' is up to date because newest input '../api/src/index.ts' is older than output '../api/tsconfig.tsbuildinfo'
+2:03:14 PM - Project '../api/tsconfig.json' is up to date because newest input '../api/src/index.ts' is older than output '../api/tsconfig.tsbuildinfo'
 
-11:34:14 AM - Project 'tsconfig.json' is out of date because output file 'tsconfig.tsbuildinfo' does not exist
+2:03:14 PM - Project 'tsconfig.json' is out of date because output file 'tsconfig.tsbuildinfo' does not exist
 
-11:34:14 AM - Building project '/home/runner/work/captcha/captcha/packages/user-access-policy/tsconfig.json'...
+2:03:14 PM - Building project '/home/runner/work/captcha/captcha/packages/user-access-policy/tsconfig.json'...
 

package/.turbo/turbo-build.log

@@ -1,9 +1,9 @@
 
-> @prosopo/user-access-policy@3.9.0 build
+> @prosopo/user-access-policy@3.10.0 build
 > npm run build:cross-env -- --mode ${NODE_ENV:-development}
 
 
-> @prosopo/user-access-policy@3.9.0 build:cross-env
+> @prosopo/user-access-policy@3.10.0 build:cross-env
 > vite build --config vite.esm.config.ts --mode production
 
 ViteEsmConfig: .
@@ -61,15 +61,15 @@ rendering chunks...
 dist/api/read/findRuleIds.js           1.54 kB
 dist/api/write/rehashRules.js          1.54 kB
 dist/ruleInput/ruleInput.js            1.72 kB
-dist/redis/reader/redisAggregate.js    1.80 kB
 dist/redis/redisClient.js              1.90 kB
 dist/redis/redisRuleIndex.js           2.01 kB
 dist/ruleInput/userScopeInput.js       2.22 kB
 dist/api/rulesApiClient.js             2.36 kB
+dist/redis/reader/redisAggregate.js    2.46 kB
 dist/transformRule.js                  2.76 kB
 dist/api/write/insertRules.js          3.22 kB
 dist/redis/redisRulesWriter.js         3.22 kB
 dist/redis/reader/redisRulesQuery.js   4.15 kB
 dist/api/ruleApiRoutes.js              4.33 kB
-dist/redis/reader/redisRulesReader.js  6.23 kB
+dist/redis/reader/redisRulesReader.js  9.40 kB
 ✓ built in 499ms

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @prosopo/user-access-policy
 
+## 3.10.0
+### Minor Changes
+
+- c1c7998: Server-side specificity rank for the access-rule lookup. Strict-match callers (the `blockMiddleware` and the verify-time `checkForHardBlock`) now issue one `FT.AGGREGATE` with `APPLY exists()` for specificity, `APPLY @type == "block"` for the severity tiebreak, `SORTBY @_rank DESC`, and `LIMIT 0 20`. Node receives at most 20 fully-populated rules — no follow-up HGETALL per candidate, no JS-side rank, no silent truncation past the LIMIT (which only applies after Redis has scored every candidate the strict filter returned).
+  
+  Supersedes both the v3.6.38 regression (`b520cd94c` — FT.AGGREGATE WITHCURSOR materialising ~1190 hashes per request, pegged provider1 at ~125% CPU on pronode10) and the 3.6.38-hotfix1 shape that reverted to FT.SEARCH (re-opened the 1000-candidate silent-truncation bug). The greedy/admin path (`matchingFieldsOnly=false`) keeps the FT.AGGREGATE+CURSOR approach with a generous `GREEDY_MAX_CANDIDATES` cap since those callers do not run on the per-request hot path.
+  
+  `packages/provider/src/api/blacklistRequestInspector.ts` flips `getPrioritisedAccessRule` to `matchingFieldsOnly: true` to engage the new path. The defensive JS `rankCandidateRules` is kept so any drift between the Redis-side score and the JS semantics surfaces as ordering, not as letting traffic through. New benchmark integration test seeds 10k rules across a realistic specificity distribution and asserts p50 < 80ms / p99 < 250ms over 200 lookups; local measurement is steady at p50 ≈ 20ms, p99 ≈ 24ms.
+
+### Patch Changes
+
+- Updated dependencies [9b91e85]
+- Updated dependencies [c80a05b]
+  - @prosopo/types@4.5.0
+  - @prosopo/api@3.4.12
+
+## 3.9.1
+### Patch Changes
+
+- b520cd9: Paginate the greedy `findRules` RediSearch query via `FT.AGGREGATE WITHCURSOR` so the candidate set is no longer truncated at `REDIS_BATCH_SIZE` (1000). Under high-volume bot traffic, a single popular ja4 fingerprint can be carried by thousands of rules; the OR-style greedy query returned > 1000 candidates and `FT.SEARCH`'s LIMIT silently dropped the tail — block rules emitted by less-frequent detectors sat past offset 1000 and never reached the JS-side specificity sort, letting matching requests through. Aggregation cursors return the full result set, so ranking sees every candidate.
+
 ## 3.9.0
 ### Minor Changes
 

package/dist/cjs/redis/reader/redisAggregate.cjs

@@ -4,19 +4,36 @@ const zod = require("zod");
 const redisRulesQuery = require("./redisRulesQuery.cjs");
 const redisClient = require("../redisClient.cjs");
 const redisRuleIndex = require("../redisRuleIndex.cjs");
-const aggregateRedisKeys = async (client, query, logger, batchHandler) => {
+const aggregateRedisKeys = async (client, query, logger, batchHandler, maxKeys) => {
   const keyField = "__key";
   const recordSchema = zod.z.object({
     // it's a reserved name for the record key
     [keyField]: zod.z.string()
   });
   const foundKeys = [];
+  let stopRequested = false;
   const addRecordKeys = async (records) => {
+    if (stopRequested) {
+      return;
+    }
     const parsedRecords = redisClient.parseRedisRecords(records, recordSchema, logger);
     const recordKeys = parsedRecords.map((record) => record[keyField]);
     if (batchHandler) {
       await batchHandler(recordKeys);
     } else {
+      if (maxKeys !== void 0 && foundKeys.length + recordKeys.length > maxKeys) {
+        const remaining = Math.max(0, maxKeys - foundKeys.length);
+        foundKeys.push(...recordKeys.slice(0, remaining));
+        stopRequested = true;
+        logger.warn(() => ({
+          msg: "Redis aggregation candidate cap hit; truncating result set. This can suppress less-frequent rules and should be investigated.",
+          data: {
+            maxKeys,
+            query
+          }
+        }));
+        return;
+      }
       foundKeys.push(...recordKeys);
       logger.debug(() => ({
         msg: "Processed aggregation batch",
@@ -35,11 +52,12 @@ const aggregateRedisKeys = async (client, query, logger, batchHandler) => {
       COUNT: redisClient.REDIS_BATCH_SIZE,
       LOAD: `@${keyField}`
     },
-    addRecordKeys
+    addRecordKeys,
+    () => stopRequested
   );
   return foundKeys;
 };
-const executeAggregation = async (client, query, aggregateOptions, handleBatch) => {
+const executeAggregation = async (client, query, aggregateOptions, handleBatch, shouldStop) => {
   const initialReply = await client.ft.aggregateWithCursor(
     redisRuleIndex.ACCESS_RULES_REDIS_INDEX_NAME,
     query,
@@ -47,7 +65,7 @@ const executeAggregation = async (client, query, aggregateOptions, handleBatch)
   );
   await handleBatch(initialReply.results);
   let cursor = initialReply.cursor;
-  while (0 !== cursor) {
+  while (0 !== cursor && !shouldStop?.()) {
     const batchReply = await client.ft.cursorRead(
       redisRuleIndex.ACCESS_RULES_REDIS_INDEX_NAME,
       cursor,

package/dist/cjs/redis/reader/redisRulesReader.cjs

@@ -5,6 +5,7 @@ const common = require("@prosopo/common");
 const redisRulesQuery = require("./redisRulesQuery.cjs");
 const redisClient = require("../redisClient.cjs");
 const redisRuleIndex = require("../redisRuleIndex.cjs");
+const rule = require("../../rule.cjs");
 const ruleInput = require("../../ruleInput/ruleInput.cjs");
 const redisAggregate = require("./redisAggregate.cjs");
 function _interopNamespaceDefault(e) {
@@ -24,6 +25,48 @@ function _interopNamespaceDefault(e) {
   return Object.freeze(n);
 }
 const util__namespace = /* @__PURE__ */ _interopNamespaceDefault(util);
+const SERVER_SIDE_RANK_TOP_N = 20;
+const GREEDY_MAX_CANDIDATES = redisClient.REDIS_BATCH_SIZE * 10;
+const SPECIFICITY_EXPR = [
+  "exists(@clientId)",
+  "exists(@userId)",
+  "exists(@ja4Hash)",
+  "exists(@headersHash)",
+  "exists(@userAgentHash)",
+  "exists(@headHash)",
+  "exists(@coords)",
+  "exists(@countryCode)",
+  "exists(@asn)",
+  "exists(@numericIp)",
+  "exists(@numericIpMaskMin)"
+].join(" + ");
+const SEVERITY_EXPR = `(@type == "${rule.AccessPolicyType.Block}")`;
+const RANK_EXPR = "(@_spec * 2) + @_sev";
+const RULE_LOAD_FIELDS = [
+  "@__key",
+  "@type",
+  "@captchaType",
+  "@description",
+  "@solvedImagesCount",
+  "@imageThreshold",
+  "@powDifficulty",
+  "@unsolvedImagesCount",
+  "@frictionlessScore",
+  "@deferToVerify",
+  "@clientId",
+  "@groupId",
+  "@userId",
+  "@ja4Hash",
+  "@headersHash",
+  "@userAgentHash",
+  "@headHash",
+  "@coords",
+  "@countryCode",
+  "@asn",
+  "@numericIp",
+  "@numericIpMaskMin",
+  "@numericIpMaskMax"
+];
 class RedisRulesReader {
   constructor(client, logger) {
     this.client = client;
@@ -52,40 +95,102 @@ class RedisRulesReader {
     if (skipEmptyUserScopes && query === "ismissing(@clientId)") {
       return [];
     }
+    if (matchingFieldsOnly) {
+      return this.findRulesRanked(filter, query);
+    }
+    return this.findRulesGreedy(filter, query);
+  }
+  async findRulesRanked(filter, query) {
     try {
-      const searchReply = await this.client.ft.searchNoContent(
+      const reply = await this.client.ft.aggregate(
         redisRuleIndex.ACCESS_RULES_REDIS_INDEX_NAME,
         query,
         {
           DIALECT: redisRulesQuery.REDIS_QUERY_DIALECT,
-          // FT.search doesn't support "unlimited" selects
-          LIMIT: {
-            from: 0,
-            size: redisClient.REDIS_BATCH_SIZE
-          }
+          LOAD: [...RULE_LOAD_FIELDS],
+          STEPS: [
+            { type: "APPLY", expression: SPECIFICITY_EXPR, AS: "_spec" },
+            { type: "APPLY", expression: SEVERITY_EXPR, AS: "_sev" },
+            { type: "APPLY", expression: RANK_EXPR, AS: "_rank" },
+            {
+              type: "SORTBY",
+              BY: [{ BY: "@_rank", DIRECTION: "DESC" }]
+              // SORTBY MAX would let RediSearch keep only the
+              // top-N in a bounded heap rather than fully
+              // sorting, but the @redis/search client
+              // serialises MAX *before* DESC and RediSearch
+              // rejects that order ("MISSING ASC or DESC
+              // after sort field (MAX)"). LIMIT below still
+              // trims; SORTBY does a full sort. Acceptable
+              // because the strict-match filter already
+              // keeps the candidate set small in practice.
+            },
+            { type: "LIMIT", from: 0, size: SERVER_SIDE_RANK_TOP_N }
+          ]
         }
       );
-      if (searchReply.total > 0) {
-        this.logger.debug(() => ({
-          msg: "Executed search query",
-          data: {
-            inspect: util__namespace.inspect(
-              {
-                filter,
-                searchReply,
-                query
-              },
-              { depth: null }
-            )
-          }
-        }));
+      if (reply.results.length === 0) {
+        return [];
       }
-      if (searchReply.documents.length === 0) {
+      this.logger.debug(() => ({
+        msg: "Executed ranked search query",
+        data: {
+          inspect: util__namespace.inspect(
+            {
+              filter,
+              foundCount: reply.results.length,
+              query
+            },
+            { depth: null }
+          )
+        }
+      }));
+      return redisClient.parseRedisRecords(reply.results, ruleInput.accessRuleInput, this.logger);
+    } catch (e) {
+      this.logger.error(() => ({
+        err: e,
+        data: {
+          inspect: util__namespace.inspect({ query, filter }, { depth: null })
+        },
+        msg: "failed to execute ranked search query"
+      }));
+      return [];
+    }
+  }
+  // Used only by admin / internal callers (matchingFieldsOnly=false).
+  // Not on the per-request blockMiddleware path any more, so the CPU
+  // cost of the HGETALL fanout is acceptable. Same shape as #2689's
+  // aggregate-with-cursor: returns every candidate up to
+  // GREEDY_MAX_CANDIDATES so we don't silently drop matches that
+  // admin tools depend on (e.g. ASN-wide rule listings).
+  async findRulesGreedy(filter, query) {
+    try {
+      const ruleKeys = await redisAggregate.aggregateRedisKeys(
+        this.client,
+        query,
+        this.logger,
+        void 0,
+        GREEDY_MAX_CANDIDATES
+      );
+      if (ruleKeys.length === 0) {
         return [];
       }
+      this.logger.debug(() => ({
+        msg: "Executed greedy search query",
+        data: {
+          inspect: util__namespace.inspect(
+            {
+              filter,
+              foundCount: ruleKeys.length,
+              query
+            },
+            { depth: null }
+          )
+        }
+      }));
       const { records } = await redisClient.fetchRedisHashRecords(
         this.client,
-        searchReply.documents,
+        ruleKeys,
         this.logger
       );
       const nonEmptyRecords = records.filter(
@@ -96,17 +201,9 @@ class RedisRulesReader {
       this.logger.error(() => ({
         err: e,
         data: {
-          inspect: util__namespace.inspect(
-            {
-              query,
-              filter
-            },
-            {
-              depth: null
-            }
-          )
+          inspect: util__namespace.inspect({ query, filter }, { depth: null })
         },
-        msg: "failed to execute search query"
+        msg: "failed to execute greedy search query"
       }));
       return [];
     }
@@ -170,14 +267,14 @@ class RedisRulesReader {
     for (const [index, ruleData] of records.entries()) {
       const isRulePresent = Object.keys(ruleData).length > 0;
       if (isRulePresent) {
-        const rule = redisClient.parseRedisRecords(
+        const rule2 = redisClient.parseRedisRecords(
           [ruleData],
           ruleInput.accessRuleInput,
           this.logger
         )[0];
-        if (rule) {
+        if (rule2) {
           entries.push({
-            rule,
+            rule: rule2,
             expiresUnixTimestamp: expirations[index]
           });
         }
@@ -237,3 +334,4 @@ class DummyRedisRulesReader {
 }
 exports.DummyRedisRulesReader = DummyRedisRulesReader;
 exports.RedisRulesReader = RedisRulesReader;
+exports.SERVER_SIDE_RANK_TOP_N = SERVER_SIDE_RANK_TOP_N;

package/dist/redis/reader/redisAggregate.d.ts

@@ -1,4 +1,4 @@
 import type { Logger } from "@prosopo/logger";
 import type { RedisClientType } from "redis";
-export declare const aggregateRedisKeys: (client: RedisClientType, query: string, logger: Logger, batchHandler?: (keys: string[]) => Promise<void>) => Promise<string[]>;
+export declare const aggregateRedisKeys: (client: RedisClientType, query: string, logger: Logger, batchHandler?: (keys: string[]) => Promise<void>, maxKeys?: number) => Promise<string[]>;
 //# sourceMappingURL=redisAggregate.d.ts.map

package/dist/redis/reader/redisAggregate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"redisAggregate.d.ts","sourceRoot":"","sources":["../../../src/redis/reader/redisAggregate.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,OAAO,CAAC;AAU7C,eAAO,MAAM,kBAAkB,WACtB,eAAe,SAChB,MAAM,UACL,MAAM,iBACC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,KAC9C,OAAO,CAAC,MAAM,EAAE,CA0ClB,CAAC"}
+{"version":3,"file":"redisAggregate.d.ts","sourceRoot":"","sources":["../../../src/redis/reader/redisAggregate.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,OAAO,CAAC;AAU7C,eAAO,MAAM,kBAAkB,WACtB,eAAe,SAChB,MAAM,UACL,MAAM,iBACC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,YACtC,MAAM,KACd,OAAO,CAAC,MAAM,EAAE,CAkElB,CAAC"}

package/dist/redis/reader/redisAggregate.js

@@ -2,19 +2,36 @@ import { z } from "zod";
 import { REDIS_QUERY_DIALECT } from "./redisRulesQuery.js";
 import { parseRedisRecords, REDIS_BATCH_SIZE } from "../redisClient.js";
 import { ACCESS_RULES_REDIS_INDEX_NAME } from "../redisRuleIndex.js";
-const aggregateRedisKeys = async (client, query, logger, batchHandler) => {
+const aggregateRedisKeys = async (client, query, logger, batchHandler, maxKeys) => {
   const keyField = "__key";
   const recordSchema = z.object({
     // it's a reserved name for the record key
     [keyField]: z.string()
   });
   const foundKeys = [];
+  let stopRequested = false;
   const addRecordKeys = async (records) => {
+    if (stopRequested) {
+      return;
+    }
     const parsedRecords = parseRedisRecords(records, recordSchema, logger);
     const recordKeys = parsedRecords.map((record) => record[keyField]);
     if (batchHandler) {
       await batchHandler(recordKeys);
     } else {
+      if (maxKeys !== void 0 && foundKeys.length + recordKeys.length > maxKeys) {
+        const remaining = Math.max(0, maxKeys - foundKeys.length);
+        foundKeys.push(...recordKeys.slice(0, remaining));
+        stopRequested = true;
+        logger.warn(() => ({
+          msg: "Redis aggregation candidate cap hit; truncating result set. This can suppress less-frequent rules and should be investigated.",
+          data: {
+            maxKeys,
+            query
+          }
+        }));
+        return;
+      }
       foundKeys.push(...recordKeys);
       logger.debug(() => ({
         msg: "Processed aggregation batch",
@@ -33,11 +50,12 @@ const aggregateRedisKeys = async (client, query, logger, batchHandler) => {
       COUNT: REDIS_BATCH_SIZE,
       LOAD: `@${keyField}`
     },
-    addRecordKeys
+    addRecordKeys,
+    () => stopRequested
   );
   return foundKeys;
 };
-const executeAggregation = async (client, query, aggregateOptions, handleBatch) => {
+const executeAggregation = async (client, query, aggregateOptions, handleBatch, shouldStop) => {
   const initialReply = await client.ft.aggregateWithCursor(
     ACCESS_RULES_REDIS_INDEX_NAME,
     query,
@@ -45,7 +63,7 @@ const executeAggregation = async (client, query, aggregateOptions, handleBatch)
   );
   await handleBatch(initialReply.results);
   let cursor = initialReply.cursor;
-  while (0 !== cursor) {
+  while (0 !== cursor && !shouldStop?.()) {
     const batchReply = await client.ft.cursorRead(
       ACCESS_RULES_REDIS_INDEX_NAME,
       cursor,

package/dist/redis/reader/redisAggregate.js.map

@@ -1 +1 @@
-{"version":3,"file":"redisAggregate.js","sourceRoot":"","sources":["../../../src/redis/reader/redisAggregate.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,yCAAyC,CAAC;AAC9E,OAAO,EACN,gBAAgB,EAChB,iBAAiB,GACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAGhF,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACtC,MAAuB,EACvB,KAAa,EACb,MAAc,EACd,YAAgD,EAC5B,EAAE;IACtB,MAAM,QAAQ,GAAG,OAAO,CAAC;IAEzB,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;QAE7B,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;KACtB,CAAC,CAAC;IAEH,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,MAAM,aAAa,GAAG,KAAK,EAAE,OAAiB,EAAE,EAAE;QACjD,MAAM,aAAa,GAAG,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QAEvE,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAEnE,IAAI,YAAY,EAAE,CAAC;YAClB,MAAM,YAAY,CAAC,UAAU,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACP,SAAS,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnB,GAAG,EAAE,6BAA6B;gBAClC,IAAI,EAAE;oBACL,IAAI,EAAE,UAAU,CAAC,MAAM;iBACvB;aACD,CAAC,CAAC,CAAC;QACL,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,kBAAkB,CACvB,MAAM,EACN,KAAK,EACL;QAEC,OAAO,EAAE,mBAAmB;QAC5B,KAAK,EAAE,gBAAgB;QACvB,IAAI,EAAE,IAAI,QAAQ,EAAE;KACpB,EACD,aAAa,CACb,CAAC;IAEF,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,KAAK,EAC/B,MAAuB,EACvB,KAAa,EACb,gBAA8C,EAC9C,WAAiD,EACjC,EAAE;IAClB,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,EAAE,CAAC,mBAAmB,CACvD,6BAA6B,EAC7B,KAAK,EACL,gBAAgB,CAChB,CAAC;IAEF,MAAM,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAExC,IAAI,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IAEjC,OAAO,CAAC,KAAK,MAAM,EAAE,CAAC;QACrB,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,EAAE,CAAC,UAAU,CAC5C,6BAA6B,EAC7B,MAAM,EACN,EAAE,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,CACjC,CAAC;QAEF,MAAM,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEtC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAC5B,CAAC;AACF,CAAC,CAAC"}
+{"version":3,"file":"redisAggregate.js","sourceRoot":"","sources":["../../../src/redis/reader/redisAggregate.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,yCAAyC,CAAC;AAC9E,OAAO,EACN,gBAAgB,EAChB,iBAAiB,GACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAGhF,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACtC,MAAuB,EACvB,KAAa,EACb,MAAc,EACd,YAAgD,EAChD,OAAgB,EACI,EAAE;IACtB,MAAM,QAAQ,GAAG,OAAO,CAAC;IAEzB,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;QAE7B,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;KACtB,CAAC,CAAC;IAEH,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,aAAa,GAAG,KAAK,CAAC;IAE1B,MAAM,aAAa,GAAG,KAAK,EAAE,OAAiB,EAAE,EAAE;QACjD,IAAI,aAAa,EAAE,CAAC;YACnB,OAAO;QACR,CAAC;QAED,MAAM,aAAa,GAAG,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QAEvE,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAEnE,IAAI,YAAY,EAAE,CAAC;YAClB,MAAM,YAAY,CAAC,UAAU,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACP,IACC,OAAO,KAAK,SAAS;gBACrB,SAAS,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,GAAG,OAAO,EAC7C,CAAC;gBACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC1D,SAAS,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBAClD,aAAa,GAAG,IAAI,CAAC;gBAErB,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;oBAClB,GAAG,EAAE,+HAA+H;oBACpI,IAAI,EAAE;wBACL,OAAO;wBACP,KAAK;qBACL;iBACD,CAAC,CAAC,CAAC;gBACJ,OAAO;YACR,CAAC;YAED,SAAS,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnB,GAAG,EAAE,6BAA6B;gBAClC,IAAI,EAAE;oBACL,IAAI,EAAE,UAAU,CAAC,MAAM;iBACvB;aACD,CAAC,CAAC,CAAC;QACL,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,kBAAkB,CACvB,MAAM,EACN,KAAK,EACL;QAEC,OAAO,EAAE,mBAAmB;QAC5B,KAAK,EAAE,gBAAgB;QACvB,IAAI,EAAE,IAAI,QAAQ,EAAE;KACpB,EACD,aAAa,EACb,GAAG,EAAE,CAAC,aAAa,CACnB,CAAC;IAEF,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,KAAK,EAC/B,MAAuB,EACvB,KAAa,EACb,gBAA8C,EAC9C,WAAiD,EACjD,UAA0B,EACV,EAAE;IAClB,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,EAAE,CAAC,mBAAmB,CACvD,6BAA6B,EAC7B,KAAK,EACL,gBAAgB,CAChB,CAAC;IAEF,MAAM,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAExC,IAAI,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IAEjC,OAAO,CAAC,KAAK,MAAM,IAAI,CAAC,UAAU,EAAE,EAAE,EAAE,CAAC;QACxC,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,EAAE,CAAC,UAAU,CAC5C,6BAA6B,EAC7B,MAAM,EACN,EAAE,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,CACjC,CAAC;QAEF,MAAM,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEtC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAC5B,CAAC;AACF,CAAC,CAAC"}

package/dist/redis/reader/redisRulesReader.d.ts

@@ -1,7 +1,8 @@
 import type { Logger } from "@prosopo/logger";
 import type { RedisClientType } from "redis";
-import type { AccessRule } from "#policy/rule.js";
+import { type AccessRule } from "#policy/rule.js";
 import type { AccessRuleEntry, AccessRulesFilter, AccessRulesReader } from "#policy/rulesStorage.js";
+export declare const SERVER_SIDE_RANK_TOP_N = 20;
 export declare class RedisRulesReader implements AccessRulesReader {
     private readonly client;
     private readonly logger;
@@ -9,6 +10,8 @@ export declare class RedisRulesReader implements AccessRulesReader {
     getMissingRuleIds(ruleIds: string[]): Promise<string[]>;
     fetchRules(ruleIds: string[]): Promise<AccessRuleEntry[]>;
     findRules(filter: AccessRulesFilter, matchingFieldsOnly?: boolean, skipEmptyUserScopes?: boolean): Promise<AccessRule[]>;
+    private findRulesRanked;
+    private findRulesGreedy;
     findRuleIds(filter: AccessRulesFilter, matchingFieldsOnly?: boolean): Promise<string[]>;
     fetchAllRuleIds(batchHandler: (ruleIds: string[]) => Promise<void>): Promise<void>;
     protected fetchRuleEntries(keys: string[]): Promise<AccessRuleEntry[]>;

package/dist/redis/reader/redisRulesReader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"redisRulesReader.d.ts","sourceRoot":"","sources":["../../../src/redis/reader/redisRulesReader.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,OAAO,CAAC;AAe7C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAElD,OAAO,KAAK,EACX,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,MAAM,yBAAyB,CAAC;AAGjC,qBAAa,gBAAiB,YAAW,iBAAiB;IAExD,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;gBADN,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,MAAM;IAG1B,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAcvD,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAazD,SAAS,CACd,MAAM,EAAE,iBAAiB,EACzB,kBAAkB,UAAQ,EAC1B,mBAAmB,UAAO,GACxB,OAAO,CAAC,UAAU,EAAE,CAAC;IA8ElB,WAAW,CAChB,MAAM,EAAE,iBAAiB,EACzB,kBAAkB,UAAQ,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAgDd,eAAe,CACpB,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAChD,OAAO,CAAC,IAAI,CAAC;cAYA,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA8B5E,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;CAGlD;AAED,qBAAa,qBAAsB,YAAW,iBAAiB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,MAAM;IAErC,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWvD,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAWzD,SAAS,CACd,MAAM,EAAE,iBAAiB,EACzB,kBAAkB,UAAQ,EAC1B,mBAAmB,UAAO,GACxB,OAAO,CAAC,UAAU,EAAE,CAAC;IAWlB,WAAW,CAChB,MAAM,EAAE,iBAAiB,EACzB,kBAAkB,UAAQ,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAWd,eAAe,CACpB,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAChD,OAAO,CAAC,IAAI,CAAC;CAKhB"}
+{"version":3,"file":"redisRulesReader.d.ts","sourceRoot":"","sources":["../../../src/redis/reader/redisRulesReader.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,OAAO,CAAC;AAe7C,OAAO,EAAoB,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAEpE,OAAO,KAAK,EACX,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,MAAM,yBAAyB,CAAC;AAoBjC,eAAO,MAAM,sBAAsB,KAAK,CAAC;AAoEzC,qBAAa,gBAAiB,YAAW,iBAAiB;IAExD,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;gBADN,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,MAAM;IAG1B,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAcvD,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAazD,SAAS,CACd,MAAM,EAAE,iBAAiB,EACzB,kBAAkB,UAAQ,EAC1B,mBAAmB,UAAO,GACxB,OAAO,CAAC,UAAU,EAAE,CAAC;YAuBV,eAAe;YA2Ef,eAAe;IAuDvB,WAAW,CAChB,MAAM,EAAE,iBAAiB,EACzB,kBAAkB,UAAQ,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAgDd,eAAe,CACpB,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAChD,OAAO,CAAC,IAAI,CAAC;cAYA,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA8B5E,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;CAGlD;AAED,qBAAa,qBAAsB,YAAW,iBAAiB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,MAAM;IAErC,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWvD,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAWzD,SAAS,CACd,MAAM,EAAE,iBAAiB,EACzB,kBAAkB,UAAQ,EAC1B,mBAAmB,UAAO,GACxB,OAAO,CAAC,UAAU,EAAE,CAAC;IAWlB,WAAW,CAChB,MAAM,EAAE,iBAAiB,EACzB,kBAAkB,UAAQ,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAWd,eAAe,CACpB,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAChD,OAAO,CAAC,IAAI,CAAC;CAKhB"}