@karmaniverous/entity-manager 4.4.3 → 5.0.0

package/README.md

@@ -373,7 +373,7 @@ Query an entity across shards.
 | options.shardQuery | <code>ShardQueryFunction</code> | Sharded query function. |
 | [options.limit] | <code>number</code> | Request limit. |
 | [options.pageKeys] | <code>object</code> | Map of shard page keys. |
-| [options.pages] | <code>number</code> \| <code>&#x27;all&#x27;</code> | Max number of pages to query. |
+| [options.pageSize] | <code>number</code> | Request page size. |
 
 <a name="module_entity-manager.EntityManager+rehydrateIndex"></a>
 
@@ -435,8 +435,8 @@ Shard query function
 | Param | Type | Description |
 | --- | --- | --- |
 | shardedKey | <code>string</code> | Sharded key. |
-| [limit] | <code>number</code> | Request limit. |
 | [pageKey] | <code>\*</code> | Page key. |
+| [limit] | <code>number</code> | Request limit. |
 
 <a name="module_entity-manager..ShardedQueryResult"></a>
 

package/dist/default/lib/EntityManager/EntityManager.js

@@ -6,11 +6,8 @@ Object.defineProperty(exports, "__esModule", {
 exports.EntityManager = void 0;
 var _zipObject2 = _interopRequireDefault(require("lodash/zipObject"));
 var _map2 = _interopRequireDefault(require("lodash/map"));
-var _size2 = _interopRequireDefault(require("lodash/size"));
 var _isEmpty2 = _interopRequireDefault(require("lodash/isEmpty"));
 var _fromPairs2 = _interopRequireDefault(require("lodash/fromPairs"));
-var _parseInt2 = _interopRequireDefault(require("lodash/parseInt"));
-var _isString2 = _interopRequireDefault(require("lodash/isString"));
 var _isPlainObject2 = _interopRequireDefault(require("lodash/isPlainObject"));
 var _isInteger2 = _interopRequireDefault(require("lodash/isInteger"));
 var _isUndefined2 = _interopRequireDefault(require("lodash/isUndefined"));
@@ -213,8 +210,8 @@ class EntityManager {
    *
    * @callback ShardQueryFunction
    * @param {string} shardedKey - Sharded key.
-   * @param {number} [limit] - Request limit.
    * @param {*} [pageKey] - Page key.
+   * @param {number} [limit] - Request limit.
    * @returns {Promise<ShardQueryResult>} Sharded query result.
    */
 
@@ -234,7 +231,7 @@ class EntityManager {
    * @param {ShardQueryFunction} options.shardQuery - Sharded query function.
    * @param {number} [options.limit] - Request limit.
    * @param {object} [options.pageKeys] - Map of shard page keys.
-   * @param {number|'all'} [options.pages] - Max number of pages to query.
+   * @param {number} [options.pageSize] - Request page size.
    * @returns {Promise<ShardedQueryResult>} Sharded query result.
    */
   async query() {
@@ -245,7 +242,7 @@ class EntityManager {
       shardQuery,
       limit,
       pageKeys,
-      pages = 1
+      pageSize
     } = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
     // Validate params.
     this.#entityManager.validateKeyToken(entityToken, keyToken);
@@ -253,13 +250,13 @@ class EntityManager {
     if (!(0, _isFunction2.default)(shardQuery)) throw new Error('shardQuery must be a function');
     if (!(0, _isUndefined2.default)(limit) && !((0, _isInteger2.default)(limit) && limit >= 1)) throw new Error('limit must be a positive integer');
     if (!(0, _isUndefined2.default)(pageKeys) && !(0, _isPlainObject2.default)(pageKeys)) throw new Error('pageKeys must be an object');
-    if (!(0, _isUndefined2.default)(pages) && !/^(?:\d+|all)$/i.test(pages.toString())) throw new Error("pages must be a positive integer or 'all'");
+    if (!(0, _isUndefined2.default)(pageSize) && !((0, _isInteger2.default)(pageSize) && pageSize >= 1)) throw new Error('pageSize must be a positive integer');
 
     // Apply default limit.
     limit ??= this.#entityManager.getEntityConfig(entityToken).defaultLimit;
 
-    // Parse pages.
-    if ((0, _isString2.default)(pages)) pages = pages.toLowerCase() === 'all' ? Infinity : (0, _parseInt2.default)(pages);
+    // Apply default pageSize.
+    pageSize ??= this.#entityManager.getEntityConfig(entityToken).defaultPageSize;
 
     // Generate default pageKeys if not provided
     pageKeys ??= (0, _fromPairs2.default)(this.getKeySpace(entityToken, keyToken, item).map(shardedKey => [shardedKey, undefined]));
@@ -270,7 +267,7 @@ class EntityManager {
       shardQuery,
       limit,
       pageKeys,
-      pages
+      pageSize
     });
 
     // Return empty result if no pageKeys.
@@ -280,9 +277,6 @@ class EntityManager {
       pageKeys: {}
     };
 
-    // Calculate shard limit.
-    const shardLimit = (0, _isUndefined2.default)(limit) ? undefined : Math.ceil(limit / (0, _size2.default)(pageKeys));
-
     // Iterate search over pages.
     let page = 0;
     const result = {
@@ -291,8 +285,12 @@ class EntityManager {
       pageKeys
     };
     do {
+      // TODO: This loop will blow up as shards scale, since at a minimum it will return shardCount x pageSize
+      // items, which may be >> limit. Probably the way to fix this is to limit the number of shards queried per
+      // iteration in order to keep shardsQueried * pageSize > (limit - items.length) but only just.
+
       // Query every shard in pageKeys.
-      const shardQueryResults = await Promise.all((0, _map2.default)(result.pageKeys, (pageKey, shardedKey) => new Promise(resolve => shardQuery(shardedKey, pageKey, shardLimit).then(_ref6 => {
+      const shardQueryResults = await Promise.all((0, _map2.default)(result.pageKeys, (pageKey, shardedKey) => new Promise(resolve => shardQuery(shardedKey, pageKey, pageSize).then(_ref6 => {
         let {
           count,
           items,
@@ -334,7 +332,7 @@ class EntityManager {
       result.items = [...result.items, ...pageResult.items];
       result.pageKeys = pageResult.pageKeys;
       page++;
-    } while (page < pages && result.items.length < limit * pages && !(0, _isEmpty2.default)(result.pageKeys));
+    } while (result.items.length < limit && !(0, _isEmpty2.default)(result.pageKeys));
     return result;
   }
 

package/dist/default/lib/EntityManager/PrivateEntityManager.js

@@ -3,7 +3,7 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.PrivateEntityManager = exports.DEFAULT_LIMIT = void 0;
+exports.PrivateEntityManager = exports.DEFAULT_PAGE_SIZE = exports.DEFAULT_LIMIT = void 0;
 var _isInteger2 = _interopRequireDefault(require("lodash/isInteger"));
 var _isPlainObject2 = _interopRequireDefault(require("lodash/isPlainObject"));
 var _range2 = _interopRequireDefault(require("lodash/range"));
@@ -26,7 +26,8 @@ var _mapValues2 = _interopRequireDefault(require("lodash/mapValues"));
 var _jsonschema = require("jsonschema");
 var _stringHash = _interopRequireDefault(require("string-hash"));
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-const DEFAULT_LIMIT = exports.DEFAULT_LIMIT = 25;
+const DEFAULT_LIMIT = exports.DEFAULT_LIMIT = 10;
+const DEFAULT_PAGE_SIZE = exports.DEFAULT_PAGE_SIZE = 10;
 
 /**
  * EntityManager config validation schema.
@@ -45,6 +46,9 @@ const configSchema = {
             defaultLimit: {
               type: 'integer'
             },
+            defaultPageSize: {
+              type: 'integer'
+            },
             indexes: {
               type: 'object',
               additionalProperties: {
@@ -175,6 +179,7 @@ class PrivateEntityManager {
       entities: (0, _mapValues2.default)(entities, _ref => {
         let {
           defaultLimit = DEFAULT_LIMIT,
+          defaultPageSize = DEFAULT_PAGE_SIZE,
           indexes = {},
           keys = {},
           sharding: {
@@ -187,6 +192,7 @@ class PrivateEntityManager {
         } = _ref;
         return {
           defaultLimit,
+          defaultPageSize,
           indexes,
           keys,
           sharding: {

package/lib/EntityManager/EntityManager.js

@@ -196,8 +196,8 @@ export class EntityManager {
    *
    * @callback ShardQueryFunction
    * @param {string} shardedKey - Sharded key.
-   * @param {number} [limit] - Request limit.
    * @param {*} [pageKey] - Page key.
+   * @param {number} [limit] - Request limit.
    * @returns {Promise<ShardQueryResult>} Sharded query result.
    */
 
@@ -217,7 +217,7 @@ export class EntityManager {
    * @param {ShardQueryFunction} options.shardQuery - Sharded query function.
    * @param {number} [options.limit] - Request limit.
    * @param {object} [options.pageKeys] - Map of shard page keys.
-   * @param {number|'all'} [options.pages] - Max number of pages to query.
+   * @param {number} [options.pageSize] - Request page size.
    * @returns {Promise<ShardedQueryResult>} Sharded query result.
    */
   async query({
@@ -227,26 +227,30 @@ export class EntityManager {
     shardQuery,
     limit,
     pageKeys,
-    pages = 1,
+    pageSize,
   } = {}) {
     // Validate params.
     this.#entityManager.validateKeyToken(entityToken, keyToken);
     this.#entityManager.validateItem(item);
+
     if (!_.isFunction(shardQuery))
       throw new Error('shardQuery must be a function');
+
     if (!_.isUndefined(limit) && !(_.isInteger(limit) && limit >= 1))
       throw new Error('limit must be a positive integer');
+
     if (!_.isUndefined(pageKeys) && !_.isPlainObject(pageKeys))
       throw new Error('pageKeys must be an object');
-    if (!_.isUndefined(pages) && !/^(?:\d+|all)$/i.test(pages.toString()))
-      throw new Error("pages must be a positive integer or 'all'");
+
+    if (!_.isUndefined(pageSize) && !(_.isInteger(pageSize) && pageSize >= 1))
+      throw new Error('pageSize must be a positive integer');
 
     // Apply default limit.
     limit ??= this.#entityManager.getEntityConfig(entityToken).defaultLimit;
 
-    // Parse pages.
-    if (_.isString(pages))
-      pages = pages.toLowerCase() === 'all' ? Infinity : _.parseInt(pages);
+    // Apply default pageSize.
+    pageSize ??=
+      this.#entityManager.getEntityConfig(entityToken).defaultPageSize;
 
     // Generate default pageKeys if not provided
     pageKeys ??= _.fromPairs(
@@ -263,29 +267,28 @@ export class EntityManager {
       shardQuery,
       limit,
       pageKeys,
-      pages,
+      pageSize,
     });
 
     // Return empty result if no pageKeys.
     if (_.isEmpty(pageKeys)) return { count: 0, items: [], pageKeys: {} };
 
-    // Calculate shard limit.
-    const shardLimit = _.isUndefined(limit)
-      ? undefined
-      : Math.ceil(limit / _.size(pageKeys));
-
     // Iterate search over pages.
     let page = 0;
     const result = { count: 0, items: [], pageKeys };
 
     do {
+      // TODO: This loop will blow up as shards scale, since at a minimum it will return shardCount x pageSize
+      // items, which may be >> limit. Probably the way to fix this is to limit the number of shards queried per
+      // iteration in order to keep shardsQueried * pageSize > (limit - items.length) but only just.
+
       // Query every shard in pageKeys.
       const shardQueryResults = await Promise.all(
         _.map(
           result.pageKeys,
           (pageKey, shardedKey) =>
             new Promise((resolve) =>
-              shardQuery(shardedKey, pageKey, shardLimit).then(
+              shardQuery(shardedKey, pageKey, pageSize).then(
                 ({ count, items, pageKey }) =>
                   resolve({ count, items, pageKey, shardedKey })
               )
@@ -315,11 +318,7 @@ export class EntityManager {
       result.items = [...result.items, ...pageResult.items];
       result.pageKeys = pageResult.pageKeys;
       page++;
-    } while (
-      page < pages &&
-      result.items.length < limit * pages &&
-      !_.isEmpty(result.pageKeys)
-    );
+    } while (result.items.length < limit && !_.isEmpty(result.pageKeys));
 
     return result;
   }

package/lib/EntityManager/PrivateEntityManager.js

@@ -2,7 +2,8 @@ import { validate } from 'jsonschema';
 import _ from 'lodash';
 import stringHash from 'string-hash';
 
-export const DEFAULT_LIMIT = 25;
+export const DEFAULT_LIMIT = 10;
+export const DEFAULT_PAGE_SIZE = 10;
 
 /**
  * EntityManager config validation schema.
@@ -21,6 +22,9 @@ const configSchema = {
             defaultLimit: {
               type: 'integer',
             },
+            defaultPageSize: {
+              type: 'integer',
+            },
             indexes: {
               type: 'object',
               additionalProperties: {
@@ -141,6 +145,7 @@ export class PrivateEntityManager {
         entities,
         ({
           defaultLimit = DEFAULT_LIMIT,
+          defaultPageSize = DEFAULT_PAGE_SIZE,
           indexes = {},
           keys = {},
           sharding: {
@@ -152,6 +157,7 @@ export class PrivateEntityManager {
           } = {},
         }) => ({
           defaultLimit,
+          defaultPageSize,
           indexes,
           keys,
           sharding: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@karmaniverous/entity-manager",
-  "version": "4.4.3",
+  "version": "5.0.0",
   "publishConfig": {
     "access": "public"
   },