@karmaniverous/entity-manager 6.7.0 → 6.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,6 +29,16 @@ class BaseShardQueryMapBuilder {
29
29
  build() {
30
30
  return radash.mapValues(this.indexParamsMap, (indexConfig, indexToken) => this.getShardQueryFunction(indexToken));
31
31
  }
32
+ async query(options) {
33
+ const { entityManager, entityToken, pageKeyMap } = this;
34
+ const shardQueryMap = this.build();
35
+ return await entityManager.query({
36
+ ...options,
37
+ entityToken,
38
+ pageKeyMap,
39
+ shardQueryMap,
40
+ });
41
+ }
32
42
  }
33
43
 
34
44
  exports.BaseShardQueryMapBuilder = BaseShardQueryMapBuilder;
@@ -87,16 +87,7 @@ class EntityManager {
87
87
  * @throws Error if {@link QueryOptions.shardQueryMapBuilder | `shardQueryMapBuilder`} `pageKeyMap` keys do not match its `shardQueryMap` keys.
88
88
  */
89
89
  async query(options) {
90
- const { shardQueryMapBuilder, ...baseOptions } = options;
91
- const { entityToken, hashKeyToken, pageKeyMap } = shardQueryMapBuilder;
92
- const shardQueryMap = shardQueryMapBuilder.build();
93
- return await query.query(this, {
94
- entityToken,
95
- hashKeyToken,
96
- pageKeyMap,
97
- shardQueryMap,
98
- ...baseOptions,
99
- });
90
+ return await query.query(this, options);
100
91
  }
101
92
  }
102
93
  _EntityManager_config = new WeakMap();
@@ -21,14 +21,13 @@ var validateEntityIndexToken = require('./validateEntityIndexToken.js');
21
21
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
22
22
  * @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
23
23
  * @param item - Partial {@link ItemMap | `ItemMap`} object.
24
- * @param omit - Array of index components to omit from the output value.
25
24
  *
26
25
  * @returns Dehydrated index value.
27
26
  *
28
27
  * @throws `Error` if `entityToken` is invalid.
29
28
  * @throws `Error` if `indexToken` is invalid.
30
29
  */
31
- function dehydrateIndexItem(entityManager, entityToken, indexToken, item, omit = []) {
30
+ function dehydrateIndexItem(entityManager, entityToken, indexToken, item) {
32
31
  try {
33
32
  const { generatedKeyDelimiter } = entityManager.config;
34
33
  // Validate params.
@@ -37,7 +36,10 @@ function dehydrateIndexItem(entityManager, entityToken, indexToken, item, omit =
37
36
  if (!item)
38
37
  return '';
39
38
  // Unwrap index elements.
40
- const elements = unwrapIndex.unwrapIndex(entityManager, entityToken, indexToken).filter((element) => !omit.includes(element));
39
+ const { hashKey } = entityManager.config.entities[entityToken].indexes[indexToken];
40
+ const elements = unwrapIndex.unwrapIndex(entityManager, entityToken, indexToken, [
41
+ hashKey,
42
+ ]);
41
43
  // Join index element values.
42
44
  const dehydrated = elements
43
45
  .map((element) => encodeEntityElement.encodeEntityElement(entityManager, entityToken, element, item))
@@ -47,7 +47,7 @@ function dehydratePageKeyMap(entityManager, entityToken, pageKeyMap) {
47
47
  let dehydrated = [];
48
48
  for (const index of indexes) {
49
49
  for (const hashKey of hashKeys) {
50
- // Undefineed pageKey.
50
+ // Undefined pageKey.
51
51
  if (!pageKeyMap[index][hashKey]) {
52
52
  dehydrated.push('');
53
53
  continue;
@@ -62,9 +62,7 @@ function dehydratePageKeyMap(entityManager, entityToken, pageKeyMap) {
62
62
  return item;
63
63
  }, {});
64
64
  // Dehydrate index from item.
65
- dehydrated.push(dehydrateIndexItem.dehydrateIndexItem(entityManager, entityToken, index, item, [
66
- entityManager.config.hashKey,
67
- ]));
65
+ dehydrated.push(dehydrateIndexItem.dehydrateIndexItem(entityManager, entityToken, index, item));
68
66
  }
69
67
  }
70
68
  // Replace with empty array if all pageKeys are empty strings.
@@ -1,13 +1,16 @@
1
1
  'use strict';
2
2
 
3
3
  var radash = require('radash');
4
- var validateEntityToken = require('./validateEntityToken.js');
4
+ var encodeGeneratedProperty = require('./encodeGeneratedProperty.js');
5
+ var validateEntityGeneratedProperty = require('./validateEntityGeneratedProperty.js');
5
6
 
6
7
  /**
7
8
  * Return an array of {@link ConfigKeys.hashKey | `entityManager.config.hashKey`} property values covering the shard space bounded by `timestampFrom` & `timestampTo`.
8
9
  *
9
10
  * @param entityManager - {@link EntityManager | `EntityManager`} instance.
10
11
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
12
+ * @param hashKeyToken - {@link ParsedConfig | `entityManager.config.hashKey`} or {@link ParsedConfig | `entityManager.config.entities.<entityToken>.generated`} key. If a generated property, must be shardable.
13
+ * @param item - Partial {@link ItemMap | `ItemMap[EntityToken]`} object. Must include all properties required to generate the hash key space.
11
14
  * @param timestampFrom - Lower timestanp limit. Defaults to `0`.
12
15
  * @param timestampTo - Upper timestamp limit. Defaults to `Date.now()`.
13
16
  *
@@ -15,24 +18,42 @@ var validateEntityToken = require('./validateEntityToken.js');
15
18
  *
16
19
  * @throws `Error` if `entityToken` is invalid.
17
20
  */
18
- function getHashKeySpace(entityManager, entityToken, timestampFrom = 0, timestampTo = Date.now()) {
21
+ function getHashKeySpace(entityManager, entityToken, hashKeyToken, item, timestampFrom = 0, timestampTo = Date.now()) {
19
22
  try {
20
- // Validate params.
21
- validateEntityToken.validateEntityToken(entityManager, entityToken);
23
+ // Validate hashKeyToken is either the global hash key or a sharded generated property.
24
+ if (hashKeyToken !== entityManager.config.hashKey)
25
+ validateEntityGeneratedProperty.validateEntityGeneratedProperty(entityManager, entityToken, hashKeyToken, true);
22
26
  const { shardBumps } = entityManager.config.entities[entityToken];
23
27
  const hashKeySpace = shardBumps
28
+ // Filter shard bumps by timestamp range.
24
29
  .filter((bump, i) => (i === shardBumps.length - 1 ||
25
30
  shardBumps[i + 1].timestamp > timestampFrom) &&
26
31
  bump.timestamp <= timestampTo)
32
+ // Generate shard key space.
27
33
  .flatMap(({ charBits, chars }) => {
28
34
  const radix = 2 ** charBits;
29
35
  return chars
30
36
  ? [...radash.range(0, radix ** chars - 1)].map((char) => char.toString(radix).padStart(chars, '0'))
31
37
  : '';
32
38
  })
33
- .map((shardKey) => `${entityToken}${entityManager.config.shardKeyDelimiter}${shardKey}`);
39
+ // Map shard keys to hash keys.
40
+ .map((shardKey) => {
41
+ // Calculate record hash key.
42
+ let hashKey = `${entityToken}${entityManager.config.shardKeyDelimiter}${shardKey}`;
43
+ // If hash key space basis is a different property, encode it.
44
+ if (hashKeyToken !== entityManager.config.hashKey)
45
+ hashKey = encodeGeneratedProperty.encodeGeneratedProperty(entityManager, entityToken, hashKeyToken, {
46
+ ...item,
47
+ [entityManager.config.hashKey]: hashKey,
48
+ });
49
+ if (!hashKey)
50
+ throw new Error('item does not support hash key space');
51
+ return hashKey;
52
+ });
34
53
  entityManager.logger.debug('generated hash key space', {
35
54
  entityToken,
55
+ hashKeyToken,
56
+ item,
36
57
  timestampFrom,
37
58
  timestampTo,
38
59
  hashKeySpace,
@@ -43,6 +64,8 @@ function getHashKeySpace(entityManager, entityToken, timestampFrom = 0, timestam
43
64
  if (error instanceof Error)
44
65
  entityManager.logger.error(error.message, {
45
66
  entityToken,
67
+ hashKeyToken,
68
+ item,
46
69
  timestampFrom,
47
70
  timestampTo,
48
71
  });
package/dist/cjs/query.js CHANGED
@@ -5,7 +5,6 @@ var lzString = require('lz-string');
5
5
  var radash = require('radash');
6
6
  var dehydratePageKeyMap = require('./dehydratePageKeyMap.js');
7
7
  var rehydratePageKeyMap = require('./rehydratePageKeyMap.js');
8
- var validateEntityGeneratedProperty = require('./validateEntityGeneratedProperty.js');
9
8
 
10
9
  const { compressToEncodedURIComponent, decompressFromEncodedURIComponent } = lzString;
11
10
  /**
@@ -27,20 +26,19 @@ const { compressToEncodedURIComponent, decompressFromEncodedURIComponent } = lzS
27
26
  *
28
27
  * @throws Error if {@link QueryOptions.pageKeyMap | `pageKeyMap`} keys do not match {@link QueryOptions.shardQueryMap | `shardQueryMap`} keys.
29
28
  */
30
- async function query(entityManager, { entityToken, hashKeyToken, limit, pageKeyMap, pageSize, shardQueryMap, sortOrder = [], timestampFrom = 0, timestampTo = Date.now(), throttle = entityManager.config.throttle, }) {
29
+ async function query(entityManager, options) {
31
30
  try {
32
31
  // Get defaults.
33
- const { defaultLimit, defaultPageSize } = entityManager.config.entities[entityToken];
34
- limit ?? (limit = defaultLimit);
35
- pageSize ?? (pageSize = defaultPageSize);
32
+ const { defaultLimit, defaultPageSize } = entityManager.config.entities[options.entityToken];
33
+ // Extract params.
34
+ const { entityToken, limit = defaultLimit, item, pageKeyMap, pageSize = defaultPageSize, shardQueryMap, sortOrder = [], timestampFrom = 0, timestampTo = Date.now(), throttle = entityManager.config.throttle, } = options;
36
35
  // Validate params.
37
- validateEntityGeneratedProperty.validateEntityGeneratedProperty(entityManager, entityToken, hashKeyToken, true);
38
36
  if (!(limit === Infinity || (radash.isInt(limit) && limit >= 1)))
39
37
  throw new Error('limit must be a positive integer or Infinity.');
40
38
  if (!(radash.isInt(pageSize) && pageSize >= 1))
41
39
  throw new Error('pageSize must be a positive integer');
42
40
  // Rehydrate pageKeyMap.
43
- const rehydratedPageKeyMap = rehydratePageKeyMap.rehydratePageKeyMap(entityManager, entityToken, Object.keys(shardQueryMap), pageKeyMap
41
+ const [hashKeyToken, rehydratedPageKeyMap] = rehydratePageKeyMap.rehydratePageKeyMap(entityManager, entityToken, Object.keys(shardQueryMap), item, pageKeyMap
44
42
  ? JSON.parse(decompressFromEncodedURIComponent(pageKeyMap))
45
43
  : undefined, timestampFrom, timestampTo);
46
44
  // Shortcut if pageKeyMap is empty.
@@ -61,18 +59,18 @@ async function query(entityManager, { entityToken, hashKeyToken, limit, pageKeyM
61
59
  // iteration in order to keep shardsQueried * pageSize > (limit - items.length) but only just.
62
60
  // TODO: Test for invalid characters (path delimiters) in index keys & shard key values.
63
61
  // Query every shard on every index in pageKeyMap.
64
- const shardQueryResults = await radash.parallel(throttle, Object.entries(rehydratedPageKeyMap).flatMap(([index, indexPageKeys]) => Object.entries(indexPageKeys).map(([hashKey, pageKey]) => [
65
- index,
62
+ const shardQueryResults = await radash.parallel(throttle, Object.entries(rehydratedPageKeyMap).flatMap(([indexToken, indexPageKeys]) => Object.entries(indexPageKeys).map(([hashKey, pageKey]) => [
63
+ indexToken,
66
64
  hashKey,
67
65
  pageKey,
68
- ])), async ([index, hashKey, pageKey]) => ({
69
- index,
70
- queryResult: await shardQueryMap[index](hashKey, pageKey, pageSize),
66
+ ])), async ([indexToken, hashKey, pageKey]) => ({
67
+ indexToken,
68
+ queryResult: await shardQueryMap[indexToken](hashKey, pageKey, pageSize),
71
69
  hashKey,
72
70
  }));
73
71
  // Reduce shardQueryResults & updateworkingRresult.
74
- workingResult = shardQueryResults.reduce(({ items, pageKeyMap }, { index, queryResult, hashKey }) => {
75
- Object.assign(rehydratedPageKeyMap[index], {
72
+ workingResult = shardQueryResults.reduce(({ items, pageKeyMap }, { indexToken, queryResult, hashKey }) => {
73
+ Object.assign(rehydratedPageKeyMap[indexToken], {
76
74
  [hashKey]: queryResult.pageKey,
77
75
  });
78
76
  return {
@@ -93,15 +91,8 @@ async function query(entityManager, { entityToken, hashKeyToken, limit, pageKeyM
93
91
  pageKeyMap: compressToEncodedURIComponent(JSON.stringify(dehydratePageKeyMap.dehydratePageKeyMap(entityManager, entityToken, workingResult.pageKeyMap))),
94
92
  };
95
93
  entityManager.logger.debug('queried entityToken across shards', {
96
- entityToken,
94
+ options,
97
95
  hashKeyToken,
98
- limit,
99
- pageKeyMap,
100
- pageSize,
101
- shardQueryMap,
102
- timestampFrom,
103
- timestampTo,
104
- throttle,
105
96
  rehydratedPageKeyMap,
106
97
  workingResult,
107
98
  result,
@@ -110,17 +101,7 @@ async function query(entityManager, { entityToken, hashKeyToken, limit, pageKeyM
110
101
  }
111
102
  catch (error) {
112
103
  if (error instanceof Error)
113
- entityManager.logger.error(error.message, {
114
- entityToken,
115
- hashKeyToken,
116
- limit,
117
- pageKeyMap,
118
- pageSize,
119
- shardQueryMap,
120
- timestampFrom,
121
- timestampTo,
122
- throttle,
123
- });
104
+ entityManager.logger.error(error.message, options);
124
105
  throw error;
125
106
  }
126
107
  }
@@ -17,20 +17,22 @@ var validateEntityIndexToken = require('./validateEntityIndexToken.js');
17
17
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
18
18
  * @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
19
19
  * @param dehydrated - Dehydrated index value.
20
- * @param omit - Array of index components omitted from `dehydrated`.
21
20
  *
22
21
  * @returns Partial {@link ItemMap | `ItemMap`} object containing rehydrated index component elements.
23
22
  *
24
23
  * @throws `Error` if `entityToken` is invalid.
25
24
  * @throws `Error` if `indexToken` is invalid.
26
25
  */
27
- function rehydrateIndexItem(entityManager, entityToken, indexToken, dehydrated, omit = []) {
26
+ function rehydrateIndexItem(entityManager, entityToken, indexToken, dehydrated) {
28
27
  try {
29
28
  const { generatedKeyDelimiter } = entityManager.config;
30
29
  // Validate params.
31
30
  validateEntityIndexToken.validateEntityIndexToken(entityManager, entityToken, indexToken);
32
31
  // Unwrap index elements.
33
- const elements = unwrapIndex.unwrapIndex(entityManager, entityToken, indexToken).filter((element) => !omit.includes(element));
32
+ const { hashKey } = entityManager.config.entities[entityToken].indexes[indexToken];
33
+ const elements = unwrapIndex.unwrapIndex(entityManager, entityToken, indexToken, [
34
+ hashKey,
35
+ ]);
34
36
  // Split dehydrated value & validate.
35
37
  const values = dehydrated.split(generatedKeyDelimiter);
36
38
  if (elements.length !== values.length)
@@ -1,6 +1,7 @@
1
1
  'use strict';
2
2
 
3
3
  var radash = require('radash');
4
+ var decodeGeneratedProperty = require('./decodeGeneratedProperty.js');
4
5
  var encodeGeneratedProperty = require('./encodeGeneratedProperty.js');
5
6
  var getHashKeySpace = require('./getHashKeySpace.js');
6
7
  var getIndexComponents = require('./getIndexComponents.js');
@@ -16,28 +17,40 @@ var validateEntityIndexToken = require('./validateEntityIndexToken.js');
16
17
  * @param entityManager - {@link EntityManager | `EntityManager`} instance.
17
18
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
18
19
  * @param indexTokens - Array of {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} keys used as keys of the original {@link PageKeyMap | `PageKeyMap`}.
20
+ * @param item - Partial item object sufficiently populated to generate index hash keys.
19
21
  * @param dehydrated - Array of dehydrated page keys or undefined if new query.
20
22
  * @param timestampFrom - Lower timestanp limit used to generate the original {@link PageKeyMap | `PageKeyMap`}. Defaults to `0`.
21
23
  * @param timestampTo - Upper timestamp limit used to generate the original {@link PageKeyMap | `PageKeyMap`}. Defaults to `Date.now()`.
22
24
  *
23
- * @returns Rehydrated {@link PageKeyMap | `PageKeyMap`} object.
25
+ * @returns A tuple of `hashKeyToken` and rehydrated {@link PageKeyMap | `PageKeyMap`} object.
24
26
  *
25
27
  * @throws `Error` if `entityToken` is invalid.
26
28
  * @throws `Error` if `indexTokens` is empty.
27
29
  * @throws `Error` if any `indexTokens` are invalid.
30
+ * @throws `Error` if `indexTokens` represent indexes with inconsistent hashKeys.
28
31
  * @throws `Error` if `dehydrated` has invalid length.
29
32
  */
30
- function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated, timestampFrom = 0, timestampTo = Date.now()) {
33
+ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, item, dehydrated, timestampFrom = 0, timestampTo = Date.now()) {
31
34
  try {
32
- // Validate params.
35
+ // Validate indexTokens populated.
33
36
  if (!indexTokens.length)
34
37
  throw new Error('indexTokens empty');
38
+ // Validate indexTokens exist.
39
+ const hashKeys = radash.unique(indexTokens.map((indexToken) => {
40
+ validateEntityIndexToken.validateEntityIndexToken(entityManager, entityToken, indexToken);
41
+ return entityManager.config.entities[entityToken].indexes[indexToken]
42
+ .hashKey;
43
+ }));
44
+ // Validate hashKeys consistent.
45
+ if (hashKeys.length > 1)
46
+ throw new Error('inconsistent hashKeys');
47
+ const [hashKeyToken] = hashKeys;
35
48
  indexTokens.map((index) => validateEntityIndexToken.validateEntityIndexToken(entityManager, entityToken, index));
36
49
  // Shortcut empty dehydrated.
37
50
  if (dehydrated && !dehydrated.length)
38
- return {};
51
+ return [hashKeyToken, {}];
39
52
  // Get hash key space.
40
- const hashKeySpace = getHashKeySpace.getHashKeySpace(entityManager, entityToken, timestampFrom, timestampTo);
53
+ const hashKeySpace = getHashKeySpace.getHashKeySpace(entityManager, entityToken, hashKeyToken, item, timestampFrom, timestampTo);
41
54
  // Default dehydrated.
42
55
  dehydrated ?? (dehydrated = [...radash.range(1, hashKeySpace.length * indexTokens.length, '')]);
43
56
  // Validate dehydrated length
@@ -47,14 +60,14 @@ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated
47
60
  const rehydrated = radash.mapValues(radash.zipToObject(indexTokens, radash.cluster(dehydrated, hashKeySpace.length)), (dehydratedIndexPageKeyMaps, index) => radash.zipToObject(hashKeySpace, (hashKey, i) => {
48
61
  if (!dehydratedIndexPageKeyMaps[i])
49
62
  return;
50
- let item = {
51
- [entityManager.config.hashKey]: hashKey,
52
- ...rehydrateIndexItem.rehydrateIndexItem(entityManager, entityToken, index, dehydratedIndexPageKeyMaps[i], [entityManager.config.hashKey]),
63
+ let pageKeyItem = {
64
+ ...decodeGeneratedProperty.decodeGeneratedProperty(entityManager, entityToken, hashKey),
65
+ ...rehydrateIndexItem.rehydrateIndexItem(entityManager, entityToken, index, dehydratedIndexPageKeyMaps[i]),
53
66
  };
54
- item = updateItemRangeKey.updateItemRangeKey(entityManager, entityToken, item);
67
+ pageKeyItem = updateItemRangeKey.updateItemRangeKey(entityManager, entityToken, pageKeyItem);
55
68
  return radash.zipToObject(getIndexComponents.getIndexComponents(entityManager, entityToken, index), (component) => entityManager.config.entities[entityToken].generated[component]
56
- ? encodeGeneratedProperty.encodeGeneratedProperty(entityManager, entityToken, component, item)
57
- : item[component]);
69
+ ? encodeGeneratedProperty.encodeGeneratedProperty(entityManager, entityToken, component, pageKeyItem)
70
+ : pageKeyItem[component]);
58
71
  }));
59
72
  entityManager.logger.debug('rehydrated page key map', {
60
73
  entityToken,
@@ -62,7 +75,7 @@ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated
62
75
  dehydrated,
63
76
  rehydrated,
64
77
  });
65
- return rehydrated;
78
+ return [hashKeyToken, rehydrated];
66
79
  }
67
80
  catch (error) {
68
81
  if (error instanceof Error)
@@ -10,28 +10,30 @@ var validateEntityIndexToken = require('./validateEntityIndexToken.js');
10
10
  * @param entityManager - {@link EntityManager | `EntityManager`} instance.
11
11
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
12
12
  * @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
13
+ * @param omit - Array of index components or elements to omit from the output value.
13
14
  *
14
15
  * @returns Deduped, sorted array of ungenerated index component elements.
15
16
  *
16
17
  * @throws `Error` if `entityToken` is invalid.
17
18
  * @throws `Error` if `indexToken` is invalid.
18
19
  */
19
- function unwrapIndex(entityManager, entityToken, indexToken) {
20
+ function unwrapIndex(entityManager, entityToken, indexToken, omit = []) {
20
21
  try {
21
22
  // Validate params.
22
23
  validateEntityIndexToken.validateEntityIndexToken(entityManager, entityToken, indexToken);
23
24
  const generated = entityManager.config.entities[entityToken].generated;
24
25
  const generatedKeys = Object.keys(radash.shake(generated));
25
- return getIndexComponents.getIndexComponents(entityManager, entityToken, indexToken)
26
+ return radash.unique(getIndexComponents.getIndexComponents(entityManager, entityToken, indexToken)
27
+ .filter((component) => !omit.includes(component))
26
28
  .map((component) => component === entityManager.config.hashKey
27
- ? entityManager.config.hashKey
29
+ ? entityManager.config.entities[entityToken].timestampProperty
28
30
  : component === entityManager.config.rangeKey
29
31
  ? entityManager.config.entities[entityToken].uniqueProperty
30
32
  : generatedKeys.includes(component)
31
33
  ? generated[component].elements
32
34
  : component)
33
35
  .flat()
34
- .sort();
36
+ .filter((element) => !omit.includes(element))).sort();
35
37
  }
36
38
  catch (error) {
37
39
  if (error instanceof Error)
package/dist/index.d.ts CHANGED
@@ -363,6 +363,137 @@ type ItemMap<M extends EntityMap, HashKey extends string = 'hashKey', RangeKey e
363
363
  } & Partial<Record<HashKey | RangeKey, string>>>;
364
364
  };
365
365
 
366
+ /**
367
+ * A result returned by a query across multiple shards, where each shard may
368
+ * receive multiple page queries via a dynamically-generated {@link ShardQueryFunction | `ShardQueryFunction`}.
369
+ *
370
+ * @category Query
371
+ */
372
+ interface QueryResult<Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string, M extends EntityMap, HashKey extends string, RangeKey extends string> {
373
+ /** Total number of records returned across all shards. */
374
+ count: number;
375
+ /** The returned records. */
376
+ items: Item[];
377
+ /**
378
+ * A compressed, two-layer map of page keys, used to query the next page of
379
+ * data for a given sort key on each shard of a given hash key.
380
+ */
381
+ pageKeyMap: string;
382
+ }
383
+
384
+ /**
385
+ * A result returned by a {@link ShardQueryFunction | `ShardQueryFunction`} querying an individual shard.
386
+ *
387
+ * @typeParam Item - The {@link Item | `Item`} type being queried.
388
+
389
+ * @category Query
390
+ */
391
+ interface ShardQueryResult<Item extends Entity> {
392
+ /** The number of records returned. */
393
+ count: number;
394
+ /** The returned records. */
395
+ items: Item[];
396
+ /** The page key for the next query on this shard. */
397
+ pageKey?: Partial<Item>;
398
+ }
399
+
400
+ /**
401
+ * A query function that returns a single page of results from an individual
402
+ * shard. This function will typically be composed dynamically to express a
403
+ * specific query index & logic. The arguments to this function will be
404
+ * provided by the {@link EntityManager.query | `EntityManager.query`} method, which assembles many returned
405
+ * pages queried across multiple shards into a single query result.
406
+ *
407
+ * @typeParam Item - The {@link Item | `Item`} type being queried.
408
+
409
+ * @param hashKey - The hash key value of the shard being queried.
410
+ * @param pageKey - The page key returned by the previous query on this shard.
411
+ * @param pageSize - The maximum number of items to return from this query.
412
+ *
413
+ * @category Query
414
+ */
415
+ type ShardQueryFunction<Item extends Entity> = (hashKey: string, pageKey?: Partial<Item>, pageSize?: number) => Promise<ShardQueryResult<Item>>;
416
+
417
+ type ShardQueryMap<Item extends Entity> = Record<string, ShardQueryFunction<Item>>;
418
+
419
+ /**
420
+ * Options passed to the {@link query | `query`} function.
421
+ *
422
+ * @category Query
423
+ */
424
+ interface QueryOptions<Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string, M extends EntityMap, HashKey extends string, RangeKey extends string> {
425
+ /** Identifies the entity to be queried. Key of {@link Config | `EntityManager.config.entities`}. */
426
+ entityToken: EntityToken;
427
+ /**
428
+ * Partial item object sufficiently populated to generate index hash keys.
429
+ */
430
+ item: Partial<Item>;
431
+ /**
432
+ * The target maximum number of records to be returned by the query across
433
+ * all shards.
434
+ *
435
+ * The actual number of records returned will be a product of {@link QueryOptions.pageSize | `pageSize`} and the
436
+ * number of shards queried, unless limited by available records in a given
437
+ * shard.
438
+ */
439
+ limit?: number;
440
+ /**
441
+ * {@link QueryResult.pageKeyMap | `pageKeyMap`} returned by the previous iteration of this query.
442
+ */
443
+ pageKeyMap?: string;
444
+ /**
445
+ * The maximum number of records to be returned by each individual query to a
446
+ * single shard (i.e. {@link ShardQueryFunction | `ShardQueryFunction`} execution).
447
+ *
448
+ * Note that, within a given {@link EntityManager.query | `query`} method execution, these queries will be
449
+ * repeated until either available data is exhausted or the {@link QueryOptions.limit | `limit`} value is
450
+ * reached.
451
+ */
452
+ pageSize?: number;
453
+ /**
454
+ * Each key in this object is a valid entity index token. Each value is a valid
455
+ * {@link ShardQueryFunction | 'ShardQueryFunction'} that specifies the query of a single page of data on a
456
+ * single shard for the mapped index.
457
+ *
458
+ * This allows simultaneous queries on multiple sort keys to share a single
459
+ * page key, e.g. to match the same string against `firstName` and `lastName`
460
+ * properties without performing a table scan for either.
461
+ */
462
+ shardQueryMap: ShardQueryMap<Item>;
463
+ /**
464
+ * A {@link SortOrder | `SortOrder`} object specifying the sort order of the result set. Defaults to `[]`.
465
+ */
466
+ sortOrder?: SortOrder<Item>;
467
+ /**
468
+ * Lower limit to query shard space.
469
+ *
470
+ * Only valid if the query is constrained along the dimension used by the
471
+ * {@link Config | `EntityManager.config.entities.<entityToken>.sharding.timestamptokens.timestamp`}
472
+ * function to generate `shardKey`.
473
+ *
474
+ * @defaultValue `0`
475
+ */
476
+ timestampFrom?: number;
477
+ /**
478
+ * Upper limit to query shard space.
479
+ *
480
+ * Only valid if the query is constrained along the dimension used by the
481
+ * {@link Config | `EntityManager.config.entities.<entityToken>.sharding.timestamptokens.timestamp`}
482
+ * function to generate `shardKey`.
483
+ *
484
+ * @defaultValue `Date.now()`
485
+ */
486
+ timestampTo?: number;
487
+ /**
488
+ * The maximum number of shards to query in parallel. Overrides options `throttle`.
489
+ *
490
+ * @defaultValue `options.throttle`
491
+ */
492
+ throttle?: number;
493
+ }
494
+
495
+ type BuilderQueryOptions<Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string, M extends EntityMap, HashKey extends string, RangeKey extends string> = Omit<QueryOptions<Item, EntityToken, M, HashKey, RangeKey>, 'entityToken' | 'pageKeyMap' | 'shardQueryMap'>;
496
+
366
497
  declare const configSchema: z.ZodEffects<z.ZodObject<{
367
498
  entities: z.ZodDefault<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodEffects<z.ZodObject<{
368
499
  defaultLimit: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
@@ -674,90 +805,6 @@ declare const configSchema: z.ZodEffects<z.ZodObject<{
674
805
  */
675
806
  type ParsedConfig = z.infer<typeof configSchema>;
676
807
 
677
- /**
678
- * Base type of options passed to the {@link EntityManager.query | `EntityManager.query`} method.
679
- *
680
- * @category Query
681
- */
682
- interface BaseQueryOptions<Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string, M extends EntityMap, HashKey extends string, RangeKey extends string> {
683
- /**
684
- * The target maximum number of records to be returned by the query across
685
- * all shards.
686
- *
687
- * The actual number of records returned will be a product of {@link QueryOptions.pageSize | `pageSize`} and the
688
- * number of shards queried, unless limited by available records in a given
689
- * shard.
690
- */
691
- limit?: number;
692
- /**
693
- * The maximum number of records to be returned by each individual query to a
694
- * single shard (i.e. {@link ShardQueryFunction | `ShardQueryFunction`} execution).
695
- *
696
- * Note that, within a given {@link EntityManager.query | `query`} method execution, these queries will be
697
- * repeated until either available data is exhausted or the {@link QueryOptions.limit | `limit`} value is
698
- * reached.
699
- */
700
- pageSize?: number;
701
- /**
702
- * A {@link SortOrder | `SortOrder`} object specifying the sort order of the result set. Defaults to `[]`.
703
- */
704
- sortOrder?: SortOrder<Item>;
705
- /**
706
- * Lower limit to query shard space.
707
- *
708
- * Only valid if the query is constrained along the dimension used by the
709
- * {@link Config | `EntityManager.config.entities.<entityToken>.sharding.timestamptokens.timestamp`}
710
- * function to generate `shardKey`.
711
- *
712
- * @defaultValue `0`
713
- */
714
- timestampFrom?: number;
715
- /**
716
- * Upper limit to query shard space.
717
- *
718
- * Only valid if the query is constrained along the dimension used by the
719
- * {@link Config | `EntityManager.config.entities.<entityToken>.sharding.timestamptokens.timestamp`}
720
- * function to generate `shardKey`.
721
- *
722
- * @defaultValue `Date.now()`
723
- */
724
- timestampTo?: number;
725
- /**
726
- * The maximum number of shards to query in parallel. Overrides options `throttle`.
727
- *
728
- * @defaultValue `options.throttle`
729
- */
730
- throttle?: number;
731
- }
732
-
733
- /**
734
- * Options passed to the {@link EntityManager.query | `EntityManager.query`} method.
735
- *
736
- * @category Query
737
- */
738
- interface QueryOptions<IndexParams, Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string, M extends EntityMap, HashKey extends string, RangeKey extends string, T extends TranscodeMap> extends BaseQueryOptions<Item, EntityToken, M, HashKey, RangeKey> {
739
- /** Instance of class extending {@link BaseShardQueryMapBuilder | `BaseShardQueryMapBuilder`}. */
740
- shardQueryMapBuilder: BaseShardQueryMapBuilder<IndexParams, Item, EntityToken, M, HashKey, RangeKey, T>;
741
- }
742
-
743
- /**
744
- * A result returned by a query across multiple shards, where each shard may
745
- * receive multiple page queries via a dynamically-generated {@link ShardQueryFunction | `ShardQueryFunction`}.
746
- *
747
- * @category Query
748
- */
749
- interface QueryResult<Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string, M extends EntityMap, HashKey extends string, RangeKey extends string> {
750
- /** Total number of records returned across all shards. */
751
- count: number;
752
- /** The returned records. */
753
- items: Item[];
754
- /**
755
- * A compressed, two-layer map of page keys, used to query the next page of
756
- * data for a given sort key on each shard of a given hash key.
757
- */
758
- pageKeyMap: string;
759
- }
760
-
761
808
  /**
762
809
  * The EntityManager class applies a configuration-driven sharded data model &
763
810
  * query strategy to NoSql data.
@@ -827,44 +874,9 @@ declare class EntityManager<M extends EntityMap, HashKey extends string, RangeKe
827
874
  *
828
875
  * @throws Error if {@link QueryOptions.shardQueryMapBuilder | `shardQueryMapBuilder`} `pageKeyMap` keys do not match its `shardQueryMap` keys.
829
876
  */
830
- query<IndexParams, Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string>(options: QueryOptions<IndexParams, Item, EntityToken, M, HashKey, RangeKey, T>): Promise<QueryResult<Item, EntityToken, M, HashKey, RangeKey>>;
877
+ query<Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string>(options: QueryOptions<Item, EntityToken, M, HashKey, RangeKey>): Promise<QueryResult<Item, EntityToken, M, HashKey, RangeKey>>;
831
878
  }
832
879
 
833
- /**
834
- * A result returned by a {@link ShardQueryFunction | `ShardQueryFunction`} querying an individual shard.
835
- *
836
- * @typeParam Item - The {@link Item | `Item`} type being queried.
837
-
838
- * @category Query
839
- */
840
- interface ShardQueryResult<Item extends Entity> {
841
- /** The number of records returned. */
842
- count: number;
843
- /** The returned records. */
844
- items: Item[];
845
- /** The page key for the next query on this shard. */
846
- pageKey?: Partial<Item>;
847
- }
848
-
849
- /**
850
- * A query function that returns a single page of results from an individual
851
- * shard. This function will typically be composed dynamically to express a
852
- * specific query index & logic. The arguments to this function will be
853
- * provided by the {@link EntityManager.query | `EntityManager.query`} method, which assembles many returned
854
- * pages queried across multiple shards into a single query result.
855
- *
856
- * @typeParam Item - The {@link Item | `Item`} type being queried.
857
-
858
- * @param hashKey - The hash key value of the shard being queried.
859
- * @param pageKey - The page key returned by the previous query on this shard.
860
- * @param pageSize - The maximum number of items to return from this query.
861
- *
862
- * @category Query
863
- */
864
- type ShardQueryFunction<Item extends Entity> = (hashKey: string, pageKey?: Partial<Item>, pageSize?: number) => Promise<ShardQueryResult<Item>>;
865
-
866
- type ShardQueryMap<Item extends Entity> = Record<string, ShardQueryFunction<Item>>;
867
-
868
880
  /**
869
881
  * Abstract base class supporting a fluent API for building a {@link ShardQueryMap | `ShardQueryMap`} using a database client.
870
882
  *
@@ -890,6 +902,7 @@ declare abstract class BaseShardQueryMapBuilder<IndexParams, Item extends ItemMa
890
902
  * @returns - The {@link ShardQueryMap | `ShardQueryMap`} object.
891
903
  */
892
904
  build(): ShardQueryMap<Item>;
905
+ query(options: BuilderQueryOptions<Item, EntityToken, M, HashKey, RangeKey>): Promise<QueryResult<Item, EntityToken, M, HashKey, RangeKey>>;
893
906
  }
894
907
 
895
908
  /**
@@ -27,6 +27,16 @@ class BaseShardQueryMapBuilder {
27
27
  build() {
28
28
  return mapValues(this.indexParamsMap, (indexConfig, indexToken) => this.getShardQueryFunction(indexToken));
29
29
  }
30
+ async query(options) {
31
+ const { entityManager, entityToken, pageKeyMap } = this;
32
+ const shardQueryMap = this.build();
33
+ return await entityManager.query({
34
+ ...options,
35
+ entityToken,
36
+ pageKeyMap,
37
+ shardQueryMap,
38
+ });
39
+ }
30
40
  }
31
41
 
32
42
  export { BaseShardQueryMapBuilder };
@@ -85,16 +85,7 @@ class EntityManager {
85
85
  * @throws Error if {@link QueryOptions.shardQueryMapBuilder | `shardQueryMapBuilder`} `pageKeyMap` keys do not match its `shardQueryMap` keys.
86
86
  */
87
87
  async query(options) {
88
- const { shardQueryMapBuilder, ...baseOptions } = options;
89
- const { entityToken, hashKeyToken, pageKeyMap } = shardQueryMapBuilder;
90
- const shardQueryMap = shardQueryMapBuilder.build();
91
- return await query(this, {
92
- entityToken,
93
- hashKeyToken,
94
- pageKeyMap,
95
- shardQueryMap,
96
- ...baseOptions,
97
- });
88
+ return await query(this, options);
98
89
  }
99
90
  }
100
91
  _EntityManager_config = new WeakMap();
@@ -19,14 +19,13 @@ import { validateEntityIndexToken } from './validateEntityIndexToken.js';
19
19
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
20
20
  * @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
21
21
  * @param item - Partial {@link ItemMap | `ItemMap`} object.
22
- * @param omit - Array of index components to omit from the output value.
23
22
  *
24
23
  * @returns Dehydrated index value.
25
24
  *
26
25
  * @throws `Error` if `entityToken` is invalid.
27
26
  * @throws `Error` if `indexToken` is invalid.
28
27
  */
29
- function dehydrateIndexItem(entityManager, entityToken, indexToken, item, omit = []) {
28
+ function dehydrateIndexItem(entityManager, entityToken, indexToken, item) {
30
29
  try {
31
30
  const { generatedKeyDelimiter } = entityManager.config;
32
31
  // Validate params.
@@ -35,7 +34,10 @@ function dehydrateIndexItem(entityManager, entityToken, indexToken, item, omit =
35
34
  if (!item)
36
35
  return '';
37
36
  // Unwrap index elements.
38
- const elements = unwrapIndex(entityManager, entityToken, indexToken).filter((element) => !omit.includes(element));
37
+ const { hashKey } = entityManager.config.entities[entityToken].indexes[indexToken];
38
+ const elements = unwrapIndex(entityManager, entityToken, indexToken, [
39
+ hashKey,
40
+ ]);
39
41
  // Join index element values.
40
42
  const dehydrated = elements
41
43
  .map((element) => encodeEntityElement(entityManager, entityToken, element, item))
@@ -45,7 +45,7 @@ function dehydratePageKeyMap(entityManager, entityToken, pageKeyMap) {
45
45
  let dehydrated = [];
46
46
  for (const index of indexes) {
47
47
  for (const hashKey of hashKeys) {
48
- // Undefineed pageKey.
48
+ // Undefined pageKey.
49
49
  if (!pageKeyMap[index][hashKey]) {
50
50
  dehydrated.push('');
51
51
  continue;
@@ -60,9 +60,7 @@ function dehydratePageKeyMap(entityManager, entityToken, pageKeyMap) {
60
60
  return item;
61
61
  }, {});
62
62
  // Dehydrate index from item.
63
- dehydrated.push(dehydrateIndexItem(entityManager, entityToken, index, item, [
64
- entityManager.config.hashKey,
65
- ]));
63
+ dehydrated.push(dehydrateIndexItem(entityManager, entityToken, index, item));
66
64
  }
67
65
  }
68
66
  // Replace with empty array if all pageKeys are empty strings.
@@ -1,11 +1,14 @@
1
1
  import { range } from 'radash';
2
- import { validateEntityToken } from './validateEntityToken.js';
2
+ import { encodeGeneratedProperty } from './encodeGeneratedProperty.js';
3
+ import { validateEntityGeneratedProperty } from './validateEntityGeneratedProperty.js';
3
4
 
4
5
  /**
5
6
  * Return an array of {@link ConfigKeys.hashKey | `entityManager.config.hashKey`} property values covering the shard space bounded by `timestampFrom` & `timestampTo`.
6
7
  *
7
8
  * @param entityManager - {@link EntityManager | `EntityManager`} instance.
8
9
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
10
+ * @param hashKeyToken - {@link ParsedConfig | `entityManager.config.hashKey`} or {@link ParsedConfig | `entityManager.config.entities.<entityToken>.generated`} key. If a generated property, must be shardable.
11
+ * @param item - Partial {@link ItemMap | `ItemMap[EntityToken]`} object. Must include all properties required to generate the hash key space.
9
12
  * @param timestampFrom - Lower timestanp limit. Defaults to `0`.
10
13
  * @param timestampTo - Upper timestamp limit. Defaults to `Date.now()`.
11
14
  *
@@ -13,24 +16,42 @@ import { validateEntityToken } from './validateEntityToken.js';
13
16
  *
14
17
  * @throws `Error` if `entityToken` is invalid.
15
18
  */
16
- function getHashKeySpace(entityManager, entityToken, timestampFrom = 0, timestampTo = Date.now()) {
19
+ function getHashKeySpace(entityManager, entityToken, hashKeyToken, item, timestampFrom = 0, timestampTo = Date.now()) {
17
20
  try {
18
- // Validate params.
19
- validateEntityToken(entityManager, entityToken);
21
+ // Validate hashKeyToken is either the global hash key or a sharded generated property.
22
+ if (hashKeyToken !== entityManager.config.hashKey)
23
+ validateEntityGeneratedProperty(entityManager, entityToken, hashKeyToken, true);
20
24
  const { shardBumps } = entityManager.config.entities[entityToken];
21
25
  const hashKeySpace = shardBumps
26
+ // Filter shard bumps by timestamp range.
22
27
  .filter((bump, i) => (i === shardBumps.length - 1 ||
23
28
  shardBumps[i + 1].timestamp > timestampFrom) &&
24
29
  bump.timestamp <= timestampTo)
30
+ // Generate shard key space.
25
31
  .flatMap(({ charBits, chars }) => {
26
32
  const radix = 2 ** charBits;
27
33
  return chars
28
34
  ? [...range(0, radix ** chars - 1)].map((char) => char.toString(radix).padStart(chars, '0'))
29
35
  : '';
30
36
  })
31
- .map((shardKey) => `${entityToken}${entityManager.config.shardKeyDelimiter}${shardKey}`);
37
+ // Map shard keys to hash keys.
38
+ .map((shardKey) => {
39
+ // Calculate record hash key.
40
+ let hashKey = `${entityToken}${entityManager.config.shardKeyDelimiter}${shardKey}`;
41
+ // If hash key space basis is a different property, encode it.
42
+ if (hashKeyToken !== entityManager.config.hashKey)
43
+ hashKey = encodeGeneratedProperty(entityManager, entityToken, hashKeyToken, {
44
+ ...item,
45
+ [entityManager.config.hashKey]: hashKey,
46
+ });
47
+ if (!hashKey)
48
+ throw new Error('item does not support hash key space');
49
+ return hashKey;
50
+ });
32
51
  entityManager.logger.debug('generated hash key space', {
33
52
  entityToken,
53
+ hashKeyToken,
54
+ item,
34
55
  timestampFrom,
35
56
  timestampTo,
36
57
  hashKeySpace,
@@ -41,6 +62,8 @@ function getHashKeySpace(entityManager, entityToken, timestampFrom = 0, timestam
41
62
  if (error instanceof Error)
42
63
  entityManager.logger.error(error.message, {
43
64
  entityToken,
65
+ hashKeyToken,
66
+ item,
44
67
  timestampFrom,
45
68
  timestampTo,
46
69
  });
package/dist/mjs/query.js CHANGED
@@ -3,7 +3,6 @@ import lzString from 'lz-string';
3
3
  import { isInt, parallel, unique } from 'radash';
4
4
  import { dehydratePageKeyMap } from './dehydratePageKeyMap.js';
5
5
  import { rehydratePageKeyMap } from './rehydratePageKeyMap.js';
6
- import { validateEntityGeneratedProperty } from './validateEntityGeneratedProperty.js';
7
6
 
8
7
  const { compressToEncodedURIComponent, decompressFromEncodedURIComponent } = lzString;
9
8
  /**
@@ -25,20 +24,19 @@ const { compressToEncodedURIComponent, decompressFromEncodedURIComponent } = lzS
25
24
  *
26
25
  * @throws Error if {@link QueryOptions.pageKeyMap | `pageKeyMap`} keys do not match {@link QueryOptions.shardQueryMap | `shardQueryMap`} keys.
27
26
  */
28
- async function query(entityManager, { entityToken, hashKeyToken, limit, pageKeyMap, pageSize, shardQueryMap, sortOrder = [], timestampFrom = 0, timestampTo = Date.now(), throttle = entityManager.config.throttle, }) {
27
+ async function query(entityManager, options) {
29
28
  try {
30
29
  // Get defaults.
31
- const { defaultLimit, defaultPageSize } = entityManager.config.entities[entityToken];
32
- limit ?? (limit = defaultLimit);
33
- pageSize ?? (pageSize = defaultPageSize);
30
+ const { defaultLimit, defaultPageSize } = entityManager.config.entities[options.entityToken];
31
+ // Extract params.
32
+ const { entityToken, limit = defaultLimit, item, pageKeyMap, pageSize = defaultPageSize, shardQueryMap, sortOrder = [], timestampFrom = 0, timestampTo = Date.now(), throttle = entityManager.config.throttle, } = options;
34
33
  // Validate params.
35
- validateEntityGeneratedProperty(entityManager, entityToken, hashKeyToken, true);
36
34
  if (!(limit === Infinity || (isInt(limit) && limit >= 1)))
37
35
  throw new Error('limit must be a positive integer or Infinity.');
38
36
  if (!(isInt(pageSize) && pageSize >= 1))
39
37
  throw new Error('pageSize must be a positive integer');
40
38
  // Rehydrate pageKeyMap.
41
- const rehydratedPageKeyMap = rehydratePageKeyMap(entityManager, entityToken, Object.keys(shardQueryMap), pageKeyMap
39
+ const [hashKeyToken, rehydratedPageKeyMap] = rehydratePageKeyMap(entityManager, entityToken, Object.keys(shardQueryMap), item, pageKeyMap
42
40
  ? JSON.parse(decompressFromEncodedURIComponent(pageKeyMap))
43
41
  : undefined, timestampFrom, timestampTo);
44
42
  // Shortcut if pageKeyMap is empty.
@@ -59,18 +57,18 @@ async function query(entityManager, { entityToken, hashKeyToken, limit, pageKeyM
59
57
  // iteration in order to keep shardsQueried * pageSize > (limit - items.length) but only just.
60
58
  // TODO: Test for invalid characters (path delimiters) in index keys & shard key values.
61
59
  // Query every shard on every index in pageKeyMap.
62
- const shardQueryResults = await parallel(throttle, Object.entries(rehydratedPageKeyMap).flatMap(([index, indexPageKeys]) => Object.entries(indexPageKeys).map(([hashKey, pageKey]) => [
63
- index,
60
+ const shardQueryResults = await parallel(throttle, Object.entries(rehydratedPageKeyMap).flatMap(([indexToken, indexPageKeys]) => Object.entries(indexPageKeys).map(([hashKey, pageKey]) => [
61
+ indexToken,
64
62
  hashKey,
65
63
  pageKey,
66
- ])), async ([index, hashKey, pageKey]) => ({
67
- index,
68
- queryResult: await shardQueryMap[index](hashKey, pageKey, pageSize),
64
+ ])), async ([indexToken, hashKey, pageKey]) => ({
65
+ indexToken,
66
+ queryResult: await shardQueryMap[indexToken](hashKey, pageKey, pageSize),
69
67
  hashKey,
70
68
  }));
71
69
  // Reduce shardQueryResults & updateworkingRresult.
72
- workingResult = shardQueryResults.reduce(({ items, pageKeyMap }, { index, queryResult, hashKey }) => {
73
- Object.assign(rehydratedPageKeyMap[index], {
70
+ workingResult = shardQueryResults.reduce(({ items, pageKeyMap }, { indexToken, queryResult, hashKey }) => {
71
+ Object.assign(rehydratedPageKeyMap[indexToken], {
74
72
  [hashKey]: queryResult.pageKey,
75
73
  });
76
74
  return {
@@ -91,15 +89,8 @@ async function query(entityManager, { entityToken, hashKeyToken, limit, pageKeyM
91
89
  pageKeyMap: compressToEncodedURIComponent(JSON.stringify(dehydratePageKeyMap(entityManager, entityToken, workingResult.pageKeyMap))),
92
90
  };
93
91
  entityManager.logger.debug('queried entityToken across shards', {
94
- entityToken,
92
+ options,
95
93
  hashKeyToken,
96
- limit,
97
- pageKeyMap,
98
- pageSize,
99
- shardQueryMap,
100
- timestampFrom,
101
- timestampTo,
102
- throttle,
103
94
  rehydratedPageKeyMap,
104
95
  workingResult,
105
96
  result,
@@ -108,17 +99,7 @@ async function query(entityManager, { entityToken, hashKeyToken, limit, pageKeyM
108
99
  }
109
100
  catch (error) {
110
101
  if (error instanceof Error)
111
- entityManager.logger.error(error.message, {
112
- entityToken,
113
- hashKeyToken,
114
- limit,
115
- pageKeyMap,
116
- pageSize,
117
- shardQueryMap,
118
- timestampFrom,
119
- timestampTo,
120
- throttle,
121
- });
102
+ entityManager.logger.error(error.message, options);
122
103
  throw error;
123
104
  }
124
105
  }
@@ -15,20 +15,22 @@ import { validateEntityIndexToken } from './validateEntityIndexToken.js';
15
15
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
16
16
  * @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
17
17
  * @param dehydrated - Dehydrated index value.
18
- * @param omit - Array of index components omitted from `dehydrated`.
19
18
  *
20
19
  * @returns Partial {@link ItemMap | `ItemMap`} object containing rehydrated index component elements.
21
20
  *
22
21
  * @throws `Error` if `entityToken` is invalid.
23
22
  * @throws `Error` if `indexToken` is invalid.
24
23
  */
25
- function rehydrateIndexItem(entityManager, entityToken, indexToken, dehydrated, omit = []) {
24
+ function rehydrateIndexItem(entityManager, entityToken, indexToken, dehydrated) {
26
25
  try {
27
26
  const { generatedKeyDelimiter } = entityManager.config;
28
27
  // Validate params.
29
28
  validateEntityIndexToken(entityManager, entityToken, indexToken);
30
29
  // Unwrap index elements.
31
- const elements = unwrapIndex(entityManager, entityToken, indexToken).filter((element) => !omit.includes(element));
30
+ const { hashKey } = entityManager.config.entities[entityToken].indexes[indexToken];
31
+ const elements = unwrapIndex(entityManager, entityToken, indexToken, [
32
+ hashKey,
33
+ ]);
32
34
  // Split dehydrated value & validate.
33
35
  const values = dehydrated.split(generatedKeyDelimiter);
34
36
  if (elements.length !== values.length)
@@ -1,4 +1,5 @@
1
- import { range, mapValues, zipToObject, cluster } from 'radash';
1
+ import { unique, range, mapValues, zipToObject, cluster } from 'radash';
2
+ import { decodeGeneratedProperty } from './decodeGeneratedProperty.js';
2
3
  import { encodeGeneratedProperty } from './encodeGeneratedProperty.js';
3
4
  import { getHashKeySpace } from './getHashKeySpace.js';
4
5
  import { getIndexComponents } from './getIndexComponents.js';
@@ -14,28 +15,40 @@ import { validateEntityIndexToken } from './validateEntityIndexToken.js';
14
15
  * @param entityManager - {@link EntityManager | `EntityManager`} instance.
15
16
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
16
17
  * @param indexTokens - Array of {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} keys used as keys of the original {@link PageKeyMap | `PageKeyMap`}.
18
+ * @param item - Partial item object sufficiently populated to generate index hash keys.
17
19
  * @param dehydrated - Array of dehydrated page keys or undefined if new query.
18
20
  * @param timestampFrom - Lower timestanp limit used to generate the original {@link PageKeyMap | `PageKeyMap`}. Defaults to `0`.
19
21
  * @param timestampTo - Upper timestamp limit used to generate the original {@link PageKeyMap | `PageKeyMap`}. Defaults to `Date.now()`.
20
22
  *
21
- * @returns Rehydrated {@link PageKeyMap | `PageKeyMap`} object.
23
+ * @returns A tuple of `hashKeyToken` and rehydrated {@link PageKeyMap | `PageKeyMap`} object.
22
24
  *
23
25
  * @throws `Error` if `entityToken` is invalid.
24
26
  * @throws `Error` if `indexTokens` is empty.
25
27
  * @throws `Error` if any `indexTokens` are invalid.
28
+ * @throws `Error` if `indexTokens` represent indexes with inconsistent hashKeys.
26
29
  * @throws `Error` if `dehydrated` has invalid length.
27
30
  */
28
- function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated, timestampFrom = 0, timestampTo = Date.now()) {
31
+ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, item, dehydrated, timestampFrom = 0, timestampTo = Date.now()) {
29
32
  try {
30
- // Validate params.
33
+ // Validate indexTokens populated.
31
34
  if (!indexTokens.length)
32
35
  throw new Error('indexTokens empty');
36
+ // Validate indexTokens exist.
37
+ const hashKeys = unique(indexTokens.map((indexToken) => {
38
+ validateEntityIndexToken(entityManager, entityToken, indexToken);
39
+ return entityManager.config.entities[entityToken].indexes[indexToken]
40
+ .hashKey;
41
+ }));
42
+ // Validate hashKeys consistent.
43
+ if (hashKeys.length > 1)
44
+ throw new Error('inconsistent hashKeys');
45
+ const [hashKeyToken] = hashKeys;
33
46
  indexTokens.map((index) => validateEntityIndexToken(entityManager, entityToken, index));
34
47
  // Shortcut empty dehydrated.
35
48
  if (dehydrated && !dehydrated.length)
36
- return {};
49
+ return [hashKeyToken, {}];
37
50
  // Get hash key space.
38
- const hashKeySpace = getHashKeySpace(entityManager, entityToken, timestampFrom, timestampTo);
51
+ const hashKeySpace = getHashKeySpace(entityManager, entityToken, hashKeyToken, item, timestampFrom, timestampTo);
39
52
  // Default dehydrated.
40
53
  dehydrated ?? (dehydrated = [...range(1, hashKeySpace.length * indexTokens.length, '')]);
41
54
  // Validate dehydrated length
@@ -45,14 +58,14 @@ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated
45
58
  const rehydrated = mapValues(zipToObject(indexTokens, cluster(dehydrated, hashKeySpace.length)), (dehydratedIndexPageKeyMaps, index) => zipToObject(hashKeySpace, (hashKey, i) => {
46
59
  if (!dehydratedIndexPageKeyMaps[i])
47
60
  return;
48
- let item = {
49
- [entityManager.config.hashKey]: hashKey,
50
- ...rehydrateIndexItem(entityManager, entityToken, index, dehydratedIndexPageKeyMaps[i], [entityManager.config.hashKey]),
61
+ let pageKeyItem = {
62
+ ...decodeGeneratedProperty(entityManager, entityToken, hashKey),
63
+ ...rehydrateIndexItem(entityManager, entityToken, index, dehydratedIndexPageKeyMaps[i]),
51
64
  };
52
- item = updateItemRangeKey(entityManager, entityToken, item);
65
+ pageKeyItem = updateItemRangeKey(entityManager, entityToken, pageKeyItem);
53
66
  return zipToObject(getIndexComponents(entityManager, entityToken, index), (component) => entityManager.config.entities[entityToken].generated[component]
54
- ? encodeGeneratedProperty(entityManager, entityToken, component, item)
55
- : item[component]);
67
+ ? encodeGeneratedProperty(entityManager, entityToken, component, pageKeyItem)
68
+ : pageKeyItem[component]);
56
69
  }));
57
70
  entityManager.logger.debug('rehydrated page key map', {
58
71
  entityToken,
@@ -60,7 +73,7 @@ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated
60
73
  dehydrated,
61
74
  rehydrated,
62
75
  });
63
- return rehydrated;
76
+ return [hashKeyToken, rehydrated];
64
77
  }
65
78
  catch (error) {
66
79
  if (error instanceof Error)
@@ -1,4 +1,4 @@
1
- import { shake } from 'radash';
1
+ import { shake, unique } from 'radash';
2
2
  import { getIndexComponents } from './getIndexComponents.js';
3
3
  import { validateEntityIndexToken } from './validateEntityIndexToken.js';
4
4
 
@@ -8,28 +8,30 @@ import { validateEntityIndexToken } from './validateEntityIndexToken.js';
8
8
  * @param entityManager - {@link EntityManager | `EntityManager`} instance.
9
9
  * @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
10
10
  * @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
11
+ * @param omit - Array of index components or elements to omit from the output value.
11
12
  *
12
13
  * @returns Deduped, sorted array of ungenerated index component elements.
13
14
  *
14
15
  * @throws `Error` if `entityToken` is invalid.
15
16
  * @throws `Error` if `indexToken` is invalid.
16
17
  */
17
- function unwrapIndex(entityManager, entityToken, indexToken) {
18
+ function unwrapIndex(entityManager, entityToken, indexToken, omit = []) {
18
19
  try {
19
20
  // Validate params.
20
21
  validateEntityIndexToken(entityManager, entityToken, indexToken);
21
22
  const generated = entityManager.config.entities[entityToken].generated;
22
23
  const generatedKeys = Object.keys(shake(generated));
23
- return getIndexComponents(entityManager, entityToken, indexToken)
24
+ return unique(getIndexComponents(entityManager, entityToken, indexToken)
25
+ .filter((component) => !omit.includes(component))
24
26
  .map((component) => component === entityManager.config.hashKey
25
- ? entityManager.config.hashKey
27
+ ? entityManager.config.entities[entityToken].timestampProperty
26
28
  : component === entityManager.config.rangeKey
27
29
  ? entityManager.config.entities[entityToken].uniqueProperty
28
30
  : generatedKeys.includes(component)
29
31
  ? generated[component].elements
30
32
  : component)
31
33
  .flat()
32
- .sort();
34
+ .filter((element) => !omit.includes(element))).sort();
33
35
  }
34
36
  catch (error) {
35
37
  if (error instanceof Error)
package/package.json CHANGED
@@ -4,9 +4,8 @@
4
4
  "url": "https://github.com/karmaniverous/entity-manager/issues"
5
5
  },
6
6
  "dependencies": {
7
- "@karmaniverous/entity-tools": "^0.4.0",
7
+ "@karmaniverous/entity-tools": "^0.4.4",
8
8
  "@karmaniverous/string-utilities": "^0.2.1",
9
- "jsonschema": "^1.4.1",
10
9
  "lz-string": "^1.5.0",
11
10
  "radash": "^12.1.0",
12
11
  "string-hash": "^1.1.3",
@@ -14,9 +13,9 @@
14
13
  },
15
14
  "description": "Rational indexing & cross-shard querying at scale in your NoSQL database so you can focus on your application logic.",
16
15
  "devDependencies": {
17
- "@dotenvx/dotenvx": "^1.21.0",
18
- "@eslint/js": "^9.13.0",
19
- "@faker-js/faker": "^9.1.0",
16
+ "@dotenvx/dotenvx": "^1.22.0",
17
+ "@eslint/js": "^9.14.0",
18
+ "@faker-js/faker": "^9.2.0",
20
19
  "@karmaniverous/mock-db": "^0.3.3",
21
20
  "@rollup/plugin-alias": "^5.1.1",
22
21
  "@rollup/plugin-commonjs": "^28.0.1",
@@ -29,35 +28,34 @@
29
28
  "@types/eslint-config-prettier": "^6.11.3",
30
29
  "@types/eslint-plugin-mocha": "^10.4.0",
31
30
  "@types/mocha": "^10.0.9",
32
- "@types/node": "^22.8.2",
31
+ "@types/node": "^22.9.0",
33
32
  "@types/string-hash": "^1.1.3",
34
33
  "auto-changelog": "^2.5.0",
35
34
  "chai": "^5.1.2",
36
- "cross-env": "^7.0.3",
37
- "eslint": "^9.13.0",
35
+ "eslint": "^9.14.0",
38
36
  "eslint-config-prettier": "^9.1.0",
39
37
  "eslint-plugin-mocha": "^10.5.0",
40
38
  "eslint-plugin-simple-import-sort": "^12.1.1",
41
39
  "eslint-plugin-tsdoc": "^0.3.0",
42
40
  "jsdom-global": "^3.0.2",
43
- "knip": "^5.34.2",
41
+ "knip": "^5.36.3",
44
42
  "lefthook": "^1.8.2",
45
- "mocha": "^10.7.3",
43
+ "mocha": "^10.8.2",
46
44
  "nyc": "^17.1.0",
47
45
  "prettier": "^3.3.3",
48
46
  "release-it": "^17.10.0",
49
47
  "rimraf": "^6.0.1",
50
- "rollup": "^4.24.2",
48
+ "rollup": "^4.24.4",
51
49
  "rollup-plugin-dts": "^6.1.1",
52
50
  "source-map-support": "^0.5.21",
53
51
  "ts-node": "^10.9.2",
54
- "tslib": "^2.8.0",
55
- "typedoc": "^0.26.10",
56
- "typedoc-plugin-mdn-links": "^3.3.5",
52
+ "tslib": "^2.8.1",
53
+ "typedoc": "^0.26.11",
54
+ "typedoc-plugin-mdn-links": "^3.3.6",
57
55
  "typedoc-plugin-replace-text": "^4.0.0",
58
56
  "typedoc-plugin-zod": "^1.2.1",
59
57
  "typescript": "^5.6.3",
60
- "typescript-eslint": "^8.12.1"
58
+ "typescript-eslint": "^8.13.0"
61
59
  },
62
60
  "exports": {
63
61
  ".": {
@@ -99,6 +97,7 @@
99
97
  "after:init": [
100
98
  "npm run lint",
101
99
  "npm run test",
100
+ "npm run knip",
102
101
  "npm run build"
103
102
  ],
104
103
  "before:npm:release": [
@@ -132,5 +131,5 @@
132
131
  },
133
132
  "type": "module",
134
133
  "types": "dist/index.d.ts",
135
- "version": "6.7.0"
134
+ "version": "6.7.2"
136
135
  }