@karmaniverous/entity-manager 6.6.0 → 6.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/BaseShardQueryMapBuilder.js +34 -0
- package/dist/cjs/EntityManager.js +2 -2
- package/dist/cjs/dehydrateIndexItem.js +5 -3
- package/dist/cjs/dehydratePageKeyMap.js +2 -4
- package/dist/cjs/getHashKeySpace.js +28 -5
- package/dist/cjs/index.js +2 -0
- package/dist/cjs/query.js +15 -34
- package/dist/cjs/rehydrateIndexItem.js +5 -3
- package/dist/cjs/rehydratePageKeyMap.js +25 -12
- package/dist/cjs/unwrapIndex.js +6 -4
- package/dist/index.d.ts +46 -20
- package/dist/mjs/BaseShardQueryMapBuilder.js +32 -0
- package/dist/mjs/EntityManager.js +2 -2
- package/dist/mjs/dehydrateIndexItem.js +5 -3
- package/dist/mjs/dehydratePageKeyMap.js +2 -4
- package/dist/mjs/getHashKeySpace.js +28 -5
- package/dist/mjs/index.js +1 -0
- package/dist/mjs/query.js +15 -34
- package/dist/mjs/rehydrateIndexItem.js +5 -3
- package/dist/mjs/rehydratePageKeyMap.js +26 -13
- package/dist/mjs/unwrapIndex.js +7 -5
- package/package.json +15 -16
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var radash = require('radash');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Abstract base class supporting a fluent API for building a {@link ShardQueryMap | `ShardQueryMap`} using a database client.
|
|
7
|
+
*
|
|
8
|
+
* @category ShardQueryMapBuilder
|
|
9
|
+
*/
|
|
10
|
+
class BaseShardQueryMapBuilder {
|
|
11
|
+
/** BaseShardQueryMapBuilder constructor. */
|
|
12
|
+
constructor(entityManager, entityToken, hashKeyToken, pageKeyMap) {
|
|
13
|
+
this.entityManager = entityManager;
|
|
14
|
+
this.entityToken = entityToken;
|
|
15
|
+
this.hashKeyToken = hashKeyToken;
|
|
16
|
+
this.pageKeyMap = pageKeyMap;
|
|
17
|
+
/**
|
|
18
|
+
* Maps `indexToken` values to database platform-specific parameters.
|
|
19
|
+
*
|
|
20
|
+
* @protected
|
|
21
|
+
*/
|
|
22
|
+
this.indexParamsMap = {};
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Builds a {@link ShardQueryMap | `ShardQueryMap`} object.
|
|
26
|
+
*
|
|
27
|
+
* @returns - The {@link ShardQueryMap | `ShardQueryMap`} object.
|
|
28
|
+
*/
|
|
29
|
+
build() {
|
|
30
|
+
return radash.mapValues(this.indexParamsMap, (indexConfig, indexToken) => this.getShardQueryFunction(indexToken));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
exports.BaseShardQueryMapBuilder = BaseShardQueryMapBuilder;
|
|
@@ -72,7 +72,7 @@ class EntityManager {
|
|
|
72
72
|
* Query a database entity across shards in a provider-generic fashion.
|
|
73
73
|
*
|
|
74
74
|
* @remarks
|
|
75
|
-
* The provided
|
|
75
|
+
* The provided `shardQueryMap` performs the actual query of individual data pages on individual index/shard combinations.
|
|
76
76
|
*
|
|
77
77
|
* Individual shard query results will be combined, deduped by {@link ConfigEntity.uniqueProperty} property value, and sorted by {@link QueryOptions.sortOrder | `sortOrder`}.
|
|
78
78
|
*
|
|
@@ -84,7 +84,7 @@ class EntityManager {
|
|
|
84
84
|
*
|
|
85
85
|
* @returns {@link QueryResult} object.
|
|
86
86
|
*
|
|
87
|
-
* @throws Error if {@link QueryOptions.
|
|
87
|
+
* @throws Error if {@link QueryOptions.shardQueryMapBuilder | `shardQueryMapBuilder`} `pageKeyMap` keys do not match its `shardQueryMap` keys.
|
|
88
88
|
*/
|
|
89
89
|
async query(options) {
|
|
90
90
|
return await query.query(this, options);
|
|
@@ -21,14 +21,13 @@ var validateEntityIndexToken = require('./validateEntityIndexToken.js');
|
|
|
21
21
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
22
22
|
* @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
|
|
23
23
|
* @param item - Partial {@link ItemMap | `ItemMap`} object.
|
|
24
|
-
* @param omit - Array of index components to omit from the output value.
|
|
25
24
|
*
|
|
26
25
|
* @returns Dehydrated index value.
|
|
27
26
|
*
|
|
28
27
|
* @throws `Error` if `entityToken` is invalid.
|
|
29
28
|
* @throws `Error` if `indexToken` is invalid.
|
|
30
29
|
*/
|
|
31
|
-
function dehydrateIndexItem(entityManager, entityToken, indexToken, item
|
|
30
|
+
function dehydrateIndexItem(entityManager, entityToken, indexToken, item) {
|
|
32
31
|
try {
|
|
33
32
|
const { generatedKeyDelimiter } = entityManager.config;
|
|
34
33
|
// Validate params.
|
|
@@ -37,7 +36,10 @@ function dehydrateIndexItem(entityManager, entityToken, indexToken, item, omit =
|
|
|
37
36
|
if (!item)
|
|
38
37
|
return '';
|
|
39
38
|
// Unwrap index elements.
|
|
40
|
-
const
|
|
39
|
+
const { hashKey } = entityManager.config.entities[entityToken].indexes[indexToken];
|
|
40
|
+
const elements = unwrapIndex.unwrapIndex(entityManager, entityToken, indexToken, [
|
|
41
|
+
hashKey,
|
|
42
|
+
]);
|
|
41
43
|
// Join index element values.
|
|
42
44
|
const dehydrated = elements
|
|
43
45
|
.map((element) => encodeEntityElement.encodeEntityElement(entityManager, entityToken, element, item))
|
|
@@ -47,7 +47,7 @@ function dehydratePageKeyMap(entityManager, entityToken, pageKeyMap) {
|
|
|
47
47
|
let dehydrated = [];
|
|
48
48
|
for (const index of indexes) {
|
|
49
49
|
for (const hashKey of hashKeys) {
|
|
50
|
-
//
|
|
50
|
+
// Undefined pageKey.
|
|
51
51
|
if (!pageKeyMap[index][hashKey]) {
|
|
52
52
|
dehydrated.push('');
|
|
53
53
|
continue;
|
|
@@ -62,9 +62,7 @@ function dehydratePageKeyMap(entityManager, entityToken, pageKeyMap) {
|
|
|
62
62
|
return item;
|
|
63
63
|
}, {});
|
|
64
64
|
// Dehydrate index from item.
|
|
65
|
-
dehydrated.push(dehydrateIndexItem.dehydrateIndexItem(entityManager, entityToken, index, item
|
|
66
|
-
entityManager.config.hashKey,
|
|
67
|
-
]));
|
|
65
|
+
dehydrated.push(dehydrateIndexItem.dehydrateIndexItem(entityManager, entityToken, index, item));
|
|
68
66
|
}
|
|
69
67
|
}
|
|
70
68
|
// Replace with empty array if all pageKeys are empty strings.
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var radash = require('radash');
|
|
4
|
-
var
|
|
4
|
+
var encodeGeneratedProperty = require('./encodeGeneratedProperty.js');
|
|
5
|
+
var validateEntityGeneratedProperty = require('./validateEntityGeneratedProperty.js');
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* Return an array of {@link ConfigKeys.hashKey | `entityManager.config.hashKey`} property values covering the shard space bounded by `timestampFrom` & `timestampTo`.
|
|
8
9
|
*
|
|
9
10
|
* @param entityManager - {@link EntityManager | `EntityManager`} instance.
|
|
10
11
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
12
|
+
* @param hashKeyToken - {@link ParsedConfig | `entityManager.config.hashKey`} or {@link ParsedConfig | `entityManager.config.entities.<entityToken>.generated`} key. If a generated property, must be shardable.
|
|
13
|
+
* @param item - Partial {@link ItemMap | `ItemMap[EntityToken]`} object. Must include all properties required to generate the hash key space.
|
|
11
14
|
* @param timestampFrom - Lower timestanp limit. Defaults to `0`.
|
|
12
15
|
* @param timestampTo - Upper timestamp limit. Defaults to `Date.now()`.
|
|
13
16
|
*
|
|
@@ -15,24 +18,42 @@ var validateEntityToken = require('./validateEntityToken.js');
|
|
|
15
18
|
*
|
|
16
19
|
* @throws `Error` if `entityToken` is invalid.
|
|
17
20
|
*/
|
|
18
|
-
function getHashKeySpace(entityManager, entityToken, timestampFrom = 0, timestampTo = Date.now()) {
|
|
21
|
+
function getHashKeySpace(entityManager, entityToken, hashKeyToken, item, timestampFrom = 0, timestampTo = Date.now()) {
|
|
19
22
|
try {
|
|
20
|
-
// Validate
|
|
21
|
-
|
|
23
|
+
// Validate hashKeyToken is either the global hash key or a sharded generated property.
|
|
24
|
+
if (hashKeyToken !== entityManager.config.hashKey)
|
|
25
|
+
validateEntityGeneratedProperty.validateEntityGeneratedProperty(entityManager, entityToken, hashKeyToken, true);
|
|
22
26
|
const { shardBumps } = entityManager.config.entities[entityToken];
|
|
23
27
|
const hashKeySpace = shardBumps
|
|
28
|
+
// Filter shard bumps by timestamp range.
|
|
24
29
|
.filter((bump, i) => (i === shardBumps.length - 1 ||
|
|
25
30
|
shardBumps[i + 1].timestamp > timestampFrom) &&
|
|
26
31
|
bump.timestamp <= timestampTo)
|
|
32
|
+
// Generate shard key space.
|
|
27
33
|
.flatMap(({ charBits, chars }) => {
|
|
28
34
|
const radix = 2 ** charBits;
|
|
29
35
|
return chars
|
|
30
36
|
? [...radash.range(0, radix ** chars - 1)].map((char) => char.toString(radix).padStart(chars, '0'))
|
|
31
37
|
: '';
|
|
32
38
|
})
|
|
33
|
-
|
|
39
|
+
// Map shard keys to hash keys.
|
|
40
|
+
.map((shardKey) => {
|
|
41
|
+
// Calculate record hash key.
|
|
42
|
+
let hashKey = `${entityToken}${entityManager.config.shardKeyDelimiter}${shardKey}`;
|
|
43
|
+
// If hash key space basis is a different property, encode it.
|
|
44
|
+
if (hashKeyToken !== entityManager.config.hashKey)
|
|
45
|
+
hashKey = encodeGeneratedProperty.encodeGeneratedProperty(entityManager, entityToken, hashKeyToken, {
|
|
46
|
+
...item,
|
|
47
|
+
[entityManager.config.hashKey]: hashKey,
|
|
48
|
+
});
|
|
49
|
+
if (!hashKey)
|
|
50
|
+
throw new Error('item does not support hash key space');
|
|
51
|
+
return hashKey;
|
|
52
|
+
});
|
|
34
53
|
entityManager.logger.debug('generated hash key space', {
|
|
35
54
|
entityToken,
|
|
55
|
+
hashKeyToken,
|
|
56
|
+
item,
|
|
36
57
|
timestampFrom,
|
|
37
58
|
timestampTo,
|
|
38
59
|
hashKeySpace,
|
|
@@ -43,6 +64,8 @@ function getHashKeySpace(entityManager, entityToken, timestampFrom = 0, timestam
|
|
|
43
64
|
if (error instanceof Error)
|
|
44
65
|
entityManager.logger.error(error.message, {
|
|
45
66
|
entityToken,
|
|
67
|
+
hashKeyToken,
|
|
68
|
+
item,
|
|
46
69
|
timestampFrom,
|
|
47
70
|
timestampTo,
|
|
48
71
|
});
|
package/dist/cjs/index.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
var BaseShardQueryMapBuilder = require('./BaseShardQueryMapBuilder.js');
|
|
3
4
|
var conditionalize = require('./conditionalize.js');
|
|
4
5
|
var EntityManager = require('./EntityManager.js');
|
|
5
6
|
|
|
6
7
|
|
|
7
8
|
|
|
9
|
+
exports.BaseShardQueryMapBuilder = BaseShardQueryMapBuilder.BaseShardQueryMapBuilder;
|
|
8
10
|
exports.conditionalize = conditionalize.conditionalize;
|
|
9
11
|
exports.EntityManager = EntityManager.EntityManager;
|
package/dist/cjs/query.js
CHANGED
|
@@ -5,7 +5,6 @@ var lzString = require('lz-string');
|
|
|
5
5
|
var radash = require('radash');
|
|
6
6
|
var dehydratePageKeyMap = require('./dehydratePageKeyMap.js');
|
|
7
7
|
var rehydratePageKeyMap = require('./rehydratePageKeyMap.js');
|
|
8
|
-
var validateEntityGeneratedProperty = require('./validateEntityGeneratedProperty.js');
|
|
9
8
|
|
|
10
9
|
const { compressToEncodedURIComponent, decompressFromEncodedURIComponent } = lzString;
|
|
11
10
|
/**
|
|
@@ -27,20 +26,19 @@ const { compressToEncodedURIComponent, decompressFromEncodedURIComponent } = lzS
|
|
|
27
26
|
*
|
|
28
27
|
* @throws Error if {@link QueryOptions.pageKeyMap | `pageKeyMap`} keys do not match {@link QueryOptions.shardQueryMap | `shardQueryMap`} keys.
|
|
29
28
|
*/
|
|
30
|
-
async function query(entityManager,
|
|
29
|
+
async function query(entityManager, options) {
|
|
31
30
|
try {
|
|
32
31
|
// Get defaults.
|
|
33
|
-
const { defaultLimit, defaultPageSize } = entityManager.config.entities[entityToken];
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
const { defaultLimit, defaultPageSize } = entityManager.config.entities[options.entityToken];
|
|
33
|
+
// Extract params.
|
|
34
|
+
const { entityToken, limit = defaultLimit, item, pageKeyMap, pageSize = defaultPageSize, shardQueryMap, sortOrder = [], timestampFrom = 0, timestampTo = Date.now(), throttle = entityManager.config.throttle, } = options;
|
|
36
35
|
// Validate params.
|
|
37
|
-
validateEntityGeneratedProperty.validateEntityGeneratedProperty(entityManager, entityToken, hashKey, true);
|
|
38
36
|
if (!(limit === Infinity || (radash.isInt(limit) && limit >= 1)))
|
|
39
37
|
throw new Error('limit must be a positive integer or Infinity.');
|
|
40
38
|
if (!(radash.isInt(pageSize) && pageSize >= 1))
|
|
41
39
|
throw new Error('pageSize must be a positive integer');
|
|
42
40
|
// Rehydrate pageKeyMap.
|
|
43
|
-
const rehydratedPageKeyMap = rehydratePageKeyMap.rehydratePageKeyMap(entityManager, entityToken, Object.keys(shardQueryMap), pageKeyMap
|
|
41
|
+
const [hashKeyToken, rehydratedPageKeyMap] = rehydratePageKeyMap.rehydratePageKeyMap(entityManager, entityToken, Object.keys(shardQueryMap), item, pageKeyMap
|
|
44
42
|
? JSON.parse(decompressFromEncodedURIComponent(pageKeyMap))
|
|
45
43
|
: undefined, timestampFrom, timestampTo);
|
|
46
44
|
// Shortcut if pageKeyMap is empty.
|
|
@@ -61,18 +59,18 @@ async function query(entityManager, { entityToken, hashKey, limit, pageKeyMap, p
|
|
|
61
59
|
// iteration in order to keep shardsQueried * pageSize > (limit - items.length) but only just.
|
|
62
60
|
// TODO: Test for invalid characters (path delimiters) in index keys & shard key values.
|
|
63
61
|
// Query every shard on every index in pageKeyMap.
|
|
64
|
-
const shardQueryResults = await radash.parallel(throttle, Object.entries(rehydratedPageKeyMap).flatMap(([
|
|
65
|
-
|
|
62
|
+
const shardQueryResults = await radash.parallel(throttle, Object.entries(rehydratedPageKeyMap).flatMap(([indexToken, indexPageKeys]) => Object.entries(indexPageKeys).map(([hashKey, pageKey]) => [
|
|
63
|
+
indexToken,
|
|
66
64
|
hashKey,
|
|
67
65
|
pageKey,
|
|
68
|
-
])), async ([
|
|
69
|
-
|
|
70
|
-
queryResult: await shardQueryMap[
|
|
66
|
+
])), async ([indexToken, hashKey, pageKey]) => ({
|
|
67
|
+
indexToken,
|
|
68
|
+
queryResult: await shardQueryMap[indexToken](hashKey, pageKey, pageSize),
|
|
71
69
|
hashKey,
|
|
72
70
|
}));
|
|
73
71
|
// Reduce shardQueryResults & updateworkingRresult.
|
|
74
|
-
workingResult = shardQueryResults.reduce(({ items, pageKeyMap }, {
|
|
75
|
-
Object.assign(rehydratedPageKeyMap[
|
|
72
|
+
workingResult = shardQueryResults.reduce(({ items, pageKeyMap }, { indexToken, queryResult, hashKey }) => {
|
|
73
|
+
Object.assign(rehydratedPageKeyMap[indexToken], {
|
|
76
74
|
[hashKey]: queryResult.pageKey,
|
|
77
75
|
});
|
|
78
76
|
return {
|
|
@@ -93,15 +91,8 @@ async function query(entityManager, { entityToken, hashKey, limit, pageKeyMap, p
|
|
|
93
91
|
pageKeyMap: compressToEncodedURIComponent(JSON.stringify(dehydratePageKeyMap.dehydratePageKeyMap(entityManager, entityToken, workingResult.pageKeyMap))),
|
|
94
92
|
};
|
|
95
93
|
entityManager.logger.debug('queried entityToken across shards', {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
limit,
|
|
99
|
-
pageKeyMap,
|
|
100
|
-
pageSize,
|
|
101
|
-
shardQueryMap,
|
|
102
|
-
timestampFrom,
|
|
103
|
-
timestampTo,
|
|
104
|
-
throttle,
|
|
94
|
+
options,
|
|
95
|
+
hashKeyToken,
|
|
105
96
|
rehydratedPageKeyMap,
|
|
106
97
|
workingResult,
|
|
107
98
|
result,
|
|
@@ -110,17 +101,7 @@ async function query(entityManager, { entityToken, hashKey, limit, pageKeyMap, p
|
|
|
110
101
|
}
|
|
111
102
|
catch (error) {
|
|
112
103
|
if (error instanceof Error)
|
|
113
|
-
entityManager.logger.error(error.message,
|
|
114
|
-
entityToken,
|
|
115
|
-
hashKey,
|
|
116
|
-
limit,
|
|
117
|
-
pageKeyMap,
|
|
118
|
-
pageSize,
|
|
119
|
-
shardQueryMap,
|
|
120
|
-
timestampFrom,
|
|
121
|
-
timestampTo,
|
|
122
|
-
throttle,
|
|
123
|
-
});
|
|
104
|
+
entityManager.logger.error(error.message, options);
|
|
124
105
|
throw error;
|
|
125
106
|
}
|
|
126
107
|
}
|
|
@@ -17,20 +17,22 @@ var validateEntityIndexToken = require('./validateEntityIndexToken.js');
|
|
|
17
17
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
18
18
|
* @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
|
|
19
19
|
* @param dehydrated - Dehydrated index value.
|
|
20
|
-
* @param omit - Array of index components omitted from `dehydrated`.
|
|
21
20
|
*
|
|
22
21
|
* @returns Partial {@link ItemMap | `ItemMap`} object containing rehydrated index component elements.
|
|
23
22
|
*
|
|
24
23
|
* @throws `Error` if `entityToken` is invalid.
|
|
25
24
|
* @throws `Error` if `indexToken` is invalid.
|
|
26
25
|
*/
|
|
27
|
-
function rehydrateIndexItem(entityManager, entityToken, indexToken, dehydrated
|
|
26
|
+
function rehydrateIndexItem(entityManager, entityToken, indexToken, dehydrated) {
|
|
28
27
|
try {
|
|
29
28
|
const { generatedKeyDelimiter } = entityManager.config;
|
|
30
29
|
// Validate params.
|
|
31
30
|
validateEntityIndexToken.validateEntityIndexToken(entityManager, entityToken, indexToken);
|
|
32
31
|
// Unwrap index elements.
|
|
33
|
-
const
|
|
32
|
+
const { hashKey } = entityManager.config.entities[entityToken].indexes[indexToken];
|
|
33
|
+
const elements = unwrapIndex.unwrapIndex(entityManager, entityToken, indexToken, [
|
|
34
|
+
hashKey,
|
|
35
|
+
]);
|
|
34
36
|
// Split dehydrated value & validate.
|
|
35
37
|
const values = dehydrated.split(generatedKeyDelimiter);
|
|
36
38
|
if (elements.length !== values.length)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var radash = require('radash');
|
|
4
|
+
var decodeGeneratedProperty = require('./decodeGeneratedProperty.js');
|
|
4
5
|
var encodeGeneratedProperty = require('./encodeGeneratedProperty.js');
|
|
5
6
|
var getHashKeySpace = require('./getHashKeySpace.js');
|
|
6
7
|
var getIndexComponents = require('./getIndexComponents.js');
|
|
@@ -16,28 +17,40 @@ var validateEntityIndexToken = require('./validateEntityIndexToken.js');
|
|
|
16
17
|
* @param entityManager - {@link EntityManager | `EntityManager`} instance.
|
|
17
18
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
18
19
|
* @param indexTokens - Array of {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} keys used as keys of the original {@link PageKeyMap | `PageKeyMap`}.
|
|
20
|
+
* @param item - Partial item object sufficiently populated to generate index hash keys.
|
|
19
21
|
* @param dehydrated - Array of dehydrated page keys or undefined if new query.
|
|
20
22
|
* @param timestampFrom - Lower timestanp limit used to generate the original {@link PageKeyMap | `PageKeyMap`}. Defaults to `0`.
|
|
21
23
|
* @param timestampTo - Upper timestamp limit used to generate the original {@link PageKeyMap | `PageKeyMap`}. Defaults to `Date.now()`.
|
|
22
24
|
*
|
|
23
|
-
* @returns
|
|
25
|
+
* @returns A tuple of `hashKeyToken` and rehydrated {@link PageKeyMap | `PageKeyMap`} object.
|
|
24
26
|
*
|
|
25
27
|
* @throws `Error` if `entityToken` is invalid.
|
|
26
28
|
* @throws `Error` if `indexTokens` is empty.
|
|
27
29
|
* @throws `Error` if any `indexTokens` are invalid.
|
|
30
|
+
* @throws `Error` if `indexTokens` represent indexes with inconsistent hashKeys.
|
|
28
31
|
* @throws `Error` if `dehydrated` has invalid length.
|
|
29
32
|
*/
|
|
30
|
-
function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated, timestampFrom = 0, timestampTo = Date.now()) {
|
|
33
|
+
function rehydratePageKeyMap(entityManager, entityToken, indexTokens, item, dehydrated, timestampFrom = 0, timestampTo = Date.now()) {
|
|
31
34
|
try {
|
|
32
|
-
// Validate
|
|
35
|
+
// Validate indexTokens populated.
|
|
33
36
|
if (!indexTokens.length)
|
|
34
37
|
throw new Error('indexTokens empty');
|
|
38
|
+
// Validate indexTokens exist.
|
|
39
|
+
const hashKeys = radash.unique(indexTokens.map((indexToken) => {
|
|
40
|
+
validateEntityIndexToken.validateEntityIndexToken(entityManager, entityToken, indexToken);
|
|
41
|
+
return entityManager.config.entities[entityToken].indexes[indexToken]
|
|
42
|
+
.hashKey;
|
|
43
|
+
}));
|
|
44
|
+
// Validate hashKeys consistent.
|
|
45
|
+
if (hashKeys.length > 1)
|
|
46
|
+
throw new Error('inconsistent hashKeys');
|
|
47
|
+
const [hashKeyToken] = hashKeys;
|
|
35
48
|
indexTokens.map((index) => validateEntityIndexToken.validateEntityIndexToken(entityManager, entityToken, index));
|
|
36
49
|
// Shortcut empty dehydrated.
|
|
37
50
|
if (dehydrated && !dehydrated.length)
|
|
38
|
-
return {};
|
|
51
|
+
return [hashKeyToken, {}];
|
|
39
52
|
// Get hash key space.
|
|
40
|
-
const hashKeySpace = getHashKeySpace.getHashKeySpace(entityManager, entityToken, timestampFrom, timestampTo);
|
|
53
|
+
const hashKeySpace = getHashKeySpace.getHashKeySpace(entityManager, entityToken, hashKeyToken, item, timestampFrom, timestampTo);
|
|
41
54
|
// Default dehydrated.
|
|
42
55
|
dehydrated ?? (dehydrated = [...radash.range(1, hashKeySpace.length * indexTokens.length, '')]);
|
|
43
56
|
// Validate dehydrated length
|
|
@@ -47,14 +60,14 @@ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated
|
|
|
47
60
|
const rehydrated = radash.mapValues(radash.zipToObject(indexTokens, radash.cluster(dehydrated, hashKeySpace.length)), (dehydratedIndexPageKeyMaps, index) => radash.zipToObject(hashKeySpace, (hashKey, i) => {
|
|
48
61
|
if (!dehydratedIndexPageKeyMaps[i])
|
|
49
62
|
return;
|
|
50
|
-
let
|
|
51
|
-
|
|
52
|
-
...rehydrateIndexItem.rehydrateIndexItem(entityManager, entityToken, index, dehydratedIndexPageKeyMaps[i]
|
|
63
|
+
let pageKeyItem = {
|
|
64
|
+
...decodeGeneratedProperty.decodeGeneratedProperty(entityManager, entityToken, hashKey),
|
|
65
|
+
...rehydrateIndexItem.rehydrateIndexItem(entityManager, entityToken, index, dehydratedIndexPageKeyMaps[i]),
|
|
53
66
|
};
|
|
54
|
-
|
|
67
|
+
pageKeyItem = updateItemRangeKey.updateItemRangeKey(entityManager, entityToken, pageKeyItem);
|
|
55
68
|
return radash.zipToObject(getIndexComponents.getIndexComponents(entityManager, entityToken, index), (component) => entityManager.config.entities[entityToken].generated[component]
|
|
56
|
-
? encodeGeneratedProperty.encodeGeneratedProperty(entityManager, entityToken, component,
|
|
57
|
-
:
|
|
69
|
+
? encodeGeneratedProperty.encodeGeneratedProperty(entityManager, entityToken, component, pageKeyItem)
|
|
70
|
+
: pageKeyItem[component]);
|
|
58
71
|
}));
|
|
59
72
|
entityManager.logger.debug('rehydrated page key map', {
|
|
60
73
|
entityToken,
|
|
@@ -62,7 +75,7 @@ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated
|
|
|
62
75
|
dehydrated,
|
|
63
76
|
rehydrated,
|
|
64
77
|
});
|
|
65
|
-
return rehydrated;
|
|
78
|
+
return [hashKeyToken, rehydrated];
|
|
66
79
|
}
|
|
67
80
|
catch (error) {
|
|
68
81
|
if (error instanceof Error)
|
package/dist/cjs/unwrapIndex.js
CHANGED
|
@@ -10,28 +10,30 @@ var validateEntityIndexToken = require('./validateEntityIndexToken.js');
|
|
|
10
10
|
* @param entityManager - {@link EntityManager | `EntityManager`} instance.
|
|
11
11
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
12
12
|
* @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
|
|
13
|
+
* @param omit - Array of index components or elements to omit from the output value.
|
|
13
14
|
*
|
|
14
15
|
* @returns Deduped, sorted array of ungenerated index component elements.
|
|
15
16
|
*
|
|
16
17
|
* @throws `Error` if `entityToken` is invalid.
|
|
17
18
|
* @throws `Error` if `indexToken` is invalid.
|
|
18
19
|
*/
|
|
19
|
-
function unwrapIndex(entityManager, entityToken, indexToken) {
|
|
20
|
+
function unwrapIndex(entityManager, entityToken, indexToken, omit = []) {
|
|
20
21
|
try {
|
|
21
22
|
// Validate params.
|
|
22
23
|
validateEntityIndexToken.validateEntityIndexToken(entityManager, entityToken, indexToken);
|
|
23
24
|
const generated = entityManager.config.entities[entityToken].generated;
|
|
24
25
|
const generatedKeys = Object.keys(radash.shake(generated));
|
|
25
|
-
return getIndexComponents.getIndexComponents(entityManager, entityToken, indexToken)
|
|
26
|
+
return radash.unique(getIndexComponents.getIndexComponents(entityManager, entityToken, indexToken)
|
|
27
|
+
.filter((component) => !omit.includes(component))
|
|
26
28
|
.map((component) => component === entityManager.config.hashKey
|
|
27
|
-
? entityManager.config.
|
|
29
|
+
? entityManager.config.entities[entityToken].timestampProperty
|
|
28
30
|
: component === entityManager.config.rangeKey
|
|
29
31
|
? entityManager.config.entities[entityToken].uniqueProperty
|
|
30
32
|
: generatedKeys.includes(component)
|
|
31
33
|
? generated[component].elements
|
|
32
34
|
: component)
|
|
33
35
|
.flat()
|
|
34
|
-
.sort();
|
|
36
|
+
.filter((element) => !omit.includes(element))).sort();
|
|
35
37
|
}
|
|
36
38
|
catch (error) {
|
|
37
39
|
if (error instanceof Error)
|
package/dist/index.d.ts
CHANGED
|
@@ -1,19 +1,6 @@
|
|
|
1
1
|
import { Entity, Exactify, TranscodeMap, PropertiesOfType, TranscodableProperties, Transcodes, DefaultTranscodeMap, SortOrder } from '@karmaniverous/entity-tools';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
|
|
4
|
-
/**
|
|
5
|
-
* Transforms a function such that it only executes when `condition` is truthy.
|
|
6
|
-
*
|
|
7
|
-
* @param fn - The function to conditionally execute.
|
|
8
|
-
* @param condition - The condition to check before executing `fn`.
|
|
9
|
-
*
|
|
10
|
-
* @typeParam F - The type of the function to conditionally execute.
|
|
11
|
-
*
|
|
12
|
-
* @returns The conditionalized function with the same signature as `fn`.
|
|
13
|
-
*
|
|
14
|
-
*/
|
|
15
|
-
declare function conditionalize<F extends (...args: Parameters<F>) => ReturnType<F>>(fn: F, condition?: unknown): (...args: Parameters<F>) => ReturnType<F> | undefined;
|
|
16
|
-
|
|
17
4
|
/**
|
|
18
5
|
* The base EntityMap type. All EntityMaps should extend this type.
|
|
19
6
|
*
|
|
@@ -723,7 +710,7 @@ type ShardQueryFunction<Item extends Entity> = (hashKey: string, pageKey?: Parti
|
|
|
723
710
|
type ShardQueryMap<Item extends Entity> = Record<string, ShardQueryFunction<Item>>;
|
|
724
711
|
|
|
725
712
|
/**
|
|
726
|
-
* Options passed to the {@link
|
|
713
|
+
* Options passed to the {@link query | `query`} function.
|
|
727
714
|
*
|
|
728
715
|
* @category Query
|
|
729
716
|
*/
|
|
@@ -731,10 +718,9 @@ interface QueryOptions<Item extends ItemMap<M, HashKey, RangeKey>[EntityToken],
|
|
|
731
718
|
/** Identifies the entity to be queried. Key of {@link Config | `EntityManager.config.entities`}. */
|
|
732
719
|
entityToken: EntityToken;
|
|
733
720
|
/**
|
|
734
|
-
*
|
|
735
|
-
* {@link Config | `EntityManager.config.entities.<entityToken>.keys`}.
|
|
721
|
+
* Partial item object sufficiently populated to generate index hash keys.
|
|
736
722
|
*/
|
|
737
|
-
|
|
723
|
+
item: Partial<Item>;
|
|
738
724
|
/**
|
|
739
725
|
* The target maximum number of records to be returned by the query across
|
|
740
726
|
* all shards.
|
|
@@ -872,7 +858,7 @@ declare class EntityManager<M extends EntityMap, HashKey extends string, RangeKe
|
|
|
872
858
|
* Query a database entity across shards in a provider-generic fashion.
|
|
873
859
|
*
|
|
874
860
|
* @remarks
|
|
875
|
-
* The provided
|
|
861
|
+
* The provided `shardQueryMap` performs the actual query of individual data pages on individual index/shard combinations.
|
|
876
862
|
*
|
|
877
863
|
* Individual shard query results will be combined, deduped by {@link ConfigEntity.uniqueProperty} property value, and sorted by {@link QueryOptions.sortOrder | `sortOrder`}.
|
|
878
864
|
*
|
|
@@ -884,9 +870,49 @@ declare class EntityManager<M extends EntityMap, HashKey extends string, RangeKe
|
|
|
884
870
|
*
|
|
885
871
|
* @returns {@link QueryResult} object.
|
|
886
872
|
*
|
|
887
|
-
* @throws Error if {@link QueryOptions.
|
|
873
|
+
* @throws Error if {@link QueryOptions.shardQueryMapBuilder | `shardQueryMapBuilder`} `pageKeyMap` keys do not match its `shardQueryMap` keys.
|
|
888
874
|
*/
|
|
889
875
|
query<Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string>(options: QueryOptions<Item, EntityToken, M, HashKey, RangeKey>): Promise<QueryResult<Item, EntityToken, M, HashKey, RangeKey>>;
|
|
890
876
|
}
|
|
891
877
|
|
|
892
|
-
|
|
878
|
+
/**
|
|
879
|
+
* Abstract base class supporting a fluent API for building a {@link ShardQueryMap | `ShardQueryMap`} using a database client.
|
|
880
|
+
*
|
|
881
|
+
* @category ShardQueryMapBuilder
|
|
882
|
+
*/
|
|
883
|
+
declare abstract class BaseShardQueryMapBuilder<IndexParams, Item extends ItemMap<M, HashKey, RangeKey>[EntityToken], EntityToken extends keyof Exactify<M> & string, M extends EntityMap, HashKey extends string, RangeKey extends string, T extends TranscodeMap> {
|
|
884
|
+
readonly entityManager: EntityManager<M, HashKey, RangeKey, T>;
|
|
885
|
+
readonly entityToken: EntityToken;
|
|
886
|
+
readonly hashKeyToken: PropertiesOfType<M[EntityToken], never> | HashKey;
|
|
887
|
+
readonly pageKeyMap?: string | undefined;
|
|
888
|
+
/**
|
|
889
|
+
* Maps `indexToken` values to database platform-specific parameters.
|
|
890
|
+
*
|
|
891
|
+
* @protected
|
|
892
|
+
*/
|
|
893
|
+
readonly indexParamsMap: Record<string, IndexParams>;
|
|
894
|
+
/** BaseShardQueryMapBuilder constructor. */
|
|
895
|
+
constructor(entityManager: EntityManager<M, HashKey, RangeKey, T>, entityToken: EntityToken, hashKeyToken: PropertiesOfType<M[EntityToken], never> | HashKey, pageKeyMap?: string | undefined);
|
|
896
|
+
protected abstract getShardQueryFunction(indexToken: string): ShardQueryFunction<Item>;
|
|
897
|
+
/**
|
|
898
|
+
* Builds a {@link ShardQueryMap | `ShardQueryMap`} object.
|
|
899
|
+
*
|
|
900
|
+
* @returns - The {@link ShardQueryMap | `ShardQueryMap`} object.
|
|
901
|
+
*/
|
|
902
|
+
build(): ShardQueryMap<Item>;
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
/**
|
|
906
|
+
* Transforms a function such that it only executes when `condition` is truthy.
|
|
907
|
+
*
|
|
908
|
+
* @param fn - The function to conditionally execute.
|
|
909
|
+
* @param condition - The condition to check before executing `fn`.
|
|
910
|
+
*
|
|
911
|
+
* @typeParam F - The type of the function to conditionally execute.
|
|
912
|
+
*
|
|
913
|
+
* @returns The conditionalized function with the same signature as `fn`.
|
|
914
|
+
*
|
|
915
|
+
*/
|
|
916
|
+
declare function conditionalize<F extends (...args: Parameters<F>) => ReturnType<F>>(fn: F, condition?: unknown): (...args: Parameters<F>) => ReturnType<F> | undefined;
|
|
917
|
+
|
|
918
|
+
export { BaseShardQueryMapBuilder, type Config, type ConfigEntities, type ConfigEntity, type ConfigEntityGenerated, type ConfigEntityIndexComponent, type ConfigKeys, type ConfigTranscodes, EntityManager, type EntityMap, type ExclusiveKey, type ItemMap, type ParsedConfig, type QueryOptions, type QueryResult, type ShardBump, type ShardQueryFunction, type ShardQueryMap, type ShardQueryResult, type Unwrap, conditionalize };
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { mapValues } from 'radash';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Abstract base class supporting a fluent API for building a {@link ShardQueryMap | `ShardQueryMap`} using a database client.
|
|
5
|
+
*
|
|
6
|
+
* @category ShardQueryMapBuilder
|
|
7
|
+
*/
|
|
8
|
+
class BaseShardQueryMapBuilder {
|
|
9
|
+
/** BaseShardQueryMapBuilder constructor. */
|
|
10
|
+
constructor(entityManager, entityToken, hashKeyToken, pageKeyMap) {
|
|
11
|
+
this.entityManager = entityManager;
|
|
12
|
+
this.entityToken = entityToken;
|
|
13
|
+
this.hashKeyToken = hashKeyToken;
|
|
14
|
+
this.pageKeyMap = pageKeyMap;
|
|
15
|
+
/**
|
|
16
|
+
* Maps `indexToken` values to database platform-specific parameters.
|
|
17
|
+
*
|
|
18
|
+
* @protected
|
|
19
|
+
*/
|
|
20
|
+
this.indexParamsMap = {};
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Builds a {@link ShardQueryMap | `ShardQueryMap`} object.
|
|
24
|
+
*
|
|
25
|
+
* @returns - The {@link ShardQueryMap | `ShardQueryMap`} object.
|
|
26
|
+
*/
|
|
27
|
+
build() {
|
|
28
|
+
return mapValues(this.indexParamsMap, (indexConfig, indexToken) => this.getShardQueryFunction(indexToken));
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export { BaseShardQueryMapBuilder };
|
|
@@ -70,7 +70,7 @@ class EntityManager {
|
|
|
70
70
|
* Query a database entity across shards in a provider-generic fashion.
|
|
71
71
|
*
|
|
72
72
|
* @remarks
|
|
73
|
-
* The provided
|
|
73
|
+
* The provided `shardQueryMap` performs the actual query of individual data pages on individual index/shard combinations.
|
|
74
74
|
*
|
|
75
75
|
* Individual shard query results will be combined, deduped by {@link ConfigEntity.uniqueProperty} property value, and sorted by {@link QueryOptions.sortOrder | `sortOrder`}.
|
|
76
76
|
*
|
|
@@ -82,7 +82,7 @@ class EntityManager {
|
|
|
82
82
|
*
|
|
83
83
|
* @returns {@link QueryResult} object.
|
|
84
84
|
*
|
|
85
|
-
* @throws Error if {@link QueryOptions.
|
|
85
|
+
* @throws Error if {@link QueryOptions.shardQueryMapBuilder | `shardQueryMapBuilder`} `pageKeyMap` keys do not match its `shardQueryMap` keys.
|
|
86
86
|
*/
|
|
87
87
|
async query(options) {
|
|
88
88
|
return await query(this, options);
|
|
@@ -19,14 +19,13 @@ import { validateEntityIndexToken } from './validateEntityIndexToken.js';
|
|
|
19
19
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
20
20
|
* @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
|
|
21
21
|
* @param item - Partial {@link ItemMap | `ItemMap`} object.
|
|
22
|
-
* @param omit - Array of index components to omit from the output value.
|
|
23
22
|
*
|
|
24
23
|
* @returns Dehydrated index value.
|
|
25
24
|
*
|
|
26
25
|
* @throws `Error` if `entityToken` is invalid.
|
|
27
26
|
* @throws `Error` if `indexToken` is invalid.
|
|
28
27
|
*/
|
|
29
|
-
function dehydrateIndexItem(entityManager, entityToken, indexToken, item
|
|
28
|
+
function dehydrateIndexItem(entityManager, entityToken, indexToken, item) {
|
|
30
29
|
try {
|
|
31
30
|
const { generatedKeyDelimiter } = entityManager.config;
|
|
32
31
|
// Validate params.
|
|
@@ -35,7 +34,10 @@ function dehydrateIndexItem(entityManager, entityToken, indexToken, item, omit =
|
|
|
35
34
|
if (!item)
|
|
36
35
|
return '';
|
|
37
36
|
// Unwrap index elements.
|
|
38
|
-
const
|
|
37
|
+
const { hashKey } = entityManager.config.entities[entityToken].indexes[indexToken];
|
|
38
|
+
const elements = unwrapIndex(entityManager, entityToken, indexToken, [
|
|
39
|
+
hashKey,
|
|
40
|
+
]);
|
|
39
41
|
// Join index element values.
|
|
40
42
|
const dehydrated = elements
|
|
41
43
|
.map((element) => encodeEntityElement(entityManager, entityToken, element, item))
|
|
@@ -45,7 +45,7 @@ function dehydratePageKeyMap(entityManager, entityToken, pageKeyMap) {
|
|
|
45
45
|
let dehydrated = [];
|
|
46
46
|
for (const index of indexes) {
|
|
47
47
|
for (const hashKey of hashKeys) {
|
|
48
|
-
//
|
|
48
|
+
// Undefined pageKey.
|
|
49
49
|
if (!pageKeyMap[index][hashKey]) {
|
|
50
50
|
dehydrated.push('');
|
|
51
51
|
continue;
|
|
@@ -60,9 +60,7 @@ function dehydratePageKeyMap(entityManager, entityToken, pageKeyMap) {
|
|
|
60
60
|
return item;
|
|
61
61
|
}, {});
|
|
62
62
|
// Dehydrate index from item.
|
|
63
|
-
dehydrated.push(dehydrateIndexItem(entityManager, entityToken, index, item
|
|
64
|
-
entityManager.config.hashKey,
|
|
65
|
-
]));
|
|
63
|
+
dehydrated.push(dehydrateIndexItem(entityManager, entityToken, index, item));
|
|
66
64
|
}
|
|
67
65
|
}
|
|
68
66
|
// Replace with empty array if all pageKeys are empty strings.
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { range } from 'radash';
|
|
2
|
-
import {
|
|
2
|
+
import { encodeGeneratedProperty } from './encodeGeneratedProperty.js';
|
|
3
|
+
import { validateEntityGeneratedProperty } from './validateEntityGeneratedProperty.js';
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* Return an array of {@link ConfigKeys.hashKey | `entityManager.config.hashKey`} property values covering the shard space bounded by `timestampFrom` & `timestampTo`.
|
|
6
7
|
*
|
|
7
8
|
* @param entityManager - {@link EntityManager | `EntityManager`} instance.
|
|
8
9
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
10
|
+
* @param hashKeyToken - {@link ParsedConfig | `entityManager.config.hashKey`} or {@link ParsedConfig | `entityManager.config.entities.<entityToken>.generated`} key. If a generated property, must be shardable.
|
|
11
|
+
* @param item - Partial {@link ItemMap | `ItemMap[EntityToken]`} object. Must include all properties required to generate the hash key space.
|
|
9
12
|
* @param timestampFrom - Lower timestanp limit. Defaults to `0`.
|
|
10
13
|
* @param timestampTo - Upper timestamp limit. Defaults to `Date.now()`.
|
|
11
14
|
*
|
|
@@ -13,24 +16,42 @@ import { validateEntityToken } from './validateEntityToken.js';
|
|
|
13
16
|
*
|
|
14
17
|
* @throws `Error` if `entityToken` is invalid.
|
|
15
18
|
*/
|
|
16
|
-
function getHashKeySpace(entityManager, entityToken, timestampFrom = 0, timestampTo = Date.now()) {
|
|
19
|
+
function getHashKeySpace(entityManager, entityToken, hashKeyToken, item, timestampFrom = 0, timestampTo = Date.now()) {
|
|
17
20
|
try {
|
|
18
|
-
// Validate
|
|
19
|
-
|
|
21
|
+
// Validate hashKeyToken is either the global hash key or a sharded generated property.
|
|
22
|
+
if (hashKeyToken !== entityManager.config.hashKey)
|
|
23
|
+
validateEntityGeneratedProperty(entityManager, entityToken, hashKeyToken, true);
|
|
20
24
|
const { shardBumps } = entityManager.config.entities[entityToken];
|
|
21
25
|
const hashKeySpace = shardBumps
|
|
26
|
+
// Filter shard bumps by timestamp range.
|
|
22
27
|
.filter((bump, i) => (i === shardBumps.length - 1 ||
|
|
23
28
|
shardBumps[i + 1].timestamp > timestampFrom) &&
|
|
24
29
|
bump.timestamp <= timestampTo)
|
|
30
|
+
// Generate shard key space.
|
|
25
31
|
.flatMap(({ charBits, chars }) => {
|
|
26
32
|
const radix = 2 ** charBits;
|
|
27
33
|
return chars
|
|
28
34
|
? [...range(0, radix ** chars - 1)].map((char) => char.toString(radix).padStart(chars, '0'))
|
|
29
35
|
: '';
|
|
30
36
|
})
|
|
31
|
-
|
|
37
|
+
// Map shard keys to hash keys.
|
|
38
|
+
.map((shardKey) => {
|
|
39
|
+
// Calculate record hash key.
|
|
40
|
+
let hashKey = `${entityToken}${entityManager.config.shardKeyDelimiter}${shardKey}`;
|
|
41
|
+
// If hash key space basis is a different property, encode it.
|
|
42
|
+
if (hashKeyToken !== entityManager.config.hashKey)
|
|
43
|
+
hashKey = encodeGeneratedProperty(entityManager, entityToken, hashKeyToken, {
|
|
44
|
+
...item,
|
|
45
|
+
[entityManager.config.hashKey]: hashKey,
|
|
46
|
+
});
|
|
47
|
+
if (!hashKey)
|
|
48
|
+
throw new Error('item does not support hash key space');
|
|
49
|
+
return hashKey;
|
|
50
|
+
});
|
|
32
51
|
entityManager.logger.debug('generated hash key space', {
|
|
33
52
|
entityToken,
|
|
53
|
+
hashKeyToken,
|
|
54
|
+
item,
|
|
34
55
|
timestampFrom,
|
|
35
56
|
timestampTo,
|
|
36
57
|
hashKeySpace,
|
|
@@ -41,6 +62,8 @@ function getHashKeySpace(entityManager, entityToken, timestampFrom = 0, timestam
|
|
|
41
62
|
if (error instanceof Error)
|
|
42
63
|
entityManager.logger.error(error.message, {
|
|
43
64
|
entityToken,
|
|
65
|
+
hashKeyToken,
|
|
66
|
+
item,
|
|
44
67
|
timestampFrom,
|
|
45
68
|
timestampTo,
|
|
46
69
|
});
|
package/dist/mjs/index.js
CHANGED
package/dist/mjs/query.js
CHANGED
|
@@ -3,7 +3,6 @@ import lzString from 'lz-string';
|
|
|
3
3
|
import { isInt, parallel, unique } from 'radash';
|
|
4
4
|
import { dehydratePageKeyMap } from './dehydratePageKeyMap.js';
|
|
5
5
|
import { rehydratePageKeyMap } from './rehydratePageKeyMap.js';
|
|
6
|
-
import { validateEntityGeneratedProperty } from './validateEntityGeneratedProperty.js';
|
|
7
6
|
|
|
8
7
|
const { compressToEncodedURIComponent, decompressFromEncodedURIComponent } = lzString;
|
|
9
8
|
/**
|
|
@@ -25,20 +24,19 @@ const { compressToEncodedURIComponent, decompressFromEncodedURIComponent } = lzS
|
|
|
25
24
|
*
|
|
26
25
|
* @throws Error if {@link QueryOptions.pageKeyMap | `pageKeyMap`} keys do not match {@link QueryOptions.shardQueryMap | `shardQueryMap`} keys.
|
|
27
26
|
*/
|
|
28
|
-
async function query(entityManager,
|
|
27
|
+
async function query(entityManager, options) {
|
|
29
28
|
try {
|
|
30
29
|
// Get defaults.
|
|
31
|
-
const { defaultLimit, defaultPageSize } = entityManager.config.entities[entityToken];
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
const { defaultLimit, defaultPageSize } = entityManager.config.entities[options.entityToken];
|
|
31
|
+
// Extract params.
|
|
32
|
+
const { entityToken, limit = defaultLimit, item, pageKeyMap, pageSize = defaultPageSize, shardQueryMap, sortOrder = [], timestampFrom = 0, timestampTo = Date.now(), throttle = entityManager.config.throttle, } = options;
|
|
34
33
|
// Validate params.
|
|
35
|
-
validateEntityGeneratedProperty(entityManager, entityToken, hashKey, true);
|
|
36
34
|
if (!(limit === Infinity || (isInt(limit) && limit >= 1)))
|
|
37
35
|
throw new Error('limit must be a positive integer or Infinity.');
|
|
38
36
|
if (!(isInt(pageSize) && pageSize >= 1))
|
|
39
37
|
throw new Error('pageSize must be a positive integer');
|
|
40
38
|
// Rehydrate pageKeyMap.
|
|
41
|
-
const rehydratedPageKeyMap = rehydratePageKeyMap(entityManager, entityToken, Object.keys(shardQueryMap), pageKeyMap
|
|
39
|
+
const [hashKeyToken, rehydratedPageKeyMap] = rehydratePageKeyMap(entityManager, entityToken, Object.keys(shardQueryMap), item, pageKeyMap
|
|
42
40
|
? JSON.parse(decompressFromEncodedURIComponent(pageKeyMap))
|
|
43
41
|
: undefined, timestampFrom, timestampTo);
|
|
44
42
|
// Shortcut if pageKeyMap is empty.
|
|
@@ -59,18 +57,18 @@ async function query(entityManager, { entityToken, hashKey, limit, pageKeyMap, p
|
|
|
59
57
|
// iteration in order to keep shardsQueried * pageSize > (limit - items.length) but only just.
|
|
60
58
|
// TODO: Test for invalid characters (path delimiters) in index keys & shard key values.
|
|
61
59
|
// Query every shard on every index in pageKeyMap.
|
|
62
|
-
const shardQueryResults = await parallel(throttle, Object.entries(rehydratedPageKeyMap).flatMap(([
|
|
63
|
-
|
|
60
|
+
const shardQueryResults = await parallel(throttle, Object.entries(rehydratedPageKeyMap).flatMap(([indexToken, indexPageKeys]) => Object.entries(indexPageKeys).map(([hashKey, pageKey]) => [
|
|
61
|
+
indexToken,
|
|
64
62
|
hashKey,
|
|
65
63
|
pageKey,
|
|
66
|
-
])), async ([
|
|
67
|
-
|
|
68
|
-
queryResult: await shardQueryMap[
|
|
64
|
+
])), async ([indexToken, hashKey, pageKey]) => ({
|
|
65
|
+
indexToken,
|
|
66
|
+
queryResult: await shardQueryMap[indexToken](hashKey, pageKey, pageSize),
|
|
69
67
|
hashKey,
|
|
70
68
|
}));
|
|
71
69
|
// Reduce shardQueryResults & updateworkingRresult.
|
|
72
|
-
workingResult = shardQueryResults.reduce(({ items, pageKeyMap }, {
|
|
73
|
-
Object.assign(rehydratedPageKeyMap[
|
|
70
|
+
workingResult = shardQueryResults.reduce(({ items, pageKeyMap }, { indexToken, queryResult, hashKey }) => {
|
|
71
|
+
Object.assign(rehydratedPageKeyMap[indexToken], {
|
|
74
72
|
[hashKey]: queryResult.pageKey,
|
|
75
73
|
});
|
|
76
74
|
return {
|
|
@@ -91,15 +89,8 @@ async function query(entityManager, { entityToken, hashKey, limit, pageKeyMap, p
|
|
|
91
89
|
pageKeyMap: compressToEncodedURIComponent(JSON.stringify(dehydratePageKeyMap(entityManager, entityToken, workingResult.pageKeyMap))),
|
|
92
90
|
};
|
|
93
91
|
entityManager.logger.debug('queried entityToken across shards', {
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
limit,
|
|
97
|
-
pageKeyMap,
|
|
98
|
-
pageSize,
|
|
99
|
-
shardQueryMap,
|
|
100
|
-
timestampFrom,
|
|
101
|
-
timestampTo,
|
|
102
|
-
throttle,
|
|
92
|
+
options,
|
|
93
|
+
hashKeyToken,
|
|
103
94
|
rehydratedPageKeyMap,
|
|
104
95
|
workingResult,
|
|
105
96
|
result,
|
|
@@ -108,17 +99,7 @@ async function query(entityManager, { entityToken, hashKey, limit, pageKeyMap, p
|
|
|
108
99
|
}
|
|
109
100
|
catch (error) {
|
|
110
101
|
if (error instanceof Error)
|
|
111
|
-
entityManager.logger.error(error.message,
|
|
112
|
-
entityToken,
|
|
113
|
-
hashKey,
|
|
114
|
-
limit,
|
|
115
|
-
pageKeyMap,
|
|
116
|
-
pageSize,
|
|
117
|
-
shardQueryMap,
|
|
118
|
-
timestampFrom,
|
|
119
|
-
timestampTo,
|
|
120
|
-
throttle,
|
|
121
|
-
});
|
|
102
|
+
entityManager.logger.error(error.message, options);
|
|
122
103
|
throw error;
|
|
123
104
|
}
|
|
124
105
|
}
|
|
@@ -15,20 +15,22 @@ import { validateEntityIndexToken } from './validateEntityIndexToken.js';
|
|
|
15
15
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
16
16
|
* @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
|
|
17
17
|
* @param dehydrated - Dehydrated index value.
|
|
18
|
-
* @param omit - Array of index components omitted from `dehydrated`.
|
|
19
18
|
*
|
|
20
19
|
* @returns Partial {@link ItemMap | `ItemMap`} object containing rehydrated index component elements.
|
|
21
20
|
*
|
|
22
21
|
* @throws `Error` if `entityToken` is invalid.
|
|
23
22
|
* @throws `Error` if `indexToken` is invalid.
|
|
24
23
|
*/
|
|
25
|
-
function rehydrateIndexItem(entityManager, entityToken, indexToken, dehydrated
|
|
24
|
+
function rehydrateIndexItem(entityManager, entityToken, indexToken, dehydrated) {
|
|
26
25
|
try {
|
|
27
26
|
const { generatedKeyDelimiter } = entityManager.config;
|
|
28
27
|
// Validate params.
|
|
29
28
|
validateEntityIndexToken(entityManager, entityToken, indexToken);
|
|
30
29
|
// Unwrap index elements.
|
|
31
|
-
const
|
|
30
|
+
const { hashKey } = entityManager.config.entities[entityToken].indexes[indexToken];
|
|
31
|
+
const elements = unwrapIndex(entityManager, entityToken, indexToken, [
|
|
32
|
+
hashKey,
|
|
33
|
+
]);
|
|
32
34
|
// Split dehydrated value & validate.
|
|
33
35
|
const values = dehydrated.split(generatedKeyDelimiter);
|
|
34
36
|
if (elements.length !== values.length)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { range, mapValues, zipToObject, cluster } from 'radash';
|
|
1
|
+
import { unique, range, mapValues, zipToObject, cluster } from 'radash';
|
|
2
|
+
import { decodeGeneratedProperty } from './decodeGeneratedProperty.js';
|
|
2
3
|
import { encodeGeneratedProperty } from './encodeGeneratedProperty.js';
|
|
3
4
|
import { getHashKeySpace } from './getHashKeySpace.js';
|
|
4
5
|
import { getIndexComponents } from './getIndexComponents.js';
|
|
@@ -14,28 +15,40 @@ import { validateEntityIndexToken } from './validateEntityIndexToken.js';
|
|
|
14
15
|
* @param entityManager - {@link EntityManager | `EntityManager`} instance.
|
|
15
16
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
16
17
|
* @param indexTokens - Array of {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} keys used as keys of the original {@link PageKeyMap | `PageKeyMap`}.
|
|
18
|
+
* @param item - Partial item object sufficiently populated to generate index hash keys.
|
|
17
19
|
* @param dehydrated - Array of dehydrated page keys or undefined if new query.
|
|
18
20
|
* @param timestampFrom - Lower timestanp limit used to generate the original {@link PageKeyMap | `PageKeyMap`}. Defaults to `0`.
|
|
19
21
|
* @param timestampTo - Upper timestamp limit used to generate the original {@link PageKeyMap | `PageKeyMap`}. Defaults to `Date.now()`.
|
|
20
22
|
*
|
|
21
|
-
* @returns
|
|
23
|
+
* @returns A tuple of `hashKeyToken` and rehydrated {@link PageKeyMap | `PageKeyMap`} object.
|
|
22
24
|
*
|
|
23
25
|
* @throws `Error` if `entityToken` is invalid.
|
|
24
26
|
* @throws `Error` if `indexTokens` is empty.
|
|
25
27
|
* @throws `Error` if any `indexTokens` are invalid.
|
|
28
|
+
* @throws `Error` if `indexTokens` represent indexes with inconsistent hashKeys.
|
|
26
29
|
* @throws `Error` if `dehydrated` has invalid length.
|
|
27
30
|
*/
|
|
28
|
-
function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated, timestampFrom = 0, timestampTo = Date.now()) {
|
|
31
|
+
function rehydratePageKeyMap(entityManager, entityToken, indexTokens, item, dehydrated, timestampFrom = 0, timestampTo = Date.now()) {
|
|
29
32
|
try {
|
|
30
|
-
// Validate
|
|
33
|
+
// Validate indexTokens populated.
|
|
31
34
|
if (!indexTokens.length)
|
|
32
35
|
throw new Error('indexTokens empty');
|
|
36
|
+
// Validate indexTokens exist.
|
|
37
|
+
const hashKeys = unique(indexTokens.map((indexToken) => {
|
|
38
|
+
validateEntityIndexToken(entityManager, entityToken, indexToken);
|
|
39
|
+
return entityManager.config.entities[entityToken].indexes[indexToken]
|
|
40
|
+
.hashKey;
|
|
41
|
+
}));
|
|
42
|
+
// Validate hashKeys consistent.
|
|
43
|
+
if (hashKeys.length > 1)
|
|
44
|
+
throw new Error('inconsistent hashKeys');
|
|
45
|
+
const [hashKeyToken] = hashKeys;
|
|
33
46
|
indexTokens.map((index) => validateEntityIndexToken(entityManager, entityToken, index));
|
|
34
47
|
// Shortcut empty dehydrated.
|
|
35
48
|
if (dehydrated && !dehydrated.length)
|
|
36
|
-
return {};
|
|
49
|
+
return [hashKeyToken, {}];
|
|
37
50
|
// Get hash key space.
|
|
38
|
-
const hashKeySpace = getHashKeySpace(entityManager, entityToken, timestampFrom, timestampTo);
|
|
51
|
+
const hashKeySpace = getHashKeySpace(entityManager, entityToken, hashKeyToken, item, timestampFrom, timestampTo);
|
|
39
52
|
// Default dehydrated.
|
|
40
53
|
dehydrated ?? (dehydrated = [...range(1, hashKeySpace.length * indexTokens.length, '')]);
|
|
41
54
|
// Validate dehydrated length
|
|
@@ -45,14 +58,14 @@ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated
|
|
|
45
58
|
const rehydrated = mapValues(zipToObject(indexTokens, cluster(dehydrated, hashKeySpace.length)), (dehydratedIndexPageKeyMaps, index) => zipToObject(hashKeySpace, (hashKey, i) => {
|
|
46
59
|
if (!dehydratedIndexPageKeyMaps[i])
|
|
47
60
|
return;
|
|
48
|
-
let
|
|
49
|
-
|
|
50
|
-
...rehydrateIndexItem(entityManager, entityToken, index, dehydratedIndexPageKeyMaps[i]
|
|
61
|
+
let pageKeyItem = {
|
|
62
|
+
...decodeGeneratedProperty(entityManager, entityToken, hashKey),
|
|
63
|
+
...rehydrateIndexItem(entityManager, entityToken, index, dehydratedIndexPageKeyMaps[i]),
|
|
51
64
|
};
|
|
52
|
-
|
|
65
|
+
pageKeyItem = updateItemRangeKey(entityManager, entityToken, pageKeyItem);
|
|
53
66
|
return zipToObject(getIndexComponents(entityManager, entityToken, index), (component) => entityManager.config.entities[entityToken].generated[component]
|
|
54
|
-
? encodeGeneratedProperty(entityManager, entityToken, component,
|
|
55
|
-
:
|
|
67
|
+
? encodeGeneratedProperty(entityManager, entityToken, component, pageKeyItem)
|
|
68
|
+
: pageKeyItem[component]);
|
|
56
69
|
}));
|
|
57
70
|
entityManager.logger.debug('rehydrated page key map', {
|
|
58
71
|
entityToken,
|
|
@@ -60,7 +73,7 @@ function rehydratePageKeyMap(entityManager, entityToken, indexTokens, dehydrated
|
|
|
60
73
|
dehydrated,
|
|
61
74
|
rehydrated,
|
|
62
75
|
});
|
|
63
|
-
return rehydrated;
|
|
76
|
+
return [hashKeyToken, rehydrated];
|
|
64
77
|
}
|
|
65
78
|
catch (error) {
|
|
66
79
|
if (error instanceof Error)
|
package/dist/mjs/unwrapIndex.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { shake } from 'radash';
|
|
1
|
+
import { shake, unique } from 'radash';
|
|
2
2
|
import { getIndexComponents } from './getIndexComponents.js';
|
|
3
3
|
import { validateEntityIndexToken } from './validateEntityIndexToken.js';
|
|
4
4
|
|
|
@@ -8,28 +8,30 @@ import { validateEntityIndexToken } from './validateEntityIndexToken.js';
|
|
|
8
8
|
* @param entityManager - {@link EntityManager | `EntityManager`} instance.
|
|
9
9
|
* @param entityToken - {@link ConfigKeys.entities | `entityManager.config.entities`} key.
|
|
10
10
|
* @param indexToken - {@link ConfigEntity.indexes | `entityManager.config.entities.<entityToken>.indexes`} key.
|
|
11
|
+
* @param omit - Array of index components or elements to omit from the output value.
|
|
11
12
|
*
|
|
12
13
|
* @returns Deduped, sorted array of ungenerated index component elements.
|
|
13
14
|
*
|
|
14
15
|
* @throws `Error` if `entityToken` is invalid.
|
|
15
16
|
* @throws `Error` if `indexToken` is invalid.
|
|
16
17
|
*/
|
|
17
|
-
function unwrapIndex(entityManager, entityToken, indexToken) {
|
|
18
|
+
function unwrapIndex(entityManager, entityToken, indexToken, omit = []) {
|
|
18
19
|
try {
|
|
19
20
|
// Validate params.
|
|
20
21
|
validateEntityIndexToken(entityManager, entityToken, indexToken);
|
|
21
22
|
const generated = entityManager.config.entities[entityToken].generated;
|
|
22
23
|
const generatedKeys = Object.keys(shake(generated));
|
|
23
|
-
return getIndexComponents(entityManager, entityToken, indexToken)
|
|
24
|
+
return unique(getIndexComponents(entityManager, entityToken, indexToken)
|
|
25
|
+
.filter((component) => !omit.includes(component))
|
|
24
26
|
.map((component) => component === entityManager.config.hashKey
|
|
25
|
-
? entityManager.config.
|
|
27
|
+
? entityManager.config.entities[entityToken].timestampProperty
|
|
26
28
|
: component === entityManager.config.rangeKey
|
|
27
29
|
? entityManager.config.entities[entityToken].uniqueProperty
|
|
28
30
|
: generatedKeys.includes(component)
|
|
29
31
|
? generated[component].elements
|
|
30
32
|
: component)
|
|
31
33
|
.flat()
|
|
32
|
-
.sort();
|
|
34
|
+
.filter((element) => !omit.includes(element))).sort();
|
|
33
35
|
}
|
|
34
36
|
catch (error) {
|
|
35
37
|
if (error instanceof Error)
|
package/package.json
CHANGED
|
@@ -4,9 +4,8 @@
|
|
|
4
4
|
"url": "https://github.com/karmaniverous/entity-manager/issues"
|
|
5
5
|
},
|
|
6
6
|
"dependencies": {
|
|
7
|
-
"@karmaniverous/entity-tools": "^0.4.
|
|
7
|
+
"@karmaniverous/entity-tools": "^0.4.4",
|
|
8
8
|
"@karmaniverous/string-utilities": "^0.2.1",
|
|
9
|
-
"jsonschema": "^1.4.1",
|
|
10
9
|
"lz-string": "^1.5.0",
|
|
11
10
|
"radash": "^12.1.0",
|
|
12
11
|
"string-hash": "^1.1.3",
|
|
@@ -14,9 +13,9 @@
|
|
|
14
13
|
},
|
|
15
14
|
"description": "Rational indexing & cross-shard querying at scale in your NoSQL database so you can focus on your application logic.",
|
|
16
15
|
"devDependencies": {
|
|
17
|
-
"@dotenvx/dotenvx": "^1.
|
|
18
|
-
"@eslint/js": "^9.
|
|
19
|
-
"@faker-js/faker": "^9.
|
|
16
|
+
"@dotenvx/dotenvx": "^1.22.0",
|
|
17
|
+
"@eslint/js": "^9.14.0",
|
|
18
|
+
"@faker-js/faker": "^9.2.0",
|
|
20
19
|
"@karmaniverous/mock-db": "^0.3.3",
|
|
21
20
|
"@rollup/plugin-alias": "^5.1.1",
|
|
22
21
|
"@rollup/plugin-commonjs": "^28.0.1",
|
|
@@ -29,35 +28,34 @@
|
|
|
29
28
|
"@types/eslint-config-prettier": "^6.11.3",
|
|
30
29
|
"@types/eslint-plugin-mocha": "^10.4.0",
|
|
31
30
|
"@types/mocha": "^10.0.9",
|
|
32
|
-
"@types/node": "^22.
|
|
31
|
+
"@types/node": "^22.9.0",
|
|
33
32
|
"@types/string-hash": "^1.1.3",
|
|
34
33
|
"auto-changelog": "^2.5.0",
|
|
35
34
|
"chai": "^5.1.2",
|
|
36
|
-
"
|
|
37
|
-
"eslint": "^9.13.0",
|
|
35
|
+
"eslint": "^9.14.0",
|
|
38
36
|
"eslint-config-prettier": "^9.1.0",
|
|
39
37
|
"eslint-plugin-mocha": "^10.5.0",
|
|
40
38
|
"eslint-plugin-simple-import-sort": "^12.1.1",
|
|
41
39
|
"eslint-plugin-tsdoc": "^0.3.0",
|
|
42
40
|
"jsdom-global": "^3.0.2",
|
|
43
|
-
"knip": "^5.
|
|
41
|
+
"knip": "^5.36.3",
|
|
44
42
|
"lefthook": "^1.8.2",
|
|
45
|
-
"mocha": "^10.
|
|
43
|
+
"mocha": "^10.8.2",
|
|
46
44
|
"nyc": "^17.1.0",
|
|
47
45
|
"prettier": "^3.3.3",
|
|
48
46
|
"release-it": "^17.10.0",
|
|
49
47
|
"rimraf": "^6.0.1",
|
|
50
|
-
"rollup": "^4.24.
|
|
48
|
+
"rollup": "^4.24.4",
|
|
51
49
|
"rollup-plugin-dts": "^6.1.1",
|
|
52
50
|
"source-map-support": "^0.5.21",
|
|
53
51
|
"ts-node": "^10.9.2",
|
|
54
|
-
"tslib": "^2.8.
|
|
55
|
-
"typedoc": "^0.26.
|
|
56
|
-
"typedoc-plugin-mdn-links": "^3.3.
|
|
52
|
+
"tslib": "^2.8.1",
|
|
53
|
+
"typedoc": "^0.26.11",
|
|
54
|
+
"typedoc-plugin-mdn-links": "^3.3.6",
|
|
57
55
|
"typedoc-plugin-replace-text": "^4.0.0",
|
|
58
56
|
"typedoc-plugin-zod": "^1.2.1",
|
|
59
57
|
"typescript": "^5.6.3",
|
|
60
|
-
"typescript-eslint": "^8.
|
|
58
|
+
"typescript-eslint": "^8.13.0"
|
|
61
59
|
},
|
|
62
60
|
"exports": {
|
|
63
61
|
".": {
|
|
@@ -99,6 +97,7 @@
|
|
|
99
97
|
"after:init": [
|
|
100
98
|
"npm run lint",
|
|
101
99
|
"npm run test",
|
|
100
|
+
"npm run knip",
|
|
102
101
|
"npm run build"
|
|
103
102
|
],
|
|
104
103
|
"before:npm:release": [
|
|
@@ -132,5 +131,5 @@
|
|
|
132
131
|
},
|
|
133
132
|
"type": "module",
|
|
134
133
|
"types": "dist/index.d.ts",
|
|
135
|
-
"version": "6.
|
|
134
|
+
"version": "6.7.1"
|
|
136
135
|
}
|