@karmaniverous/entity-manager 6.14.2 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +171 -259
- package/dist/cjs/BaseEntityClient/BaseEntityClient.js +1 -1
- package/dist/cjs/BaseQueryBuilder/BaseQueryBuilder.js +3 -2
- package/dist/cjs/EntityManager/EntityManager.js +2 -2
- package/dist/cjs/EntityManager/createEntityManager.js +42 -0
- package/dist/cjs/EntityManager/decodeGeneratedProperty.js +4 -2
- package/dist/cjs/EntityManager/dehydratePageKeyMap.js +5 -4
- package/dist/cjs/EntityManager/getIndexComponents.js +6 -1
- package/dist/cjs/EntityManager/getPrimaryKey.js +51 -7
- package/dist/cjs/EntityManager/query.js +26 -11
- package/dist/cjs/EntityManager/rehydratePageKeyMap.js +6 -3
- package/dist/cjs/index.js +2 -0
- package/dist/index.d.ts +319 -132
- package/dist/mjs/BaseEntityClient/BaseEntityClient.js +1 -1
- package/dist/mjs/BaseQueryBuilder/BaseQueryBuilder.js +3 -2
- package/dist/mjs/EntityManager/EntityManager.js +2 -2
- package/dist/mjs/EntityManager/createEntityManager.js +40 -0
- package/dist/mjs/EntityManager/decodeGeneratedProperty.js +4 -2
- package/dist/mjs/EntityManager/dehydratePageKeyMap.js +5 -4
- package/dist/mjs/EntityManager/getIndexComponents.js +6 -1
- package/dist/mjs/EntityManager/getPrimaryKey.js +51 -7
- package/dist/mjs/EntityManager/query.js +26 -11
- package/dist/mjs/EntityManager/rehydratePageKeyMap.js +6 -3
- package/dist/mjs/index.js +1 -0
- package/package.json +132 -135
package/README.md
CHANGED
|
@@ -1,163 +1,72 @@
|
|
|
1
|
-
# entity-manager
|
|
1
|
+
# entity-manager
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/@karmaniverous/entity-manager)  <!-- TYPEDOC_EXCLUDE --> [](https://docs.karmanivero.us/entity-manager) [](https://github.com/karmaniverous/entity-manager/tree/main/CHANGELOG.md)<!-- /TYPEDOC_EXCLUDE --> [](https://github.com/karmaniverous/entity-manager/tree/main/LICENSE.md)
|
|
4
|
+
|
|
5
|
+
Entity Manager implements rational indexing & cross‑shard querying at scale in your NoSQL database so you can focus on application logic. It is provider‑agnostic (great fit for DynamoDB) and TypeScript‑first with strong types and runtime validation.
|
|
2
6
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
EntityManager implements rational indexing & cross-shard querying at scale in your NoSQL database so you can focus on your application logic.
|
|
6
|
-
|
|
7
|
-
If you have any questions, please [start a discussion](https://github.com/karmaniverous/entity-manager/discussions). Otherwise stay tuned!
|
|
8
|
-
|
|
9
|
-
## What is this?
|
|
7
|
+
Key links:
|
|
10
8
|
|
|
11
|
-
|
|
9
|
+
- Docs: https://docs.karmanivero.us/entity-manager
|
|
10
|
+
- Discussions: https://github.com/karmaniverous/entity-manager/discussions
|
|
12
11
|
|
|
13
|
-
|
|
14
|
-
- Encode/decode indexable elements via transcodes so strings sort like their original types.
|
|
15
|
-
- Configure a time‑based sharding strategy (shard bumps) that grows as you scale.
|
|
16
|
-
- Query across many shards and indexes in parallel through injected “shard query” functions — results are combined, de‑duplicated, sorted, and returned with a compact, dehydrated page key for the next request.
|
|
12
|
+
## Why this library?
|
|
17
13
|
|
|
18
|
-
|
|
14
|
+
Modern NoSQL puts the burden of indexing, sharding, and pagination on the application. Entity Manager gives you:
|
|
19
15
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
-
|
|
23
|
-
-
|
|
24
|
-
-
|
|
25
|
-
|
|
26
|
-
## Features
|
|
27
|
-
|
|
28
|
-
- Global model for generated properties, indexes, and property transcodes
|
|
29
|
-
- generatedProperties.sharded and generatedProperties.unsharded
|
|
30
|
-
- Global indexes: indexToken → { hashKey, rangeKey, projections? }
|
|
31
|
-
- Global propertyTranscodes: property → transcodeName
|
|
32
|
-
- Deterministic sharding
|
|
33
|
-
- Time‑windowed shard bumps: { timestamp, charBits, chars }
|
|
34
|
-
- Full shard space assignment per bump (uses radix\*\*chars placeholders)
|
|
35
|
-
- Cross‑bump query enumeration over all applicable shards
|
|
36
|
-
- Page key dehydration/rehydration
|
|
37
|
-
- Compact string arrays and lz‑string compression for transport
|
|
38
|
-
- Rehydrate back to pageKey objects for each index+shard
|
|
39
|
-
- Provider‑agnostic parallel query orchestration
|
|
40
|
-
- Inject shard query functions for each index
|
|
41
|
-
- Parallel fan‑out with configurable throttle
|
|
42
|
-
- Combine, dedupe by unique property, and sort results
|
|
43
|
-
- Strong typing + runtime validation
|
|
44
|
-
- Zod‑validated config parsing
|
|
45
|
-
- Robust TypeScript surface for config, items, keys, queries
|
|
16
|
+
- Values‑first configuration with runtime validation (Zod) and best‑in‑class inference.
|
|
17
|
+
- Token‑aware and index‑aware types end‑to‑end (entities, keys, page keys, queries).
|
|
18
|
+
- Deterministic sharding with a time‑based scale‑up schedule.
|
|
19
|
+
- Cross‑shard, multi‑index query orchestration with dedupe and sorting.
|
|
20
|
+
- Dehydration/rehydration of page keys to pass a single compact token between calls.
|
|
46
21
|
|
|
47
22
|
## Install
|
|
48
23
|
|
|
49
24
|
```bash
|
|
50
25
|
npm install @karmaniverous/entity-manager
|
|
51
|
-
# optional
|
|
26
|
+
# optional (tests/demo helpers)
|
|
52
27
|
npm install --save-dev @karmaniverous/mock-db
|
|
53
28
|
```
|
|
54
29
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
-
|
|
64
|
-
-
|
|
65
|
-
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
Below is a minimal end‑to‑end example showing shape and intent. It mirrors the current implementation’s global config model.
|
|
83
|
-
|
|
84
|
-
```ts
|
|
85
|
-
import {
|
|
86
|
-
defaultTranscodes,
|
|
87
|
-
type ConfigMap,
|
|
88
|
-
} from '@karmaniverous/entity-manager';
|
|
89
|
-
import { EntityManager } from '@karmaniverous/entity-manager';
|
|
90
|
-
|
|
91
|
-
// 1) Entity definitions (Typescript types)
|
|
92
|
-
interface User {
|
|
93
|
-
userId: string; // unique property
|
|
94
|
-
created: number; // timestamp property
|
|
95
|
-
updated: number;
|
|
96
|
-
firstNameCanonical: string;
|
|
97
|
-
lastNameCanonical: string;
|
|
98
|
-
// Generated properties exist on stored items but are configured globally:
|
|
99
|
-
// e.g., firstNameRK (unsharded), lastNameRK (unsharded), userPK (sharded)
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
type MyConfigMap = ConfigMap<{
|
|
103
|
-
EntityMap: { user: User };
|
|
104
|
-
HashKey: 'hashKey'; // defaults are 'hashKey' / 'rangeKey' if omitted
|
|
105
|
-
RangeKey: 'rangeKey';
|
|
106
|
-
ShardedKeys: 'userPK'; // token(s) for sharded generated properties
|
|
107
|
-
UnshardedKeys: 'firstNameRK' | 'lastNameRK';
|
|
108
|
-
TranscodedProperties:
|
|
109
|
-
| 'userId'
|
|
110
|
-
| 'created'
|
|
111
|
-
| 'updated'
|
|
112
|
-
| 'firstNameCanonical'
|
|
113
|
-
| 'lastNameCanonical';
|
|
114
|
-
}>;
|
|
115
|
-
|
|
116
|
-
// 2) Build a config with the global model
|
|
117
|
-
const now = Date.now();
|
|
118
|
-
const manager = new EntityManager<MyConfigMap>({
|
|
119
|
-
// Per-entity: unique + timestamp + shard schedule (+ optional defaults)
|
|
120
|
-
entities: {
|
|
121
|
-
user: {
|
|
122
|
-
uniqueProperty: 'userId',
|
|
123
|
-
timestampProperty: 'created',
|
|
124
|
-
shardBumps: [
|
|
125
|
-
// records with timestamp < now → effectively unsharded
|
|
126
|
-
{ timestamp: now, charBits: 1, chars: 0 },
|
|
127
|
-
// records with timestamp ≥ now → 1 char at radix 4 (2^2) gives 4 shards
|
|
128
|
-
{ timestamp: now, charBits: 2, chars: 1 },
|
|
129
|
-
],
|
|
130
|
-
// optional defaults (used by query if omitted in options)
|
|
131
|
-
defaultLimit: 10,
|
|
132
|
-
defaultPageSize: 10,
|
|
133
|
-
},
|
|
134
|
-
},
|
|
30
|
+
## DX highlights
|
|
31
|
+
|
|
32
|
+
- Values‑first + schema‑first config:
|
|
33
|
+
- Use a config literal (prefer `as const` and `satisfies`) to preserve literal tokens.
|
|
34
|
+
- Optionally provide Zod schemas (`entitiesSchema`) to infer entity shapes without generics.
|
|
35
|
+
- Token‑aware helpers:
|
|
36
|
+
- `addKeys`, `getPrimaryKey`, `removeKeys` narrow types by entity token (no casts).
|
|
37
|
+
- Index‑aware page keys (optional CF channel):
|
|
38
|
+
- Provide a values‑first config literal (CF) with `indexes` and get typed page keys per index.
|
|
39
|
+
- Use `QueryOptionsByCF` and `ShardQueryMapByCF` to derive index token unions directly from CF.
|
|
40
|
+
- CC-based DX sugar (values-first captured config):
|
|
41
|
+
- Use `QueryOptionsByCC` and `ShardQueryMapByCC` to derive index token unions from a captured config type (via `IndexTokensFrom`), while still benefiting from page-key narrowing.
|
|
42
|
+
|
|
43
|
+
## Quick start (values‑first + schema‑first)
|
|
44
|
+
|
|
45
|
+
import { z } from 'zod';
|
|
46
|
+
import { createEntityManager, defaultTranscodes } from '@karmaniverous/entity-manager';
|
|
47
|
+
|
|
48
|
+
// 1) Schema-first entity shapes (non-generated fields only)
|
|
49
|
+
const userSchema = z.object({
|
|
50
|
+
userId: z.string(), // unique property
|
|
51
|
+
created: z.number(), // timestamp property
|
|
52
|
+
updated: z.number().optional(),
|
|
53
|
+
firstNameCanonical: z.string(),
|
|
54
|
+
lastNameCanonical: z.string(),
|
|
55
|
+
});
|
|
135
56
|
|
|
136
|
-
|
|
57
|
+
// 2) Values-first config literal (prefer `as const`)
|
|
58
|
+
const config = {
|
|
59
|
+
hashKey: 'hashKey2',
|
|
60
|
+
rangeKey: 'rangeKey',
|
|
137
61
|
generatedProperties: {
|
|
138
62
|
sharded: {
|
|
139
|
-
userPK: ['userId']
|
|
63
|
+
userPK: ['userId'] as const,
|
|
140
64
|
},
|
|
141
65
|
unsharded: {
|
|
142
|
-
firstNameRK: ['firstNameCanonical', 'lastNameCanonical'
|
|
143
|
-
lastNameRK: ['lastNameCanonical', 'firstNameCanonical'
|
|
66
|
+
firstNameRK: ['firstNameCanonical', 'lastNameCanonical'] as const,
|
|
67
|
+
lastNameRK: ['lastNameCanonical', 'firstNameCanonical'] as const,
|
|
144
68
|
},
|
|
145
69
|
},
|
|
146
|
-
|
|
147
|
-
// Global key tokens
|
|
148
|
-
hashKey: 'hashKey',
|
|
149
|
-
rangeKey: 'rangeKey',
|
|
150
|
-
|
|
151
|
-
// Global indexes (hashKey, rangeKey must match allowed token sets)
|
|
152
|
-
indexes: {
|
|
153
|
-
created: { hashKey: 'hashKey', rangeKey: 'created' },
|
|
154
|
-
updated: { hashKey: 'hashKey', rangeKey: 'updated' },
|
|
155
|
-
firstName: { hashKey: 'hashKey', rangeKey: 'firstNameRK' },
|
|
156
|
-
lastName: { hashKey: 'hashKey', rangeKey: 'lastNameRK' },
|
|
157
|
-
userCreated: { hashKey: 'userPK', rangeKey: 'created' }, // sharded alt hash
|
|
158
|
-
},
|
|
159
|
-
|
|
160
|
-
// Transcode mapping for scalar/unsharded elements and properties
|
|
161
70
|
propertyTranscodes: {
|
|
162
71
|
userId: 'string',
|
|
163
72
|
created: 'timestamp',
|
|
@@ -165,179 +74,183 @@ const manager = new EntityManager<MyConfigMap>({
|
|
|
165
74
|
firstNameCanonical: 'string',
|
|
166
75
|
lastNameCanonical: 'string',
|
|
167
76
|
},
|
|
168
|
-
|
|
169
|
-
|
|
77
|
+
indexes: {
|
|
78
|
+
created: { hashKey: 'hashKey2', rangeKey: 'created' },
|
|
79
|
+
userCreated: { hashKey: 'userPK', rangeKey: 'created' },
|
|
80
|
+
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
81
|
+
lastName: { hashKey: 'hashKey2', rangeKey: 'lastNameRK' },
|
|
82
|
+
} as const,
|
|
83
|
+
entities: {
|
|
84
|
+
user: {
|
|
85
|
+
uniqueProperty: 'userId',
|
|
86
|
+
timestampProperty: 'created',
|
|
87
|
+
shardBumps: [{ timestamp: Date.now(), charBits: 2, chars: 1 }],
|
|
88
|
+
},
|
|
89
|
+
},
|
|
90
|
+
entitiesSchema: { user: userSchema },
|
|
170
91
|
transcodes: defaultTranscodes,
|
|
92
|
+
} as const;
|
|
171
93
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
generatedValueDelimiter: '#',
|
|
175
|
-
shardKeyDelimiter: '!',
|
|
176
|
-
throttle: 10,
|
|
177
|
-
});
|
|
94
|
+
// 3) Create the manager — types captured from values, shapes from schemas
|
|
95
|
+
const manager = createEntityManager(config);
|
|
178
96
|
```
|
|
179
97
|
|
|
180
|
-
###
|
|
98
|
+
### Token‑aware helpers
|
|
181
99
|
|
|
182
100
|
```ts
|
|
183
|
-
//
|
|
101
|
+
// Input item (no generated keys yet)
|
|
184
102
|
const user = {
|
|
185
|
-
userId: '
|
|
103
|
+
userId: 'u1',
|
|
186
104
|
created: Date.now(),
|
|
187
|
-
updated: Date.now(),
|
|
188
105
|
firstNameCanonical: 'lee',
|
|
189
106
|
lastNameCanonical: 'zhang',
|
|
190
107
|
};
|
|
191
108
|
|
|
192
|
-
// Add hashKey
|
|
193
|
-
const record = manager.addKeys('user', user);
|
|
109
|
+
// Add generated keys (hashKey/rangeKey + index tokens)
|
|
110
|
+
const record = manager.addKeys('user', user);
|
|
194
111
|
|
|
195
|
-
//
|
|
196
|
-
const
|
|
112
|
+
// Compute one or more primary keys
|
|
113
|
+
const keys = manager.getPrimaryKey('user', { userId: 'u1' });
|
|
197
114
|
|
|
198
|
-
//
|
|
199
|
-
const
|
|
115
|
+
// Strip generated keys after read
|
|
116
|
+
const item = manager.removeKeys('user', record);
|
|
200
117
|
```
|
|
201
118
|
|
|
202
|
-
|
|
119
|
+
Types narrow automatically from the entity token (`'user'`). No casts required.
|
|
203
120
|
|
|
204
|
-
|
|
205
|
-
// Encode an unsharded generated property (always returns a string)
|
|
206
|
-
const fn = manager.encodeGeneratedProperty('firstNameRK', record);
|
|
207
|
-
// e.g. "firstNameCanonical#lee|lastNameCanonical#zhang|created#000001711234567"
|
|
121
|
+
## Index‑aware querying (CF channel)
|
|
208
122
|
|
|
209
|
-
|
|
210
|
-
import { decodeGeneratedProperty } from '@karmaniverous/entity-manager';
|
|
211
|
-
const decoded = decodeGeneratedProperty(manager, fn); // { firstNameCanonical: 'lee', ... }
|
|
212
|
-
```
|
|
123
|
+
When you author a values‑first config literal with `indexes` (prefer `as const`), Entity Manager can:
|
|
213
124
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
- page‑key rehydration → parallel shard queries → de‑duplication and sorting → page‑key dehydration.
|
|
125
|
+
- Constrain `shardQueryMap` keys to the index key union.
|
|
126
|
+
- Narrow page‑key shapes per index (only its component tokens).
|
|
127
|
+
- Derive ITS (index token subset) automatically from CF via `QueryOptionsByCF` and `ShardQueryMapByCF`.
|
|
219
128
|
|
|
220
129
|
```ts
|
|
221
130
|
import type {
|
|
222
|
-
QueryOptions,
|
|
223
131
|
ShardQueryFunction,
|
|
132
|
+
ShardQueryMapByCF,
|
|
133
|
+
QueryOptionsByCF,
|
|
224
134
|
} from '@karmaniverous/entity-manager';
|
|
225
135
|
|
|
226
|
-
//
|
|
227
|
-
const
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
136
|
+
// CF: capture index tokens from a values-first literal
|
|
137
|
+
const cf = {
|
|
138
|
+
indexes: {
|
|
139
|
+
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
140
|
+
lastName: { hashKey: 'hashKey2', rangeKey: 'lastNameRK' },
|
|
141
|
+
},
|
|
142
|
+
} as const;
|
|
143
|
+
type CF = typeof cf;
|
|
144
|
+
|
|
145
|
+
// SQFs are typed; pageKey is narrowed to index components per IT
|
|
146
|
+
const firstNameSQF: ShardQueryFunction<MyConfigMap, 'user', 'firstName', CF> =
|
|
147
|
+
async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
148
|
+
const lastNameSQF: ShardQueryFunction<MyConfigMap, 'user', 'lastName', CF> =
|
|
149
|
+
async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
150
|
+
|
|
151
|
+
// CF-aware shardQueryMap — only 'firstName' | 'lastName' allowed
|
|
152
|
+
const shardQueryMap: ShardQueryMapByCF<MyConfigMap, 'user', CF> = {
|
|
153
|
+
firstName: firstNameSQF,
|
|
154
|
+
lastName: lastNameSQF,
|
|
236
155
|
};
|
|
237
156
|
|
|
238
|
-
//
|
|
239
|
-
const
|
|
157
|
+
// Derive ITS from CF for options
|
|
158
|
+
const options: QueryOptionsByCF<MyConfigMap, 'user', CF> = {
|
|
240
159
|
entityToken: 'user',
|
|
241
|
-
item: {},
|
|
242
|
-
shardQueryMap
|
|
160
|
+
item: {},
|
|
161
|
+
shardQueryMap,
|
|
243
162
|
limit: 50,
|
|
244
163
|
pageSize: 10,
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const result = await manager.query(options);
|
|
167
|
+
// result.pageKeyMap is a compact string — pass it to the next call’s options.pageKeyMap
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### CC-based aliases
|
|
171
|
+
|
|
172
|
+
You can also derive ITS (index token subset) directly from a values‑first captured config type (CC) using `QueryOptionsByCC` and `ShardQueryMapByCC`. This mirrors the CF helpers but drives ITS from the CC type (via `IndexTokensFrom`) and passes the same CC through the CF channel for page‑key narrowing.
|
|
173
|
+
|
|
174
|
+
```ts
|
|
175
|
+
import type {
|
|
176
|
+
ShardQueryFunction,
|
|
177
|
+
ShardQueryMapByCC,
|
|
178
|
+
QueryOptionsByCC,
|
|
179
|
+
} from '@karmaniverous/entity-manager';
|
|
180
|
+
|
|
181
|
+
// A values-first config literal capturing index tokens (the same shape used for CF)
|
|
182
|
+
const cc = {
|
|
183
|
+
indexes: {
|
|
184
|
+
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
185
|
+
lastName: { hashKey: 'hashKey2', rangeKey: 'lastNameRK' },
|
|
186
|
+
},
|
|
187
|
+
} as const;
|
|
188
|
+
type CC = typeof cc;
|
|
189
|
+
|
|
190
|
+
// Reuse typed SQFs (pageKey narrowed per index)
|
|
191
|
+
const firstNameSQF: ShardQueryFunction<MyConfigMap, 'user', 'firstName', CC> =
|
|
192
|
+
async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
193
|
+
const lastNameSQF: ShardQueryFunction<MyConfigMap, 'user', 'lastName', CC> =
|
|
194
|
+
async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
195
|
+
|
|
196
|
+
// CC-aware shardQueryMap — only 'firstName' | 'lastName' allowed
|
|
197
|
+
const shardQueryMapCC: ShardQueryMapByCC<MyConfigMap, 'user', CC> = {
|
|
198
|
+
firstName: firstNameSQF,
|
|
199
|
+
lastName: lastNameSQF,
|
|
200
|
+
};
|
|
201
|
+
const optionsCC: QueryOptionsByCC<MyConfigMap, 'user', CC> = { entityToken: 'user', item: {}, shardQueryMap: shardQueryMapCC };
|
|
202
|
+
const resultCC = await manager.query(optionsCC);
|
|
248
203
|
```
|
|
249
204
|
|
|
250
205
|
Notes:
|
|
251
206
|
|
|
252
|
-
-
|
|
253
|
-
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
-
|
|
259
|
-
- generatedProperties:
|
|
260
|
-
- sharded: Record<ShardedKey, TranscodedProperties[]>
|
|
261
|
-
- unsharded: Record<UnshardedKey, TranscodedProperties[]>
|
|
262
|
-
- indexes: Record<indexToken, {
|
|
263
|
-
- hashKey: HashKey | ShardedKey
|
|
264
|
-
- rangeKey: RangeKey | UnshardedKey | TranscodedProperties
|
|
265
|
-
- projections?: string[]
|
|
266
|
-
}>
|
|
267
|
-
- propertyTranscodes: Record<TranscodedProperties, keyof TranscodeMap>
|
|
268
|
-
- transcodes: Record<transcodeName, { encode, decode }> (defaults to defaultTranscodes)
|
|
269
|
-
- hashKey: HashKey (e.g., 'hashKey')
|
|
270
|
-
- rangeKey: RangeKey (e.g., 'rangeKey')
|
|
271
|
-
- generatedKeyDelimiter: string (default '|', must match /\W+/)
|
|
272
|
-
- generatedValueDelimiter: string (default '#', must match /\W+/)
|
|
273
|
-
- shardKeyDelimiter: string (default '!', must match /\W+/)
|
|
274
|
-
- throttle: number (default 10)
|
|
275
|
-
|
|
276
|
-
Validation highlights:
|
|
277
|
-
|
|
278
|
-
- Delimiters must not contain each other.
|
|
279
|
-
- Keys and tokens must be mutually exclusive as required.
|
|
280
|
-
- Generated property element lists are non‑empty and have no duplicates.
|
|
281
|
-
- propertyTranscodes values must exist in transcodes.
|
|
282
|
-
- Index hashKey/rangeKey must use valid token sets.
|
|
283
|
-
- shardBumps are sorted, include a zero‑timestamp bump if missing, and chars must increase monotonically with timestamp.
|
|
284
|
-
|
|
285
|
-
Sharding:
|
|
286
|
-
|
|
287
|
-
- For assignment: a record always uses all placeholders for its applicable bump; suffix space is (2**charBits) ** chars.
|
|
288
|
-
- For queries: hash key space spans all bumps overlapping [timestampFrom, timestampTo].
|
|
207
|
+
- Entity Manager enumerates hash‑key space for the time window, rehydrates page keys (when present), executes shard queries in parallel (throttled), dedupes by unique property, sorts, and dehydrates a new pageKeyMap.
|
|
208
|
+
- For provider integration, the SQF lambda encapsulates the platform‑specific query for one index + shard page. See tests and entity‑client‑dynamodb for examples.
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
- `rehydratePageKeyMap` decodes a dehydrated array (compressed string) into a two‑layer map of `{ indexToken: { hashKeyValue: pageKey | undefined } }`.
|
|
212
|
+
- `dehydratePageKeyMap` performs the inverse and emits a compact array (compressed in `query()`).
|
|
213
|
+
- You rarely call these directly — `query()` composes them for you — but the API is exposed for advanced flows.
|
|
289
214
|
|
|
290
215
|
## ESM / CJS
|
|
291
216
|
|
|
292
217
|
```ts
|
|
293
218
|
// ESM
|
|
294
|
-
import {
|
|
295
|
-
EntityManager,
|
|
296
|
-
defaultTranscodes,
|
|
297
|
-
} from '@karmaniverous/entity-manager';
|
|
219
|
+
import { createEntityManager, defaultTranscodes } from '@karmaniverous/entity-manager';
|
|
298
220
|
|
|
299
221
|
// CJS
|
|
300
|
-
const {
|
|
301
|
-
EntityManager,
|
|
302
|
-
defaultTranscodes,
|
|
303
|
-
} = require('@karmaniverous/entity-manager');
|
|
222
|
+
const { createEntityManager, defaultTranscodes } = require('@karmaniverous/entity-manager');
|
|
304
223
|
```
|
|
305
224
|
|
|
306
225
|
## Logging
|
|
307
226
|
|
|
308
|
-
|
|
227
|
+
Entity Manager logs debug and error details via the injected logger (defaults to `console`).
|
|
309
228
|
|
|
310
229
|
```ts
|
|
311
230
|
const logger = { debug: () => undefined, error: console.error };
|
|
312
|
-
const manager =
|
|
231
|
+
const manager = createEntityManager(config, logger);
|
|
313
232
|
```
|
|
314
233
|
|
|
315
|
-
##
|
|
316
|
-
|
|
317
|
-
Generated key/value delimiters and the shard key delimiter are used when composing strings:
|
|
318
|
-
|
|
319
|
-
- generatedKeyDelimiter: '|' (between pairs)
|
|
320
|
-
- generatedValueDelimiter: '#' (between key and value)
|
|
321
|
-
- shardKeyDelimiter: '!' (between entity token and shard suffix)
|
|
322
|
-
|
|
323
|
-
Your scalar property values used in generated properties should not include these delimiters. If they must, set custom delimiters (must match /\W+/ and not contain each other).
|
|
324
|
-
|
|
325
|
-
## Types you’ll use most
|
|
234
|
+
## Types you’ll reach for
|
|
326
235
|
|
|
327
|
-
-
|
|
328
|
-
-
|
|
329
|
-
-
|
|
330
|
-
-
|
|
331
|
-
-
|
|
236
|
+
- Values/schema capture
|
|
237
|
+
- `createEntityManager(config, logger?)`
|
|
238
|
+
- `ConfigInput` (values‑first), `CapturedConfigMapFrom`, `EntitiesFromSchema`
|
|
239
|
+
- Token aware
|
|
240
|
+
- `EntityToken<CC>`, `EntityItemByToken<CC, ET>`, `EntityRecordByToken<CC, ET>`
|
|
241
|
+
- Index aware (CF channel)
|
|
242
|
+
- `PageKeyByIndex<CC, ET, IT, CF>`
|
|
243
|
+
- `ShardQueryFunction<CC, ET, IT, CF>`, `ShardQueryMap<CC, ET, ITS, CF>`
|
|
244
|
+
- `QueryOptions<CC, ET, ITS, CF>`, `QueryResult<CC, ET, ITS>`
|
|
245
|
+
- DX sugar: `IndexTokensOf<CF>`, `QueryOptionsByCF`, `ShardQueryMapByCF`, `IndexTokensFrom<CC>`, `QueryOptionsByCC`, `ShardQueryMapByCC`
|
|
332
246
|
|
|
333
247
|
See the full API: https://docs.karmanivero.us/entity-manager
|
|
334
248
|
|
|
335
249
|
## Scripts (repo)
|
|
336
250
|
|
|
337
|
-
- build: rollup outputs ESM/CJS + .d.ts
|
|
338
|
-
- test: vitest with coverage
|
|
339
|
-
- lint: ESLint (type‑aware) + Prettier integration
|
|
340
|
-
- docs: TypeDoc (links to external type docs for shared utility packages)
|
|
251
|
+
- build: rollup outputs ESM/CJS + .d.ts
|
|
341
|
-
|
|
252
|
+
- lint: ESLint (type‑aware) + Prettier
|
|
253
|
+
- docs: TypeDoc
|
|
342
254
|
- typecheck: tsc + tsd (type‑level tests)
|
|
343
255
|
|
|
344
256
|
## License
|
|
@@ -346,4 +259,4 @@ BSD‑3‑Clause (see package.json).
|
|
|
346
259
|
|
|
347
260
|
---
|
|
348
261
|
|
|
349
|
-
Built for you with ❤️ on Bali! Find more great tools & templates on [my GitHub Profile](https://github.com/karmaniverous).
|
|
262
|
+
Built for you with ❤️ on Bali! Find more great tools & templates on [my GitHub Profile](https://github.com/karmaniverous).
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* Base EntityClient class. Integrates {@link EntityManager | `EntityManager`} with injected logging & enhanced batch processing.
|
|
5
5
|
*
|
|
6
|
-
* @typeParam
|
|
6
|
+
* @typeParam CC - {@link ConfigMap | `ConfigMap`} that defines an {@link Config | `EntityManager configuration`}'s {@link EntityMap | `EntityMap`}, key properties, and {@link TranscodeRegistry | `TranscodeRegistry`}. If omitted, defaults to {@link BaseConfigMap | `BaseConfigMap`}.
|
|
7
7
|
*
|
|
8
8
|
* @category EntityClient
|
|
9
9
|
*/
|
|
@@ -5,9 +5,10 @@ var radash = require('radash');
|
|
|
5
5
|
/**
|
|
6
6
|
* Abstract base class supporting a fluent API for building a {@link ShardQueryMap | `ShardQueryMap`} using a database client.
|
|
7
7
|
*
|
|
8
|
-
* @typeParam
|
|
8
|
+
* @typeParam CC - {@link ConfigMap | `ConfigMap`} that defines an {@link Config | `EntityManager configuration`}'s {@link EntityMap | `EntityMap`}, key properties, and {@link TranscodeRegistry | `TranscodeRegistry`}. If omitted, defaults to {@link BaseConfigMap | `BaseConfigMap`}.
|
|
9
9
|
* @typeParam EntityClient - {@link BaseEntityClient | `BaseEntityClient`} derived class instance.
|
|
10
10
|
* @typeParam IndexParams - Database platform-specific, index-specific query parameters.
|
|
11
|
+
* @typeParam CF - Optional values-first config literal type for page key narrowing.
|
|
11
12
|
*
|
|
12
13
|
* @category QueryBuilder
|
|
13
14
|
*/
|
|
@@ -32,7 +33,7 @@ class BaseQueryBuilder {
|
|
|
32
33
|
* @returns - The {@link ShardQueryMap | `ShardQueryMap`} object.
|
|
33
34
|
*/
|
|
34
35
|
build() {
|
|
35
|
-
return radash.mapValues(this.indexParamsMap, (
|
|
36
|
+
return radash.mapValues(this.indexParamsMap, (_indexConfig, indexToken) => this.getShardQueryFunction(indexToken));
|
|
36
37
|
}
|
|
37
38
|
async query(options) {
|
|
38
39
|
const { entityClient: { entityManager }, entityToken, pageKeyMap, } = this;
|
|
@@ -14,7 +14,7 @@ var _EntityManager_config;
|
|
|
14
14
|
* The EntityManager class applies a configuration-driven sharded data model &
|
|
15
15
|
* query strategy to NoSql data.
|
|
16
16
|
*
|
|
17
|
-
* @typeParam
|
|
17
|
+
* @typeParam CC - {@link ConfigMap | `ConfigMap`} that defines the configuration's {@link EntityMap | `EntityMap`}, key properties, and {@link TranscodeRegistry | `TranscodeRegistry`}. If omitted, defaults to {@link BaseConfigMap | `BaseConfigMap`}.
|
|
18
18
|
*
|
|
19
19
|
* @remarks
|
|
20
20
|
* While the {@link EntityManager.query | `query`} method is `public`, normally it should not be called directly. The `query` method is used by a platform-specific {@link BaseQueryBuilder.query | `QueryBuilder.query`} method to provide a fluent query API.
|
|
@@ -70,7 +70,7 @@ class EntityManager {
|
|
|
70
70
|
}
|
|
71
71
|
getPrimaryKey(entityToken, i, overwrite = false) {
|
|
72
72
|
if (Array.isArray(i)) {
|
|
73
|
-
return i.
|
|
73
|
+
return i.flatMap((item) => getPrimaryKey.getPrimaryKey(this, entityToken, item, overwrite));
|
|
74
74
|
}
|
|
75
75
|
return getPrimaryKey.getPrimaryKey(this, entityToken, i, overwrite);
|
|
76
76
|
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var EntityManager = require('./EntityManager.js');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Values-first factory that captures literal tokens and index names directly
|
|
7
|
+
* from the provided config value. Runtime config parsing/validation is
|
|
8
|
+
* unchanged (performed in the EntityManager constructor).
|
|
9
|
+
*
|
|
10
|
+
* @typeParam CC - Captured config input (values-first). Prefer `as const` and
|
|
11
|
+
* `satisfies` at call sites to preserve literal keys.
|
|
12
|
+
* @typeParam EM - EntityMap for the manager. Defaults to a minimal derived map
|
|
13
|
+
* from `CC.entitiesSchema` when present; otherwise falls back to EntityMap.
|
|
14
|
+
*/
|
|
15
|
+
function createEntityManager(config, logger = console) {
|
|
16
|
+
// Cast to the existing Config<C> shape for runtime parsing; Zod validation
|
|
17
|
+
// remains authoritative at construction time.
|
|
18
|
+
// Optional dev guardrail: cross-check entitiesSchema keys vs config.entities keys.
|
|
19
|
+
try {
|
|
20
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
21
|
+
if (config && 'entitiesSchema' in config && config.entitiesSchema) {
|
|
22
|
+
const schemaKeys = Object.keys(config
|
|
23
|
+
.entitiesSchema ?? {});
|
|
24
|
+
const entitiesKeys = Object.keys(config
|
|
25
|
+
.entities ?? {});
|
|
26
|
+
const missingInEntities = schemaKeys.filter((k) => !entitiesKeys.includes(k));
|
|
27
|
+
const missingInSchema = entitiesKeys.filter((k) => !schemaKeys.includes(k));
|
|
28
|
+
if (missingInEntities.length || missingInSchema.length) {
|
|
29
|
+
logger.debug('entitiesSchema keys mismatch with config.entities', {
|
|
30
|
+
missingInEntities,
|
|
31
|
+
missingInSchema,
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
// Best-effort warning only; never block construction.
|
|
38
|
+
}
|
|
39
|
+
return new EntityManager.EntityManager(config, logger);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
exports.createEntityManager = createEntityManager;
|
|
@@ -7,14 +7,14 @@ var decodeElement = require('./decodeElement.js');
|
|
|
7
7
|
* Decode a generated property value. Returns an {@link EntityItem | `EntityItem`}.
|
|
8
8
|
*
|
|
9
9
|
* @param entityManager - {@link EntityManager | `EntityManager`} instance.
|
|
10
|
-
* @param entityToken - `entityManager.config.entities` key.
|
|
10
|
+
* @param entityToken - {@link Config.entities | `entityManager.config.entities`} key.
|
|
11
11
|
* @param encoded - Encoded generated property value.
|
|
12
12
|
*
|
|
13
13
|
* @returns {@link EntityItem | `EntityItem`} object with updated properties decoded from `encoded`.
|
|
14
14
|
*
|
|
15
15
|
* @throws `Error` if `entityToken` is invalid.
|
|
16
16
|
*/
|
|
17
|
-
function decodeGeneratedProperty(entityManager, encoded) {
|
|
17
|
+
function decodeGeneratedProperty(entityManager, entityToken, encoded) {
|
|
18
18
|
try {
|
|
19
19
|
const { generatedKeyDelimiter, generatedValueDelimiter, hashKey, shardKeyDelimiter, } = entityManager.config;
|
|
20
20
|
// Handle degenerate case.
|
|
@@ -39,6 +39,8 @@ function decodeGeneratedProperty(entityManager, encoded) {
|
|
|
39
39
|
encoded,
|
|
40
40
|
decoded,
|
|
41
41
|
});
|
|
42
|
+
// entityToken used for typing only (ET-narrowed result).
|
|
43
|
+
void entityToken;
|
|
42
44
|
return decoded;
|
|
43
45
|
}
|
|
44
46
|
catch (error) {
|