@karmaniverous/entity-manager 7.1.0 → 7.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -1,393 +1,393 @@
|
|
|
1
|
-
# entity-manager
|
|
2
|
-
|
|
3
|
-
[](https://www.npmjs.com/package/@karmaniverous/entity-manager)  <!-- TYPEDOC_EXCLUDE --> [](https://docs.karmanivero.us/entity-manager) [](https://github.com/karmaniverous/entity-manager/tree/main/CHANGELOG.md)<!-- /TYPEDOC_EXCLUDE --> [](https://github.com/karmaniverous/entity-manager/tree/main/LICENSE.md)
|
|
4
|
-
|
|
5
|
-
Entity Manager implements rational indexing & cross‑shard querying at scale in your NoSQL database so you can focus on application logic. It is provider‑agnostic (great fit for DynamoDB) and TypeScript‑first with strong types and runtime validation.
|
|
6
|
-
|
|
7
|
-
Key links:
|
|
8
|
-
|
|
9
|
-
- Docs: https://docs.karmanivero.us/entity-manager
|
|
10
|
-
- Discussions: https://github.com/karmaniverous/entity-manager/discussions
|
|
11
|
-
|
|
12
|
-
## Why this library?
|
|
13
|
-
|
|
14
|
-
Modern NoSQL puts the burden of indexing, sharding, and pagination on the application. Entity Manager gives you:
|
|
15
|
-
|
|
16
|
-
- Values‑first configuration with runtime validation (Zod) and best‑in‑class inference.
|
|
17
|
-
- Token‑aware and index‑aware types end‑to‑end (entities, keys, page keys, queries).
|
|
18
|
-
- Deterministic sharding with a time‑based scale‑up schedule.
|
|
19
|
-
- Cross‑shard, multi‑index query orchestration with dedupe and sorting.
|
|
20
|
-
- Dehydration/rehydration of page keys to pass a single compact token between calls.
|
|
21
|
-
|
|
22
|
-
## Install
|
|
23
|
-
|
|
24
|
-
```bash
|
|
25
|
-
npm install @karmaniverous/entity-manager
|
|
26
|
-
# optional (tests/demo helpers)
|
|
27
|
-
npm install --save-dev @karmaniverous/mock-db
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
## DX highlights
|
|
31
|
-
|
|
32
|
-
- Values‑first + schema‑first config:
|
|
33
|
-
- Use a config literal (prefer `as const` and `satisfies`) to preserve literal tokens.
|
|
34
|
-
- Optionally provide Zod schemas (`entitiesSchema`) to infer entity shapes without generics.
|
|
35
|
-
- Projection‑aware typing (type‑only K):
|
|
36
|
-
- Pass attributes as a const tuple (K) through your query types to narrow result items to Pick<…> of those properties.
|
|
37
|
-
- No runtime change; adapters execute projections. Adapters should auto‑include `uniqueProperty` and any explicit sort keys when callers omit them to preserve dedupe/sort invariants.
|
|
38
|
-
- Index‑aware typing (CF
|
|
39
|
-
- CF: drive index token unions and page‑key narrowing directly from a values‑first config literal (`QueryOptionsByCF`, `ShardQueryMapByCF`).
|
|
40
|
-
- CC: derive index tokens from a captured config type while reusing CF for narrowing (`QueryOptionsByCC`, `ShardQueryMapByCC`).
|
|
41
|
-
- Token‑aware helpers:
|
|
42
|
-
- `addKeys`, `getPrimaryKey`, `removeKeys` narrow types by entity token (no casts).
|
|
43
|
-
- Index‑aware page keys (optional CF channel):
|
|
44
|
-
- Provide a values‑first config literal (CF) with `indexes` and get typed page keys per index.
|
|
45
|
-
- Use `QueryOptionsByCF` and `ShardQueryMapByCF` to derive index token unions directly from CF.
|
|
46
|
-
- CC-based DX sugar (values-first captured config):
|
|
47
|
-
- Use `QueryOptionsByCC` and `ShardQueryMapByCC` to derive index token unions from a captured config type (via `IndexTokensFrom`), while still benefiting from page-key narrowing.
|
|
48
|
-
|
|
49
|
-
## Quick start (values‑first + schema‑first)
|
|
50
|
-
|
|
51
|
-
```ts
|
|
52
|
-
import { z } from 'zod';
|
|
53
|
-
import {
|
|
54
|
-
createEntityManager,
|
|
55
|
-
defaultTranscodes,
|
|
56
|
-
} from '@karmaniverous/entity-manager';
|
|
57
|
-
|
|
58
|
-
// 1) Schema-first entity shapes (non-generated fields only)
|
|
59
|
-
const userSchema = z.object({
|
|
60
|
-
userId: z.string(), // unique property
|
|
61
|
-
created: z.number(), // timestamp property
|
|
62
|
-
updated: z.number().optional(),
|
|
63
|
-
firstNameCanonical: z.string(),
|
|
64
|
-
lastNameCanonical: z.string(),
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
// 2) Values-first config literal (prefer `as const`)
|
|
68
|
-
const config = {
|
|
69
|
-
hashKey: 'hashKey2',
|
|
70
|
-
rangeKey: 'rangeKey',
|
|
71
|
-
generatedProperties: {
|
|
72
|
-
sharded: {
|
|
73
|
-
userPK: ['userId'] as const,
|
|
74
|
-
},
|
|
75
|
-
unsharded: {
|
|
76
|
-
firstNameRK: ['firstNameCanonical', 'lastNameCanonical'] as const,
|
|
77
|
-
lastNameRK: ['lastNameCanonical', 'firstNameCanonical'] as const,
|
|
78
|
-
},
|
|
79
|
-
},
|
|
80
|
-
propertyTranscodes: {
|
|
81
|
-
userId: 'string',
|
|
82
|
-
created: 'timestamp',
|
|
83
|
-
updated: 'timestamp',
|
|
84
|
-
firstNameCanonical: 'string',
|
|
85
|
-
lastNameCanonical: 'string',
|
|
86
|
-
},
|
|
87
|
-
indexes: {
|
|
88
|
-
created: { hashKey: 'hashKey2', rangeKey: 'created' },
|
|
89
|
-
userCreated: { hashKey: 'userPK', rangeKey: 'created' },
|
|
90
|
-
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
91
|
-
lastName: { hashKey: 'hashKey2', rangeKey: 'lastNameRK' },
|
|
92
|
-
} as const,
|
|
93
|
-
entities: {
|
|
94
|
-
user: {
|
|
95
|
-
uniqueProperty: 'userId',
|
|
96
|
-
timestampProperty: 'created',
|
|
97
|
-
shardBumps: [{ timestamp: Date.now(), charBits: 2, chars: 1 }],
|
|
98
|
-
},
|
|
99
|
-
},
|
|
100
|
-
entitiesSchema: { user: userSchema },
|
|
101
|
-
transcodes: defaultTranscodes,
|
|
102
|
-
} as const;
|
|
103
|
-
|
|
104
|
-
// 3) Create the manager — types captured from values, shapes from schemas
|
|
105
|
-
const manager = createEntityManager(config);
|
|
106
|
-
```
|
|
107
|
-
|
|
108
|
-
### Token‑aware helpers
|
|
109
|
-
|
|
110
|
-
```ts
|
|
111
|
-
// Input item (no generated keys yet)
|
|
112
|
-
const user = {
|
|
113
|
-
userId: 'u1',
|
|
114
|
-
created: Date.now(),
|
|
115
|
-
firstNameCanonical: 'lee',
|
|
116
|
-
lastNameCanonical: 'zhang',
|
|
117
|
-
};
|
|
118
|
-
|
|
119
|
-
// Add generated keys (hashKey/rangeKey + index tokens)
|
|
120
|
-
const record = manager.addKeys('user', user);
|
|
121
|
-
|
|
122
|
-
// Compute one or more primary keys
|
|
123
|
-
const keys = manager.getPrimaryKey('user', { userId: 'u1' });
|
|
124
|
-
|
|
125
|
-
// Strip generated keys after read
|
|
126
|
-
const item = manager.removeKeys('user', record);
|
|
127
|
-
```
|
|
128
|
-
|
|
129
|
-
Types narrow automatically from the entity token (`'user'`). No casts required.
|
|
130
|
-
|
|
131
|
-
## Index‑aware querying (CF channel)
|
|
132
|
-
|
|
133
|
-
When you author a values‑first config literal with `indexes` (prefer `as const`), Entity Manager can:
|
|
134
|
-
|
|
135
|
-
- Constrain `shardQueryMap` keys to the index key union.
|
|
136
|
-
- Narrow page‑key shapes per index (only its component tokens).
|
|
137
|
-
- Derive ITS (index token subset) automatically from CF via `QueryOptionsByCF` and `ShardQueryMapByCF`.
|
|
138
|
-
|
|
139
|
-
```ts
|
|
140
|
-
import type {
|
|
141
|
-
ShardQueryFunction,
|
|
142
|
-
ShardQueryMapByCF,
|
|
143
|
-
QueryOptionsByCF,
|
|
144
|
-
} from '@karmaniverous/entity-manager';
|
|
145
|
-
|
|
146
|
-
// CF: capture index tokens from a values-first literal
|
|
147
|
-
const cf = {
|
|
148
|
-
indexes: {
|
|
149
|
-
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
150
|
-
lastName: { hashKey: 'hashKey2', rangeKey: 'lastNameRK' },
|
|
151
|
-
},
|
|
152
|
-
} as const;
|
|
153
|
-
type CF = typeof cf;
|
|
154
|
-
|
|
155
|
-
// SQFs are typed; pageKey is narrowed to index components per IT
|
|
156
|
-
const firstNameSQF: ShardQueryFunction<
|
|
157
|
-
MyConfigMap,
|
|
158
|
-
'user',
|
|
159
|
-
'firstName',
|
|
160
|
-
CF
|
|
161
|
-
> = async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
162
|
-
const lastNameSQF: ShardQueryFunction<
|
|
163
|
-
MyConfigMap,
|
|
164
|
-
'user',
|
|
165
|
-
'lastName',
|
|
166
|
-
CF
|
|
167
|
-
> = async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
168
|
-
|
|
169
|
-
// CF-aware shardQueryMap — only 'firstName' | 'lastName' allowed
|
|
170
|
-
const shardQueryMap: ShardQueryMapByCF<MyConfigMap, 'user', CF> = {
|
|
171
|
-
firstName: firstNameSQF,
|
|
172
|
-
lastName: lastNameSQF,
|
|
173
|
-
};
|
|
174
|
-
|
|
175
|
-
// Derive ITS from CF for options
|
|
176
|
-
const options: QueryOptionsByCF<MyConfigMap, 'user', CF> = {
|
|
177
|
-
entityToken: 'user',
|
|
178
|
-
item: {},
|
|
179
|
-
shardQueryMap,
|
|
180
|
-
limit: 50,
|
|
181
|
-
pageSize: 10,
|
|
182
|
-
};
|
|
183
|
-
|
|
184
|
-
const result = await manager.query(options);
|
|
185
|
-
// result.pageKeyMap is a compact string — pass it to the next call’s options.pageKeyMap
|
|
186
|
-
```
|
|
187
|
-
|
|
188
|
-
### CC
|
|
189
|
-
|
|
190
|
-
You can also derive ITS (index token subset) directly from a values‑first captured config type (CC) using `QueryOptionsByCC` and `ShardQueryMapByCC`. This mirrors the CF helpers but drives ITS from the CC type (via `IndexTokensFrom`) and passes the same CC through the CF channel for page‑key narrowing.
|
|
191
|
-
|
|
192
|
-
```ts
|
|
193
|
-
import type {
|
|
194
|
-
ShardQueryFunction,
|
|
195
|
-
ShardQueryMapByCC,
|
|
196
|
-
QueryOptionsByCC,
|
|
197
|
-
} from '@karmaniverous/entity-manager';
|
|
198
|
-
|
|
199
|
-
// A values-first config literal capturing index tokens (the same shape used for CF)
|
|
200
|
-
const cc = {
|
|
201
|
-
indexes: {
|
|
202
|
-
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
203
|
-
lastName: { hashKey: 'hashKey2', rangeKey: 'lastNameRK' },
|
|
204
|
-
},
|
|
205
|
-
} as const;
|
|
206
|
-
type CC = typeof cc;
|
|
207
|
-
|
|
208
|
-
// Reuse typed SQFs (pageKey narrowed per index)
|
|
209
|
-
const firstNameSQF: ShardQueryFunction<
|
|
210
|
-
MyConfigMap,
|
|
211
|
-
'user',
|
|
212
|
-
'firstName',
|
|
213
|
-
CC
|
|
214
|
-
> = async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
215
|
-
const lastNameSQF: ShardQueryFunction<
|
|
216
|
-
MyConfigMap,
|
|
217
|
-
'user',
|
|
218
|
-
'lastName',
|
|
219
|
-
CC
|
|
220
|
-
> = async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
221
|
-
|
|
222
|
-
// CC-aware shardQueryMap — only 'firstName' | 'lastName' allowed
|
|
223
|
-
const shardQueryMapCC: ShardQueryMapByCC<MyConfigMap, 'user', CC> = {
|
|
224
|
-
firstName: firstNameSQF,
|
|
225
|
-
lastName: lastNameSQF,
|
|
226
|
-
};
|
|
227
|
-
const optionsCC: QueryOptionsByCC<MyConfigMap, 'user', CC> = {
|
|
228
|
-
entityToken: 'user',
|
|
229
|
-
item: {},
|
|
230
|
-
shardQueryMap: shardQueryMapCC,
|
|
231
|
-
};
|
|
232
|
-
const resultCC = await manager.query(optionsCC);
|
|
233
|
-
```
|
|
234
|
-
|
|
235
|
-
Notes:
|
|
236
|
-
|
|
237
|
-
- Entity Manager enumerates hash‑key space for the time window, rehydrates page keys (when present), executes shard queries in parallel (throttled), dedupes by unique property, sorts, and dehydrates a new pageKeyMap.
|
|
238
|
-
- For provider integration, the SQF lambda encapsulates the platform‑specific query for one index + shard page. See tests and entity‑client‑dynamodb for examples.
|
|
239
|
-
|
|
240
|
-
## Projection‑aware typing (K)
|
|
241
|
-
|
|
242
|
-
Entity Manager supports a type‑only projection channel K that narrows result item shapes when a provider adapter projects a subset of attributes at runtime. Pass your attributes as a const tuple and thread K through `ShardQueryFunction/Map`, `QueryOptions`, and `QueryResult`.
|
|
243
|
-
|
|
244
|
-
```ts
|
|
245
|
-
import type {
|
|
246
|
-
ConfigMap,
|
|
247
|
-
EntityItemByToken,
|
|
248
|
-
QueryOptions,
|
|
249
|
-
QueryResult,
|
|
250
|
-
ShardQueryFunction,
|
|
251
|
-
ShardQueryMap,
|
|
252
|
-
} from '@karmaniverous/entity-manager';
|
|
253
|
-
|
|
254
|
-
// Minimal entities (example)
|
|
255
|
-
interface Email {
|
|
256
|
-
created: number;
|
|
257
|
-
email: string;
|
|
258
|
-
userId: string;
|
|
259
|
-
}
|
|
260
|
-
interface User {
|
|
261
|
-
beneficiaryId: string;
|
|
262
|
-
created: number;
|
|
263
|
-
firstNameCanonical: string;
|
|
264
|
-
lastNameCanonical: string;
|
|
265
|
-
phone?: string;
|
|
266
|
-
updated: number;
|
|
267
|
-
userId: string;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
type MyConfigMap = ConfigMap<{
|
|
271
|
-
EntityMap: { email: Email; user: User };
|
|
272
|
-
HashKey: 'hashKey2';
|
|
273
|
-
RangeKey: 'rangeKey';
|
|
274
|
-
ShardedKeys: 'beneficiaryPK' | 'userPK';
|
|
275
|
-
UnshardedKeys: 'firstNameRK' | 'lastNameRK' | 'phoneRK';
|
|
276
|
-
TranscodedProperties:
|
|
277
|
-
| 'beneficiaryId'
|
|
278
|
-
| 'created'
|
|
279
|
-
| 'email'
|
|
280
|
-
| 'firstNameCanonical'
|
|
281
|
-
| 'lastNameCanonical'
|
|
282
|
-
| 'phone'
|
|
283
|
-
| 'updated'
|
|
284
|
-
| 'userId';
|
|
285
|
-
}>;
|
|
286
|
-
|
|
287
|
-
// CF capturing a single index
|
|
288
|
-
const cf = {
|
|
289
|
-
indexes: {
|
|
290
|
-
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
291
|
-
},
|
|
292
|
-
} as const;
|
|
293
|
-
type CF = typeof cf;
|
|
294
|
-
|
|
295
|
-
// Projection attributes as const tuple — narrows K.
|
|
296
|
-
const attrs = ['userId', 'created'] as const;
|
|
297
|
-
type K = typeof attrs;
|
|
298
|
-
|
|
299
|
-
// A typed SQF: pageKey narrows via CF; items narrow via K (type-only).
|
|
300
|
-
const sqf: ShardQueryFunction<MyConfigMap, 'user', 'firstName', CF, K> = async (
|
|
301
|
-
_hashKey,
|
|
302
|
-
_pageKey,
|
|
303
|
-
_pageSize,
|
|
304
|
-
) => ({
|
|
305
|
-
count: 0,
|
|
306
|
-
items: [], // never[] is assignable to the projected array
|
|
307
|
-
pageKey: _pageKey,
|
|
308
|
-
});
|
|
309
|
-
|
|
310
|
-
// ShardQueryMap carrying CF and K.
|
|
311
|
-
const map: ShardQueryMap<MyConfigMap, 'user', 'firstName', CF, K> = {
|
|
312
|
-
firstName: sqf,
|
|
313
|
-
};
|
|
314
|
-
const options: QueryOptions<MyConfigMap, 'user', 'firstName', CF, K> = {
|
|
315
|
-
entityToken: 'user',
|
|
316
|
-
item: {},
|
|
317
|
-
shardQueryMap: map,
|
|
318
|
-
};
|
|
319
|
-
const result: QueryResult<MyConfigMap, 'user', 'firstName', K> =
|
|
320
|
-
await manager.query(options);
|
|
321
|
-
// result.items: Pick<EntityItemByToken<MyConfigMap, 'user'>, 'userId' | 'created'>[]
|
|
322
|
-
```
|
|
323
|
-
|
|
324
|
-
Notes:
|
|
325
|
-
|
|
326
|
-
- K is a type‑only channel; it does not change runtime behavior. Providers (e.g., DynamoDB adapters) execute projections.
|
|
327
|
-
- Dedupe/sort invariants: Entity Manager dedupes by `uniqueProperty` and applies `QueryOptions.sortOrder`. If your adapter projects attributes, ensure it auto‑includes `uniqueProperty` and any explicit sort keys when callers omit them from K.
|
|
328
|
-
|
|
329
|
-
## Page keys in a nutshell
|
|
330
|
-
|
|
331
|
-
- `rehydratePageKeyMap` decodes a dehydrated array (compressed string) into a two‑layer map of `{ indexToken: { hashKeyValue: pageKey | undefined } }`.
|
|
332
|
-
- `dehydratePageKeyMap` performs the inverse and emits a compact array (compressed in `query()`).
|
|
333
|
-
- You rarely call these directly — `query()` composes them for you — but the API is exposed for advanced flows.
|
|
334
|
-
|
|
335
|
-
## ESM / CJS
|
|
336
|
-
|
|
337
|
-
```ts
|
|
338
|
-
// ESM
|
|
339
|
-
import {
|
|
340
|
-
createEntityManager,
|
|
341
|
-
defaultTranscodes,
|
|
342
|
-
} from '@karmaniverous/entity-manager';
|
|
343
|
-
|
|
344
|
-
// CJS
|
|
345
|
-
const {
|
|
346
|
-
createEntityManager,
|
|
347
|
-
defaultTranscodes,
|
|
348
|
-
} = require('@karmaniverous/entity-manager');
|
|
349
|
-
```
|
|
350
|
-
|
|
351
|
-
## Logging
|
|
352
|
-
|
|
353
|
-
Entity Manager logs debug and error details via the injected logger (defaults to `console`).
|
|
354
|
-
|
|
355
|
-
```ts
|
|
356
|
-
const logger = { debug: () => undefined, error: console.error };
|
|
357
|
-
const manager = createEntityManager(config, logger);
|
|
358
|
-
```
|
|
359
|
-
|
|
360
|
-
## Types you’ll reach for
|
|
361
|
-
|
|
362
|
-
- Values/schema capture
|
|
363
|
-
- `createEntityManager(config, logger?)`
|
|
364
|
-
- `ConfigInput` (values‑first), `CapturedConfigMapFrom`, `EntitiesFromSchema`
|
|
365
|
-
- Token aware
|
|
366
|
-
- `EntityToken<CC>`, `EntityItemByToken<CC, ET>`, `EntityRecordByToken<CC, ET>`
|
|
367
|
-
- Index aware (CF channel)
|
|
368
|
-
- `PageKeyByIndex<CC, ET, IT, CF>`
|
|
369
|
-
- `ShardQueryFunction<CC, ET, IT, CF>`, `ShardQueryMap<CC, ET, ITS, CF>`
|
|
370
|
-
- `QueryOptions<CC, ET, ITS, CF>`, `QueryResult<CC, ET, ITS>`
|
|
371
|
-
- DX sugar: `IndexTokensOf<CF>`, `QueryOptionsByCF`, `ShardQueryMapByCF`, `IndexTokensFrom<CC>`, `QueryOptionsByCC`, `ShardQueryMapByCC`
|
|
372
|
-
- Projection helpers
|
|
373
|
-
- `KeysFrom<K>`
|
|
374
|
-
- `Projected<T, K>`
|
|
375
|
-
- `ProjectedItemByToken<CC, ET, K>`
|
|
376
|
-
|
|
377
|
-
See the full API: https://docs.karmanivero.us/entity-manager
|
|
378
|
-
|
|
379
|
-
## Scripts (repo)
|
|
380
|
-
|
|
381
|
-
- build: rollup outputs ESM/CJS + .d.ts
|
|
382
|
-
- test: vitest with coverage
|
|
383
|
-
- lint: ESLint (type‑aware) + Prettier
|
|
384
|
-
- docs: TypeDoc
|
|
385
|
-
- typecheck: tsc + tsd (type‑level tests)
|
|
386
|
-
|
|
387
|
-
## License
|
|
388
|
-
|
|
389
|
-
BSD‑3‑Clause (see package.json).
|
|
390
|
-
|
|
391
|
-
---
|
|
392
|
-
|
|
393
|
-
Built for you with ❤️ on Bali! Find more great tools & templates on [my GitHub Profile](https://github.com/karmaniverous).
|
|
1
|
+
# entity-manager
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/@karmaniverous/entity-manager)  <!-- TYPEDOC_EXCLUDE --> [](https://docs.karmanivero.us/entity-manager) [](https://github.com/karmaniverous/entity-manager/tree/main/CHANGELOG.md)<!-- /TYPEDOC_EXCLUDE --> [](https://github.com/karmaniverous/entity-manager/tree/main/LICENSE.md)
|
|
4
|
+
|
|
5
|
+
Entity Manager implements rational indexing & cross‑shard querying at scale in your NoSQL database so you can focus on application logic. It is provider‑agnostic (great fit for DynamoDB) and TypeScript‑first with strong types and runtime validation.
|
|
6
|
+
|
|
7
|
+
Key links:
|
|
8
|
+
|
|
9
|
+
- Docs: https://docs.karmanivero.us/entity-manager
|
|
10
|
+
- Discussions: https://github.com/karmaniverous/entity-manager/discussions
|
|
11
|
+
|
|
12
|
+
## Why this library?
|
|
13
|
+
|
|
14
|
+
Modern NoSQL puts the burden of indexing, sharding, and pagination on the application. Entity Manager gives you:
|
|
15
|
+
|
|
16
|
+
- Values‑first configuration with runtime validation (Zod) and best‑in‑class inference.
|
|
17
|
+
- Token‑aware and index‑aware types end‑to‑end (entities, keys, page keys, queries).
|
|
18
|
+
- Deterministic sharding with a time‑based scale‑up schedule.
|
|
19
|
+
- Cross‑shard, multi‑index query orchestration with dedupe and sorting.
|
|
20
|
+
- Dehydration/rehydration of page keys to pass a single compact token between calls.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
npm install @karmaniverous/entity-manager
|
|
26
|
+
# optional (tests/demo helpers)
|
|
27
|
+
npm install --save-dev @karmaniverous/mock-db
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## DX highlights
|
|
31
|
+
|
|
32
|
+
- Values‑first + schema‑first config:
|
|
33
|
+
- Use a config literal (prefer `as const` and `satisfies`) to preserve literal tokens.
|
|
34
|
+
- Optionally provide Zod schemas (`entitiesSchema`) to infer entity shapes without generics.
|
|
35
|
+
- Projection‑aware typing (type‑only K):
|
|
36
|
+
- Pass attributes as a const tuple (K) through your query types to narrow result items to Pick<…> of those properties.
|
|
37
|
+
- No runtime change; adapters execute projections. Adapters should auto‑include `uniqueProperty` and any explicit sort keys when callers omit them to preserve dedupe/sort invariants.
|
|
38
|
+
- Index‑aware typing (values‑first config literal “CF” and captured config “CC” helpers):
|
|
39
|
+
- CF (values‑first config literal): drive index token unions and page‑key narrowing directly from a values‑first config literal (`QueryOptionsByCF`, `ShardQueryMapByCF`).
|
|
40
|
+
- CC (Captured Config): derive index tokens from a captured config type while reusing CF for narrowing (`QueryOptionsByCC`, `ShardQueryMapByCC`).
|
|
41
|
+
- Token‑aware helpers:
|
|
42
|
+
- `addKeys`, `getPrimaryKey`, `removeKeys` narrow types by entity token (no casts).
|
|
43
|
+
- Index‑aware page keys (optional CF channel):
|
|
44
|
+
- Provide a values‑first config literal (CF) with `indexes` and get typed page keys per index.
|
|
45
|
+
- Use `QueryOptionsByCF` and `ShardQueryMapByCF` to derive index token unions directly from CF.
|
|
46
|
+
- CC-based DX sugar (values-first captured config):
|
|
47
|
+
- Use `QueryOptionsByCC` and `ShardQueryMapByCC` to derive index token unions from a captured config type (via `IndexTokensFrom`), while still benefiting from page-key narrowing.
|
|
48
|
+
|
|
49
|
+
## Quick start (values‑first + schema‑first)
|
|
50
|
+
|
|
51
|
+
```ts
|
|
52
|
+
import { z } from 'zod';
|
|
53
|
+
import {
|
|
54
|
+
createEntityManager,
|
|
55
|
+
defaultTranscodes,
|
|
56
|
+
} from '@karmaniverous/entity-manager';
|
|
57
|
+
|
|
58
|
+
// 1) Schema-first entity shapes (non-generated fields only)
|
|
59
|
+
const userSchema = z.object({
|
|
60
|
+
userId: z.string(), // unique property
|
|
61
|
+
created: z.number(), // timestamp property
|
|
62
|
+
updated: z.number().optional(),
|
|
63
|
+
firstNameCanonical: z.string(),
|
|
64
|
+
lastNameCanonical: z.string(),
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
// 2) Values-first config literal (prefer `as const`)
|
|
68
|
+
const config = {
|
|
69
|
+
hashKey: 'hashKey2',
|
|
70
|
+
rangeKey: 'rangeKey',
|
|
71
|
+
generatedProperties: {
|
|
72
|
+
sharded: {
|
|
73
|
+
userPK: ['userId'] as const,
|
|
74
|
+
},
|
|
75
|
+
unsharded: {
|
|
76
|
+
firstNameRK: ['firstNameCanonical', 'lastNameCanonical'] as const,
|
|
77
|
+
lastNameRK: ['lastNameCanonical', 'firstNameCanonical'] as const,
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
propertyTranscodes: {
|
|
81
|
+
userId: 'string',
|
|
82
|
+
created: 'timestamp',
|
|
83
|
+
updated: 'timestamp',
|
|
84
|
+
firstNameCanonical: 'string',
|
|
85
|
+
lastNameCanonical: 'string',
|
|
86
|
+
},
|
|
87
|
+
indexes: {
|
|
88
|
+
created: { hashKey: 'hashKey2', rangeKey: 'created' },
|
|
89
|
+
userCreated: { hashKey: 'userPK', rangeKey: 'created' },
|
|
90
|
+
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
91
|
+
lastName: { hashKey: 'hashKey2', rangeKey: 'lastNameRK' },
|
|
92
|
+
} as const,
|
|
93
|
+
entities: {
|
|
94
|
+
user: {
|
|
95
|
+
uniqueProperty: 'userId',
|
|
96
|
+
timestampProperty: 'created',
|
|
97
|
+
shardBumps: [{ timestamp: Date.now(), charBits: 2, chars: 1 }],
|
|
98
|
+
},
|
|
99
|
+
},
|
|
100
|
+
entitiesSchema: { user: userSchema },
|
|
101
|
+
transcodes: defaultTranscodes,
|
|
102
|
+
} as const;
|
|
103
|
+
|
|
104
|
+
// 3) Create the manager — types captured from values, shapes from schemas
|
|
105
|
+
const manager = createEntityManager(config);
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### Token‑aware helpers
|
|
109
|
+
|
|
110
|
+
```ts
|
|
111
|
+
// Input item (no generated keys yet)
|
|
112
|
+
const user = {
|
|
113
|
+
userId: 'u1',
|
|
114
|
+
created: Date.now(),
|
|
115
|
+
firstNameCanonical: 'lee',
|
|
116
|
+
lastNameCanonical: 'zhang',
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
// Add generated keys (hashKey/rangeKey + index tokens)
|
|
120
|
+
const record = manager.addKeys('user', user);
|
|
121
|
+
|
|
122
|
+
// Compute one or more primary keys
|
|
123
|
+
const keys = manager.getPrimaryKey('user', { userId: 'u1' });
|
|
124
|
+
|
|
125
|
+
// Strip generated keys after read
|
|
126
|
+
const item = manager.removeKeys('user', record);
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Types narrow automatically from the entity token (`'user'`). No casts required.
|
|
130
|
+
|
|
131
|
+
## Index‑aware querying (values‑first config literal, “CF” channel)
|
|
132
|
+
|
|
133
|
+
When you author a values‑first config literal with `indexes` (prefer `as const`), Entity Manager can:
|
|
134
|
+
|
|
135
|
+
- Constrain `shardQueryMap` keys to the index key union.
|
|
136
|
+
- Narrow page‑key shapes per index (only its component tokens).
|
|
137
|
+
- Derive ITS (index token subset) automatically from CF via `QueryOptionsByCF` and `ShardQueryMapByCF`.
|
|
138
|
+
|
|
139
|
+
```ts
|
|
140
|
+
import type {
|
|
141
|
+
ShardQueryFunction,
|
|
142
|
+
ShardQueryMapByCF,
|
|
143
|
+
QueryOptionsByCF,
|
|
144
|
+
} from '@karmaniverous/entity-manager';
|
|
145
|
+
|
|
146
|
+
// CF: capture index tokens from a values-first literal
|
|
147
|
+
const cf = {
|
|
148
|
+
indexes: {
|
|
149
|
+
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
150
|
+
lastName: { hashKey: 'hashKey2', rangeKey: 'lastNameRK' },
|
|
151
|
+
},
|
|
152
|
+
} as const;
|
|
153
|
+
type CF = typeof cf;
|
|
154
|
+
|
|
155
|
+
// SQFs are typed; pageKey is narrowed to index components per IT
|
|
156
|
+
const firstNameSQF: ShardQueryFunction<
|
|
157
|
+
MyConfigMap,
|
|
158
|
+
'user',
|
|
159
|
+
'firstName',
|
|
160
|
+
CF
|
|
161
|
+
> = async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
162
|
+
const lastNameSQF: ShardQueryFunction<
|
|
163
|
+
MyConfigMap,
|
|
164
|
+
'user',
|
|
165
|
+
'lastName',
|
|
166
|
+
CF
|
|
167
|
+
> = async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
168
|
+
|
|
169
|
+
// CF-aware shardQueryMap — only 'firstName' | 'lastName' allowed
|
|
170
|
+
const shardQueryMap: ShardQueryMapByCF<MyConfigMap, 'user', CF> = {
|
|
171
|
+
firstName: firstNameSQF,
|
|
172
|
+
lastName: lastNameSQF,
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
// Derive ITS from CF for options
|
|
176
|
+
const options: QueryOptionsByCF<MyConfigMap, 'user', CF> = {
|
|
177
|
+
entityToken: 'user',
|
|
178
|
+
item: {},
|
|
179
|
+
shardQueryMap,
|
|
180
|
+
limit: 50,
|
|
181
|
+
pageSize: 10,
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
const result = await manager.query(options);
|
|
185
|
+
// result.pageKeyMap is a compact string — pass it to the next call’s options.pageKeyMap
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
### Captured Config (“CC”) aliases
|
|
189
|
+
|
|
190
|
+
You can also derive ITS (index token subset) directly from a values‑first captured config type (CC) using `QueryOptionsByCC` and `ShardQueryMapByCC`. This mirrors the CF helpers but drives ITS from the CC type (via `IndexTokensFrom`) and passes the same CC through the CF channel for page‑key narrowing.
|
|
191
|
+
|
|
192
|
+
```ts
|
|
193
|
+
import type {
|
|
194
|
+
ShardQueryFunction,
|
|
195
|
+
ShardQueryMapByCC,
|
|
196
|
+
QueryOptionsByCC,
|
|
197
|
+
} from '@karmaniverous/entity-manager';
|
|
198
|
+
|
|
199
|
+
// A values-first config literal capturing index tokens (the same shape used for CF)
|
|
200
|
+
const cc = {
|
|
201
|
+
indexes: {
|
|
202
|
+
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
203
|
+
lastName: { hashKey: 'hashKey2', rangeKey: 'lastNameRK' },
|
|
204
|
+
},
|
|
205
|
+
} as const;
|
|
206
|
+
type CC = typeof cc;
|
|
207
|
+
|
|
208
|
+
// Reuse typed SQFs (pageKey narrowed per index)
|
|
209
|
+
const firstNameSQF: ShardQueryFunction<
|
|
210
|
+
MyConfigMap,
|
|
211
|
+
'user',
|
|
212
|
+
'firstName',
|
|
213
|
+
CC
|
|
214
|
+
> = async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
215
|
+
const lastNameSQF: ShardQueryFunction<
|
|
216
|
+
MyConfigMap,
|
|
217
|
+
'user',
|
|
218
|
+
'lastName',
|
|
219
|
+
CC
|
|
220
|
+
> = async (hashKey, pageKey, pageSize) => ({ count: 0, items: [], pageKey });
|
|
221
|
+
|
|
222
|
+
// CC-aware shardQueryMap — only 'firstName' | 'lastName' allowed
|
|
223
|
+
const shardQueryMapCC: ShardQueryMapByCC<MyConfigMap, 'user', CC> = {
|
|
224
|
+
firstName: firstNameSQF,
|
|
225
|
+
lastName: lastNameSQF,
|
|
226
|
+
};
|
|
227
|
+
const optionsCC: QueryOptionsByCC<MyConfigMap, 'user', CC> = {
|
|
228
|
+
entityToken: 'user',
|
|
229
|
+
item: {},
|
|
230
|
+
shardQueryMap: shardQueryMapCC,
|
|
231
|
+
};
|
|
232
|
+
const resultCC = await manager.query(optionsCC);
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Notes:
|
|
236
|
+
|
|
237
|
+
- Entity Manager enumerates hash‑key space for the time window, rehydrates page keys (when present), executes shard queries in parallel (throttled), dedupes by unique property, sorts, and dehydrates a new pageKeyMap.
|
|
238
|
+
- For provider integration, the SQF lambda encapsulates the platform‑specific query for one index + shard page. See tests and entity‑client‑dynamodb for examples.
|
|
239
|
+
|
|
240
|
+
## Projection‑aware typing (K)
|
|
241
|
+
|
|
242
|
+
Entity Manager supports a type‑only projection channel K that narrows result item shapes when a provider adapter projects a subset of attributes at runtime. Pass your attributes as a const tuple and thread K through `ShardQueryFunction/Map`, `QueryOptions`, and `QueryResult`.
|
|
243
|
+
|
|
244
|
+
```ts
|
|
245
|
+
import type {
|
|
246
|
+
ConfigMap,
|
|
247
|
+
EntityItemByToken,
|
|
248
|
+
QueryOptions,
|
|
249
|
+
QueryResult,
|
|
250
|
+
ShardQueryFunction,
|
|
251
|
+
ShardQueryMap,
|
|
252
|
+
} from '@karmaniverous/entity-manager';
|
|
253
|
+
|
|
254
|
+
// Minimal entities (example)
|
|
255
|
+
interface Email {
|
|
256
|
+
created: number;
|
|
257
|
+
email: string;
|
|
258
|
+
userId: string;
|
|
259
|
+
}
|
|
260
|
+
interface User {
|
|
261
|
+
beneficiaryId: string;
|
|
262
|
+
created: number;
|
|
263
|
+
firstNameCanonical: string;
|
|
264
|
+
lastNameCanonical: string;
|
|
265
|
+
phone?: string;
|
|
266
|
+
updated: number;
|
|
267
|
+
userId: string;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
type MyConfigMap = ConfigMap<{
|
|
271
|
+
EntityMap: { email: Email; user: User };
|
|
272
|
+
HashKey: 'hashKey2';
|
|
273
|
+
RangeKey: 'rangeKey';
|
|
274
|
+
ShardedKeys: 'beneficiaryPK' | 'userPK';
|
|
275
|
+
UnshardedKeys: 'firstNameRK' | 'lastNameRK' | 'phoneRK';
|
|
276
|
+
TranscodedProperties:
|
|
277
|
+
| 'beneficiaryId'
|
|
278
|
+
| 'created'
|
|
279
|
+
| 'email'
|
|
280
|
+
| 'firstNameCanonical'
|
|
281
|
+
| 'lastNameCanonical'
|
|
282
|
+
| 'phone'
|
|
283
|
+
| 'updated'
|
|
284
|
+
| 'userId';
|
|
285
|
+
}>;
|
|
286
|
+
|
|
287
|
+
// CF capturing a single index
|
|
288
|
+
const cf = {
|
|
289
|
+
indexes: {
|
|
290
|
+
firstName: { hashKey: 'hashKey2', rangeKey: 'firstNameRK' },
|
|
291
|
+
},
|
|
292
|
+
} as const;
|
|
293
|
+
type CF = typeof cf;
|
|
294
|
+
|
|
295
|
+
// Projection attributes as const tuple — narrows K.
|
|
296
|
+
const attrs = ['userId', 'created'] as const;
|
|
297
|
+
type K = typeof attrs;
|
|
298
|
+
|
|
299
|
+
// A typed SQF: pageKey narrows via CF; items narrow via K (type-only).
|
|
300
|
+
const sqf: ShardQueryFunction<MyConfigMap, 'user', 'firstName', CF, K> = async (
|
|
301
|
+
_hashKey,
|
|
302
|
+
_pageKey,
|
|
303
|
+
_pageSize,
|
|
304
|
+
) => ({
|
|
305
|
+
count: 0,
|
|
306
|
+
items: [], // never[] is assignable to the projected array
|
|
307
|
+
pageKey: _pageKey,
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
// ShardQueryMap carrying CF and K.
|
|
311
|
+
const map: ShardQueryMap<MyConfigMap, 'user', 'firstName', CF, K> = {
|
|
312
|
+
firstName: sqf,
|
|
313
|
+
};
|
|
314
|
+
const options: QueryOptions<MyConfigMap, 'user', 'firstName', CF, K> = {
|
|
315
|
+
entityToken: 'user',
|
|
316
|
+
item: {},
|
|
317
|
+
shardQueryMap: map,
|
|
318
|
+
};
|
|
319
|
+
const result: QueryResult<MyConfigMap, 'user', 'firstName', K> =
|
|
320
|
+
await manager.query(options);
|
|
321
|
+
// result.items: Pick<EntityItemByToken<MyConfigMap, 'user'>, 'userId' | 'created'>[]
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
Notes:
|
|
325
|
+
|
|
326
|
+
- K is a type‑only channel; it does not change runtime behavior. Providers (e.g., DynamoDB adapters) execute projections.
|
|
327
|
+
- Dedupe/sort invariants: Entity Manager dedupes by `uniqueProperty` and applies `QueryOptions.sortOrder`. If your adapter projects attributes, ensure it auto‑includes `uniqueProperty` and any explicit sort keys when callers omit them from K.
|
|
328
|
+
|
|
329
|
+
## Page keys in a nutshell
|
|
330
|
+
|
|
331
|
+
- `rehydratePageKeyMap` decodes a dehydrated array (compressed string) into a two‑layer map of `{ indexToken: { hashKeyValue: pageKey | undefined } }`.
|
|
332
|
+
- `dehydratePageKeyMap` performs the inverse and emits a compact array (compressed in `query()`).
|
|
333
|
+
- You rarely call these directly — `query()` composes them for you — but the API is exposed for advanced flows.
|
|
334
|
+
|
|
335
|
+
## ESM / CJS
|
|
336
|
+
|
|
337
|
+
```ts
|
|
338
|
+
// ESM
|
|
339
|
+
import {
|
|
340
|
+
createEntityManager,
|
|
341
|
+
defaultTranscodes,
|
|
342
|
+
} from '@karmaniverous/entity-manager';
|
|
343
|
+
|
|
344
|
+
// CJS
|
|
345
|
+
const {
|
|
346
|
+
createEntityManager,
|
|
347
|
+
defaultTranscodes,
|
|
348
|
+
} = require('@karmaniverous/entity-manager');
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
## Logging
|
|
352
|
+
|
|
353
|
+
Entity Manager logs debug and error details via the injected logger (defaults to `console`).
|
|
354
|
+
|
|
355
|
+
```ts
|
|
356
|
+
const logger = { debug: () => undefined, error: console.error };
|
|
357
|
+
const manager = createEntityManager(config, logger);
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
## Types you’ll reach for
|
|
361
|
+
|
|
362
|
+
- Values/schema capture
|
|
363
|
+
- `createEntityManager(config, logger?)`
|
|
364
|
+
- `ConfigInput` (values‑first), `CapturedConfigMapFrom`, `EntitiesFromSchema`
|
|
365
|
+
- Token aware
|
|
366
|
+
- `EntityToken<CC>`, `EntityItemByToken<CC, ET>`, `EntityRecordByToken<CC, ET>`
|
|
367
|
+
- Index aware (values‑first config literal, “CF” channel)
|
|
368
|
+
- `PageKeyByIndex<CC, ET, IT, CF>`
|
|
369
|
+
- `ShardQueryFunction<CC, ET, IT, CF>`, `ShardQueryMap<CC, ET, ITS, CF>`
|
|
370
|
+
- `QueryOptions<CC, ET, ITS, CF>`, `QueryResult<CC, ET, ITS>`
|
|
371
|
+
- DX sugar: `IndexTokensOf<CF>`, `QueryOptionsByCF`, `ShardQueryMapByCF`, `IndexTokensFrom<CC>`, `QueryOptionsByCC`, `ShardQueryMapByCC`
|
|
372
|
+
- Projection helpers
|
|
373
|
+
- `KeysFrom<K>`
|
|
374
|
+
- `Projected<T, K>`
|
|
375
|
+
- `ProjectedItemByToken<CC, ET, K>`
|
|
376
|
+
|
|
377
|
+
See the full API: https://docs.karmanivero.us/entity-manager
|
|
378
|
+
|
|
379
|
+
## Scripts (repo)
|
|
380
|
+
|
|
381
|
+
- build: rollup outputs ESM/CJS + .d.ts
|
|
382
|
+
- test: vitest with coverage
|
|
383
|
+
- lint: ESLint (type‑aware) + Prettier
|
|
384
|
+
- docs: TypeDoc
|
|
385
|
+
- typecheck: tsc + tsd (type‑level tests)
|
|
386
|
+
|
|
387
|
+
## License
|
|
388
|
+
|
|
389
|
+
BSD‑3‑Clause (see package.json).
|
|
390
|
+
|
|
391
|
+
---
|
|
392
|
+
|
|
393
|
+
Built for you with ❤️ on Bali! Find more great tools & templates on [my GitHub Profile](https://github.com/karmaniverous).
|
|
@@ -30,7 +30,14 @@ class EntityManager {
|
|
|
30
30
|
*/
|
|
31
31
|
constructor(config, logger = console) {
|
|
32
32
|
_EntityManager_config.set(this, void 0);
|
|
33
|
-
|
|
33
|
+
// Accept a compile-time-only `entitiesSchema` key on values-first configs.
|
|
34
|
+
// We strip it here so the same literal config can be used with either the
|
|
35
|
+
// factory or the direct constructor without tripping Zod's strict parser.
|
|
36
|
+
//
|
|
37
|
+
// This keeps runtime semantics unchanged and avoids widening ParsedConfig.
|
|
38
|
+
const cfgWithOptionalES = config;
|
|
39
|
+
const { entitiesSchema: _ignored, ...configForParse } = cfgWithOptionalES;
|
|
40
|
+
tslib.__classPrivateFieldSet(this, _EntityManager_config, ParsedConfig.configSchema.parse(configForParse), "f");
|
|
34
41
|
this.logger = logger;
|
|
35
42
|
}
|
|
36
43
|
/**
|
|
@@ -28,7 +28,14 @@ class EntityManager {
|
|
|
28
28
|
*/
|
|
29
29
|
constructor(config, logger = console) {
|
|
30
30
|
_EntityManager_config.set(this, void 0);
|
|
31
|
-
|
|
31
|
+
// Accept a compile-time-only `entitiesSchema` key on values-first configs.
|
|
32
|
+
// We strip it here so the same literal config can be used with either the
|
|
33
|
+
// factory or the direct constructor without tripping Zod's strict parser.
|
|
34
|
+
//
|
|
35
|
+
// This keeps runtime semantics unchanged and avoids widening ParsedConfig.
|
|
36
|
+
const cfgWithOptionalES = config;
|
|
37
|
+
const { entitiesSchema: _ignored, ...configForParse } = cfgWithOptionalES;
|
|
38
|
+
__classPrivateFieldSet(this, _EntityManager_config, configSchema.parse(configForParse), "f");
|
|
32
39
|
this.logger = logger;
|
|
33
40
|
}
|
|
34
41
|
/**
|
package/package.json
CHANGED