@karmaniverous/entity-manager 6.14.0 → 6.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +330 -16
- package/dist/cjs/EntityManager/ParsedConfig.js +46 -55
- package/dist/cjs/EntityManager/addKeys.js +1 -0
- package/dist/cjs/EntityManager/decodeElement.js +2 -1
- package/dist/cjs/EntityManager/dehydratePageKeyMap.js +3 -1
- package/dist/cjs/EntityManager/encodeElement.js +2 -2
- package/dist/cjs/EntityManager/encodeGeneratedProperty.js +1 -1
- package/dist/cjs/EntityManager/rehydratePageKeyMap.js +3 -1
- package/dist/cjs/EntityManager/removeKeys.js +10 -8
- package/dist/cjs/EntityManager/updateItemHashKey.js +4 -1
- package/dist/index.d.ts +17 -208
- package/dist/mjs/EntityManager/ParsedConfig.js +46 -55
- package/dist/mjs/EntityManager/addKeys.js +1 -0
- package/dist/mjs/EntityManager/decodeElement.js +2 -1
- package/dist/mjs/EntityManager/dehydratePageKeyMap.js +3 -1
- package/dist/mjs/EntityManager/encodeElement.js +2 -2
- package/dist/mjs/EntityManager/encodeGeneratedProperty.js +1 -1
- package/dist/mjs/EntityManager/rehydratePageKeyMap.js +3 -1
- package/dist/mjs/EntityManager/removeKeys.js +10 -8
- package/dist/mjs/EntityManager/updateItemHashKey.js +4 -1
- package/package.json +44 -45
package/README.md
CHANGED
|
@@ -1,34 +1,348 @@
|
|
|
1
|
-
|
|
1
|
+
# entity-manager
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/@karmaniverous/entity-manager)  <!-- TYPEDOC_EXCLUDE --> [](https://docs.karmanivero.us/entity-manager) [](https://github.com/karmaniverous/entity-manager/tree/main/CHANGELOG.md)<!-- /TYPEDOC_EXCLUDE --> [](https://github.com/karmaniverous/entity-manager/tree/main/LICENSE.md)
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
EntityManager implements rational indexing & cross-shard querying at scale in your NoSQL database so you can focus on your application logic.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
If you have any questions, please [start a discussion](https://github.com/karmaniverous/entity-manager/discussions). Otherwise stay tuned!
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
## What is this?
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
Entity Manager is a TypeScript-first library that applies a provider‑agnostic, highly opinionated single‑table design to your NoSQL data. It lets you:
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
- Define a global hash key and range key, plus additional generated properties used by your indexes.
|
|
14
|
+
- Encode/decode indexable elements via transcodes so strings sort like their original types.
|
|
15
|
+
- Configure a time‑based sharding strategy (shard bumps) that grows as you scale.
|
|
16
|
+
- Query across many shards and indexes in parallel through injected “shard query” functions — results are combined, de‑duplicated, sorted, and returned with a compact, dehydrated page key for the next request.
|
|
17
|
+
|
|
18
|
+
It is designed to work with stores like DynamoDB but keeps the orchestration provider‑neutral.
|
|
19
|
+
|
|
20
|
+
Key links:
|
|
21
|
+
|
|
22
|
+
- API: https://docs.karmanivero.us/entity-manager
|
|
23
|
+
- Requirements: see .stan/system/stan.requirements.md (authoritative for v6.14.0)
|
|
24
|
+
- Example test configuration: see test/config.ts
|
|
25
|
+
|
|
26
|
+
## Features
|
|
27
|
+
|
|
28
|
+
- Global model for generated properties, indexes, and property transcodes
|
|
29
|
+
- generatedProperties.sharded and generatedProperties.unsharded
|
|
30
|
+
- Global indexes: indexToken → { hashKey, rangeKey, projections? }
|
|
31
|
+
- Global propertyTranscodes: property → transcodeName
|
|
32
|
+
- Deterministic sharding
|
|
33
|
+
- Time‑windowed shard bumps: { timestamp, charBits, chars }
|
|
34
|
+
- Full shard space assignment per bump (uses radix\*\*chars placeholders)
|
|
35
|
+
- Cross‑bump query enumeration over all applicable shards
|
|
36
|
+
- Page key dehydration/rehydration
|
|
37
|
+
- Compact string arrays and lz‑string compression for transport
|
|
38
|
+
- Rehydrate back to pageKey objects for each index+shard
|
|
39
|
+
- Provider‑agnostic parallel query orchestration
|
|
40
|
+
- Inject shard query functions for each index
|
|
41
|
+
- Parallel fan‑out with configurable throttle
|
|
42
|
+
- Combine, dedupe by unique property, and sort results
|
|
43
|
+
- Strong typing + runtime validation
|
|
44
|
+
- Zod‑validated config parsing
|
|
45
|
+
- Robust TypeScript surface for config, items, keys, queries
|
|
46
|
+
|
|
47
|
+
## Install
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
npm install @karmaniverous/entity-manager
|
|
51
|
+
# optional: testing support used in the repo
|
|
52
|
+
npm install --save-dev @karmaniverous/mock-db
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
TypeScript is strongly recommended. The library will validate configuration at runtime for JavaScript users, but you lose compile‑time guarantees.
|
|
56
|
+
|
|
57
|
+
## Usage overview
|
|
58
|
+
|
|
59
|
+
The pattern has three parts:
|
|
60
|
+
|
|
61
|
+
1. Define your entity types and a config map
|
|
62
|
+
|
|
63
|
+
- Each entity type lists all properties that exist on your records.
|
|
64
|
+
- Entity‑level behaviors live in the config’s entities block (timestamp property, unique property, shard bumps).
|
|
65
|
+
- Generated properties, indexes, and transcodes are defined globally.
|
|
66
|
+
|
|
67
|
+
2. Create an EntityManager instance
|
|
68
|
+
|
|
69
|
+
- Pass your config (validated with Zod).
|
|
70
|
+
- Optionally inject a logger with debug/error methods (defaults to console).
|
|
71
|
+
|
|
72
|
+
3. Use EntityManager helpers
|
|
73
|
+
|
|
74
|
+
- addKeys / getPrimaryKey / removeKeys
|
|
75
|
+
- encodeGeneratedProperty / decodeGeneratedProperty
|
|
76
|
+
- getIndexComponents / unwrapIndex / dehydrateIndexItem / rehydrateIndexItem
|
|
77
|
+
- dehydratePageKeyMap / rehydratePageKeyMap
|
|
78
|
+
- query(options) to orchestrate cross‑shard multi‑index queries
|
|
79
|
+
|
|
80
|
+
## Quick start (TypeScript)
|
|
81
|
+
|
|
82
|
+
Below is a minimal end‑to‑end example showing shape and intent. It mirrors the current implementation’s global config model.
|
|
83
|
+
|
|
84
|
+
```ts
|
|
85
|
+
import {
|
|
86
|
+
defaultTranscodes,
|
|
87
|
+
type ConfigMap,
|
|
88
|
+
} from '@karmaniverous/entity-manager';
|
|
89
|
+
import { EntityManager } from '@karmaniverous/entity-manager';
|
|
90
|
+
|
|
91
|
+
// 1) Entity definitions (Typescript types)
|
|
92
|
+
interface User {
|
|
93
|
+
userId: string; // unique property
|
|
94
|
+
created: number; // timestamp property
|
|
95
|
+
updated: number;
|
|
96
|
+
firstNameCanonical: string;
|
|
97
|
+
lastNameCanonical: string;
|
|
98
|
+
// Generated properties exist on stored items but are configured globally:
|
|
99
|
+
// e.g., firstNameRK (unsharded), lastNameRK (unsharded), userPK (sharded)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
type MyConfigMap = ConfigMap<{
|
|
103
|
+
EntityMap: { user: User };
|
|
104
|
+
HashKey: 'hashKey'; // defaults are 'hashKey' / 'rangeKey' if omitted
|
|
105
|
+
RangeKey: 'rangeKey';
|
|
106
|
+
ShardedKeys: 'userPK'; // token(s) for sharded generated properties
|
|
107
|
+
UnshardedKeys: 'firstNameRK' | 'lastNameRK';
|
|
108
|
+
TranscodedProperties:
|
|
109
|
+
| 'userId'
|
|
110
|
+
| 'created'
|
|
111
|
+
| 'updated'
|
|
112
|
+
| 'firstNameCanonical'
|
|
113
|
+
| 'lastNameCanonical';
|
|
114
|
+
}>;
|
|
115
|
+
|
|
116
|
+
// 2) Build a config with the global model
|
|
117
|
+
const now = Date.now();
|
|
118
|
+
const manager = new EntityManager<MyConfigMap>({
|
|
119
|
+
// Per-entity: unique + timestamp + shard schedule (+ optional defaults)
|
|
120
|
+
entities: {
|
|
121
|
+
user: {
|
|
122
|
+
uniqueProperty: 'userId',
|
|
123
|
+
timestampProperty: 'created',
|
|
124
|
+
shardBumps: [
|
|
125
|
+
// records with timestamp < now → effectively unsharded
|
|
126
|
+
{ timestamp: now, charBits: 1, chars: 0 },
|
|
127
|
+
// records with timestamp ≥ now → 1 char at radix 4 (2^2) gives 4 shards
|
|
128
|
+
{ timestamp: now, charBits: 2, chars: 1 },
|
|
129
|
+
],
|
|
130
|
+
// optional defaults (used by query if omitted in options)
|
|
131
|
+
defaultLimit: 10,
|
|
132
|
+
defaultPageSize: 10,
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
|
|
136
|
+
// Global generated properties (tokens → element lists)
|
|
137
|
+
generatedProperties: {
|
|
138
|
+
sharded: {
|
|
139
|
+
userPK: ['userId'], // atomic (all required)
|
|
140
|
+
},
|
|
141
|
+
unsharded: {
|
|
142
|
+
firstNameRK: ['firstNameCanonical', 'lastNameCanonical', 'created'],
|
|
143
|
+
lastNameRK: ['lastNameCanonical', 'firstNameCanonical', 'created'],
|
|
144
|
+
},
|
|
145
|
+
},
|
|
146
|
+
|
|
147
|
+
// Global key tokens
|
|
148
|
+
hashKey: 'hashKey',
|
|
149
|
+
rangeKey: 'rangeKey',
|
|
150
|
+
|
|
151
|
+
// Global indexes (hashKey, rangeKey must match allowed token sets)
|
|
152
|
+
indexes: {
|
|
153
|
+
created: { hashKey: 'hashKey', rangeKey: 'created' },
|
|
154
|
+
updated: { hashKey: 'hashKey', rangeKey: 'updated' },
|
|
155
|
+
firstName: { hashKey: 'hashKey', rangeKey: 'firstNameRK' },
|
|
156
|
+
lastName: { hashKey: 'hashKey', rangeKey: 'lastNameRK' },
|
|
157
|
+
userCreated: { hashKey: 'userPK', rangeKey: 'created' }, // sharded alt hash
|
|
158
|
+
},
|
|
159
|
+
|
|
160
|
+
// Transcode mapping for scalar/unsharded elements and properties
|
|
161
|
+
propertyTranscodes: {
|
|
162
|
+
userId: 'string',
|
|
163
|
+
created: 'timestamp',
|
|
164
|
+
updated: 'timestamp',
|
|
165
|
+
firstNameCanonical: 'string',
|
|
166
|
+
lastNameCanonical: 'string',
|
|
167
|
+
},
|
|
168
|
+
|
|
169
|
+
// Transcodes (can override/extend defaultTranscodes)
|
|
170
|
+
transcodes: defaultTranscodes,
|
|
171
|
+
|
|
172
|
+
// Delimiters and query throttle (defaults shown)
|
|
173
|
+
generatedKeyDelimiter: '|',
|
|
174
|
+
generatedValueDelimiter: '#',
|
|
175
|
+
shardKeyDelimiter: '!',
|
|
176
|
+
throttle: 10,
|
|
177
|
+
});
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### Generate keys on items
|
|
181
|
+
|
|
182
|
+
```ts
|
|
183
|
+
// A partial item (no keys yet)
|
|
184
|
+
const user = {
|
|
185
|
+
userId: 'u123',
|
|
186
|
+
created: Date.now(),
|
|
187
|
+
updated: Date.now(),
|
|
188
|
+
firstNameCanonical: 'lee',
|
|
189
|
+
lastNameCanonical: 'zhang',
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
// Add hashKey, rangeKey, and generated properties
|
|
193
|
+
const record = manager.addKeys('user', user); // returns EntityRecord<...>
|
|
194
|
+
|
|
195
|
+
// Get just the primary key
|
|
196
|
+
const keyOnly = manager.getPrimaryKey('user', user); // { hashKey, rangeKey }
|
|
197
|
+
|
|
198
|
+
// Remove generated keys from a stored record
|
|
199
|
+
const pruned = manager.removeKeys('user', record);
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Encode/decode generated property strings
|
|
203
|
+
|
|
204
|
+
```ts
|
|
205
|
+
// Encode an unsharded generated property (always returns a string)
|
|
206
|
+
const fn = manager.encodeGeneratedProperty('firstNameRK', record);
|
|
207
|
+
// e.g. "firstNameCanonical#lee|lastNameCanonical#zhang|created#000001711234567"
|
|
208
|
+
|
|
209
|
+
// Decode back into an object fragment
|
|
210
|
+
import { decodeGeneratedProperty } from '@karmaniverous/entity-manager';
|
|
211
|
+
const decoded = decodeGeneratedProperty(manager, fn); // { firstNameCanonical: 'lee', ... }
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### Query across shards and indexes
|
|
215
|
+
|
|
216
|
+
Entity Manager relies on injected shard query functions to perform provider‑specific queries on each shard/index page. The library orchestrates:
|
|
217
|
+
|
|
218
|
+
- page‑key rehydration → parallel shard queries → de‑duplication and sorting → page‑key dehydration.
|
|
219
|
+
|
|
220
|
+
```ts
|
|
221
|
+
import type {
|
|
222
|
+
QueryOptions,
|
|
223
|
+
ShardQueryFunction,
|
|
224
|
+
} from '@karmaniverous/entity-manager';
|
|
225
|
+
|
|
226
|
+
// Example shard query using a made-up client (see @karmaniverous/mock-db in repo tests)
|
|
227
|
+
const firstNameQuery: ShardQueryFunction<MyConfigMap> = async (
|
|
228
|
+
hashKey,
|
|
229
|
+
pageKey,
|
|
230
|
+
pageSize,
|
|
231
|
+
) => {
|
|
232
|
+
// Return { count, items, pageKey? } for this shard+index page
|
|
233
|
+
// pageKey is a partial item object with necessary index components
|
|
234
|
+
// ... perform provider-specific work here ...
|
|
235
|
+
return { count: 0, items: [], pageKey };
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
// Invoke query with shardQueryMap
|
|
239
|
+
const result = await manager.query({
|
|
240
|
+
entityToken: 'user',
|
|
241
|
+
item: {}, // often used to supply elements for alternate hash keys
|
|
242
|
+
shardQueryMap: { firstName: firstNameQuery },
|
|
243
|
+
limit: 50,
|
|
244
|
+
pageSize: 10,
|
|
245
|
+
// optional: pageKeyMap: previousResult.pageKeyMap,
|
|
246
|
+
// optional: timestampFrom / timestampTo for shard-space windowing
|
|
247
|
+
});
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Notes:
|
|
251
|
+
|
|
252
|
+
- The result includes a compressed pageKeyMap string for the next call.
|
|
253
|
+
- Entity Manager enumerates the hash key space for the time window, rehydrates the prior page keys (if any), and fans out queries across all shard+index pairs in `shardQueryMap` (up to `throttle`).
|
|
254
|
+
- Items are deduplicated by the entity’s unique property and sorted by `sortOrder` (if provided).
|
|
255
|
+
|
|
256
|
+
## Configuration reference (current model, v6.14.0)
|
|
257
|
+
|
|
258
|
+
- entities: Record<entityToken, { timestampProperty, uniqueProperty, shardBumps?, defaultLimit?, defaultPageSize? }>
|
|
259
|
+
- generatedProperties:
|
|
260
|
+
- sharded: Record<ShardedKey, TranscodedProperties[]>
|
|
261
|
+
- unsharded: Record<UnshardedKey, TranscodedProperties[]>
|
|
262
|
+
- indexes: Record<indexToken, {
|
|
263
|
+
- hashKey: HashKey | ShardedKey
|
|
264
|
+
- rangeKey: RangeKey | UnshardedKey | TranscodedProperties
|
|
265
|
+
- projections?: string[]
|
|
266
|
+
}>
|
|
267
|
+
- propertyTranscodes: Record<TranscodedProperties, keyof TranscodeMap>
|
|
268
|
+
- transcodes: Record<transcodeName, { encode, decode }> (defaults to defaultTranscodes)
|
|
269
|
+
- hashKey: HashKey (e.g., 'hashKey')
|
|
270
|
+
- rangeKey: RangeKey (e.g., 'rangeKey')
|
|
271
|
+
- generatedKeyDelimiter: string (default '|', must match /\W+/)
|
|
272
|
+
- generatedValueDelimiter: string (default '#', must match /\W+/)
|
|
273
|
+
- shardKeyDelimiter: string (default '!', must match /\W+/)
|
|
274
|
+
- throttle: number (default 10)
|
|
275
|
+
|
|
276
|
+
Validation highlights:
|
|
277
|
+
|
|
278
|
+
- Delimiters must not contain each other.
|
|
279
|
+
- Keys and tokens must be mutually exclusive as required.
|
|
280
|
+
- Generated property element lists are non‑empty and have no duplicates.
|
|
281
|
+
- propertyTranscodes values must exist in transcodes.
|
|
282
|
+
- Index hashKey/rangeKey must use valid token sets.
|
|
283
|
+
- shardBumps are sorted, include a zero‑timestamp bump if missing, and chars must increase monotonically with timestamp.
|
|
284
|
+
|
|
285
|
+
Sharding:
|
|
286
|
+
|
|
287
|
+
- For assignment: a record always uses all placeholders for its applicable bump; suffix space is (2**charBits) ** chars.
|
|
288
|
+
- For queries: hash key space spans all bumps overlapping [timestampFrom, timestampTo].
|
|
289
|
+
|
|
290
|
+
## ESM / CJS
|
|
291
|
+
|
|
292
|
+
```ts
|
|
293
|
+
// ESM
|
|
294
|
+
import {
|
|
295
|
+
EntityManager,
|
|
296
|
+
defaultTranscodes,
|
|
297
|
+
} from '@karmaniverous/entity-manager';
|
|
298
|
+
|
|
299
|
+
// CJS
|
|
300
|
+
const {
|
|
301
|
+
EntityManager,
|
|
302
|
+
defaultTranscodes,
|
|
303
|
+
} = require('@karmaniverous/entity-manager');
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
## Logging
|
|
307
|
+
|
|
308
|
+
All helpers log debug context and error detail via the injected logger (defaults to `console`). In tests, you may supply a quiet logger:
|
|
309
|
+
|
|
310
|
+
```ts
|
|
311
|
+
const logger = { debug: () => undefined, error: console.error };
|
|
312
|
+
const manager = new EntityManager(config, logger);
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
## Delimiter safety
|
|
316
|
+
|
|
317
|
+
Generated key/value delimiters and the shard key delimiter are used when composing strings:
|
|
14
318
|
|
|
15
|
-
|
|
319
|
+
- generatedKeyDelimiter: '|' (between pairs)
|
|
320
|
+
- generatedValueDelimiter: '#' (between key and value)
|
|
321
|
+
- shardKeyDelimiter: '!' (between entity token and shard suffix)
|
|
16
322
|
|
|
17
|
-
|
|
323
|
+
Your scalar property values used in generated properties should not include these delimiters. If they must, set custom delimiters (must match /\W+/ and not contain each other).
|
|
18
324
|
|
|
19
|
-
|
|
325
|
+
## Types you’ll use most
|
|
20
326
|
|
|
21
|
-
|
|
327
|
+
- ConfigMap<M>
|
|
328
|
+
- EntityItem<C>, EntityRecord<C>, EntityKey<C>, EntityToken<C>
|
|
329
|
+
- QueryOptions<C>, QueryResult<C>
|
|
330
|
+
- PageKey<C>, PageKeyMap<C>
|
|
331
|
+
- ShardQueryFunction<C>, ShardQueryMap<C>, ShardBump
|
|
22
332
|
|
|
23
|
-
|
|
333
|
+
See the full API: https://docs.karmanivero.us/entity-manager
|
|
24
334
|
|
|
25
|
-
|
|
335
|
+
## Scripts (repo)
|
|
26
336
|
|
|
27
|
-
-
|
|
337
|
+
- build: rollup outputs ESM/CJS + .d.ts
|
|
338
|
+
- test: vitest with coverage
|
|
339
|
+
- lint: ESLint (type‑aware) + Prettier integration
|
|
340
|
+
- docs: TypeDoc (links to external type docs for shared utility packages)
|
|
341
|
+
- typecheck: tsc + tsd (type‑level tests)
|
|
28
342
|
|
|
29
|
-
|
|
343
|
+
## License
|
|
30
344
|
|
|
31
|
-
|
|
345
|
+
BSD‑3‑Clause (see package.json).
|
|
32
346
|
|
|
33
347
|
---
|
|
34
348
|
|
|
@@ -10,7 +10,7 @@ const validateArrayUnique = (arr, ctx, identity = (item) => item, path = []) =>
|
|
|
10
10
|
for (const [element, count] of Object.entries(counts)) {
|
|
11
11
|
if (count > 1)
|
|
12
12
|
ctx.addIssue({
|
|
13
|
-
code:
|
|
13
|
+
code: 'custom',
|
|
14
14
|
message: `duplicate array element`,
|
|
15
15
|
params: { element },
|
|
16
16
|
path,
|
|
@@ -21,7 +21,7 @@ const validateKeysExclusive = (keys, label, ref, ctx) => {
|
|
|
21
21
|
const intersection = keys.filter((key) => ref.includes(key));
|
|
22
22
|
if (intersection.length)
|
|
23
23
|
ctx.addIssue({
|
|
24
|
-
code:
|
|
24
|
+
code: 'custom',
|
|
25
25
|
message: `${label} key collision: ${intersection.toString()}`,
|
|
26
26
|
});
|
|
27
27
|
};
|
|
@@ -32,26 +32,14 @@ const componentArray = zod.z
|
|
|
32
32
|
const configSchema = zod.z
|
|
33
33
|
.object({
|
|
34
34
|
entities: zod.z
|
|
35
|
-
.record(zod.z
|
|
35
|
+
.record(zod.z.string(), zod.z
|
|
36
36
|
.object({
|
|
37
|
-
defaultLimit: zod.z
|
|
38
|
-
|
|
39
|
-
.int()
|
|
40
|
-
.positive()
|
|
41
|
-
.safe()
|
|
42
|
-
.optional()
|
|
43
|
-
.default(10),
|
|
44
|
-
defaultPageSize: zod.z
|
|
45
|
-
.number()
|
|
46
|
-
.int()
|
|
47
|
-
.positive()
|
|
48
|
-
.safe()
|
|
49
|
-
.optional()
|
|
50
|
-
.default(10),
|
|
37
|
+
defaultLimit: zod.z.number().int().positive().optional().default(10),
|
|
38
|
+
defaultPageSize: zod.z.number().int().positive().optional().default(10),
|
|
51
39
|
shardBumps: zod.z
|
|
52
40
|
.array(zod.z
|
|
53
41
|
.object({
|
|
54
|
-
timestamp: zod.z.number().
|
|
42
|
+
timestamp: zod.z.number().int().nonnegative(),
|
|
55
43
|
charBits: zod.z.number().int().min(1).max(5),
|
|
56
44
|
chars: zod.z.number().int().min(0).max(40),
|
|
57
45
|
})
|
|
@@ -79,7 +67,7 @@ const configSchema = zod.z
|
|
|
79
67
|
for (let i = 1; i < val.length; i++)
|
|
80
68
|
if (val[i].chars <= val[i - 1].chars)
|
|
81
69
|
ctx.addIssue({
|
|
82
|
-
code:
|
|
70
|
+
code: 'custom',
|
|
83
71
|
message: `shardBump chars do not monotonically increase at timestamp ${val[i].timestamp.toString()}`,
|
|
84
72
|
path: [i],
|
|
85
73
|
});
|
|
@@ -93,14 +81,14 @@ const configSchema = zod.z
|
|
|
93
81
|
.default({}),
|
|
94
82
|
generatedProperties: zod.z
|
|
95
83
|
.object({
|
|
96
|
-
sharded: zod.z.record(componentArray).optional().default({}),
|
|
97
|
-
unsharded: zod.z.record(componentArray).optional().default({}),
|
|
84
|
+
sharded: zod.z.record(zod.z.string(), componentArray).optional().default({}),
|
|
85
|
+
unsharded: zod.z.record(zod.z.string(), componentArray).optional().default({}),
|
|
98
86
|
})
|
|
99
87
|
.optional()
|
|
100
88
|
.default({ sharded: {}, unsharded: {} }),
|
|
101
89
|
hashKey: zod.z.string(),
|
|
102
90
|
indexes: zod.z
|
|
103
|
-
.record(zod.z.object({
|
|
91
|
+
.record(zod.z.string(), zod.z.object({
|
|
104
92
|
hashKey: zod.z.string().min(1),
|
|
105
93
|
rangeKey: zod.z.string().min(1),
|
|
106
94
|
projections: zod.z
|
|
@@ -112,15 +100,18 @@ const configSchema = zod.z
|
|
|
112
100
|
.default({}),
|
|
113
101
|
generatedKeyDelimiter: zod.z.string().regex(/\W+/).optional().default('|'),
|
|
114
102
|
generatedValueDelimiter: zod.z.string().regex(/\W+/).optional().default('#'),
|
|
115
|
-
propertyTranscodes: zod.z.record(zod.z.string()).optional().default({}),
|
|
103
|
+
propertyTranscodes: zod.z.record(zod.z.string(), zod.z.string()).optional().default({}),
|
|
116
104
|
rangeKey: zod.z.string(),
|
|
117
105
|
shardKeyDelimiter: zod.z.string().regex(/\W+/).optional().default('!'),
|
|
118
|
-
throttle: zod.z.number().int().positive().
|
|
106
|
+
throttle: zod.z.number().int().positive().optional().default(10),
|
|
119
107
|
transcodes: zod.z
|
|
120
|
-
.record(zod.z
|
|
108
|
+
.record(zod.z.string(), zod.z
|
|
121
109
|
.object({
|
|
122
|
-
|
|
123
|
-
|
|
110
|
+
// Accept function shapes without relying on z.function()
|
|
111
|
+
// to avoid TS inference conflicts across Zod versions and
|
|
112
|
+
// to remain compatible with narrower parameter types.
|
|
113
|
+
encode: zod.z.custom((fn) => typeof fn === 'function'),
|
|
114
|
+
decode: zod.z.custom((fn) => typeof fn === 'function'),
|
|
124
115
|
})
|
|
125
116
|
.strict())
|
|
126
117
|
.optional()
|
|
@@ -131,7 +122,7 @@ const configSchema = zod.z
|
|
|
131
122
|
// validate no generated key delimiter collision
|
|
132
123
|
if (data.generatedKeyDelimiter.includes(data.generatedValueDelimiter))
|
|
133
124
|
ctx.addIssue({
|
|
134
|
-
code:
|
|
125
|
+
code: 'custom',
|
|
135
126
|
message: 'generatedKeyDelimiter contains generatedValueDelimiter',
|
|
136
127
|
params: {
|
|
137
128
|
generatedKeyDelimiter: data.generatedKeyDelimiter,
|
|
@@ -141,7 +132,7 @@ const configSchema = zod.z
|
|
|
141
132
|
});
|
|
142
133
|
if (data.generatedKeyDelimiter.includes(data.shardKeyDelimiter))
|
|
143
134
|
ctx.addIssue({
|
|
144
|
-
code:
|
|
135
|
+
code: 'custom',
|
|
145
136
|
message: 'generatedKeyDelimiter contains shardKeyDelimiter',
|
|
146
137
|
params: {
|
|
147
138
|
generatedKeyDelimiter: data.generatedKeyDelimiter,
|
|
@@ -152,7 +143,7 @@ const configSchema = zod.z
|
|
|
152
143
|
// validate no generated value delimiter collision
|
|
153
144
|
if (data.generatedValueDelimiter.includes(data.generatedKeyDelimiter))
|
|
154
145
|
ctx.addIssue({
|
|
155
|
-
code:
|
|
146
|
+
code: 'custom',
|
|
156
147
|
message: 'generatedValueDelimiter contains generatedKeyDelimiter',
|
|
157
148
|
params: {
|
|
158
149
|
generatedValueDelimiter: data.generatedValueDelimiter,
|
|
@@ -162,7 +153,7 @@ const configSchema = zod.z
|
|
|
162
153
|
});
|
|
163
154
|
if (data.generatedValueDelimiter.includes(data.shardKeyDelimiter))
|
|
164
155
|
ctx.addIssue({
|
|
165
|
-
code:
|
|
156
|
+
code: 'custom',
|
|
166
157
|
message: 'generatedValueDelimiter contains shardKeyDelimiter',
|
|
167
158
|
params: {
|
|
168
159
|
generatedValueDelimiter: data.generatedValueDelimiter,
|
|
@@ -173,7 +164,7 @@ const configSchema = zod.z
|
|
|
173
164
|
// validate no shard key delimiter collision
|
|
174
165
|
if (data.shardKeyDelimiter.includes(data.generatedKeyDelimiter))
|
|
175
166
|
ctx.addIssue({
|
|
176
|
-
code:
|
|
167
|
+
code: 'custom',
|
|
177
168
|
message: 'shardKeyDelimiter contains generatedKeyDelimiter',
|
|
178
169
|
params: {
|
|
179
170
|
generatedKeyDelimiter: data.generatedKeyDelimiter,
|
|
@@ -183,7 +174,7 @@ const configSchema = zod.z
|
|
|
183
174
|
});
|
|
184
175
|
if (data.shardKeyDelimiter.includes(data.generatedValueDelimiter))
|
|
185
176
|
ctx.addIssue({
|
|
186
|
-
code:
|
|
177
|
+
code: 'custom',
|
|
187
178
|
message: 'shardKeyDelimiter contains generatedValueDelimiter',
|
|
188
179
|
params: {
|
|
189
180
|
generatedValueDelimiter: data.generatedValueDelimiter,
|
|
@@ -213,19 +204,17 @@ const configSchema = zod.z
|
|
|
213
204
|
for (const [property, transcode] of Object.entries(data.propertyTranscodes))
|
|
214
205
|
if (!transcodes.includes(transcode))
|
|
215
206
|
ctx.addIssue({
|
|
216
|
-
code:
|
|
217
|
-
|
|
207
|
+
code: 'custom',
|
|
208
|
+
message: `propertyTranscodes['${property}'] references unknown transcode '${transcode}'`,
|
|
218
209
|
path: ['propertyTranscodes', property],
|
|
219
|
-
received: transcode,
|
|
220
210
|
});
|
|
221
211
|
// Validate all sharded property elements are transcoded properties.
|
|
222
212
|
for (const [property, elements] of Object.entries(data.generatedProperties.sharded))
|
|
223
213
|
for (const element of elements)
|
|
224
214
|
if (!transcodedProperties.includes(element))
|
|
225
215
|
ctx.addIssue({
|
|
226
|
-
code:
|
|
227
|
-
|
|
228
|
-
received: element,
|
|
216
|
+
code: 'custom',
|
|
217
|
+
message: `generatedProperties.sharded['${property}'] contains non-transcoded element '${element}'`,
|
|
229
218
|
path: ['generatedProperties', 'sharded', property],
|
|
230
219
|
});
|
|
231
220
|
// Validate all unsharded property elements are transcoded properties.
|
|
@@ -233,9 +222,8 @@ const configSchema = zod.z
|
|
|
233
222
|
for (const element of elements)
|
|
234
223
|
if (!transcodedProperties.includes(element))
|
|
235
224
|
ctx.addIssue({
|
|
236
|
-
code:
|
|
237
|
-
|
|
238
|
-
received: element,
|
|
225
|
+
code: 'custom',
|
|
226
|
+
message: `generatedProperties.unsharded['${property}'] contains non-transcoded element '${element}'`,
|
|
239
227
|
path: ['generatedProperties', 'unsharded', property],
|
|
240
228
|
});
|
|
241
229
|
// Validate indexes.
|
|
@@ -244,19 +232,24 @@ const configSchema = zod.z
|
|
|
244
232
|
// Validate hash key is sharded.
|
|
245
233
|
if (![data.hashKey, ...shardedKeys].includes(hashKey)) {
|
|
246
234
|
ctx.addIssue({
|
|
247
|
-
code:
|
|
248
|
-
|
|
235
|
+
code: 'custom',
|
|
236
|
+
message: `index '${indexKey}' hashKey '${hashKey}' must be one of [${[
|
|
237
|
+
data.hashKey,
|
|
238
|
+
...shardedKeys,
|
|
239
|
+
].join(', ')}]`,
|
|
249
240
|
path: ['indexes', indexKey, 'hashKey'],
|
|
250
|
-
received: hashKey,
|
|
251
241
|
});
|
|
252
242
|
}
|
|
253
243
|
// Validate range key is unsharded or transcodable.
|
|
254
244
|
if (![data.rangeKey, ...unshardedKeys, ...transcodedProperties].includes(rangeKey)) {
|
|
255
245
|
ctx.addIssue({
|
|
256
|
-
code:
|
|
257
|
-
|
|
246
|
+
code: 'custom',
|
|
247
|
+
message: `index '${indexKey}' rangeKey '${rangeKey}' must be one of [${[
|
|
248
|
+
data.rangeKey,
|
|
249
|
+
...unshardedKeys,
|
|
250
|
+
...transcodedProperties,
|
|
251
|
+
].join(', ')}]`,
|
|
258
252
|
path: ['indexes', indexKey, 'rangeKey'],
|
|
259
|
-
received: rangeKey,
|
|
260
253
|
});
|
|
261
254
|
}
|
|
262
255
|
// Validate no index projections are keys.
|
|
@@ -271,7 +264,7 @@ const configSchema = zod.z
|
|
|
271
264
|
...unshardedKeys,
|
|
272
265
|
].includes(projection))
|
|
273
266
|
ctx.addIssue({
|
|
274
|
-
code:
|
|
267
|
+
code: 'custom',
|
|
275
268
|
message: 'index projection is a key',
|
|
276
269
|
params: { projection },
|
|
277
270
|
path: ['indexes', indexKey, 'projections'],
|
|
@@ -282,18 +275,16 @@ const configSchema = zod.z
|
|
|
282
275
|
// validate timestampProperty is a transcoded property.
|
|
283
276
|
if (!transcodedProperties.includes(timestampProperty))
|
|
284
277
|
ctx.addIssue({
|
|
285
|
-
code:
|
|
286
|
-
|
|
278
|
+
code: 'custom',
|
|
279
|
+
message: `entities['${entityToken}'].timestampProperty '${timestampProperty}' must be one of [${transcodedProperties.join(', ')}]`,
|
|
287
280
|
path: ['entities', entityToken, 'timestampProperty'],
|
|
288
|
-
received: timestampProperty,
|
|
289
281
|
});
|
|
290
282
|
// validate uniqueProperty is a transcoded property.
|
|
291
283
|
if (!transcodedProperties.includes(uniqueProperty))
|
|
292
284
|
ctx.addIssue({
|
|
293
|
-
code:
|
|
294
|
-
|
|
285
|
+
code: 'custom',
|
|
286
|
+
message: `entities['${entityToken}'].uniqueProperty '${uniqueProperty}' must be one of [${transcodedProperties.join(', ')}]`,
|
|
295
287
|
path: ['entities', entityToken, 'uniqueProperty'],
|
|
296
|
-
received: uniqueProperty,
|
|
297
288
|
});
|
|
298
289
|
}
|
|
299
290
|
});
|
|
@@ -24,7 +24,8 @@ function decodeElement(entityManager, element, value) {
|
|
|
24
24
|
if (!value)
|
|
25
25
|
return;
|
|
26
26
|
const { propertyTranscodes, transcodes } = entityManager.config;
|
|
27
|
-
const
|
|
27
|
+
const decodeFn = transcodes[propertyTranscodes[element]].decode;
|
|
28
|
+
const decoded = decodeFn(value);
|
|
28
29
|
entityManager.logger.debug('decoded entity element', {
|
|
29
30
|
element,
|
|
30
31
|
value,
|
|
@@ -40,7 +40,9 @@ function dehydratePageKeyMap(entityManager, entityToken, pageKeyMap) {
|
|
|
40
40
|
}
|
|
41
41
|
// Extract, sort & validate indexs.
|
|
42
42
|
const indexes = Object.keys(pageKeyMap).sort();
|
|
43
|
-
indexes.map((index) =>
|
|
43
|
+
indexes.map((index) => {
|
|
44
|
+
validateIndexToken.validateIndexToken(entityManager, index);
|
|
45
|
+
});
|
|
44
46
|
// Extract & sort hash keys.
|
|
45
47
|
const hashKeys = Object.keys(pageKeyMap[indexes[0]]);
|
|
46
48
|
// Dehydrate page keys.
|
|
@@ -21,8 +21,8 @@ function encodeElement(entityManager, element, item) {
|
|
|
21
21
|
const value = item[element];
|
|
22
22
|
if (value === undefined || [hashKey, rangeKey].includes(element))
|
|
23
23
|
return value;
|
|
24
|
-
const
|
|
25
|
-
|
|
24
|
+
const encodeFn = transcodes[propertyTranscodes[element]].encode;
|
|
25
|
+
const encoded = encodeFn(item[element]) || undefined;
|
|
26
26
|
entityManager.logger.debug('encoded entity element', {
|
|
27
27
|
element,
|
|
28
28
|
item,
|
|
@@ -30,7 +30,7 @@ function encodeGeneratedProperty(entityManager, property, item) {
|
|
|
30
30
|
...(sharded
|
|
31
31
|
? [item[entityManager.config.hashKey]]
|
|
32
32
|
: []),
|
|
33
|
-
...elementMap.map(([element, value]) => [element,
|
|
33
|
+
...elementMap.map(([element, value]) => [element, `${value ?? ''}`].join(entityManager.config.generatedValueDelimiter)),
|
|
34
34
|
].join(entityManager.config.generatedKeyDelimiter);
|
|
35
35
|
entityManager.logger.debug('encoded generated property', {
|
|
36
36
|
property,
|