edinburgh 0.5.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +309 -246
  2. package/build/src/datapack.d.ts +9 -9
  3. package/build/src/datapack.js +9 -9
  4. package/build/src/edinburgh.d.ts +21 -7
  5. package/build/src/edinburgh.js +53 -67
  6. package/build/src/edinburgh.js.map +1 -1
  7. package/build/src/indexes.d.ts +85 -205
  8. package/build/src/indexes.js +150 -503
  9. package/build/src/indexes.js.map +1 -1
  10. package/build/src/migrate.js +8 -10
  11. package/build/src/migrate.js.map +1 -1
  12. package/build/src/models.d.ts +152 -107
  13. package/build/src/models.js +433 -144
  14. package/build/src/models.js.map +1 -1
  15. package/build/src/types.d.ts +30 -48
  16. package/build/src/types.js +25 -24
  17. package/build/src/types.js.map +1 -1
  18. package/build/src/utils.d.ts +4 -4
  19. package/build/src/utils.js +4 -4
  20. package/package.json +2 -2
  21. package/skill/AnyModelClass.md +7 -0
  22. package/skill/FindOptions.md +37 -0
  23. package/skill/Lifecycle Hooks.md +24 -0
  24. package/skill/{Model_delete.md → Lifecycle Hooks_delete.md } +1 -1
  25. package/skill/{Model_getPrimaryKeyHash.md → Lifecycle Hooks_getPrimaryKeyHash.md } +1 -1
  26. package/skill/{Model_isValid.md → Lifecycle Hooks_isValid.md } +1 -1
  27. package/skill/Lifecycle Hooks_migrate.md +26 -0
  28. package/skill/{Model_preCommit.md → Lifecycle Hooks_preCommit.md } +2 -2
  29. package/skill/{Model_preventPersist.md → Lifecycle Hooks_preventPersist.md } +1 -1
  30. package/skill/{Model_validate.md → Lifecycle Hooks_validate.md } +2 -2
  31. package/skill/ModelBase.md +7 -0
  32. package/skill/ModelClass.md +8 -0
  33. package/skill/SKILL.md +180 -132
  34. package/skill/Schema Evolution.md +19 -0
  35. package/skill/TypeWrapper_containsNull.md +11 -0
  36. package/skill/TypeWrapper_deserialize.md +9 -0
  37. package/skill/TypeWrapper_getError.md +11 -0
  38. package/skill/TypeWrapper_serialize.md +10 -0
  39. package/skill/TypeWrapper_serializeType.md +9 -0
  40. package/skill/array.md +2 -2
  41. package/skill/defineModel.md +3 -2
  42. package/skill/deleteEverything.md +8 -0
  43. package/skill/field.md +3 -3
  44. package/skill/link.md +3 -3
  45. package/skill/literal.md +1 -1
  46. package/skill/opt.md +1 -1
  47. package/skill/or.md +1 -1
  48. package/skill/record.md +1 -1
  49. package/skill/set.md +2 -2
  50. package/skill/setOnSaveCallback.md +5 -2
  51. package/skill/transact.md +1 -1
  52. package/src/datapack.ts +9 -9
  53. package/src/edinburgh.ts +68 -68
  54. package/src/indexes.ts +251 -599
  55. package/src/migrate.ts +9 -10
  56. package/src/models.ts +528 -231
  57. package/src/types.ts +36 -34
  58. package/src/utils.ts +4 -4
  59. package/skill/BaseIndex.md +0 -16
  60. package/skill/BaseIndex_batchProcess.md +0 -10
  61. package/skill/BaseIndex_find.md +0 -7
  62. package/skill/BaseIndex_find_2.md +0 -7
  63. package/skill/BaseIndex_find_3.md +0 -7
  64. package/skill/BaseIndex_find_4.md +0 -7
  65. package/skill/Model.md +0 -20
  66. package/skill/Model_batchProcess.md +0 -8
  67. package/skill/Model_migrate.md +0 -32
  68. package/skill/Model_replaceInto.md +0 -16
  69. package/skill/NonPrimaryIndex.md +0 -10
  70. package/skill/SecondaryIndex.md +0 -9
  71. package/skill/UniqueIndex.md +0 -9
  72. package/skill/dump.md +0 -8
package/src/indexes.ts CHANGED
@@ -1,15 +1,26 @@
1
1
  import * as lowlevel from "olmdb/lowlevel";
2
2
  import { DatabaseError } from "olmdb/lowlevel";
3
3
  import DataPack from "./datapack.js";
4
- import { FieldConfig, Model, Transaction, currentTxn } from "./models.js";
5
- import { scheduleInit, transact } from "./edinburgh.js";
4
+ import { currentTxn, transact, type Transaction } from "./edinburgh.js";
6
5
  import { assert, logLevel, dbGet, dbPut, dbDel, hashBytes, hashFunction, bytesEqual, toBuffer } from "./utils.js";
7
6
  import { deserializeType, serializeType, TypeWrapper } from "./types.js";
8
7
 
9
- // Index system types and utilities
10
- type IndexArgTypes<M extends typeof Model<any>, F extends readonly (keyof InstanceType<M> & string)[]> = {
11
- [I in keyof F]: InstanceType<M>[F[I]]
12
- }
8
+ type IndexItem = {
9
+ _setLoadedField(fieldName: string, value: any): void;
10
+ _restoreLazyFields?(): void;
11
+ };
12
+ type PrimaryKeyItem = IndexItem & {
13
+ _oldValues: Record<string, any> | undefined | null | false;
14
+ _primaryKey: Uint8Array | undefined;
15
+ _txn: Transaction;
16
+ _setPrimaryKey(key: Uint8Array, hash?: number): void;
17
+ };
18
+ type FieldTypes = ReadonlyMap<string, TypeWrapper<any>>;
19
+ type LoadPrimary<ITEM> = (txn: Transaction, primaryKey: Uint8Array, loadNow: boolean | Uint8Array) => ITEM | undefined;
20
+ type QueueInitialization = () => void;
21
+ type IndexArgTypes<ITEM, F extends readonly (keyof ITEM & string)[]> = {
22
+ [I in keyof F]: ITEM[F[I]]
23
+ };
13
24
 
14
25
  const MAX_INDEX_ID_PREFIX = -1;
15
26
  const INDEX_ID_PREFIX = -2;
@@ -22,7 +33,7 @@ export interface VersionInfo {
22
33
  migrateHash: number;
23
34
  /** Non-key field names → TypeWrappers for deserialization of this version's data. */
24
35
  nonKeyFields: Map<string, TypeWrapper<any>>;
25
- /** Set of serialized secondary index signatures that existed in this version. */
36
+ /** Set of serialized secondary index signatures that existed in this version's data. */
26
37
  secondaryKeys: Set<string>;
27
38
  }
28
39
 
@@ -31,21 +42,18 @@ export interface VersionInfo {
31
42
  * Handles common iteration logic for both primary and unique indexes.
32
43
  * Extends built-in Iterator to provide map/filter/reduce/toArray/etc.
33
44
  */
34
- export class IndexRangeIterator<M extends typeof Model> extends Iterator<InstanceType<M>> {
45
+ export class IndexRangeIterator<ITEM> extends Iterator<ITEM> {
35
46
  constructor(
36
47
  private txn: Transaction,
37
48
  private iteratorId: number,
38
- private indexId: number,
39
- private parentIndex: BaseIndex<M, any>
49
+ private parentIndex: BaseIndex<ITEM, any>
40
50
  ) {
41
51
  super();
42
52
  }
43
53
 
44
- // This is also in Iterator<InstanceType<M>>, but we'll repeat it here for deps that
45
- // don't have ESNext.Iterator in their TypeScript lib set.
46
54
  [Symbol.iterator](): this { return this; }
47
55
 
48
- next(): IteratorResult<InstanceType<M>> {
56
+ next(): IteratorResult<ITEM> {
49
57
  if (this.iteratorId < 0) return { done: true, value: undefined };
50
58
  const raw = lowlevel.readIterator(this.iteratorId);
51
59
  if (!raw) {
@@ -53,10 +61,8 @@ export class IndexRangeIterator<M extends typeof Model> extends Iterator<Instanc
53
61
  this.iteratorId = -1;
54
62
  return { done: true, value: undefined };
55
63
  }
56
-
57
- // Dispatches to the _pairToInstance specific to the index type
58
- const model = this.parentIndex._pairToInstance(this.txn, raw.key, raw.value);
59
64
 
65
+ const model = this.parentIndex._pairToInstance(this.txn, raw.key, raw.value);
60
66
  return { done: false, value: model };
61
67
  }
62
68
 
@@ -66,15 +72,27 @@ export class IndexRangeIterator<M extends typeof Model> extends Iterator<Instanc
66
72
  return result;
67
73
  }
68
74
 
69
- fetch(): InstanceType<M> | undefined {
75
+ fetch(): ITEM | undefined {
70
76
  for (const model of this) {
71
- return model; // Return the first model found
77
+ return model;
72
78
  }
73
79
  }
74
80
  }
75
81
 
76
82
  type ArrayOrOnlyItem<ARG_TYPES extends readonly any[]> = ARG_TYPES extends readonly [infer A] ? (A | Partial<ARG_TYPES>) : Partial<ARG_TYPES>;
77
83
 
84
+ /**
85
+ * Range-query options accepted by `find()`, `findBy()`, `batchProcess()`, and `batchProcessBy()`.
86
+ *
87
+ * Supports exact-match lookups via `is`, inclusive bounds via `from` / `to`,
88
+ * exclusive bounds via `after` / `before`, and reverse scans.
89
+ *
90
+ * For single-field indexes, values can be passed directly. For composite indexes,
91
+ * pass tuples or partial tuples for prefix matching.
92
+ *
93
+ * @template ARG_TYPES - Tuple of index argument types.
94
+ * @template FETCH - Optional fetch mode used by overloads that return one row.
95
+ */
78
96
  export type FindOptions<ARG_TYPES extends readonly any[], FETCH extends 'first' | 'single' | undefined = undefined> = (
79
97
  (
80
98
  {is: ArrayOrOnlyItem<ARG_TYPES>;} // Shortcut for setting `from` and `to` to the same value
@@ -103,76 +121,66 @@ export type FindOptions<ARG_TYPES extends readonly any[], FETCH extends 'first'
103
121
  & (FETCH extends undefined ? { fetch?: undefined } : { fetch: FETCH })
104
122
  );
105
123
 
106
-
107
124
  /**
108
125
  * Base class for database indexes for efficient lookups on model fields.
109
- *
126
+ *
110
127
  * Indexes enable fast queries on specific field combinations and enforce uniqueness constraints.
111
- *
112
- * @template M - The model class this index belongs to.
113
- * @template F - The field names that make up this index.
114
128
  */
115
- export abstract class BaseIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[], ARGS extends readonly any[] = IndexArgTypes<M, F>> {
116
- public _MyModel: M;
117
- public _fieldTypes: Map<keyof InstanceType<M> & string, TypeWrapper<any>> = new Map();
118
- public _fieldCount!: number;
129
+ export abstract class BaseIndex<ITEM, const F extends readonly (keyof ITEM & string)[], ARGS extends readonly any[] = IndexArgTypes<ITEM, F>> {
130
+ public tableName!: string;
131
+ public _indexFields: Map<F[number], TypeWrapper<any>> = new Map();
119
132
  _computeFn?: (data: any) => any[];
120
-
121
- /**
122
- * Create a new index.
123
- * @param MyModel - The model class this index belongs to.
124
- * @param _fieldNames - Array of field names that make up this index.
125
- */
126
- constructor(MyModel: M, public _fieldNames: F) {
127
- this._MyModel = MyModel;
133
+ _indexId?: number;
134
+ _signature?: string;
135
+
136
+ constructor(tableName: string, fieldNames: F) {
137
+ this.tableName = tableName;
138
+ this._indexFields = new Map(fieldNames.map(fieldName => [fieldName, undefined as unknown as TypeWrapper<any>]));
128
139
  }
129
140
 
130
- async _delayedInit() {
131
- if (this._indexId != null) return; // Already initialized
141
+ async _initializeIndex(fields: FieldTypes, reset = false, primaryFieldTypes?: FieldTypes) {
142
+ const fieldNames = [...this._indexFields.keys()];
143
+ if (reset) {
144
+ this._indexId = undefined;
145
+ this._signature = undefined;
146
+ } else if (this._indexId != null) {
147
+ return;
148
+ }
149
+
132
150
  if (this._computeFn) {
133
- this._fieldCount = 1;
151
+ this._indexFields = new Map();
134
152
  } else {
135
- for(const fieldName of this._fieldNames) {
153
+ this._indexFields = new Map();
154
+ for (const fieldName of fieldNames) {
136
155
  assert(typeof fieldName === 'string', 'Field names must be strings');
137
- this._fieldTypes.set(fieldName, this._MyModel.fields[fieldName].type);
156
+ const fieldType = fields.get(fieldName);
157
+ assert(fieldType, `Unknown field '${fieldName}' in ${this}`);
158
+ this._indexFields.set(fieldName, fieldType);
138
159
  }
139
- this._fieldCount = this._fieldNames.length;
140
160
  }
141
- await this._retrieveIndexId();
142
161
 
143
- // Human-readable signature for version tracking, e.g. "secondary category:string"
162
+ await this._retrieveIndexId(fields, primaryFieldTypes);
163
+
144
164
  if (this._computeFn) {
145
165
  this._signature = this._getTypeName() + ' ' + hashFunction(this._computeFn);
146
166
  } else {
147
167
  this._signature = this._getTypeName() + ' ' +
148
- Array.from(this._fieldTypes.entries()).map(([n, t]) => n + ':' + t).join(' ');
168
+ Array.from(this._indexFields.entries()).map(([name, fieldType]) => name + ':' + fieldType).join(' ');
149
169
  }
150
170
  }
151
171
 
152
- _indexId?: number;
153
-
154
- /** Human-readable signature for version tracking, e.g. "secondary category:string" */
155
- _signature?: string;
156
-
157
- /**
158
- * Serialize array of key values to a (index-id prefixed) Bytes instance that can be used as a key.
159
- * @param args - Field values to serialize (can be partial for range queries).
160
- * @returns A Bytes instance containing the index id and serialized key parts.
161
- * @internal
162
- */
163
172
  _argsToKeyBytes(args: [], allowPartial: boolean): DataPack;
164
173
  _argsToKeyBytes(args: Partial<ARGS>, allowPartial: boolean): DataPack;
165
-
166
174
  _argsToKeyBytes(args: any, allowPartial: boolean) {
167
- assert(allowPartial ? args.length <= this._fieldCount : args.length === this._fieldCount);
175
+ const expectedCount = this._computeFn ? 1 : this._indexFields.size;
176
+ assert(allowPartial ? args.length <= expectedCount : args.length === expectedCount);
168
177
  const bytes = new DataPack();
169
178
  bytes.write(this._indexId!);
170
179
  if (this._computeFn) {
171
180
  if (args.length > 0) bytes.write(args[0]);
172
181
  } else {
173
182
  let index = 0;
174
- for(const fieldType of this._fieldTypes.values()) {
175
- // For partial keys, undefined values are acceptable and represent open range suffixes
183
+ for (const fieldType of this._indexFields.values()) {
176
184
  if (index >= args.length) break;
177
185
  fieldType.serialize(args[index++], bytes);
178
186
  }
@@ -180,36 +188,25 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
180
188
  return bytes;
181
189
  }
182
190
 
183
- /**
184
- * Extract model from iterator entry - implemented differently by each index type.
185
- * @param keyBuffer - Key bytes (including index id).
186
- * @param valueBuffer - Value bytes from the entry.
187
- * @returns Model instance or undefined.
188
- * @internal
189
- */
190
- abstract _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): InstanceType<M>;
191
+ abstract _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): ITEM;
191
192
 
192
- /**
193
- * Retrieve (or create) a stable index ID from the DB, with retry on transaction races.
194
- * Sets `this._indexId` on success.
195
- */
196
- async _retrieveIndexId(): Promise<void> {
197
- const indexNameBytes = new DataPack().write(INDEX_ID_PREFIX).write(this._MyModel.tableName).write(this._getTypeName());
193
+ abstract _getTypeName(): string;
194
+
195
+ async _retrieveIndexId(fields: FieldTypes, primaryFieldTypes?: FieldTypes): Promise<void> {
196
+ const indexNameBytes = new DataPack().write(INDEX_ID_PREFIX).write(this.tableName).write(this._getTypeName());
198
197
  if (this._computeFn) {
199
198
  indexNameBytes.write(hashFunction(this._computeFn));
200
199
  } else {
201
- for(let name of this._fieldNames) {
200
+ for (const name of this._indexFields.keys()) {
202
201
  indexNameBytes.write(name);
203
- serializeType(this._MyModel.fields[name].type, indexNameBytes);
202
+ serializeType(fields.get(name)!, indexNameBytes);
204
203
  }
205
204
  }
206
- // For non-primary indexes, include primary key field info to avoid misinterpreting
207
- // values when the primary key schema changes.
208
- if (this._MyModel._primary !== (this as any)) {
209
- indexNameBytes.write(undefined); // separator
210
- for (const name of this._MyModel._primary._fieldNames) {
205
+ if (primaryFieldTypes) {
206
+ indexNameBytes.write(undefined);
207
+ for (const [name, fieldType] of primaryFieldTypes.entries()) {
211
208
  indexNameBytes.write(name);
212
- serializeType(this._MyModel.fields[name].type, indexNameBytes);
209
+ serializeType(fieldType, indexNameBytes);
213
210
  }
214
211
  }
215
212
  const indexNameBuf = indexNameBytes.toUint8Array();
@@ -242,65 +239,6 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
242
239
  }
243
240
  }
244
241
 
245
-
246
- /**
247
- * Find model instances using flexible range query options.
248
- *
249
- * Supports exact matches, inclusive/exclusive range queries, and reverse iteration.
250
- * For single-field indexes, you can pass values directly or in arrays.
251
- * For multi-field indexes, pass arrays or partial arrays for prefix matching.
252
- *
253
- * @param opts - Query options object
254
- * @param opts.is - Exact match (sets both `from` and `to` to same value)
255
- * @param opts.from - Range start (inclusive)
256
- * @param opts.after - Range start (exclusive)
257
- * @param opts.to - Range end (inclusive)
258
- * @param opts.before - Range end (exclusive)
259
- * @param opts.reverse - Whether to iterate in reverse order
260
- * @returns An iterable of model instances matching the query
261
- *
262
- * @example
263
- * ```typescript
264
- * // Exact match
265
- * for (const user of User.byEmail.find({is: "john@example.com"})) {
266
- * console.log(user.name);
267
- * }
268
- *
269
- * // Range query (inclusive)
270
- * for (const user of User.byEmail.find({from: "a@", to: "m@"})) {
271
- * console.log(user.email);
272
- * }
273
- *
274
- * // Range query (exclusive)
275
- * for (const user of User.byEmail.find({after: "a@", before: "m@"})) {
276
- * console.log(user.email);
277
- * }
278
- *
279
- * // Open-ended ranges
280
- * for (const user of User.byEmail.find({from: "m@"})) { // m@ and later
281
- * console.log(user.email);
282
- * }
283
- *
284
- * for (const user of User.byEmail.find({to: "m@"})) { // up to and including m@
285
- * console.log(user.email);
286
- * }
287
- *
288
- * // Reverse iteration
289
- * for (const user of User.byEmail.find({reverse: true})) {
290
- * console.log(user.email); // Z to A order
291
- * }
292
- *
293
- * // Multi-field index prefix matching
294
- * for (const item of CompositeModel.find({from: ["electronics", "phones"]})) {
295
- * console.log(item.name); // All electronics/phones items
296
- * }
297
- *
298
- * // For single-field indexes, you can use the value directly
299
- * for (const user of User.byEmail.find({is: "john@example.com"})) {
300
- * console.log(user.name);
301
- * }
302
- * ```
303
- */
304
242
  _computeKeyBounds(opts: FindOptions<ARGS>): [DataPack | undefined, DataPack | undefined] | null {
305
243
  let startKey: DataPack | undefined;
306
244
  let endKey: DataPack | undefined;
@@ -327,27 +265,39 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
327
265
  return [startKey, endKey];
328
266
  }
329
267
 
330
- public find(opts?: FindOptions<ARGS, 'first'>): InstanceType<M> | undefined;
331
- public find(opts: FindOptions<ARGS, 'single'>): InstanceType<M>;
332
- public find(opts?: FindOptions<ARGS>): IndexRangeIterator<M>;
333
- public find(opts: any = {}): IndexRangeIterator<M> | InstanceType<M> | undefined {
268
+ /**
269
+ * Find rows using exact-match or range-query options.
270
+ *
271
+ * Supports exact matches, inclusive and exclusive bounds, open-ended ranges,
272
+ * and reverse iteration. For single-field indexes, values can be passed
273
+ * directly. For composite indexes, pass tuples or partial tuples.
274
+ *
275
+ * @example
276
+ * ```typescript
277
+ * const exact = User.find({ is: "user-123", fetch: "first" });
278
+ * const email = [...User.findBy("email", { from: "a@test.com", to: "m@test.com" })];
279
+ * const reverse = [...Product.findBy("category", { is: "electronics", reverse: true })];
280
+ * ```
281
+ */
282
+ public find(opts: FindOptions<ARGS, 'first'>): ITEM | undefined;
283
+ public find(opts: FindOptions<ARGS, 'single'>): ITEM;
284
+ public find(opts?: FindOptions<ARGS>): IndexRangeIterator<ITEM>;
285
+ public find(opts: any = {}): IndexRangeIterator<ITEM> | ITEM | undefined {
334
286
  const txn = currentTxn();
335
- const indexId = this._indexId!;
336
287
 
337
288
  const bounds = this._computeKeyBounds(opts);
338
289
  if (!bounds) {
339
290
  if (opts.fetch === 'single') throw new DatabaseError('Expected exactly one result, got none', 'NOT_FOUND');
340
291
  if (opts.fetch === 'first') return undefined;
341
- return new IndexRangeIterator(txn, -1, indexId, this);
292
+ return new IndexRangeIterator(txn, -1, this);
342
293
  }
343
294
  const [startKey, endKey] = bounds;
344
295
 
345
- // For reverse scans, swap start/end keys since OLMDB expects it
346
296
  const scanStart = opts.reverse ? endKey : startKey;
347
297
  const scanEnd = opts.reverse ? startKey : endKey;
348
298
 
349
299
  if (logLevel >= 3) {
350
- console.log(`[edinburgh] Scan ${this} start=${scanStart} end=${scanEnd} reverse=${opts.reverse||false}`);
300
+ console.log(`[edinburgh] Scan ${this} start=${scanStart} end=${scanEnd} reverse=${opts.reverse || false}`);
351
301
  }
352
302
  const startBuf = scanStart?.toUint8Array();
353
303
  const endBuf = scanEnd?.toUint8Array();
@@ -357,8 +307,8 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
357
307
  endBuf ? toBuffer(endBuf) : undefined,
358
308
  opts.reverse || false,
359
309
  );
360
-
361
- const iter = new IndexRangeIterator(txn, iteratorId, indexId, this);
310
+
311
+ const iter = new IndexRangeIterator(txn, iteratorId, this);
362
312
  if (opts.fetch === 'first') return iter.fetch();
363
313
  if (opts.fetch === 'single') {
364
314
  const first = iter.fetch();
@@ -370,19 +320,17 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
370
320
  }
371
321
 
372
322
  /**
373
- * Process all matching rows in batched transactions.
323
+ * Process matching rows in batched transactions.
374
324
  *
375
- * Uses the same query options as {@link find}. The batch is committed and a new
376
- * transaction started once either `limitSeconds` or `limitRows` is exceeded.
325
+ * Uses the same range options as {@link find}, plus optional row and time
326
+ * limits that control when the current transaction is committed and a new one starts.
377
327
  *
378
- * @param opts - Query options (same as `find()`), plus:
379
- * @param opts.limitSeconds - Max seconds per transaction batch (default: 1)
380
- * @param opts.limitRows - Max rows per transaction batch (default: 4096)
381
- * @param callback - Called for each matching row within a transaction
328
+ * @param opts Query options plus batch limits.
329
+ * @param callback Called for each matching row inside a transaction.
382
330
  */
383
331
  public async batchProcess(
384
332
  opts: FindOptions<ARGS> & { limitSeconds?: number; limitRows?: number } = {} as any,
385
- callback: (row: InstanceType<M>) => void | Promise<void>
333
+ callback: (row: ITEM) => void | Promise<void>
386
334
  ): Promise<void> {
387
335
  const limitMs = (opts.limitSeconds ?? 1) * 1000;
388
336
  const limitRows = opts.limitRows ?? 4096;
@@ -424,10 +372,10 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
424
372
  lowlevel.closeIterator(iteratorId);
425
373
  }
426
374
 
427
- lastRawKey = lastRawKey.slice(); // Copy, as lastRawKey points at OLMDB's internal read-only mmap
428
- if (reverse) return lastRawKey!;
429
- const nk = new DataPack(lastRawKey!);
430
- return nk.increment() ? nk.toUint8Array() : null;
375
+ lastRawKey = lastRawKey.slice();
376
+ if (reverse) return lastRawKey;
377
+ const nextKey = new DataPack(lastRawKey);
378
+ return nextKey.increment() ? nextKey.toUint8Array() : null;
431
379
  });
432
380
 
433
381
  if (next === null) break;
@@ -435,104 +383,27 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
435
383
  }
436
384
  }
437
385
 
438
- abstract _getTypeName(): string;
439
-
440
386
  toString() {
441
- return `${this._indexId}:${this._MyModel.tableName}:${this._getTypeName()}[${Array.from(this._fieldTypes.keys()).join(',')}]`;
387
+ return `${this._indexId}:${this.tableName}:${this._getTypeName()}[${Array.from(this._indexFields.keys()).join(',')}]`;
442
388
  }
443
389
  }
444
390
 
445
- function toArray<ARG_TYPES extends readonly any[]>(args: ArrayOrOnlyItem<ARG_TYPES>): Partial<ARG_TYPES> {
446
- // Convert single value or array to array format compatible with Partial<ARG_TYPES>
447
- return (Array.isArray(args) ? args : [args]) as Partial<ARG_TYPES>;
448
- }
449
-
450
- /**
451
- * Primary index that stores the actual model data.
452
- *
453
- * @template M - The model class this index belongs to.
454
- * @template F - The field names that make up this index.
455
- */
456
- export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[]> extends BaseIndex<M, F, IndexArgTypes<M, F>> {
457
-
458
- _nonKeyFields!: (keyof InstanceType<M> & string)[];
459
- _lazyDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
460
- _resetDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
461
- _freezePrimaryKeyDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
462
-
463
- /** Current version number for this primary index's value format. */
464
- _currentVersion!: number;
465
- /** Hash of the current migrate() function source, or 0 if none. */
466
- _currentMigrateHash!: number;
467
- /** Cached version info for old versions (loaded on demand). */
468
- _versions: Map<number, VersionInfo> = new Map();
469
-
470
- constructor(MyModel: M, fieldNames: F) {
471
- super(MyModel, fieldNames);
472
- if (MyModel._primary) {
473
- throw new DatabaseError(`There's already a primary index defined: ${MyModel._primary}. This error may also indicate that your tsconfig.json needs to have "target": "ES2022" set.`, 'INIT_ERROR');
474
- }
475
- MyModel._primary = this;
391
+ export abstract class PrimaryKey<ITEM extends PrimaryKeyItem, const F extends readonly (keyof ITEM & string)[], ARGS extends readonly any[] = IndexArgTypes<ITEM, F>> extends BaseIndex<ITEM, F, ARGS> {
392
+ _getTypeName(): string {
393
+ return 'primary';
476
394
  }
477
395
 
478
- async _delayedInit() {
479
- if (this._indexId != null) return; // Already initialized
480
- await super._delayedInit();
481
- const MyModel = this._MyModel;
482
- this._nonKeyFields = Object.keys(MyModel.fields).filter(fieldName => !this._fieldNames.includes(fieldName as any)) as any;
483
-
484
- for(const fieldName of this._nonKeyFields) {
485
- this._lazyDescriptors[fieldName] = {
486
- configurable: true,
487
- enumerable: true,
488
- get(this: InstanceType<M>) {
489
- this.constructor._primary._lazyNow(this);
490
- return this[fieldName];
491
- },
492
- set(this: InstanceType<M>, value: any) {
493
- this.constructor._primary._lazyNow(this);
494
- this[fieldName] = value;
495
- }
496
- };
497
- this._resetDescriptors[fieldName] = {
498
- writable: true,
499
- enumerable: true
500
- };
501
- }
502
-
503
- for(const fieldName of this._fieldNames) {
504
- this._freezePrimaryKeyDescriptors[fieldName] = {
505
- writable: false,
506
- enumerable: true
507
- };
508
- }
509
-
510
- }
396
+ abstract _serializeValue(data: Record<string, any>): Uint8Array;
511
397
 
512
- /** Serialize the current version fingerprint as a DataPack object. */
513
- _serializeVersionValue(): Uint8Array {
514
- const fields: [string, Uint8Array][] = [];
515
- for (const fieldName of this._nonKeyFields) {
516
- const tp = new DataPack();
517
- serializeType(this._MyModel.fields[fieldName].type, tp);
518
- fields.push([fieldName, tp.toUint8Array()]);
519
- }
520
- return new DataPack().write({
521
- migrateHash: this._currentMigrateHash,
522
- fields,
523
- secondaryKeys: new Set((this._MyModel._secondaries || []).map(sec => sec._signature!)),
524
- }).toUint8Array();
398
+ _versionInfoKey(version: number): Uint8Array {
399
+ return new DataPack()
400
+ .write(VERSION_INFO_PREFIX)
401
+ .write(this._indexId!)
402
+ .write(version)
403
+ .toUint8Array();
525
404
  }
526
405
 
527
- /** Look up or create the current version number for this primary index. */
528
- async _initVersioning(): Promise<void> {
529
- // Compute migrate hash from function source
530
- const migrateFn = (this._MyModel as any)._original?.migrate ?? (this._MyModel as any).migrate;
531
- this._currentMigrateHash = migrateFn ? hashFunction(migrateFn) : 0;
532
-
533
- const currentValueBytes = this._serializeVersionValue();
534
-
535
- // Scan last 20 version info rows for this primary index
406
+ async _ensureVersionEntry(currentValueBytes: Uint8Array): Promise<{ version: number; created: boolean }> {
536
407
  const scanStart = new DataPack().write(VERSION_INFO_PREFIX).write(this._indexId!);
537
408
  const scanEnd = scanStart.clone(true).increment();
538
409
 
@@ -543,12 +414,12 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
543
414
  txnId,
544
415
  scanEnd ? toBuffer(scanEnd.toUint8Array()) : undefined,
545
416
  toBuffer(scanStart.toUint8Array()),
546
- true // reverse - scan newest versions first
417
+ true,
547
418
  );
548
419
 
549
420
  let count = 0;
550
421
  let maxVersion = 0;
551
- let found = false;
422
+ let matchingVersion: number | undefined;
552
423
 
553
424
  try {
554
425
  while (count < 20) {
@@ -557,15 +428,14 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
557
428
  count++;
558
429
 
559
430
  const keyPack = new DataPack(new Uint8Array(raw.key));
560
- keyPack.readNumber(); // skip VERSION_INFO_PREFIX
561
- keyPack.readNumber(); // skip indexId
431
+ keyPack.readNumber();
432
+ keyPack.readNumber();
562
433
  const versionNum = keyPack.readNumber();
563
434
  maxVersion = Math.max(maxVersion, versionNum);
564
435
 
565
436
  const valueBytes = new Uint8Array(raw.value);
566
437
  if (bytesEqual(valueBytes, currentValueBytes)) {
567
- this._currentVersion = versionNum;
568
- found = true;
438
+ matchingVersion = versionNum;
569
439
  break;
570
440
  }
571
441
  }
@@ -573,25 +443,18 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
573
443
  lowlevel.closeIterator(iteratorId);
574
444
  }
575
445
 
576
- if (found) {
446
+ if (matchingVersion !== undefined) {
577
447
  lowlevel.abortTransaction(txnId);
578
- return;
448
+ return { version: matchingVersion, created: false };
579
449
  }
580
450
 
581
- // No match found - create new version
582
- this._currentVersion = maxVersion + 1;
583
- const versionKey = new DataPack()
584
- .write(VERSION_INFO_PREFIX)
585
- .write(this._indexId!)
586
- .write(this._currentVersion)
587
- .toUint8Array();
588
- dbPut(txnId, versionKey, currentValueBytes);
589
- if (logLevel >= 1) console.log(`[edinburgh] Create version ${this._currentVersion} for ${this}`);
451
+ const version = maxVersion + 1;
452
+ dbPut(txnId, this._versionInfoKey(version), currentValueBytes);
453
+ if (logLevel >= 1) console.log(`[edinburgh] Create version ${version} for ${this}`);
590
454
 
591
455
  const commitResult = lowlevel.commitTransaction(txnId);
592
456
  const commitSeq = typeof commitResult === 'number' ? commitResult : await commitResult;
593
- if (commitSeq > 0) return;
594
- // Race - retry
457
+ if (commitSeq > 0) return { version, created: true };
595
458
  } catch (e) {
596
459
  try { lowlevel.abortTransaction(txnId); } catch {}
597
460
  throw e;
@@ -599,229 +462,34 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
599
462
  }
600
463
  }
601
464
 
602
- /**
603
- * Get a model instance by primary key values.
604
- * @param args - The primary key values.
605
- * @returns The model instance if found, undefined otherwise.
606
- *
607
- * @example
608
- * ```typescript
609
- * const user = User.get("john_doe");
610
- * ```
611
- */
612
- get(...args: IndexArgTypes<M, F>): InstanceType<M> | undefined {
613
- return this._get(currentTxn(), args, true);
614
- }
615
-
616
- /**
617
- * Does the same as as `get()`, but will delay loading the instance from disk until the first
618
- * property access. In case it turns out the instance doesn't exist, an error will be thrown
619
- * at that time.
620
- * @param args Primary key field values. (Or a single Uint8Array containing the key.)
621
- * @returns The (lazily loaded) model instance.
622
- */
623
- getLazy(...args: IndexArgTypes<M, F>): InstanceType<M> {
624
- return this._get(currentTxn(), args, false);
625
- }
626
-
627
- _get(txn: Transaction, args: IndexArgTypes<M, F> | Uint8Array, loadNow: false | Uint8Array): InstanceType<M>;
628
- _get(txn: Transaction, args: IndexArgTypes<M, F> | Uint8Array, loadNow: true): InstanceType<M> | undefined;
629
- _get(txn: Transaction, args: IndexArgTypes<M, F> | Uint8Array, loadNow: boolean | Uint8Array) {
630
- let key: Uint8Array, keyParts;
631
- if (args instanceof Uint8Array) {
632
- key = args;
633
- } else {
634
- key = this._argsToKeyBytes(args as IndexArgTypes<M, F>, false).toUint8Array();
635
- keyParts = args;
636
- }
637
-
638
- const keyHash = hashBytes(key);
639
- const cached = txn.instancesByPk.get(keyHash) as InstanceType<M>;
640
- if (cached) {
641
- if (loadNow && loadNow !== true) {
642
- // The object already exists, but it may still be lazy-loaded
643
- Object.defineProperties(cached, this._resetDescriptors);
644
- this._setNonKeyValues(cached, loadNow);
645
- }
646
- return cached;
647
- }
648
-
649
- let valueBuffer: Uint8Array | undefined;
650
- if (loadNow) {
651
- if (loadNow === true) {
652
- valueBuffer = dbGet(txn.id, key);
653
- if (logLevel >= 3) {
654
- console.log(`[edinburgh] Get ${this} key=${new DataPack(key)} result=${valueBuffer && new DataPack(valueBuffer)}`);
655
- }
656
- if (!valueBuffer) return;
657
- } else {
658
- valueBuffer = loadNow; // Uint8Array
659
- }
660
- }
661
-
662
- // This is a primary index. So we can now deserialize all primary and non-primary fields into instance values.
663
- const model = new (this._MyModel as any)(undefined, txn) as InstanceType<M>;
664
-
665
- // Set to the original value for all fields that are loaded by _setLoadedField
666
- model._oldValues = {};
667
-
668
- // Set the primary key fields on the model
669
- if (keyParts) {
670
- let index = 0;
671
- for(const fieldName of this._fieldTypes.keys()) {
672
- model._setLoadedField(fieldName, keyParts[index++] as any);
673
- }
674
- } else {
675
- const bytes = new DataPack(key);
676
- assert(bytes.readNumber() === this._MyModel._primary._indexId); // Skip index id
677
- for(const [fieldName, fieldType] of this._fieldTypes.entries()) {
678
- model._setLoadedField(fieldName, fieldType.deserialize(bytes));
679
- }
680
- }
681
-
682
- // Store the primary key on the model, set the hash, and freeze the primary key fields.
683
- model._setPrimaryKey(key, keyHash);
684
-
685
- if (valueBuffer) {
686
- // Non-lazy load. Set other fields
687
- this._setNonKeyValues(model, valueBuffer);
688
- } else {
689
- // Lazy - set getters for other fields
690
- Object.defineProperties(model, this._lazyDescriptors);
691
- // When creating a lazy instance, we don't need to add it to txn.instances yet, as only the
692
- // primary key fields are loaded, and they cannot be modified (so we don't need to check).
693
- // When any other field is set, that will trigger a lazy-load, adding the instance to
694
- // txn.instances.
695
- }
696
-
697
- txn.instancesByPk.set(keyHash, model);
698
- return model;
699
- }
700
-
701
- /**
702
- * Serialize primary key bytes from field values: indexId + typed field values.
703
- */
704
- _serializeKey(data: Record<string, any>): DataPack {
465
+ _serializePK(data: Record<string, any>): DataPack {
705
466
  const bytes = new DataPack();
706
467
  bytes.write(this._indexId!);
707
- for (const [fieldName, fieldType] of this._fieldTypes.entries()) {
468
+ for (const [fieldName, fieldType] of this._indexFields.entries()) {
708
469
  fieldType.serialize(data[fieldName], bytes);
709
470
  }
710
471
  return bytes;
711
472
  }
712
473
 
713
- _lazyNow(model: InstanceType<M>) {
714
- let valueBuffer = dbGet(model._txn.id, model._primaryKey!);
715
- if (logLevel >= 3) {
716
- console.log(`[edinburgh] Lazy retrieve ${this} key=${new DataPack(model._primaryKey)} result=${valueBuffer && new DataPack(valueBuffer)}`);
717
- }
718
- if (!valueBuffer) throw new DatabaseError(`Lazy-loaded ${model.constructor.name}#${model._primaryKey} does not exist`, 'LAZY_FAIL');
719
- Object.defineProperties(model, this._resetDescriptors);
720
- this._setNonKeyValues(model, valueBuffer);
721
- }
722
-
723
- _setNonKeyValues(model: InstanceType<M>, valueArray: Uint8Array) {
724
- const fieldConfigs = this._MyModel.fields;
725
- const valuePack = new DataPack(valueArray);
726
- const version = valuePack.readNumber();
727
-
728
- if (version === this._currentVersion) {
729
- for (const fieldName of this._nonKeyFields) {
730
- model._setLoadedField(fieldName, fieldConfigs[fieldName].type.deserialize(valuePack));
731
- }
732
- } else {
733
- this._migrateFromVersion(model, version, valuePack);
734
- }
735
- }
736
-
737
- /** Load a version's info from DB, caching the result. */
738
- _loadVersionInfo(txnId: number, version: number): VersionInfo {
739
- let info = this._versions.get(version);
740
- if (info) return info;
741
-
742
- const key = new DataPack()
743
- .write(VERSION_INFO_PREFIX)
744
- .write(this._indexId!)
745
- .write(version)
746
- .toUint8Array();
747
- const raw = dbGet(txnId, key);
748
- if (!raw) throw new DatabaseError(`Version ${version} info not found for index ${this}`, 'CONSISTENCY_ERROR');
749
-
750
- const obj = new DataPack(raw).read() as any;
751
- if (!obj || typeof obj.migrateHash !== 'number' || !Array.isArray(obj.fields) || !(obj.secondaryKeys instanceof Set))
752
- throw new DatabaseError(`Version ${version} info is corrupted for index ${this}`, 'CONSISTENCY_ERROR');
753
-
754
- const nonKeyFields = new Map<string, TypeWrapper<any>>();
755
- for (const [name, typeBytes] of obj.fields) {
756
- nonKeyFields.set(name, deserializeType(new DataPack(typeBytes), 0));
757
- }
758
-
759
- info = { migrateHash: obj.migrateHash, nonKeyFields, secondaryKeys: obj.secondaryKeys as Set<string> };
760
- this._versions.set(version, info);
761
- return info;
762
- }
763
-
764
- /** Deserialize and migrate a row from an old version. */
765
- _migrateFromVersion(model: InstanceType<M>, version: number, valuePack: DataPack) {
766
- const versionInfo = this._loadVersionInfo(model._txn.id, version);
767
-
768
- // Deserialize using old field types into a plain record
769
- const record: Record<string, any> = {};
770
- for (const [name] of this._fieldTypes.entries()) record[name] = (model as any)[name]; // pk fields
771
- for (const [name, type] of versionInfo.nonKeyFields.entries()) {
772
- record[name] = type.deserialize(valuePack);
773
- }
774
-
775
- // Run migrate() if it exists
776
- const migrateFn = (this._MyModel as any).migrate;
777
- if (migrateFn) migrateFn(record);
778
-
779
- // Set non-key fields on model from the (possibly migrated) record
780
- for (const fieldName of this._nonKeyFields) {
781
- if (fieldName in record) {
782
- model._setLoadedField(fieldName, record[fieldName]);
783
- } else if (fieldName in model) {
784
- // Instantiate the default value
785
- model._setLoadedField(fieldName, (model as any)[fieldName]);
786
- } else {
787
- throw new DatabaseError(`Field ${fieldName} is missing in migrated data for ${model}`, 'MIGRATION_ERROR');
788
- }
789
- }
790
- }
791
-
792
- _keyToArray(key: Uint8Array): IndexArgTypes<M, F> {
474
+ _pkToArray(key: Uint8Array): ARGS {
793
475
  const bytes = new DataPack(key);
794
476
  assert(bytes.readNumber() === this._indexId);
795
477
  const result = [] as any[];
796
- for (const fieldType of this._fieldTypes.values()) {
478
+ for (const fieldType of this._indexFields.values()) {
797
479
  result.push(fieldType.deserialize(bytes));
798
480
  }
799
- return result as any;
800
- }
801
-
802
- _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): InstanceType<M> {
803
- return this._get(txn, new Uint8Array(keyBuffer), new Uint8Array(valueBuffer));
804
- }
805
-
806
- _getTypeName(): string {
807
- return 'primary';
481
+ return result as unknown as ARGS;
808
482
  }
809
483
 
810
- _write(txn: Transaction, primaryKey: Uint8Array, data: Record<string, any>) {
811
- let valueBytes = new DataPack();
812
- valueBytes.write(this._currentVersion);
813
- const fieldConfigs = this._MyModel.fields as any;
814
- for (const fieldName of this._nonKeyFields) {
815
- const fieldConfig = fieldConfigs[fieldName] as FieldConfig<unknown>;
816
- fieldConfig.type.serialize(data[fieldName], valueBytes);
817
- }
484
+ _writePK(txn: Transaction, primaryKey: Uint8Array, data: Record<string, any>) {
485
+ const valueBytes = this._serializeValue(data);
818
486
  if (logLevel >= 2) {
819
- console.log(`[edinburgh] Write ${this} key=${new DataPack(primaryKey)} value=${valueBytes}`);
487
+ console.log(`[edinburgh] Write ${this} key=${new DataPack(primaryKey)} value=${new DataPack(valueBytes)}`);
820
488
  }
821
- dbPut(txn.id, primaryKey, valueBytes.toUint8Array());
489
+ dbPut(txn.id, primaryKey, valueBytes);
822
490
  }
823
491
 
824
- _delete(txn: Transaction, primaryKey: Uint8Array, _data: Record<string, any>) {
492
+ _deletePK(txn: Transaction, primaryKey: Uint8Array, _data: Record<string, any>) {
825
493
  if (logLevel >= 2) {
826
494
  console.log(`[edinburgh] Delete ${this} key=${new DataPack(primaryKey)}`);
827
495
  }
@@ -829,38 +497,33 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
829
497
  }
830
498
  }
831
499
 
832
- // OLMDB does not support storing empty values, so we use a single byte value for secondary indexes.
500
+ function toArray<ARG_TYPES extends readonly any[]>(args: ArrayOrOnlyItem<ARG_TYPES>): Partial<ARG_TYPES> {
501
+ return (Array.isArray(args) ? args : [args]) as Partial<ARG_TYPES>;
502
+ }
503
+
833
504
  const SECONDARY_VALUE = new DataPack().write(undefined).toUint8Array();
834
505
 
835
- /**
836
- * Abstract base for all non-primary indexes (unique and secondary).
837
- * Provides shared key serialization, write/delete/update logic.
838
- */
839
- export abstract class NonPrimaryIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[], ARGS extends readonly any[] = IndexArgTypes<M, F>> extends BaseIndex<M, F, ARGS> {
506
+ export abstract class NonPrimaryIndex<ITEM extends IndexItem, const F extends readonly (keyof ITEM & string)[], ARGS extends readonly any[] = IndexArgTypes<ITEM, F>> extends BaseIndex<ITEM, F, ARGS> {
840
507
  _resetIndexFieldDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
841
508
 
842
- constructor(MyModel: M, fieldsOrFn: F | ((data: any) => any[])) {
843
- super(MyModel, typeof fieldsOrFn === 'function' ? [] as any : fieldsOrFn);
509
+ constructor(tableName: string, fieldsOrFn: F | ((data: any) => any[]), protected _loadPrimary: LoadPrimary<ITEM>, queueInitialization: QueueInitialization) {
510
+ super(tableName, typeof fieldsOrFn === 'function' ? [] as any : fieldsOrFn);
844
511
  if (typeof fieldsOrFn === 'function') this._computeFn = fieldsOrFn;
845
- (this._MyModel._secondaries ||= []).push(this);
846
- scheduleInit();
512
+ queueInitialization();
847
513
  }
848
514
 
849
- async _delayedInit() {
850
- await super._delayedInit();
851
- for (const fieldName of this._fieldTypes.keys()) {
515
+ async _initializeIndex(fields: FieldTypes, reset = false, primaryFieldTypes?: FieldTypes) {
516
+ if (reset) this._resetIndexFieldDescriptors = {};
517
+ await super._initializeIndex(fields, reset, primaryFieldTypes);
518
+ for (const fieldName of this._indexFields.keys()) {
852
519
  this._resetIndexFieldDescriptors[fieldName] = {
853
520
  writable: true,
854
521
  configurable: true,
855
- enumerable: true
522
+ enumerable: true,
856
523
  };
857
524
  }
858
525
  }
859
526
 
860
- /**
861
- * Build DataPack key prefixes (indexId + field/computed values). Returns [] to skip indexing.
862
- * SecondaryIndex appends the primary key to each pack before converting to Uint8Array.
863
- */
864
527
  _buildKeyPacks(data: Record<string, any>): DataPack[] {
865
528
  if (this._computeFn) {
866
529
  return this._computeFn(data).map((value: any) => {
@@ -870,48 +533,41 @@ export abstract class NonPrimaryIndex<M extends typeof Model, const F extends re
870
533
  return bytes;
871
534
  });
872
535
  }
873
- for (const fieldName of this._fieldTypes.keys()) {
536
+ for (const fieldName of this._indexFields.keys()) {
874
537
  if (data[fieldName] == null) return [];
875
538
  }
876
539
  const bytes = new DataPack();
877
540
  bytes.write(this._indexId!);
878
- for (const [fieldName, fieldType] of this._fieldTypes.entries()) {
541
+ for (const [fieldName, fieldType] of this._indexFields.entries()) {
879
542
  fieldType.serialize(data[fieldName], bytes);
880
543
  }
881
544
  return [bytes];
882
545
  }
883
546
 
884
- /** Serialize all index keys. Default: key = indexId + fields. */
885
547
  _serializeKeys(primaryKey: Uint8Array, data: Record<string, any>): Uint8Array[] {
886
- return this._buildKeyPacks(data).map(p => p.toUint8Array());
548
+ return this._buildKeyPacks(data).map(pack => pack.toUint8Array());
887
549
  }
888
550
 
889
- /** Write a single pre-serialized key — subclasses define value and uniqueness check. */
890
551
  abstract _writeKey(txn: Transaction, key: Uint8Array, primaryKey: Uint8Array): void;
891
552
 
892
- _write(txn: Transaction, primaryKey: Uint8Array, model: InstanceType<M>): void {
553
+ _write(txn: Transaction, primaryKey: Uint8Array, model: ITEM): void {
893
554
  for (const key of this._serializeKeys(primaryKey, model as any)) {
894
555
  if (logLevel >= 2) console.log(`[edinburgh] Write ${this} key=${new DataPack(key)}`);
895
556
  this._writeKey(txn, key, primaryKey);
896
557
  }
897
558
  }
898
559
 
899
- _delete(txn: Transaction, primaryKey: Uint8Array, model: InstanceType<M>): void {
560
+ _delete(txn: Transaction, primaryKey: Uint8Array, model: ITEM): void {
900
561
  for (const key of this._serializeKeys(primaryKey, model as any)) {
901
562
  if (logLevel >= 2) console.log(`[edinburgh] Delete ${this} key=${new DataPack(key)}`);
902
563
  dbDel(txn.id, key);
903
564
  }
904
565
  }
905
566
 
906
- /**
907
- * Granular update: diff old vs new keys and only insert/delete what changed.
908
- * For non-computed indexes, uses a fast path that checks which fields changed.
909
- */
910
- _update(txn: Transaction, primaryKey: Uint8Array, newData: InstanceType<M>, oldData: Record<string, any>): number {
567
+ _update(txn: Transaction, primaryKey: Uint8Array, newData: ITEM, oldData: Record<string, any>): number {
911
568
  const oldKeys = this._serializeKeys(primaryKey, oldData);
912
569
  const newKeys = this._serializeKeys(primaryKey, newData as any);
913
570
 
914
- // Fast path: no changes and max 1 key
915
571
  if (oldKeys.length === newKeys.length && (oldKeys.length === 0 || bytesEqual(oldKeys[0], newKeys[0]))) {
916
572
  return 0;
917
573
  }
@@ -939,23 +595,26 @@ export abstract class NonPrimaryIndex<M extends typeof Model, const F extends re
939
595
  }
940
596
  }
941
597
 
942
- /**
943
- * Unique index that stores references to the primary key.
944
- */
945
- export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[], ARGS extends readonly any[] = IndexArgTypes<M, F>> extends NonPrimaryIndex<M, F, ARGS> {
598
+ export class UniqueIndex<ITEM extends IndexItem, const F extends readonly (keyof ITEM & string)[], ARGS extends readonly any[] = IndexArgTypes<ITEM, F>> extends NonPrimaryIndex<ITEM, F, ARGS> {
599
+ constructor(tableName: string, fieldsOrFn: F | ((data: any) => any[]), loadPrimary: LoadPrimary<ITEM>, queueInitialization: QueueInitialization) {
600
+ super(tableName, fieldsOrFn, loadPrimary, queueInitialization);
601
+ }
946
602
 
947
- get(...args: ARGS): InstanceType<M> | undefined {
603
+ _getTypeName(): string {
604
+ return this._computeFn ? 'fn-unique' : 'unique';
605
+ }
606
+
607
+ getPK(...args: ARGS): ITEM | undefined {
948
608
  const txn = currentTxn();
949
- let keyBuffer = this._argsToKeyBytes(args, false).toUint8Array();
609
+ const keyBuffer = this._argsToKeyBytes(args, false).toUint8Array();
950
610
 
951
- let valueBuffer = dbGet(txn.id, keyBuffer);
611
+ const valueBuffer = dbGet(txn.id, keyBuffer);
952
612
  if (logLevel >= 3) {
953
613
  console.log(`[edinburgh] Get ${this} key=${new DataPack(keyBuffer)} result=${valueBuffer}`);
954
614
  }
955
615
  if (!valueBuffer) return;
956
616
 
957
- const pk = this._MyModel._primary!;
958
- const result = pk._get(txn, valueBuffer, true);
617
+ const result = this._loadPrimary(txn, valueBuffer, true);
959
618
  if (!result) throw new DatabaseError(`Unique index ${this} points at non-existing primary for key: ${args.join(', ')}`, 'CONSISTENCY_ERROR');
960
619
  return result;
961
620
  }
@@ -965,45 +624,46 @@ export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof
965
624
  dbPut(txn.id, key, primaryKey);
966
625
  }
967
626
 
968
- _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): InstanceType<M> {
969
- const pk = this._MyModel._primary!;
970
- const model = pk._get(txn, new Uint8Array(valueBuffer), false);
627
+ _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): ITEM {
628
+ const model = this._loadPrimary(txn, new Uint8Array(valueBuffer), false)!;
971
629
 
972
- if (this._fieldTypes.size > 0) {
630
+ if (this._indexFields.size > 0) {
973
631
  const keyPack = new DataPack(new Uint8Array(keyBuffer));
974
- keyPack.readNumber(); // discard index id
632
+ keyPack.readNumber();
975
633
 
976
634
  Object.defineProperties(model, this._resetIndexFieldDescriptors);
977
- for(const [name, fieldType] of this._fieldTypes.entries()) {
635
+ for (const [name, fieldType] of this._indexFields.entries()) {
978
636
  model._setLoadedField(name, fieldType.deserialize(keyPack));
979
637
  }
980
638
  }
981
639
 
640
+ model._restoreLazyFields?.();
641
+
982
642
  return model;
983
643
  }
644
+ }
984
645
 
985
- _getTypeName(): string {
986
- return this._computeFn ? 'fn-unique' : 'unique';
646
+ export class SecondaryIndex<ITEM extends IndexItem, const F extends readonly (keyof ITEM & string)[], ARGS extends readonly any[] = IndexArgTypes<ITEM, F>> extends NonPrimaryIndex<ITEM, F, ARGS> {
647
+ constructor(tableName: string, fieldsOrFn: F | ((data: any) => any[]), loadPrimary: LoadPrimary<ITEM>, queueInitialization: QueueInitialization) {
648
+ super(tableName, fieldsOrFn, loadPrimary, queueInitialization);
987
649
  }
988
- }
989
650
 
990
- /**
991
- * Secondary index for non-unique lookups.
992
- */
993
- export class SecondaryIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[], ARGS extends readonly any[] = IndexArgTypes<M, F>> extends NonPrimaryIndex<M, F, ARGS> {
651
+ _getTypeName(): string {
652
+ return this._computeFn ? 'fn-secondary' : 'secondary';
653
+ }
994
654
 
995
- _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, _valueBuffer: ArrayBuffer): InstanceType<M> {
655
+ _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, _valueBuffer: ArrayBuffer): ITEM {
996
656
  const keyPack = new DataPack(new Uint8Array(keyBuffer));
997
- keyPack.readNumber(); // discard index id
998
-
999
- const indexFields = new Map();
1000
- for (const [name, type] of this._fieldTypes.entries()) {
1001
- indexFields.set(name, type.deserialize(keyPack));
657
+ keyPack.readNumber();
658
+
659
+ const indexFields = new Map<string, any>();
660
+ for (const [name, fieldType] of this._indexFields.entries()) {
661
+ indexFields.set(name, fieldType.deserialize(keyPack));
1002
662
  }
1003
- if (this._computeFn) keyPack.read(); // skip computed value
663
+ if (this._computeFn) keyPack.read();
1004
664
 
1005
665
  const primaryKey = keyPack.readUint8Array();
1006
- const model = this._MyModel._primary!._get(txn, primaryKey, false);
666
+ const model = this._loadPrimary(txn, primaryKey, false)!;
1007
667
 
1008
668
  if (indexFields.size > 0) {
1009
669
  Object.defineProperties(model, this._resetIndexFieldDescriptors);
@@ -1012,94 +672,86 @@ export class SecondaryIndex<M extends typeof Model, const F extends readonly (ke
1012
672
  }
1013
673
  }
1014
674
 
675
+ model._restoreLazyFields?.();
676
+
1015
677
  return model;
1016
678
  }
1017
679
 
1018
680
  _serializeKeys(primaryKey: Uint8Array, data: Record<string, any>): Uint8Array[] {
1019
- return this._buildKeyPacks(data).map(p => { p.write(primaryKey); return p.toUint8Array(); });
681
+ return this._buildKeyPacks(data).map(pack => {
682
+ pack.write(primaryKey);
683
+ return pack.toUint8Array();
684
+ });
1020
685
  }
1021
686
 
1022
687
  _writeKey(txn: Transaction, key: Uint8Array, _primaryKey: Uint8Array): void {
1023
688
  dbPut(txn.id, key, SECONDARY_VALUE);
1024
689
  }
1025
-
1026
- _getTypeName(): string {
1027
- return this._computeFn ? 'fn-secondary' : 'secondary';
1028
- }
1029
690
  }
1030
691
 
1031
- /**
1032
- * Dump database contents for debugging.
1033
- *
1034
- * Prints all indexes and their data to the console for inspection.
1035
- * This is primarily useful for development and debugging purposes.
1036
- */
1037
692
  export function dump() {
1038
693
  const txn = currentTxn();
1039
- let indexesById = new Map<number, {name: string, type: string, fields: Record<string, TypeWrapper<any>>}>();
1040
- let versions = new Map<number, Map<number, Map<string, TypeWrapper<any>>>>();
1041
- console.log("--- edinburgh database dump ---")
694
+ const indexesById = new Map<number, {name: string, type: string, fields: Record<string, TypeWrapper<any>>}>();
695
+ const versions = new Map<number, Map<number, Map<string, TypeWrapper<any>>>>();
696
+ console.log("--- edinburgh database dump ---");
1042
697
  const iteratorId = lowlevel.createIterator(txn.id, undefined, undefined, false);
1043
698
  try {
1044
- while (true) {
1045
- const raw = lowlevel.readIterator(iteratorId);
1046
- if (!raw) break;
1047
- const kb = new DataPack(new Uint8Array(raw.key));
1048
- const vb = new DataPack(new Uint8Array(raw.value));
1049
- const indexId = kb.readNumber();
1050
- if (indexId === MAX_INDEX_ID_PREFIX) {
1051
- console.log("* Max index id", vb.readNumber());
1052
- } else if (indexId === VERSION_INFO_PREFIX) {
1053
- const idxId = kb.readNumber();
1054
- const version = kb.readNumber();
1055
- const obj = vb.read() as any;
1056
- const nonKeyFields = new Map<string, TypeWrapper<any>>();
1057
- for (const [name, typeBytes] of obj.fields) {
1058
- nonKeyFields.set(name, deserializeType(new DataPack(typeBytes), 0));
1059
- }
1060
- if (!versions.has(idxId)) versions.set(idxId, new Map());
1061
- versions.get(idxId)!.set(version, nonKeyFields);
1062
- console.log(`* Version ${version} for index ${idxId}: fields=[${[...nonKeyFields.keys()].join(',')}]`);
1063
- } else if (indexId === INDEX_ID_PREFIX) {
1064
- const name = kb.readString();
1065
- const type = kb.readString();
1066
- const fields: Record<string, TypeWrapper<any>> = {};
1067
- while(kb.readAvailable()) {
1068
- const name = kb.read();
1069
- if (typeof name !== 'string') break; // undefined separator or computed hash
1070
- fields[name] = deserializeType(kb, 0);
1071
- }
699
+ while (true) {
700
+ const raw = lowlevel.readIterator(iteratorId);
701
+ if (!raw) break;
702
+ const kb = new DataPack(new Uint8Array(raw.key));
703
+ const vb = new DataPack(new Uint8Array(raw.value));
704
+ const indexId = kb.readNumber();
705
+ if (indexId === MAX_INDEX_ID_PREFIX) {
706
+ console.log("* Max index id", vb.readNumber());
707
+ } else if (indexId === VERSION_INFO_PREFIX) {
708
+ const idxId = kb.readNumber();
709
+ const version = kb.readNumber();
710
+ const obj = vb.read() as any;
711
+ const nonKeyFields = new Map<string, TypeWrapper<any>>();
712
+ for (const [name, typeBytes] of obj.fields) {
713
+ nonKeyFields.set(name, deserializeType(new DataPack(typeBytes), 0));
714
+ }
715
+ if (!versions.has(idxId)) versions.set(idxId, new Map());
716
+ versions.get(idxId)!.set(version, nonKeyFields);
717
+ console.log(`* Version ${version} for index ${idxId}: fields=[${[...nonKeyFields.keys()].join(',')}]`);
718
+ } else if (indexId === INDEX_ID_PREFIX) {
719
+ const name = kb.readString();
720
+ const type = kb.readString();
721
+ const fields: Record<string, TypeWrapper<any>> = {};
722
+ while (kb.readAvailable()) {
723
+ const fieldName = kb.read();
724
+ if (typeof fieldName !== 'string') break;
725
+ fields[fieldName] = deserializeType(kb, 0);
726
+ }
1072
727
 
1073
- const indexId = vb.readNumber();
1074
- console.log(`* Index definition ${indexId}:${name}:${type}[${Object.keys(fields).join(',')}]`);
1075
- indexesById.set(indexId, {name, type, fields});
1076
- } else if (indexId > 0 && indexesById.has(indexId)) {
1077
- const index = indexesById.get(indexId)!;
1078
- let name, type, rowKey: any, rowValue: any;
1079
- if (index) {
1080
- name = index.name;
1081
- type = index.type;
1082
- const fields = index.fields;
1083
- rowKey = {};
1084
- for(const [fieldName, fieldType] of Object.entries(fields)) {
728
+ const definedIndexId = vb.readNumber();
729
+ console.log(`* Index definition ${definedIndexId}:${name}:${type}[${Object.keys(fields).join(',')}]`);
730
+ indexesById.set(definedIndexId, {name, type, fields});
731
+ } else if (indexId > 0 && indexesById.has(indexId)) {
732
+ const index = indexesById.get(indexId)!;
733
+ let rowKey: any = {};
734
+ let rowValue: any;
735
+ for (const [fieldName, fieldType] of Object.entries(index.fields)) {
1085
736
  rowKey[fieldName] = fieldType.deserialize(kb);
1086
737
  }
1087
- if (type === 'primary') {
738
+ if (index.type === 'primary') {
1088
739
  const version = vb.readNumber();
1089
- const vFields = versions.get(indexId)?.get(version);
1090
- if (vFields) {
740
+ const valueFields = versions.get(indexId)?.get(version);
741
+ if (valueFields) {
1091
742
  rowValue = {};
1092
- for (const [fieldName, fieldType] of vFields) {
743
+ for (const [fieldName, fieldType] of valueFields) {
1093
744
  rowValue[fieldName] = fieldType.deserialize(vb);
1094
745
  }
1095
746
  }
1096
747
  }
748
+ console.log(`* Row for ${indexId}:${index.name}:${index.type}`, rowKey ?? kb, rowValue ?? vb);
749
+ } else {
750
+ console.log(`* Unhandled '${indexId}' key=${kb} value=${vb}`);
1097
751
  }
1098
- console.log(`* Row for ${indexId}:${name}:${type}`, rowKey ?? kb, rowValue ?? vb);
1099
- } else {
1100
- console.log(`* Unhandled '${indexId}' key=${kb} value=${vb}`);
1101
752
  }
753
+ } finally {
754
+ lowlevel.closeIterator(iteratorId);
1102
755
  }
1103
- } finally { lowlevel.closeIterator(iteratorId); }
1104
- console.log("--- end ---")
756
+ console.log("--- end ---");
1105
757
  }