edinburgh 0.4.2 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +251 -170
- package/build/src/datapack.d.ts +17 -1
- package/build/src/datapack.js +44 -5
- package/build/src/datapack.js.map +1 -1
- package/build/src/edinburgh.d.ts +1 -1
- package/build/src/edinburgh.js +1 -1
- package/build/src/edinburgh.js.map +1 -1
- package/build/src/indexes.d.ts +34 -17
- package/build/src/indexes.js +126 -55
- package/build/src/indexes.js.map +1 -1
- package/build/src/migrate-cli.d.ts +1 -16
- package/build/src/migrate-cli.js +56 -42
- package/build/src/migrate-cli.js.map +1 -1
- package/build/src/migrate.d.ts +15 -11
- package/build/src/migrate.js +62 -32
- package/build/src/migrate.js.map +1 -1
- package/build/src/models.d.ts +1 -1
- package/build/src/models.js +20 -7
- package/build/src/models.js.map +1 -1
- package/build/src/types.d.ts +13 -1
- package/build/src/types.js +89 -1
- package/build/src/types.js.map +1 -1
- package/build/src/utils.d.ts +2 -0
- package/build/src/utils.js +12 -0
- package/build/src/utils.js.map +1 -1
- package/package.json +7 -4
- package/skill/BaseIndex.md +16 -0
- package/skill/BaseIndex_batchProcess.md +10 -0
- package/skill/BaseIndex_find.md +7 -0
- package/skill/DatabaseError.md +9 -0
- package/skill/Model.md +22 -0
- package/skill/Model_delete.md +14 -0
- package/skill/Model_findAll.md +12 -0
- package/skill/Model_getPrimaryKeyHash.md +5 -0
- package/skill/Model_isValid.md +14 -0
- package/skill/Model_migrate.md +34 -0
- package/skill/Model_preCommit.md +28 -0
- package/skill/Model_preventPersist.md +15 -0
- package/skill/Model_replaceInto.md +16 -0
- package/skill/Model_validate.md +21 -0
- package/skill/PrimaryIndex.md +8 -0
- package/skill/PrimaryIndex_get.md +17 -0
- package/skill/PrimaryIndex_getLazy.md +13 -0
- package/skill/SKILL.md +158 -664
- package/skill/SecondaryIndex.md +9 -0
- package/skill/UniqueIndex.md +9 -0
- package/skill/UniqueIndex_get.md +17 -0
- package/skill/array.md +23 -0
- package/skill/dump.md +8 -0
- package/skill/field.md +29 -0
- package/skill/index.md +32 -0
- package/skill/init.md +17 -0
- package/skill/link.md +27 -0
- package/skill/literal.md +22 -0
- package/skill/opt.md +22 -0
- package/skill/or.md +22 -0
- package/skill/primary.md +26 -0
- package/skill/record.md +21 -0
- package/skill/registerModel.md +26 -0
- package/skill/runMigration.md +10 -0
- package/skill/set.md +23 -0
- package/skill/setMaxRetryCount.md +10 -0
- package/skill/setOnSaveCallback.md +12 -0
- package/skill/transact.md +49 -0
- package/skill/unique.md +32 -0
- package/src/datapack.ts +49 -7
- package/src/edinburgh.ts +2 -0
- package/src/indexes.ts +143 -71
- package/src/migrate-cli.ts +44 -46
- package/src/migrate.ts +71 -39
- package/src/models.ts +19 -7
- package/src/types.ts +97 -1
- package/src/utils.ts +12 -0
package/src/indexes.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { DatabaseError } from "olmdb/lowlevel";
|
|
|
3
3
|
import DataPack from "./datapack.js";
|
|
4
4
|
import { FieldConfig, getMockModel, Model, Transaction, currentTxn } from "./models.js";
|
|
5
5
|
import { scheduleInit, transact } from "./edinburgh.js";
|
|
6
|
-
import { assert, logLevel, dbGet, dbPut, dbDel, hashBytes, toBuffer } from "./utils.js";
|
|
6
|
+
import { assert, logLevel, dbGet, dbPut, dbDel, hashBytes, hashFunction, bytesEqual, toBuffer } from "./utils.js";
|
|
7
7
|
import { deserializeType, serializeType, TypeWrapper } from "./types.js";
|
|
8
8
|
|
|
9
9
|
// Index system types and utilities
|
|
@@ -26,14 +26,6 @@ interface VersionInfo {
|
|
|
26
26
|
secondaryKeys: Set<string>;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
|
|
30
|
-
if (a.length !== b.length) return false;
|
|
31
|
-
for (let i = 0; i < a.length; i++) {
|
|
32
|
-
if (a[i] !== b[i]) return false;
|
|
33
|
-
}
|
|
34
|
-
return true;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
29
|
/**
|
|
38
30
|
* Iterator for range queries on indexes.
|
|
39
31
|
* Handles common iteration logic for both primary and unique indexes.
|
|
@@ -118,11 +110,12 @@ type FindOptions<ARG_TYPES extends readonly any[]> = (
|
|
|
118
110
|
* @template M - The model class this index belongs to.
|
|
119
111
|
* @template F - The field names that make up this index.
|
|
120
112
|
*/
|
|
121
|
-
export abstract class BaseIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[]
|
|
113
|
+
export abstract class BaseIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[], ARGS extends readonly any[] = IndexArgTypes<M, F>> {
|
|
122
114
|
public _MyModel: M;
|
|
123
115
|
public _fieldTypes: Map<keyof InstanceType<M> & string, TypeWrapper<any>> = new Map();
|
|
124
116
|
public _fieldCount!: number;
|
|
125
117
|
_resetIndexFieldDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
|
|
118
|
+
_computeFn?: (data: any) => any[];
|
|
126
119
|
|
|
127
120
|
/**
|
|
128
121
|
* Create a new index.
|
|
@@ -135,16 +128,24 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
|
|
|
135
128
|
|
|
136
129
|
async _delayedInit() {
|
|
137
130
|
if (this._indexId != null) return; // Already initialized
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
131
|
+
if (this._computeFn) {
|
|
132
|
+
this._fieldCount = 1;
|
|
133
|
+
} else {
|
|
134
|
+
for(const fieldName of this._fieldNames) {
|
|
135
|
+
assert(typeof fieldName === 'string', 'Field names must be strings');
|
|
136
|
+
this._fieldTypes.set(fieldName, this._MyModel.fields[fieldName].type);
|
|
137
|
+
}
|
|
138
|
+
this._fieldCount = this._fieldNames.length;
|
|
141
139
|
}
|
|
142
|
-
this._fieldCount = this._fieldNames.length;
|
|
143
140
|
await this._retrieveIndexId();
|
|
144
141
|
|
|
145
142
|
// Human-readable signature for version tracking, e.g. "secondary category:string"
|
|
146
|
-
|
|
147
|
-
|
|
143
|
+
if (this._computeFn) {
|
|
144
|
+
this._signature = this._getTypeName() + ' ' + hashFunction(this._computeFn);
|
|
145
|
+
} else {
|
|
146
|
+
this._signature = this._getTypeName() + ' ' +
|
|
147
|
+
Array.from(this._fieldTypes.entries()).map(([n, t]) => n + ':' + t).join(' ');
|
|
148
|
+
}
|
|
148
149
|
|
|
149
150
|
for(const fieldName of this._fieldTypes.keys()) {
|
|
150
151
|
this._resetIndexFieldDescriptors[fieldName] = {
|
|
@@ -167,17 +168,21 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
|
|
|
167
168
|
* @internal
|
|
168
169
|
*/
|
|
169
170
|
_argsToKeyBytes(args: [], allowPartial: boolean): DataPack;
|
|
170
|
-
_argsToKeyBytes(args: Partial<
|
|
171
|
+
_argsToKeyBytes(args: Partial<ARGS>, allowPartial: boolean): DataPack;
|
|
171
172
|
|
|
172
173
|
_argsToKeyBytes(args: any, allowPartial: boolean) {
|
|
173
174
|
assert(allowPartial ? args.length <= this._fieldCount : args.length === this._fieldCount);
|
|
174
175
|
const bytes = new DataPack();
|
|
175
176
|
bytes.write(this._indexId!);
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
fieldType.
|
|
177
|
+
if (this._computeFn) {
|
|
178
|
+
if (args.length > 0) bytes.write(args[0]);
|
|
179
|
+
} else {
|
|
180
|
+
let index = 0;
|
|
181
|
+
for(const fieldType of this._fieldTypes.values()) {
|
|
182
|
+
// For partial keys, undefined values are acceptable and represent open range suffixes
|
|
183
|
+
if (index >= args.length) break;
|
|
184
|
+
fieldType.serialize(args[index++], bytes);
|
|
185
|
+
}
|
|
181
186
|
}
|
|
182
187
|
return bytes;
|
|
183
188
|
}
|
|
@@ -192,6 +197,7 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
|
|
|
192
197
|
abstract _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): InstanceType<M>;
|
|
193
198
|
|
|
194
199
|
_hasNullIndexValues(data: Record<string, any>) {
|
|
200
|
+
assert(!this._computeFn);
|
|
195
201
|
for(const fieldName of this._fieldTypes.keys()) {
|
|
196
202
|
if (data[fieldName] == null) return true;
|
|
197
203
|
}
|
|
@@ -206,8 +212,12 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
|
|
|
206
212
|
_serializeKeyFields(data: Record<string, any>): DataPack {
|
|
207
213
|
const bytes = new DataPack();
|
|
208
214
|
bytes.write(this._indexId!);
|
|
209
|
-
|
|
210
|
-
|
|
215
|
+
if (this._computeFn) {
|
|
216
|
+
for (const v of this._computeFn(data)) bytes.write(v);
|
|
217
|
+
} else {
|
|
218
|
+
for(const [fieldName, fieldType] of this._fieldTypes.entries()) {
|
|
219
|
+
fieldType.serialize(data[fieldName], bytes);
|
|
220
|
+
}
|
|
211
221
|
}
|
|
212
222
|
return bytes;
|
|
213
223
|
}
|
|
@@ -218,9 +228,13 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
|
|
|
218
228
|
*/
|
|
219
229
|
async _retrieveIndexId(): Promise<void> {
|
|
220
230
|
const indexNameBytes = new DataPack().write(INDEX_ID_PREFIX).write(this._MyModel.tableName).write(this._getTypeName());
|
|
221
|
-
|
|
222
|
-
indexNameBytes.write(
|
|
223
|
-
|
|
231
|
+
if (this._computeFn) {
|
|
232
|
+
indexNameBytes.write(hashFunction(this._computeFn));
|
|
233
|
+
} else {
|
|
234
|
+
for(let name of this._fieldNames) {
|
|
235
|
+
indexNameBytes.write(name);
|
|
236
|
+
serializeType(this._MyModel.fields[name].type, indexNameBytes);
|
|
237
|
+
}
|
|
224
238
|
}
|
|
225
239
|
// For non-primary indexes, include primary key field info to avoid misinterpreting
|
|
226
240
|
// values when the primary key schema changes.
|
|
@@ -323,7 +337,7 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
|
|
|
323
337
|
* }
|
|
324
338
|
* ```
|
|
325
339
|
*/
|
|
326
|
-
_computeKeyBounds(opts: FindOptions<
|
|
340
|
+
_computeKeyBounds(opts: FindOptions<ARGS>): [DataPack | undefined, DataPack | undefined] | null {
|
|
327
341
|
let startKey: DataPack | undefined;
|
|
328
342
|
let endKey: DataPack | undefined;
|
|
329
343
|
if ('is' in opts) {
|
|
@@ -349,7 +363,7 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
|
|
|
349
363
|
return [startKey, endKey];
|
|
350
364
|
}
|
|
351
365
|
|
|
352
|
-
public find(opts: FindOptions<
|
|
366
|
+
public find(opts: FindOptions<ARGS> = {}): IndexRangeIterator<M> {
|
|
353
367
|
const txn = currentTxn();
|
|
354
368
|
const indexId = this._indexId!;
|
|
355
369
|
|
|
@@ -388,7 +402,7 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
|
|
|
388
402
|
* @param callback - Called for each matching row within a transaction
|
|
389
403
|
*/
|
|
390
404
|
public async batchProcess(
|
|
391
|
-
opts: FindOptions<
|
|
405
|
+
opts: FindOptions<ARGS> & { limitSeconds?: number; limitRows?: number } = {} as any,
|
|
392
406
|
callback: (row: InstanceType<M>) => void | Promise<void>
|
|
393
407
|
): Promise<void> {
|
|
394
408
|
const limitMs = (opts.limitSeconds ?? 1) * 1000;
|
|
@@ -460,7 +474,7 @@ function toArray<ARG_TYPES extends readonly any[]>(args: ArrayOrOnlyItem<ARG_TYP
|
|
|
460
474
|
* @template M - The model class this index belongs to.
|
|
461
475
|
* @template F - The field names that make up this index.
|
|
462
476
|
*/
|
|
463
|
-
export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[]> extends BaseIndex<M, F
|
|
477
|
+
export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[]> extends BaseIndex<M, F, IndexArgTypes<M, F>> {
|
|
464
478
|
|
|
465
479
|
_nonKeyFields!: (keyof InstanceType<M> & string)[];
|
|
466
480
|
_lazyDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
|
|
@@ -535,7 +549,7 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
|
|
|
535
549
|
async _initVersioning(): Promise<void> {
|
|
536
550
|
// Compute migrate hash from function source
|
|
537
551
|
const migrateFn = (this._MyModel as any)._original?.migrate ?? (this._MyModel as any).migrate;
|
|
538
|
-
this._currentMigrateHash = migrateFn ?
|
|
552
|
+
this._currentMigrateHash = migrateFn ? hashFunction(migrateFn) : 0;
|
|
539
553
|
|
|
540
554
|
const currentValueBytes = this._serializeVersionValue();
|
|
541
555
|
|
|
@@ -834,10 +848,11 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
|
|
|
834
848
|
* @template M - The model class this index belongs to.
|
|
835
849
|
* @template F - The field names that make up this index.
|
|
836
850
|
*/
|
|
837
|
-
export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[]
|
|
851
|
+
export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[], ARGS extends readonly any[] = IndexArgTypes<M, F>> extends BaseIndex<M, F, ARGS> {
|
|
838
852
|
|
|
839
|
-
constructor(MyModel: M,
|
|
840
|
-
super(MyModel,
|
|
853
|
+
constructor(MyModel: M, fieldsOrFn: F | ((data: any) => any[])) {
|
|
854
|
+
super(MyModel, typeof fieldsOrFn === 'function' ? [] as any : fieldsOrFn);
|
|
855
|
+
if (typeof fieldsOrFn === 'function') this._computeFn = fieldsOrFn;
|
|
841
856
|
(this._MyModel._secondaries ||= []).push(this);
|
|
842
857
|
scheduleInit();
|
|
843
858
|
}
|
|
@@ -852,7 +867,7 @@ export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof
|
|
|
852
867
|
* const userByEmail = User.byEmail.get("john@example.com");
|
|
853
868
|
* ```
|
|
854
869
|
*/
|
|
855
|
-
get(...args:
|
|
870
|
+
get(...args: ARGS): InstanceType<M> | undefined {
|
|
856
871
|
const txn = currentTxn();
|
|
857
872
|
let keyBuffer = this._argsToKeyBytes(args, false).toUint8Array();
|
|
858
873
|
|
|
@@ -873,6 +888,14 @@ export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof
|
|
|
873
888
|
}
|
|
874
889
|
|
|
875
890
|
_delete(txn: Transaction, primaryKey: Uint8Array, data: Record<string, any>) {
|
|
891
|
+
if (this._computeFn) {
|
|
892
|
+
for (const value of this._computeFn(data)) {
|
|
893
|
+
const key = new DataPack().write(this._indexId!).write(value).toUint8Array();
|
|
894
|
+
if (logLevel >= 2) console.log(`[edinburgh] Delete ${this} fn-key=${key}`);
|
|
895
|
+
dbDel(txn.id, key);
|
|
896
|
+
}
|
|
897
|
+
return;
|
|
898
|
+
}
|
|
876
899
|
if (!this._hasNullIndexValues(data)) {
|
|
877
900
|
const key = this._serializeKey(primaryKey, data);
|
|
878
901
|
if (logLevel >= 2) {
|
|
@@ -883,6 +906,15 @@ export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof
|
|
|
883
906
|
}
|
|
884
907
|
|
|
885
908
|
_write(txn: Transaction, primaryKey: Uint8Array, data: Record<string, any>) {
|
|
909
|
+
if (this._computeFn) {
|
|
910
|
+
for (const value of this._computeFn(data)) {
|
|
911
|
+
const key = new DataPack().write(this._indexId!).write(value).toUint8Array();
|
|
912
|
+
if (logLevel >= 2) console.log(`[edinburgh] Write ${this} fn-key=${key} value=${new DataPack(primaryKey)}`);
|
|
913
|
+
if (dbGet(txn.id, key)) throw new DatabaseError(`Unique constraint violation for ${this} key ${key}`, 'UNIQUE_CONSTRAINT');
|
|
914
|
+
dbPut(txn.id, key, primaryKey);
|
|
915
|
+
}
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
886
918
|
if (!this._hasNullIndexValues(data)) {
|
|
887
919
|
const key = this._serializeKey(primaryKey, data);
|
|
888
920
|
if (logLevel >= 2) {
|
|
@@ -898,26 +930,28 @@ export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof
|
|
|
898
930
|
_pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): InstanceType<M> {
|
|
899
931
|
// For unique indexes, the value contains the primary key
|
|
900
932
|
|
|
901
|
-
const keyPack = new DataPack(new Uint8Array(keyBuffer));
|
|
902
|
-
keyPack.readNumber(); // discard index id
|
|
903
|
-
|
|
904
933
|
const pk = this._MyModel._primary!;
|
|
905
934
|
const model = pk._get(txn, new Uint8Array(valueBuffer), false);
|
|
906
935
|
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
936
|
+
if (!this._computeFn) {
|
|
937
|
+
const keyPack = new DataPack(new Uint8Array(keyBuffer));
|
|
938
|
+
keyPack.readNumber(); // discard index id
|
|
910
939
|
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
940
|
+
// _get will have created lazy-load getters for our indexed fields. Let's turn them back into
|
|
941
|
+
// regular properties:
|
|
942
|
+
Object.defineProperties(model, this._resetIndexFieldDescriptors);
|
|
943
|
+
|
|
944
|
+
// Set the values for our indexed fields
|
|
945
|
+
for(const [name, fieldType] of this._fieldTypes.entries()) {
|
|
946
|
+
model._setLoadedField(name, fieldType.deserialize(keyPack));
|
|
947
|
+
}
|
|
914
948
|
}
|
|
915
949
|
|
|
916
950
|
return model;
|
|
917
951
|
}
|
|
918
952
|
|
|
919
953
|
_getTypeName(): string {
|
|
920
|
-
return 'unique';
|
|
954
|
+
return this._computeFn ? 'fn-unique' : 'unique';
|
|
921
955
|
}
|
|
922
956
|
}
|
|
923
957
|
|
|
@@ -930,10 +964,11 @@ const SECONDARY_VALUE = new DataPack().write(undefined).toUint8Array(); // Singl
|
|
|
930
964
|
* @template M - The model class this index belongs to.
|
|
931
965
|
* @template F - The field names that make up this index.
|
|
932
966
|
*/
|
|
933
|
-
export class SecondaryIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[]
|
|
967
|
+
export class SecondaryIndex<M extends typeof Model, const F extends readonly (keyof InstanceType<M> & string)[], ARGS extends readonly any[] = IndexArgTypes<M, F>> extends BaseIndex<M, F, ARGS> {
|
|
934
968
|
|
|
935
|
-
constructor(MyModel: M,
|
|
936
|
-
super(MyModel,
|
|
969
|
+
constructor(MyModel: M, fieldsOrFn: F | ((data: any) => any[])) {
|
|
970
|
+
super(MyModel, typeof fieldsOrFn === 'function' ? [] as any : fieldsOrFn);
|
|
971
|
+
if (typeof fieldsOrFn === 'function') this._computeFn = fieldsOrFn;
|
|
937
972
|
(this._MyModel._secondaries ||= []).push(this);
|
|
938
973
|
scheduleInit();
|
|
939
974
|
}
|
|
@@ -944,23 +979,28 @@ export class SecondaryIndex<M extends typeof Model, const F extends readonly (ke
|
|
|
944
979
|
const keyPack = new DataPack(new Uint8Array(keyBuffer));
|
|
945
980
|
keyPack.readNumber(); // discard index id
|
|
946
981
|
|
|
947
|
-
// Read the index fields
|
|
982
|
+
// Read the index fields (or skip computed value)
|
|
948
983
|
const indexFields = new Map();
|
|
949
|
-
|
|
950
|
-
|
|
984
|
+
if (this._computeFn) {
|
|
985
|
+
keyPack.read(); // skip computed value
|
|
986
|
+
} else {
|
|
987
|
+
for(const [name, type] of this._fieldTypes.entries()) {
|
|
988
|
+
indexFields.set(name, type.deserialize(keyPack));
|
|
989
|
+
}
|
|
951
990
|
}
|
|
952
991
|
|
|
953
992
|
const primaryKey = keyPack.readUint8Array();
|
|
954
993
|
const model = this._MyModel._primary!._get(txn, primaryKey, false);
|
|
955
994
|
|
|
995
|
+
if (indexFields.size > 0) {
|
|
996
|
+
// _get will have created lazy-load getters for our indexed fields. Let's turn them back into
|
|
997
|
+
// regular properties:
|
|
998
|
+
Object.defineProperties(model, this._resetIndexFieldDescriptors);
|
|
956
999
|
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
// Set the values for our indexed fields
|
|
962
|
-
for(const [name, value] of indexFields) {
|
|
963
|
-
model._setLoadedField(name, value);
|
|
1000
|
+
// Set the values for our indexed fields
|
|
1001
|
+
for(const [name, value] of indexFields) {
|
|
1002
|
+
model._setLoadedField(name, value);
|
|
1003
|
+
}
|
|
964
1004
|
}
|
|
965
1005
|
|
|
966
1006
|
return model;
|
|
@@ -974,6 +1014,14 @@ export class SecondaryIndex<M extends typeof Model, const F extends readonly (ke
|
|
|
974
1014
|
}
|
|
975
1015
|
|
|
976
1016
|
_write(txn: Transaction, primaryKey: Uint8Array, model: InstanceType<M>) {
|
|
1017
|
+
if (this._computeFn) {
|
|
1018
|
+
for (const value of this._computeFn(model)) {
|
|
1019
|
+
const key = new DataPack().write(this._indexId!).write(value).write(primaryKey).toUint8Array();
|
|
1020
|
+
if (logLevel >= 2) console.log(`[edinburgh] Write ${this} fn-key=${key}`);
|
|
1021
|
+
dbPut(txn.id, key, SECONDARY_VALUE);
|
|
1022
|
+
}
|
|
1023
|
+
return;
|
|
1024
|
+
}
|
|
977
1025
|
if (this._hasNullIndexValues(model)) return;
|
|
978
1026
|
const key = this._serializeKey(primaryKey, model);
|
|
979
1027
|
if (logLevel >= 2) {
|
|
@@ -983,6 +1031,14 @@ export class SecondaryIndex<M extends typeof Model, const F extends readonly (ke
|
|
|
983
1031
|
}
|
|
984
1032
|
|
|
985
1033
|
_delete(txn: Transaction, primaryKey: Uint8Array, model: InstanceType<M>): void {
|
|
1034
|
+
if (this._computeFn) {
|
|
1035
|
+
for (const value of this._computeFn(model)) {
|
|
1036
|
+
const key = new DataPack().write(this._indexId!).write(value).write(primaryKey).toUint8Array();
|
|
1037
|
+
if (logLevel >= 2) console.log(`[edinburgh] Delete ${this} fn-key=${key}`);
|
|
1038
|
+
dbDel(txn.id, key);
|
|
1039
|
+
}
|
|
1040
|
+
return;
|
|
1041
|
+
}
|
|
986
1042
|
if (this._hasNullIndexValues(model)) return;
|
|
987
1043
|
const key = this._serializeKey(primaryKey, model);
|
|
988
1044
|
if (logLevel >= 2) {
|
|
@@ -992,7 +1048,7 @@ export class SecondaryIndex<M extends typeof Model, const F extends readonly (ke
|
|
|
992
1048
|
}
|
|
993
1049
|
|
|
994
1050
|
_getTypeName(): string {
|
|
995
|
-
return 'secondary';
|
|
1051
|
+
return this._computeFn ? 'fn-secondary' : 'secondary';
|
|
996
1052
|
}
|
|
997
1053
|
}
|
|
998
1054
|
|
|
@@ -1026,13 +1082,19 @@ export function primary(MyModel: typeof Model, fields: any): PrimaryIndex<any, a
|
|
|
1026
1082
|
}
|
|
1027
1083
|
|
|
1028
1084
|
/**
|
|
1029
|
-
* Create a unique index on model fields.
|
|
1085
|
+
* Create a unique index on model fields, or a computed unique index using a function.
|
|
1086
|
+
*
|
|
1087
|
+
* For field-based indexes, pass a field name or array of field names.
|
|
1088
|
+
* For computed indexes, pass a function that takes a model instance and returns an array of
|
|
1089
|
+
* index keys. Return `[]` to skip indexing for that instance. Each array element creates a
|
|
1090
|
+
* separate index entry, enabling multi-value indexes (e.g., indexing by each word in a name).
|
|
1091
|
+
*
|
|
1030
1092
|
* @template M - The model class.
|
|
1093
|
+
* @template V - The computed index value type (for function-based indexes).
|
|
1031
1094
|
* @template F - The field name (for single field index).
|
|
1032
1095
|
* @template FS - The field names array (for composite index).
|
|
1033
1096
|
* @param MyModel - The model class to create the index for.
|
|
1034
|
-
* @param field -
|
|
1035
|
-
* @param fields - Array of field names for composite indexes.
|
|
1097
|
+
* @param field - Field name, array of field names, or a compute function.
|
|
1036
1098
|
* @returns A new UniqueIndex instance.
|
|
1037
1099
|
*
|
|
1038
1100
|
* @example
|
|
@@ -1040,24 +1102,32 @@ export function primary(MyModel: typeof Model, fields: any): PrimaryIndex<any, a
|
|
|
1040
1102
|
* class User extends E.Model<User> {
|
|
1041
1103
|
* static byEmail = E.unique(User, "email");
|
|
1042
1104
|
* static byNameAge = E.unique(User, ["name", "age"]);
|
|
1105
|
+
* static byFullName = E.unique(User, (u: User) => [`${u.firstName} ${u.lastName}`]);
|
|
1043
1106
|
* }
|
|
1044
1107
|
* ```
|
|
1045
1108
|
*/
|
|
1109
|
+
export function unique<M extends typeof Model, V>(MyModel: M, fn: (instance: InstanceType<M>) => V[]): UniqueIndex<M, [], [V]>;
|
|
1046
1110
|
export function unique<M extends typeof Model, const F extends (keyof InstanceType<M> & string)>(MyModel: M, field: F): UniqueIndex<M, [F]>;
|
|
1047
1111
|
export function unique<M extends typeof Model, const FS extends readonly (keyof InstanceType<M> & string)[]>(MyModel: M, fields: FS): UniqueIndex<M, FS>;
|
|
1048
1112
|
|
|
1049
|
-
export function unique(MyModel: typeof Model, fields: any): UniqueIndex<any, any> {
|
|
1050
|
-
return new UniqueIndex(MyModel,
|
|
1113
|
+
export function unique(MyModel: typeof Model, fields: any): UniqueIndex<any, any, any> {
|
|
1114
|
+
return new UniqueIndex(MyModel, typeof fields === 'string' ? [fields] : fields);
|
|
1051
1115
|
}
|
|
1052
1116
|
|
|
1053
1117
|
/**
|
|
1054
|
-
* Create a secondary index on model fields.
|
|
1118
|
+
* Create a secondary index on model fields, or a computed secondary index using a function.
|
|
1119
|
+
*
|
|
1120
|
+
* For field-based indexes, pass a field name or array of field names.
|
|
1121
|
+
* For computed indexes, pass a function that takes a model instance and returns an array of
|
|
1122
|
+
* index keys. Return `[]` to skip indexing for that instance. Each array element creates a
|
|
1123
|
+
* separate index entry, enabling multi-value indexes (e.g., indexing by each word in a name).
|
|
1124
|
+
*
|
|
1055
1125
|
* @template M - The model class.
|
|
1126
|
+
* @template V - The computed index value type (for function-based indexes).
|
|
1056
1127
|
* @template F - The field name (for single field index).
|
|
1057
1128
|
* @template FS - The field names array (for composite index).
|
|
1058
1129
|
* @param MyModel - The model class to create the index for.
|
|
1059
|
-
* @param field -
|
|
1060
|
-
* @param fields - Array of field names for composite indexes.
|
|
1130
|
+
* @param field - Field name, array of field names, or a compute function.
|
|
1061
1131
|
* @returns A new SecondaryIndex instance.
|
|
1062
1132
|
*
|
|
1063
1133
|
* @example
|
|
@@ -1065,14 +1135,16 @@ export function unique(MyModel: typeof Model, fields: any): UniqueIndex<any, any
|
|
|
1065
1135
|
* class User extends E.Model<User> {
|
|
1066
1136
|
* static byAge = E.index(User, "age");
|
|
1067
1137
|
* static byTagsDate = E.index(User, ["tags", "createdAt"]);
|
|
1138
|
+
* static byWord = E.index(User, (u: User) => u.name.split(" "));
|
|
1068
1139
|
* }
|
|
1069
1140
|
* ```
|
|
1070
1141
|
*/
|
|
1142
|
+
export function index<M extends typeof Model, V>(MyModel: M, fn: (instance: InstanceType<M>) => V[]): SecondaryIndex<M, [], [V]>;
|
|
1071
1143
|
export function index<M extends typeof Model, const F extends (keyof InstanceType<M> & string)>(MyModel: M, field: F): SecondaryIndex<M, [F]>;
|
|
1072
1144
|
export function index<M extends typeof Model, const FS extends readonly (keyof InstanceType<M> & string)[]>(MyModel: M, fields: FS): SecondaryIndex<M, FS>;
|
|
1073
1145
|
|
|
1074
|
-
export function index(MyModel: typeof Model, fields: any): SecondaryIndex<any, any> {
|
|
1075
|
-
return new SecondaryIndex(MyModel,
|
|
1146
|
+
export function index(MyModel: typeof Model, fields: any): SecondaryIndex<any, any, any> {
|
|
1147
|
+
return new SecondaryIndex(MyModel, typeof fields === 'string' ? [fields] : fields);
|
|
1076
1148
|
}
|
|
1077
1149
|
|
|
1078
1150
|
/**
|
|
@@ -1113,7 +1185,7 @@ export function dump() {
|
|
|
1113
1185
|
const fields: Record<string, TypeWrapper<any>> = {};
|
|
1114
1186
|
while(kb.readAvailable()) {
|
|
1115
1187
|
const name = kb.read();
|
|
1116
|
-
if (name
|
|
1188
|
+
if (typeof name !== 'string') break; // undefined separator or computed hash
|
|
1117
1189
|
fields[name] = deserializeType(kb, 0);
|
|
1118
1190
|
}
|
|
1119
1191
|
|
package/src/migrate-cli.ts
CHANGED
|
@@ -1,22 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* migrate-edinburgh
|
|
5
|
-
*
|
|
6
|
-
* Runs database migrations: upgrades all rows to the latest schema version,
|
|
7
|
-
* converts old primary indices, and cleans up orphaned secondary indices.
|
|
8
|
-
*
|
|
9
|
-
* Usage:
|
|
10
|
-
* npx migrate-edinburgh --import ./src/models.ts [options]
|
|
11
|
-
*
|
|
12
|
-
* Options:
|
|
13
|
-
* --import <path> Path to the module that registers all models (required)
|
|
14
|
-
* --db <path> Database directory (default: .edinburgh)
|
|
15
|
-
* --tables <names> Comma-separated list of table names to migrate
|
|
16
|
-
* --batch-size <n> Number of rows per transaction batch (default: 500)
|
|
17
|
-
* --no-convert Skip converting old primary indices
|
|
18
|
-
* --no-cleanup Skip deleting orphaned secondary indices
|
|
19
|
-
* --no-upgrade Skip upgrading rows to latest version
|
|
4
|
+
* See `npx migrate-edinburgh --help` for usage.
|
|
20
5
|
*/
|
|
21
6
|
|
|
22
7
|
import { runMigration, type MigrationOptions } from './migrate.js';
|
|
@@ -24,43 +9,49 @@ import { runMigration, type MigrationOptions } from './migrate.js';
|
|
|
24
9
|
function parseArgs(args: string[]): { importPath: string, options: MigrationOptions & { dbDir?: string } } {
|
|
25
10
|
let importPath = '';
|
|
26
11
|
const options: MigrationOptions & { dbDir?: string } = {};
|
|
12
|
+
const tables: string[] = [];
|
|
27
13
|
|
|
28
14
|
for (let i = 0; i < args.length; i++) {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
importPath = args[++i];
|
|
32
|
-
break;
|
|
15
|
+
const arg = args[i];
|
|
16
|
+
switch (arg) {
|
|
33
17
|
case '--db':
|
|
34
18
|
options.dbDir = args[++i];
|
|
35
19
|
break;
|
|
36
|
-
case '
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
case '
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
case '
|
|
43
|
-
|
|
44
|
-
break;
|
|
45
|
-
case '--no-upgrade':
|
|
46
|
-
options.upgradeVersions = false;
|
|
47
|
-
break;
|
|
20
|
+
case '+secondaries': options.populateSecondaries = true; break;
|
|
21
|
+
case '-secondaries': options.populateSecondaries = false; break;
|
|
22
|
+
case '+primaries': options.migratePrimaries = true; break;
|
|
23
|
+
case '-primaries': options.migratePrimaries = false; break;
|
|
24
|
+
case '+data': options.rewriteData = true; break;
|
|
25
|
+
case '-data': options.rewriteData = false; break;
|
|
26
|
+
case '+orphans': options.removeOrphans = true; break;
|
|
27
|
+
case '-orphans': options.removeOrphans = false; break;
|
|
48
28
|
default:
|
|
49
|
-
if (
|
|
50
|
-
console.error(`Unknown option: ${
|
|
29
|
+
if (arg.startsWith('-') || arg.startsWith('+')) {
|
|
30
|
+
console.error(`Unknown option: ${arg}`);
|
|
51
31
|
process.exit(1);
|
|
52
32
|
}
|
|
33
|
+
if (!importPath) {
|
|
34
|
+
importPath = arg;
|
|
35
|
+
} else {
|
|
36
|
+
tables.push(arg);
|
|
37
|
+
}
|
|
53
38
|
}
|
|
54
39
|
}
|
|
55
40
|
|
|
41
|
+
if (tables.length > 0) options.tables = tables;
|
|
42
|
+
|
|
56
43
|
if (!importPath) {
|
|
57
|
-
console.error('Usage: npx migrate-edinburgh
|
|
58
|
-
console.error('
|
|
59
|
-
console.error('
|
|
60
|
-
console.error('
|
|
61
|
-
console.error('
|
|
62
|
-
console.error('
|
|
63
|
-
console.error(' --
|
|
44
|
+
console.error('Usage: npx migrate-edinburgh <import_path> [<table> ...] [options]');
|
|
45
|
+
console.error('');
|
|
46
|
+
console.error(' <import_path> Module that registers all models (required)');
|
|
47
|
+
console.error(' <table> Table names to migrate (default: all)');
|
|
48
|
+
console.error('');
|
|
49
|
+
console.error('Options:');
|
|
50
|
+
console.error(' --db <path> Database directory (default: .edinburgh)');
|
|
51
|
+
console.error(' -secondaries Skip populating secondary indexes');
|
|
52
|
+
console.error(' -primaries Skip migrating old primary indexes');
|
|
53
|
+
console.error(' -orphans Skip removing orphaned index entries');
|
|
54
|
+
console.error(' +data Rewrite all row data to latest schema version');
|
|
64
55
|
process.exit(1);
|
|
65
56
|
}
|
|
66
57
|
|
|
@@ -99,14 +90,21 @@ async function main() {
|
|
|
99
90
|
|
|
100
91
|
// Report results
|
|
101
92
|
if (Object.keys(result.secondaries).length > 0) {
|
|
102
|
-
console.log('
|
|
93
|
+
console.log('Populated secondary indexes:');
|
|
103
94
|
for (const [table, count] of Object.entries(result.secondaries)) {
|
|
104
95
|
console.log(` ${table}: ${count}`);
|
|
105
96
|
}
|
|
106
97
|
}
|
|
107
98
|
|
|
99
|
+
if (Object.keys(result.rewritten).length > 0) {
|
|
100
|
+
console.log('Rewritten rows:');
|
|
101
|
+
for (const [table, count] of Object.entries(result.rewritten)) {
|
|
102
|
+
console.log(` ${table}: ${count}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
108
106
|
if (Object.keys(result.primaries).length > 0) {
|
|
109
|
-
console.log('
|
|
107
|
+
console.log('Migrated old primary rows:');
|
|
110
108
|
for (const [table, count] of Object.entries(result.primaries)) {
|
|
111
109
|
console.log(` ${table}: ${count}`);
|
|
112
110
|
}
|
|
@@ -121,11 +119,11 @@ async function main() {
|
|
|
121
119
|
}
|
|
122
120
|
}
|
|
123
121
|
|
|
124
|
-
if (result.
|
|
125
|
-
console.log(`Deleted ${result.
|
|
122
|
+
if (result.orphans > 0) {
|
|
123
|
+
console.log(`Deleted ${result.orphans} orphaned index entries`);
|
|
126
124
|
}
|
|
127
125
|
|
|
128
|
-
if (Object.keys(result.secondaries).length === 0 && Object.keys(result.primaries).length === 0 && result.
|
|
126
|
+
if (Object.keys(result.secondaries).length === 0 && Object.keys(result.primaries).length === 0 && Object.keys(result.rewritten).length === 0 && result.orphans === 0) {
|
|
129
127
|
console.log('No migration needed - database is up to date.');
|
|
130
128
|
}
|
|
131
129
|
|