@mauryasumit/driftdb 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -45
- package/dist/db.d.ts +19 -0
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +66 -13
- package/dist/db.js.map +1 -1
- package/dist/orm/repository.d.ts +3 -0
- package/dist/orm/repository.d.ts.map +1 -1
- package/dist/orm/repository.js +32 -6
- package/dist/orm/repository.js.map +1 -1
- package/dist/storage/s3-adapter.d.ts +4 -4
- package/dist/storage/s3-adapter.d.ts.map +1 -1
- package/dist/storage/s3-adapter.js +8 -8
- package/dist/storage/s3-adapter.js.map +1 -1
- package/dist/sync/engine.d.ts.map +1 -1
- package/dist/sync/engine.js +12 -9
- package/dist/sync/engine.js.map +1 -1
- package/dist/sync/snapshot-manager.d.ts +2 -1
- package/dist/sync/snapshot-manager.d.ts.map +1 -1
- package/dist/sync/snapshot-manager.js +6 -5
- package/dist/sync/snapshot-manager.js.map +1 -1
- package/dist/types.d.ts +3 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/db.ts +97 -36
- package/src/orm/repository.ts +39 -8
- package/src/storage/s3-adapter.ts +8 -8
- package/src/sync/engine.ts +44 -35
- package/src/sync/snapshot-manager.ts +34 -31
- package/src/types.ts +133 -130
package/dist/sync/engine.js
CHANGED
|
@@ -36,7 +36,7 @@ class SyncEngine {
|
|
|
36
36
|
compress: config.compression !== false,
|
|
37
37
|
encryptionKey: config.encryption?.key,
|
|
38
38
|
};
|
|
39
|
-
this.snapshotManager = new snapshot_manager_js_1.SnapshotManager(db, this.s3, nodeId, config.sqlitePath, uploadOptions);
|
|
39
|
+
this.snapshotManager = new snapshot_manager_js_1.SnapshotManager(db, this.s3, config.dbName, nodeId, config.sqlitePath, uploadOptions);
|
|
40
40
|
}
|
|
41
41
|
else {
|
|
42
42
|
this.s3 = null;
|
|
@@ -111,7 +111,7 @@ class SyncEngine {
|
|
|
111
111
|
if (pending.length === 0)
|
|
112
112
|
return;
|
|
113
113
|
const batch = this.changeLog.buildBatch(pending);
|
|
114
|
-
const s3Key = this.s3.logKey(this.nodeId, batch.fromSequence, batch.toSequence);
|
|
114
|
+
const s3Key = this.s3.logKey(this.config.dbName, this.nodeId, batch.fromSequence, batch.toSequence);
|
|
115
115
|
const alreadyQueued = this.queue.hasPendingOfType('upload_log');
|
|
116
116
|
if (!alreadyQueued) {
|
|
117
117
|
const payload = {
|
|
@@ -140,9 +140,10 @@ class SyncEngine {
|
|
|
140
140
|
const batchBuffer = Buffer.from(JSON.stringify(p.batch), 'utf8');
|
|
141
141
|
await this.s3.upload(p.s3Key, batchBuffer, uploadOptions);
|
|
142
142
|
this.changeLog.markSynced(p.fromSequence, p.toSequence);
|
|
143
|
-
const manifest = await this.s3.getManifest(this.
|
|
143
|
+
const manifest = await this.s3.getManifest(this.config.dbName);
|
|
144
144
|
const latestSeq = Math.max(manifest?.latestLogSequence ?? 0, p.toSequence);
|
|
145
|
-
await this.s3.putManifest(this.
|
|
145
|
+
await this.s3.putManifest(this.config.dbName, {
|
|
146
|
+
dbName: this.config.dbName,
|
|
146
147
|
nodeId: this.nodeId,
|
|
147
148
|
latestSnapshotKey: manifest?.latestSnapshotKey ?? null,
|
|
148
149
|
latestSnapshotTimestamp: manifest?.latestSnapshotTimestamp ?? null,
|
|
@@ -157,8 +158,9 @@ class SyncEngine {
|
|
|
157
158
|
const p = JSON.parse(job.payload);
|
|
158
159
|
if (this.snapshotManager) {
|
|
159
160
|
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
160
|
-
const manifest = await this.s3.getManifest(this.
|
|
161
|
-
await this.s3.putManifest(this.
|
|
161
|
+
const manifest = await this.s3.getManifest(this.config.dbName);
|
|
162
|
+
await this.s3.putManifest(this.config.dbName, {
|
|
163
|
+
dbName: this.config.dbName,
|
|
162
164
|
nodeId: this.nodeId,
|
|
163
165
|
latestSnapshotKey: key,
|
|
164
166
|
latestSnapshotTimestamp: timestamp,
|
|
@@ -184,7 +186,7 @@ class SyncEngine {
|
|
|
184
186
|
if (!this.queue.hasPendingOfType('upload_snapshot')) {
|
|
185
187
|
const payload = {
|
|
186
188
|
timestamp: Date.now(),
|
|
187
|
-
s3Key: this.s3.snapshotKey(this.nodeId, Date.now()),
|
|
189
|
+
s3Key: this.s3.snapshotKey(this.config.dbName, this.nodeId, Date.now()),
|
|
188
190
|
dbPath: this.config.sqlitePath,
|
|
189
191
|
};
|
|
190
192
|
this.queue.enqueue('upload_snapshot', payload);
|
|
@@ -195,8 +197,9 @@ class SyncEngine {
|
|
|
195
197
|
if (!this.snapshotManager || !this.s3)
|
|
196
198
|
return;
|
|
197
199
|
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
198
|
-
const manifest = await this.s3.getManifest(this.
|
|
199
|
-
await this.s3.putManifest(this.
|
|
200
|
+
const manifest = await this.s3.getManifest(this.config.dbName);
|
|
201
|
+
await this.s3.putManifest(this.config.dbName, {
|
|
202
|
+
dbName: this.config.dbName,
|
|
200
203
|
nodeId: this.nodeId,
|
|
201
204
|
latestSnapshotKey: key,
|
|
202
205
|
latestSnapshotTimestamp: timestamp,
|
package/dist/sync/engine.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../src/sync/engine.ts"],"names":[],"mappings":";;;AAEA,gDAA8C;AAC9C,mDAA4C;AAC5C,+DAAwD;AACxD,4DAAqD;AACrD,gDAA8C;AAC9C,2BAA0C;AAE1C,MAAM,wBAAwB,GAAG,IAAK,CAAC;AACvC,MAAM,6BAA6B,GAAG,IAAK,CAAC;AAC5C,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,oBAAoB,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,UAAU,EAAE,KAAM,EAAE,CAAC;AAErF,MAAa,UAAU;IAsBrB,YAAY,EAAqB,EAAE,MAAc,EAAE,MAAgB;QAb3D,UAAK,GAA0C,IAAI,CAAC;QACpD,iBAAY,GAAG,KAAK,CAAC;QAErB,YAAO,GAAgB;YAC7B,UAAU,EAAE,IAAI;YAChB,cAAc,EAAE,IAAI;YACpB,cAAc,EAAE,CAAC;YACjB,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,KAAK;SACjB,CAAC;QAGA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,WAAW,GAAG,EAAE,GAAG,oBAAoB,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QACvE,IAAI,CAAC,KAAK,GAAG,IAAI,oBAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,IAAI,yBAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAE3C,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,EAAE,GAAG,IAAI,yBAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,aAAa,GAAG;gBACpB,QAAQ,EAAE,MAAM,CAAC,WAAW,KAAK,KAAK;gBACtC,aAAa,EAAE,MAAM,CAAC,UAAU,EAAE,GAAG;aACtC,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,IAAI,qCAAe,CACxC,EAAE,EACF,IAAI,CAAC,EAAE,EACP,MAAM,EACN,MAAM,CAAC,UAAU,EACjB,aAAa,CACd,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;QACnB,CAAC,EAAE,UAAU,CAAC,CAAC;QAEf,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAED,UAAU;QACR,OAAO;YACL,GAAG,IAAI,CAAC,OAAO;YACf,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;YAC7C,WAAW,EAAE,IAAI,CAAC,cAAc,EAAE;SACnC,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,IAAA,eAAU,EAAC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,IAAA,aAAQ,EAAC,IAAI,CAAC,CAAC,IAAI,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAC1C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAEzB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,sBAAsB,CAAC;QACpE,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,EAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../src/sync/engine.ts"],"names":[],"mappings":";;;AAEA,gDAA8C;AAC9C,mDAA4C;AAC5C,+DAAwD;AACxD,4DAAqD;AACrD,gDAA8C;AAC9C,2BAA0C;AAE1C,MAAM,wBAAwB,GAAG,IAAK,CAAC;AACvC,MAAM,6BAA6B,GAAG,IAAK,CAAC;AAC5C,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,oBAAoB,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,UAAU,EAAE,KAAM,EAAE,CAAC;AAErF,MAAa,UAAU;IAsBrB,YAAY,EAAqB,EAAE,MAAc,EAAE,MAAgB;QAb3D,UAAK,GAA0C,IAAI,CAAC;QACpD,iBAAY,GAAG,KAAK,CAAC;QAErB,YAAO,GAAgB;YAC7B,UAAU,EAAE,IAAI;YAChB,cAAc,EAAE,IAAI;YACpB,cAAc,EAAE,CAAC;YACjB,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,KAAK;SACjB,CAAC;QAGA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,WAAW,GAAG,EAAE,GAAG,oBAAoB,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QACvE,IAAI,CAAC,KAAK,GAAG,IAAI,oBAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,IAAI,yBAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAE3C,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,EAAE,GAAG,IAAI,yBAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,aAAa,GAAG;gBACpB,QAAQ,EAAE,MAAM,CAAC,WAAW,KAAK,KAAK;gBACtC,aAAa,EAAE,MAAM,CAAC,UAAU,EAAE,GAAG;aACtC,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,IAAI,qCAAe,CACxC,EAAE,EACF,IAAI,CAAC,EAAE,EACP,MAAM,CAAC,MAAM,EACb,MAAM,EACN,MAAM,CAAC,UAAU,EACjB,aAAa,CACd,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;QACnB,CAAC,EAAE,UAAU,CAAC,CAAC;QAEf,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAED,UAAU;QACR,OAAO;YACL,GAAG,IAAI,CAAC,OAAO;YACf,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;YAC7C,WAAW,EAAE,IAAI,CAAC,cAAc,EAAE;SACnC,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,IAAA,eAAU,EAAC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,IAAA,aAAQ,EAAC,IAAI,CAAC,CAAC,IAAI,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAC1C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAEzB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,sBAAsB,CAAC;QACpE,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,EAAG,CAAC,MAAM,CAC3B,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB,IAAI,CAAC,MAAM,EACX,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,UAAU,CACjB,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;QAChE,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,OAAO,GAAqB;gBAChC,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,KAAK;aACN,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEnC,MAAM,OAAO,CAAC,UAAU,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CACxC,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,GAAY;QACnC,MAAM,WAAW,GAAG,EAAE,GAAG,oBAAoB,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC5E,MAAM,aAAa,GAAG;YACpB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,KAAK,KAAK;YAC3C,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG;SAC3C,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,IAAA,oBAAS,EACb,KAAK,IAAI,EAAE;gBACT,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAA0C,CAAC;oBAC3E,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;oBACjE,MAAM,IAAI,CAAC,EAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;oBAC3D,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;oBAExD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAG,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAChE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,QAAQ,EAAE,iBAAiB,IAAI,CAAC,EAChC,CAAC,CAAC,UAAU,CACb,CAAC;oBACF,MAAM,IAAI,CAAC,EAAG,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;wBAC7C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;wBAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,iBAAiB,EAAE,QAAQ,EAAE,iBAAiB,IAAI,IAAI;wBACtD,uBAAuB,EAAE,QAAQ,EAAE,uBAAuB,IAAI,IAAI;wBAClE,iBAAiB,EAAE,SAAS;wBAC5B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB,CAAC,CAAC;oBAEH,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;oBAChE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAErC,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;gBACtC,CAAC;qBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;oBAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAA0B,CAAC;oBAC3D,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;wBACzB,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;wBACtE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAG,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBAChE,MAAM,IAAI,CAAC,EAAG,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;4BAC7C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;4BAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,iBAAiB,EAAE,GAAG;4BACtB,uBAAuB,EAAE,SAAS;4BAClC,iBAAiB,EAAE,QAAQ,EAAE,iBAAiB,IAAI,CAAC;4BACnD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;yBACtB,CAAC,CAAC;wBACH,IAAI,CAAC,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC3C,CAAC;oBACD,KAAK,CAAC,CAAC;gBACT,CAAC;YACH,CAAC,EACD,WAAW,CACZ,CAAC;YAEF,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,cAAsB;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,6BAA6B,CAAC;QAClF,IAAI,cAAc,GAAG,CAAC,IAAI,cAAc,GAAG,SAAS,KAAK,CAAC,EAAE,CAAC;YAC3D,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACpD,MAAM,OAAO,GAA0B;oBACrC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,KAAK,EAAE,IAAI,CAAC,EAAG,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;oBACxE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;iBAC/B,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAC9C,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;QACtE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YAC5C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,iBAAiB,EAAE,GAAG;YACtB,uBAAuB,EAAE,SAAS;YAClC,iBAAiB,EAAE,QAAQ,EAAE,iBAAiB,IAAI,CAAC;YACnD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3C,CAAC;CACF;AAnPD,gCAmPC"}
|
|
@@ -4,10 +4,11 @@ import type { S3UploadOptions } from '../storage/s3-adapter.js';
|
|
|
4
4
|
export declare class SnapshotManager {
|
|
5
5
|
private readonly db;
|
|
6
6
|
private readonly s3;
|
|
7
|
+
private readonly dbName;
|
|
7
8
|
private readonly nodeId;
|
|
8
9
|
private readonly sqlitePath;
|
|
9
10
|
private readonly uploadOptions;
|
|
10
|
-
constructor(db: Database.Database, s3: S3Adapter, nodeId: string, sqlitePath: string, uploadOptions: S3UploadOptions);
|
|
11
|
+
constructor(db: Database.Database, s3: S3Adapter, dbName: string, nodeId: string, sqlitePath: string, uploadOptions: S3UploadOptions);
|
|
11
12
|
takeAndUpload(): Promise<{
|
|
12
13
|
key: string;
|
|
13
14
|
timestamp: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-manager.d.ts","sourceRoot":"","sources":["../../src/sync/snapshot-manager.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAEhE,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IACvC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkB;gBAG9C,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,EAAE,EAAE,SAAS,EACb,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,eAAe;
|
|
1
|
+
{"version":3,"file":"snapshot-manager.d.ts","sourceRoot":"","sources":["../../src/sync/snapshot-manager.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAEhE,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IACvC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkB;gBAG9C,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,EAAE,EAAE,SAAS,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,eAAe;IAU1B,aAAa,IAAI,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAiC5D,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC;CAiBxC"}
|
|
@@ -39,9 +39,10 @@ const path_1 = require("path");
|
|
|
39
39
|
const os_1 = require("os");
|
|
40
40
|
const fs_2 = require("fs");
|
|
41
41
|
class SnapshotManager {
|
|
42
|
-
constructor(db, s3, nodeId, sqlitePath, uploadOptions) {
|
|
42
|
+
constructor(db, s3, dbName, nodeId, sqlitePath, uploadOptions) {
|
|
43
43
|
this.db = db;
|
|
44
44
|
this.s3 = s3;
|
|
45
|
+
this.dbName = dbName;
|
|
45
46
|
this.nodeId = nodeId;
|
|
46
47
|
this.sqlitePath = sqlitePath;
|
|
47
48
|
this.uploadOptions = uploadOptions;
|
|
@@ -53,14 +54,14 @@ class SnapshotManager {
|
|
|
53
54
|
this.db.exec('PRAGMA wal_checkpoint(FULL)');
|
|
54
55
|
if (this.sqlitePath === ':memory:') {
|
|
55
56
|
const backup = this.db.serialize();
|
|
56
|
-
await this.s3.upload(this.s3.snapshotKey(this.nodeId, timestamp), Buffer.from(backup), this.uploadOptions);
|
|
57
|
+
await this.s3.upload(this.s3.snapshotKey(this.dbName, this.nodeId, timestamp), Buffer.from(backup), this.uploadOptions);
|
|
57
58
|
}
|
|
58
59
|
else {
|
|
59
60
|
(0, fs_2.copyFileSync)(this.sqlitePath, tempPath);
|
|
60
61
|
const data = (0, fs_1.readFileSync)(tempPath);
|
|
61
|
-
await this.s3.upload(this.s3.snapshotKey(this.nodeId, timestamp), data, this.uploadOptions);
|
|
62
|
+
await this.s3.upload(this.s3.snapshotKey(this.dbName, this.nodeId, timestamp), data, this.uploadOptions);
|
|
62
63
|
}
|
|
63
|
-
const key = this.s3.snapshotKey(this.nodeId, timestamp);
|
|
64
|
+
const key = this.s3.snapshotKey(this.dbName, this.nodeId, timestamp);
|
|
64
65
|
return { key, timestamp };
|
|
65
66
|
}
|
|
66
67
|
finally {
|
|
@@ -73,7 +74,7 @@ class SnapshotManager {
|
|
|
73
74
|
}
|
|
74
75
|
}
|
|
75
76
|
async restoreLatest() {
|
|
76
|
-
const manifest = await this.s3.getManifest(this.
|
|
77
|
+
const manifest = await this.s3.getManifest(this.dbName);
|
|
77
78
|
if (!manifest?.latestSnapshotKey)
|
|
78
79
|
return false;
|
|
79
80
|
const data = await this.s3.download(manifest.latestSnapshotKey, this.uploadOptions);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-manager.js","sourceRoot":"","sources":["../../src/sync/snapshot-manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2BAA8C;AAC9C,+BAAqC;AACrC,2BAA4B;AAC5B,2BAAyD;AAKzD,MAAa,eAAe;
|
|
1
|
+
{"version":3,"file":"snapshot-manager.js","sourceRoot":"","sources":["../../src/sync/snapshot-manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2BAA8C;AAC9C,+BAAqC;AACrC,2BAA4B;AAC5B,2BAAyD;AAKzD,MAAa,eAAe;IAQ1B,YACE,EAAqB,EACrB,EAAa,EACb,MAAc,EACd,MAAc,EACd,UAAkB,EAClB,aAA8B;QAE9B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,IAAA,WAAM,GAAE,EAAE,gBAAgB,IAAI,CAAC,MAAM,IAAI,SAAS,SAAS,CAAC,CAAC;QAEnF,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAE5C,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;gBACnC,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAClB,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EACxD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EACnB,IAAI,CAAC,aAAa,CACnB,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAA,iBAAY,EAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;gBACxC,MAAM,IAAI,GAAG,IAAA,iBAAY,EAAC,QAAQ,CAAC,CAAC;gBACpC,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAClB,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EACxD,IAAI,EACJ,IAAI,CAAC,aAAa,CACnB,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACrE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,IAAI,IAAA,eAAU,EAAC,QAAQ,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC;oBAAC,IAAA,eAAU,EAAC,QAAQ,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,iBAAiB;YAAE,OAAO,KAAK,CAAC;QAE/C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAEpF,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5E,IAAI,CAAC,WAAW;YAAE,OAAO,KAAK,CAAC;QAE/B,MAAM,GAAG,GAAG,IAAA,cAAO,EAAC,WAAW,CAAC,CAAC;QACjC,IAAA,cAAS,EAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpC,MAAM,EAAE,aAAa,EAAE,GAAG,wDAAa,IAAI,GAAC,CAAC;QAC7C,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAEjC,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AA1ED,0CA0EC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -48,6 +48,7 @@ export interface EncryptionConfig {
|
|
|
48
48
|
key: string;
|
|
49
49
|
}
|
|
50
50
|
export interface DBConfig {
|
|
51
|
+
dbName: string;
|
|
51
52
|
sqlitePath: string;
|
|
52
53
|
s3Config?: S3Config;
|
|
53
54
|
nodeId?: string;
|
|
@@ -58,6 +59,7 @@ export interface DBConfig {
|
|
|
58
59
|
encryption?: EncryptionConfig;
|
|
59
60
|
retryConfig?: RetryConfig;
|
|
60
61
|
autoSync?: boolean;
|
|
62
|
+
restoreFromS3?: boolean;
|
|
61
63
|
}
|
|
62
64
|
export interface ChangeLogEntry {
|
|
63
65
|
sequence: number;
|
|
@@ -92,6 +94,7 @@ export interface SyncJob {
|
|
|
92
94
|
error: string | null;
|
|
93
95
|
}
|
|
94
96
|
export interface SyncManifest {
|
|
97
|
+
dbName: string;
|
|
95
98
|
nodeId: string;
|
|
96
99
|
latestSnapshotKey: string | null;
|
|
97
100
|
latestSnapshotTimestamp: number | null;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAE1E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;IAC3C,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEpD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,UAAU,CAAC,CAAC,IACpB,CAAC,GACD;IAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,CAAA;CAAE,CAAC;AAEjF,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI;KAC1B,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CAClC,CAAC;AAEF,MAAM,WAAW,WAAW,CAAC,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,QAAQ;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAE1E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;IAC3C,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEpD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,UAAU,CAAC,CAAC,IACpB,CAAC,GACD;IAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,CAAA;CAAE,CAAC;AAEjF,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI;KAC1B,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CAClC,CAAC;AAEF,MAAM,WAAW,WAAW,CAAC,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC1C,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;CACf;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,CAAC,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,KAAK,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;QAC1C,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;KACtC,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,YAAY,GAAG,iBAAiB,CAAC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,GAAG,YAAY,GAAG,MAAM,GAAG,QAAQ,CAAC;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB"}
|
package/package.json
CHANGED
package/src/db.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import BetterSqlite3 from 'better-sqlite3';
|
|
2
2
|
import type Database from 'better-sqlite3';
|
|
3
|
-
import { mkdirSync, existsSync } from 'fs';
|
|
3
|
+
import { mkdirSync, existsSync, writeFileSync } from 'fs';
|
|
4
4
|
import { dirname } from 'path';
|
|
5
5
|
import type { DBConfig, ModelSchema, SyncMetrics } from './types.js';
|
|
6
6
|
import { Repository } from './orm/repository.js';
|
|
7
7
|
import { SyncEngine } from './sync/engine.js';
|
|
8
|
+
import { S3Adapter } from './storage/s3-adapter.js';
|
|
8
9
|
import { generateNodeId } from './utils/id.js';
|
|
9
10
|
import type { Model, ModelStatic } from './orm/model.js';
|
|
10
11
|
import type { BaseRecord } from './types.js';
|
|
@@ -16,24 +17,32 @@ const META_SCHEMA = `
|
|
|
16
17
|
);
|
|
17
18
|
`;
|
|
18
19
|
|
|
19
|
-
export class DB {
|
|
20
|
+
export class DB {
|
|
20
21
|
private readonly sqliteDb: Database.Database;
|
|
21
22
|
private readonly config: DBConfig;
|
|
22
23
|
private readonly nodeId: string;
|
|
23
24
|
private readonly syncEngine: SyncEngine;
|
|
24
25
|
private readonly repos = new Map<string, Repository<BaseRecord>>();
|
|
25
26
|
|
|
26
|
-
constructor(config: DBConfig) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
27
|
+
constructor(config: DBConfig) {
|
|
28
|
+
const dbName = config.dbName.trim();
|
|
29
|
+
if (!dbName) {
|
|
30
|
+
throw new Error('DBConfig.dbName is required');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
this.config = {
|
|
34
|
+
...config,
|
|
35
|
+
dbName,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
if (this.config.sqlitePath !== ':memory:') {
|
|
39
|
+
const dir = dirname(this.config.sqlitePath);
|
|
40
|
+
if (dir && dir !== '.') {
|
|
41
|
+
mkdirSync(dir, { recursive: true });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
this.sqliteDb = new BetterSqlite3(this.config.sqlitePath);
|
|
37
46
|
this.sqliteDb.pragma('journal_mode = WAL');
|
|
38
47
|
this.sqliteDb.pragma('synchronous = NORMAL');
|
|
39
48
|
this.sqliteDb.pragma('foreign_keys = ON');
|
|
@@ -42,35 +51,87 @@ export class DB {
|
|
|
42
51
|
|
|
43
52
|
this.sqliteDb.exec(META_SCHEMA);
|
|
44
53
|
|
|
45
|
-
this.nodeId = this.getOrCreateNodeId(config.nodeId);
|
|
46
|
-
this.syncEngine = new SyncEngine(this.sqliteDb, this.nodeId, config);
|
|
47
|
-
|
|
48
|
-
if (config.autoSync !== false && config.s3Config) {
|
|
49
|
-
this.syncEngine.start();
|
|
54
|
+
this.nodeId = this.getOrCreateNodeId(this.config.nodeId);
|
|
55
|
+
this.syncEngine = new SyncEngine(this.sqliteDb, this.nodeId, this.config);
|
|
56
|
+
|
|
57
|
+
if (this.config.autoSync !== false && this.config.s3Config) {
|
|
58
|
+
this.syncEngine.start();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Async factory — use this instead of `new DB()` when you need S3 restore on startup.
|
|
64
|
+
*
|
|
65
|
+
* - If `restoreFromS3: true` and the local SQLite file does not exist, it downloads
|
|
66
|
+
* the latest snapshot from S3 before opening the database.
|
|
67
|
+
* - `dbName` is required and namespaces each logical database in S3.
|
|
68
|
+
* - `nodeId` identifies the current local node within that logical database.
|
|
69
|
+
*
|
|
70
|
+
* @example
|
|
71
|
+
* const db = await DB.open({
|
|
72
|
+
* dbName: 'my-app-db',
|
|
73
|
+
* sqlitePath: './data/app.sqlite',
|
|
74
|
+
* nodeId: 'server-1',
|
|
75
|
+
* restoreFromS3: true, // auto-restore if local file is missing
|
|
76
|
+
* s3Config: { bucket: '...', region: '...' },
|
|
77
|
+
* });
|
|
78
|
+
*/
|
|
79
|
+
static async open(config: DBConfig): Promise<DB> {
|
|
80
|
+
if (
|
|
81
|
+
config.restoreFromS3 &&
|
|
82
|
+
config.s3Config &&
|
|
83
|
+
config.sqlitePath !== ':memory:' &&
|
|
84
|
+
!existsSync(config.sqlitePath)
|
|
85
|
+
) {
|
|
86
|
+
await DB.restoreSnapshot(config);
|
|
50
87
|
}
|
|
88
|
+
return new DB(config);
|
|
51
89
|
}
|
|
52
90
|
|
|
53
|
-
private
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
91
|
+
private static async restoreSnapshot(config: DBConfig): Promise<boolean> {
|
|
92
|
+
const s3 = new S3Adapter(config.s3Config!);
|
|
93
|
+
const uploadOptions = {
|
|
94
|
+
compress: config.compression !== false,
|
|
95
|
+
encryptionKey: config.encryption?.key,
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const manifest = await s3.getManifest(config.dbName);
|
|
99
|
+
if (!manifest?.latestSnapshotKey) {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const data = await s3.download(manifest.latestSnapshotKey, uploadOptions);
|
|
104
|
+
const dir = dirname(config.sqlitePath);
|
|
105
|
+
if (dir && dir !== '.') {
|
|
106
|
+
mkdirSync(dir, { recursive: true });
|
|
59
107
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
.prepare(`SELECT value FROM _driftdb_meta WHERE key = 'nodeId'`)
|
|
63
|
-
.get() as { value: string } | undefined;
|
|
64
|
-
|
|
65
|
-
if (row) return row.value;
|
|
66
|
-
|
|
67
|
-
const id = generateNodeId();
|
|
68
|
-
this.sqliteDb
|
|
69
|
-
.prepare(`INSERT INTO _driftdb_meta (key, value) VALUES ('nodeId', ?)`)
|
|
70
|
-
.run(id);
|
|
71
|
-
return id;
|
|
108
|
+
writeFileSync(config.sqlitePath, data);
|
|
109
|
+
return true;
|
|
72
110
|
}
|
|
73
111
|
|
|
112
|
+
private getOrCreateNodeId(preferred?: string): string {
|
|
113
|
+
const metaKey = `nodeId:${this.config.dbName}`;
|
|
114
|
+
|
|
115
|
+
if (preferred) {
|
|
116
|
+
this.sqliteDb
|
|
117
|
+
.prepare(`INSERT OR REPLACE INTO _driftdb_meta (key, value) VALUES (?, ?)`)
|
|
118
|
+
.run(metaKey, preferred);
|
|
119
|
+
return preferred;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const row = this.sqliteDb
|
|
123
|
+
.prepare(`SELECT value FROM _driftdb_meta WHERE key = ? OR key = 'nodeId' ORDER BY CASE WHEN key = ? THEN 0 ELSE 1 END LIMIT 1`)
|
|
124
|
+
.get(metaKey, metaKey) as { value: string } | undefined;
|
|
125
|
+
|
|
126
|
+
if (row) return row.value;
|
|
127
|
+
|
|
128
|
+
const id = generateNodeId();
|
|
129
|
+
this.sqliteDb
|
|
130
|
+
.prepare(`INSERT INTO _driftdb_meta (key, value) VALUES (?, ?)`)
|
|
131
|
+
.run(metaKey, id);
|
|
132
|
+
return id;
|
|
133
|
+
}
|
|
134
|
+
|
|
74
135
|
define<S extends ModelSchema>(
|
|
75
136
|
tableName: string,
|
|
76
137
|
schema: S
|
package/src/orm/repository.ts
CHANGED
|
@@ -17,11 +17,14 @@ export type RecordOf<S extends ModelSchema> = BaseRecord & {
|
|
|
17
17
|
: string;
|
|
18
18
|
};
|
|
19
19
|
|
|
20
|
+
type SqliteBindable = string | number | bigint | Buffer | null;
|
|
21
|
+
|
|
20
22
|
export class Repository<T extends BaseRecord> {
|
|
21
23
|
private readonly db: Database.Database;
|
|
22
24
|
private readonly tableName: string;
|
|
23
25
|
private readonly schema: ModelSchema;
|
|
24
26
|
private readonly changeLog: ChangeLog | null;
|
|
27
|
+
private readonly booleanColumns: Set<string>;
|
|
25
28
|
|
|
26
29
|
constructor(
|
|
27
30
|
db: Database.Database,
|
|
@@ -33,6 +36,11 @@ export class Repository<T extends BaseRecord> {
|
|
|
33
36
|
this.tableName = tableName;
|
|
34
37
|
this.schema = normalizeSchema(schema);
|
|
35
38
|
this.changeLog = changeLog;
|
|
39
|
+
this.booleanColumns = new Set(
|
|
40
|
+
Object.entries(this.schema)
|
|
41
|
+
.filter(([, def]) => def.type === 'BOOLEAN')
|
|
42
|
+
.map(([key]) => key)
|
|
43
|
+
);
|
|
36
44
|
this.initTable();
|
|
37
45
|
}
|
|
38
46
|
|
|
@@ -41,6 +49,28 @@ export class Repository<T extends BaseRecord> {
|
|
|
41
49
|
this.db.exec(sql);
|
|
42
50
|
}
|
|
43
51
|
|
|
52
|
+
private serialize(key: string, value: unknown): SqliteBindable {
|
|
53
|
+
if (value === undefined || value === null) return null;
|
|
54
|
+
if (this.booleanColumns.has(key) && typeof value === 'boolean') {
|
|
55
|
+
return value ? 1 : 0;
|
|
56
|
+
}
|
|
57
|
+
if (typeof value === 'boolean') return value ? 1 : 0;
|
|
58
|
+
if (typeof value === 'object' && !Buffer.isBuffer(value)) {
|
|
59
|
+
return JSON.stringify(value);
|
|
60
|
+
}
|
|
61
|
+
return value as SqliteBindable;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
private deserializeRow(row: Record<string, unknown>): T {
|
|
65
|
+
const out: Record<string, unknown> = { ...row };
|
|
66
|
+
for (const col of this.booleanColumns) {
|
|
67
|
+
if (col in out) {
|
|
68
|
+
out[col] = out[col] === 1 || out[col] === true;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return out as T;
|
|
72
|
+
}
|
|
73
|
+
|
|
44
74
|
async create(data: Partial<Omit<T, 'id' | 'createdAt' | 'updatedAt'>>): Promise<T> {
|
|
45
75
|
const now = Date.now();
|
|
46
76
|
const id = generateId();
|
|
@@ -49,7 +79,7 @@ export class Repository<T extends BaseRecord> {
|
|
|
49
79
|
const keys = Object.keys(record);
|
|
50
80
|
const placeholders = keys.map(() => '?').join(', ');
|
|
51
81
|
const cols = keys.map((k) => `"${k}"`).join(', ');
|
|
52
|
-
const values = keys.map((k) => (record as Record<string, unknown>)[k]);
|
|
82
|
+
const values = keys.map((k) => this.serialize(k, (record as Record<string, unknown>)[k]));
|
|
53
83
|
|
|
54
84
|
this.db
|
|
55
85
|
.prepare(`INSERT INTO "${this.tableName}" (${cols}) VALUES (${placeholders})`)
|
|
@@ -57,25 +87,26 @@ export class Repository<T extends BaseRecord> {
|
|
|
57
87
|
|
|
58
88
|
this.changeLog?.append(this.tableName, 'insert', record as Record<string, unknown>);
|
|
59
89
|
|
|
60
|
-
return record;
|
|
90
|
+
return this.deserializeRow(record as Record<string, unknown>);
|
|
61
91
|
}
|
|
62
92
|
|
|
63
93
|
async findById(id: string): Promise<T | null> {
|
|
64
94
|
const row = this.db
|
|
65
95
|
.prepare(`SELECT * FROM "${this.tableName}" WHERE id = ?`)
|
|
66
|
-
.get(id) as
|
|
67
|
-
return row
|
|
96
|
+
.get(id) as Record<string, unknown> | undefined;
|
|
97
|
+
return row ? this.deserializeRow(row) : null;
|
|
68
98
|
}
|
|
69
99
|
|
|
70
100
|
async findOne(where: WhereClause<T>): Promise<T | null> {
|
|
71
101
|
const { sql, params } = buildSelectSQL<T>(this.tableName, { where, limit: 1 });
|
|
72
|
-
const row = this.db.prepare(sql).get(...params) as
|
|
73
|
-
return row
|
|
102
|
+
const row = this.db.prepare(sql).get(...params) as Record<string, unknown> | undefined;
|
|
103
|
+
return row ? this.deserializeRow(row) : null;
|
|
74
104
|
}
|
|
75
105
|
|
|
76
106
|
async find(options: FindOptions<T> = {}): Promise<T[]> {
|
|
77
107
|
const { sql, params } = buildSelectSQL<T>(this.tableName, options);
|
|
78
|
-
|
|
108
|
+
const rows = this.db.prepare(sql).all(...params) as Record<string, unknown>[];
|
|
109
|
+
return rows.map((r) => this.deserializeRow(r));
|
|
79
110
|
}
|
|
80
111
|
|
|
81
112
|
async filter(where: WhereClause<T> = {}, options: Omit<FindOptions<T>, 'where'> = {}): Promise<T[]> {
|
|
@@ -89,7 +120,7 @@ export class Repository<T extends BaseRecord> {
|
|
|
89
120
|
const setCols = Object.keys(updateData)
|
|
90
121
|
.map((k) => `"${k}" = ?`)
|
|
91
122
|
.join(', ');
|
|
92
|
-
const setValues = Object.
|
|
123
|
+
const setValues = Object.entries(updateData).map(([k, v]) => this.serialize(k, v));
|
|
93
124
|
|
|
94
125
|
const { sql: whereSQL, params: whereParams } = buildWhereClause(where);
|
|
95
126
|
const sql = `UPDATE "${this.tableName}" SET ${setCols} ${whereSQL}`;
|
|
@@ -159,23 +159,23 @@ export class S3Adapter {
|
|
|
159
159
|
return keys;
|
|
160
160
|
}
|
|
161
161
|
|
|
162
|
-
async putManifest(
|
|
162
|
+
async putManifest(dbName: string, manifest: SyncManifest): Promise<void> {
|
|
163
163
|
const data = Buffer.from(JSON.stringify(manifest), 'utf8');
|
|
164
|
-
await this.upload(`
|
|
164
|
+
await this.upload(`databases/${dbName}/manifest.json`, data);
|
|
165
165
|
}
|
|
166
166
|
|
|
167
|
-
async getManifest(
|
|
168
|
-
const path = `
|
|
167
|
+
async getManifest(dbName: string): Promise<SyncManifest | null> {
|
|
168
|
+
const path = `databases/${dbName}/manifest.json`;
|
|
169
169
|
if (!(await this.exists(path))) return null;
|
|
170
170
|
const data = await this.download(path);
|
|
171
171
|
return JSON.parse(data.toString('utf8')) as SyncManifest;
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
-
logKey(nodeId: string, fromSeq: number, toSeq: number): string {
|
|
175
|
-
return `nodes/${nodeId}/logs/${String(fromSeq).padStart(12, '0')}-${String(toSeq).padStart(12, '0')}.json`;
|
|
174
|
+
logKey(dbName: string, nodeId: string, fromSeq: number, toSeq: number): string {
|
|
175
|
+
return `databases/${dbName}/nodes/${nodeId}/logs/${String(fromSeq).padStart(12, '0')}-${String(toSeq).padStart(12, '0')}.json`;
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
-
snapshotKey(nodeId: string, timestamp: number): string {
|
|
179
|
-
return `nodes/${nodeId}/snapshots/${timestamp}.sqlite`;
|
|
178
|
+
snapshotKey(dbName: string, nodeId: string, timestamp: number): string {
|
|
179
|
+
return `databases/${dbName}/nodes/${nodeId}/snapshots/${timestamp}.sqlite`;
|
|
180
180
|
}
|
|
181
181
|
}
|
package/src/sync/engine.ts
CHANGED
|
@@ -49,12 +49,13 @@ export class SyncEngine {
|
|
|
49
49
|
compress: config.compression !== false,
|
|
50
50
|
encryptionKey: config.encryption?.key,
|
|
51
51
|
};
|
|
52
|
-
this.snapshotManager = new SnapshotManager(
|
|
53
|
-
db,
|
|
54
|
-
this.s3,
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
52
|
+
this.snapshotManager = new SnapshotManager(
|
|
53
|
+
db,
|
|
54
|
+
this.s3,
|
|
55
|
+
config.dbName,
|
|
56
|
+
nodeId,
|
|
57
|
+
config.sqlitePath,
|
|
58
|
+
uploadOptions
|
|
58
59
|
);
|
|
59
60
|
} else {
|
|
60
61
|
this.s3 = null;
|
|
@@ -134,8 +135,13 @@ export class SyncEngine {
|
|
|
134
135
|
const pending = this.changeLog.pendingEntries(maxBatch);
|
|
135
136
|
if (pending.length === 0) return;
|
|
136
137
|
|
|
137
|
-
const batch = this.changeLog.buildBatch(pending);
|
|
138
|
-
const s3Key = this.s3!.logKey(
|
|
138
|
+
const batch = this.changeLog.buildBatch(pending);
|
|
139
|
+
const s3Key = this.s3!.logKey(
|
|
140
|
+
this.config.dbName,
|
|
141
|
+
this.nodeId,
|
|
142
|
+
batch.fromSequence,
|
|
143
|
+
batch.toSequence
|
|
144
|
+
);
|
|
139
145
|
|
|
140
146
|
const alreadyQueued = this.queue.hasPendingOfType('upload_log');
|
|
141
147
|
if (!alreadyQueued) {
|
|
@@ -174,15 +180,16 @@ export class SyncEngine {
|
|
|
174
180
|
await this.s3!.upload(p.s3Key, batchBuffer, uploadOptions);
|
|
175
181
|
this.changeLog.markSynced(p.fromSequence, p.toSequence);
|
|
176
182
|
|
|
177
|
-
const manifest = await this.s3!.getManifest(this.
|
|
178
|
-
const latestSeq = Math.max(
|
|
179
|
-
manifest?.latestLogSequence ?? 0,
|
|
180
|
-
p.toSequence
|
|
181
|
-
);
|
|
182
|
-
await this.s3!.putManifest(this.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
183
|
+
const manifest = await this.s3!.getManifest(this.config.dbName);
|
|
184
|
+
const latestSeq = Math.max(
|
|
185
|
+
manifest?.latestLogSequence ?? 0,
|
|
186
|
+
p.toSequence
|
|
187
|
+
);
|
|
188
|
+
await this.s3!.putManifest(this.config.dbName, {
|
|
189
|
+
dbName: this.config.dbName,
|
|
190
|
+
nodeId: this.nodeId,
|
|
191
|
+
latestSnapshotKey: manifest?.latestSnapshotKey ?? null,
|
|
192
|
+
latestSnapshotTimestamp: manifest?.latestSnapshotTimestamp ?? null,
|
|
186
193
|
latestLogSequence: latestSeq,
|
|
187
194
|
updatedAt: Date.now(),
|
|
188
195
|
});
|
|
@@ -194,12 +201,13 @@ export class SyncEngine {
|
|
|
194
201
|
} else if (job.type === 'upload_snapshot') {
|
|
195
202
|
const p = JSON.parse(job.payload) as UploadSnapshotPayload;
|
|
196
203
|
if (this.snapshotManager) {
|
|
197
|
-
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
198
|
-
const manifest = await this.s3!.getManifest(this.
|
|
199
|
-
await this.s3!.putManifest(this.
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
204
|
+
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
205
|
+
const manifest = await this.s3!.getManifest(this.config.dbName);
|
|
206
|
+
await this.s3!.putManifest(this.config.dbName, {
|
|
207
|
+
dbName: this.config.dbName,
|
|
208
|
+
nodeId: this.nodeId,
|
|
209
|
+
latestSnapshotKey: key,
|
|
210
|
+
latestSnapshotTimestamp: timestamp,
|
|
203
211
|
latestLogSequence: manifest?.latestLogSequence ?? 0,
|
|
204
212
|
updatedAt: Date.now(),
|
|
205
213
|
});
|
|
@@ -223,24 +231,25 @@ export class SyncEngine {
|
|
|
223
231
|
const threshold = this.config.snapshotEveryNLogs ?? DEFAULT_SNAPSHOT_EVERY_N_LOGS;
|
|
224
232
|
if (latestSequence > 0 && latestSequence % threshold === 0) {
|
|
225
233
|
if (!this.queue.hasPendingOfType('upload_snapshot')) {
|
|
226
|
-
const payload: UploadSnapshotPayload = {
|
|
227
|
-
timestamp: Date.now(),
|
|
228
|
-
s3Key: this.s3!.snapshotKey(this.nodeId, Date.now()),
|
|
229
|
-
dbPath: this.config.sqlitePath,
|
|
230
|
-
};
|
|
234
|
+
const payload: UploadSnapshotPayload = {
|
|
235
|
+
timestamp: Date.now(),
|
|
236
|
+
s3Key: this.s3!.snapshotKey(this.config.dbName, this.nodeId, Date.now()),
|
|
237
|
+
dbPath: this.config.sqlitePath,
|
|
238
|
+
};
|
|
231
239
|
this.queue.enqueue('upload_snapshot', payload);
|
|
232
240
|
}
|
|
233
241
|
}
|
|
234
242
|
}
|
|
235
243
|
|
|
236
244
|
async triggerSnapshot(): Promise<void> {
|
|
237
|
-
if (!this.snapshotManager || !this.s3) return;
|
|
238
|
-
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
239
|
-
const manifest = await this.s3.getManifest(this.
|
|
240
|
-
await this.s3.putManifest(this.
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
245
|
+
if (!this.snapshotManager || !this.s3) return;
|
|
246
|
+
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
247
|
+
const manifest = await this.s3.getManifest(this.config.dbName);
|
|
248
|
+
await this.s3.putManifest(this.config.dbName, {
|
|
249
|
+
dbName: this.config.dbName,
|
|
250
|
+
nodeId: this.nodeId,
|
|
251
|
+
latestSnapshotKey: key,
|
|
252
|
+
latestSnapshotTimestamp: timestamp,
|
|
244
253
|
latestLogSequence: manifest?.latestLogSequence ?? 0,
|
|
245
254
|
updatedAt: Date.now(),
|
|
246
255
|
});
|