@mauryasumit/driftdb 2.0.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -45
- package/dist/db.d.ts +19 -0
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +66 -13
- package/dist/db.js.map +1 -1
- package/dist/storage/s3-adapter.d.ts +4 -4
- package/dist/storage/s3-adapter.d.ts.map +1 -1
- package/dist/storage/s3-adapter.js +8 -8
- package/dist/storage/s3-adapter.js.map +1 -1
- package/dist/sync/engine.d.ts.map +1 -1
- package/dist/sync/engine.js +12 -9
- package/dist/sync/engine.js.map +1 -1
- package/dist/sync/snapshot-manager.d.ts +2 -1
- package/dist/sync/snapshot-manager.d.ts.map +1 -1
- package/dist/sync/snapshot-manager.js +6 -5
- package/dist/sync/snapshot-manager.js.map +1 -1
- package/dist/types.d.ts +3 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/db.ts +97 -36
- package/src/storage/s3-adapter.ts +8 -8
- package/src/sync/engine.ts +44 -35
- package/src/sync/snapshot-manager.ts +34 -31
- package/src/types.ts +133 -130
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-manager.js","sourceRoot":"","sources":["../../src/sync/snapshot-manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2BAA8C;AAC9C,+BAAqC;AACrC,2BAA4B;AAC5B,2BAAyD;AAKzD,MAAa,eAAe;
|
|
1
|
+
{"version":3,"file":"snapshot-manager.js","sourceRoot":"","sources":["../../src/sync/snapshot-manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2BAA8C;AAC9C,+BAAqC;AACrC,2BAA4B;AAC5B,2BAAyD;AAKzD,MAAa,eAAe;IAQ1B,YACE,EAAqB,EACrB,EAAa,EACb,MAAc,EACd,MAAc,EACd,UAAkB,EAClB,aAA8B;QAE9B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,IAAA,WAAM,GAAE,EAAE,gBAAgB,IAAI,CAAC,MAAM,IAAI,SAAS,SAAS,CAAC,CAAC;QAEnF,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAE5C,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;gBACnC,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAClB,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EACxD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EACnB,IAAI,CAAC,aAAa,CACnB,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAA,iBAAY,EAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;gBACxC,MAAM,IAAI,GAAG,IAAA,iBAAY,EAAC,QAAQ,CAAC,CAAC;gBACpC,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAClB,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EACxD,IAAI,EACJ,IAAI,CAAC,aAAa,CACnB,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACrE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,IAAI,IAAA,eAAU,EAAC,QAAQ,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC;oBAAC,IAAA,eAAU,EAAC,QAAQ,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,iBAAiB;YAAE,OAAO,KAAK,CAAC;QAE/C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAEpF,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5E,IAAI,CAAC,WAAW;YAAE,OAAO,KAAK,CAAC;QAE/B,MAAM,GAAG,GAAG,IAAA,cAAO,EAAC,WAAW,CAAC,CAAC;QACjC,IAAA,cAAS,EAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpC,MAAM,EAAE,aAAa,EAAE,GAAG,wDAAa,IAAI,GAAC,CAAC;QAC7C,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAEjC,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AA1ED,0CA0EC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -48,6 +48,7 @@ export interface EncryptionConfig {
|
|
|
48
48
|
key: string;
|
|
49
49
|
}
|
|
50
50
|
export interface DBConfig {
|
|
51
|
+
dbName: string;
|
|
51
52
|
sqlitePath: string;
|
|
52
53
|
s3Config?: S3Config;
|
|
53
54
|
nodeId?: string;
|
|
@@ -58,6 +59,7 @@ export interface DBConfig {
|
|
|
58
59
|
encryption?: EncryptionConfig;
|
|
59
60
|
retryConfig?: RetryConfig;
|
|
60
61
|
autoSync?: boolean;
|
|
62
|
+
restoreFromS3?: boolean;
|
|
61
63
|
}
|
|
62
64
|
export interface ChangeLogEntry {
|
|
63
65
|
sequence: number;
|
|
@@ -92,6 +94,7 @@ export interface SyncJob {
|
|
|
92
94
|
error: string | null;
|
|
93
95
|
}
|
|
94
96
|
export interface SyncManifest {
|
|
97
|
+
dbName: string;
|
|
95
98
|
nodeId: string;
|
|
96
99
|
latestSnapshotKey: string | null;
|
|
97
100
|
latestSnapshotTimestamp: number | null;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAE1E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;IAC3C,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEpD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,UAAU,CAAC,CAAC,IACpB,CAAC,GACD;IAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,CAAA;CAAE,CAAC;AAEjF,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI;KAC1B,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CAClC,CAAC;AAEF,MAAM,WAAW,WAAW,CAAC,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,QAAQ;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAE1E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;IAC3C,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEpD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,UAAU,CAAC,CAAC,IACpB,CAAC,GACD;IAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,CAAA;CAAE,CAAC;AAEjF,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI;KAC1B,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CAClC,CAAC;AAEF,MAAM,WAAW,WAAW,CAAC,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC1C,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;CACf;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,CAAC,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,KAAK,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;QAC1C,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;KACtC,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,YAAY,GAAG,iBAAiB,CAAC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,GAAG,YAAY,GAAG,MAAM,GAAG,QAAQ,CAAC;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB"}
|
package/package.json
CHANGED
package/src/db.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import BetterSqlite3 from 'better-sqlite3';
|
|
2
2
|
import type Database from 'better-sqlite3';
|
|
3
|
-
import { mkdirSync, existsSync } from 'fs';
|
|
3
|
+
import { mkdirSync, existsSync, writeFileSync } from 'fs';
|
|
4
4
|
import { dirname } from 'path';
|
|
5
5
|
import type { DBConfig, ModelSchema, SyncMetrics } from './types.js';
|
|
6
6
|
import { Repository } from './orm/repository.js';
|
|
7
7
|
import { SyncEngine } from './sync/engine.js';
|
|
8
|
+
import { S3Adapter } from './storage/s3-adapter.js';
|
|
8
9
|
import { generateNodeId } from './utils/id.js';
|
|
9
10
|
import type { Model, ModelStatic } from './orm/model.js';
|
|
10
11
|
import type { BaseRecord } from './types.js';
|
|
@@ -16,24 +17,32 @@ const META_SCHEMA = `
|
|
|
16
17
|
);
|
|
17
18
|
`;
|
|
18
19
|
|
|
19
|
-
export class DB {
|
|
20
|
+
export class DB {
|
|
20
21
|
private readonly sqliteDb: Database.Database;
|
|
21
22
|
private readonly config: DBConfig;
|
|
22
23
|
private readonly nodeId: string;
|
|
23
24
|
private readonly syncEngine: SyncEngine;
|
|
24
25
|
private readonly repos = new Map<string, Repository<BaseRecord>>();
|
|
25
26
|
|
|
26
|
-
constructor(config: DBConfig) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
27
|
+
constructor(config: DBConfig) {
|
|
28
|
+
const dbName = config.dbName.trim();
|
|
29
|
+
if (!dbName) {
|
|
30
|
+
throw new Error('DBConfig.dbName is required');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
this.config = {
|
|
34
|
+
...config,
|
|
35
|
+
dbName,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
if (this.config.sqlitePath !== ':memory:') {
|
|
39
|
+
const dir = dirname(this.config.sqlitePath);
|
|
40
|
+
if (dir && dir !== '.') {
|
|
41
|
+
mkdirSync(dir, { recursive: true });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
this.sqliteDb = new BetterSqlite3(this.config.sqlitePath);
|
|
37
46
|
this.sqliteDb.pragma('journal_mode = WAL');
|
|
38
47
|
this.sqliteDb.pragma('synchronous = NORMAL');
|
|
39
48
|
this.sqliteDb.pragma('foreign_keys = ON');
|
|
@@ -42,35 +51,87 @@ export class DB {
|
|
|
42
51
|
|
|
43
52
|
this.sqliteDb.exec(META_SCHEMA);
|
|
44
53
|
|
|
45
|
-
this.nodeId = this.getOrCreateNodeId(config.nodeId);
|
|
46
|
-
this.syncEngine = new SyncEngine(this.sqliteDb, this.nodeId, config);
|
|
47
|
-
|
|
48
|
-
if (config.autoSync !== false && config.s3Config) {
|
|
49
|
-
this.syncEngine.start();
|
|
54
|
+
this.nodeId = this.getOrCreateNodeId(this.config.nodeId);
|
|
55
|
+
this.syncEngine = new SyncEngine(this.sqliteDb, this.nodeId, this.config);
|
|
56
|
+
|
|
57
|
+
if (this.config.autoSync !== false && this.config.s3Config) {
|
|
58
|
+
this.syncEngine.start();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Async factory — use this instead of `new DB()` when you need S3 restore on startup.
|
|
64
|
+
*
|
|
65
|
+
* - If `restoreFromS3: true` and the local SQLite file does not exist, it downloads
|
|
66
|
+
* the latest snapshot from S3 before opening the database.
|
|
67
|
+
* - `dbName` is required and namespaces each logical database in S3.
|
|
68
|
+
* - `nodeId` identifies the current local node within that logical database.
|
|
69
|
+
*
|
|
70
|
+
* @example
|
|
71
|
+
* const db = await DB.open({
|
|
72
|
+
* dbName: 'my-app-db',
|
|
73
|
+
* sqlitePath: './data/app.sqlite',
|
|
74
|
+
* nodeId: 'server-1',
|
|
75
|
+
* restoreFromS3: true, // auto-restore if local file is missing
|
|
76
|
+
* s3Config: { bucket: '...', region: '...' },
|
|
77
|
+
* });
|
|
78
|
+
*/
|
|
79
|
+
static async open(config: DBConfig): Promise<DB> {
|
|
80
|
+
if (
|
|
81
|
+
config.restoreFromS3 &&
|
|
82
|
+
config.s3Config &&
|
|
83
|
+
config.sqlitePath !== ':memory:' &&
|
|
84
|
+
!existsSync(config.sqlitePath)
|
|
85
|
+
) {
|
|
86
|
+
await DB.restoreSnapshot(config);
|
|
50
87
|
}
|
|
88
|
+
return new DB(config);
|
|
51
89
|
}
|
|
52
90
|
|
|
53
|
-
private
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
91
|
+
private static async restoreSnapshot(config: DBConfig): Promise<boolean> {
|
|
92
|
+
const s3 = new S3Adapter(config.s3Config!);
|
|
93
|
+
const uploadOptions = {
|
|
94
|
+
compress: config.compression !== false,
|
|
95
|
+
encryptionKey: config.encryption?.key,
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const manifest = await s3.getManifest(config.dbName);
|
|
99
|
+
if (!manifest?.latestSnapshotKey) {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const data = await s3.download(manifest.latestSnapshotKey, uploadOptions);
|
|
104
|
+
const dir = dirname(config.sqlitePath);
|
|
105
|
+
if (dir && dir !== '.') {
|
|
106
|
+
mkdirSync(dir, { recursive: true });
|
|
59
107
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
.prepare(`SELECT value FROM _driftdb_meta WHERE key = 'nodeId'`)
|
|
63
|
-
.get() as { value: string } | undefined;
|
|
64
|
-
|
|
65
|
-
if (row) return row.value;
|
|
66
|
-
|
|
67
|
-
const id = generateNodeId();
|
|
68
|
-
this.sqliteDb
|
|
69
|
-
.prepare(`INSERT INTO _driftdb_meta (key, value) VALUES ('nodeId', ?)`)
|
|
70
|
-
.run(id);
|
|
71
|
-
return id;
|
|
108
|
+
writeFileSync(config.sqlitePath, data);
|
|
109
|
+
return true;
|
|
72
110
|
}
|
|
73
111
|
|
|
112
|
+
private getOrCreateNodeId(preferred?: string): string {
|
|
113
|
+
const metaKey = `nodeId:${this.config.dbName}`;
|
|
114
|
+
|
|
115
|
+
if (preferred) {
|
|
116
|
+
this.sqliteDb
|
|
117
|
+
.prepare(`INSERT OR REPLACE INTO _driftdb_meta (key, value) VALUES (?, ?)`)
|
|
118
|
+
.run(metaKey, preferred);
|
|
119
|
+
return preferred;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const row = this.sqliteDb
|
|
123
|
+
.prepare(`SELECT value FROM _driftdb_meta WHERE key = ? OR key = 'nodeId' ORDER BY CASE WHEN key = ? THEN 0 ELSE 1 END LIMIT 1`)
|
|
124
|
+
.get(metaKey, metaKey) as { value: string } | undefined;
|
|
125
|
+
|
|
126
|
+
if (row) return row.value;
|
|
127
|
+
|
|
128
|
+
const id = generateNodeId();
|
|
129
|
+
this.sqliteDb
|
|
130
|
+
.prepare(`INSERT INTO _driftdb_meta (key, value) VALUES (?, ?)`)
|
|
131
|
+
.run(metaKey, id);
|
|
132
|
+
return id;
|
|
133
|
+
}
|
|
134
|
+
|
|
74
135
|
define<S extends ModelSchema>(
|
|
75
136
|
tableName: string,
|
|
76
137
|
schema: S
|
|
@@ -159,23 +159,23 @@ export class S3Adapter {
|
|
|
159
159
|
return keys;
|
|
160
160
|
}
|
|
161
161
|
|
|
162
|
-
async putManifest(
|
|
162
|
+
async putManifest(dbName: string, manifest: SyncManifest): Promise<void> {
|
|
163
163
|
const data = Buffer.from(JSON.stringify(manifest), 'utf8');
|
|
164
|
-
await this.upload(`
|
|
164
|
+
await this.upload(`databases/${dbName}/manifest.json`, data);
|
|
165
165
|
}
|
|
166
166
|
|
|
167
|
-
async getManifest(
|
|
168
|
-
const path = `
|
|
167
|
+
async getManifest(dbName: string): Promise<SyncManifest | null> {
|
|
168
|
+
const path = `databases/${dbName}/manifest.json`;
|
|
169
169
|
if (!(await this.exists(path))) return null;
|
|
170
170
|
const data = await this.download(path);
|
|
171
171
|
return JSON.parse(data.toString('utf8')) as SyncManifest;
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
-
logKey(nodeId: string, fromSeq: number, toSeq: number): string {
|
|
175
|
-
return `nodes/${nodeId}/logs/${String(fromSeq).padStart(12, '0')}-${String(toSeq).padStart(12, '0')}.json`;
|
|
174
|
+
logKey(dbName: string, nodeId: string, fromSeq: number, toSeq: number): string {
|
|
175
|
+
return `databases/${dbName}/nodes/${nodeId}/logs/${String(fromSeq).padStart(12, '0')}-${String(toSeq).padStart(12, '0')}.json`;
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
-
snapshotKey(nodeId: string, timestamp: number): string {
|
|
179
|
-
return `nodes/${nodeId}/snapshots/${timestamp}.sqlite`;
|
|
178
|
+
snapshotKey(dbName: string, nodeId: string, timestamp: number): string {
|
|
179
|
+
return `databases/${dbName}/nodes/${nodeId}/snapshots/${timestamp}.sqlite`;
|
|
180
180
|
}
|
|
181
181
|
}
|
package/src/sync/engine.ts
CHANGED
|
@@ -49,12 +49,13 @@ export class SyncEngine {
|
|
|
49
49
|
compress: config.compression !== false,
|
|
50
50
|
encryptionKey: config.encryption?.key,
|
|
51
51
|
};
|
|
52
|
-
this.snapshotManager = new SnapshotManager(
|
|
53
|
-
db,
|
|
54
|
-
this.s3,
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
52
|
+
this.snapshotManager = new SnapshotManager(
|
|
53
|
+
db,
|
|
54
|
+
this.s3,
|
|
55
|
+
config.dbName,
|
|
56
|
+
nodeId,
|
|
57
|
+
config.sqlitePath,
|
|
58
|
+
uploadOptions
|
|
58
59
|
);
|
|
59
60
|
} else {
|
|
60
61
|
this.s3 = null;
|
|
@@ -134,8 +135,13 @@ export class SyncEngine {
|
|
|
134
135
|
const pending = this.changeLog.pendingEntries(maxBatch);
|
|
135
136
|
if (pending.length === 0) return;
|
|
136
137
|
|
|
137
|
-
const batch = this.changeLog.buildBatch(pending);
|
|
138
|
-
const s3Key = this.s3!.logKey(
|
|
138
|
+
const batch = this.changeLog.buildBatch(pending);
|
|
139
|
+
const s3Key = this.s3!.logKey(
|
|
140
|
+
this.config.dbName,
|
|
141
|
+
this.nodeId,
|
|
142
|
+
batch.fromSequence,
|
|
143
|
+
batch.toSequence
|
|
144
|
+
);
|
|
139
145
|
|
|
140
146
|
const alreadyQueued = this.queue.hasPendingOfType('upload_log');
|
|
141
147
|
if (!alreadyQueued) {
|
|
@@ -174,15 +180,16 @@ export class SyncEngine {
|
|
|
174
180
|
await this.s3!.upload(p.s3Key, batchBuffer, uploadOptions);
|
|
175
181
|
this.changeLog.markSynced(p.fromSequence, p.toSequence);
|
|
176
182
|
|
|
177
|
-
const manifest = await this.s3!.getManifest(this.
|
|
178
|
-
const latestSeq = Math.max(
|
|
179
|
-
manifest?.latestLogSequence ?? 0,
|
|
180
|
-
p.toSequence
|
|
181
|
-
);
|
|
182
|
-
await this.s3!.putManifest(this.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
183
|
+
const manifest = await this.s3!.getManifest(this.config.dbName);
|
|
184
|
+
const latestSeq = Math.max(
|
|
185
|
+
manifest?.latestLogSequence ?? 0,
|
|
186
|
+
p.toSequence
|
|
187
|
+
);
|
|
188
|
+
await this.s3!.putManifest(this.config.dbName, {
|
|
189
|
+
dbName: this.config.dbName,
|
|
190
|
+
nodeId: this.nodeId,
|
|
191
|
+
latestSnapshotKey: manifest?.latestSnapshotKey ?? null,
|
|
192
|
+
latestSnapshotTimestamp: manifest?.latestSnapshotTimestamp ?? null,
|
|
186
193
|
latestLogSequence: latestSeq,
|
|
187
194
|
updatedAt: Date.now(),
|
|
188
195
|
});
|
|
@@ -194,12 +201,13 @@ export class SyncEngine {
|
|
|
194
201
|
} else if (job.type === 'upload_snapshot') {
|
|
195
202
|
const p = JSON.parse(job.payload) as UploadSnapshotPayload;
|
|
196
203
|
if (this.snapshotManager) {
|
|
197
|
-
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
198
|
-
const manifest = await this.s3!.getManifest(this.
|
|
199
|
-
await this.s3!.putManifest(this.
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
204
|
+
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
205
|
+
const manifest = await this.s3!.getManifest(this.config.dbName);
|
|
206
|
+
await this.s3!.putManifest(this.config.dbName, {
|
|
207
|
+
dbName: this.config.dbName,
|
|
208
|
+
nodeId: this.nodeId,
|
|
209
|
+
latestSnapshotKey: key,
|
|
210
|
+
latestSnapshotTimestamp: timestamp,
|
|
203
211
|
latestLogSequence: manifest?.latestLogSequence ?? 0,
|
|
204
212
|
updatedAt: Date.now(),
|
|
205
213
|
});
|
|
@@ -223,24 +231,25 @@ export class SyncEngine {
|
|
|
223
231
|
const threshold = this.config.snapshotEveryNLogs ?? DEFAULT_SNAPSHOT_EVERY_N_LOGS;
|
|
224
232
|
if (latestSequence > 0 && latestSequence % threshold === 0) {
|
|
225
233
|
if (!this.queue.hasPendingOfType('upload_snapshot')) {
|
|
226
|
-
const payload: UploadSnapshotPayload = {
|
|
227
|
-
timestamp: Date.now(),
|
|
228
|
-
s3Key: this.s3!.snapshotKey(this.nodeId, Date.now()),
|
|
229
|
-
dbPath: this.config.sqlitePath,
|
|
230
|
-
};
|
|
234
|
+
const payload: UploadSnapshotPayload = {
|
|
235
|
+
timestamp: Date.now(),
|
|
236
|
+
s3Key: this.s3!.snapshotKey(this.config.dbName, this.nodeId, Date.now()),
|
|
237
|
+
dbPath: this.config.sqlitePath,
|
|
238
|
+
};
|
|
231
239
|
this.queue.enqueue('upload_snapshot', payload);
|
|
232
240
|
}
|
|
233
241
|
}
|
|
234
242
|
}
|
|
235
243
|
|
|
236
244
|
async triggerSnapshot(): Promise<void> {
|
|
237
|
-
if (!this.snapshotManager || !this.s3) return;
|
|
238
|
-
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
239
|
-
const manifest = await this.s3.getManifest(this.
|
|
240
|
-
await this.s3.putManifest(this.
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
245
|
+
if (!this.snapshotManager || !this.s3) return;
|
|
246
|
+
const { key, timestamp } = await this.snapshotManager.takeAndUpload();
|
|
247
|
+
const manifest = await this.s3.getManifest(this.config.dbName);
|
|
248
|
+
await this.s3.putManifest(this.config.dbName, {
|
|
249
|
+
dbName: this.config.dbName,
|
|
250
|
+
nodeId: this.nodeId,
|
|
251
|
+
latestSnapshotKey: key,
|
|
252
|
+
latestSnapshotTimestamp: timestamp,
|
|
244
253
|
latestLogSequence: manifest?.latestLogSequence ?? 0,
|
|
245
254
|
updatedAt: Date.now(),
|
|
246
255
|
});
|
|
@@ -8,21 +8,24 @@ import type { S3UploadOptions } from '../storage/s3-adapter.js';
|
|
|
8
8
|
|
|
9
9
|
export class SnapshotManager {
|
|
10
10
|
private readonly db: Database.Database;
|
|
11
|
-
private readonly s3: S3Adapter;
|
|
12
|
-
private readonly
|
|
11
|
+
private readonly s3: S3Adapter;
|
|
12
|
+
private readonly dbName: string;
|
|
13
|
+
private readonly nodeId: string;
|
|
13
14
|
private readonly sqlitePath: string;
|
|
14
15
|
private readonly uploadOptions: S3UploadOptions;
|
|
15
16
|
|
|
16
17
|
constructor(
|
|
17
|
-
db: Database.Database,
|
|
18
|
-
s3: S3Adapter,
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
this.
|
|
25
|
-
this.
|
|
18
|
+
db: Database.Database,
|
|
19
|
+
s3: S3Adapter,
|
|
20
|
+
dbName: string,
|
|
21
|
+
nodeId: string,
|
|
22
|
+
sqlitePath: string,
|
|
23
|
+
uploadOptions: S3UploadOptions
|
|
24
|
+
) {
|
|
25
|
+
this.db = db;
|
|
26
|
+
this.s3 = s3;
|
|
27
|
+
this.dbName = dbName;
|
|
28
|
+
this.nodeId = nodeId;
|
|
26
29
|
this.sqlitePath = sqlitePath;
|
|
27
30
|
this.uploadOptions = uploadOptions;
|
|
28
31
|
}
|
|
@@ -35,24 +38,24 @@ export class SnapshotManager {
|
|
|
35
38
|
this.db.exec('PRAGMA wal_checkpoint(FULL)');
|
|
36
39
|
|
|
37
40
|
if (this.sqlitePath === ':memory:') {
|
|
38
|
-
const backup = this.db.serialize();
|
|
39
|
-
await this.s3.upload(
|
|
40
|
-
this.s3.snapshotKey(this.nodeId, timestamp),
|
|
41
|
-
Buffer.from(backup),
|
|
42
|
-
this.uploadOptions
|
|
43
|
-
);
|
|
44
|
-
} else {
|
|
45
|
-
copyFileSync(this.sqlitePath, tempPath);
|
|
46
|
-
const data = readFileSync(tempPath);
|
|
47
|
-
await this.s3.upload(
|
|
48
|
-
this.s3.snapshotKey(this.nodeId, timestamp),
|
|
49
|
-
data,
|
|
50
|
-
this.uploadOptions
|
|
51
|
-
);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
const key = this.s3.snapshotKey(this.nodeId, timestamp);
|
|
55
|
-
return { key, timestamp };
|
|
41
|
+
const backup = this.db.serialize();
|
|
42
|
+
await this.s3.upload(
|
|
43
|
+
this.s3.snapshotKey(this.dbName, this.nodeId, timestamp),
|
|
44
|
+
Buffer.from(backup),
|
|
45
|
+
this.uploadOptions
|
|
46
|
+
);
|
|
47
|
+
} else {
|
|
48
|
+
copyFileSync(this.sqlitePath, tempPath);
|
|
49
|
+
const data = readFileSync(tempPath);
|
|
50
|
+
await this.s3.upload(
|
|
51
|
+
this.s3.snapshotKey(this.dbName, this.nodeId, timestamp),
|
|
52
|
+
data,
|
|
53
|
+
this.uploadOptions
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const key = this.s3.snapshotKey(this.dbName, this.nodeId, timestamp);
|
|
58
|
+
return { key, timestamp };
|
|
56
59
|
} finally {
|
|
57
60
|
if (existsSync(tempPath)) {
|
|
58
61
|
try { unlinkSync(tempPath); } catch { /* ignore */ }
|
|
@@ -60,8 +63,8 @@ export class SnapshotManager {
|
|
|
60
63
|
}
|
|
61
64
|
}
|
|
62
65
|
|
|
63
|
-
async restoreLatest(): Promise<boolean> {
|
|
64
|
-
const manifest = await this.s3.getManifest(this.
|
|
66
|
+
async restoreLatest(): Promise<boolean> {
|
|
67
|
+
const manifest = await this.s3.getManifest(this.dbName);
|
|
65
68
|
if (!manifest?.latestSnapshotKey) return false;
|
|
66
69
|
|
|
67
70
|
const data = await this.s3.download(manifest.latestSnapshotKey, this.uploadOptions);
|