@mauryasumit/driftdb 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +810 -0
  2. package/dist/db.d.ts +30 -0
  3. package/dist/db.d.ts.map +1 -0
  4. package/dist/db.js +115 -0
  5. package/dist/db.js.map +1 -0
  6. package/dist/index.d.ts +8 -0
  7. package/dist/index.d.ts.map +1 -0
  8. package/dist/index.js +12 -0
  9. package/dist/index.js.map +1 -0
  10. package/dist/orm/model.d.ts +35 -0
  11. package/dist/orm/model.d.ts.map +1 -0
  12. package/dist/orm/model.js +34 -0
  13. package/dist/orm/model.js.map +1 -0
  14. package/dist/orm/query-builder.d.ts +8 -0
  15. package/dist/orm/query-builder.d.ts.map +1 -0
  16. package/dist/orm/query-builder.js +90 -0
  17. package/dist/orm/query-builder.js.map +1 -0
  18. package/dist/orm/repository.d.ts +38 -0
  19. package/dist/orm/repository.d.ts.map +1 -0
  20. package/dist/orm/repository.js +107 -0
  21. package/dist/orm/repository.js.map +1 -0
  22. package/dist/orm/schema.d.ts +20 -0
  23. package/dist/orm/schema.d.ts.map +1 -0
  24. package/dist/orm/schema.js +81 -0
  25. package/dist/orm/schema.js.map +1 -0
  26. package/dist/queue/queue.d.ts +17 -0
  27. package/dist/queue/queue.d.ts.map +1 -0
  28. package/dist/queue/queue.js +109 -0
  29. package/dist/queue/queue.js.map +1 -0
  30. package/dist/storage/s3-adapter.d.ts +21 -0
  31. package/dist/storage/s3-adapter.d.ts.map +1 -0
  32. package/dist/storage/s3-adapter.js +133 -0
  33. package/dist/storage/s3-adapter.js.map +1 -0
  34. package/dist/sync/change-log.d.ts +15 -0
  35. package/dist/sync/change-log.d.ts.map +1 -0
  36. package/dist/sync/change-log.js +78 -0
  37. package/dist/sync/change-log.js.map +1 -0
  38. package/dist/sync/engine.d.ts +31 -0
  39. package/dist/sync/engine.d.ts.map +1 -0
  40. package/dist/sync/engine.js +210 -0
  41. package/dist/sync/engine.js.map +1 -0
  42. package/dist/sync/snapshot-manager.d.ts +17 -0
  43. package/dist/sync/snapshot-manager.d.ts.map +1 -0
  44. package/dist/sync/snapshot-manager.js +91 -0
  45. package/dist/sync/snapshot-manager.js.map +1 -0
  46. package/dist/types.d.ts +120 -0
  47. package/dist/types.d.ts.map +1 -0
  48. package/dist/types.js +3 -0
  49. package/dist/types.js.map +1 -0
  50. package/dist/utils/compress.d.ts +3 -0
  51. package/dist/utils/compress.d.ts.map +1 -0
  52. package/dist/utils/compress.js +16 -0
  53. package/dist/utils/compress.js.map +1 -0
  54. package/dist/utils/crypto.d.ts +4 -0
  55. package/dist/utils/crypto.d.ts.map +1 -0
  56. package/dist/utils/crypto.js +35 -0
  57. package/dist/utils/crypto.js.map +1 -0
  58. package/dist/utils/id.d.ts +3 -0
  59. package/dist/utils/id.d.ts.map +1 -0
  60. package/dist/utils/id.js +13 -0
  61. package/dist/utils/id.js.map +1 -0
  62. package/dist/utils/retry.d.ts +5 -0
  63. package/dist/utils/retry.d.ts.map +1 -0
  64. package/dist/utils/retry.js +36 -0
  65. package/dist/utils/retry.js.map +1 -0
  66. package/package.json +55 -0
  67. package/src/db.ts +154 -0
  68. package/src/index.ts +24 -0
  69. package/src/orm/model.ts +95 -0
  70. package/src/orm/query-builder.ts +100 -0
  71. package/src/orm/repository.ts +156 -0
  72. package/src/orm/schema.ts +92 -0
  73. package/src/queue/queue.ts +138 -0
  74. package/src/storage/s3-adapter.ts +181 -0
  75. package/src/sync/change-log.ts +101 -0
  76. package/src/sync/engine.ts +249 -0
  77. package/src/sync/snapshot-manager.ts +80 -0
  78. package/src/types.ts +130 -0
  79. package/src/utils/compress.ts +14 -0
  80. package/src/utils/crypto.ts +33 -0
  81. package/src/utils/id.ts +10 -0
  82. package/src/utils/retry.ts +38 -0
@@ -0,0 +1,81 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Column = exports.ColumnBuilder = void 0;
4
+ exports.buildCreateTableSQL = buildCreateTableSQL;
5
+ exports.normalizeSchema = normalizeSchema;
6
+ class ColumnBuilder {
7
+ constructor(type) {
8
+ this.def = { type };
9
+ }
10
+ required() {
11
+ this.def.notNull = true;
12
+ return this;
13
+ }
14
+ unique() {
15
+ this.def.unique = true;
16
+ return this;
17
+ }
18
+ default(value) {
19
+ this.def.default = value;
20
+ return this;
21
+ }
22
+ index() {
23
+ this.def.index = true;
24
+ return this;
25
+ }
26
+ build() {
27
+ return { ...this.def };
28
+ }
29
+ }
30
+ exports.ColumnBuilder = ColumnBuilder;
31
+ exports.Column = {
32
+ text() {
33
+ return new ColumnBuilder('TEXT');
34
+ },
35
+ integer() {
36
+ return new ColumnBuilder('INTEGER');
37
+ },
38
+ real() {
39
+ return new ColumnBuilder('REAL');
40
+ },
41
+ blob() {
42
+ return new ColumnBuilder('BLOB');
43
+ },
44
+ boolean() {
45
+ return new ColumnBuilder('BOOLEAN');
46
+ },
47
+ };
48
+ function buildCreateTableSQL(tableName, schema) {
49
+ const cols = [
50
+ 'id TEXT PRIMARY KEY',
51
+ 'createdAt INTEGER NOT NULL',
52
+ 'updatedAt INTEGER NOT NULL',
53
+ ];
54
+ const indices = [];
55
+ for (const [name, rawDef] of Object.entries(schema)) {
56
+ const def = rawDef instanceof ColumnBuilder ? rawDef.build() : rawDef;
57
+ let col = `"${name}" ${def.type}`;
58
+ if (def.notNull)
59
+ col += ' NOT NULL';
60
+ if (def.unique)
61
+ col += ' UNIQUE';
62
+ if (def.default !== undefined) {
63
+ const v = def.default;
64
+ col += ` DEFAULT ${typeof v === 'string' ? `'${v}'` : String(v)}`;
65
+ }
66
+ cols.push(col);
67
+ if (def.index) {
68
+ indices.push(`CREATE INDEX IF NOT EXISTS "idx_${tableName}_${name}" ON "${tableName}" ("${name}");`);
69
+ }
70
+ }
71
+ const create = `CREATE TABLE IF NOT EXISTS "${tableName}" (\n ${cols.join(',\n ')}\n);`;
72
+ return [create, ...indices].join('\n');
73
+ }
74
+ function normalizeSchema(schema) {
75
+ const normalized = {};
76
+ for (const [key, val] of Object.entries(schema)) {
77
+ normalized[key] = val instanceof ColumnBuilder ? val.build() : val;
78
+ }
79
+ return normalized;
80
+ }
81
+ //# sourceMappingURL=schema.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/orm/schema.ts"],"names":[],"mappings":";;;AAoDA,kDA+BC;AAED,0CAMC;AAzFD,MAAa,aAAa;IAGxB,YAAY,IAAgB;QAC1B,IAAI,CAAC,GAAG,GAAG,EAAE,IAAI,EAAE,CAAC;IACtB,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,KAAuC;QAC7C,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,CAAC;CACF;AA9BD,sCA8BC;AAEY,QAAA,MAAM,GAAG;IACpB,IAAI;QACF,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,OAAO;QACL,OAAO,IAAI,aAAa,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IACD,IAAI;QACF,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,IAAI;QACF,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,OAAO;QACL,OAAO,IAAI,aAAa,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;CACF,CAAC;AAEF,SAAgB,mBAAmB,CAAC,SAAiB,EAAE,MAAmB;IACxE,MAAM,IAAI,GAAa;QACrB,qBAAqB;QACrB,4BAA4B;QAC5B,4BAA4B;KAC7B,CAAC;IAEF,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,MAAM,GAAG,GACP,MAAM,YAAY,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAE5D,IAAI,GAAG,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;QAClC,IAAI,GAAG,CAAC,OAAO;YAAE,GAAG,IAAI,WAAW,CAAC;QACpC,IAAI,GAAG,CAAC,MAAM;YAAE,GAAG,IAAI,SAAS,CAAC;QACjC,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;YACtB,GAAG,IAAI,YAAY,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEf,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,OAAO,CAAC,IAAI,CACV,mCAAmC,SAAS,IAAI,IAAI,SAAS,SAAS,OAAO,IAAI,KAAK,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,+BAA+B,SAAS,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC1F,OAAO,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC;AAED,SAAgB,eAAe,CAAC,MAAmB;IACjD,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,YAAY,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IACrE,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}
@@ -0,0 +1,17 @@
1
+ import type Database from 'better-sqlite3';
2
+ import type { SyncJob } from '../types.js';
3
+ import type { RetryConfig } from '../types.js';
4
+ export declare class SyncQueue {
5
+ private readonly db;
6
+ private readonly retryConfig;
7
+ constructor(db: Database.Database, retryConfig: RetryConfig);
8
+ enqueue(type: SyncJob['type'], payload: object): SyncJob;
9
+ dequeue(limit?: number): SyncJob[];
10
+ markDone(id: string): void;
11
+ markFailed(id: string, error: string): void;
12
+ resetStuck(): void;
13
+ pendingCount(): number;
14
+ purgeCompleted(olderThanMs?: number): void;
15
+ hasPendingOfType(type: SyncJob['type']): boolean;
16
+ }
17
+ //# sourceMappingURL=queue.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../../src/queue/queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAgB/C,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;gBAE9B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW;IAM3D,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO;IAsBxD,OAAO,CAAC,KAAK,SAAI,GAAG,OAAO,EAAE;IAuB7B,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAM1B,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAqB3C,UAAU,IAAI,IAAI;IAWlB,YAAY,IAAI,MAAM;IAStB,cAAc,CAAC,WAAW,SAAsB,GAAG,IAAI;IAOvD,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO;CAQjD"}
@@ -0,0 +1,109 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.SyncQueue = void 0;
4
+ const id_js_1 = require("../utils/id.js");
5
+ const retry_js_1 = require("../utils/retry.js");
6
+ const SCHEMA = `
7
+ CREATE TABLE IF NOT EXISTS _driftdb_queue (
8
+ id TEXT PRIMARY KEY,
9
+ type TEXT NOT NULL,
10
+ payload TEXT NOT NULL,
11
+ status TEXT NOT NULL DEFAULT 'pending',
12
+ attempts INTEGER NOT NULL DEFAULT 0,
13
+ nextRetryAt INTEGER NOT NULL DEFAULT 0,
14
+ createdAt INTEGER NOT NULL,
15
+ error TEXT
16
+ );
17
+ CREATE INDEX IF NOT EXISTS idx_driftdb_queue_status ON _driftdb_queue (status, nextRetryAt);
18
+ `;
19
+ class SyncQueue {
20
+ constructor(db, retryConfig) {
21
+ this.db = db;
22
+ this.retryConfig = retryConfig;
23
+ db.exec(SCHEMA);
24
+ }
25
+ enqueue(type, payload) {
26
+ const job = {
27
+ id: (0, id_js_1.generateId)(),
28
+ type,
29
+ payload: JSON.stringify(payload),
30
+ status: 'pending',
31
+ attempts: 0,
32
+ nextRetryAt: 0,
33
+ createdAt: Date.now(),
34
+ error: null,
35
+ };
36
+ this.db
37
+ .prepare(`INSERT INTO _driftdb_queue (id, type, payload, status, attempts, nextRetryAt, createdAt, error)
38
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
39
+ .run(job.id, job.type, job.payload, job.status, job.attempts, job.nextRetryAt, job.createdAt, job.error);
40
+ return job;
41
+ }
42
+ dequeue(limit = 5) {
43
+ const now = Date.now();
44
+ const rows = this.db
45
+ .prepare(`SELECT * FROM _driftdb_queue
46
+ WHERE status IN ('pending', 'failed')
47
+ AND nextRetryAt <= ?
48
+ ORDER BY createdAt ASC
49
+ LIMIT ?`)
50
+ .all(now, limit);
51
+ if (rows.length === 0)
52
+ return [];
53
+ const ids = rows.map((r) => r.id);
54
+ const placeholders = ids.map(() => '?').join(',');
55
+ this.db
56
+ .prepare(`UPDATE _driftdb_queue SET status = 'processing' WHERE id IN (${placeholders})`)
57
+ .run(...ids);
58
+ return rows.map((r) => ({ ...r, status: 'processing' }));
59
+ }
60
+ markDone(id) {
61
+ this.db
62
+ .prepare(`UPDATE _driftdb_queue SET status = 'done', error = NULL WHERE id = ?`)
63
+ .run(id);
64
+ }
65
+ markFailed(id, error) {
66
+ const job = this.db
67
+ .prepare(`SELECT attempts FROM _driftdb_queue WHERE id = ?`)
68
+ .get(id);
69
+ if (!job)
70
+ return;
71
+ const attempts = job.attempts + 1;
72
+ const willRetry = attempts <= this.retryConfig.maxRetries;
73
+ const status = willRetry ? 'failed' : 'failed';
74
+ const retryAt = willRetry ? (0, retry_js_1.nextRetryAt)(attempts, this.retryConfig) : 0;
75
+ this.db
76
+ .prepare(`UPDATE _driftdb_queue
77
+ SET status = ?, attempts = ?, nextRetryAt = ?, error = ?
78
+ WHERE id = ?`)
79
+ .run(status, attempts, retryAt, error.slice(0, 1000), id);
80
+ }
81
+ resetStuck() {
82
+ const staleThreshold = Date.now() - 5 * 60 * 1000;
83
+ this.db
84
+ .prepare(`UPDATE _driftdb_queue
85
+ SET status = 'pending', nextRetryAt = 0
86
+ WHERE status = 'processing' AND createdAt < ?`)
87
+ .run(staleThreshold);
88
+ }
89
+ pendingCount() {
90
+ const row = this.db
91
+ .prepare(`SELECT COUNT(*) as cnt FROM _driftdb_queue WHERE status IN ('pending', 'processing', 'failed')`)
92
+ .get();
93
+ return row.cnt;
94
+ }
95
+ purgeCompleted(olderThanMs = 24 * 60 * 60 * 1000) {
96
+ const cutoff = Date.now() - olderThanMs;
97
+ this.db
98
+ .prepare(`DELETE FROM _driftdb_queue WHERE status = 'done' AND createdAt < ?`)
99
+ .run(cutoff);
100
+ }
101
+ hasPendingOfType(type) {
102
+ const row = this.db
103
+ .prepare(`SELECT 1 FROM _driftdb_queue WHERE type = ? AND status IN ('pending', 'processing', 'failed') LIMIT 1`)
104
+ .get(type);
105
+ return row !== undefined;
106
+ }
107
+ }
108
+ exports.SyncQueue = SyncQueue;
109
+ //# sourceMappingURL=queue.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"queue.js","sourceRoot":"","sources":["../../src/queue/queue.ts"],"names":[],"mappings":";;;AAEA,0CAA4C;AAC5C,gDAAgD;AAGhD,MAAM,MAAM,GAAG;;;;;;;;;;;;CAYd,CAAC;AAEF,MAAa,SAAS;IAIpB,YAAY,EAAqB,EAAE,WAAwB;QACzD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,IAAqB,EAAE,OAAe;QAC5C,MAAM,GAAG,GAAY;YACnB,EAAE,EAAE,IAAA,kBAAU,GAAE;YAChB,IAAI;YACJ,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YAChC,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,CAAC;YACX,WAAW,EAAE,CAAC;YACd,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,KAAK,EAAE,IAAI;SACZ,CAAC;QAEF,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;yCACiC,CAClC;aACA,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QAE3G,OAAO,GAAG,CAAC;IACb,CAAC;IAED,OAAO,CAAC,KAAK,GAAG,CAAC;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;;;iBAIS,CACV;aACA,GAAG,CAAC,GAAG,EAAE,KAAK,CAAc,CAAC;QAEhC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,gEAAgE,YAAY,GAAG,CAAC;aACxF,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;QAEf,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,YAAqB,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,QAAQ,CAAC,EAAU;QACjB,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,sEAAsE,CAAC;aAC/E,GAAG,CAAC,EAAE,CAAC,CAAC;IACb,CAAC;IAED,UAAU,CAAC,EAAU,EAAE,KAAa;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,kDAAkD,CAAC;aAC3D,GAAG,CAAC,EAAE,CAA0C,CAAC;QAEpD,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClC,MAAM,SAAS,GAAG,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC/C,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAA,sBAAW,EAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAExE,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;sBAEc,CACf;aACA,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,UAAU;QACR,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;QAClD,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;uDAE+C,CAChD;aACA,GAAG,CAAC,cAAc,CAAC,CAAC;IACzB,CAAC;IAED,YAAY;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CACN,gGAAgG,CACjG;aACA,GAAG,EAAqB,CAAC;QAC5B,OAAO,GAAG,CAAC,GAAG,CAAC;IACjB,CAAC;IAED,cAAc,CAAC,WAAW,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;QAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;QACxC,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,oEAAoE,CAAC;aAC7E,GAAG,CAAC,MAAM,CAAC,CAAC;IACjB,CAAC;IAED,gBAAgB,CAAC,IAAqB;QACpC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CACN,uGAAuG,CACxG;aACA,GAAG,CAAC,IAAI,CAA8B,CAAC;QAC1C,OAAO,GAAG,KAAK,SAAS,CAAC;IAC3B,CAAC;CACF;AArHD,8BAqHC"}
@@ -0,0 +1,21 @@
1
+ import type { S3Config, SyncManifest } from '../types.js';
2
+ export interface S3UploadOptions {
3
+ compress?: boolean;
4
+ encryptionKey?: string;
5
+ }
6
+ export declare class S3Adapter {
7
+ private readonly client;
8
+ private readonly bucket;
9
+ private readonly prefix;
10
+ constructor(config: S3Config);
11
+ private key;
12
+ upload(path: string, data: Buffer, options?: S3UploadOptions): Promise<void>;
13
+ download(path: string, options?: S3UploadOptions): Promise<Buffer>;
14
+ exists(path: string): Promise<boolean>;
15
+ listKeys(prefix: string): Promise<string[]>;
16
+ putManifest(nodeId: string, manifest: SyncManifest): Promise<void>;
17
+ getManifest(nodeId: string): Promise<SyncManifest | null>;
18
+ logKey(nodeId: string, fromSeq: number, toSeq: number): string;
19
+ snapshotKey(nodeId: string, timestamp: number): string;
20
+ }
21
+ //# sourceMappingURL=s3-adapter.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"s3-adapter.d.ts","sourceRoot":"","sources":["../../src/storage/s3-adapter.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAI1D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEpB,MAAM,EAAE,QAAQ;IAsB5B,OAAO,CAAC,GAAG;IAIL,MAAM,CACV,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,IAAI,CAAC;IAyCV,QAAQ,CACZ,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC;IA0BZ,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWtC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA0B3C,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlE,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAO/D,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM;IAI9D,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;CAGvD"}
@@ -0,0 +1,133 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.S3Adapter = void 0;
4
+ const client_s3_1 = require("@aws-sdk/client-s3");
5
+ const lib_storage_1 = require("@aws-sdk/lib-storage");
6
+ const stream_1 = require("stream");
7
+ const compress_js_1 = require("../utils/compress.js");
8
+ const crypto_js_1 = require("../utils/crypto.js");
9
+ class S3Adapter {
10
+ constructor(config) {
11
+ const clientConfig = {
12
+ region: config.region,
13
+ };
14
+ if (config.accessKeyId && config.secretAccessKey) {
15
+ clientConfig.credentials = {
16
+ accessKeyId: config.accessKeyId,
17
+ secretAccessKey: config.secretAccessKey,
18
+ };
19
+ }
20
+ if (config.endpoint) {
21
+ clientConfig.endpoint = config.endpoint;
22
+ clientConfig.forcePathStyle = config.forcePathStyle ?? true;
23
+ }
24
+ this.client = new client_s3_1.S3Client(clientConfig);
25
+ this.bucket = config.bucket;
26
+ this.prefix = config.prefix ? config.prefix.replace(/\/$/, '') : 'driftdb';
27
+ }
28
+ key(path) {
29
+ return `${this.prefix}/${path}`;
30
+ }
31
+ async upload(path, data, options = {}) {
32
+ let payload = data;
33
+ if (options.compress) {
34
+ payload = await (0, compress_js_1.compress)(payload);
35
+ }
36
+ if (options.encryptionKey) {
37
+ payload = (0, crypto_js_1.encrypt)(payload, options.encryptionKey);
38
+ }
39
+ const contentEncoding = options.compress ? 'gzip' : undefined;
40
+ const metadata = {};
41
+ if (options.compress)
42
+ metadata['x-driftdb-compressed'] = '1';
43
+ if (options.encryptionKey)
44
+ metadata['x-driftdb-encrypted'] = '1';
45
+ if (payload.length > 5 * 1024 * 1024) {
46
+ const upload = new lib_storage_1.Upload({
47
+ client: this.client,
48
+ params: {
49
+ Bucket: this.bucket,
50
+ Key: this.key(path),
51
+ Body: stream_1.Readable.from(payload),
52
+ ContentEncoding: contentEncoding,
53
+ Metadata: metadata,
54
+ },
55
+ });
56
+ await upload.done();
57
+ }
58
+ else {
59
+ await this.client.send(new client_s3_1.PutObjectCommand({
60
+ Bucket: this.bucket,
61
+ Key: this.key(path),
62
+ Body: payload,
63
+ ContentEncoding: contentEncoding,
64
+ Metadata: metadata,
65
+ }));
66
+ }
67
+ }
68
+ async download(path, options = {}) {
69
+ const response = await this.client.send(new client_s3_1.GetObjectCommand({ Bucket: this.bucket, Key: this.key(path) }));
70
+ if (!response.Body) {
71
+ throw new Error(`Empty response body for key: ${path}`);
72
+ }
73
+ const chunks = [];
74
+ for await (const chunk of response.Body) {
75
+ chunks.push(Buffer.from(chunk));
76
+ }
77
+ let data = Buffer.concat(chunks);
78
+ if (options.encryptionKey) {
79
+ data = Buffer.from((0, crypto_js_1.decrypt)(data, options.encryptionKey));
80
+ }
81
+ if (options.compress) {
82
+ data = Buffer.from(await (0, compress_js_1.decompress)(data));
83
+ }
84
+ return data;
85
+ }
86
+ async exists(path) {
87
+ try {
88
+ await this.client.send(new client_s3_1.HeadObjectCommand({ Bucket: this.bucket, Key: this.key(path) }));
89
+ return true;
90
+ }
91
+ catch {
92
+ return false;
93
+ }
94
+ }
95
+ async listKeys(prefix) {
96
+ const fullPrefix = this.key(prefix);
97
+ const keys = [];
98
+ let continuationToken;
99
+ do {
100
+ const response = await this.client.send(new client_s3_1.ListObjectsV2Command({
101
+ Bucket: this.bucket,
102
+ Prefix: fullPrefix,
103
+ ContinuationToken: continuationToken,
104
+ }));
105
+ for (const obj of response.Contents ?? []) {
106
+ if (obj.Key) {
107
+ keys.push(obj.Key.slice(this.prefix.length + 1));
108
+ }
109
+ }
110
+ continuationToken = response.NextContinuationToken;
111
+ } while (continuationToken);
112
+ return keys;
113
+ }
114
+ async putManifest(nodeId, manifest) {
115
+ const data = Buffer.from(JSON.stringify(manifest), 'utf8');
116
+ await this.upload(`nodes/${nodeId}/manifest.json`, data);
117
+ }
118
+ async getManifest(nodeId) {
119
+ const path = `nodes/${nodeId}/manifest.json`;
120
+ if (!(await this.exists(path)))
121
+ return null;
122
+ const data = await this.download(path);
123
+ return JSON.parse(data.toString('utf8'));
124
+ }
125
+ logKey(nodeId, fromSeq, toSeq) {
126
+ return `nodes/${nodeId}/logs/${String(fromSeq).padStart(12, '0')}-${String(toSeq).padStart(12, '0')}.json`;
127
+ }
128
+ snapshotKey(nodeId, timestamp) {
129
+ return `nodes/${nodeId}/snapshots/${timestamp}.sqlite`;
130
+ }
131
+ }
132
+ exports.S3Adapter = S3Adapter;
133
+ //# sourceMappingURL=s3-adapter.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"s3-adapter.js","sourceRoot":"","sources":["../../src/storage/s3-adapter.ts"],"names":[],"mappings":";;;AAAA,kDAO4B;AAC5B,sDAA8C;AAC9C,mCAAkC;AAElC,sDAA4D;AAC5D,kDAAsD;AAOtD,MAAa,SAAS;IAKpB,YAAY,MAAgB;QAC1B,MAAM,YAAY,GAAmB;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC;QAEF,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YACjD,YAAY,CAAC,WAAW,GAAG;gBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;aACxC,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,YAAY,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YACxC,YAAY,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,oBAAQ,CAAC,YAAY,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7E,CAAC;IAEO,GAAG,CAAC,IAAY;QACtB,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,MAAM,CACV,IAAY,EACZ,IAAY,EACZ,UAA2B,EAAE;QAE7B,IAAI,OAAO,GAAG,IAAI,CAAC;QAEnB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,OAAO,GAAG,MAAM,IAAA,sBAAQ,EAAC,OAAO,CAAC,CAAC;QACpC,CAAC;QAED,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAC1B,OAAO,GAAG,IAAA,mBAAO,EAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9D,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,QAAQ;YAAE,QAAQ,CAAC,sBAAsB,CAAC,GAAG,GAAG,CAAC;QAC7D,IAAI,OAAO,CAAC,aAAa;YAAE,QAAQ,CAAC,qBAAqB,CAAC,GAAG,GAAG,CAAC;QAEjE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,IAAI,oBAAM,CAAC;gBACxB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,iBAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;oBAC5B,eAAe,EAAE,eAAe;oBAChC,QAAQ,EAAE,QAAQ;iBACnB;aACF,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACpB,IAAI,4BAAgB,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACnB,IAAI,EAAE,OAAO;gBACb,eAAe,EAAE,eAAe;gBAChC,QAAQ,EAAE,QAAQ;aACnB,CAAC,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,IAAY,EACZ,UAA2B,EAAE;QAE7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACrC,IAAI,4BAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CACnE,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAAiC,EAAE,CAAC;YACrE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEjC,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAC1B,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAA,mBAAO,EAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAA,wBAAU,EAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAY;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACpB,IAAI,6BAAiB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CACpE,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAc;QAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,IAAI,iBAAqC,CAAC;QAE1C,GAAG,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACrC,IAAI,gCAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,UAAU;gBAClB,iBAAiB,EAAE,iBAAiB;aACrC,CAAC,CACH,CAAC;YAEF,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;gBAC1C,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;oBACZ,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;YAED,iBAAiB,GAAG,QAAQ,CAAC,qBAAqB,CAAC;QACrD,CAAC,QAAQ,iBAAiB,EAAE;QAE5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,QAAsB;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,MAAM,gBAAgB,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc;QAC9B,MAAM,IAAI,GAAG,SAAS,MAAM,gBAAgB,CAAC;QAC7C,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAiB,CAAC;IAC3D,CAAC;IAED,MAAM,CAAC,MAAc,EAAE,OAAe,EAAE,KAAa;QACnD,OAAO,SAAS,MAAM,SAAS,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC;IAC7G,CAAC;IAED,WAAW,CAAC,MAAc,EAAE,SAAiB;QAC3C,OAAO,SAAS,MAAM,cAAc,SAAS,SAAS,CAAC;IACzD,CAAC;CACF;AAjKD,8BAiKC"}
@@ -0,0 +1,15 @@
1
+ import type Database from 'better-sqlite3';
2
+ import type { ChangeLogEntry, LogBatch } from '../types.js';
3
+ export declare class ChangeLog {
4
+ private readonly db;
5
+ private readonly nodeId;
6
+ constructor(db: Database.Database, nodeId: string);
7
+ append(table: string, operation: ChangeLogEntry['operation'], data: Record<string, unknown> | null): number;
8
+ pendingEntries(limit: number): ChangeLogEntry[];
9
+ markSynced(fromSequence: number, toSequence: number): void;
10
+ pendingCount(): number;
11
+ latestSyncedSequence(): number;
12
+ buildBatch(entries: ChangeLogEntry[]): LogBatch;
13
+ purgeOldSynced(keepLatest?: number): void;
14
+ }
15
+ //# sourceMappingURL=change-log.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"change-log.d.ts","sourceRoot":"","sources":["../../src/sync/change-log.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAe5D,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEpB,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM;IAMjD,MAAM,CACJ,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,cAAc,CAAC,WAAW,CAAC,EACtC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GACnC,MAAM;IAWT,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE;IAQ/C,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAQ1D,YAAY,IAAI,MAAM;IAOtB,oBAAoB,IAAI,MAAM;IAO9B,UAAU,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,QAAQ;IAgB/C,cAAc,CAAC,UAAU,SAAO,GAAG,IAAI;CAaxC"}
@@ -0,0 +1,78 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ChangeLog = void 0;
4
+ const SCHEMA = `
5
+ CREATE TABLE IF NOT EXISTS _driftdb_log (
6
+ sequence INTEGER PRIMARY KEY AUTOINCREMENT,
7
+ timestamp INTEGER NOT NULL,
8
+ nodeId TEXT NOT NULL,
9
+ \`table\` TEXT NOT NULL,
10
+ operation TEXT NOT NULL,
11
+ data TEXT,
12
+ synced INTEGER NOT NULL DEFAULT 0
13
+ );
14
+ CREATE INDEX IF NOT EXISTS idx_driftdb_log_synced ON _driftdb_log (synced, sequence);
15
+ `;
16
+ class ChangeLog {
17
+ constructor(db, nodeId) {
18
+ this.db = db;
19
+ this.nodeId = nodeId;
20
+ db.exec(SCHEMA);
21
+ }
22
+ append(table, operation, data) {
23
+ const result = this.db
24
+ .prepare(`INSERT INTO _driftdb_log (timestamp, nodeId, \`table\`, operation, data, synced)
25
+ VALUES (?, ?, ?, ?, ?, 0)`)
26
+ .run(Date.now(), this.nodeId, table, operation, data ? JSON.stringify(data) : null);
27
+ return Number(result.lastInsertRowid);
28
+ }
29
+ pendingEntries(limit) {
30
+ return this.db
31
+ .prepare(`SELECT * FROM _driftdb_log WHERE synced = 0 ORDER BY sequence ASC LIMIT ?`)
32
+ .all(limit);
33
+ }
34
+ markSynced(fromSequence, toSequence) {
35
+ this.db
36
+ .prepare(`UPDATE _driftdb_log SET synced = 1 WHERE sequence >= ? AND sequence <= ?`)
37
+ .run(fromSequence, toSequence);
38
+ }
39
+ pendingCount() {
40
+ const row = this.db
41
+ .prepare(`SELECT COUNT(*) as cnt FROM _driftdb_log WHERE synced = 0`)
42
+ .get();
43
+ return row.cnt;
44
+ }
45
+ latestSyncedSequence() {
46
+ const row = this.db
47
+ .prepare(`SELECT MAX(sequence) as seq FROM _driftdb_log WHERE synced = 1`)
48
+ .get();
49
+ return row.seq ?? 0;
50
+ }
51
+ buildBatch(entries) {
52
+ return {
53
+ version: 1,
54
+ nodeId: this.nodeId,
55
+ fromSequence: entries[0]?.sequence ?? 0,
56
+ toSequence: entries[entries.length - 1]?.sequence ?? 0,
57
+ entries: entries.map((e) => ({
58
+ sequence: e.sequence,
59
+ timestamp: e.timestamp,
60
+ table: e.table,
61
+ operation: e.operation,
62
+ data: e.data ? JSON.parse(e.data) : null,
63
+ })),
64
+ };
65
+ }
66
+ purgeOldSynced(keepLatest = 5000) {
67
+ const row = this.db
68
+ .prepare(`SELECT sequence FROM _driftdb_log WHERE synced = 1 ORDER BY sequence DESC LIMIT 1 OFFSET ?`)
69
+ .get(keepLatest);
70
+ if (row) {
71
+ this.db
72
+ .prepare(`DELETE FROM _driftdb_log WHERE synced = 1 AND sequence <= ?`)
73
+ .run(row.sequence);
74
+ }
75
+ }
76
+ }
77
+ exports.ChangeLog = ChangeLog;
78
+ //# sourceMappingURL=change-log.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"change-log.js","sourceRoot":"","sources":["../../src/sync/change-log.ts"],"names":[],"mappings":";;;AAGA,MAAM,MAAM,GAAG;;;;;;;;;;;CAWd,CAAC;AAEF,MAAa,SAAS;IAIpB,YAAY,EAAqB,EAAE,MAAc;QAC/C,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,CACJ,KAAa,EACb,SAAsC,EACtC,IAAoC;QAEpC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE;aACnB,OAAO,CACN;mCAC2B,CAC5B;aACA,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEtF,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAED,cAAc,CAAC,KAAa;QAC1B,OAAO,IAAI,CAAC,EAAE;aACX,OAAO,CACN,2EAA2E,CAC5E;aACA,GAAG,CAAC,KAAK,CAAqB,CAAC;IACpC,CAAC;IAED,UAAU,CAAC,YAAoB,EAAE,UAAkB;QACjD,IAAI,CAAC,EAAE;aACJ,OAAO,CACN,0EAA0E,CAC3E;aACA,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IACnC,CAAC;IAED,YAAY;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,2DAA2D,CAAC;aACpE,GAAG,EAAqB,CAAC;QAC5B,OAAO,GAAG,CAAC,GAAG,CAAC;IACjB,CAAC;IAED,oBAAoB;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,gEAAgE,CAAC;aACzE,GAAG,EAA4B,CAAC;QACnC,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACtB,CAAC;IAED,UAAU,CAAC,OAAyB;QAClC,OAAO;YACL,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,CAAC;YACvC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,QAAQ,IAAI,CAAC;YACtD,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3B,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAA6B,CAAC,CAAC,CAAC,IAAI;aACtE,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;IAED,cAAc,CAAC,UAAU,GAAG,IAAI;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CACN,4FAA4F,CAC7F;aACA,GAAG,CAAC,UAAU,CAAqC,CAAC;QAEvD,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,EAAE;iBACJ,OAAO,CAAC,6DAA6D,CAAC;iBACtE,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;CACF;AApFD,8BAoFC"}
@@ -0,0 +1,31 @@
1
+ import type Database from 'better-sqlite3';
2
+ import type { DBConfig, SyncMetrics } from '../types.js';
3
+ import { SyncQueue } from '../queue/queue.js';
4
+ import { ChangeLog } from './change-log.js';
5
+ export declare class SyncEngine {
6
+ private readonly db;
7
+ private readonly config;
8
+ private readonly nodeId;
9
+ private readonly queue;
10
+ private readonly changeLog;
11
+ private readonly snapshotManager;
12
+ private readonly s3;
13
+ private timer;
14
+ private isProcessing;
15
+ private metrics;
16
+ constructor(db: Database.Database, nodeId: string, config: DBConfig);
17
+ getChangeLog(): ChangeLog;
18
+ getQueue(): SyncQueue;
19
+ start(): void;
20
+ stop(): void;
21
+ flush(): Promise<void>;
22
+ getMetrics(): Readonly<SyncMetrics>;
23
+ private getDbSizeBytes;
24
+ private tick;
25
+ private enqueuePendingLogs;
26
+ private processQueue;
27
+ private processJob;
28
+ private maybeSnapshot;
29
+ triggerSnapshot(): Promise<void>;
30
+ }
31
+ //# sourceMappingURL=engine.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/sync/engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAW,WAAW,EAA2C,MAAM,aAAa,CAAC;AAC3G,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAW5C,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAY;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAyB;IACzD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmB;IAEtC,OAAO,CAAC,KAAK,CAA+C;IAC5D,OAAO,CAAC,YAAY,CAAS;IAE7B,OAAO,CAAC,OAAO,CAQb;gBAEU,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ;IA4BnE,YAAY,IAAI,SAAS;IAIzB,QAAQ,IAAI,SAAS;IAIrB,KAAK,IAAI,IAAI;IAeb,IAAI,IAAI,IAAI;IAQN,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,UAAU,IAAI,QAAQ,CAAC,WAAW,CAAC;IAQnC,OAAO,CAAC,cAAc;YAUR,IAAI;YAcJ,kBAAkB;YAmBlB,YAAY;YAUZ,UAAU;YA6DV,aAAa;IAcrB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;CAavC"}