@raft-hlc-sync-protocol/raft-sync-lib 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -56
- package/README.zh-CN.md +124 -56
- package/dialects.js +286 -0
- package/index.js +5 -2
- package/package.json +2 -2
- package/sync-engine.js +20 -8
- package/sync-protocol.js +12 -0
package/README.md
CHANGED
|
@@ -33,74 +33,73 @@ cp -r sync-lib/ your-project/sync-lib/
|
|
|
33
33
|
|
|
34
34
|
### Step 1: Implement DatabaseAdapter
|
|
35
35
|
|
|
36
|
-
The adapter
|
|
36
|
+
The adapter provides basic query methods. Dialect-specific SQL (transactions, upsert, schema, triggers) is handled automatically via the `dialect` parameter.
|
|
37
|
+
|
|
38
|
+
**SQLite:**
|
|
37
39
|
|
|
38
40
|
```js
|
|
39
41
|
import Database from 'better-sqlite3';
|
|
40
|
-
import { getInfraSchemaSQL } from 'sync-lib';
|
|
41
42
|
|
|
42
43
|
const rawDb = new Database(':memory:');
|
|
43
44
|
rawDb.pragma('journal_mode = WAL');
|
|
44
45
|
|
|
45
46
|
const db = {
|
|
46
|
-
// --- Query (required) ---
|
|
47
47
|
run(sql, params = []) { return rawDb.prepare(sql).run(...params); },
|
|
48
48
|
get(sql, params = []) { return rawDb.prepare(sql).get(...params) || null; },
|
|
49
49
|
all(sql, params = []) { return rawDb.prepare(sql).all(...params); },
|
|
50
50
|
exec(sql) { rawDb.exec(sql); },
|
|
51
|
+
};
|
|
52
|
+
```
|
|
51
53
|
|
|
52
|
-
|
|
53
|
-
beginTransaction() { rawDb.exec('BEGIN IMMEDIATE'); },
|
|
54
|
-
commit() { rawDb.exec('COMMIT'); },
|
|
55
|
-
rollback() { rawDb.exec('ROLLBACK'); },
|
|
54
|
+
**PostgreSQL:**
|
|
56
55
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const ph = columns.map(() => '?').join(', ');
|
|
61
|
-
return `INSERT OR REPLACE INTO ${table} (${cols}) VALUES (${ph})`;
|
|
62
|
-
},
|
|
56
|
+
```js
|
|
57
|
+
import pg from 'pg';
|
|
58
|
+
const pool = new pg.Pool({ connectionString: '...' });
|
|
63
59
|
|
|
64
|
-
|
|
65
|
-
|
|
60
|
+
function pgConvert(sql) {
|
|
61
|
+
let i = 0;
|
|
62
|
+
return sql.replace(/\?/g, () => '$' + (++i));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const db = {
|
|
66
|
+
async run(sql, p = []) { const r = await pool.query(pgConvert(sql), p); return { changes: r.rowCount }; },
|
|
67
|
+
async get(sql, p = []) { const r = await pool.query(pgConvert(sql), p); return r.rows[0] || null; },
|
|
68
|
+
async all(sql, p = []) { const r = await pool.query(pgConvert(sql), p); return r.rows; },
|
|
69
|
+
async exec(sql) { await pool.query(sql); },
|
|
66
70
|
};
|
|
67
71
|
```
|
|
68
72
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
#### Why `upsertSQL`?
|
|
73
|
+
**MySQL:**
|
|
72
74
|
|
|
73
|
-
|
|
75
|
+
```js
|
|
76
|
+
import mysql from 'mysql2/promise';
|
|
77
|
+
const pool = await mysql.createPool({ host: '...', user: '...', password: '...', database: '...' });
|
|
74
78
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
79
|
+
const db = {
|
|
80
|
+
async run(sql, p = []) { const [r] = await pool.execute(sql, p); return { changes: r.affectedRows }; },
|
|
81
|
+
async get(sql, p = []) { const [rows] = await pool.execute(sql, p); return rows[0] || null; },
|
|
82
|
+
async all(sql, p = []) { const [rows] = await pool.execute(sql, p); return rows; },
|
|
83
|
+
async exec(sql) { await pool.query(sql); },
|
|
84
|
+
};
|
|
85
|
+
```
|
|
80
86
|
|
|
81
|
-
|
|
82
|
-
- `table` — target table name
|
|
83
|
-
- `columns` — all columns to insert (including keys)
|
|
84
|
-
- `keyColumns` — primary key columns, used for conflict detection. On conflict, all non-key columns are updated.
|
|
87
|
+
That's it — just 4 methods. The `dialect` parameter (see Step 2) fills in everything else automatically.
|
|
85
88
|
|
|
86
|
-
|
|
87
|
-
1. Syncing remote data into business tables
|
|
88
|
-
2. Writing/updating tombstones (`_tombstones` table)
|
|
89
|
-
3. Updating peer sync positions (`_sync_peers` table)
|
|
89
|
+
> For custom databases, see [Full DatabaseAdapter Interface](#databaseadapter-interface) and [Custom Adapter Examples](#custom-adapter-examples) below.
|
|
90
90
|
|
|
91
|
-
####
|
|
91
|
+
#### Built-in dialect support
|
|
92
92
|
|
|
93
|
-
sync-lib
|
|
93
|
+
sync-lib has built-in support for **SQLite**, **PostgreSQL**, and **MySQL**. When you specify `dialect: 'sqlite'` (or `'postgresql'` / `'mysql'`), all dialect-specific methods are auto-filled on your adapter:
|
|
94
94
|
|
|
95
|
-
|
|
95
|
+
- `beginTransaction()` / `commit()` / `rollback()` — transaction control
|
|
96
|
+
- `upsertSQL(table, columns, keyColumns)` — conflict-aware insert/update SQL
|
|
97
|
+
- `infraSchemaSQL()` — DDL for internal tables (`_sync_log`, `_sync_peers`, `_tombstones`)
|
|
98
|
+
- `triggersSQL(tableName, def)` / `dropTriggersSQL(tableName)` — auto-logging triggers
|
|
96
99
|
|
|
97
|
-
|
|
98
|
-
|--|--------|------------|-------|
|
|
99
|
-
| Auto-increment | `AUTOINCREMENT` | `BIGSERIAL` | `AUTO_INCREMENT` |
|
|
100
|
-
| Timestamp default | `datetime('now')` | `NOW()` | `CURRENT_TIMESTAMP` |
|
|
101
|
-
| String type | `TEXT` | `TEXT` | `VARCHAR(255)` |
|
|
100
|
+
**If you don't specify a dialect (or specify an unsupported one), you must implement ALL of these methods yourself.** Missing methods will throw an error at construction time.
|
|
102
101
|
|
|
103
|
-
|
|
102
|
+
You can override any auto-filled method by defining it on your `db` adapter — your explicit implementation always takes priority over the built-in dialect.
|
|
104
103
|
|
|
105
104
|
### Step 2: Create engine and register tables
|
|
106
105
|
|
|
@@ -110,6 +109,7 @@ import { SyncEngine } from 'sync-lib';
|
|
|
110
109
|
const engine = new SyncEngine({
|
|
111
110
|
nodeId: 'node-1',
|
|
112
111
|
db,
|
|
112
|
+
dialect: 'sqlite', // or 'postgresql' or 'mysql'
|
|
113
113
|
|
|
114
114
|
// Required callbacks
|
|
115
115
|
onSendToPeer: (peerId, msg) => yourTransport.send(peerId, msg),
|
|
@@ -130,8 +130,7 @@ engine.registerTable('users', {
|
|
|
130
130
|
});
|
|
131
131
|
|
|
132
132
|
engine.initSchema(); // Creates _sync_log, _sync_peers, _tombstones
|
|
133
|
-
engine.initTriggers(); //
|
|
134
|
-
// Skip for PostgreSQL/MySQL, use logChange() instead
|
|
133
|
+
engine.initTriggers(); // Creates auto-logging triggers (dialect-aware)
|
|
135
134
|
engine.start(); // Starts leader election
|
|
136
135
|
```
|
|
137
136
|
|
|
@@ -162,7 +161,7 @@ setInterval(() => engine.tickCleanup(), 3600_000); // Cleanup old logs every 1
|
|
|
162
161
|
|
|
163
162
|
### Step 5: Write data
|
|
164
163
|
|
|
165
|
-
|
|
164
|
+
With `dialect` set and `initTriggers()` called, all databases work the same way — triggers automatically log changes:
|
|
166
165
|
|
|
167
166
|
```js
|
|
168
167
|
const ts = engine.hlc.tick();
|
|
@@ -171,7 +170,7 @@ db.run('INSERT INTO users (user_id, name, _hlc) VALUES (?, ?, ?)',
|
|
|
171
170
|
engine.notifyLocalWrite(); // pushes to peers
|
|
172
171
|
```
|
|
173
172
|
|
|
174
|
-
**
|
|
173
|
+
**Without triggers** (manual logChange fallback):
|
|
175
174
|
|
|
176
175
|
```js
|
|
177
176
|
import { logChange } from 'sync-lib';
|
|
@@ -232,7 +231,11 @@ The leader executes the request via the `onExecuteProxyRequest` callback.
|
|
|
232
231
|
|
|
233
232
|
---
|
|
234
233
|
|
|
235
|
-
##
|
|
234
|
+
## Custom Adapter Examples
|
|
235
|
+
|
|
236
|
+
When using a built-in dialect (`sqlite`, `postgresql`, `mysql`), you only need `run/get/all/exec` on your adapter — everything else is auto-filled. The examples below show what the built-in dialect provides, and are useful as reference if you need to customize or use an unsupported database.
|
|
237
|
+
|
|
238
|
+
### PostgreSQL (full manual adapter)
|
|
236
239
|
|
|
237
240
|
```js
|
|
238
241
|
import pg from 'pg';
|
|
@@ -291,12 +294,40 @@ const db = {
|
|
|
291
294
|
_hlc TEXT NOT NULL, PRIMARY KEY (table_name, row_key))`,
|
|
292
295
|
];
|
|
293
296
|
},
|
|
297
|
+
|
|
298
|
+
// Optional: inject PG triggers so initTriggers() works
|
|
299
|
+
triggersSQL(tableName, def) {
|
|
300
|
+
const keyExpr = def.keyColumns.map(c => `'${c}', NEW.${c}`).join(', ');
|
|
301
|
+
const dataExpr = def.dataColumns.map(c => `'${c}', NEW.${c}`).join(', ');
|
|
302
|
+
return [
|
|
303
|
+
`CREATE OR REPLACE FUNCTION _sync_trg_${tableName}_fn() RETURNS TRIGGER AS $$
|
|
304
|
+
BEGIN
|
|
305
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
306
|
+
VALUES ('${tableName}', TG_OP,
|
|
307
|
+
json_build_object(${keyExpr})::text,
|
|
308
|
+
CASE WHEN TG_OP = 'DELETE' THEN NULL
|
|
309
|
+
ELSE json_build_object(${dataExpr})::text END,
|
|
310
|
+
COALESCE(NEW._hlc, OLD._hlc));
|
|
311
|
+
RETURN COALESCE(NEW, OLD);
|
|
312
|
+
END; $$ LANGUAGE plpgsql`,
|
|
313
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName} ON ${tableName}`,
|
|
314
|
+
`CREATE TRIGGER _sync_trg_${tableName}
|
|
315
|
+
AFTER INSERT OR UPDATE OR DELETE ON ${tableName}
|
|
316
|
+
FOR EACH ROW EXECUTE FUNCTION _sync_trg_${tableName}_fn()`,
|
|
317
|
+
];
|
|
318
|
+
},
|
|
319
|
+
dropTriggersSQL(tableName) {
|
|
320
|
+
return [
|
|
321
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName} ON ${tableName}`,
|
|
322
|
+
`DROP FUNCTION IF EXISTS _sync_trg_${tableName}_fn`,
|
|
323
|
+
];
|
|
324
|
+
},
|
|
294
325
|
};
|
|
295
326
|
```
|
|
296
327
|
|
|
297
328
|
---
|
|
298
329
|
|
|
299
|
-
|
|
330
|
+
### MySQL (full manual adapter)
|
|
300
331
|
|
|
301
332
|
```js
|
|
302
333
|
import mysql from 'mysql2/promise';
|
|
@@ -347,6 +378,34 @@ const db = {
|
|
|
347
378
|
_hlc VARCHAR(64) NOT NULL, PRIMARY KEY (table_name, row_key(255)))`,
|
|
348
379
|
];
|
|
349
380
|
},
|
|
381
|
+
|
|
382
|
+
// Optional: inject MySQL triggers so initTriggers() works
|
|
383
|
+
triggersSQL(tableName, def) {
|
|
384
|
+
const keyExpr = def.keyColumns.map(c => `'${c}', NEW.${c}`).join(', ');
|
|
385
|
+
const dataExpr = def.dataColumns.map(c => `'${c}', NEW.${c}`).join(', ');
|
|
386
|
+
const delKeyExpr = def.keyColumns.map(c => `'${c}', OLD.${c}`).join(', ');
|
|
387
|
+
return [
|
|
388
|
+
`CREATE TRIGGER _sync_trg_${tableName}_insert
|
|
389
|
+
AFTER INSERT ON ${tableName} FOR EACH ROW
|
|
390
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
391
|
+
VALUES ('${tableName}', 'INSERT', JSON_OBJECT(${keyExpr}), JSON_OBJECT(${dataExpr}), NEW._hlc)`,
|
|
392
|
+
`CREATE TRIGGER _sync_trg_${tableName}_update
|
|
393
|
+
AFTER UPDATE ON ${tableName} FOR EACH ROW
|
|
394
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
395
|
+
VALUES ('${tableName}', 'UPDATE', JSON_OBJECT(${keyExpr}), JSON_OBJECT(${dataExpr}), NEW._hlc)`,
|
|
396
|
+
`CREATE TRIGGER _sync_trg_${tableName}_delete
|
|
397
|
+
AFTER DELETE ON ${tableName} FOR EACH ROW
|
|
398
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
399
|
+
VALUES ('${tableName}', 'DELETE', JSON_OBJECT(${delKeyExpr}), NULL, OLD._hlc)`,
|
|
400
|
+
];
|
|
401
|
+
},
|
|
402
|
+
dropTriggersSQL(tableName) {
|
|
403
|
+
return [
|
|
404
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName}_insert`,
|
|
405
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName}_update`,
|
|
406
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName}_delete`,
|
|
407
|
+
];
|
|
408
|
+
},
|
|
350
409
|
};
|
|
351
410
|
```
|
|
352
411
|
|
|
@@ -385,14 +444,17 @@ const db = {
|
|
|
385
444
|
### Standalone utilities
|
|
386
445
|
|
|
387
446
|
```js
|
|
388
|
-
import { HLC, logChange, getInfraSchemaSQL, getTriggersSQL } from 'sync-lib';
|
|
447
|
+
import { HLC, DIALECTS, applyDialect, logChange, getInfraSchemaSQL, getTriggersSQL } from 'sync-lib';
|
|
389
448
|
```
|
|
390
449
|
|
|
391
450
|
| Export | Description |
|
|
392
451
|
|--------|-------------|
|
|
393
452
|
| `HLC` | Hybrid Logical Clock class |
|
|
394
453
|
| `LeaderElection` | Leader election state machine |
|
|
395
|
-
| `
|
|
454
|
+
| `DIALECTS` | Built-in dialect definitions (sqlite, postgresql, mysql) |
|
|
455
|
+
| `applyDialect(db, name)` | Apply a dialect's methods to a db adapter |
|
|
456
|
+
| `validateDialectMethods(db)` | Validate all required dialect methods exist |
|
|
457
|
+
| `logChange(db, table, op, key, data, hlc)` | Manually log a change (alternative to triggers) |
|
|
396
458
|
| `getInfraSchemaSQL()` | Built-in SQLite DDL for infrastructure tables |
|
|
397
459
|
| `getTriggersSQL(table, def)` | Generate SQLite sync triggers |
|
|
398
460
|
| `applyEntries(db, entries, hlc, registry)` | Apply remote entries idempotently |
|
|
@@ -401,26 +463,31 @@ import { HLC, logChange, getInfraSchemaSQL, getTriggersSQL } from 'sync-lib';
|
|
|
401
463
|
|
|
402
464
|
## DatabaseAdapter Interface
|
|
403
465
|
|
|
404
|
-
Every adapter must implement these methods:
|
|
405
|
-
|
|
406
466
|
```
|
|
407
467
|
┌───────────────────────────────────────────────────────────────┐
|
|
468
|
+
│ Always required (you must implement): │
|
|
469
|
+
│ │
|
|
408
470
|
│ db.run(sql, params) → { changes: number } │
|
|
409
471
|
│ db.get(sql, params) → Object | null │
|
|
410
472
|
│ db.all(sql, params) → Object[] │
|
|
411
473
|
│ db.exec(sql) → void │
|
|
412
474
|
│ │
|
|
475
|
+
├───────────────────────────────────────────────────────────────┤
|
|
476
|
+
│ Auto-filled by dialect (or manually required): │
|
|
477
|
+
│ │
|
|
413
478
|
│ db.beginTransaction() → void │
|
|
414
479
|
│ db.commit() → void │
|
|
415
480
|
│ db.rollback() → void │
|
|
416
|
-
│ │
|
|
417
481
|
│ db.upsertSQL(table, cols, keyColumns) → string │
|
|
418
|
-
│
|
|
419
|
-
│ db.
|
|
482
|
+
│ db.infraSchemaSQL() → string[] │
|
|
483
|
+
│ db.triggersSQL(table, def) → string[] │
|
|
484
|
+
│ db.dropTriggersSQL(table) → string[] │
|
|
420
485
|
└───────────────────────────────────────────────────────────────┘
|
|
421
486
|
```
|
|
422
487
|
|
|
423
|
-
|
|
488
|
+
With `dialect: 'sqlite' | 'postgresql' | 'mysql'`, the bottom 7 methods are auto-filled. Without a dialect, you must implement all of them yourself.
|
|
489
|
+
|
|
490
|
+
SQL uses `?` placeholders. If your database uses `$1/$2` (e.g. PostgreSQL), convert inside `run/get/all`.
|
|
424
491
|
|
|
425
492
|
---
|
|
426
493
|
|
|
@@ -446,7 +513,8 @@ sync-lib/
|
|
|
446
513
|
├── hlc.js ← Hybrid Logical Clock (pure algorithm)
|
|
447
514
|
├── leader-election.js ← Raft-simplified election state machine
|
|
448
515
|
├── sync-protocol.js ← Message codec + idempotent data apply
|
|
449
|
-
|
|
516
|
+
├── sync-engine.js ← Main engine (composes all above)
|
|
517
|
+
└── dialects.js ← Built-in SQL dialects (SQLite/PG/MySQL)
|
|
450
518
|
```
|
|
451
519
|
|
|
452
520
|
**Design principles**:
|
package/README.zh-CN.md
CHANGED
|
@@ -33,74 +33,73 @@ cp -r sync-lib/ your-project/sync-lib/
|
|
|
33
33
|
|
|
34
34
|
### 第 1 步:实现 DatabaseAdapter
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
适配器只需提供基础查询方法。方言相关的 SQL(事务、upsert、schema、触发器)由 `dialect` 参数自动处理。
|
|
37
|
+
|
|
38
|
+
**SQLite:**
|
|
37
39
|
|
|
38
40
|
```js
|
|
39
41
|
import Database from 'better-sqlite3';
|
|
40
|
-
import { getInfraSchemaSQL } from 'sync-lib';
|
|
41
42
|
|
|
42
43
|
const rawDb = new Database(':memory:');
|
|
43
44
|
rawDb.pragma('journal_mode = WAL');
|
|
44
45
|
|
|
45
46
|
const db = {
|
|
46
|
-
// --- 查询(必需)---
|
|
47
47
|
run(sql, params = []) { return rawDb.prepare(sql).run(...params); },
|
|
48
48
|
get(sql, params = []) { return rawDb.prepare(sql).get(...params) || null; },
|
|
49
49
|
all(sql, params = []) { return rawDb.prepare(sql).all(...params); },
|
|
50
50
|
exec(sql) { rawDb.exec(sql); },
|
|
51
|
+
};
|
|
52
|
+
```
|
|
51
53
|
|
|
52
|
-
|
|
53
|
-
beginTransaction() { rawDb.exec('BEGIN IMMEDIATE'); },
|
|
54
|
-
commit() { rawDb.exec('COMMIT'); },
|
|
55
|
-
rollback() { rawDb.exec('ROLLBACK'); },
|
|
54
|
+
**PostgreSQL:**
|
|
56
55
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const ph = columns.map(() => '?').join(', ');
|
|
61
|
-
return `INSERT OR REPLACE INTO ${table} (${cols}) VALUES (${ph})`;
|
|
62
|
-
},
|
|
56
|
+
```js
|
|
57
|
+
import pg from 'pg';
|
|
58
|
+
const pool = new pg.Pool({ connectionString: '...' });
|
|
63
59
|
|
|
64
|
-
|
|
65
|
-
|
|
60
|
+
function pgConvert(sql) {
|
|
61
|
+
let i = 0;
|
|
62
|
+
return sql.replace(/\?/g, () => '$' + (++i));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const db = {
|
|
66
|
+
async run(sql, p = []) { const r = await pool.query(pgConvert(sql), p); return { changes: r.rowCount }; },
|
|
67
|
+
async get(sql, p = []) { const r = await pool.query(pgConvert(sql), p); return r.rows[0] || null; },
|
|
68
|
+
async all(sql, p = []) { const r = await pool.query(pgConvert(sql), p); return r.rows; },
|
|
69
|
+
async exec(sql) { await pool.query(sql); },
|
|
66
70
|
};
|
|
67
71
|
```
|
|
68
72
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
#### 为什么需要 `upsertSQL`?
|
|
73
|
+
**MySQL:**
|
|
72
74
|
|
|
73
|
-
|
|
75
|
+
```js
|
|
76
|
+
import mysql from 'mysql2/promise';
|
|
77
|
+
const pool = await mysql.createPool({ host: '...', user: '...', password: '...', database: '...' });
|
|
74
78
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
79
|
+
const db = {
|
|
80
|
+
async run(sql, p = []) { const [r] = await pool.execute(sql, p); return { changes: r.affectedRows }; },
|
|
81
|
+
async get(sql, p = []) { const [rows] = await pool.execute(sql, p); return rows[0] || null; },
|
|
82
|
+
async all(sql, p = []) { const [rows] = await pool.execute(sql, p); return rows; },
|
|
83
|
+
async exec(sql) { await pool.query(sql); },
|
|
84
|
+
};
|
|
85
|
+
```
|
|
80
86
|
|
|
81
|
-
|
|
82
|
-
- `table` — 目标表名
|
|
83
|
-
- `columns` — 要插入的所有列(包括主键列)
|
|
84
|
-
- `keyColumns` — 主键列,用于冲突检测。冲突时更新所有非主键列。
|
|
87
|
+
只需 4 个方法。`dialect` 参数(见第 2 步)会自动填充其余所有方言方法。
|
|
85
88
|
|
|
86
|
-
|
|
87
|
-
1. 将远程数据同步到业务表
|
|
88
|
-
2. 写入/更新墓碑记录(`_tombstones` 表)
|
|
89
|
-
3. 更新对等节点同步位置(`_sync_peers` 表)
|
|
89
|
+
> 自定义数据库请参见下方 [DatabaseAdapter 接口](#databaseadapter-接口) 和 [自定义适配器示例](#自定义适配器示例)。
|
|
90
90
|
|
|
91
|
-
####
|
|
91
|
+
#### 内置方言支持
|
|
92
92
|
|
|
93
|
-
sync-lib
|
|
93
|
+
sync-lib 内置 **SQLite**、**PostgreSQL** 和 **MySQL** 三种方言。指定 `dialect: 'sqlite'`(或 `'postgresql'` / `'mysql'`)后,以下方法会自动填充到你的适配器上:
|
|
94
94
|
|
|
95
|
-
|
|
95
|
+
- `beginTransaction()` / `commit()` / `rollback()` — 事务控制
|
|
96
|
+
- `upsertSQL(table, columns, keyColumns)` — 冲突感知的 insert/update SQL
|
|
97
|
+
- `infraSchemaSQL()` — 内部表(`_sync_log`、`_sync_peers`、`_tombstones`)的 DDL
|
|
98
|
+
- `triggersSQL(tableName, def)` / `dropTriggersSQL(tableName)` — 自动记录变更的触发器
|
|
96
99
|
|
|
97
|
-
|
|
98
|
-
|--|--------|------------|-------|
|
|
99
|
-
| 自增 | `AUTOINCREMENT` | `BIGSERIAL` | `AUTO_INCREMENT` |
|
|
100
|
-
| 时间戳默认值 | `datetime('now')` | `NOW()` | `CURRENT_TIMESTAMP` |
|
|
101
|
-
| 字符串类型 | `TEXT` | `TEXT` | `VARCHAR(255)` |
|
|
100
|
+
**如果不指定 dialect(或指定不支持的值),必须自行实现以上所有方法。** 缺失方法会在构造时报错。
|
|
102
101
|
|
|
103
|
-
|
|
102
|
+
你也可以在 `db` 适配器上显式定义任何方法来覆盖内置实现 — 你的显式实现始终优先于内置方言。
|
|
104
103
|
|
|
105
104
|
### 第 2 步:创建引擎并注册表
|
|
106
105
|
|
|
@@ -110,6 +109,7 @@ import { SyncEngine } from 'sync-lib';
|
|
|
110
109
|
const engine = new SyncEngine({
|
|
111
110
|
nodeId: 'node-1',
|
|
112
111
|
db,
|
|
112
|
+
dialect: 'sqlite', // 或 'postgresql' 或 'mysql'
|
|
113
113
|
|
|
114
114
|
// 必需回调
|
|
115
115
|
onSendToPeer: (peerId, msg) => yourTransport.send(peerId, msg),
|
|
@@ -130,8 +130,7 @@ engine.registerTable('users', {
|
|
|
130
130
|
});
|
|
131
131
|
|
|
132
132
|
engine.initSchema(); // 创建 _sync_log、_sync_peers、_tombstones
|
|
133
|
-
engine.initTriggers(); //
|
|
134
|
-
// PostgreSQL/MySQL 跳过此步,改用 logChange()
|
|
133
|
+
engine.initTriggers(); // 创建同步触发器(方言感知)
|
|
135
134
|
engine.start(); // 启动 Leader 选举
|
|
136
135
|
```
|
|
137
136
|
|
|
@@ -162,7 +161,7 @@ setInterval(() => engine.tickCleanup(), 3600_000); // 每 1 小时清理旧日
|
|
|
162
161
|
|
|
163
162
|
### 第 5 步:写入数据
|
|
164
163
|
|
|
165
|
-
|
|
164
|
+
指定了 `dialect` 并调用 `initTriggers()` 后,所有数据库的写入方式完全一致 — 触发器自动记录变更:
|
|
166
165
|
|
|
167
166
|
```js
|
|
168
167
|
const ts = engine.hlc.tick();
|
|
@@ -171,7 +170,7 @@ db.run('INSERT INTO users (user_id, name, _hlc) VALUES (?, ?, ?)',
|
|
|
171
170
|
engine.notifyLocalWrite(); // 推送到对等节点
|
|
172
171
|
```
|
|
173
172
|
|
|
174
|
-
|
|
173
|
+
**不使用触发器时**(手动 logChange 备选方案):
|
|
175
174
|
|
|
176
175
|
```js
|
|
177
176
|
import { logChange } from 'sync-lib';
|
|
@@ -232,7 +231,11 @@ Leader 通过 `onExecuteProxyRequest` 回调执行请求。
|
|
|
232
231
|
|
|
233
232
|
---
|
|
234
233
|
|
|
235
|
-
##
|
|
234
|
+
## 自定义适配器示例
|
|
235
|
+
|
|
236
|
+
使用内置方言(`sqlite`、`postgresql`、`mysql`)时,适配器只需 `run/get/all/exec` 四个方法。以下完整示例供参考,适用于需要自定义或使用不支持的数据库的场景。
|
|
237
|
+
|
|
238
|
+
### PostgreSQL(完整手动适配器)
|
|
236
239
|
|
|
237
240
|
```js
|
|
238
241
|
import pg from 'pg';
|
|
@@ -291,12 +294,40 @@ const db = {
|
|
|
291
294
|
_hlc TEXT NOT NULL, PRIMARY KEY (table_name, row_key))`,
|
|
292
295
|
];
|
|
293
296
|
},
|
|
297
|
+
|
|
298
|
+
// 可选:注入 PG 触发器,使 initTriggers() 自动创建
|
|
299
|
+
triggersSQL(tableName, def) {
|
|
300
|
+
const keyExpr = def.keyColumns.map(c => `'${c}', NEW.${c}`).join(', ');
|
|
301
|
+
const dataExpr = def.dataColumns.map(c => `'${c}', NEW.${c}`).join(', ');
|
|
302
|
+
return [
|
|
303
|
+
`CREATE OR REPLACE FUNCTION _sync_trg_${tableName}_fn() RETURNS TRIGGER AS $$
|
|
304
|
+
BEGIN
|
|
305
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
306
|
+
VALUES ('${tableName}', TG_OP,
|
|
307
|
+
json_build_object(${keyExpr})::text,
|
|
308
|
+
CASE WHEN TG_OP = 'DELETE' THEN NULL
|
|
309
|
+
ELSE json_build_object(${dataExpr})::text END,
|
|
310
|
+
COALESCE(NEW._hlc, OLD._hlc));
|
|
311
|
+
RETURN COALESCE(NEW, OLD);
|
|
312
|
+
END; $$ LANGUAGE plpgsql`,
|
|
313
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName} ON ${tableName}`,
|
|
314
|
+
`CREATE TRIGGER _sync_trg_${tableName}
|
|
315
|
+
AFTER INSERT OR UPDATE OR DELETE ON ${tableName}
|
|
316
|
+
FOR EACH ROW EXECUTE FUNCTION _sync_trg_${tableName}_fn()`,
|
|
317
|
+
];
|
|
318
|
+
},
|
|
319
|
+
dropTriggersSQL(tableName) {
|
|
320
|
+
return [
|
|
321
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName} ON ${tableName}`,
|
|
322
|
+
`DROP FUNCTION IF EXISTS _sync_trg_${tableName}_fn`,
|
|
323
|
+
];
|
|
324
|
+
},
|
|
294
325
|
};
|
|
295
326
|
```
|
|
296
327
|
|
|
297
328
|
---
|
|
298
329
|
|
|
299
|
-
|
|
330
|
+
### MySQL(完整手动适配器)
|
|
300
331
|
|
|
301
332
|
```js
|
|
302
333
|
import mysql from 'mysql2/promise';
|
|
@@ -347,6 +378,34 @@ const db = {
|
|
|
347
378
|
_hlc VARCHAR(64) NOT NULL, PRIMARY KEY (table_name, row_key(255)))`,
|
|
348
379
|
];
|
|
349
380
|
},
|
|
381
|
+
|
|
382
|
+
// 可选:注入 MySQL 触发器,使 initTriggers() 自动创建
|
|
383
|
+
triggersSQL(tableName, def) {
|
|
384
|
+
const keyExpr = def.keyColumns.map(c => `'${c}', NEW.${c}`).join(', ');
|
|
385
|
+
const dataExpr = def.dataColumns.map(c => `'${c}', NEW.${c}`).join(', ');
|
|
386
|
+
const delKeyExpr = def.keyColumns.map(c => `'${c}', OLD.${c}`).join(', ');
|
|
387
|
+
return [
|
|
388
|
+
`CREATE TRIGGER _sync_trg_${tableName}_insert
|
|
389
|
+
AFTER INSERT ON ${tableName} FOR EACH ROW
|
|
390
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
391
|
+
VALUES ('${tableName}', 'INSERT', JSON_OBJECT(${keyExpr}), JSON_OBJECT(${dataExpr}), NEW._hlc)`,
|
|
392
|
+
`CREATE TRIGGER _sync_trg_${tableName}_update
|
|
393
|
+
AFTER UPDATE ON ${tableName} FOR EACH ROW
|
|
394
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
395
|
+
VALUES ('${tableName}', 'UPDATE', JSON_OBJECT(${keyExpr}), JSON_OBJECT(${dataExpr}), NEW._hlc)`,
|
|
396
|
+
`CREATE TRIGGER _sync_trg_${tableName}_delete
|
|
397
|
+
AFTER DELETE ON ${tableName} FOR EACH ROW
|
|
398
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
399
|
+
VALUES ('${tableName}', 'DELETE', JSON_OBJECT(${delKeyExpr}), NULL, OLD._hlc)`,
|
|
400
|
+
];
|
|
401
|
+
},
|
|
402
|
+
dropTriggersSQL(tableName) {
|
|
403
|
+
return [
|
|
404
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName}_insert`,
|
|
405
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName}_update`,
|
|
406
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName}_delete`,
|
|
407
|
+
];
|
|
408
|
+
},
|
|
350
409
|
};
|
|
351
410
|
```
|
|
352
411
|
|
|
@@ -385,14 +444,17 @@ const db = {
|
|
|
385
444
|
### 独立工具函数
|
|
386
445
|
|
|
387
446
|
```js
|
|
388
|
-
import { HLC, logChange, getInfraSchemaSQL, getTriggersSQL } from 'sync-lib';
|
|
447
|
+
import { HLC, DIALECTS, applyDialect, logChange, getInfraSchemaSQL, getTriggersSQL } from 'sync-lib';
|
|
389
448
|
```
|
|
390
449
|
|
|
391
450
|
| 导出 | 说明 |
|
|
392
451
|
|------|------|
|
|
393
452
|
| `HLC` | 混合逻辑时钟类 |
|
|
394
453
|
| `LeaderElection` | Leader 选举状态机 |
|
|
395
|
-
| `
|
|
454
|
+
| `DIALECTS` | 内置方言定义(sqlite、postgresql、mysql) |
|
|
455
|
+
| `applyDialect(db, name)` | 将方言方法应用到 db 适配器 |
|
|
456
|
+
| `validateDialectMethods(db)` | 校验 db 是否具备所有必需的方言方法 |
|
|
457
|
+
| `logChange(db, table, op, key, data, hlc)` | 手动记录变更(触发器的替代方案) |
|
|
396
458
|
| `getInfraSchemaSQL()` | 内置 SQLite DDL(基础设施表) |
|
|
397
459
|
| `getTriggersSQL(table, def)` | 生成 SQLite 同步触发器 |
|
|
398
460
|
| `applyEntries(db, entries, hlc, registry)` | 幂等地应用远程条目 |
|
|
@@ -401,26 +463,31 @@ import { HLC, logChange, getInfraSchemaSQL, getTriggersSQL } from 'sync-lib';
|
|
|
401
463
|
|
|
402
464
|
## DatabaseAdapter 接口
|
|
403
465
|
|
|
404
|
-
每个适配器必须实现以下方法:
|
|
405
|
-
|
|
406
466
|
```
|
|
407
467
|
┌───────────────────────────────────────────────────────────────┐
|
|
468
|
+
│ 始终必须实现: │
|
|
469
|
+
│ │
|
|
408
470
|
│ db.run(sql, params) → { changes: number } │
|
|
409
471
|
│ db.get(sql, params) → Object | null │
|
|
410
472
|
│ db.all(sql, params) → Object[] │
|
|
411
473
|
│ db.exec(sql) → void │
|
|
412
474
|
│ │
|
|
475
|
+
├───────────────────────────────────────────────────────────────┤
|
|
476
|
+
│ 由 dialect 自动填充(或手动实现): │
|
|
477
|
+
│ │
|
|
413
478
|
│ db.beginTransaction() → void │
|
|
414
479
|
│ db.commit() → void │
|
|
415
480
|
│ db.rollback() → void │
|
|
416
|
-
│ │
|
|
417
481
|
│ db.upsertSQL(table, cols, keyColumns) → string │
|
|
418
|
-
│
|
|
419
|
-
│ db.
|
|
482
|
+
│ db.infraSchemaSQL() → string[] │
|
|
483
|
+
│ db.triggersSQL(table, def) → string[] │
|
|
484
|
+
│ db.dropTriggersSQL(table) → string[] │
|
|
420
485
|
└───────────────────────────────────────────────────────────────┘
|
|
421
486
|
```
|
|
422
487
|
|
|
423
|
-
|
|
488
|
+
指定 `dialect: 'sqlite' | 'postgresql' | 'mysql'` 后,下面 7 个方法自动填充。不指定 dialect 则必须全部自行实现。
|
|
489
|
+
|
|
490
|
+
SQL 使用 `?` 占位符。如果你的数据库使用 `$1/$2`(如 PostgreSQL),请在 `run/get/all` 内部转换。
|
|
424
491
|
|
|
425
492
|
---
|
|
426
493
|
|
|
@@ -446,7 +513,8 @@ sync-lib/
|
|
|
446
513
|
├── hlc.js ← 混合逻辑时钟(纯算法)
|
|
447
514
|
├── leader-election.js ← Raft 简化版选举状态机
|
|
448
515
|
├── sync-protocol.js ← 消息编解码 + 幂等数据应用
|
|
449
|
-
|
|
516
|
+
├── sync-engine.js ← 主引擎(组合以上模块)
|
|
517
|
+
└── dialects.js ← 内置 SQL 方言(SQLite/PG/MySQL)
|
|
450
518
|
```
|
|
451
519
|
|
|
452
520
|
**设计原则**:
|
package/dialects.js
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 内置 SQL 方言定义
|
|
3
|
+
*
|
|
4
|
+
* 支持 sqlite、postgresql、mysql 三种方言。
|
|
5
|
+
* 每种方言提供以下 DatabaseAdapter 方法的默认实现:
|
|
6
|
+
* - beginTransaction / commit / rollback(通过 SQL 字符串 + db.exec)
|
|
7
|
+
* - upsertSQL
|
|
8
|
+
* - infraSchemaSQL
|
|
9
|
+
* - triggersSQL / dropTriggersSQL
|
|
10
|
+
*
|
|
11
|
+
* 使用方式:
|
|
12
|
+
* new SyncEngine({ dialect: 'sqlite', db: { run, get, all, exec }, ... })
|
|
13
|
+
*
|
|
14
|
+
* 指定 dialect 后,db 上缺失的方言方法由内置实现自动填充。
|
|
15
|
+
* db 上已有的方法优先级更高(不会被覆盖)。
|
|
16
|
+
*
|
|
17
|
+
* 若不指定 dialect 或指定不认识的值,所有方言方法必须由 db 自行实现,否则报错。
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { getInfraSchemaSQL, getTriggersSQL, dropTriggersSQL } from './sync-protocol.js';
|
|
21
|
+
|
|
22
|
+
// ═══════════════════════════════════════════════════
|
|
23
|
+
// SQLite
|
|
24
|
+
// ═══════════════════════════════════════════════════
|
|
25
|
+
|
|
26
|
+
const SQLITE = {
|
|
27
|
+
beginTransactionSQL: 'BEGIN IMMEDIATE',
|
|
28
|
+
commitSQL: 'COMMIT',
|
|
29
|
+
rollbackSQL: 'ROLLBACK',
|
|
30
|
+
|
|
31
|
+
upsertSQL(table, columns, _keyColumns) {
|
|
32
|
+
const cols = columns.join(', ');
|
|
33
|
+
const ph = columns.map(() => '?').join(', ');
|
|
34
|
+
return `INSERT OR REPLACE INTO ${table} (${cols}) VALUES (${ph})`;
|
|
35
|
+
},
|
|
36
|
+
|
|
37
|
+
infraSchemaSQL: getInfraSchemaSQL,
|
|
38
|
+
triggersSQL: getTriggersSQL,
|
|
39
|
+
dropTriggersSQL,
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
// ═══════════════════════════════════════════════════
|
|
43
|
+
// PostgreSQL
|
|
44
|
+
// ═══════════════════════════════════════════════════
|
|
45
|
+
|
|
46
|
+
const POSTGRESQL = {
|
|
47
|
+
beginTransactionSQL: 'BEGIN',
|
|
48
|
+
commitSQL: 'COMMIT',
|
|
49
|
+
rollbackSQL: 'ROLLBACK',
|
|
50
|
+
|
|
51
|
+
upsertSQL(table, columns, keyColumns) {
|
|
52
|
+
const cols = columns.join(', ');
|
|
53
|
+
const ph = columns.map(() => '?').join(', ');
|
|
54
|
+
const keys = keyColumns.join(', ');
|
|
55
|
+
const updates = columns
|
|
56
|
+
.filter(c => !keyColumns.includes(c))
|
|
57
|
+
.map(c => `${c} = EXCLUDED.${c}`);
|
|
58
|
+
return updates.length === 0
|
|
59
|
+
? `INSERT INTO ${table} (${cols}) VALUES (${ph}) ON CONFLICT (${keys}) DO NOTHING`
|
|
60
|
+
: `INSERT INTO ${table} (${cols}) VALUES (${ph}) ON CONFLICT (${keys}) DO UPDATE SET ${updates.join(', ')}`;
|
|
61
|
+
},
|
|
62
|
+
|
|
63
|
+
infraSchemaSQL() {
|
|
64
|
+
return [
|
|
65
|
+
`CREATE TABLE IF NOT EXISTS _sync_log (
|
|
66
|
+
id BIGSERIAL PRIMARY KEY,
|
|
67
|
+
table_name TEXT NOT NULL,
|
|
68
|
+
operation TEXT NOT NULL,
|
|
69
|
+
row_key TEXT NOT NULL,
|
|
70
|
+
row_data TEXT,
|
|
71
|
+
_hlc TEXT NOT NULL,
|
|
72
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
73
|
+
)`,
|
|
74
|
+
`CREATE INDEX IF NOT EXISTS idx_sync_log_id ON _sync_log (id)`,
|
|
75
|
+
`CREATE TABLE IF NOT EXISTS _sync_peers (
|
|
76
|
+
peer_key TEXT PRIMARY KEY,
|
|
77
|
+
last_sync_id BIGINT DEFAULT 0,
|
|
78
|
+
last_sync_at TIMESTAMPTZ
|
|
79
|
+
)`,
|
|
80
|
+
`CREATE TABLE IF NOT EXISTS _tombstones (
|
|
81
|
+
table_name TEXT NOT NULL,
|
|
82
|
+
row_key TEXT NOT NULL,
|
|
83
|
+
_hlc TEXT NOT NULL,
|
|
84
|
+
PRIMARY KEY (table_name, row_key)
|
|
85
|
+
)`,
|
|
86
|
+
];
|
|
87
|
+
},
|
|
88
|
+
|
|
89
|
+
triggersSQL(tableName, def) {
|
|
90
|
+
const blobSet = new Set(def.blobColumns || []);
|
|
91
|
+
|
|
92
|
+
const keyExprNew = def.keyColumns.map(c => `'${c}', NEW.${c}`).join(', ');
|
|
93
|
+
const keyExprOld = def.keyColumns.map(c => `'${c}', OLD.${c}`).join(', ');
|
|
94
|
+
|
|
95
|
+
const dataExpr = def.dataColumns.map(c => {
|
|
96
|
+
if (blobSet.has(c)) return `'${c}', encode(NEW.${c}, 'hex')`;
|
|
97
|
+
return `'${c}', NEW.${c}`;
|
|
98
|
+
}).join(', ');
|
|
99
|
+
|
|
100
|
+
return [
|
|
101
|
+
`CREATE OR REPLACE FUNCTION _sync_trg_${tableName}_fn() RETURNS TRIGGER AS $$
|
|
102
|
+
BEGIN
|
|
103
|
+
IF TG_OP = 'DELETE' THEN
|
|
104
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
105
|
+
VALUES ('${tableName}', 'DELETE', json_build_object(${keyExprOld})::text, NULL, OLD._hlc);
|
|
106
|
+
RETURN OLD;
|
|
107
|
+
ELSE
|
|
108
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
109
|
+
VALUES ('${tableName}', TG_OP, json_build_object(${keyExprNew})::text, json_build_object(${dataExpr})::text, NEW._hlc);
|
|
110
|
+
RETURN NEW;
|
|
111
|
+
END IF;
|
|
112
|
+
END; $$ LANGUAGE plpgsql`,
|
|
113
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName} ON ${tableName}`,
|
|
114
|
+
`CREATE TRIGGER _sync_trg_${tableName}
|
|
115
|
+
AFTER INSERT OR UPDATE OR DELETE ON ${tableName}
|
|
116
|
+
FOR EACH ROW EXECUTE FUNCTION _sync_trg_${tableName}_fn()`,
|
|
117
|
+
];
|
|
118
|
+
},
|
|
119
|
+
|
|
120
|
+
dropTriggersSQL(tableName) {
|
|
121
|
+
return [
|
|
122
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName} ON ${tableName}`,
|
|
123
|
+
`DROP FUNCTION IF EXISTS _sync_trg_${tableName}_fn`,
|
|
124
|
+
];
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
// ═══════════════════════════════════════════════════
|
|
129
|
+
// MySQL
|
|
130
|
+
// ═══════════════════════════════════════════════════
|
|
131
|
+
|
|
132
|
+
const MYSQL = {
|
|
133
|
+
beginTransactionSQL: 'START TRANSACTION',
|
|
134
|
+
commitSQL: 'COMMIT',
|
|
135
|
+
rollbackSQL: 'ROLLBACK',
|
|
136
|
+
|
|
137
|
+
upsertSQL(table, columns, keyColumns) {
|
|
138
|
+
const cols = columns.join(', ');
|
|
139
|
+
const ph = columns.map(() => '?').join(', ');
|
|
140
|
+
const updates = columns
|
|
141
|
+
.filter(c => !keyColumns.includes(c))
|
|
142
|
+
.map(c => `${c} = VALUES(${c})`);
|
|
143
|
+
return updates.length === 0
|
|
144
|
+
? `INSERT IGNORE INTO ${table} (${cols}) VALUES (${ph})`
|
|
145
|
+
: `INSERT INTO ${table} (${cols}) VALUES (${ph}) ON DUPLICATE KEY UPDATE ${updates.join(', ')}`;
|
|
146
|
+
},
|
|
147
|
+
|
|
148
|
+
infraSchemaSQL() {
|
|
149
|
+
return [
|
|
150
|
+
`CREATE TABLE IF NOT EXISTS _sync_log (
|
|
151
|
+
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
|
152
|
+
table_name VARCHAR(255) NOT NULL,
|
|
153
|
+
operation VARCHAR(10) NOT NULL,
|
|
154
|
+
row_key TEXT NOT NULL,
|
|
155
|
+
row_data LONGTEXT,
|
|
156
|
+
_hlc VARCHAR(64) NOT NULL,
|
|
157
|
+
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
158
|
+
)`,
|
|
159
|
+
`CREATE TABLE IF NOT EXISTS _sync_peers (
|
|
160
|
+
peer_key VARCHAR(255) PRIMARY KEY,
|
|
161
|
+
last_sync_id BIGINT DEFAULT 0,
|
|
162
|
+
last_sync_at DATETIME
|
|
163
|
+
)`,
|
|
164
|
+
`CREATE TABLE IF NOT EXISTS _tombstones (
|
|
165
|
+
table_name VARCHAR(255) NOT NULL,
|
|
166
|
+
row_key VARCHAR(1024) NOT NULL,
|
|
167
|
+
_hlc VARCHAR(64) NOT NULL,
|
|
168
|
+
PRIMARY KEY (table_name, row_key(255))
|
|
169
|
+
)`,
|
|
170
|
+
];
|
|
171
|
+
},
|
|
172
|
+
|
|
173
|
+
triggersSQL(tableName, def) {
|
|
174
|
+
const blobSet = new Set(def.blobColumns || []);
|
|
175
|
+
|
|
176
|
+
const keyExpr = (ref) => def.keyColumns.map(c => `'${c}', ${ref}.${c}`).join(', ');
|
|
177
|
+
const dataExpr = (ref) => def.dataColumns.map(c => {
|
|
178
|
+
if (blobSet.has(c)) return `'${c}', HEX(${ref}.${c})`;
|
|
179
|
+
return `'${c}', ${ref}.${c}`;
|
|
180
|
+
}).join(', ');
|
|
181
|
+
|
|
182
|
+
return [
|
|
183
|
+
`CREATE TRIGGER _sync_trg_${tableName}_insert
|
|
184
|
+
AFTER INSERT ON ${tableName} FOR EACH ROW
|
|
185
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
186
|
+
VALUES ('${tableName}', 'INSERT', JSON_OBJECT(${keyExpr('NEW')}), JSON_OBJECT(${dataExpr('NEW')}), NEW._hlc)`,
|
|
187
|
+
`CREATE TRIGGER _sync_trg_${tableName}_update
|
|
188
|
+
AFTER UPDATE ON ${tableName} FOR EACH ROW
|
|
189
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
190
|
+
VALUES ('${tableName}', 'UPDATE', JSON_OBJECT(${keyExpr('NEW')}), JSON_OBJECT(${dataExpr('NEW')}), NEW._hlc)`,
|
|
191
|
+
`CREATE TRIGGER _sync_trg_${tableName}_delete
|
|
192
|
+
AFTER DELETE ON ${tableName} FOR EACH ROW
|
|
193
|
+
INSERT INTO _sync_log (table_name, operation, row_key, row_data, _hlc)
|
|
194
|
+
VALUES ('${tableName}', 'DELETE', JSON_OBJECT(${keyExpr('OLD')}), NULL, OLD._hlc)`,
|
|
195
|
+
];
|
|
196
|
+
},
|
|
197
|
+
|
|
198
|
+
dropTriggersSQL(tableName) {
|
|
199
|
+
return [
|
|
200
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName}_insert`,
|
|
201
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName}_update`,
|
|
202
|
+
`DROP TRIGGER IF EXISTS _sync_trg_${tableName}_delete`,
|
|
203
|
+
];
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
// ═══════════════════════════════════════════════════
|
|
208
|
+
// 公共接口
|
|
209
|
+
// ═══════════════════════════════════════════════════
|
|
210
|
+
|
|
211
|
+
/** 所有内置方言 */
|
|
212
|
+
export const DIALECTS = {
|
|
213
|
+
sqlite: SQLITE,
|
|
214
|
+
postgresql: POSTGRESQL,
|
|
215
|
+
postgres: POSTGRESQL, // 别名
|
|
216
|
+
pg: POSTGRESQL, // 别名
|
|
217
|
+
mysql: MYSQL,
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
/** 需要方言提供的全部方法名 */
|
|
221
|
+
const DIALECT_METHODS = [
|
|
222
|
+
'beginTransaction', 'commit', 'rollback',
|
|
223
|
+
'upsertSQL', 'infraSchemaSQL',
|
|
224
|
+
'triggersSQL', 'dropTriggersSQL',
|
|
225
|
+
];
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* 将内置方言应用到 db 适配器。
|
|
229
|
+
*
|
|
230
|
+
* - 已有的方法不覆盖(用户显式注入的优先级最高)。
|
|
231
|
+
* - beginTransaction / commit / rollback 通过 db.exec(sql) 桥接。
|
|
232
|
+
*
|
|
233
|
+
* @param {object} db - DatabaseAdapter 实例
|
|
234
|
+
* @param {string} dialectName - 方言名称
|
|
235
|
+
*/
|
|
236
|
+
export function applyDialect(db, dialectName) {
|
|
237
|
+
const dialect = DIALECTS[dialectName];
|
|
238
|
+
if (!dialect) {
|
|
239
|
+
const supported = Object.keys(DIALECTS).filter(k => !['postgres', 'pg'].includes(k)).join(', ');
|
|
240
|
+
throw new Error(
|
|
241
|
+
`Unknown dialect "${dialectName}". Supported: ${supported}. ` +
|
|
242
|
+
`Or omit dialect and implement all adapter methods manually.`
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// 事务控制:通过 SQL 字符串 + db.exec() 桥接
|
|
247
|
+
if (typeof db.beginTransaction !== 'function') {
|
|
248
|
+
db.beginTransaction = () => db.exec(dialect.beginTransactionSQL);
|
|
249
|
+
}
|
|
250
|
+
if (typeof db.commit !== 'function') {
|
|
251
|
+
db.commit = () => db.exec(dialect.commitSQL);
|
|
252
|
+
}
|
|
253
|
+
if (typeof db.rollback !== 'function') {
|
|
254
|
+
db.rollback = () => db.exec(dialect.rollbackSQL);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// 函数方法:直接挂载
|
|
258
|
+
if (typeof db.upsertSQL !== 'function') {
|
|
259
|
+
db.upsertSQL = dialect.upsertSQL;
|
|
260
|
+
}
|
|
261
|
+
if (typeof db.infraSchemaSQL !== 'function') {
|
|
262
|
+
db.infraSchemaSQL = dialect.infraSchemaSQL;
|
|
263
|
+
}
|
|
264
|
+
if (typeof db.triggersSQL !== 'function') {
|
|
265
|
+
db.triggersSQL = dialect.triggersSQL;
|
|
266
|
+
}
|
|
267
|
+
if (typeof db.dropTriggersSQL !== 'function') {
|
|
268
|
+
db.dropTriggersSQL = dialect.dropTriggersSQL;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* 校验 db 适配器是否具备所有必需的方言方法。
|
|
274
|
+
* 如果缺失任何方法,抛出包含缺失列表的错误。
|
|
275
|
+
*
|
|
276
|
+
* @param {object} db - DatabaseAdapter 实例
|
|
277
|
+
*/
|
|
278
|
+
export function validateDialectMethods(db) {
|
|
279
|
+
const missing = DIALECT_METHODS.filter(m => typeof db[m] !== 'function');
|
|
280
|
+
if (missing.length > 0) {
|
|
281
|
+
throw new Error(
|
|
282
|
+
`DatabaseAdapter missing required methods: ${missing.join(', ')}. ` +
|
|
283
|
+
`Either specify a dialect (e.g. dialect: 'sqlite') or implement these methods on your db adapter.`
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
}
|
package/index.js
CHANGED
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
*
|
|
56
56
|
* // 4. 初始化 Schema & 触发器
|
|
57
57
|
* engine.initSchema(); // 创建 _sync_log, _sync_peers, _tombstones
|
|
58
|
-
* engine.initTriggers(); //
|
|
58
|
+
* engine.initTriggers(); // 创建同步触发器(由 dialect 自动选择对应方言)
|
|
59
59
|
* engine.start();
|
|
60
60
|
*
|
|
61
61
|
* // 5. 外部传输层对接
|
|
@@ -72,7 +72,7 @@
|
|
|
72
72
|
* db.run('INSERT INTO users ...');
|
|
73
73
|
* engine.notifyLocalWrite();
|
|
74
74
|
*
|
|
75
|
-
* // 8.
|
|
75
|
+
* // 8. 未使用触发器时的备选方案:手动记录变更
|
|
76
76
|
* db.run('INSERT INTO users ...');
|
|
77
77
|
* engine.logChange('users', 'INSERT', { user_id: 'u1' }, { user_id: 'u1', name: 'Alice', ... });
|
|
78
78
|
* engine.notifyLocalWrite();
|
|
@@ -116,6 +116,9 @@ export { SyncEngine } from './sync-engine.js';
|
|
|
116
116
|
export { HLC } from './hlc.js';
|
|
117
117
|
export { LeaderElection } from './leader-election.js';
|
|
118
118
|
|
|
119
|
+
// 内置方言
|
|
120
|
+
export { DIALECTS, applyDialect, validateDialectMethods } from './dialects.js';
|
|
121
|
+
|
|
119
122
|
// 协议层工具(高级用途:自定义消息处理)
|
|
120
123
|
export {
|
|
121
124
|
// 消息构造
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@raft-hlc-sync-protocol/raft-sync-lib",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"description": "pure javascript raft and hlc sync protocol",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"raft",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
],
|
|
14
14
|
"license": "MIT",
|
|
15
15
|
"author": "xnet",
|
|
16
|
-
"type": "
|
|
16
|
+
"type": "module",
|
|
17
17
|
"main": "index.js",
|
|
18
18
|
"scripts": {
|
|
19
19
|
"test": "echo \"Error: no test specified\" && exit 1"
|
package/sync-engine.js
CHANGED
|
@@ -91,7 +91,7 @@
|
|
|
91
91
|
*
|
|
92
92
|
* // 3. 初始化
|
|
93
93
|
* engine.initSchema(); // 创建 _sync_log 等基础设施表
|
|
94
|
-
* engine.initTriggers(); //
|
|
94
|
+
* engine.initTriggers(); // 创建同步触发器(由 dialect 决定方言)
|
|
95
95
|
* engine.start(); // 启动 Leader Election
|
|
96
96
|
*
|
|
97
97
|
* // 4. 外部传输层事件
|
|
@@ -113,6 +113,7 @@
|
|
|
113
113
|
|
|
114
114
|
import { HLC } from './hlc.js';
|
|
115
115
|
import { LeaderElection } from './leader-election.js';
|
|
116
|
+
import { applyDialect, validateDialectMethods } from './dialects.js';
|
|
116
117
|
import {
|
|
117
118
|
makeHandshake,
|
|
118
119
|
makeSync,
|
|
@@ -183,6 +184,11 @@ export class SyncEngine {
|
|
|
183
184
|
* @param {function(string, Error): void} [options.onError] - 错误通知
|
|
184
185
|
* @param {function(object): Promise<object>} [options.onExecuteProxyRequest] - 代理请求执行
|
|
185
186
|
*
|
|
187
|
+
* ── 方言 ──
|
|
188
|
+
* @param {string} [options.dialect] - 数据库方言:'sqlite' | 'postgresql' | 'mysql'
|
|
189
|
+
* 指定后自动填充 db 上缺失的方言方法(beginTransaction/commit/rollback/upsertSQL/infraSchemaSQL/triggersSQL/dropTriggersSQL)。
|
|
190
|
+
* 不指定时所有方言方法必须由 db 自行实现,否则报错。
|
|
191
|
+
*
|
|
186
192
|
* ── 可选配置 ──
|
|
187
193
|
* @param {number} [options.numShards=16] - 虚拟分片数
|
|
188
194
|
* @param {number} [options.syncBatchSize=500] - 每批同步最大条目数
|
|
@@ -200,6 +206,12 @@ export class SyncEngine {
|
|
|
200
206
|
this.db = options.db;
|
|
201
207
|
this.hlc = new HLC(options.nodeId);
|
|
202
208
|
|
|
209
|
+
// ===== 方言处理 =====
|
|
210
|
+
if (options.dialect) {
|
|
211
|
+
applyDialect(this.db, options.dialect);
|
|
212
|
+
}
|
|
213
|
+
validateDialectMethods(this.db);
|
|
214
|
+
|
|
203
215
|
// ===== 必须回调 =====
|
|
204
216
|
this._onSendToPeer = options.onSendToPeer;
|
|
205
217
|
this._onClosePeer = options.onClosePeer;
|
|
@@ -294,18 +306,18 @@ export class SyncEngine {
|
|
|
294
306
|
|
|
295
307
|
/** 创建同步基础设施表(_sync_log, _sync_peers, _tombstones) */
|
|
296
308
|
initSchema() {
|
|
297
|
-
const
|
|
298
|
-
? this.db.infraSchemaSQL()
|
|
299
|
-
: getInfraSchemaSQL();
|
|
300
|
-
for (const sql of sqls) {
|
|
309
|
+
for (const sql of this.db.infraSchemaSQL()) {
|
|
301
310
|
this.db.exec(sql);
|
|
302
311
|
}
|
|
303
312
|
}
|
|
304
313
|
|
|
305
|
-
/**
|
|
314
|
+
/**
|
|
315
|
+
* 为所有已注册表创建同步触发器。
|
|
316
|
+
* 触发器 SQL 由 db.triggersSQL(tableName, def) 提供(内置方言或外部注入)。
|
|
317
|
+
*/
|
|
306
318
|
initTriggers() {
|
|
307
319
|
for (const [tableName, def] of this._tableRegistry) {
|
|
308
|
-
for (const sql of
|
|
320
|
+
for (const sql of this.db.triggersSQL(tableName, def)) {
|
|
309
321
|
this.db.exec(sql);
|
|
310
322
|
}
|
|
311
323
|
}
|
|
@@ -314,7 +326,7 @@ export class SyncEngine {
|
|
|
314
326
|
/** 删除所有已注册表的同步触发器 */
|
|
315
327
|
dropTriggers() {
|
|
316
328
|
for (const tableName of this._tableRegistry.keys()) {
|
|
317
|
-
for (const sql of dropTriggersSQL(tableName)) {
|
|
329
|
+
for (const sql of this.db.dropTriggersSQL(tableName)) {
|
|
318
330
|
this.db.exec(sql);
|
|
319
331
|
}
|
|
320
332
|
}
|
package/sync-protocol.js
CHANGED
|
@@ -44,6 +44,18 @@
|
|
|
44
44
|
* 返回同步基础设施表的 DDL 语句数组。
|
|
45
45
|
* 不实现则由 sync-lib 提供内置 SQLite DDL 作为降级。
|
|
46
46
|
*
|
|
47
|
+
* ── 触发器(可选注入)──
|
|
48
|
+
*
|
|
49
|
+
* db.triggersSQL?(tableName, def) → string[]
|
|
50
|
+
* 返回指定表的同步触发器 DDL 语句数组。
|
|
51
|
+
* 不实现则由 sync-lib 提供内置 SQLite 触发器。
|
|
52
|
+
* PG / MySQL 用户可注入各自方言的触发器,
|
|
53
|
+
* 从而使用 initTriggers() 自动记录变更,无需手动调用 logChange()。
|
|
54
|
+
*
|
|
55
|
+
* db.dropTriggersSQL?(tableName) → string[]
|
|
56
|
+
* 返回删除指定表同步触发器的 SQL 语句数组。
|
|
57
|
+
* 不实现则由 sync-lib 提供内置 SQLite 版本。
|
|
58
|
+
*
|
|
47
59
|
* ═══════════════════════════════════════════════════════
|
|
48
60
|
* 表定义 TableDef(通过 SyncEngine.registerTable 注册):
|
|
49
61
|
* ═══════════════════════════════════════════════════════
|