meadow-integration 1.0.41 → 1.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BUILDING-AND-PUBLISHING.md +2 -2
- package/README.md +27 -8
- package/docs/README.md +1 -1
- package/docs/_cover.md +1 -1
- package/docs/_topbar.md +1 -1
- package/docs/_version.json +3 -3
- package/docs/architecture.md +14 -225
- package/docs/css/docuserve.css +327 -0
- package/docs/data-clone/diagrams/architecture-diagram.excalidraw +1756 -0
- package/docs/data-clone/diagrams/architecture-diagram.mmd +8 -0
- package/docs/data-clone/diagrams/architecture-diagram.svg +2 -0
- package/docs/data-clone/overview.md +2 -32
- package/docs/diagrams/configuration-cascade-2.excalidraw +831 -0
- package/docs/diagrams/configuration-cascade-2.mmd +8 -0
- package/docs/diagrams/configuration-cascade-2.svg +2 -0
- package/docs/diagrams/configuration-cascade.excalidraw +831 -0
- package/docs/diagrams/configuration-cascade.mmd +8 -0
- package/docs/diagrams/configuration-cascade.svg +2 -0
- package/docs/diagrams/data-synchronization-pipeline.excalidraw +3278 -0
- package/docs/diagrams/data-synchronization-pipeline.mmd +42 -0
- package/docs/diagrams/data-synchronization-pipeline.svg +2 -0
- package/docs/diagrams/data-transformation-pipeline.excalidraw +2929 -0
- package/docs/diagrams/data-transformation-pipeline.mmd +31 -0
- package/docs/diagrams/data-transformation-pipeline.svg +2 -0
- package/docs/diagrams/docker-deployment.excalidraw +1963 -0
- package/docs/diagrams/docker-deployment.mmd +23 -0
- package/docs/diagrams/docker-deployment.svg +2 -0
- package/docs/diagrams/high-level-system-architecture.excalidraw +5752 -0
- package/docs/diagrams/high-level-system-architecture.mmd +66 -0
- package/docs/diagrams/high-level-system-architecture.svg +2 -0
- package/docs/diagrams/module-structure.excalidraw +15206 -0
- package/docs/diagrams/module-structure.mmd +56 -0
- package/docs/diagrams/module-structure.svg +2 -0
- package/docs/diagrams/sync-mode-comparison.excalidraw +3660 -0
- package/docs/diagrams/sync-mode-comparison.mmd +33 -0
- package/docs/diagrams/sync-mode-comparison.svg +2 -0
- package/docs/implementation-reference.md +2 -58
- package/docs/index.html +2 -2
- package/docs/retold-catalog.json +388 -284
- package/docs/retold-keyword-index.json +24830 -16291
- package/example-applications/mapping-demo/README.md +2 -10
- package/example-applications/mapping-demo/diagrams/architecture.excalidraw +1866 -0
- package/example-applications/mapping-demo/diagrams/architecture.mmd +8 -0
- package/example-applications/mapping-demo/diagrams/architecture.svg +2 -0
- package/example-applications/mapping-demo/package.json +0 -3
- package/example-applications/mapping-demo/web/mapping-demo-editor.js +5 -5
- package/example-applications/mapping-demo/web/mapping-demo-editor.min.js +2 -2
- package/example-applications/mapping-demo/web/pict.min.js +2 -2
- package/package.json +7 -7
- package/source/Meadow-Integration-Engine.js +34 -0
- package/source/services/clone/Meadow-Service-DeleteCursorStore.js +105 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-OngoingEventualConsistency.js +327 -92
- package/source/services/clone/Meadow-Service-Sync.js +2 -0
- package/test/Meadow-Integration-BisectionSync_test.js +15 -5
- package/test/Meadow-Integration-NewStrategies_test.js +15 -5
- package/test/Meadow-Integration-OngoingEventualConsistencyDeleteCursor_test.js +228 -0
- package/test/Meadow-Integration-OngoingEventualConsistencyDeleteSync_test.js +311 -0
|
@@ -428,18 +428,28 @@ function setupSQLiteProvider(pFable, fCallback)
|
|
|
428
428
|
|
|
429
429
|
function seedLocalBooks(pFable, pBooks)
|
|
430
430
|
{
|
|
431
|
-
|
|
431
|
+
// node:sqlite's DatabaseSync has no `.transaction(fn)` helper (that was
|
|
432
|
+
// a better-sqlite3 idiom). Bracket the bulk insert manually so seeding
|
|
433
|
+
// many thousands of rows doesn't pay per-row commit overhead.
|
|
434
|
+
const tmpDB = pFable.MeadowSQLiteProvider.db;
|
|
435
|
+
const tmpInsert = tmpDB.prepare(`
|
|
432
436
|
INSERT OR REPLACE INTO Book (IDBook, GUIDBook, CreateDate, CreatingIDUser, UpdateDate, UpdatingIDUser, Deleted, DeleteDate, DeletingIDUser, Title, Type, Genre, PublicationYear)
|
|
433
437
|
VALUES (@IDBook, @GUIDBook, @CreateDate, @CreatingIDUser, @UpdateDate, @UpdatingIDUser, @Deleted, @DeleteDate, @DeletingIDUser, @Title, @Type, @Genre, @PublicationYear)
|
|
434
438
|
`);
|
|
435
|
-
|
|
439
|
+
tmpDB.exec('BEGIN');
|
|
440
|
+
try
|
|
436
441
|
{
|
|
437
|
-
for (const tmpRecord of
|
|
442
|
+
for (const tmpRecord of pBooks)
|
|
438
443
|
{
|
|
439
444
|
tmpInsert.run(tmpRecord);
|
|
440
445
|
}
|
|
441
|
-
|
|
442
|
-
|
|
446
|
+
tmpDB.exec('COMMIT');
|
|
447
|
+
}
|
|
448
|
+
catch (pError)
|
|
449
|
+
{
|
|
450
|
+
tmpDB.exec('ROLLBACK');
|
|
451
|
+
throw pError;
|
|
452
|
+
}
|
|
443
453
|
}
|
|
444
454
|
|
|
445
455
|
function getLocalBookCount(pFable)
|
|
@@ -552,18 +552,28 @@ function setupSQLiteProvider(pFable, fCallback)
|
|
|
552
552
|
|
|
553
553
|
function seedLocalBooks(pFable, pBooks)
|
|
554
554
|
{
|
|
555
|
-
|
|
555
|
+
// node:sqlite's DatabaseSync has no `.transaction(fn)` helper (that was
|
|
556
|
+
// a better-sqlite3 idiom). Bracket the bulk insert manually so seeding
|
|
557
|
+
// many thousands of rows doesn't pay per-row commit overhead.
|
|
558
|
+
const tmpDB = pFable.MeadowSQLiteProvider.db;
|
|
559
|
+
const tmpInsert = tmpDB.prepare(`
|
|
556
560
|
INSERT OR REPLACE INTO Book (IDBook, GUIDBook, CreateDate, CreatingIDUser, UpdateDate, UpdatingIDUser, Deleted, DeleteDate, DeletingIDUser, Title, Type, Genre, PublicationYear)
|
|
557
561
|
VALUES (@IDBook, @GUIDBook, @CreateDate, @CreatingIDUser, @UpdateDate, @UpdatingIDUser, @Deleted, @DeleteDate, @DeletingIDUser, @Title, @Type, @Genre, @PublicationYear)
|
|
558
562
|
`);
|
|
559
|
-
|
|
563
|
+
tmpDB.exec('BEGIN');
|
|
564
|
+
try
|
|
560
565
|
{
|
|
561
|
-
for (const tmpRecord of
|
|
566
|
+
for (const tmpRecord of pBooks)
|
|
562
567
|
{
|
|
563
568
|
tmpInsert.run(tmpRecord);
|
|
564
569
|
}
|
|
565
|
-
|
|
566
|
-
|
|
570
|
+
tmpDB.exec('COMMIT');
|
|
571
|
+
}
|
|
572
|
+
catch (pError)
|
|
573
|
+
{
|
|
574
|
+
tmpDB.exec('ROLLBACK');
|
|
575
|
+
throw pError;
|
|
576
|
+
}
|
|
567
577
|
}
|
|
568
578
|
|
|
569
579
|
function getLocalBookCount(pFable)
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Unit tests for the resumable (head/tail) delete cursor.
|
|
3
|
+
|
|
4
|
+
Drives OngoingEventualConsistency.syncDeletedRecords with DeleteCursorStatePath
|
|
5
|
+
set, simulating successive "runs" (each call re-reads the JSON state file, as a
|
|
6
|
+
fresh container would). Verifies the tail drains across runs, the head pass
|
|
7
|
+
picks up new high-id deletions, the caught-up steady state, and that state
|
|
8
|
+
persists in the JSON file.
|
|
9
|
+
|
|
10
|
+
Mock server serves the keyset deleted-page queries
|
|
11
|
+
(FBV~Deleted~EQ~1[~FBV~IDBook~GT~N][~FBV~IDBook~LT~M]~FSF~IDBook~DESC~DESC).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const Chai = require('chai');
|
|
15
|
+
const Expect = Chai.expect;
|
|
16
|
+
|
|
17
|
+
const libHTTP = require('http');
|
|
18
|
+
const libFS = require('fs');
|
|
19
|
+
const libOS = require('os');
|
|
20
|
+
const libPath = require('path');
|
|
21
|
+
const libFable = require('fable');
|
|
22
|
+
const libMeadowConnectionSQLite = require('meadow-connection-sqlite');
|
|
23
|
+
|
|
24
|
+
const libMeadowCloneRestClient = require('../source/services/clone/Meadow-Service-RestClient.js');
|
|
25
|
+
const libMeadowSync = require('../source/services/clone/Meadow-Service-Sync.js');
|
|
26
|
+
|
|
27
|
+
const MOCK_PORT = 18095;
|
|
28
|
+
const MOCK_BASE_URL = `http://localhost:${MOCK_PORT}/1.0/`;
|
|
29
|
+
const STATE_PATH = libPath.join(libOS.tmpdir(), `oec-delete-cursor-${process.pid}.json`);
|
|
30
|
+
|
|
31
|
+
const _BookSchema =
|
|
32
|
+
{
|
|
33
|
+
TableName: 'Book',
|
|
34
|
+
Columns:
|
|
35
|
+
[
|
|
36
|
+
{ Column: 'IDBook', DataType: 'int' },
|
|
37
|
+
{ Column: 'GUIDBook', DataType: 'GUID' },
|
|
38
|
+
{ Column: 'CreateDate', DataType: 'DateTime' },
|
|
39
|
+
{ Column: 'CreatingIDUser', DataType: 'int' },
|
|
40
|
+
{ Column: 'UpdateDate', DataType: 'DateTime' },
|
|
41
|
+
{ Column: 'UpdatingIDUser', DataType: 'int' },
|
|
42
|
+
{ Column: 'Deleted', DataType: 'int' },
|
|
43
|
+
{ Column: 'DeleteDate', DataType: 'DateTime' },
|
|
44
|
+
{ Column: 'DeletingIDUser', DataType: 'int' },
|
|
45
|
+
{ Column: 'Title', DataType: 'String' }
|
|
46
|
+
],
|
|
47
|
+
MeadowSchema:
|
|
48
|
+
{
|
|
49
|
+
Scope: 'Book', DefaultIdentifier: 'IDBook', Domain: 'Default',
|
|
50
|
+
Schema:
|
|
51
|
+
[
|
|
52
|
+
{ Column: 'IDBook', Type: 'AutoIdentity', Size: 'Default' },
|
|
53
|
+
{ Column: 'GUIDBook', Type: 'AutoGUID', Size: '128' },
|
|
54
|
+
{ Column: 'CreateDate', Type: 'CreateDate', Size: 'Default' },
|
|
55
|
+
{ Column: 'CreatingIDUser', Type: 'CreateIDUser', Size: 'int' },
|
|
56
|
+
{ Column: 'UpdateDate', Type: 'UpdateDate', Size: 'Default' },
|
|
57
|
+
{ Column: 'UpdatingIDUser', Type: 'UpdateIDUser', Size: 'int' },
|
|
58
|
+
{ Column: 'Deleted', Type: 'Deleted', Size: 'Default' },
|
|
59
|
+
{ Column: 'DeleteDate', Type: 'DeleteDate', Size: 'Default' },
|
|
60
|
+
{ Column: 'DeletingIDUser', Type: 'DeleteIDUser', Size: 'int' },
|
|
61
|
+
{ Column: 'Title', Type: 'String', Size: '200' }
|
|
62
|
+
],
|
|
63
|
+
DefaultObject: { IDBook: 0, GUIDBook: '', CreateDate: null, CreatingIDUser: 0, UpdateDate: null, UpdatingIDUser: 0, Deleted: 0, DeleteDate: null, DeletingIDUser: 0, Title: '' },
|
|
64
|
+
JsonSchema: { title: 'Book', type: 'object', properties: { IDBook: { type: 'integer' } }, required: ['IDBook'] }
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
// Server-deleted ids: 10,20,...,100 (mutable so a test can add a new one).
|
|
69
|
+
let _ServerDeletedIDs = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100];
|
|
70
|
+
function serverDeletedRecord(pID)
|
|
71
|
+
{
|
|
72
|
+
return { IDBook: pID, GUIDBook: `GUID-${pID}`, Deleted: 1, DeleteDate: '2025-07-01T00:00:00.000Z', DeletingIDUser: 1, Title: `Deleted-${pID}` };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function createMockServer()
|
|
76
|
+
{
|
|
77
|
+
return libHTTP.createServer((pRequest, pResponse) =>
|
|
78
|
+
{
|
|
79
|
+
const tmpURL = pRequest.url;
|
|
80
|
+
pResponse.setHeader('Content-Type', 'application/json');
|
|
81
|
+
|
|
82
|
+
const tmpMatch = tmpURL.match(/\/1\.0\/Books\/FilteredTo\/(.+?)\/(\d+)\/(\d+)(\?|$)/);
|
|
83
|
+
if (tmpMatch && tmpMatch[1].indexOf('FBV~Deleted~EQ~1') > -1)
|
|
84
|
+
{
|
|
85
|
+
const tmpFilter = tmpMatch[1];
|
|
86
|
+
const tmpOffset = parseInt(tmpMatch[2], 10) || 0;
|
|
87
|
+
const tmpPageSize = parseInt(tmpMatch[3], 10) || 100;
|
|
88
|
+
const tmpGT = tmpFilter.match(/FBV~IDBook~GT~(\d+)/);
|
|
89
|
+
const tmpLT = tmpFilter.match(/FBV~IDBook~LT~(\d+)/);
|
|
90
|
+
|
|
91
|
+
let tmpIDs = _ServerDeletedIDs.slice();
|
|
92
|
+
if (tmpGT) { const n = parseInt(tmpGT[1], 10); tmpIDs = tmpIDs.filter((id) => id > n); }
|
|
93
|
+
if (tmpLT) { const n = parseInt(tmpLT[1], 10); tmpIDs = tmpIDs.filter((id) => id < n); }
|
|
94
|
+
tmpIDs.sort((a, b) => b - a); // DESC
|
|
95
|
+
const tmpPage = tmpIDs.slice(tmpOffset, tmpOffset + tmpPageSize).map(serverDeletedRecord);
|
|
96
|
+
pResponse.end(JSON.stringify(tmpPage));
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
pResponse.statusCode = 404;
|
|
101
|
+
pResponse.end(JSON.stringify({ Error: `Unknown endpoint: ${tmpURL}` }));
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function createTestFable()
|
|
106
|
+
{
|
|
107
|
+
const tmpFable = new libFable({ Product: 'OECDeleteCursorTest', MeadowProvider: 'SQLite', SQLite: { SQLiteFilePath: ':memory:' }, LogStreams: [{ streamtype: 'console', level: 'error' }] });
|
|
108
|
+
tmpFable.ProgramConfiguration = {};
|
|
109
|
+
return tmpFable;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function readState() { try { return JSON.parse(libFS.readFileSync(STATE_PATH, 'utf8')); } catch (e) { return {}; } }
|
|
113
|
+
function deletedCount(pFable) { return pFable.MeadowSQLiteProvider.db.prepare('SELECT COUNT(*) c FROM Book WHERE Deleted=1').get().c; }
|
|
114
|
+
|
|
115
|
+
suite
|
|
116
|
+
(
|
|
117
|
+
'OngoingEventualConsistency resumable delete cursor',
|
|
118
|
+
() =>
|
|
119
|
+
{
|
|
120
|
+
let _MockServer = null;
|
|
121
|
+
let _Fable = null;
|
|
122
|
+
let _Entity = null;
|
|
123
|
+
|
|
124
|
+
suiteSetup((fDone) => { _MockServer = createMockServer(); _MockServer.listen(MOCK_PORT, fDone); });
|
|
125
|
+
suiteTeardown((fDone) => { try { libFS.unlinkSync(STATE_PATH); } catch (e) {} if (_MockServer) { _MockServer.close(fDone); } else { return fDone(); } });
|
|
126
|
+
|
|
127
|
+
setup((fDone) =>
|
|
128
|
+
{
|
|
129
|
+
_ServerDeletedIDs = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100];
|
|
130
|
+
try { libFS.unlinkSync(STATE_PATH); } catch (e) {}
|
|
131
|
+
|
|
132
|
+
_Fable = createTestFable();
|
|
133
|
+
_Fable.serviceManager.addServiceType('MeadowSQLiteProvider', libMeadowConnectionSQLite);
|
|
134
|
+
_Fable.serviceManager.instantiateServiceProvider('MeadowSQLiteProvider');
|
|
135
|
+
_Fable.MeadowSQLiteProvider.connectAsync((pErr) =>
|
|
136
|
+
{
|
|
137
|
+
if (pErr) return fDone(pErr);
|
|
138
|
+
_Fable.MeadowSQLiteProvider.db.exec(`CREATE TABLE IF NOT EXISTS Book (
|
|
139
|
+
IDBook INTEGER PRIMARY KEY AUTOINCREMENT, GUIDBook TEXT DEFAULT '', CreateDate TEXT DEFAULT '',
|
|
140
|
+
CreatingIDUser INTEGER DEFAULT 0, UpdateDate TEXT DEFAULT '', UpdatingIDUser INTEGER DEFAULT 0,
|
|
141
|
+
Deleted INTEGER DEFAULT 0, DeleteDate TEXT DEFAULT '', DeletingIDUser INTEGER DEFAULT 0, Title TEXT DEFAULT '');`);
|
|
142
|
+
|
|
143
|
+
_Fable.serviceManager.addServiceType('MeadowCloneRestClient', libMeadowCloneRestClient);
|
|
144
|
+
_Fable.serviceManager.instantiateServiceProvider('MeadowCloneRestClient', { ServerURL: MOCK_BASE_URL });
|
|
145
|
+
_Fable.serviceManager.addServiceType('MeadowSync', libMeadowSync);
|
|
146
|
+
// PageSize 3 + MaxRecordsPerEntity 3 => one page per "run" so we can watch it resume.
|
|
147
|
+
_Fable.serviceManager.instantiateServiceProvider('MeadowSync',
|
|
148
|
+
{ PageSize: 3, SyncDeletedRecords: true, BackSyncTimeLimit: 999999, MaxRecordsPerEntity: 3, DeleteCursorStatePath: STATE_PATH });
|
|
149
|
+
_Fable.MeadowSync.SyncMode = 'OngoingEventualConsistency';
|
|
150
|
+
_Fable.MeadowSync.SyncDeletedRecords = true;
|
|
151
|
+
_Fable.MeadowSync.BackSyncTimeLimit = 999999;
|
|
152
|
+
_Fable.MeadowSync.loadMeadowSchema({ Tables: { Book: _BookSchema } }, (pSchemaErr) =>
|
|
153
|
+
{
|
|
154
|
+
if (pSchemaErr) return fDone(pSchemaErr);
|
|
155
|
+
_Entity = _Fable.MeadowSync.MeadowSyncEntities['Book'];
|
|
156
|
+
_Entity.syncResults = { Created: 0, Updated: 0, Deleted: 0 };
|
|
157
|
+
// Seed all 10 as ACTIVE so each can be flagged when the cursor reaches it.
|
|
158
|
+
const tmpIns = _Fable.MeadowSQLiteProvider.db.prepare('INSERT INTO Book (IDBook, GUIDBook, Deleted, Title) VALUES (?, ?, 0, ?)');
|
|
159
|
+
for (const id of [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]) { tmpIns.run(id, `GUID-${id}`, `Active-${id}`); }
|
|
160
|
+
return fDone();
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
// Helper: run syncDeletedRecords once (a "run").
|
|
166
|
+
function run(fNext) { _Entity.syncResults = { Created: 0, Updated: 0, Deleted: 0 }; _Entity.syncDeletedRecords(fNext); }
|
|
167
|
+
|
|
168
|
+
test('drains the backlog across runs, resuming each time (no re-walk)', function (fDone)
|
|
169
|
+
{
|
|
170
|
+
this.timeout(20000);
|
|
171
|
+
// 10 deleted, 3 per run → ~4 runs to drain.
|
|
172
|
+
run(() =>
|
|
173
|
+
{
|
|
174
|
+
// Run 1: newest 3 (100,90,80) flagged; cursor advanced, not caught up.
|
|
175
|
+
Expect(deletedCount(_Fable)).to.equal(3, 'run 1 flags newest 3');
|
|
176
|
+
const s1 = readState().Book;
|
|
177
|
+
Expect(s1).to.be.an('object');
|
|
178
|
+
Expect(s1.HeadID).to.equal(100, 'head established at the top');
|
|
179
|
+
Expect(s1.TailID).to.equal(80, 'tail advanced to lowest examined');
|
|
180
|
+
Expect(s1.CaughtUp).to.equal(false);
|
|
181
|
+
|
|
182
|
+
run(() =>
|
|
183
|
+
{
|
|
184
|
+
Expect(deletedCount(_Fable)).to.equal(6, 'run 2 resumes: +3 (70,60,50)');
|
|
185
|
+
Expect(readState().Book.TailID).to.equal(50);
|
|
186
|
+
run(() =>
|
|
187
|
+
{
|
|
188
|
+
Expect(deletedCount(_Fable)).to.equal(9, 'run 3 resumes: +3 (40,30,20)');
|
|
189
|
+
run(() =>
|
|
190
|
+
{
|
|
191
|
+
// Run 4: only 10 left → flagged, then exhausted → caught up.
|
|
192
|
+
Expect(deletedCount(_Fable)).to.equal(10, 'run 4 flags the last one');
|
|
193
|
+
const s4 = readState().Book;
|
|
194
|
+
Expect(s4.CaughtUp).to.equal(true, 'tail reached the bottom → caught up');
|
|
195
|
+
return fDone();
|
|
196
|
+
});
|
|
197
|
+
});
|
|
198
|
+
});
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
test('once caught up, the head pass picks up a new high-id deletion cheaply', function (fDone)
|
|
203
|
+
{
|
|
204
|
+
this.timeout(20000);
|
|
205
|
+
// Drain fully first (4 runs).
|
|
206
|
+
run(() => run(() => run(() => run(() =>
|
|
207
|
+
{
|
|
208
|
+
Expect(deletedCount(_Fable)).to.equal(10);
|
|
209
|
+
Expect(readState().Book.CaughtUp).to.equal(true);
|
|
210
|
+
const tmpHeadBefore = readState().Book.HeadID;
|
|
211
|
+
Expect(tmpHeadBefore).to.equal(100);
|
|
212
|
+
|
|
213
|
+
// A new record (id 110) is created and deleted on the server; seed it locally as active.
|
|
214
|
+
_ServerDeletedIDs.push(110);
|
|
215
|
+
_Fable.MeadowSQLiteProvider.db.prepare('INSERT INTO Book (IDBook, GUIDBook, Deleted, Title) VALUES (110, ?, 0, ?)').run('GUID-110', 'Active-110');
|
|
216
|
+
|
|
217
|
+
run(() =>
|
|
218
|
+
{
|
|
219
|
+
// Head pass (id > 100) catches 110; tail stays caught up.
|
|
220
|
+
Expect(_Fable.MeadowSQLiteProvider.db.prepare('SELECT Deleted FROM Book WHERE IDBook=110').get().Deleted).to.equal(1, 'new high-id deletion flagged by head pass');
|
|
221
|
+
Expect(readState().Book.HeadID).to.equal(110, 'head advanced to the new max');
|
|
222
|
+
Expect(readState().Book.CaughtUp).to.equal(true, 'still caught up');
|
|
223
|
+
return fDone();
|
|
224
|
+
});
|
|
225
|
+
}))));
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
);
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Unit tests for OngoingEventualConsistency delete reconciliation.
|
|
3
|
+
|
|
4
|
+
Focused on the correctness fix: deleted server rows are matched to local
|
|
5
|
+
rows by GUID (not identity), so a cloned database that holds a record under
|
|
6
|
+
a DIFFERENT auto-increment id (the real-world failure mode) still gets the
|
|
7
|
+
row flagged deleted instead of attempting a CREATE that collides on the GUID
|
|
8
|
+
unique index.
|
|
9
|
+
|
|
10
|
+
Also covers: already-deleted rows are skipped, rows never synced live are
|
|
11
|
+
skipped (not created), and the structured report receives the real count.
|
|
12
|
+
|
|
13
|
+
Uses a mock HTTP server for the deleted-record API and an in-memory SQLite
|
|
14
|
+
database as the local clone destination. Calls syncDeletedRecords() directly
|
|
15
|
+
so the forward-sync phases don't need mocking.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const Chai = require('chai');
|
|
19
|
+
const Expect = Chai.expect;
|
|
20
|
+
|
|
21
|
+
const libHTTP = require('http');
|
|
22
|
+
const libFable = require('fable');
|
|
23
|
+
const libMeadowConnectionSQLite = require('meadow-connection-sqlite');
|
|
24
|
+
|
|
25
|
+
const libMeadowCloneRestClient = require('../source/services/clone/Meadow-Service-RestClient.js');
|
|
26
|
+
const libMeadowSync = require('../source/services/clone/Meadow-Service-Sync.js');
|
|
27
|
+
|
|
28
|
+
const MOCK_PORT = 18097;
|
|
29
|
+
const MOCK_BASE_URL = `http://localhost:${MOCK_PORT}/1.0/`;
|
|
30
|
+
|
|
31
|
+
const _BookSchema =
|
|
32
|
+
{
|
|
33
|
+
TableName: 'Book',
|
|
34
|
+
Columns:
|
|
35
|
+
[
|
|
36
|
+
{ Column: 'IDBook', DataType: 'int' },
|
|
37
|
+
{ Column: 'GUIDBook', DataType: 'GUID' },
|
|
38
|
+
{ Column: 'CreateDate', DataType: 'DateTime' },
|
|
39
|
+
{ Column: 'CreatingIDUser', DataType: 'int' },
|
|
40
|
+
{ Column: 'UpdateDate', DataType: 'DateTime' },
|
|
41
|
+
{ Column: 'UpdatingIDUser', DataType: 'int' },
|
|
42
|
+
{ Column: 'Deleted', DataType: 'int' },
|
|
43
|
+
{ Column: 'DeleteDate', DataType: 'DateTime' },
|
|
44
|
+
{ Column: 'DeletingIDUser', DataType: 'int' },
|
|
45
|
+
{ Column: 'Title', DataType: 'String' }
|
|
46
|
+
],
|
|
47
|
+
MeadowSchema:
|
|
48
|
+
{
|
|
49
|
+
Scope: 'Book',
|
|
50
|
+
DefaultIdentifier: 'IDBook',
|
|
51
|
+
Domain: 'Default',
|
|
52
|
+
Schema:
|
|
53
|
+
[
|
|
54
|
+
{ Column: 'IDBook', Type: 'AutoIdentity', Size: 'Default' },
|
|
55
|
+
{ Column: 'GUIDBook', Type: 'AutoGUID', Size: '128' },
|
|
56
|
+
{ Column: 'CreateDate', Type: 'CreateDate', Size: 'Default' },
|
|
57
|
+
{ Column: 'CreatingIDUser', Type: 'CreateIDUser', Size: 'int' },
|
|
58
|
+
{ Column: 'UpdateDate', Type: 'UpdateDate', Size: 'Default' },
|
|
59
|
+
{ Column: 'UpdatingIDUser', Type: 'UpdateIDUser', Size: 'int' },
|
|
60
|
+
{ Column: 'Deleted', Type: 'Deleted', Size: 'Default' },
|
|
61
|
+
{ Column: 'DeleteDate', Type: 'DeleteDate', Size: 'Default' },
|
|
62
|
+
{ Column: 'DeletingIDUser', Type: 'DeleteIDUser', Size: 'int' },
|
|
63
|
+
{ Column: 'Title', Type: 'String', Size: '200' }
|
|
64
|
+
],
|
|
65
|
+
DefaultObject:
|
|
66
|
+
{
|
|
67
|
+
IDBook: 0, GUIDBook: '', CreateDate: null, CreatingIDUser: 0,
|
|
68
|
+
UpdateDate: null, UpdatingIDUser: 0, Deleted: 0,
|
|
69
|
+
DeleteDate: null, DeletingIDUser: 0, Title: ''
|
|
70
|
+
},
|
|
71
|
+
JsonSchema:
|
|
72
|
+
{
|
|
73
|
+
title: 'Book', type: 'object',
|
|
74
|
+
properties:
|
|
75
|
+
{
|
|
76
|
+
IDBook: { type: 'integer' }, GUIDBook: { type: 'string' },
|
|
77
|
+
CreateDate: { type: 'string' }, CreatingIDUser: { type: 'integer' },
|
|
78
|
+
UpdateDate: { type: 'string' }, UpdatingIDUser: { type: 'integer' },
|
|
79
|
+
Deleted: { type: 'boolean' }, DeleteDate: { type: 'string' },
|
|
80
|
+
DeletingIDUser: { type: 'integer' }, Title: { type: 'string' }
|
|
81
|
+
},
|
|
82
|
+
required: ['IDBook']
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
// Server-side deleted set (ids 100-103). Returned newest-id-first.
|
|
88
|
+
function makeServerDeleted(pID)
|
|
89
|
+
{
|
|
90
|
+
return {
|
|
91
|
+
IDBook: pID,
|
|
92
|
+
GUIDBook: `GUID-BOOK-${pID}`,
|
|
93
|
+
CreateDate: '2025-01-01T00:00:00.000Z',
|
|
94
|
+
CreatingIDUser: 1,
|
|
95
|
+
UpdateDate: '2025-06-15T12:00:00.000Z',
|
|
96
|
+
UpdatingIDUser: 1,
|
|
97
|
+
Deleted: 1,
|
|
98
|
+
DeleteDate: '2025-07-01T00:00:00.000Z',
|
|
99
|
+
DeletingIDUser: 7,
|
|
100
|
+
Title: `Server-Deleted-${pID}`
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
const SERVER_DELETED = [ makeServerDeleted(100), makeServerDeleted(101), makeServerDeleted(102), makeServerDeleted(103) ];
|
|
104
|
+
|
|
105
|
+
function createMockServer()
|
|
106
|
+
{
|
|
107
|
+
return libHTTP.createServer(
|
|
108
|
+
(pRequest, pResponse) =>
|
|
109
|
+
{
|
|
110
|
+
const tmpURL = pRequest.url;
|
|
111
|
+
pResponse.setHeader('Content-Type', 'application/json');
|
|
112
|
+
|
|
113
|
+
// Count of deleted records
|
|
114
|
+
if (tmpURL.match(/\/1\.0\/Books\/Count\/FilteredTo\/FBV~Deleted~EQ~1/))
|
|
115
|
+
{
|
|
116
|
+
pResponse.end(JSON.stringify({ Count: SERVER_DELETED.length }));
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Deleted page, newest-id first (DESC) — this is the new ordering.
|
|
121
|
+
if (tmpURL.match(/\/1\.0\/Books\/FilteredTo\/FBV~Deleted~EQ~1~FSF~IDBook~DESC~DESC/))
|
|
122
|
+
{
|
|
123
|
+
const tmpParts = tmpURL.split('?')[0].split('/');
|
|
124
|
+
const tmpOffset = parseInt(tmpParts[tmpParts.length - 2], 10) || 0;
|
|
125
|
+
const tmpPageSize = parseInt(tmpParts[tmpParts.length - 1], 10) || 100;
|
|
126
|
+
const tmpSortedDesc = SERVER_DELETED.slice().sort((a, b) => b.IDBook - a.IDBook);
|
|
127
|
+
pResponse.end(JSON.stringify(tmpSortedDesc.slice(tmpOffset, tmpOffset + tmpPageSize)));
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
pResponse.statusCode = 404;
|
|
132
|
+
pResponse.end(JSON.stringify({ Error: `Unknown endpoint: ${tmpURL}` }));
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function createTestFable()
|
|
137
|
+
{
|
|
138
|
+
const tmpFable = new libFable(
|
|
139
|
+
{
|
|
140
|
+
Product: 'OECDeleteSyncTest',
|
|
141
|
+
ProductVersion: '1.0.0',
|
|
142
|
+
MeadowProvider: 'SQLite',
|
|
143
|
+
SQLite: { SQLiteFilePath: ':memory:' },
|
|
144
|
+
LogStreams: [{ streamtype: 'console', level: 'error' }]
|
|
145
|
+
});
|
|
146
|
+
tmpFable.ProgramConfiguration = {};
|
|
147
|
+
return tmpFable;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function setupSQLite(pFable, fCallback)
|
|
151
|
+
{
|
|
152
|
+
pFable.serviceManager.addServiceType('MeadowSQLiteProvider', libMeadowConnectionSQLite);
|
|
153
|
+
pFable.serviceManager.instantiateServiceProvider('MeadowSQLiteProvider');
|
|
154
|
+
pFable.MeadowSQLiteProvider.connectAsync(
|
|
155
|
+
(pError) =>
|
|
156
|
+
{
|
|
157
|
+
if (pError) return fCallback(pError);
|
|
158
|
+
pFable.MeadowSQLiteProvider.db.exec(`
|
|
159
|
+
CREATE TABLE IF NOT EXISTS Book (
|
|
160
|
+
IDBook INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
161
|
+
GUIDBook TEXT DEFAULT '',
|
|
162
|
+
CreateDate TEXT DEFAULT '',
|
|
163
|
+
CreatingIDUser INTEGER DEFAULT 0,
|
|
164
|
+
UpdateDate TEXT DEFAULT '',
|
|
165
|
+
UpdatingIDUser INTEGER DEFAULT 0,
|
|
166
|
+
Deleted INTEGER DEFAULT 0,
|
|
167
|
+
DeleteDate TEXT DEFAULT '',
|
|
168
|
+
DeletingIDUser INTEGER DEFAULT 0,
|
|
169
|
+
Title TEXT DEFAULT ''
|
|
170
|
+
);
|
|
171
|
+
`);
|
|
172
|
+
return fCallback();
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function seedBook(pFable, pID, pGUID, pDeleted, pTitle)
|
|
177
|
+
{
|
|
178
|
+
pFable.MeadowSQLiteProvider.db.prepare(
|
|
179
|
+
'INSERT INTO Book (IDBook, GUIDBook, Deleted, DeleteDate, Title) VALUES (?, ?, ?, ?, ?)')
|
|
180
|
+
.run(pID, pGUID, pDeleted, pDeleted ? '2025-07-01T00:00:00.000Z' : '', pTitle);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function allBooks(pFable)
|
|
184
|
+
{
|
|
185
|
+
return pFable.MeadowSQLiteProvider.db.prepare('SELECT * FROM Book ORDER BY IDBook').all();
|
|
186
|
+
}
|
|
187
|
+
function bookByGUID(pFable, pGUID)
|
|
188
|
+
{
|
|
189
|
+
return pFable.MeadowSQLiteProvider.db.prepare('SELECT * FROM Book WHERE GUIDBook = ?').all(pGUID);
|
|
190
|
+
}
|
|
191
|
+
function bookByID(pFable, pID)
|
|
192
|
+
{
|
|
193
|
+
return pFable.MeadowSQLiteProvider.db.prepare('SELECT * FROM Book WHERE IDBook = ?').get(pID);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
suite
|
|
197
|
+
(
|
|
198
|
+
'OngoingEventualConsistency delete reconciliation',
|
|
199
|
+
() =>
|
|
200
|
+
{
|
|
201
|
+
let _MockServer = null;
|
|
202
|
+
|
|
203
|
+
suiteSetup((fDone) => { _MockServer = createMockServer(); _MockServer.listen(MOCK_PORT, fDone); });
|
|
204
|
+
suiteTeardown((fDone) => { if (_MockServer) { _MockServer.close(fDone); } else { return fDone(); } });
|
|
205
|
+
|
|
206
|
+
let _Fable = null;
|
|
207
|
+
let _Entity = null;
|
|
208
|
+
|
|
209
|
+
setup
|
|
210
|
+
(
|
|
211
|
+
(fDone) =>
|
|
212
|
+
{
|
|
213
|
+
_Fable = createTestFable();
|
|
214
|
+
setupSQLite(_Fable,
|
|
215
|
+
(pError) =>
|
|
216
|
+
{
|
|
217
|
+
if (pError) return fDone(pError);
|
|
218
|
+
|
|
219
|
+
_Fable.serviceManager.addServiceType('MeadowCloneRestClient', libMeadowCloneRestClient);
|
|
220
|
+
_Fable.serviceManager.instantiateServiceProvider('MeadowCloneRestClient', { ServerURL: MOCK_BASE_URL });
|
|
221
|
+
|
|
222
|
+
// Wire through MeadowSync so fable.Meadow is established and the
|
|
223
|
+
// entity is built/initialized exactly as in production.
|
|
224
|
+
_Fable.serviceManager.addServiceType('MeadowSync', libMeadowSync);
|
|
225
|
+
_Fable.serviceManager.instantiateServiceProvider('MeadowSync',
|
|
226
|
+
{ PageSize: 100, SyncDeletedRecords: true, BackSyncTimeLimit: 999999 });
|
|
227
|
+
_Fable.MeadowSync.SyncMode = 'OngoingEventualConsistency';
|
|
228
|
+
_Fable.MeadowSync.SyncDeletedRecords = true;
|
|
229
|
+
_Fable.MeadowSync.BackSyncTimeLimit = 999999;
|
|
230
|
+
|
|
231
|
+
_Fable.MeadowSync.loadMeadowSchema({ Tables: { Book: _BookSchema } },
|
|
232
|
+
(pSchemaError) =>
|
|
233
|
+
{
|
|
234
|
+
if (pSchemaError) return fDone(pSchemaError);
|
|
235
|
+
_Entity = _Fable.MeadowSync.MeadowSyncEntities['Book'];
|
|
236
|
+
_Entity.syncResults = { Created: 0, Updated: 0, Deleted: 0 };
|
|
237
|
+
|
|
238
|
+
// Seed local clone state AFTER the table exists. Server has
|
|
239
|
+
// deleted ids 100, 101, 102, 103.
|
|
240
|
+
// id 101 active -> matches a deleted id -> mark deleted
|
|
241
|
+
// id 102 already deleted -> skip
|
|
242
|
+
// id 100 absent -> not in clone -> skip (no create)
|
|
243
|
+
// id 777 active, GUID twin -> shares GUID-BOOK-103 with the deleted
|
|
244
|
+
// id 103 but under a DIFFERENT id. MUST
|
|
245
|
+
// NOT be touched (the duplicate-GUID trap).
|
|
246
|
+
// id 50 unrelated active -> untouched
|
|
247
|
+
seedBook(_Fable, 101, 'GUID-BOOK-101', 0, 'Normal-101');
|
|
248
|
+
seedBook(_Fable, 102, 'GUID-BOOK-102', 1, 'Already-102');
|
|
249
|
+
seedBook(_Fable, 777, 'GUID-BOOK-103', 0, 'GuidTwin-of-deleted-103');
|
|
250
|
+
seedBook(_Fable, 50, 'GUID-BOOK-50', 0, 'Untouched-50');
|
|
251
|
+
|
|
252
|
+
return fDone();
|
|
253
|
+
});
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
);
|
|
257
|
+
|
|
258
|
+
test
|
|
259
|
+
(
|
|
260
|
+
'flags the row whose id matches the server-deleted id, and NEVER a GUID twin under a different id',
|
|
261
|
+
(fDone) =>
|
|
262
|
+
{
|
|
263
|
+
_Entity.syncDeletedRecords(
|
|
264
|
+
() =>
|
|
265
|
+
{
|
|
266
|
+
// id 101 matched a server-deleted id -> flagged deleted, DeleteDate stamped
|
|
267
|
+
const tmp101 = bookByID(_Fable, 101);
|
|
268
|
+
Expect(tmp101.Deleted).to.equal(1, 'id 101 flagged deleted');
|
|
269
|
+
Expect(tmp101.DeleteDate).to.be.a('string').and.not.equal('', 'DeleteDate stamped on delete');
|
|
270
|
+
|
|
271
|
+
// THE SAFETY CASE: id 777 shares GUID-BOOK-103 with the server-deleted id
|
|
272
|
+
// 103, but is a DIFFERENT, active record. It must be left untouched —
|
|
273
|
+
// deleting it would be data corruption from a duplicate GUID.
|
|
274
|
+
const tmp777 = bookByID(_Fable, 777);
|
|
275
|
+
Expect(tmp777.Deleted).to.equal(0, 'GUID twin under a different id MUST NOT be deleted');
|
|
276
|
+
|
|
277
|
+
// No local row at the server-deleted ids 100 or 103 was created.
|
|
278
|
+
Expect(bookByID(_Fable, 100)).to.be.undefined;
|
|
279
|
+
Expect(bookByID(_Fable, 103)).to.be.undefined;
|
|
280
|
+
return fDone();
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
);
|
|
284
|
+
|
|
285
|
+
test
|
|
286
|
+
(
|
|
287
|
+
'skips already-deleted and not-in-clone rows; creates nothing; reports the real count',
|
|
288
|
+
(fDone) =>
|
|
289
|
+
{
|
|
290
|
+
_Entity.syncDeletedRecords(
|
|
291
|
+
() =>
|
|
292
|
+
{
|
|
293
|
+
// id 102 already deleted — still exactly one row, still deleted
|
|
294
|
+
const tmp102 = bookByGUID(_Fable, 'GUID-BOOK-102');
|
|
295
|
+
Expect(tmp102.length).to.equal(1);
|
|
296
|
+
Expect(tmp102[0].Deleted).to.equal(1);
|
|
297
|
+
|
|
298
|
+
// Unrelated active row untouched
|
|
299
|
+
Expect(bookByID(_Fable, 50).Deleted).to.equal(0);
|
|
300
|
+
|
|
301
|
+
// No rows created at all — total stays at the 4 seeded (101, 102, 777, 50)
|
|
302
|
+
Expect(allBooks(_Fable).length).to.equal(4, 'no ghost rows created');
|
|
303
|
+
|
|
304
|
+
// Only id 101 was newly flagged (102 already; 100/103 not in clone)
|
|
305
|
+
Expect(_Entity.syncResults.Deleted).to.equal(1, 'reports exactly the newly-flagged count');
|
|
306
|
+
return fDone();
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
);
|