meadow-integration 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/source/cli/commands/Meadow-Integration-Command-DataClone.js +88 -0
- package/source/services/clone/Meadow-Service-ConnectionManager.js +14 -0
- package/source/services/clone/Meadow-Service-IndexConvergence.js +181 -0
- package/source/services/clone/Meadow-Service-IndexPolicy.js +165 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js +37 -57
- package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js +37 -40
- package/source/services/clone/Meadow-Service-Sync-Entity-OngoingEventualConsistency.js +2 -2
- package/source/services/clone/Meadow-Service-Sync-Entity-TrueUp.js +1 -1
- package/source/services/clone/Meadow-Service-Sync-PoolLimit.js +76 -0
- package/source/services/clone/Meadow-Service-Sync.js +12 -1
- package/test/Meadow-Integration-IndexConvergence-SQLite_test.js +144 -0
- package/test/Meadow-Integration-IndexConvergence_test.js +163 -0
- package/test/Meadow-Integration-IndexPolicy_test.js +133 -0
- package/test/Meadow-Integration-SyncConcurrency_test.js +181 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const libFableServiceProviderBase = require('fable-serviceproviderbase');
|
|
2
2
|
const libMeadowOperation = require('./Meadow-Service-Operation.js');
|
|
3
|
+
const libSyncPoolLimit = require('./Meadow-Service-Sync-PoolLimit.js');
|
|
3
4
|
|
|
4
5
|
class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
5
6
|
{
|
|
@@ -60,6 +61,13 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
|
60
61
|
? this.options.DateTimePrecisionMS
|
|
61
62
|
: 1000;
|
|
62
63
|
|
|
64
|
+
// How many records to reconcile/upsert concurrently within a table.
|
|
65
|
+
// Defaults to the DB connection pool size (capped) so fan-out never
|
|
66
|
+
// oversubscribes the pool (a 2-connection clone must not fan out 5-wide)
|
|
67
|
+
// nor runs unbounded on a very large pool. Overridable globally or
|
|
68
|
+
// per-table via SyncEntityOptions.SyncRecordConcurrency.
|
|
69
|
+
this.SyncRecordConcurrency = this._resolveSyncRecordConcurrency(this.options.SyncRecordConcurrency);
|
|
70
|
+
|
|
63
71
|
this.Meadow = false;
|
|
64
72
|
|
|
65
73
|
this.operation = new libMeadowOperation(this.fable);
|
|
@@ -67,6 +75,27 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
|
67
75
|
this.skipSync = false;
|
|
68
76
|
}
|
|
69
77
|
|
|
78
|
+
/**
|
|
79
|
+
* Resolve the per-record fan-out concurrency for this entity.
|
|
80
|
+
*
|
|
81
|
+
* Precedence: an explicit configured value (global or per-table) wins and is
|
|
82
|
+
* used as-is; otherwise fall back to the automatic default (pool size, capped)
|
|
83
|
+
* — see resolveDefaultConcurrency in Meadow-Service-Sync-PoolLimit.js.
|
|
84
|
+
*
|
|
85
|
+
* @param {number|string} pConfigured - configured concurrency, if any
|
|
86
|
+
* @returns {number} a positive integer concurrency
|
|
87
|
+
*/
|
|
88
|
+
_resolveSyncRecordConcurrency(pConfigured)
|
|
89
|
+
{
|
|
90
|
+
let tmpConfigured = parseInt(pConfigured, 10);
|
|
91
|
+
if (!isNaN(tmpConfigured) && tmpConfigured > 0)
|
|
92
|
+
{
|
|
93
|
+
return tmpConfigured;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return libSyncPoolLimit.resolveDefaultConcurrency(this.fable);
|
|
97
|
+
}
|
|
98
|
+
|
|
70
99
|
initialize(fCallback)
|
|
71
100
|
{
|
|
72
101
|
if (this.fable.hasOwnProperty('Meadow'))
|
|
@@ -78,46 +107,14 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
|
78
107
|
|
|
79
108
|
if (this.Meadow && this.Meadow.provider)
|
|
80
109
|
{
|
|
110
|
+
// Init only ensures the table exists. Index provisioning is handled
|
|
111
|
+
// separately and authoritatively by the index-convergence step
|
|
112
|
+
// (Meadow-Service-IndexConvergence), so indexes have a single writer —
|
|
113
|
+
// the legacy per-column MeadowConnectionManager.createIndex path was
|
|
114
|
+
// retired to avoid two mechanisms fighting over the same indexes.
|
|
81
115
|
return this.Meadow.provider.getProvider().createTable(this.EntitySchema, (pCreateError) =>
|
|
82
116
|
{
|
|
83
|
-
|
|
84
|
-
// in the DataCloner flow. See the comment in
|
|
85
|
-
// Meadow-Service-Sync-Entity-Initial.js for the rationale —
|
|
86
|
-
// indexes go through the provider's createIndices path
|
|
87
|
-
// (triggered by DataCloner's /indices/create endpoint) rather
|
|
88
|
-
// than via MeadowConnectionManager, which isn't registered in
|
|
89
|
-
// the DataCloner service container.
|
|
90
|
-
const tmpGUIDColumn = this.EntitySchema.Columns.find((c) => c.DataType == 'GUID');
|
|
91
|
-
const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
|
|
92
|
-
|
|
93
|
-
let tmpCanCreateIndexes = (tmpGUIDColumn || tmpDeletedColumn)
|
|
94
|
-
&& this.fable.MeadowConnectionManager
|
|
95
|
-
&& this.fable.MeadowConnectionManager.ConnectionPool
|
|
96
|
-
&& typeof(this.fable.MeadowConnectionManager.createIndex) === 'function';
|
|
97
|
-
|
|
98
|
-
if (!tmpCanCreateIndexes)
|
|
99
|
-
{
|
|
100
|
-
return fCallback(pCreateError);
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
let tmpAnticipate = this.fable.newAnticipate();
|
|
104
|
-
if (tmpGUIDColumn)
|
|
105
|
-
{
|
|
106
|
-
tmpAnticipate.anticipate(
|
|
107
|
-
(fNext) =>
|
|
108
|
-
{
|
|
109
|
-
return this.fable.MeadowConnectionManager.createIndex(this.EntitySchema, tmpGUIDColumn, true, fNext);
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
if (tmpDeletedColumn)
|
|
113
|
-
{
|
|
114
|
-
tmpAnticipate.anticipate(
|
|
115
|
-
(fNext) =>
|
|
116
|
-
{
|
|
117
|
-
return this.fable.MeadowConnectionManager.createIndex(this.EntitySchema, tmpDeletedColumn, false, fNext);
|
|
118
|
-
});
|
|
119
|
-
}
|
|
120
|
-
tmpAnticipate.wait((pIndexError) => { return fCallback(pCreateError); });
|
|
117
|
+
return fCallback(pCreateError);
|
|
121
118
|
});
|
|
122
119
|
}
|
|
123
120
|
return fCallback();
|
|
@@ -467,7 +464,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
|
467
464
|
return fCallback(null, tmpSyncedCount);
|
|
468
465
|
}
|
|
469
466
|
|
|
470
|
-
this.fable.Utility.eachLimit(pRecords,
|
|
467
|
+
this.fable.Utility.eachLimit(pRecords, this.SyncRecordConcurrency,
|
|
471
468
|
(pRecord, fRecordDone) =>
|
|
472
469
|
{
|
|
473
470
|
this._upsertRecord(pRecord,
|
|
@@ -723,7 +720,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
|
723
720
|
return fCallback();
|
|
724
721
|
}
|
|
725
722
|
|
|
726
|
-
this.fable.Utility.eachLimit(pBody,
|
|
723
|
+
this.fable.Utility.eachLimit(pBody, this.SyncRecordConcurrency,
|
|
727
724
|
(pEntityRecord, fRecordComplete) =>
|
|
728
725
|
{
|
|
729
726
|
const tmpRecordID = pEntityRecord[this.DefaultIdentifier];
|
|
@@ -260,7 +260,7 @@ class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngo
|
|
|
260
260
|
return fFinish('complete');
|
|
261
261
|
}
|
|
262
262
|
|
|
263
|
-
this.fable.Utility.eachLimit(pPageBody,
|
|
263
|
+
this.fable.Utility.eachLimit(pPageBody, this.SyncRecordConcurrency,
|
|
264
264
|
(pEntityRecord, fRecordComplete) =>
|
|
265
265
|
{
|
|
266
266
|
tmpCounters.seen++;
|
|
@@ -504,7 +504,7 @@ class MeadowSyncEntityOngoingEventualConsistency extends libMeadowSyncEntityOngo
|
|
|
504
504
|
tmpMaxID = pPageBody[0][this.DefaultIdentifier]; // DESC → first row is the highest id
|
|
505
505
|
}
|
|
506
506
|
|
|
507
|
-
this.fable.Utility.eachLimit(pPageBody,
|
|
507
|
+
this.fable.Utility.eachLimit(pPageBody, this.SyncRecordConcurrency,
|
|
508
508
|
(pEntityRecord, fRecordComplete) =>
|
|
509
509
|
{
|
|
510
510
|
tmpCounters.seen++;
|
|
@@ -130,7 +130,7 @@ class MeadowSyncEntityTrueUp extends libMeadowSyncEntityOngoing
|
|
|
130
130
|
return fStageComplete();
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
-
this.fable.Utility.eachLimit(pRecords,
|
|
133
|
+
this.fable.Utility.eachLimit(pRecords, this.SyncRecordConcurrency,
|
|
134
134
|
(pRecord, fRecordDone) =>
|
|
135
135
|
{
|
|
136
136
|
this._upsertRecord(pRecord,
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Connection providers that expose a connection pool size. The clone/sync
|
|
4
|
+
// engine defaults its fan-out (per-record reconcile, per-table init) to
|
|
5
|
+
// whichever of these is the active clone target so it never oversubscribes the
|
|
6
|
+
// pool. Only one is instantiated per clone; SQLite is intentionally absent
|
|
7
|
+
// (no pool — single writer).
|
|
8
|
+
const POOLED_CONNECTION_PROVIDER_SERVICES = [ 'MeadowMSSQLProvider', 'MeadowMySQLProvider', 'MeadowPostgreSQLProvider' ];
|
|
9
|
+
|
|
10
|
+
// Fan-out used when no pool size is reported (e.g. SQLite — single writer).
|
|
11
|
+
const DEFAULT_CONCURRENCY = 5;
|
|
12
|
+
|
|
13
|
+
// Ceiling on the AUTO-derived default fan-out. Sizing to the pool avoids
|
|
14
|
+
// oversubscribing a small pool, but a very large pool (say 100) should not make
|
|
15
|
+
// the sync fan out 100-wide by default: the concurrency also gates source-API
|
|
16
|
+
// reads, and that many in-flight reconciles would hammer the source and balloon
|
|
17
|
+
// memory. An explicit SyncRecordConcurrency / SyncTableInitConcurrency bypasses
|
|
18
|
+
// this cap — it only tempers the automatic default.
|
|
19
|
+
const MAX_DEFAULT_CONCURRENCY = 16;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Resolve the active clone target's connection pool size.
|
|
23
|
+
*
|
|
24
|
+
* Returns the pool size of whichever pooled connection provider is registered on
|
|
25
|
+
* the fable, or 0 when the active provider does not report one (e.g. SQLite) so
|
|
26
|
+
* callers can apply their own default.
|
|
27
|
+
*
|
|
28
|
+
* @param {object} pFable - the fable instance the sync is running on
|
|
29
|
+
* @returns {number} the pool size, or 0 if none is available
|
|
30
|
+
*/
|
|
31
|
+
function resolveConnectionPoolLimit(pFable)
|
|
32
|
+
{
|
|
33
|
+
if (!pFable)
|
|
34
|
+
{
|
|
35
|
+
return 0;
|
|
36
|
+
}
|
|
37
|
+
for (let i = 0; i < POOLED_CONNECTION_PROVIDER_SERVICES.length; i++)
|
|
38
|
+
{
|
|
39
|
+
let tmpProvider = pFable[POOLED_CONNECTION_PROVIDER_SERVICES[i]];
|
|
40
|
+
if (tmpProvider && (typeof(tmpProvider.connectionPoolLimit) === 'number') && tmpProvider.connectionPoolLimit > 0)
|
|
41
|
+
{
|
|
42
|
+
return tmpProvider.connectionPoolLimit;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Resolve the automatic default fan-out concurrency for the active clone target.
|
|
50
|
+
*
|
|
51
|
+
* Sizes to the connection pool so the sync neither oversubscribes a small pool
|
|
52
|
+
* nor fans out unbounded on a very large one (capped at MAX_DEFAULT_CONCURRENCY),
|
|
53
|
+
* and falls back to DEFAULT_CONCURRENCY when the provider reports no pool (e.g.
|
|
54
|
+
* SQLite). Callers apply this only when no explicit concurrency is configured;
|
|
55
|
+
* an explicit value is honored as-is and is not subject to the cap.
|
|
56
|
+
*
|
|
57
|
+
* @param {object} pFable - the fable instance the sync is running on
|
|
58
|
+
* @returns {number} a positive default concurrency
|
|
59
|
+
*/
|
|
60
|
+
function resolveDefaultConcurrency(pFable)
|
|
61
|
+
{
|
|
62
|
+
let tmpPoolLimit = resolveConnectionPoolLimit(pFable);
|
|
63
|
+
if (tmpPoolLimit > 0)
|
|
64
|
+
{
|
|
65
|
+
return Math.min(tmpPoolLimit, MAX_DEFAULT_CONCURRENCY);
|
|
66
|
+
}
|
|
67
|
+
return DEFAULT_CONCURRENCY;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
module.exports = {
|
|
71
|
+
POOLED_CONNECTION_PROVIDER_SERVICES: POOLED_CONNECTION_PROVIDER_SERVICES,
|
|
72
|
+
DEFAULT_CONCURRENCY: DEFAULT_CONCURRENCY,
|
|
73
|
+
MAX_DEFAULT_CONCURRENCY: MAX_DEFAULT_CONCURRENCY,
|
|
74
|
+
resolveConnectionPoolLimit: resolveConnectionPoolLimit,
|
|
75
|
+
resolveDefaultConcurrency: resolveDefaultConcurrency
|
|
76
|
+
};
|
|
@@ -4,6 +4,7 @@ const libMeadowSyncEntityOngoing = require('./Meadow-Service-Sync-Entity-Ongoing
|
|
|
4
4
|
const libMeadowSyncEntityOngoingEventualConsistency = require('./Meadow-Service-Sync-Entity-OngoingEventualConsistency.js');
|
|
5
5
|
const libMeadowSyncEntityTrueUp = require('./Meadow-Service-Sync-Entity-TrueUp.js');
|
|
6
6
|
const libMeadowSyncEntityComparisonOnly = require('./Meadow-Service-Sync-Entity-ComparisonOnly.js');
|
|
7
|
+
const libSyncPoolLimit = require('./Meadow-Service-Sync-PoolLimit.js');
|
|
7
8
|
|
|
8
9
|
class MeadowSync extends libFableServiceProviderBase
|
|
9
10
|
{
|
|
@@ -165,7 +166,16 @@ class MeadowSync extends libFableServiceProviderBase
|
|
|
165
166
|
let tmpErrorCount = 0;
|
|
166
167
|
let tmpSuccessCount = 0;
|
|
167
168
|
|
|
168
|
-
|
|
169
|
+
// Bound table-initialization fan-out (createTable per table) to the DB
|
|
170
|
+
// connection pool size so deploy doesn't oversubscribe a small pool.
|
|
171
|
+
// Overridable via SyncTableInitConcurrency.
|
|
172
|
+
let tmpTableInitConcurrency = parseInt(this.options.SyncTableInitConcurrency, 10);
|
|
173
|
+
if (isNaN(tmpTableInitConcurrency) || tmpTableInitConcurrency < 1)
|
|
174
|
+
{
|
|
175
|
+
tmpTableInitConcurrency = libSyncPoolLimit.resolveDefaultConcurrency(this.fable);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
this.fable.Utility.eachLimit(this.MeadowSchemaTableList, tmpTableInitConcurrency,
|
|
169
179
|
(pEntitySchemaName, fSyncInitializationComplete) =>
|
|
170
180
|
{
|
|
171
181
|
tmpEntityIndex++;
|
|
@@ -186,6 +196,7 @@ class MeadowSync extends libFableServiceProviderBase
|
|
|
186
196
|
TrueUpPageSize: this.TrueUpPageSize,
|
|
187
197
|
DeleteCursorStatePath: this.options.DeleteCursorStatePath,
|
|
188
198
|
DeleteResweepIntervalHours: this.options.DeleteResweepIntervalHours,
|
|
199
|
+
SyncRecordConcurrency: this.options.SyncRecordConcurrency,
|
|
189
200
|
};
|
|
190
201
|
|
|
191
202
|
// Apply per-entity option overrides if configured
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Live-database smoke test for index convergence.
|
|
5
|
+
*
|
|
6
|
+
* Unlike the stub-provider unit test, this runs the convergence engine against a
|
|
7
|
+
* REAL connector + REAL database (Node's built-in node:sqlite via
|
|
8
|
+
* meadow-connection-sqlite) — exercising the actual introspectTableIndices,
|
|
9
|
+
* generateCreateIndexStatements, createIndex, and the NEW dropIndex end to end.
|
|
10
|
+
* No external server / credentials required.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const Chai = require('chai');
|
|
14
|
+
const Expect = Chai.expect;
|
|
15
|
+
|
|
16
|
+
const libFs = require('fs');
|
|
17
|
+
const libPath = require('path');
|
|
18
|
+
const libOs = require('os');
|
|
19
|
+
|
|
20
|
+
const libFable = require('fable');
|
|
21
|
+
const libMeadowConnectionSQLite = require('meadow-connection-sqlite');
|
|
22
|
+
const libIndexPolicy = require('../source/services/clone/Meadow-Service-IndexPolicy.js');
|
|
23
|
+
const libConvergence = require('../source/services/clone/Meadow-Service-IndexConvergence.js');
|
|
24
|
+
|
|
25
|
+
const _Schema =
|
|
26
|
+
{
|
|
27
|
+
TableName: 'Document',
|
|
28
|
+
DefaultIdentifier: 'IDDocument',
|
|
29
|
+
Columns:
|
|
30
|
+
[
|
|
31
|
+
{ Column: 'IDDocument', DataType: 'ID' },
|
|
32
|
+
{ Column: 'GUIDDocument', DataType: 'GUID', Size: 36 },
|
|
33
|
+
{ Column: 'Deleted', DataType: 'Boolean' },
|
|
34
|
+
{ Column: 'Name', DataType: 'String', Size: 128 }
|
|
35
|
+
]
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const COMPOSITE = 'IX_M_SYNC_Document_Deleted_IDDocument';
|
|
39
|
+
const GUIDIDX = 'IX_M_SYNC_Document_GUIDDocument';
|
|
40
|
+
|
|
41
|
+
suite('Meadow Integration - IndexConvergence (live SQLite)',
|
|
42
|
+
() =>
|
|
43
|
+
{
|
|
44
|
+
let _Fable = null;
|
|
45
|
+
let _Provider = null;
|
|
46
|
+
let _DbPath = '';
|
|
47
|
+
|
|
48
|
+
setup((fDone) =>
|
|
49
|
+
{
|
|
50
|
+
_DbPath = libPath.join(libOs.tmpdir(), `mi-index-converge-${process.pid}-${_Fable ? 1 : 0}.sqlite`);
|
|
51
|
+
if (libFs.existsSync(_DbPath)) { libFs.unlinkSync(_DbPath); }
|
|
52
|
+
|
|
53
|
+
_Fable = new libFable({ Product: 'IndexConvergenceSmoke', SQLite: { SQLiteFilePath: _DbPath } });
|
|
54
|
+
_Fable.serviceManager.addServiceType('MeadowSQLiteProvider', libMeadowConnectionSQLite);
|
|
55
|
+
_Fable.serviceManager.instantiateServiceProvider('MeadowSQLiteProvider');
|
|
56
|
+
|
|
57
|
+
_Fable.MeadowSQLiteProvider.connectAsync((pError) =>
|
|
58
|
+
{
|
|
59
|
+
if (pError) { return fDone(pError); }
|
|
60
|
+
_Provider = _Fable.MeadowSQLiteProvider;
|
|
61
|
+
// Create the table (real DDL) before each test.
|
|
62
|
+
_Provider.db.exec(_Provider.generateCreateTableStatement(_Schema));
|
|
63
|
+
return fDone();
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
let converge = (pOptions, fCallback) =>
|
|
68
|
+
{
|
|
69
|
+
let tmpDesired = libIndexPolicy.resolveDesiredIndexes(_Schema, { StandardOperationalIndexes: true });
|
|
70
|
+
libConvergence.convergeTableIndexes(_Provider, _Schema, tmpDesired,
|
|
71
|
+
Object.assign({ PruneScope: 'managed', log: _Fable.log }, pOptions || {}), fCallback);
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
test('fresh table: creates the operational indexes (composite + non-unique GUID), drops nothing',
|
|
75
|
+
(fDone) =>
|
|
76
|
+
{
|
|
77
|
+
converge({}, (pError, pResult) =>
|
|
78
|
+
{
|
|
79
|
+
if (pError) { return fDone(pError); }
|
|
80
|
+
_Provider.introspectTableIndices('Document', (pIntrospectError, pActual) =>
|
|
81
|
+
{
|
|
82
|
+
try {
|
|
83
|
+
if (pIntrospectError) { return fDone(pIntrospectError); }
|
|
84
|
+
let tmpNames = pActual.map((pIndex) => { return pIndex.Name; });
|
|
85
|
+
Expect(tmpNames, 'composite present').to.include(COMPOSITE);
|
|
86
|
+
Expect(tmpNames, 'GUID index present').to.include(GUIDIDX);
|
|
87
|
+
let tmpGUID = pActual.find((pIndex) => { return pIndex.Name === GUIDIDX; });
|
|
88
|
+
Expect(tmpGUID.Unique, 'GUID index is non-unique').to.not.equal(true);
|
|
89
|
+
Expect(pResult.dropped, 'nothing dropped on a fresh table').to.have.length(0);
|
|
90
|
+
fDone();
|
|
91
|
+
} catch (e) { fDone(e); }
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test('managed prune: replaces a precursor [Deleted] index, leaves an unmanaged index alone',
|
|
97
|
+
(fDone) =>
|
|
98
|
+
{
|
|
99
|
+
// Seed a precursor-style single-column index + an external/unmanaged one.
|
|
100
|
+
_Provider.db.exec('CREATE INDEX IF NOT EXISTS "Deleted" ON "Document" ("Deleted")');
|
|
101
|
+
_Provider.db.exec('CREATE INDEX IF NOT EXISTS "IX_dba_custom" ON "Document" ("Name")');
|
|
102
|
+
|
|
103
|
+
converge({}, (pError, pResult) =>
|
|
104
|
+
{
|
|
105
|
+
if (pError) { return fDone(pError); }
|
|
106
|
+
_Provider.introspectTableIndices('Document', (pIntrospectError, pActual) =>
|
|
107
|
+
{
|
|
108
|
+
try {
|
|
109
|
+
if (pIntrospectError) { return fDone(pIntrospectError); }
|
|
110
|
+
let tmpNames = pActual.map((pIndex) => { return pIndex.Name; });
|
|
111
|
+
Expect(tmpNames, 'new composite created').to.include(COMPOSITE);
|
|
112
|
+
Expect(tmpNames, 'precursor [Deleted] dropped (managed)').to.not.include('Deleted');
|
|
113
|
+
Expect(tmpNames, 'unmanaged index untouched').to.include('IX_dba_custom');
|
|
114
|
+
Expect(pResult.dropped, 'precursor was dropped').to.include('Deleted');
|
|
115
|
+
fDone();
|
|
116
|
+
} catch (e) { fDone(e); }
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test('idempotent: a second convergence run creates and drops nothing',
|
|
122
|
+
(fDone) =>
|
|
123
|
+
{
|
|
124
|
+
converge({}, (pFirstError) =>
|
|
125
|
+
{
|
|
126
|
+
if (pFirstError) { return fDone(pFirstError); }
|
|
127
|
+
converge({}, (pError, pResult) =>
|
|
128
|
+
{
|
|
129
|
+
try {
|
|
130
|
+
if (pError) { return fDone(pError); }
|
|
131
|
+
Expect(pResult.created, 'nothing created on 2nd run').to.have.length(0);
|
|
132
|
+
Expect(pResult.dropped, 'nothing dropped on 2nd run').to.have.length(0);
|
|
133
|
+
fDone();
|
|
134
|
+
} catch (e) { fDone(e); }
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
teardown(() =>
|
|
140
|
+
{
|
|
141
|
+
try { if (_Provider && _Provider.db) { _Provider.db.close(); } } catch (e) {}
|
|
142
|
+
try { if (_DbPath && libFs.existsSync(_DbPath)) { libFs.unlinkSync(_DbPath); } } catch (e) {}
|
|
143
|
+
});
|
|
144
|
+
});
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const Chai = require('chai');
|
|
4
|
+
const Expect = Chai.expect;
|
|
5
|
+
|
|
6
|
+
const libConvergence = require('../source/services/clone/Meadow-Service-IndexConvergence.js');
|
|
7
|
+
const libIndexPolicy = require('../source/services/clone/Meadow-Service-IndexPolicy.js');
|
|
8
|
+
|
|
9
|
+
const _DocSchema =
|
|
10
|
+
{
|
|
11
|
+
TableName: 'Document',
|
|
12
|
+
DefaultIdentifier: 'IDDocument',
|
|
13
|
+
Columns:
|
|
14
|
+
[
|
|
15
|
+
{ Column: 'IDDocument', DataType: 'ID' },
|
|
16
|
+
{ Column: 'GUIDDocument', DataType: 'GUID' },
|
|
17
|
+
{ Column: 'Deleted', DataType: 'Boolean' }
|
|
18
|
+
]
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
// Desired set from the policy: (Deleted, IDDocument) composite + GUID lookup.
|
|
22
|
+
const _Desired = libIndexPolicy.resolveDesiredIndexes(_DocSchema, {});
|
|
23
|
+
const COMPOSITE = 'IX_M_SYNC_Document_Deleted_IDDocument';
|
|
24
|
+
const GUIDIDX = 'IX_M_SYNC_Document_GUIDDocument';
|
|
25
|
+
|
|
26
|
+
// Stub connection provider recording the create/drop call order.
|
|
27
|
+
function makeStubProvider(pActual, pFailCreateNames)
|
|
28
|
+
{
|
|
29
|
+
let tmpFail = new Set(pFailCreateNames || []);
|
|
30
|
+
let tmpCalls = [];
|
|
31
|
+
return {
|
|
32
|
+
calls: tmpCalls,
|
|
33
|
+
introspectTableIndices: (pTable, fCallback) => { return fCallback(null, (pActual || []).slice()); },
|
|
34
|
+
generateCreateIndexStatements: (pSchema) =>
|
|
35
|
+
{
|
|
36
|
+
return (pSchema.Indices || []).map((pIndex) =>
|
|
37
|
+
{
|
|
38
|
+
return { Name: pIndex.Name, Statement: `CREATE INDEX ${pIndex.Name}`, CheckStatement: `CHECK ${pIndex.Name}` };
|
|
39
|
+
});
|
|
40
|
+
},
|
|
41
|
+
createIndex: (pStatement, fCallback) =>
|
|
42
|
+
{
|
|
43
|
+
tmpCalls.push('create:' + pStatement.Name);
|
|
44
|
+
if (tmpFail.has(pStatement.Name)) { return fCallback(new Error('simulated create failure')); }
|
|
45
|
+
return fCallback();
|
|
46
|
+
},
|
|
47
|
+
dropIndex: (pTableName, pIndexName, fCallback) =>
|
|
48
|
+
{
|
|
49
|
+
tmpCalls.push('drop:' + pIndexName);
|
|
50
|
+
return fCallback();
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
suite('Meadow Integration - IndexConvergence',
|
|
56
|
+
() =>
|
|
57
|
+
{
|
|
58
|
+
test('additive: creates the missing desired indexes, drops nothing',
|
|
59
|
+
(fDone) =>
|
|
60
|
+
{
|
|
61
|
+
let tmpProvider = makeStubProvider([]);
|
|
62
|
+
libConvergence.convergeTableIndexes(tmpProvider, _DocSchema, _Desired, { PruneScope: 'managed' },
|
|
63
|
+
(pError, pResult) =>
|
|
64
|
+
{
|
|
65
|
+
try {
|
|
66
|
+
Expect(pError).to.equal(null);
|
|
67
|
+
Expect(pResult.created).to.have.members([ COMPOSITE, GUIDIDX ]);
|
|
68
|
+
Expect(pResult.dropped).to.have.length(0);
|
|
69
|
+
fDone();
|
|
70
|
+
} catch (e) { fDone(e); }
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test('managed convergence: replaces precursor artifacts, create BEFORE drop',
|
|
75
|
+
(fDone) =>
|
|
76
|
+
{
|
|
77
|
+
let tmpActual = [ { Name: 'Deleted', Columns: [ 'Deleted' ] }, { Name: 'GUIDDocument', Columns: [ 'GUIDDocument' ] } ];
|
|
78
|
+
let tmpProvider = makeStubProvider(tmpActual);
|
|
79
|
+
libConvergence.convergeTableIndexes(tmpProvider, _DocSchema, _Desired, { PruneScope: 'managed' },
|
|
80
|
+
(pError, pResult) =>
|
|
81
|
+
{
|
|
82
|
+
try {
|
|
83
|
+
Expect(pError).to.equal(null);
|
|
84
|
+
Expect(pResult.created).to.have.members([ COMPOSITE, GUIDIDX ]);
|
|
85
|
+
Expect(pResult.dropped).to.have.members([ 'Deleted', 'GUIDDocument' ]);
|
|
86
|
+
// Ordering: every create precedes every drop.
|
|
87
|
+
let tmpFirstDrop = tmpProvider.calls.findIndex((pCall) => pCall.indexOf('drop:') === 0);
|
|
88
|
+
let tmpLastCreate = tmpProvider.calls.map((pCall, i) => pCall.indexOf('create:') === 0 ? i : -1).reduce((a, b) => Math.max(a, b), -1);
|
|
89
|
+
Expect(tmpLastCreate).to.be.lessThan(tmpFirstDrop);
|
|
90
|
+
fDone();
|
|
91
|
+
} catch (e) { fDone(e); }
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test('no-loss-on-failure: a failed create suppresses ALL drops this run',
|
|
96
|
+
(fDone) =>
|
|
97
|
+
{
|
|
98
|
+
let tmpActual = [ { Name: 'Deleted', Columns: [ 'Deleted' ] } ];
|
|
99
|
+
let tmpProvider = makeStubProvider(tmpActual, [ COMPOSITE ]); // composite build fails
|
|
100
|
+
libConvergence.convergeTableIndexes(tmpProvider, _DocSchema, _Desired, { PruneScope: 'managed' },
|
|
101
|
+
(pError, pResult) =>
|
|
102
|
+
{
|
|
103
|
+
try {
|
|
104
|
+
Expect(pError).to.equal(null);
|
|
105
|
+
Expect(pResult.created).to.deep.equal([ GUIDIDX ]); // the GUID one still built
|
|
106
|
+
Expect(pResult.skipped).to.include(COMPOSITE); // the failed create
|
|
107
|
+
Expect(pResult.dropped).to.have.length(0); // no drops — coverage preserved
|
|
108
|
+
Expect(pResult.skipped).to.include('Deleted'); // the drop was deferred
|
|
109
|
+
Expect(tmpProvider.calls.some((pCall) => pCall.indexOf('drop:') === 0)).to.equal(false);
|
|
110
|
+
fDone();
|
|
111
|
+
} catch (e) { fDone(e); }
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test("prune 'none' never drops, even a managed undeclared index",
|
|
116
|
+
(fDone) =>
|
|
117
|
+
{
|
|
118
|
+
let tmpActual = [ { Name: 'Deleted', Columns: [ 'Deleted' ] } ];
|
|
119
|
+
let tmpProvider = makeStubProvider(tmpActual);
|
|
120
|
+
libConvergence.convergeTableIndexes(tmpProvider, _DocSchema, _Desired, { PruneScope: 'none' },
|
|
121
|
+
(pError, pResult) =>
|
|
122
|
+
{
|
|
123
|
+
try {
|
|
124
|
+
Expect(pError).to.equal(null);
|
|
125
|
+
Expect(pResult.dropped).to.have.length(0);
|
|
126
|
+
fDone();
|
|
127
|
+
} catch (e) { fDone(e); }
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test("prune 'managed' leaves a truly external index untouched",
|
|
132
|
+
(fDone) =>
|
|
133
|
+
{
|
|
134
|
+
let tmpActual = [ { Name: 'IX_dba_custom_reporting', Columns: [ 'Status' ] } ];
|
|
135
|
+
let tmpProvider = makeStubProvider(tmpActual);
|
|
136
|
+
libConvergence.convergeTableIndexes(tmpProvider, _DocSchema, _Desired, { PruneScope: 'managed' },
|
|
137
|
+
(pError, pResult) =>
|
|
138
|
+
{
|
|
139
|
+
try {
|
|
140
|
+
Expect(pError).to.equal(null);
|
|
141
|
+
Expect(pResult.dropped).to.have.length(0); // unmanaged → not dropped
|
|
142
|
+
Expect(pResult.created).to.have.members([ COMPOSITE, GUIDIDX ]);
|
|
143
|
+
fDone();
|
|
144
|
+
} catch (e) { fDone(e); }
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("prune 'all' drops any undeclared index",
|
|
149
|
+
(fDone) =>
|
|
150
|
+
{
|
|
151
|
+
let tmpActual = [ { Name: 'IX_dba_custom_reporting', Columns: [ 'Status' ] } ];
|
|
152
|
+
let tmpProvider = makeStubProvider(tmpActual);
|
|
153
|
+
libConvergence.convergeTableIndexes(tmpProvider, _DocSchema, _Desired, { PruneScope: 'all' },
|
|
154
|
+
(pError, pResult) =>
|
|
155
|
+
{
|
|
156
|
+
try {
|
|
157
|
+
Expect(pError).to.equal(null);
|
|
158
|
+
Expect(pResult.dropped).to.deep.equal([ 'IX_dba_custom_reporting' ]);
|
|
159
|
+
fDone();
|
|
160
|
+
} catch (e) { fDone(e); }
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
});
|