meadow-integration 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/publish-image.yml +2 -2
- package/BUILDING-AND-PUBLISHING.md +5 -5
- package/CONTRIBUTING.md +1 -1
- package/README.md +1 -1
- package/package.json +10 -9
- package/source/cli/commands/Meadow-Integration-Command-DataClone.js +88 -0
- package/source/services/clone/Meadow-Service-ConnectionManager.js +14 -0
- package/source/services/clone/Meadow-Service-IndexConvergence.js +181 -0
- package/source/services/clone/Meadow-Service-IndexPolicy.js +165 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js +46 -58
- package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js +37 -40
- package/source/services/clone/Meadow-Service-Sync-Entity-OngoingEventualConsistency.js +2 -2
- package/source/services/clone/Meadow-Service-Sync-Entity-TrueUp.js +1 -1
- package/source/services/clone/Meadow-Service-Sync-PoolLimit.js +76 -0
- package/source/services/clone/Meadow-Service-Sync.js +12 -1
- package/source/services/guid/Meadow-Integration-GUIDStrategy.js +32 -3
- package/test/Meadow-Integration-ComprehensionPush_test.js +2 -1
- package/test/Meadow-Integration-GUIDStrategy_test.js +23 -0
- package/test/Meadow-Integration-IndexConvergence-SQLite_test.js +144 -0
- package/test/Meadow-Integration-IndexConvergence_test.js +163 -0
- package/test/Meadow-Integration-IndexPolicy_test.js +133 -0
- package/test/Meadow-Integration-SyncConcurrency_test.js +181 -0
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
# - One step to wire: GITHUB_TOKEN already has `packages: write`
|
|
10
10
|
#
|
|
11
11
|
# Image lands at:
|
|
12
|
-
# ghcr.io/
|
|
13
|
-
# ghcr.io/
|
|
12
|
+
# ghcr.io/fable-retold/meadow-integration:<version>
|
|
13
|
+
# ghcr.io/fable-retold/meadow-integration:latest (only on the
|
|
14
14
|
# highest stable tag)
|
|
15
15
|
#
|
|
16
16
|
# Manual run: `Actions → Publish container image → Run workflow`,
|
|
@@ -170,7 +170,7 @@ postpublish: BUILD_DOCKER=1 → tag + push ← image trigger
|
|
|
170
170
|
↓ (tag arrives at GitHub)
|
|
171
171
|
.github/workflows/publish-image.yml fires:
|
|
172
172
|
- docker buildx build --platform linux/amd64,linux/arm64
|
|
173
|
-
- docker push ghcr.io/
|
|
173
|
+
- docker push ghcr.io/fable-retold/meadow-integration:<version>
|
|
174
174
|
- tags: <version>, <major>.<minor>, <major>, latest
|
|
175
175
|
```
|
|
176
176
|
|
|
@@ -189,14 +189,14 @@ After `release:patch` completes:
|
|
|
189
189
|
2. **GHCR workflow**: visit
|
|
190
190
|
`https://github.com/fable-retold/meadow-integration/actions` and
|
|
191
191
|
confirm the "Publish container image" run succeeded.
|
|
192
|
-
3. **Image**: `docker pull ghcr.io/
|
|
192
|
+
3. **Image**: `docker pull ghcr.io/fable-retold/meadow-integration:<version>`
|
|
193
193
|
should succeed. The image is also tagged as `latest`, `<major>`, and
|
|
194
194
|
`<major>.<minor>`.
|
|
195
195
|
4. **Smoke test** (one-shot — runs the bundled BookStore default schema
|
|
196
196
|
against a non-existent API; expect it to fail at the data-fetch step,
|
|
197
197
|
which proves the binary started correctly):
|
|
198
198
|
```bash
|
|
199
|
-
docker run --rm ghcr.io/
|
|
199
|
+
docker run --rm ghcr.io/fable-retold/meadow-integration:latest
|
|
200
200
|
```
|
|
201
201
|
|
|
202
202
|
---
|
|
@@ -280,7 +280,7 @@ whatever stability level fits.
|
|
|
280
280
|
### Pull and run (one-shot)
|
|
281
281
|
|
|
282
282
|
```bash
|
|
283
|
-
docker pull ghcr.io/
|
|
283
|
+
docker pull ghcr.io/fable-retold/meadow-integration:latest
|
|
284
284
|
|
|
285
285
|
# Single sync run; container exits when done
|
|
286
286
|
docker run --rm \
|
|
@@ -293,7 +293,7 @@ docker run --rm \
|
|
|
293
293
|
-e MEADOW_INTEGRATION_DB_NAME=mydatabase \
|
|
294
294
|
-e MEADOW_INTEGRATION_SCHEMA_PATH=/schemas/my-schema.json \
|
|
295
295
|
-v $(pwd)/schemas:/schemas:ro \
|
|
296
|
-
ghcr.io/
|
|
296
|
+
ghcr.io/fable-retold/meadow-integration:latest
|
|
297
297
|
```
|
|
298
298
|
|
|
299
299
|
### Configuration via env vars
|
package/CONTRIBUTING.md
CHANGED
|
@@ -47,4 +47,4 @@ When in doubt, match what the surrounding code does.
|
|
|
47
47
|
|
|
48
48
|
## Repository Structure
|
|
49
49
|
|
|
50
|
-
Each module is its own git repository. The [retold](https://github.com/
|
|
50
|
+
Each module is its own git repository. The [retold](https://github.com/fable-retold/retold) repository tracks module organization but does not contain module source code. Direct your pull request to the specific module repository where your change belongs.
|
package/README.md
CHANGED
|
@@ -361,4 +361,4 @@ MIT
|
|
|
361
361
|
|
|
362
362
|
## Contributing
|
|
363
363
|
|
|
364
|
-
Pull requests are welcome. For details on our code of conduct, contribution process, and testing requirements, see the [Retold Contributing Guide](https://github.com/
|
|
364
|
+
Pull requests are welcome. For details on our code of conduct, contribution process, and testing requirements, see the [Retold Contributing Guide](https://github.com/fable-retold/retold/blob/main/docs/contributing.md).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "meadow-integration",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.3",
|
|
4
4
|
"description": "Meadow Data Integration",
|
|
5
5
|
"retoldBeacon": {
|
|
6
6
|
"displayName": "Meadow Integration",
|
|
@@ -58,9 +58,10 @@
|
|
|
58
58
|
"author": "steven velozo <steven@velozo.com>",
|
|
59
59
|
"license": "MIT",
|
|
60
60
|
"devDependencies": {
|
|
61
|
-
"meadow-connection-sqlite": "^1.0.
|
|
61
|
+
"meadow-connection-sqlite": "^1.0.21",
|
|
62
62
|
"pict-docuserve": "^1.4.19",
|
|
63
|
-
"quackage": "^1.3.0"
|
|
63
|
+
"quackage": "^1.3.0",
|
|
64
|
+
"retold-harness": "^1.1.13"
|
|
64
65
|
},
|
|
65
66
|
"mocha": {
|
|
66
67
|
"diff": true,
|
|
@@ -81,19 +82,19 @@
|
|
|
81
82
|
]
|
|
82
83
|
},
|
|
83
84
|
"dependencies": {
|
|
84
|
-
"fable": "^3.1.
|
|
85
|
+
"fable": "^3.1.79",
|
|
85
86
|
"fable-serviceproviderbase": "^3.0.19",
|
|
86
87
|
"fast-xml-parser": "^4.4.1",
|
|
87
|
-
"meadow": "^2.0.
|
|
88
|
-
"meadow-connection-mssql": "^1.0.
|
|
89
|
-
"meadow-connection-mysql": "^1.0.
|
|
88
|
+
"meadow": "^2.0.47",
|
|
89
|
+
"meadow-connection-mssql": "^1.0.24",
|
|
90
|
+
"meadow-connection-mysql": "^1.0.20",
|
|
90
91
|
"orator": "^6.1.2",
|
|
91
92
|
"orator-serviceserver-restify": "^2.0.11",
|
|
92
93
|
"pict-provider-theme": "^1.1.2",
|
|
93
94
|
"pict-section-flow": "^1.0.1",
|
|
94
|
-
"pict-section-modal": "^1.
|
|
95
|
+
"pict-section-modal": "^1.3.2",
|
|
95
96
|
"pict-section-theme": "^1.1.1",
|
|
96
|
-
"pict-service-commandlineutility": "^1.0.
|
|
97
|
+
"pict-service-commandlineutility": "^1.0.20",
|
|
97
98
|
"pict-sessionmanager": "^1.0.2",
|
|
98
99
|
"pict-view": "^1.0.68",
|
|
99
100
|
"xlsx": "^0.18.5"
|
|
@@ -7,6 +7,8 @@ const libMeadowConnectionManager = require('../../services/clone/Meadow-Service-
|
|
|
7
7
|
const libMeadowCloneRestClient = require('../../services/clone/Meadow-Service-RestClient.js');
|
|
8
8
|
const libMeadowSync = require('../../services/clone/Meadow-Service-Sync.js');
|
|
9
9
|
const libSessionManagerSetup = require('../../Meadow-Integration-SessionManagerSetup.js');
|
|
10
|
+
const libIndexPolicy = require('../../services/clone/Meadow-Service-IndexPolicy.js');
|
|
11
|
+
const libIndexConvergence = require('../../services/clone/Meadow-Service-IndexConvergence.js');
|
|
10
12
|
|
|
11
13
|
// Resolve an env var with the standard `_FILE` suffix fallback for
|
|
12
14
|
// secrets — so docker / k8s secret mounts work without bespoke wiring.
|
|
@@ -65,6 +67,8 @@ class DataClone extends libCLICommandLineCommand
|
|
|
65
67
|
|
|
66
68
|
this.options.CommandOptions.push({ Name: '-w, --post_run_delay [post_run_delay]', Description: 'Minutes to wait after sync before exiting. Default is 0.', DefaultValue: '0' });
|
|
67
69
|
|
|
70
|
+
this.options.CommandOptions.push({ Name: '--index_config [index_config]', Description: 'Converge clone indexes before syncing. "all" applies the standard operational indexes to every table; or pass a path to a JSON index-policy file ({ StandardOperationalIndexes, PruneScope, TableIndexes }). Omit to leave indexes untouched.' });
|
|
71
|
+
|
|
68
72
|
this.addCommand();
|
|
69
73
|
}
|
|
70
74
|
|
|
@@ -373,6 +377,11 @@ class DataClone extends libCLICommandLineCommand
|
|
|
373
377
|
});
|
|
374
378
|
},
|
|
375
379
|
(fStageComplete) =>
|
|
380
|
+
{
|
|
381
|
+
// Converge indexes to the policy before syncing (opt-in via --index_config).
|
|
382
|
+
this._convergeIndexes(tmpSchemaModel, fStageComplete);
|
|
383
|
+
},
|
|
384
|
+
(fStageComplete) =>
|
|
376
385
|
{
|
|
377
386
|
// Execute the sync
|
|
378
387
|
this.fable.MeadowSync.syncAll(
|
|
@@ -405,6 +414,85 @@ class DataClone extends libCLICommandLineCommand
|
|
|
405
414
|
}, tmpPostRunDelay * 60 * 1000);
|
|
406
415
|
});
|
|
407
416
|
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Converge the clone's indexes to the configured policy before syncing.
|
|
420
|
+
* Opt-in via --index_config: "all" applies the standard operational indexes
|
|
421
|
+
* to every table; a path loads a JSON index-policy file. Uses the same
|
|
422
|
+
* IndexPolicy + IndexConvergence the headless DataCloner pipeline uses.
|
|
423
|
+
*
|
|
424
|
+
* @param {object} pSchemaModel - loaded Meadow schema ({ Tables: {...} })
|
|
425
|
+
* @param {Function} fCallback
|
|
426
|
+
*/
|
|
427
|
+
_convergeIndexes(pSchemaModel, fCallback)
|
|
428
|
+
{
|
|
429
|
+
let tmpIndexConfigArg = this.CommandOptions.index_config;
|
|
430
|
+
if (!tmpIndexConfigArg)
|
|
431
|
+
{
|
|
432
|
+
return fCallback();
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
let tmpIndexConfig;
|
|
436
|
+
if (String(tmpIndexConfigArg).toLowerCase() === 'all')
|
|
437
|
+
{
|
|
438
|
+
tmpIndexConfig = { StandardOperationalIndexes: true, PruneScope: 'managed' };
|
|
439
|
+
}
|
|
440
|
+
else
|
|
441
|
+
{
|
|
442
|
+
try
|
|
443
|
+
{
|
|
444
|
+
tmpIndexConfig = require(libPath.resolve(tmpIndexConfigArg));
|
|
445
|
+
}
|
|
446
|
+
catch (pReadError)
|
|
447
|
+
{
|
|
448
|
+
this.log.error(`Could not load --index_config ${tmpIndexConfigArg}: ${pReadError.message}`);
|
|
449
|
+
return fCallback();
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
let tmpProviderName = this.fable.MeadowConnectionManager && this.fable.MeadowConnectionManager.Provider;
|
|
454
|
+
let tmpProvider = tmpProviderName ? this.fable[`Meadow${tmpProviderName}Provider`] : null;
|
|
455
|
+
if (!tmpProvider || typeof(tmpProvider.introspectTableIndices) !== 'function')
|
|
456
|
+
{
|
|
457
|
+
this.log.warn(`Index convergence requested but provider '${tmpProviderName || '(unknown)'}' does not expose the index API; skipping.`);
|
|
458
|
+
return fCallback();
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
let tmpTables = (pSchemaModel && pSchemaModel.Tables) ? pSchemaModel.Tables : {};
|
|
462
|
+
// Converge only the tables actually deployed — respect SyncEntityList (a
|
|
463
|
+
// subset clone) so we never introspect/converge a table that was never
|
|
464
|
+
// created. Mirrors loadMeadowSchema's filter and the headless endpoint's
|
|
465
|
+
// "deployed tables only" scope. Empty list = all tables in the schema.
|
|
466
|
+
let tmpSyncList = (this.fable.MeadowSync && Array.isArray(this.fable.MeadowSync.SyncEntityList)) ? this.fable.MeadowSync.SyncEntityList : [];
|
|
467
|
+
let tmpTableNames = (tmpSyncList.length > 0)
|
|
468
|
+
? tmpSyncList.filter((pName) => { return tmpTables[pName]; })
|
|
469
|
+
: Object.keys(tmpTables);
|
|
470
|
+
let tmpPruneScope = tmpIndexConfig.PruneScope || libIndexConvergence.PRUNE_MANAGED;
|
|
471
|
+
|
|
472
|
+
this.log.info(`Converging indexes for ${tmpTableNames.length} deployed table(s) (prune scope: ${tmpPruneScope})...`);
|
|
473
|
+
|
|
474
|
+
this.fable.Utility.eachLimit(tmpTableNames, 1,
|
|
475
|
+
(pTableName, fNextTable) =>
|
|
476
|
+
{
|
|
477
|
+
let tmpTableSchema = tmpTables[pTableName];
|
|
478
|
+
let tmpDesired = libIndexPolicy.resolveDesiredIndexes(tmpTableSchema, tmpIndexConfig);
|
|
479
|
+
libIndexConvergence.convergeTableIndexes(tmpProvider, tmpTableSchema, tmpDesired,
|
|
480
|
+
{ PruneScope: tmpPruneScope, log: this.log },
|
|
481
|
+
(pConvergeError, pResult) =>
|
|
482
|
+
{
|
|
483
|
+
if (pConvergeError)
|
|
484
|
+
{
|
|
485
|
+
this.log.warn(`Index convergence for ${pTableName} failed: ${pConvergeError}`);
|
|
486
|
+
}
|
|
487
|
+
else if (pResult && (pResult.created.length > 0 || pResult.dropped.length > 0))
|
|
488
|
+
{
|
|
489
|
+
this.log.info(`${pTableName} — created [${pResult.created.join(', ')}], dropped [${pResult.dropped.join(', ')}]`);
|
|
490
|
+
}
|
|
491
|
+
return fNextTable();
|
|
492
|
+
});
|
|
493
|
+
},
|
|
494
|
+
() => { return fCallback(); });
|
|
495
|
+
}
|
|
408
496
|
}
|
|
409
497
|
|
|
410
498
|
module.exports = DataClone;
|
|
@@ -73,6 +73,15 @@ class MeadowConnectionManager extends libFableServiceProviderBase
|
|
|
73
73
|
// Apply MySQL settings to fable settings for meadow provider
|
|
74
74
|
this.fable.settings.MySQL = tmpConfig;
|
|
75
75
|
|
|
76
|
+
// Point the meadow ORM at this provider. MeadowConnectionManager
|
|
77
|
+
// connects the connector (fable.MeadowMSSQLProvider/MySQLProvider),
|
|
78
|
+
// but the meadow instance built in Meadow-Service-Sync selects its
|
|
79
|
+
// provider from fable.settings.MeadowProvider (default 'None').
|
|
80
|
+
// Without this the ORM stays on the 'None' provider and createTable
|
|
81
|
+
// (via provider.getProvider()) has nothing to call. Set before the
|
|
82
|
+
// sync service is instantiated so meadow picks it up.
|
|
83
|
+
this.fable.settings.MeadowProvider = this.Provider;
|
|
84
|
+
|
|
76
85
|
this.fable.serviceManager.addServiceType('MeadowMySQLProvider', libMeadowConnectionMySQL);
|
|
77
86
|
this.fable.serviceManager.instantiateServiceProvider('MeadowMySQLProvider', tmpConfig);
|
|
78
87
|
|
|
@@ -108,6 +117,11 @@ class MeadowConnectionManager extends libFableServiceProviderBase
|
|
|
108
117
|
// Apply MSSQL settings to fable settings for meadow provider
|
|
109
118
|
this.fable.settings.MSSQL = tmpConfig;
|
|
110
119
|
|
|
120
|
+
// Point the meadow ORM at this provider (see the MySQL branch for
|
|
121
|
+
// the full rationale) — without it the ORM stays on 'None' and
|
|
122
|
+
// createTable has no underlying connector to call.
|
|
123
|
+
this.fable.settings.MeadowProvider = this.Provider;
|
|
124
|
+
|
|
111
125
|
this.fable.serviceManager.addServiceType('MeadowMSSQLProvider', libMeadowConnectionMSSQL);
|
|
112
126
|
this.fable.serviceManager.instantiateServiceProvider('MeadowMSSQLProvider', tmpConfig);
|
|
113
127
|
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Index convergence for a clone table.
|
|
5
|
+
*
|
|
6
|
+
* Given a table's DESIRED index set (from Meadow-Service-IndexPolicy) and a
|
|
7
|
+
* connection provider, bring the table's actual indexes in line with the
|
|
8
|
+
* desired set:
|
|
9
|
+
*
|
|
10
|
+
* introspect actual → diff by name → create missing → drop undeclared
|
|
11
|
+
*
|
|
12
|
+
* Two safety properties:
|
|
13
|
+
* 1. Create-before-drop: new indexes are built BEFORE any old one is removed,
|
|
14
|
+
* so there is never a window with no covering index (matters because the
|
|
15
|
+
* sync's range-count walk runs right after).
|
|
16
|
+
* 2. No-loss-on-failure: if any desired index fails to build this run, NO
|
|
17
|
+
* drops happen — we never remove a still-needed index whose replacement is
|
|
18
|
+
* not yet in place. The drop is retried next run once the create succeeds.
|
|
19
|
+
*
|
|
20
|
+
* Prune scope controls what may be dropped:
|
|
21
|
+
* - 'none' : additive only (never drop).
|
|
22
|
+
* - 'managed' : drop only indexes the policy owns or legacy artifacts
|
|
23
|
+
* (see IndexPolicy.isManagedIndexName) — a truly external,
|
|
24
|
+
* hand-authored index is left alone. DEFAULT.
|
|
25
|
+
* - 'all' : drop any undeclared index. (introspectTableIndices already
|
|
26
|
+
* excludes the primary key, so the PK is never a drop target.)
|
|
27
|
+
*
|
|
28
|
+
* @license MIT
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
const libIndexPolicy = require('./Meadow-Service-IndexPolicy.js');
|
|
32
|
+
|
|
33
|
+
const PRUNE_NONE = 'none';
|
|
34
|
+
const PRUNE_MANAGED = 'managed';
|
|
35
|
+
const PRUNE_ALL = 'all';
|
|
36
|
+
|
|
37
|
+
const NOOP_LOG = { info: () => {}, warn: () => {}, error: () => {} };
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Converge a single table's indexes to the desired set.
|
|
41
|
+
*
|
|
42
|
+
* @param {object} pProvider - connection provider exposing introspectTableIndices,
|
|
43
|
+
* generateCreateIndexStatements, createIndex, dropIndex
|
|
44
|
+
* @param {object} pTableSchema - { TableName, Columns[] }
|
|
45
|
+
* @param {Array} pDesiredIndexes - desired defs ({ Name, TableName, Columns[], Unique, Strategy })
|
|
46
|
+
* @param {object} [pOptions] - { PruneScope?: string, log?: object }
|
|
47
|
+
* @param {Function} fCallback - callback(pError, { table, created[], dropped[], skipped[] })
|
|
48
|
+
*/
|
|
49
|
+
function convergeTableIndexes(pProvider, pTableSchema, pDesiredIndexes, pOptions, fCallback)
|
|
50
|
+
{
|
|
51
|
+
let tmpOptions = pOptions || {};
|
|
52
|
+
let tmpLog = tmpOptions.log || NOOP_LOG;
|
|
53
|
+
let tmpPruneScope = tmpOptions.PruneScope || PRUNE_MANAGED;
|
|
54
|
+
let tmpTableName = pTableSchema.TableName;
|
|
55
|
+
let tmpDesired = Array.isArray(pDesiredIndexes) ? pDesiredIndexes : [];
|
|
56
|
+
|
|
57
|
+
if (!pProvider
|
|
58
|
+
|| typeof(pProvider.introspectTableIndices) !== 'function'
|
|
59
|
+
|| typeof(pProvider.generateCreateIndexStatements) !== 'function'
|
|
60
|
+
|| typeof(pProvider.createIndex) !== 'function'
|
|
61
|
+
|| typeof(pProvider.dropIndex) !== 'function')
|
|
62
|
+
{
|
|
63
|
+
return fCallback(new Error('IndexConvergence requires a provider with introspectTableIndices / generateCreateIndexStatements / createIndex / dropIndex'));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
pProvider.introspectTableIndices(tmpTableName,
|
|
67
|
+
(pIntrospectError, pActualIndices) =>
|
|
68
|
+
{
|
|
69
|
+
if (pIntrospectError)
|
|
70
|
+
{
|
|
71
|
+
return fCallback(pIntrospectError);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
let tmpActual = Array.isArray(pActualIndices) ? pActualIndices : [];
|
|
75
|
+
let tmpActualNames = new Set(tmpActual.map((pIndex) => { return pIndex.Name; }));
|
|
76
|
+
let tmpDesiredNames = new Set(tmpDesired.map((pIndex) => { return pIndex.Name; }));
|
|
77
|
+
|
|
78
|
+
// Missing desired indexes to create.
|
|
79
|
+
let tmpToCreate = tmpDesired.filter((pIndex) => { return !tmpActualNames.has(pIndex.Name); });
|
|
80
|
+
|
|
81
|
+
// Undeclared actual indexes to drop, scoped by prune policy.
|
|
82
|
+
let tmpToDrop = [];
|
|
83
|
+
if (tmpPruneScope !== PRUNE_NONE)
|
|
84
|
+
{
|
|
85
|
+
tmpToDrop = tmpActual.filter((pIndex) =>
|
|
86
|
+
{
|
|
87
|
+
if (tmpDesiredNames.has(pIndex.Name))
|
|
88
|
+
{
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
if (tmpPruneScope === PRUNE_ALL)
|
|
92
|
+
{
|
|
93
|
+
return true;
|
|
94
|
+
}
|
|
95
|
+
return libIndexPolicy.isManagedIndexName(pIndex.Name, pTableSchema);
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
let tmpResult = { table: tmpTableName, created: [], dropped: [], skipped: [] };
|
|
100
|
+
|
|
101
|
+
// Render CREATE statements for exactly the to-create set. Passing
|
|
102
|
+
// Columns:[] means the connector's per-column auto-derivation (GUID /
|
|
103
|
+
// FK / Indexed) contributes nothing — only the explicit Indices[] we
|
|
104
|
+
// hand it are rendered.
|
|
105
|
+
let tmpCreateStatements = pProvider.generateCreateIndexStatements(
|
|
106
|
+
{ TableName: tmpTableName, Columns: [], Indices: tmpToCreate });
|
|
107
|
+
|
|
108
|
+
// -- Phase B: drop (runs after all creates) --
|
|
109
|
+
let fDropPhase = (pCreateFailures) =>
|
|
110
|
+
{
|
|
111
|
+
if (tmpToDrop.length < 1)
|
|
112
|
+
{
|
|
113
|
+
return fCallback(null, tmpResult);
|
|
114
|
+
}
|
|
115
|
+
if (pCreateFailures > 0)
|
|
116
|
+
{
|
|
117
|
+
tmpLog.warn(`IndexConvergence: ${pCreateFailures} create(s) failed on ${tmpTableName}; skipping ${tmpToDrop.length} drop(s) this run to preserve coverage.`);
|
|
118
|
+
for (let i = 0; i < tmpToDrop.length; i++)
|
|
119
|
+
{
|
|
120
|
+
tmpResult.skipped.push(tmpToDrop[i].Name);
|
|
121
|
+
}
|
|
122
|
+
return fCallback(null, tmpResult);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
let fDropNext = (pIndex) =>
|
|
126
|
+
{
|
|
127
|
+
if (pIndex >= tmpToDrop.length)
|
|
128
|
+
{
|
|
129
|
+
return fCallback(null, tmpResult);
|
|
130
|
+
}
|
|
131
|
+
let tmpDropName = tmpToDrop[pIndex].Name;
|
|
132
|
+
pProvider.dropIndex(tmpTableName, tmpDropName,
|
|
133
|
+
(pDropError) =>
|
|
134
|
+
{
|
|
135
|
+
if (pDropError)
|
|
136
|
+
{
|
|
137
|
+
tmpLog.warn(`IndexConvergence: drop ${tmpDropName} on ${tmpTableName} failed: ${pDropError}`);
|
|
138
|
+
tmpResult.skipped.push(tmpDropName);
|
|
139
|
+
}
|
|
140
|
+
else
|
|
141
|
+
{
|
|
142
|
+
tmpResult.dropped.push(tmpDropName);
|
|
143
|
+
}
|
|
144
|
+
return fDropNext(pIndex + 1);
|
|
145
|
+
});
|
|
146
|
+
};
|
|
147
|
+
fDropNext(0);
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
// -- Phase A: create --
|
|
151
|
+
let fCreateNext = (pIndex, pCreateFailures) =>
|
|
152
|
+
{
|
|
153
|
+
if (pIndex >= tmpCreateStatements.length)
|
|
154
|
+
{
|
|
155
|
+
return fDropPhase(pCreateFailures);
|
|
156
|
+
}
|
|
157
|
+
let tmpStatement = tmpCreateStatements[pIndex];
|
|
158
|
+
pProvider.createIndex(tmpStatement,
|
|
159
|
+
(pCreateError) =>
|
|
160
|
+
{
|
|
161
|
+
if (pCreateError)
|
|
162
|
+
{
|
|
163
|
+
tmpLog.warn(`IndexConvergence: create ${tmpStatement.Name} on ${tmpTableName} failed: ${pCreateError}`);
|
|
164
|
+
tmpResult.skipped.push(tmpStatement.Name);
|
|
165
|
+
return fCreateNext(pIndex + 1, pCreateFailures + 1);
|
|
166
|
+
}
|
|
167
|
+
tmpResult.created.push(tmpStatement.Name);
|
|
168
|
+
return fCreateNext(pIndex + 1, pCreateFailures);
|
|
169
|
+
});
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
fCreateNext(0, 0);
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
module.exports = {
|
|
177
|
+
PRUNE_NONE: PRUNE_NONE,
|
|
178
|
+
PRUNE_MANAGED: PRUNE_MANAGED,
|
|
179
|
+
PRUNE_ALL: PRUNE_ALL,
|
|
180
|
+
convergeTableIndexes: convergeTableIndexes
|
|
181
|
+
};
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Index policy for the clone/sync workload.
|
|
5
|
+
*
|
|
6
|
+
* This is the single source of truth for which indexes a synced table SHOULD
|
|
7
|
+
* have, independent of any one execution flow. Both the headless DataCloner
|
|
8
|
+
* pipeline and the standalone CLI clone tool resolve their desired index set
|
|
9
|
+
* from here so they converge to the same shape.
|
|
10
|
+
*
|
|
11
|
+
* The desired set for a table is the union of:
|
|
12
|
+
* 1. Standard operational indexes (applied to every eligible table):
|
|
13
|
+
* - (Deleted, ID<Table>) composite — makes the OngoingEventualConsistency
|
|
14
|
+
* range-count / delete-reconcile walk a seek instead of a scan.
|
|
15
|
+
* - a GUID lookup index — speeds meadow's app-side GUID
|
|
16
|
+
* precheck on insert. NON-UNIQUE by policy: the clone is a derived
|
|
17
|
+
* replica of data with known duplicate GUIDs at the source, and a
|
|
18
|
+
* unique index would fail to build on a table that holds a dup pair.
|
|
19
|
+
* 2. Caller-declared per-table indexes (the "observed required" extras):
|
|
20
|
+
* config.TableIndexes[<TableName>] = [ { Columns, Unique?, Name?, Strategy? } ]
|
|
21
|
+
*
|
|
22
|
+
* Every policy-produced index carries the IX_M_SYNC_ prefix so the convergence
|
|
23
|
+
* pass can recognize the set it manages. Definitions match the shape the
|
|
24
|
+
* connectors' generateCreateIndexStatements consumes:
|
|
25
|
+
* { Name, TableName, Columns[], Unique, Strategy }
|
|
26
|
+
*
|
|
27
|
+
* @license MIT
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
// All policy-managed indexes share this prefix so convergence can scope itself.
|
|
31
|
+
const INDEX_POLICY_PREFIX = 'IX_M_SYNC_';
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Resolve the identity/PK column for a table.
|
|
35
|
+
* Prefers the schema's DefaultIdentifier, then an AutoIdentity/ID column, then
|
|
36
|
+
* the ID<Table> naming convention.
|
|
37
|
+
*
|
|
38
|
+
* @param {object} pTableSchema
|
|
39
|
+
* @param {Set<string>} pColumnNames
|
|
40
|
+
* @returns {string}
|
|
41
|
+
*/
|
|
42
|
+
function resolveIdentityColumn(pTableSchema, pColumnNames)
|
|
43
|
+
{
|
|
44
|
+
let tmpIdentity = pTableSchema.DefaultIdentifier;
|
|
45
|
+
if (tmpIdentity && pColumnNames.has(tmpIdentity))
|
|
46
|
+
{
|
|
47
|
+
return tmpIdentity;
|
|
48
|
+
}
|
|
49
|
+
let tmpColumns = Array.isArray(pTableSchema.Columns) ? pTableSchema.Columns : [];
|
|
50
|
+
let tmpIdentityCol = tmpColumns.find((pColumn) => { return (pColumn.DataType === 'AutoIdentity' || pColumn.DataType === 'ID'); });
|
|
51
|
+
return tmpIdentityCol ? tmpIdentityCol.Column : `ID${pTableSchema.TableName}`;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Resolve the desired index definitions for a single table.
|
|
56
|
+
*
|
|
57
|
+
* @param {object} pTableSchema - Meadow table schema ({ TableName, Columns[], DefaultIdentifier })
|
|
58
|
+
* @param {object} [pIndexConfig] - { StandardOperationalIndexes?: boolean, TableIndexes?: { [table]: Array } }
|
|
59
|
+
* @returns {Array<{Name:string,TableName:string,Columns:string[],Unique:boolean,Strategy:string}>}
|
|
60
|
+
*/
|
|
61
|
+
function resolveDesiredIndexes(pTableSchema, pIndexConfig)
|
|
62
|
+
{
|
|
63
|
+
let tmpConfig = pIndexConfig || {};
|
|
64
|
+
let tmpTableName = pTableSchema.TableName;
|
|
65
|
+
let tmpColumns = Array.isArray(pTableSchema.Columns) ? pTableSchema.Columns : [];
|
|
66
|
+
let tmpColumnNames = new Set(tmpColumns.map((pColumn) => { return pColumn.Column; }));
|
|
67
|
+
let tmpIndices = [];
|
|
68
|
+
|
|
69
|
+
// -- 1. Standard operational indexes (all eligible tables) --
|
|
70
|
+
// Enabled by default; set StandardOperationalIndexes:false to opt a run out.
|
|
71
|
+
if (tmpConfig.StandardOperationalIndexes !== false)
|
|
72
|
+
{
|
|
73
|
+
let tmpIdentity = resolveIdentityColumn(pTableSchema, tmpColumnNames);
|
|
74
|
+
|
|
75
|
+
// (Deleted, ID) composite — equality on Deleted then range on the identity.
|
|
76
|
+
if (tmpColumnNames.has('Deleted') && tmpColumnNames.has(tmpIdentity))
|
|
77
|
+
{
|
|
78
|
+
tmpIndices.push(
|
|
79
|
+
{
|
|
80
|
+
Name: `${INDEX_POLICY_PREFIX}${tmpTableName}_Deleted_${tmpIdentity}`,
|
|
81
|
+
TableName: tmpTableName,
|
|
82
|
+
Columns: [ 'Deleted', tmpIdentity ],
|
|
83
|
+
Unique: false,
|
|
84
|
+
Strategy: ''
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// GUID lookup — NON-UNIQUE by policy (see file header).
|
|
89
|
+
let tmpGUIDColumn = tmpColumns.find((pColumn) => { return pColumn.DataType === 'GUID'; });
|
|
90
|
+
if (tmpGUIDColumn)
|
|
91
|
+
{
|
|
92
|
+
tmpIndices.push(
|
|
93
|
+
{
|
|
94
|
+
Name: `${INDEX_POLICY_PREFIX}${tmpTableName}_${tmpGUIDColumn.Column}`,
|
|
95
|
+
TableName: tmpTableName,
|
|
96
|
+
Columns: [ tmpGUIDColumn.Column ],
|
|
97
|
+
Unique: false,
|
|
98
|
+
Strategy: ''
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// -- 2. Caller-declared per-table extras --
|
|
104
|
+
let tmpTableExtras = (tmpConfig.TableIndexes && Array.isArray(tmpConfig.TableIndexes[tmpTableName]))
|
|
105
|
+
? tmpConfig.TableIndexes[tmpTableName]
|
|
106
|
+
: [];
|
|
107
|
+
for (let i = 0; i < tmpTableExtras.length; i++)
|
|
108
|
+
{
|
|
109
|
+
let tmpExtra = tmpTableExtras[i];
|
|
110
|
+
let tmpCols = Array.isArray(tmpExtra.Columns) ? tmpExtra.Columns : [ tmpExtra.Columns ];
|
|
111
|
+
// Skip a declared index whose columns are not all present on the table.
|
|
112
|
+
if (!tmpCols.every((pCol) => { return tmpColumnNames.has(pCol); }))
|
|
113
|
+
{
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
tmpIndices.push(
|
|
117
|
+
{
|
|
118
|
+
Name: tmpExtra.Name || `${INDEX_POLICY_PREFIX}${tmpTableName}_${tmpCols.join('_')}`,
|
|
119
|
+
TableName: tmpTableName,
|
|
120
|
+
Columns: tmpCols,
|
|
121
|
+
Unique: !!tmpExtra.Unique,
|
|
122
|
+
Strategy: tmpExtra.Strategy || ''
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return tmpIndices;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Whether an index name is one this policy manages — used by convergence's
|
|
131
|
+
* "managed" prune scope so it only ever drops indexes it owns. Recognizes:
|
|
132
|
+
* - policy-created indexes (IX_M_SYNC_ prefix),
|
|
133
|
+
* - meadow schema-managed indexes (AK_M / IX_M prefixes),
|
|
134
|
+
* - the legacy precursor / MeadowConnectionManager artifacts, which name a
|
|
135
|
+
* single-column index after the column itself (e.g. [Deleted], [GUIDDocument]).
|
|
136
|
+
* A truly external, hand-authored index (unusual here, but possible) is NOT
|
|
137
|
+
* managed and is left untouched.
|
|
138
|
+
*
|
|
139
|
+
* @param {string} pIndexName
|
|
140
|
+
* @param {object} pTableSchema
|
|
141
|
+
* @returns {boolean}
|
|
142
|
+
*/
|
|
143
|
+
function isManagedIndexName(pIndexName, pTableSchema)
|
|
144
|
+
{
|
|
145
|
+
if (!pIndexName)
|
|
146
|
+
{
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
if (pIndexName.indexOf(INDEX_POLICY_PREFIX) === 0
|
|
150
|
+
|| pIndexName.indexOf('AK_M') === 0
|
|
151
|
+
|| pIndexName.indexOf('IX_M') === 0)
|
|
152
|
+
{
|
|
153
|
+
return true;
|
|
154
|
+
}
|
|
155
|
+
// Legacy artifact: index named exactly after one of the table's columns.
|
|
156
|
+
let tmpColumns = Array.isArray(pTableSchema && pTableSchema.Columns) ? pTableSchema.Columns : [];
|
|
157
|
+
return tmpColumns.some((pColumn) => { return pColumn.Column === pIndexName; });
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
module.exports = {
|
|
161
|
+
INDEX_POLICY_PREFIX: INDEX_POLICY_PREFIX,
|
|
162
|
+
resolveIdentityColumn: resolveIdentityColumn,
|
|
163
|
+
resolveDesiredIndexes: resolveDesiredIndexes,
|
|
164
|
+
isManagedIndexName: isManagedIndexName
|
|
165
|
+
};
|