meadow-integration 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +11 -0
- package/Docker-Build.sh +2 -0
- package/Docker-Compose.sh +2 -0
- package/Docker-Push.sh +2 -0
- package/Docker-Tag.sh +2 -0
- package/Dockerfile +28 -0
- package/Dockerfile_LUXURYCode +23 -0
- package/README.md +139 -25
- package/docker-compose.yml +16 -0
- package/docs/README.md +65 -18
- package/docs/_cover.md +3 -2
- package/docs/_sidebar.md +52 -7
- package/docs/_topbar.md +2 -0
- package/docs/api/clone-rest-client.md +278 -0
- package/docs/api/connection-manager.md +179 -0
- package/docs/api/guid-map.md +234 -0
- package/docs/api/integration-adapter.md +283 -0
- package/docs/api/operation.md +241 -0
- package/docs/api/sync-entity-initial.md +227 -0
- package/docs/api/sync-entity-ongoing.md +244 -0
- package/docs/api/sync.md +213 -0
- package/docs/api/tabular-check.md +213 -0
- package/docs/api/tabular-transform.md +316 -0
- package/docs/architecture.md +423 -0
- package/docs/cli/comprehensionarray.md +111 -0
- package/docs/cli/comprehensionintersect.md +132 -0
- package/docs/cli/csvcheck.md +111 -0
- package/docs/cli/csvtransform.md +170 -0
- package/docs/cli/data-clone.md +277 -0
- package/docs/cli/jsonarraytransform.md +166 -0
- package/docs/cli/load-comprehension.md +129 -0
- package/docs/cli/objectarraytocsv.md +159 -0
- package/docs/cli/overview.md +96 -0
- package/docs/cli/serve.md +102 -0
- package/docs/cli/tsvtransform.md +144 -0
- package/docs/data-clone/configuration.md +357 -0
- package/docs/data-clone/connection-manager.md +206 -0
- package/docs/data-clone/docker.md +290 -0
- package/docs/data-clone/overview.md +173 -0
- package/docs/data-clone/sync-modes.md +186 -0
- package/docs/implementation-reference.md +311 -0
- package/docs/overview.md +156 -0
- package/docs/quickstart.md +233 -0
- package/docs/rest/comprehension-push.md +209 -0
- package/docs/rest/comprehension.md +506 -0
- package/docs/rest/csv.md +255 -0
- package/docs/rest/entity-generation.md +158 -0
- package/docs/rest/json-array.md +243 -0
- package/docs/rest/overview.md +120 -0
- package/docs/rest/status.md +63 -0
- package/docs/rest/tsv.md +241 -0
- package/docs/retold-catalog.json +93 -3
- package/docs/retold-keyword-index.json +23683 -1901
- package/package.json +6 -3
- package/scripts/run.sh +18 -0
- package/source/Meadow-Integration.js +15 -1
- package/source/cli/Default-Meadow-Integration-Configuration.json +37 -2
- package/source/cli/Meadow-Integration-CLI-Program.js +4 -1
- package/source/cli/commands/Meadow-Integration-Command-DataClone.js +284 -0
- package/source/services/clone/Meadow-Service-ConnectionManager.js +251 -0
- package/source/services/clone/Meadow-Service-Operation.js +196 -0
- package/source/services/clone/Meadow-Service-RestClient.js +364 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js +367 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js +457 -0
- package/source/services/clone/Meadow-Service-Sync.js +142 -0
- /package/docs/examples/bookstore/{mapping_books_Author.json → mapping_books_author.json} +0 -0
- /package/docs/examples/bookstore/{mapping_books_Book.json → mapping_books_book.json} +0 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "meadow-integration",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.6",
|
|
4
4
|
"description": "Meadow Data Integration",
|
|
5
5
|
"bin": {
|
|
6
6
|
"mdwint": "source/cli/Meadow-Integration-CLI-Run.js"
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"author": "steven velozo <steven@velozo.com>",
|
|
17
17
|
"license": "MIT",
|
|
18
18
|
"devDependencies": {
|
|
19
|
-
"quackage": "^1.0.
|
|
19
|
+
"quackage": "^1.0.61"
|
|
20
20
|
},
|
|
21
21
|
"mocha": {
|
|
22
22
|
"diff": true,
|
|
@@ -39,7 +39,10 @@
|
|
|
39
39
|
"dependencies": {
|
|
40
40
|
"fable": "^3.1.63",
|
|
41
41
|
"fable-serviceproviderbase": "^3.0.19",
|
|
42
|
-
"
|
|
42
|
+
"meadow": "^2.0.28",
|
|
43
|
+
"meadow-connection-mysql": "^1.0.13",
|
|
44
|
+
"meadow-connection-mssql": "^1.0.15",
|
|
45
|
+
"orator": "^6.0.4",
|
|
43
46
|
"orator-serviceserver-restify": "^2.0.9",
|
|
44
47
|
"pict-service-commandlineutility": "^1.0.19"
|
|
45
48
|
}
|
package/scripts/run.sh
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# capture SIGTERM and SIGINT for graceful shutdown
|
|
4
|
+
trap 'kill -TERM $PID' TERM INT
|
|
5
|
+
|
|
6
|
+
if [ -z "$RUN_LOCAL_DEV" ]; then
|
|
7
|
+
node source/cli/Meadow-Integration-CLI-Run.js data-clone &
|
|
8
|
+
PID=$!
|
|
9
|
+
else
|
|
10
|
+
node --inspect="0.0.0.0:9229" source/cli/Meadow-Integration-CLI-Run.js data-clone &
|
|
11
|
+
PID=$!
|
|
12
|
+
fi
|
|
13
|
+
|
|
14
|
+
wait $PID
|
|
15
|
+
trap - TERM INT
|
|
16
|
+
wait $PID
|
|
17
|
+
EXIT_STATUS=$?
|
|
18
|
+
echo "Service exited with status ${EXIT_STATUS}"
|
|
@@ -1,8 +1,22 @@
|
|
|
1
1
|
const libTabularCheck = require(`./services/tabular/Service-TabularCheck.js`);
|
|
2
2
|
const libIntegrationServer = require(`./restserver/Meadow-Integration-Server.js`);
|
|
3
3
|
|
|
4
|
+
const libConnectionManager = require(`./services/clone/Meadow-Service-ConnectionManager.js`);
|
|
5
|
+
const libCloneRestClient = require(`./services/clone/Meadow-Service-RestClient.js`);
|
|
6
|
+
const libSync = require(`./services/clone/Meadow-Service-Sync.js`);
|
|
7
|
+
const libSyncEntityInitial = require(`./services/clone/Meadow-Service-Sync-Entity-Initial.js`);
|
|
8
|
+
const libSyncEntityOngoing = require(`./services/clone/Meadow-Service-Sync-Entity-Ongoing.js`);
|
|
9
|
+
const libOperation = require(`./services/clone/Meadow-Service-Operation.js`);
|
|
10
|
+
|
|
4
11
|
module.exports = (
|
|
5
12
|
{
|
|
6
13
|
TabularCheck: libTabularCheck,
|
|
7
|
-
IntegrationServer: libIntegrationServer
|
|
14
|
+
IntegrationServer: libIntegrationServer,
|
|
15
|
+
|
|
16
|
+
ConnectionManager: libConnectionManager,
|
|
17
|
+
CloneRestClient: libCloneRestClient,
|
|
18
|
+
Sync: libSync,
|
|
19
|
+
SyncEntityInitial: libSyncEntityInitial,
|
|
20
|
+
SyncEntityOngoing: libSyncEntityOngoing,
|
|
21
|
+
Operation: libOperation
|
|
8
22
|
});
|
|
@@ -1,3 +1,38 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
"Source":
|
|
3
|
+
{
|
|
4
|
+
"ServerURL": "https://localhost:8080/1.0/",
|
|
5
|
+
"UserID": false,
|
|
6
|
+
"Password": false
|
|
7
|
+
},
|
|
8
|
+
"Destination":
|
|
9
|
+
{
|
|
10
|
+
"Provider": "MySQL",
|
|
11
|
+
"MySQL":
|
|
12
|
+
{
|
|
13
|
+
"server": "127.0.0.1",
|
|
14
|
+
"port": 3306,
|
|
15
|
+
"user": "root",
|
|
16
|
+
"password": "",
|
|
17
|
+
"database": "meadow",
|
|
18
|
+
"connectionLimit": 20
|
|
19
|
+
},
|
|
20
|
+
"MSSQL":
|
|
21
|
+
{
|
|
22
|
+
"server": "127.0.0.1",
|
|
23
|
+
"port": 1433,
|
|
24
|
+
"user": "sa",
|
|
25
|
+
"password": "",
|
|
26
|
+
"database": "meadow",
|
|
27
|
+
"ConnectionPoolLimit": 20
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"SchemaPath": "./schema/Model-Extended.json",
|
|
31
|
+
"Sync":
|
|
32
|
+
{
|
|
33
|
+
"DefaultSyncMode": "Initial",
|
|
34
|
+
"PageSize": 100,
|
|
35
|
+
"SyncEntityList": [],
|
|
36
|
+
"SyncEntityOptions": {}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -10,7 +10,7 @@ let _PictCLIProgram = new libCLIProgram(
|
|
|
10
10
|
|
|
11
11
|
DefaultProgramConfiguration: require('./Default-Meadow-Integration-Configuration.json'),
|
|
12
12
|
|
|
13
|
-
ProgramConfigurationFileName: '.meadow
|
|
13
|
+
ProgramConfigurationFileName: '.meadow.config.json',
|
|
14
14
|
AutoGatherProgramConfiguration: true,
|
|
15
15
|
AutoAddConfigurationExplanationCommand: true
|
|
16
16
|
},
|
|
@@ -32,6 +32,9 @@ let _PictCLIProgram = new libCLIProgram(
|
|
|
32
32
|
|
|
33
33
|
require('./commands/Meadow-Integration-Command-ComprehensionPush.js'),
|
|
34
34
|
|
|
35
|
+
// Data clone
|
|
36
|
+
require('./commands/Meadow-Integration-Command-DataClone.js'),
|
|
37
|
+
|
|
35
38
|
// REST server
|
|
36
39
|
require('./commands/Meadow-Integration-Command-Serve.js')
|
|
37
40
|
]);
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
const libCLICommandLineCommand = require('pict-service-commandlineutility').ServiceCommandLineCommand;
|
|
2
|
+
|
|
3
|
+
const libPath = require('path');
|
|
4
|
+
|
|
5
|
+
const libMeadowConnectionManager = require('../../services/clone/Meadow-Service-ConnectionManager.js');
|
|
6
|
+
const libMeadowCloneRestClient = require('../../services/clone/Meadow-Service-RestClient.js');
|
|
7
|
+
const libMeadowSync = require('../../services/clone/Meadow-Service-Sync.js');
|
|
8
|
+
|
|
9
|
+
class DataClone extends libCLICommandLineCommand
|
|
10
|
+
{
|
|
11
|
+
constructor(pFable, pOptions, pServiceHash)
|
|
12
|
+
{
|
|
13
|
+
super(pFable, pOptions, pServiceHash);
|
|
14
|
+
|
|
15
|
+
this.options.CommandKeyword = 'data-clone';
|
|
16
|
+
this.options.Description = 'Clone data from a Meadow API source to a local database.';
|
|
17
|
+
this.options.Aliases.push('clone');
|
|
18
|
+
this.options.Aliases.push('sync');
|
|
19
|
+
|
|
20
|
+
this.options.CommandOptions.push({ Name: '-a, --api_server [api_server]', Description: 'The source API server URL.' });
|
|
21
|
+
this.options.CommandOptions.push({ Name: '-u, --api_username [api_username]', Description: 'The API username to authenticate with.' });
|
|
22
|
+
this.options.CommandOptions.push({ Name: '-p, --api_password [api_password]', Description: 'The API password to authenticate with.' });
|
|
23
|
+
|
|
24
|
+
this.options.CommandOptions.push({ Name: '-d, --db_provider [db_provider]', Description: 'The database provider (MySQL or MSSQL). Default is MySQL.', DefaultValue: 'MySQL' });
|
|
25
|
+
this.options.CommandOptions.push({ Name: '-dh, --db_host [db_host]', Description: 'The database host address.' });
|
|
26
|
+
this.options.CommandOptions.push({ Name: '-dp, --db_port [db_port]', Description: 'The database port.' });
|
|
27
|
+
this.options.CommandOptions.push({ Name: '-du, --db_username [db_username]', Description: 'The database username.' });
|
|
28
|
+
this.options.CommandOptions.push({ Name: '-dw, --db_password [db_password]', Description: 'The database password.' });
|
|
29
|
+
this.options.CommandOptions.push({ Name: '-dn, --db_name [db_name]', Description: 'The database name.' });
|
|
30
|
+
|
|
31
|
+
this.options.CommandOptions.push({ Name: '-sp, --schema_path [schema_path]', Description: 'Path to the Meadow extended schema JSON file.' });
|
|
32
|
+
|
|
33
|
+
this.options.CommandOptions.push({ Name: '-s, --sync_mode [sync_mode]', Description: 'The sync mode: "Initial" or "Ongoing". Default is "Initial".', DefaultValue: 'Initial' });
|
|
34
|
+
|
|
35
|
+
this.options.CommandOptions.push({ Name: '-w, --post_run_delay [post_run_delay]', Description: 'Minutes to wait after sync before exiting. Default is 0.', DefaultValue: '0' });
|
|
36
|
+
|
|
37
|
+
this.addCommand();
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
_resolveConfig()
|
|
41
|
+
{
|
|
42
|
+
const tmpConfig = JSON.parse(JSON.stringify(this.fable.ProgramConfiguration));
|
|
43
|
+
|
|
44
|
+
// Apply command-line overrides for Source (API)
|
|
45
|
+
if (!tmpConfig.Source)
|
|
46
|
+
{
|
|
47
|
+
tmpConfig.Source = {};
|
|
48
|
+
}
|
|
49
|
+
if (this.CommandOptions.api_server)
|
|
50
|
+
{
|
|
51
|
+
tmpConfig.Source.ServerURL = this.CommandOptions.api_server;
|
|
52
|
+
}
|
|
53
|
+
if (this.CommandOptions.api_username)
|
|
54
|
+
{
|
|
55
|
+
tmpConfig.Source.UserID = this.CommandOptions.api_username;
|
|
56
|
+
}
|
|
57
|
+
if (this.CommandOptions.api_password)
|
|
58
|
+
{
|
|
59
|
+
tmpConfig.Source.Password = this.CommandOptions.api_password;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Apply command-line overrides for Destination (Database)
|
|
63
|
+
if (!tmpConfig.Destination)
|
|
64
|
+
{
|
|
65
|
+
tmpConfig.Destination = {};
|
|
66
|
+
}
|
|
67
|
+
if (this.CommandOptions.db_provider)
|
|
68
|
+
{
|
|
69
|
+
tmpConfig.Destination.Provider = this.CommandOptions.db_provider;
|
|
70
|
+
}
|
|
71
|
+
if (!tmpConfig.Destination.Provider)
|
|
72
|
+
{
|
|
73
|
+
tmpConfig.Destination.Provider = 'MySQL';
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const tmpProvider = tmpConfig.Destination.Provider;
|
|
77
|
+
if (!tmpConfig.Destination[tmpProvider])
|
|
78
|
+
{
|
|
79
|
+
tmpConfig.Destination[tmpProvider] = {};
|
|
80
|
+
}
|
|
81
|
+
if (this.CommandOptions.db_host)
|
|
82
|
+
{
|
|
83
|
+
tmpConfig.Destination[tmpProvider].server = this.CommandOptions.db_host;
|
|
84
|
+
}
|
|
85
|
+
if (this.CommandOptions.db_port)
|
|
86
|
+
{
|
|
87
|
+
tmpConfig.Destination[tmpProvider].port = parseInt(this.CommandOptions.db_port);
|
|
88
|
+
}
|
|
89
|
+
if (this.CommandOptions.db_username)
|
|
90
|
+
{
|
|
91
|
+
tmpConfig.Destination[tmpProvider].user = this.CommandOptions.db_username;
|
|
92
|
+
}
|
|
93
|
+
if (this.CommandOptions.db_password)
|
|
94
|
+
{
|
|
95
|
+
tmpConfig.Destination[tmpProvider].password = this.CommandOptions.db_password;
|
|
96
|
+
}
|
|
97
|
+
if (this.CommandOptions.db_name)
|
|
98
|
+
{
|
|
99
|
+
tmpConfig.Destination[tmpProvider].database = this.CommandOptions.db_name;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Schema path override
|
|
103
|
+
if (this.CommandOptions.schema_path)
|
|
104
|
+
{
|
|
105
|
+
tmpConfig.SchemaPath = this.CommandOptions.schema_path;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Sync config
|
|
109
|
+
if (!tmpConfig.Sync)
|
|
110
|
+
{
|
|
111
|
+
tmpConfig.Sync = {};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return tmpConfig;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
onRunAsync(fCallback)
|
|
118
|
+
{
|
|
119
|
+
const tmpConfig = this._resolveConfig();
|
|
120
|
+
|
|
121
|
+
// Validate required configuration
|
|
122
|
+
if (!tmpConfig.Source || !tmpConfig.Source.ServerURL)
|
|
123
|
+
{
|
|
124
|
+
this.log.error('No source API server URL configured. Set Source.ServerURL in .meadow.config.json or use --api_server.');
|
|
125
|
+
return fCallback(new Error('Missing Source.ServerURL configuration.'));
|
|
126
|
+
}
|
|
127
|
+
if (!tmpConfig.SchemaPath)
|
|
128
|
+
{
|
|
129
|
+
this.log.error('No schema path configured. Set SchemaPath in .meadow.config.json or use --schema_path.');
|
|
130
|
+
return fCallback(new Error('Missing SchemaPath configuration.'));
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const tmpProvider = tmpConfig.Destination.Provider;
|
|
134
|
+
const tmpDbConfig = tmpConfig.Destination[tmpProvider];
|
|
135
|
+
if (!tmpDbConfig || !tmpDbConfig.server || !tmpDbConfig.database)
|
|
136
|
+
{
|
|
137
|
+
this.log.error(`Database configuration incomplete for provider ${tmpProvider}. Set Destination.${tmpProvider} in .meadow.config.json or use --db_host/--db_name.`);
|
|
138
|
+
return fCallback(new Error(`Missing ${tmpProvider} database configuration.`));
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
this.log.info(`Data Clone: ${tmpProvider} database [${tmpDbConfig.database}] from [${tmpConfig.Source.ServerURL}]`);
|
|
142
|
+
|
|
143
|
+
// Load schema
|
|
144
|
+
let tmpSchemaModel;
|
|
145
|
+
try
|
|
146
|
+
{
|
|
147
|
+
const tmpSchemaPath = libPath.resolve(tmpConfig.SchemaPath);
|
|
148
|
+
this.log.info(`Loading schema from ${tmpSchemaPath}...`);
|
|
149
|
+
tmpSchemaModel = require(tmpSchemaPath);
|
|
150
|
+
}
|
|
151
|
+
catch (pError)
|
|
152
|
+
{
|
|
153
|
+
this.log.error(`Error loading schema from ${tmpConfig.SchemaPath}: ${pError.message}`);
|
|
154
|
+
return fCallback(pError);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Register and instantiate services
|
|
158
|
+
this.fable.serviceManager.addServiceType('MeadowCloneRestClient', libMeadowCloneRestClient);
|
|
159
|
+
this.fable.serviceManager.instantiateServiceProvider('MeadowCloneRestClient', tmpConfig.Source);
|
|
160
|
+
|
|
161
|
+
this.fable.serviceManager.addServiceType('MeadowConnectionManager', libMeadowConnectionManager);
|
|
162
|
+
this.fable.serviceManager.instantiateServiceProvider('MeadowConnectionManager', tmpConfig.Destination);
|
|
163
|
+
|
|
164
|
+
this.fable.serviceManager.addServiceType('MeadowSync', libMeadowSync);
|
|
165
|
+
|
|
166
|
+
this.fable.Utility.waterfall(
|
|
167
|
+
[
|
|
168
|
+
(fStageComplete) =>
|
|
169
|
+
{
|
|
170
|
+
// Authenticate with the source API
|
|
171
|
+
if (tmpConfig.Source.UserID && tmpConfig.Source.Password)
|
|
172
|
+
{
|
|
173
|
+
this.log.info('Authenticating with source API...');
|
|
174
|
+
this.fable.MeadowCloneRestClient.authenticate(
|
|
175
|
+
(pError, pResponse) =>
|
|
176
|
+
{
|
|
177
|
+
if (pError)
|
|
178
|
+
{
|
|
179
|
+
this.log.error('Error authenticating with source API.', pError);
|
|
180
|
+
}
|
|
181
|
+
else
|
|
182
|
+
{
|
|
183
|
+
this.log.info(`Authenticated with source API as [${tmpConfig.Source.UserID}] at [${tmpConfig.Source.ServerURL}]`);
|
|
184
|
+
}
|
|
185
|
+
return fStageComplete();
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
else
|
|
189
|
+
{
|
|
190
|
+
this.log.info('No credentials configured; skipping authentication.');
|
|
191
|
+
return fStageComplete();
|
|
192
|
+
}
|
|
193
|
+
},
|
|
194
|
+
(fStageComplete) =>
|
|
195
|
+
{
|
|
196
|
+
// Connect to database
|
|
197
|
+
this.log.info(`Connecting to ${tmpProvider}...`);
|
|
198
|
+
this.fable.MeadowConnectionManager.connect(
|
|
199
|
+
(pError, pConnectionPool) =>
|
|
200
|
+
{
|
|
201
|
+
if (pError)
|
|
202
|
+
{
|
|
203
|
+
this.log.error(`Error connecting to ${tmpProvider}: ${pError}`, pError);
|
|
204
|
+
return fStageComplete(pError);
|
|
205
|
+
}
|
|
206
|
+
return fStageComplete(null, pConnectionPool);
|
|
207
|
+
});
|
|
208
|
+
},
|
|
209
|
+
(pConnectionPool, fStageComplete) =>
|
|
210
|
+
{
|
|
211
|
+
// Construct and configure the sync service
|
|
212
|
+
const tmpSyncConfig = Object.assign(
|
|
213
|
+
{
|
|
214
|
+
ConnectionPool: pConnectionPool,
|
|
215
|
+
PageSize: tmpConfig.Sync.PageSize || 100,
|
|
216
|
+
},
|
|
217
|
+
tmpConfig.Sync);
|
|
218
|
+
|
|
219
|
+
this.fable.serviceManager.instantiateServiceProvider('MeadowSync', tmpSyncConfig, 'SyncService');
|
|
220
|
+
|
|
221
|
+
const tmpSyncMode = this.CommandOptions.sync_mode || tmpConfig.Sync.DefaultSyncMode || 'Initial';
|
|
222
|
+
switch (tmpSyncMode)
|
|
223
|
+
{
|
|
224
|
+
case 'Ongoing':
|
|
225
|
+
this.fable.MeadowSync.SyncMode = 'Ongoing';
|
|
226
|
+
break;
|
|
227
|
+
case 'Initial':
|
|
228
|
+
default:
|
|
229
|
+
this.fable.MeadowSync.SyncMode = 'Initial';
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
this.log.info(`Sync mode: ${this.fable.MeadowSync.SyncMode}`);
|
|
234
|
+
|
|
235
|
+
// Load the schema
|
|
236
|
+
this.fable.MeadowSync.loadMeadowSchema(tmpSchemaModel,
|
|
237
|
+
(pLoadSchemaError) =>
|
|
238
|
+
{
|
|
239
|
+
if (pLoadSchemaError)
|
|
240
|
+
{
|
|
241
|
+
this.log.error(`Error loading schema: ${pLoadSchemaError}`, pLoadSchemaError);
|
|
242
|
+
return fStageComplete(pLoadSchemaError);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
this.log.info(`${this.fable.MeadowSync.SyncEntityList.length} schema(s) loaded successfully.`);
|
|
246
|
+
return fStageComplete();
|
|
247
|
+
});
|
|
248
|
+
},
|
|
249
|
+
(fStageComplete) =>
|
|
250
|
+
{
|
|
251
|
+
// Execute the sync
|
|
252
|
+
this.fable.MeadowSync.syncAll(
|
|
253
|
+
(pSyncError) =>
|
|
254
|
+
{
|
|
255
|
+
if (pSyncError)
|
|
256
|
+
{
|
|
257
|
+
this.log.error(`Error syncing: ${pSyncError}`, pSyncError);
|
|
258
|
+
return fStageComplete(pSyncError);
|
|
259
|
+
}
|
|
260
|
+
this.log.info('Sync complete.');
|
|
261
|
+
return fStageComplete();
|
|
262
|
+
});
|
|
263
|
+
},
|
|
264
|
+
],
|
|
265
|
+
(pError) =>
|
|
266
|
+
{
|
|
267
|
+
const tmpPostRunDelay = parseInt(this.CommandOptions.post_run_delay) || 0;
|
|
268
|
+
if (tmpPostRunDelay > 0)
|
|
269
|
+
{
|
|
270
|
+
this.log.info(`Waiting ${tmpPostRunDelay} minutes before exiting...`);
|
|
271
|
+
}
|
|
272
|
+
setTimeout(() =>
|
|
273
|
+
{
|
|
274
|
+
fCallback(pError);
|
|
275
|
+
setTimeout(() =>
|
|
276
|
+
{
|
|
277
|
+
process.exit(pError ? 1 : 0);
|
|
278
|
+
}, 10000);
|
|
279
|
+
}, tmpPostRunDelay * 60 * 1000);
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
module.exports = DataClone;
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
const libFableServiceProviderBase = require('fable-serviceproviderbase');
|
|
2
|
+
|
|
3
|
+
const defaultConnectionManagerOptions = (
|
|
4
|
+
{
|
|
5
|
+
Provider: 'MySQL',
|
|
6
|
+
|
|
7
|
+
MySQL:
|
|
8
|
+
{
|
|
9
|
+
server: '127.0.0.1',
|
|
10
|
+
port: 3306,
|
|
11
|
+
user: 'root',
|
|
12
|
+
password: '',
|
|
13
|
+
database: 'meadow',
|
|
14
|
+
connectionLimit: 20,
|
|
15
|
+
},
|
|
16
|
+
|
|
17
|
+
MSSQL:
|
|
18
|
+
{
|
|
19
|
+
server: '127.0.0.1',
|
|
20
|
+
port: 1433,
|
|
21
|
+
user: 'sa',
|
|
22
|
+
password: '',
|
|
23
|
+
database: 'meadow',
|
|
24
|
+
ConnectionPoolLimit: 20,
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
class MeadowConnectionManager extends libFableServiceProviderBase
|
|
29
|
+
{
|
|
30
|
+
constructor(pFable, pOptions, pServiceHash)
|
|
31
|
+
{
|
|
32
|
+
const tmpOptions = Object.assign({}, defaultConnectionManagerOptions, pOptions);
|
|
33
|
+
super(pFable, tmpOptions, pServiceHash);
|
|
34
|
+
|
|
35
|
+
this.serviceType = 'MeadowConnectionManager';
|
|
36
|
+
|
|
37
|
+
this.Provider = this.options.Provider;
|
|
38
|
+
this.ConnectionPool = false;
|
|
39
|
+
this._Connected = false;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
get connected()
|
|
43
|
+
{
|
|
44
|
+
return this._Connected;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
connect(fCallback)
|
|
48
|
+
{
|
|
49
|
+
switch (this.Provider)
|
|
50
|
+
{
|
|
51
|
+
case 'MySQL':
|
|
52
|
+
return this._connectMySQL(fCallback);
|
|
53
|
+
case 'MSSQL':
|
|
54
|
+
return this._connectMSSQL(fCallback);
|
|
55
|
+
default:
|
|
56
|
+
return fCallback(new Error(`Unsupported database provider: ${this.Provider}`));
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
_connectMySQL(fCallback)
|
|
61
|
+
{
|
|
62
|
+
try
|
|
63
|
+
{
|
|
64
|
+
const libMeadowConnectionMySQL = require('meadow-connection-mysql');
|
|
65
|
+
const tmpConfig = Object.assign({}, this.options.MySQL);
|
|
66
|
+
|
|
67
|
+
// meadow-mysql treats connectionLimit as a global setting
|
|
68
|
+
if (tmpConfig.connectionLimit && typeof(this.fable.settings.connectionLimit) !== 'number')
|
|
69
|
+
{
|
|
70
|
+
this.fable.settings.connectionLimit = tmpConfig.connectionLimit;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Apply MySQL settings to fable settings for meadow provider
|
|
74
|
+
this.fable.settings.MySQL = tmpConfig;
|
|
75
|
+
|
|
76
|
+
this.fable.serviceManager.addServiceType('MeadowMySQLProvider', libMeadowConnectionMySQL);
|
|
77
|
+
this.fable.serviceManager.instantiateServiceProvider('MeadowMySQLProvider', tmpConfig);
|
|
78
|
+
|
|
79
|
+
this.fable.MeadowMySQLProvider.connectAsync(
|
|
80
|
+
(pError, pConnectionPool) =>
|
|
81
|
+
{
|
|
82
|
+
if (pError)
|
|
83
|
+
{
|
|
84
|
+
this.log.error(`Error connecting to MySQL: ${pError}`, pError);
|
|
85
|
+
return fCallback(pError);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
this.ConnectionPool = pConnectionPool;
|
|
89
|
+
this._Connected = true;
|
|
90
|
+
this.log.info('Connected to MySQL successfully.');
|
|
91
|
+
return fCallback(null, pConnectionPool);
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
catch (pError)
|
|
95
|
+
{
|
|
96
|
+
this.log.error(`Failed to load MySQL provider. Ensure meadow-connection-mysql is installed: ${pError.message}`);
|
|
97
|
+
return fCallback(pError);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
_connectMSSQL(fCallback)
|
|
102
|
+
{
|
|
103
|
+
try
|
|
104
|
+
{
|
|
105
|
+
const libMeadowConnectionMSSQL = require('meadow-connection-mssql');
|
|
106
|
+
const tmpConfig = Object.assign({}, this.options.MSSQL);
|
|
107
|
+
|
|
108
|
+
// Apply MSSQL settings to fable settings for meadow provider
|
|
109
|
+
this.fable.settings.MSSQL = tmpConfig;
|
|
110
|
+
|
|
111
|
+
this.fable.serviceManager.addServiceType('MeadowMSSQLProvider', libMeadowConnectionMSSQL);
|
|
112
|
+
this.fable.serviceManager.instantiateServiceProvider('MeadowMSSQLProvider', tmpConfig);
|
|
113
|
+
|
|
114
|
+
this.fable.MeadowMSSQLProvider.connectAsync(
|
|
115
|
+
(pError, pConnectionPool) =>
|
|
116
|
+
{
|
|
117
|
+
if (pError)
|
|
118
|
+
{
|
|
119
|
+
this.log.error(`Error connecting to MSSQL: ${pError}`, pError);
|
|
120
|
+
return fCallback(pError);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
this.ConnectionPool = pConnectionPool;
|
|
124
|
+
this._Connected = true;
|
|
125
|
+
this.log.info('Connected to MSSQL successfully.');
|
|
126
|
+
return fCallback(null, pConnectionPool);
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
catch (pError)
|
|
130
|
+
{
|
|
131
|
+
this.log.error(`Failed to load MSSQL provider. Ensure meadow-connection-mssql is installed: ${pError.message}`);
|
|
132
|
+
return fCallback(pError);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
createIndex(pEntitySchema, pColumn, pIsUnique, fCallback)
|
|
137
|
+
{
|
|
138
|
+
switch (this.Provider)
|
|
139
|
+
{
|
|
140
|
+
case 'MySQL':
|
|
141
|
+
return this._createMySQLIndex(pEntitySchema, pColumn, pIsUnique, fCallback);
|
|
142
|
+
case 'MSSQL':
|
|
143
|
+
return this._createMSSQLIndex(pEntitySchema, pColumn, pIsUnique, fCallback);
|
|
144
|
+
default:
|
|
145
|
+
return fCallback();
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
_createMySQLIndex(pEntitySchema, pColumn, pIsUnique, fCallback)
|
|
150
|
+
{
|
|
151
|
+
if (!this.ConnectionPool)
|
|
152
|
+
{
|
|
153
|
+
this.log.error(`No connection pool available; skipping index creation for ${pEntitySchema.TableName}`);
|
|
154
|
+
return fCallback();
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const tmpTableName = pEntitySchema.TableName;
|
|
158
|
+
|
|
159
|
+
if (!pColumn || (typeof(pColumn) != 'object') || !pColumn.hasOwnProperty('Column') || (typeof(pColumn.Column) != 'string') || (pColumn.Column.length < 1))
|
|
160
|
+
{
|
|
161
|
+
this.log.error(`No column information passed to createIndex for ${tmpTableName}; skipping index creation`);
|
|
162
|
+
return fCallback();
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const tmpColumnName = pColumn.Column;
|
|
166
|
+
const tmpIndexName = `AK_${tmpTableName}_${tmpColumnName}`;
|
|
167
|
+
const tmpIndexIsUnique = (typeof(pIsUnique) == 'boolean') ? pIsUnique : false;
|
|
168
|
+
const tmpCheckIndexSQL = `
|
|
169
|
+
SELECT COUNT(1) AS IndexCount FROM INFORMATION_SCHEMA.STATISTICS
|
|
170
|
+
WHERE
|
|
171
|
+
table_name = '${tmpTableName}'
|
|
172
|
+
AND index_name='${tmpIndexName}'
|
|
173
|
+
AND table_schema = DATABASE()
|
|
174
|
+
`;
|
|
175
|
+
this.ConnectionPool.query(tmpCheckIndexSQL,
|
|
176
|
+
(pError, pResult) =>
|
|
177
|
+
{
|
|
178
|
+
if (pError)
|
|
179
|
+
{
|
|
180
|
+
this.log.error(`Error checking for existing index ${tmpIndexName} on ${tmpTableName}:`, pError);
|
|
181
|
+
return fCallback(pError);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const tmpIndexExists = pResult[0].IndexCount > 0;
|
|
185
|
+
if (tmpIndexExists)
|
|
186
|
+
{
|
|
187
|
+
this.log.info(`Index ${tmpIndexName} already exists on ${tmpTableName}; skipping creation.`);
|
|
188
|
+
return fCallback();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
let tmpCreateIndexSQL = `CREATE `;
|
|
192
|
+
if (tmpIndexIsUnique)
|
|
193
|
+
{
|
|
194
|
+
tmpCreateIndexSQL += `UNIQUE `;
|
|
195
|
+
}
|
|
196
|
+
tmpCreateIndexSQL += `INDEX ${tmpIndexName} ON ${tmpTableName} (${tmpColumnName})`;
|
|
197
|
+
this.ConnectionPool.query(tmpCreateIndexSQL,
|
|
198
|
+
(pCreateIndexError) =>
|
|
199
|
+
{
|
|
200
|
+
if (pCreateIndexError && pCreateIndexError.code !== 'ER_DUP_KEYNAME')
|
|
201
|
+
{
|
|
202
|
+
this.log.error(`Error creating index ${tmpIndexName} on ${tmpTableName}:`, pCreateIndexError);
|
|
203
|
+
return fCallback(pCreateIndexError);
|
|
204
|
+
}
|
|
205
|
+
this.log.info(`Index ${tmpIndexName} created on ${tmpTableName}.`);
|
|
206
|
+
return fCallback();
|
|
207
|
+
});
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
_createMSSQLIndex(pEntitySchema, pColumn, pIsUnique, fCallback)
|
|
212
|
+
{
|
|
213
|
+
if (!this.ConnectionPool)
|
|
214
|
+
{
|
|
215
|
+
this.log.error(`No connection pool available; skipping index creation for ${pEntitySchema.TableName}`);
|
|
216
|
+
return fCallback();
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const tmpTableName = pEntitySchema.TableName;
|
|
220
|
+
|
|
221
|
+
if (!pColumn || (typeof(pColumn) != 'object') || !pColumn.hasOwnProperty('Column') || (typeof(pColumn.Column) != 'string') || (pColumn.Column.length < 1))
|
|
222
|
+
{
|
|
223
|
+
this.log.error(`No column information passed to createIndex for ${tmpTableName}; skipping index creation`);
|
|
224
|
+
return fCallback();
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const tmpColumnName = pColumn.Column;
|
|
228
|
+
const tmpIndexSQL = `
|
|
229
|
+
IF NOT EXISTS(SELECT * FROM sys.indexes WHERE name = '${tmpColumnName}' AND object_id = OBJECT_ID('${tmpTableName}'))
|
|
230
|
+
BEGIN
|
|
231
|
+
CREATE INDEX [${tmpColumnName}] ON [dbo].[${tmpTableName}] ([${tmpColumnName}])
|
|
232
|
+
END;
|
|
233
|
+
`;
|
|
234
|
+
|
|
235
|
+
this.ConnectionPool.query(tmpIndexSQL)
|
|
236
|
+
.then(() =>
|
|
237
|
+
{
|
|
238
|
+
this.log.info(`Index ${tmpColumnName} created on ${tmpTableName}.`);
|
|
239
|
+
return fCallback();
|
|
240
|
+
})
|
|
241
|
+
.catch((pError) =>
|
|
242
|
+
{
|
|
243
|
+
this.log.error(`Error creating index for ${tmpTableName}: ${pError.message}`, { Error: pError });
|
|
244
|
+
return fCallback();
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
module.exports = MeadowConnectionManager;
|
|
250
|
+
|
|
251
|
+
module.exports.default_configuration = defaultConnectionManagerOptions;
|