meadow-integration 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +11 -0
- package/Docker-Build.sh +2 -0
- package/Docker-Compose.sh +2 -0
- package/Docker-Push.sh +2 -0
- package/Docker-Tag.sh +2 -0
- package/Dockerfile +28 -0
- package/Dockerfile_LUXURYCode +23 -0
- package/README.md +139 -25
- package/docker-compose.yml +16 -0
- package/docs/README.md +65 -18
- package/docs/{cover.md → _cover.md} +3 -2
- package/docs/_sidebar.md +52 -7
- package/docs/_topbar.md +2 -0
- package/docs/api/clone-rest-client.md +278 -0
- package/docs/api/connection-manager.md +179 -0
- package/docs/api/guid-map.md +234 -0
- package/docs/api/integration-adapter.md +283 -0
- package/docs/api/operation.md +241 -0
- package/docs/api/sync-entity-initial.md +227 -0
- package/docs/api/sync-entity-ongoing.md +244 -0
- package/docs/api/sync.md +213 -0
- package/docs/api/tabular-check.md +213 -0
- package/docs/api/tabular-transform.md +316 -0
- package/docs/architecture.md +423 -0
- package/docs/cli/comprehensionarray.md +111 -0
- package/docs/cli/comprehensionintersect.md +132 -0
- package/docs/cli/csvcheck.md +111 -0
- package/docs/cli/csvtransform.md +170 -0
- package/docs/cli/data-clone.md +277 -0
- package/docs/cli/jsonarraytransform.md +166 -0
- package/docs/cli/load-comprehension.md +129 -0
- package/docs/cli/objectarraytocsv.md +159 -0
- package/docs/cli/overview.md +96 -0
- package/docs/cli/serve.md +102 -0
- package/docs/cli/tsvtransform.md +144 -0
- package/docs/data-clone/configuration.md +357 -0
- package/docs/data-clone/connection-manager.md +206 -0
- package/docs/data-clone/docker.md +290 -0
- package/docs/data-clone/overview.md +173 -0
- package/docs/data-clone/sync-modes.md +186 -0
- package/docs/implementation-reference.md +311 -0
- package/docs/overview.md +156 -0
- package/docs/quickstart.md +233 -0
- package/docs/rest/comprehension-push.md +209 -0
- package/docs/rest/comprehension.md +506 -0
- package/docs/rest/csv.md +255 -0
- package/docs/rest/entity-generation.md +158 -0
- package/docs/rest/json-array.md +243 -0
- package/docs/rest/overview.md +120 -0
- package/docs/rest/status.md +63 -0
- package/docs/rest/tsv.md +241 -0
- package/docs/retold-catalog.json +93 -3
- package/docs/retold-keyword-index.json +23683 -1901
- package/package.json +13 -10
- package/scripts/run.sh +18 -0
- package/source/Meadow-Integration.js +15 -1
- package/source/cli/Default-Meadow-Integration-Configuration.json +37 -2
- package/source/cli/Meadow-Integration-CLI-Program.js +4 -1
- package/source/cli/commands/Meadow-Integration-Command-DataClone.js +284 -0
- package/source/services/clone/Meadow-Service-ConnectionManager.js +251 -0
- package/source/services/clone/Meadow-Service-Operation.js +196 -0
- package/source/services/clone/Meadow-Service-RestClient.js +364 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js +367 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js +457 -0
- package/source/services/clone/Meadow-Service-Sync.js +142 -0
- /package/docs/examples/bookstore/{mapping_books_Author.json → mapping_books_author.json} +0 -0
- /package/docs/examples/bookstore/{mapping_books_Book.json → mapping_books_book.json} +0 -0
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
const libFableServiceProviderBase = require('fable-serviceproviderbase');
|
|
2
|
+
const libMeadowOperation = require('./Meadow-Service-Operation.js');
|
|
3
|
+
|
|
4
|
+
class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
5
|
+
{
|
|
6
|
+
constructor(pFable, pOptions, pServiceHash)
|
|
7
|
+
{
|
|
8
|
+
super(pFable, pOptions, pServiceHash);
|
|
9
|
+
|
|
10
|
+
this.serviceType = 'MeadowSyncEntityOngoing';
|
|
11
|
+
|
|
12
|
+
if (!this.options.hasOwnProperty('MeadowEntitySchema'))
|
|
13
|
+
{
|
|
14
|
+
throw new Error('MeadowSyncEntityOngoing requires a valid MeadowEntitySchema option.');
|
|
15
|
+
}
|
|
16
|
+
if (typeof(this.options.MeadowEntitySchema) != 'object')
|
|
17
|
+
{
|
|
18
|
+
throw new Error(`MeadowSyncEntityOngoing requires MeadowEntitySchema to be an object; got ${typeof(this.options.MeadowEntitySchema)}.`);
|
|
19
|
+
}
|
|
20
|
+
if (!this.options.MeadowEntitySchema.hasOwnProperty('TableName') ||
|
|
21
|
+
typeof(this.options.MeadowEntitySchema.TableName) != 'string' ||
|
|
22
|
+
this.options.MeadowEntitySchema.TableName.length < 1)
|
|
23
|
+
{
|
|
24
|
+
throw new Error('MeadowSyncEntityOngoing requires a valid MeadowEntitySchema.TableName.');
|
|
25
|
+
}
|
|
26
|
+
if (!this.options.MeadowEntitySchema.hasOwnProperty('Columns') ||
|
|
27
|
+
!Array.isArray(this.options.MeadowEntitySchema.Columns) ||
|
|
28
|
+
this.options.MeadowEntitySchema.Columns.length < 1)
|
|
29
|
+
{
|
|
30
|
+
throw new Error('MeadowSyncEntityOngoing requires a valid MeadowEntitySchema.Columns array.');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
this.EntitySchema = JSON.parse(JSON.stringify(this.options.MeadowEntitySchema));
|
|
34
|
+
|
|
35
|
+
if (!this.EntitySchema.hasOwnProperty('MeadowSchema'))
|
|
36
|
+
{
|
|
37
|
+
throw new Error('MeadowSyncEntityOngoing requires MeadowEntitySchema.MeadowSchema; please update stricture and recompile the extended JSON.');
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
this.DefaultIdentifier = this.EntitySchema.MeadowSchema.DefaultIdentifier;
|
|
41
|
+
this.PageSize = this.options.PageSize || 100;
|
|
42
|
+
|
|
43
|
+
this.Meadow = false;
|
|
44
|
+
|
|
45
|
+
this.operation = new libMeadowOperation(this.fable);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
initialize(fCallback)
|
|
49
|
+
{
|
|
50
|
+
if (this.fable.hasOwnProperty('Meadow'))
|
|
51
|
+
{
|
|
52
|
+
this.Meadow = this.fable.Meadow.loadFromPackageObject(this.EntitySchema.MeadowSchema);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
this.log.info(`Sync for ${this.EntitySchema.TableName} creating table if it doesn't exist...`);
|
|
56
|
+
|
|
57
|
+
if (this.Meadow && this.Meadow.provider)
|
|
58
|
+
{
|
|
59
|
+
return this.Meadow.provider.getProvider().createTable(this.EntitySchema, (pCreateError) =>
|
|
60
|
+
{
|
|
61
|
+
const tmpGUIDColumn = this.EntitySchema.Columns.find((c) => c.DataType == 'GUID');
|
|
62
|
+
const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
|
|
63
|
+
|
|
64
|
+
if (!tmpGUIDColumn && !tmpDeletedColumn)
|
|
65
|
+
{
|
|
66
|
+
this.log.info(`No GUID or Deleted columns for ${this.EntitySchema.TableName}; skipping index creation`);
|
|
67
|
+
return fCallback(pCreateError);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (!this.fable.MeadowConnectionManager || !this.fable.MeadowConnectionManager.ConnectionPool)
|
|
71
|
+
{
|
|
72
|
+
this.log.info(`No connection manager available; skipping index creation for ${this.EntitySchema.TableName}`);
|
|
73
|
+
return fCallback(pCreateError);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
let tmpAnticipate = this.fable.newAnticipate();
|
|
77
|
+
if (tmpGUIDColumn)
|
|
78
|
+
{
|
|
79
|
+
tmpAnticipate.anticipate(
|
|
80
|
+
(fNext) =>
|
|
81
|
+
{
|
|
82
|
+
return this.fable.MeadowConnectionManager.createIndex(this.EntitySchema, tmpGUIDColumn, true, fNext);
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
if (tmpDeletedColumn)
|
|
86
|
+
{
|
|
87
|
+
tmpAnticipate.anticipate(
|
|
88
|
+
(fNext) =>
|
|
89
|
+
{
|
|
90
|
+
return this.fable.MeadowConnectionManager.createIndex(this.EntitySchema, tmpDeletedColumn, false, fNext);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
tmpAnticipate.wait(fCallback);
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
return fCallback();
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
marshalRecord(pSourceRecord)
|
|
100
|
+
{
|
|
101
|
+
const tmpRecordToCommit = {};
|
|
102
|
+
|
|
103
|
+
for (const tmpColumn of this.EntitySchema.Columns)
|
|
104
|
+
{
|
|
105
|
+
if (pSourceRecord.hasOwnProperty(tmpColumn.Column))
|
|
106
|
+
{
|
|
107
|
+
switch (typeof(pSourceRecord[tmpColumn.Column]))
|
|
108
|
+
{
|
|
109
|
+
case 'null':
|
|
110
|
+
case 'undefined':
|
|
111
|
+
break;
|
|
112
|
+
case 'object':
|
|
113
|
+
if (pSourceRecord[tmpColumn.Column])
|
|
114
|
+
{
|
|
115
|
+
tmpRecordToCommit[tmpColumn.Column] = JSON.stringify(pSourceRecord[tmpColumn.Column]);
|
|
116
|
+
}
|
|
117
|
+
break;
|
|
118
|
+
default:
|
|
119
|
+
if (pSourceRecord[tmpColumn.Column] !== '')
|
|
120
|
+
{
|
|
121
|
+
tmpRecordToCommit[tmpColumn.Column] = pSourceRecord[tmpColumn.Column];
|
|
122
|
+
}
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
else if (tmpColumn.Column.endsWith('JSON') && typeof pSourceRecord[tmpColumn.Column.substring(0, tmpColumn.Column.length - 4)] === 'object')
|
|
127
|
+
{
|
|
128
|
+
tmpRecordToCommit[tmpColumn.Column] = JSON.stringify(pSourceRecord[tmpColumn.Column.substring(0, tmpColumn.Column.length - 4)]);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return tmpRecordToCommit;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
addSyncAnticipateEntry(tmpSyncState, tmpAnticipate)
|
|
136
|
+
{
|
|
137
|
+
tmpAnticipate.anticipate(
|
|
138
|
+
(fNext) =>
|
|
139
|
+
{
|
|
140
|
+
const tmpURLPartial = `${this.EntitySchema.TableName}s/FilteredTo/FBV~${this.DefaultIdentifier}~GT~${tmpSyncState.LastRequestedID}~FSF~${this.DefaultIdentifier}~ASC~ASC/0/${this.PageSize}`;
|
|
141
|
+
this.fable.MeadowCloneRestClient.getJSON(tmpURLPartial,
|
|
142
|
+
(pDownloadError, pResponse, pBody) =>
|
|
143
|
+
{
|
|
144
|
+
if (pDownloadError)
|
|
145
|
+
{
|
|
146
|
+
this.fable.log.error(`Error getting URL Partial [${tmpURLPartial}]: ${pDownloadError}`, { Error: pDownloadError });
|
|
147
|
+
return fNext();
|
|
148
|
+
}
|
|
149
|
+
if (pBody && pBody.length > 0)
|
|
150
|
+
{
|
|
151
|
+
for (let i = 0; i < pBody.length; i++)
|
|
152
|
+
{
|
|
153
|
+
const tmpRecord = pBody[i];
|
|
154
|
+
|
|
155
|
+
tmpAnticipate.anticipate(
|
|
156
|
+
(fNextEntityRecordSync) =>
|
|
157
|
+
{
|
|
158
|
+
const tmpQuery = this.Meadow.query;
|
|
159
|
+
|
|
160
|
+
if (tmpRecord[this.DefaultIdentifier] > tmpSyncState.LastRequestedID)
|
|
161
|
+
{
|
|
162
|
+
tmpSyncState.LastRequestedID = tmpRecord[this.DefaultIdentifier];
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if ((typeof(tmpRecord[this.DefaultIdentifier]) !== 'undefined') && (tmpRecord[this.DefaultIdentifier] > 0))
|
|
166
|
+
{
|
|
167
|
+
tmpQuery.addFilter(this.DefaultIdentifier, tmpRecord[this.DefaultIdentifier]);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
this.Meadow.doRead(tmpQuery,
|
|
171
|
+
(pReadError, pQuery, pRecord) =>
|
|
172
|
+
{
|
|
173
|
+
if (pReadError)
|
|
174
|
+
{
|
|
175
|
+
this.fable.log.error(`Error reading record ${this.EntitySchema.TableName}: ${pReadError}`, { Error: pReadError, PassedRecord: tmpRecord });
|
|
176
|
+
return fNextEntityRecordSync();
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
if (pRecord)
|
|
180
|
+
{
|
|
181
|
+
const tmpAgeDifference = this.fable.Dates.dayJS(tmpRecord.UpdateDate).diff(this.fable.Dates.dayJS(pRecord.UpdateDate));
|
|
182
|
+
|
|
183
|
+
if (Math.abs(tmpAgeDifference) < 5)
|
|
184
|
+
{
|
|
185
|
+
return fNextEntityRecordSync();
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
this.fable.log.info(`Syncing ${this.EntitySchema.TableName} record ${tmpRecord[this.DefaultIdentifier]} with age difference of ${tmpAgeDifference} ms.`);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const tmpRecordToCommit = this.marshalRecord(tmpRecord);
|
|
192
|
+
|
|
193
|
+
const tmpSyncQuery = this.Meadow.query.addRecord(tmpRecordToCommit);
|
|
194
|
+
|
|
195
|
+
tmpSyncQuery.setDisableAutoIdentity(true);
|
|
196
|
+
tmpSyncQuery.setDisableAutoDateStamp(true);
|
|
197
|
+
tmpSyncQuery.setDisableAutoUserStamp(true);
|
|
198
|
+
tmpSyncQuery.setDisableDeleteTracking(true);
|
|
199
|
+
|
|
200
|
+
if (!pRecord)
|
|
201
|
+
{
|
|
202
|
+
// Record not found -- create
|
|
203
|
+
tmpSyncQuery.AllowIdentityInsert = true;
|
|
204
|
+
|
|
205
|
+
this.Meadow.doCreate(tmpSyncQuery,
|
|
206
|
+
(pCreateError) =>
|
|
207
|
+
{
|
|
208
|
+
if (pCreateError)
|
|
209
|
+
{
|
|
210
|
+
this.log.error(`Error creating record ${this.EntitySchema.TableName}: ${pCreateError}`, pCreateError);
|
|
211
|
+
return fNextEntityRecordSync();
|
|
212
|
+
}
|
|
213
|
+
this.operation.incrementProgressTrackerStatus(`UpdateSync-${this.EntitySchema.TableName}`, 1);
|
|
214
|
+
return fNextEntityRecordSync();
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
else
|
|
218
|
+
{
|
|
219
|
+
// Record found -- update
|
|
220
|
+
this.Meadow.doUpdate(tmpSyncQuery,
|
|
221
|
+
(pUpdateError) =>
|
|
222
|
+
{
|
|
223
|
+
if (pUpdateError)
|
|
224
|
+
{
|
|
225
|
+
this.log.error(`Error updating record ${this.EntitySchema.TableName}: ${pUpdateError}`, pUpdateError);
|
|
226
|
+
return fNextEntityRecordSync();
|
|
227
|
+
}
|
|
228
|
+
this.operation.incrementProgressTrackerStatus(`UpdateSync-${this.EntitySchema.TableName}`, 1);
|
|
229
|
+
return fNextEntityRecordSync();
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
tmpSyncState.RequestsPerformed++;
|
|
236
|
+
if (tmpSyncState.RequestsPerformed < tmpSyncState.EstimatedRequestCount)
|
|
237
|
+
{
|
|
238
|
+
this.fable.log.info(`Syncing ${this.EntitySchema.TableName} request ${tmpSyncState.RequestsPerformed} of ${tmpSyncState.EstimatedRequestCount}...`);
|
|
239
|
+
this.addSyncAnticipateEntry(tmpSyncState, tmpAnticipate);
|
|
240
|
+
}
|
|
241
|
+
return fNext();
|
|
242
|
+
}
|
|
243
|
+
else
|
|
244
|
+
{
|
|
245
|
+
return fNext();
|
|
246
|
+
}
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
sync(fCallback)
|
|
252
|
+
{
|
|
253
|
+
this.operation.createTimeStamp('EntityOngoingSync');
|
|
254
|
+
|
|
255
|
+
let tmpAnticipate = this.fable.newAnticipate();
|
|
256
|
+
|
|
257
|
+
const tmpSyncState = (
|
|
258
|
+
{
|
|
259
|
+
Local: { MaxIDEntity: -1, RecordCount: 0, HasUpdateDate: false, LatestUpdateDate: false },
|
|
260
|
+
Server: { MaxIDEntity: -1, RecordCount: 0, HasUpdateDate: false, LatestUpdateDate: false },
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
this.fable.Utility.waterfall(
|
|
264
|
+
[
|
|
265
|
+
(fStageComplete) =>
|
|
266
|
+
{
|
|
267
|
+
if (!this.EntitySchema || !this.EntitySchema.MeadowSchema || !Array.isArray(this.EntitySchema.MeadowSchema.Schema))
|
|
268
|
+
{
|
|
269
|
+
return fStageComplete('MeadowSyncEntityOngoing requires a valid MeadowEntitySchema.MeadowSchema.Schema.');
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
for (let i = 0; i < this.EntitySchema.MeadowSchema.Schema.length; i++)
|
|
273
|
+
{
|
|
274
|
+
const tmpColumn = this.EntitySchema.MeadowSchema.Schema[i];
|
|
275
|
+
if (tmpColumn.Column == 'UpdateDate')
|
|
276
|
+
{
|
|
277
|
+
tmpSyncState.Local.HasUpdateDate = true;
|
|
278
|
+
tmpSyncState.Server.HasUpdateDate = true;
|
|
279
|
+
this.log.info(`Entity ${this.EntitySchema.TableName} has UpdateDate column.`);
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
this.log.info(`Syncing with UPDATE STRATEGY entity ${this.EntitySchema.TableName}...`);
|
|
285
|
+
return fStageComplete();
|
|
286
|
+
},
|
|
287
|
+
(fStageComplete) =>
|
|
288
|
+
{
|
|
289
|
+
// Get the Max ID from local database
|
|
290
|
+
const tmpQuery = this.Meadow.query;
|
|
291
|
+
tmpQuery.setSort({ Column: this.DefaultIdentifier, Direction: 'Descending' });
|
|
292
|
+
tmpQuery.setCap(1);
|
|
293
|
+
this.Meadow.doRead(tmpQuery,
|
|
294
|
+
(pReadError, pQuery, pRecord) =>
|
|
295
|
+
{
|
|
296
|
+
if (pReadError)
|
|
297
|
+
{
|
|
298
|
+
this.fable.log.error(`Error reading local max entity ID ${this.EntitySchema.TableName}: ${pReadError}`, { Error: pReadError });
|
|
299
|
+
return fStageComplete(`Error reading local max entity ID ${this.EntitySchema.TableName}: ${pReadError}`);
|
|
300
|
+
}
|
|
301
|
+
if (!pRecord)
|
|
302
|
+
{
|
|
303
|
+
this.fable.log.warn(`No records found in local ${this.EntitySchema.TableName}.`);
|
|
304
|
+
return fStageComplete();
|
|
305
|
+
}
|
|
306
|
+
this.fable.log.info(`Found local max entity ID ${this.EntitySchema.TableName}: ${pRecord[this.DefaultIdentifier]}`);
|
|
307
|
+
tmpSyncState.Local.MaxIDEntity = pRecord[this.DefaultIdentifier];
|
|
308
|
+
return fStageComplete();
|
|
309
|
+
});
|
|
310
|
+
},
|
|
311
|
+
(fStageComplete) =>
|
|
312
|
+
{
|
|
313
|
+
// Get the Max UpdateDate from local database
|
|
314
|
+
const tmpQuery = this.Meadow.query;
|
|
315
|
+
tmpQuery.setSort({ Column: 'UpdateDate', Direction: 'Descending' });
|
|
316
|
+
tmpQuery.setCap(1);
|
|
317
|
+
this.Meadow.doRead(tmpQuery,
|
|
318
|
+
(pReadError, pQuery, pRecord) =>
|
|
319
|
+
{
|
|
320
|
+
if (pReadError)
|
|
321
|
+
{
|
|
322
|
+
this.fable.log.error(`Error reading local max UpdateDate ${this.EntitySchema.TableName}: ${pReadError}`, { Error: pReadError });
|
|
323
|
+
return fStageComplete(`Error reading local max UpdateDate ${this.EntitySchema.TableName}: ${pReadError}`);
|
|
324
|
+
}
|
|
325
|
+
if (!pRecord)
|
|
326
|
+
{
|
|
327
|
+
this.fable.log.warn(`No records found in local checking UpdateDate ${this.EntitySchema.TableName}.`);
|
|
328
|
+
return fStageComplete();
|
|
329
|
+
}
|
|
330
|
+
this.fable.log.info(`Found local max UpdateDate ${this.EntitySchema.TableName}: ${pRecord.UpdateDate}`);
|
|
331
|
+
tmpSyncState.Local.MaxUpdateDate = pRecord.UpdateDate;
|
|
332
|
+
return fStageComplete();
|
|
333
|
+
});
|
|
334
|
+
},
|
|
335
|
+
(fStageComplete) =>
|
|
336
|
+
{
|
|
337
|
+
// Get the count from local database
|
|
338
|
+
const tmpQuery = this.Meadow.query;
|
|
339
|
+
this.Meadow.doCount(tmpQuery,
|
|
340
|
+
(pCountError, pQuery, pCount) =>
|
|
341
|
+
{
|
|
342
|
+
if (pCountError)
|
|
343
|
+
{
|
|
344
|
+
this.fable.log.error(`Error getting local count of ${this.EntitySchema.TableName}: ${pCountError}`, { Error: pCountError });
|
|
345
|
+
return fStageComplete(`Error getting local count of ${this.EntitySchema.TableName}: ${pCountError}`);
|
|
346
|
+
}
|
|
347
|
+
tmpSyncState.Local.RecordCount = pCount;
|
|
348
|
+
return fStageComplete();
|
|
349
|
+
});
|
|
350
|
+
},
|
|
351
|
+
(fStageComplete) =>
|
|
352
|
+
{
|
|
353
|
+
// Get the Max ID from server
|
|
354
|
+
this.fable.MeadowCloneRestClient.getJSON(`${this.EntitySchema.TableName}/Max/${this.DefaultIdentifier}`,
|
|
355
|
+
(pError, pResponse, pBody) =>
|
|
356
|
+
{
|
|
357
|
+
if (pError)
|
|
358
|
+
{
|
|
359
|
+
this.fable.log.error(`Error getting server max entity ID ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
|
|
360
|
+
return fStageComplete(`Error getting server max entity ID ${this.EntitySchema.TableName}: ${pError}`);
|
|
361
|
+
}
|
|
362
|
+
if (pBody && pBody.hasOwnProperty(this.DefaultIdentifier))
|
|
363
|
+
{
|
|
364
|
+
this.fable.log.info(`Found server max entity ID ${this.EntitySchema.TableName}: ${pBody[this.DefaultIdentifier]}`);
|
|
365
|
+
tmpSyncState.Server.MaxIDEntity = pBody[this.DefaultIdentifier];
|
|
366
|
+
}
|
|
367
|
+
else
|
|
368
|
+
{
|
|
369
|
+
this.fable.log.warn(`No records found in server for max entity ID of ${this.EntitySchema.TableName}.`);
|
|
370
|
+
}
|
|
371
|
+
return fStageComplete();
|
|
372
|
+
});
|
|
373
|
+
},
|
|
374
|
+
(fStageComplete) =>
|
|
375
|
+
{
|
|
376
|
+
// Get the Max UpdateDate from server
|
|
377
|
+
this.fable.MeadowCloneRestClient.getJSON(`${this.EntitySchema.TableName}/Max/UpdateDate`,
|
|
378
|
+
(pError, pResponse, pBody) =>
|
|
379
|
+
{
|
|
380
|
+
if (pError)
|
|
381
|
+
{
|
|
382
|
+
this.fable.log.error(`Error getting server max UpdateDate ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
|
|
383
|
+
return fStageComplete(`Error getting server max UpdateDate ${this.EntitySchema.TableName}: ${pError}`);
|
|
384
|
+
}
|
|
385
|
+
if (pBody && pBody.hasOwnProperty(this.DefaultIdentifier))
|
|
386
|
+
{
|
|
387
|
+
this.fable.log.info(`Found server max UpdateDate ${this.EntitySchema.TableName}: ${pBody['UpdateDate']}`);
|
|
388
|
+
tmpSyncState.Server.MaxUpdateDate = pBody.UpdateDate;
|
|
389
|
+
}
|
|
390
|
+
else
|
|
391
|
+
{
|
|
392
|
+
this.fable.log.warn(`No records found in server for max UpdateDate of ${this.EntitySchema.TableName}.`);
|
|
393
|
+
}
|
|
394
|
+
return fStageComplete();
|
|
395
|
+
});
|
|
396
|
+
},
|
|
397
|
+
(fStageComplete) =>
|
|
398
|
+
{
|
|
399
|
+
// Get the count from server
|
|
400
|
+
this.fable.MeadowCloneRestClient.getJSON(`${this.EntitySchema.TableName}s/Count`,
|
|
401
|
+
(pError, pResponse, pBody) =>
|
|
402
|
+
{
|
|
403
|
+
if (pError)
|
|
404
|
+
{
|
|
405
|
+
this.fable.log.error(`Error getting server count for ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
|
|
406
|
+
return fStageComplete(`Error getting server count for ${this.EntitySchema.TableName}: ${pError}`);
|
|
407
|
+
}
|
|
408
|
+
if (pBody && pBody.hasOwnProperty('Count'))
|
|
409
|
+
{
|
|
410
|
+
this.fable.log.info(`Found server count for ${this.EntitySchema.TableName}: ${pBody.Count}`);
|
|
411
|
+
tmpSyncState.Server.RecordCount = pBody.Count;
|
|
412
|
+
}
|
|
413
|
+
else
|
|
414
|
+
{
|
|
415
|
+
this.fable.log.warn(`No records found in server based on count for ${this.EntitySchema.TableName}.`);
|
|
416
|
+
}
|
|
417
|
+
return fStageComplete();
|
|
418
|
+
});
|
|
419
|
+
},
|
|
420
|
+
(fStageComplete) =>
|
|
421
|
+
{
|
|
422
|
+
tmpSyncState.EstimatedRequestCount = Math.ceil(tmpSyncState.Server.RecordCount / this.PageSize);
|
|
423
|
+
tmpSyncState.RequestsPerformed = 0;
|
|
424
|
+
tmpSyncState.LastRequestedID = 0;
|
|
425
|
+
|
|
426
|
+
this.operation.createProgressTracker(tmpSyncState.EstimatedRequestCount, `UpdateSync-${this.EntitySchema.TableName}`);
|
|
427
|
+
this.operation.printProgressTrackerStatus(`UpdateSync-${this.EntitySchema.TableName}`);
|
|
428
|
+
|
|
429
|
+
return fStageComplete();
|
|
430
|
+
},
|
|
431
|
+
(fStageComplete) =>
|
|
432
|
+
{
|
|
433
|
+
if (tmpSyncState.EstimatedRequestCount < 1)
|
|
434
|
+
{
|
|
435
|
+
this.fable.log.info(`No records to update sync for ${this.EntitySchema.TableName}.`);
|
|
436
|
+
return fStageComplete();
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
this.addSyncAnticipateEntry(tmpSyncState, tmpAnticipate);
|
|
440
|
+
|
|
441
|
+
tmpAnticipate.wait(fStageComplete);
|
|
442
|
+
},
|
|
443
|
+
],
|
|
444
|
+
(pError) =>
|
|
445
|
+
{
|
|
446
|
+
if (pError)
|
|
447
|
+
{
|
|
448
|
+
this.fable.log.error(`Error performing Update sync ${this.EntitySchema.TableName}: ${pError}`, { Error: pError });
|
|
449
|
+
return fCallback();
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
return fCallback();
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
module.exports = MeadowSyncEntityOngoing;
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
const libFableServiceProviderBase = require('fable-serviceproviderbase');
|
|
2
|
+
const libMeadowSyncEntityInitial = require('./Meadow-Service-Sync-Entity-Initial.js');
|
|
3
|
+
const libMeadowSyncEntityOngoing = require('./Meadow-Service-Sync-Entity-Ongoing.js');
|
|
4
|
+
|
|
5
|
+
class MeadowSync extends libFableServiceProviderBase
|
|
6
|
+
{
|
|
7
|
+
constructor(pFable, pOptions, pServiceHash)
|
|
8
|
+
{
|
|
9
|
+
super(pFable, pOptions, pServiceHash);
|
|
10
|
+
|
|
11
|
+
this.serviceType = 'MeadowSync';
|
|
12
|
+
|
|
13
|
+
if (!this.fable.ServiceManager.servicesMap.hasOwnProperty('MeadowSyncEntityInitial'))
|
|
14
|
+
{
|
|
15
|
+
this.fable.ServiceManager.addServiceType('MeadowSyncEntityInitial', libMeadowSyncEntityInitial);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
if (!this.fable.ServiceManager.servicesMap.hasOwnProperty('MeadowSyncEntityOngoing'))
|
|
19
|
+
{
|
|
20
|
+
this.fable.ServiceManager.addServiceType('MeadowSyncEntityOngoing', libMeadowSyncEntityOngoing);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// If this is empty, we will sync everything in the loaded Schema.
|
|
24
|
+
// Otherwise, we will go through this list and sync them in this order.
|
|
25
|
+
this.SyncEntityList = [];
|
|
26
|
+
if (this.fable.ProgramConfiguration.hasOwnProperty('SyncEntityList') && Array.isArray(this.fable.ProgramConfiguration.SyncEntityList))
|
|
27
|
+
{
|
|
28
|
+
this.SyncEntityList = JSON.parse(JSON.stringify(this.fable.ProgramConfiguration.SyncEntityList));
|
|
29
|
+
}
|
|
30
|
+
else if (this.options.hasOwnProperty('SyncEntityList') && Array.isArray(this.options.SyncEntityList))
|
|
31
|
+
{
|
|
32
|
+
this.SyncEntityList = JSON.parse(JSON.stringify(this.options.SyncEntityList));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Per-entity sync options.
|
|
36
|
+
this.SyncEntityOptions = {};
|
|
37
|
+
if (this.fable.ProgramConfiguration.hasOwnProperty('SyncEntityOptions') && typeof(this.fable.ProgramConfiguration.SyncEntityOptions) == 'object')
|
|
38
|
+
{
|
|
39
|
+
this.SyncEntityOptions = JSON.parse(JSON.stringify(this.fable.ProgramConfiguration.SyncEntityOptions));
|
|
40
|
+
}
|
|
41
|
+
else if (this.options.hasOwnProperty('SyncEntityOptions') && typeof(this.options.SyncEntityOptions) == 'object')
|
|
42
|
+
{
|
|
43
|
+
this.SyncEntityOptions = JSON.parse(JSON.stringify(this.options.SyncEntityOptions));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
this.MeadowSchema = false;
|
|
47
|
+
this.MeadowSchemaTableList = false;
|
|
48
|
+
|
|
49
|
+
this.MeadowSyncEntities = {};
|
|
50
|
+
|
|
51
|
+
this.SyncMode = 'Initial';
|
|
52
|
+
|
|
53
|
+
this.fable._MeadowPrototype = require('meadow');
|
|
54
|
+
this.fable.Meadow = this.fable._MeadowPrototype.new(this.fable, 'MeadowSync-Prototype');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
loadMeadowSchema(pSchema, fCallback)
|
|
58
|
+
{
|
|
59
|
+
this.meadowSchema = pSchema;
|
|
60
|
+
this.MeadowSchemaTableList = Object.keys(this.meadowSchema.Tables);
|
|
61
|
+
|
|
62
|
+
this.fable.Utility.eachLimit(this.MeadowSchemaTableList, 1,
|
|
63
|
+
(pEntitySchemaName, fSyncInitializationComplete) =>
|
|
64
|
+
{
|
|
65
|
+
const tmpEntitySchema = this.meadowSchema.Tables[pEntitySchemaName];
|
|
66
|
+
// If this is in the entity list or none is specified, create the sync entity object.
|
|
67
|
+
if (this.SyncEntityList.length < 1 || this.SyncEntityList.indexOf(tmpEntitySchema.TableName) > -1)
|
|
68
|
+
{
|
|
69
|
+
const tmpSyncEntityOptions = {
|
|
70
|
+
MeadowEntitySchema: tmpEntitySchema,
|
|
71
|
+
ConnectionPool: this.options.ConnectionPool,
|
|
72
|
+
PageSize: this.options.PageSize || 100,
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
let tmpSyncEntity;
|
|
76
|
+
|
|
77
|
+
if (this.SyncMode == 'Ongoing')
|
|
78
|
+
{
|
|
79
|
+
tmpSyncEntity = this.fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityOngoing', tmpSyncEntityOptions, `SyncEntity-${tmpEntitySchema.TableName}`);
|
|
80
|
+
}
|
|
81
|
+
else
|
|
82
|
+
{
|
|
83
|
+
tmpSyncEntity = this.fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityInitial', tmpSyncEntityOptions, `SyncEntity-${tmpEntitySchema.TableName}`);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
this.MeadowSyncEntities[tmpEntitySchema.TableName] = tmpSyncEntity;
|
|
87
|
+
|
|
88
|
+
return tmpSyncEntity.initialize(fSyncInitializationComplete);
|
|
89
|
+
}
|
|
90
|
+
else
|
|
91
|
+
{
|
|
92
|
+
return fSyncInitializationComplete();
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
(pSyncInitializationError) =>
|
|
96
|
+
{
|
|
97
|
+
if (pSyncInitializationError)
|
|
98
|
+
{
|
|
99
|
+
this.log.error(`MeadowSync Error creating sync objects: ${pSyncInitializationError}`, pSyncInitializationError);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
this.log.info('Entity sync objects created!');
|
|
103
|
+
|
|
104
|
+
if (this.SyncEntityList.length < 1)
|
|
105
|
+
{
|
|
106
|
+
this.SyncEntityList = Object.keys(this.MeadowSyncEntities);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return fCallback(pSyncInitializationError);
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
syncEntity(pEntityHash, fCallback)
|
|
114
|
+
{
|
|
115
|
+
if (!this.MeadowSyncEntities.hasOwnProperty(pEntityHash))
|
|
116
|
+
{
|
|
117
|
+
this.log.warn(`MeadowSync.syncEntity called for an entity that does not exist: ${pEntityHash}`);
|
|
118
|
+
return fCallback();
|
|
119
|
+
}
|
|
120
|
+
this.MeadowSyncEntities[pEntityHash].sync(fCallback);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
syncAll(fCallback)
|
|
124
|
+
{
|
|
125
|
+
this.fable.Utility.eachLimit(this.SyncEntityList, 1,
|
|
126
|
+
(pEntityHash, fSyncEntityComplete) =>
|
|
127
|
+
{
|
|
128
|
+
this.syncEntity(pEntityHash, fSyncEntityComplete);
|
|
129
|
+
},
|
|
130
|
+
(pSyncError) =>
|
|
131
|
+
{
|
|
132
|
+
if (pSyncError)
|
|
133
|
+
{
|
|
134
|
+
this.log.error(`MeadowSync Error syncing entities: ${pSyncError}`, pSyncError);
|
|
135
|
+
}
|
|
136
|
+
this.log.info('Entity sync complete!');
|
|
137
|
+
return fCallback(pSyncError);
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
module.exports = MeadowSync;
|
|
File without changes
|
|
File without changes
|