meadow-integration 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +11 -0
- package/Docker-Build.sh +2 -0
- package/Docker-Compose.sh +2 -0
- package/Docker-Push.sh +2 -0
- package/Docker-Tag.sh +2 -0
- package/Dockerfile +28 -0
- package/Dockerfile_LUXURYCode +23 -0
- package/README.md +139 -25
- package/docker-compose.yml +16 -0
- package/docs/README.md +65 -18
- package/docs/{cover.md → _cover.md} +3 -2
- package/docs/_sidebar.md +52 -7
- package/docs/_topbar.md +2 -0
- package/docs/api/clone-rest-client.md +278 -0
- package/docs/api/connection-manager.md +179 -0
- package/docs/api/guid-map.md +234 -0
- package/docs/api/integration-adapter.md +283 -0
- package/docs/api/operation.md +241 -0
- package/docs/api/sync-entity-initial.md +227 -0
- package/docs/api/sync-entity-ongoing.md +244 -0
- package/docs/api/sync.md +213 -0
- package/docs/api/tabular-check.md +213 -0
- package/docs/api/tabular-transform.md +316 -0
- package/docs/architecture.md +423 -0
- package/docs/cli/comprehensionarray.md +111 -0
- package/docs/cli/comprehensionintersect.md +132 -0
- package/docs/cli/csvcheck.md +111 -0
- package/docs/cli/csvtransform.md +170 -0
- package/docs/cli/data-clone.md +277 -0
- package/docs/cli/jsonarraytransform.md +166 -0
- package/docs/cli/load-comprehension.md +129 -0
- package/docs/cli/objectarraytocsv.md +159 -0
- package/docs/cli/overview.md +96 -0
- package/docs/cli/serve.md +102 -0
- package/docs/cli/tsvtransform.md +144 -0
- package/docs/data-clone/configuration.md +357 -0
- package/docs/data-clone/connection-manager.md +206 -0
- package/docs/data-clone/docker.md +290 -0
- package/docs/data-clone/overview.md +173 -0
- package/docs/data-clone/sync-modes.md +186 -0
- package/docs/implementation-reference.md +311 -0
- package/docs/overview.md +156 -0
- package/docs/quickstart.md +233 -0
- package/docs/rest/comprehension-push.md +209 -0
- package/docs/rest/comprehension.md +506 -0
- package/docs/rest/csv.md +255 -0
- package/docs/rest/entity-generation.md +158 -0
- package/docs/rest/json-array.md +243 -0
- package/docs/rest/overview.md +120 -0
- package/docs/rest/status.md +63 -0
- package/docs/rest/tsv.md +241 -0
- package/docs/retold-catalog.json +93 -3
- package/docs/retold-keyword-index.json +23683 -1901
- package/package.json +13 -10
- package/scripts/run.sh +18 -0
- package/source/Meadow-Integration.js +15 -1
- package/source/cli/Default-Meadow-Integration-Configuration.json +37 -2
- package/source/cli/Meadow-Integration-CLI-Program.js +4 -1
- package/source/cli/commands/Meadow-Integration-Command-DataClone.js +284 -0
- package/source/services/clone/Meadow-Service-ConnectionManager.js +251 -0
- package/source/services/clone/Meadow-Service-Operation.js +196 -0
- package/source/services/clone/Meadow-Service-RestClient.js +364 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js +367 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js +457 -0
- package/source/services/clone/Meadow-Service-Sync.js +142 -0
- /package/docs/examples/bookstore/{mapping_books_Author.json → mapping_books_author.json} +0 -0
- /package/docs/examples/bookstore/{mapping_books_Book.json → mapping_books_book.json} +0 -0
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
# MeadowSyncEntityInitial
|
|
2
|
+
|
|
3
|
+
Performs a full initial synchronization of a single entity from a remote Meadow API server to a local database. Downloads all records that do not yet exist locally by comparing max IDs, then inserts missing records in paginated batches.
|
|
4
|
+
|
|
5
|
+
**Source:** `source/services/clone/Meadow-Service-Sync-Entity-Initial.js`
|
|
6
|
+
|
|
7
|
+
**Extends:** `fable-serviceproviderbase`
|
|
8
|
+
|
|
9
|
+
**Service Type:** `MeadowSyncEntityInitial`
|
|
10
|
+
|
|
11
|
+
## Constructor
|
|
12
|
+
|
|
13
|
+
```js
|
|
14
|
+
const syncEntity = fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityInitial', pOptions, pServiceHash);
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
### Options
|
|
18
|
+
|
|
19
|
+
| Option | Type | Required | Default | Description |
|
|
20
|
+
|--------|------|----------|---------|-------------|
|
|
21
|
+
| `MeadowEntitySchema` | `object` | Yes | -- | Entity schema object containing `TableName`, `Columns` (array), and `MeadowSchema`. |
|
|
22
|
+
| `ConnectionPool` | `object` | No | -- | Database connection pool (used for table/index creation). |
|
|
23
|
+
| `PageSize` | `number` | No | `100` | Number of records per paginated download request. |
|
|
24
|
+
|
|
25
|
+
**Validation:** The constructor throws an `Error` if:
|
|
26
|
+
- `MeadowEntitySchema` is missing or not an object.
|
|
27
|
+
- `MeadowEntitySchema.TableName` is missing or empty.
|
|
28
|
+
- `MeadowEntitySchema.Columns` is missing or not a non-empty array.
|
|
29
|
+
- `MeadowEntitySchema.MeadowSchema` is missing.
|
|
30
|
+
|
|
31
|
+
## Properties
|
|
32
|
+
|
|
33
|
+
### `EntitySchema`
|
|
34
|
+
|
|
35
|
+
`object` -- Deep copy of the provided `MeadowEntitySchema` option.
|
|
36
|
+
|
|
37
|
+
### `DefaultIdentifier`
|
|
38
|
+
|
|
39
|
+
`string` -- The default identifier column name from `EntitySchema.MeadowSchema.DefaultIdentifier` (typically `'IDAnimal'`, `'IDCustomer'`, etc.).
|
|
40
|
+
|
|
41
|
+
### `PageSize`
|
|
42
|
+
|
|
43
|
+
`number` -- Records per download page.
|
|
44
|
+
|
|
45
|
+
### `Meadow`
|
|
46
|
+
|
|
47
|
+
The Meadow ORM instance for this entity, loaded from the entity's `MeadowSchema` package object. Set by `initialize()`.
|
|
48
|
+
|
|
49
|
+
### `operation`
|
|
50
|
+
|
|
51
|
+
`MeadowOperation` -- Instance of the operation utility for timestamps and progress tracking.
|
|
52
|
+
|
|
53
|
+
## Methods
|
|
54
|
+
|
|
55
|
+
### `initialize(fCallback)`
|
|
56
|
+
|
|
57
|
+
Prepares the entity for synchronization by creating the local database table (if it does not exist) and setting up indexes on GUID and Deleted columns.
|
|
58
|
+
|
|
59
|
+
| Parameter | Type | Description |
|
|
60
|
+
|-----------|------|-------------|
|
|
61
|
+
| `fCallback` | `function(pError)` | Callback invoked when initialization is complete. |
|
|
62
|
+
|
|
63
|
+
**Behavior:**
|
|
64
|
+
1. Loads the Meadow ORM instance from the entity's schema package.
|
|
65
|
+
2. Calls the provider's `createTable()` to ensure the table exists.
|
|
66
|
+
3. If the schema has a GUID column (`DataType == 'GUID'`), creates a unique index on it.
|
|
67
|
+
4. If the schema has a `Deleted` column, creates a non-unique index on it.
|
|
68
|
+
5. Index creation requires `fable.MeadowConnectionManager` to be available.
|
|
69
|
+
|
|
70
|
+
### `marshalRecord(pSourceRecord)`
|
|
71
|
+
|
|
72
|
+
Transforms a server-side record into a local record suitable for insertion, based on the entity schema columns.
|
|
73
|
+
|
|
74
|
+
| Parameter | Type | Description |
|
|
75
|
+
|-----------|------|-------------|
|
|
76
|
+
| `pSourceRecord` | `object` | The record object from the server. |
|
|
77
|
+
|
|
78
|
+
**Returns:** `object` -- The marshaled record with only schema-defined columns.
|
|
79
|
+
|
|
80
|
+
**Transformation rules:**
|
|
81
|
+
- `null`/`undefined` values are skipped.
|
|
82
|
+
- Object values are JSON-stringified.
|
|
83
|
+
- `DateTime` columns are formatted to `'YYYY-MM-DD HH:mm:ss.SSS'` in UTC.
|
|
84
|
+
- Empty string values are skipped (except for DateTime).
|
|
85
|
+
- Columns ending in `JSON` auto-stringify the corresponding non-JSON property from the source (e.g., `MetadataJSON` from `Metadata`).
|
|
86
|
+
|
|
87
|
+
### `sync(fCallback)`
|
|
88
|
+
|
|
89
|
+
Executes the full initial sync algorithm for this entity.
|
|
90
|
+
|
|
91
|
+
| Parameter | Type | Description |
|
|
92
|
+
|-----------|------|-------------|
|
|
93
|
+
| `fCallback` | `function()` | Callback invoked when sync completes (errors are logged, not passed). |
|
|
94
|
+
|
|
95
|
+
**Algorithm:**
|
|
96
|
+
|
|
97
|
+
1. **Get local max ID** -- Reads the highest-ID record from the local database.
|
|
98
|
+
2. **Get local count** -- Counts all local records.
|
|
99
|
+
3. **Get server max ID** -- Requests max ID from `{Entity}/Max/{DefaultIdentifier}`.
|
|
100
|
+
4. **Get server count** -- Requests count from `{Entity}s/Count`.
|
|
101
|
+
5. **Calculate estimated records** -- `Server.RecordCount - Local.RecordCount`.
|
|
102
|
+
6. **Generate paginated URLs** -- Creates URL partials filtered to `{DefaultIdentifier} > {LocalMaxID}`, sorted ascending, paginated by `PageSize`.
|
|
103
|
+
7. **Download and insert** -- For each page, downloads records and for each record:
|
|
104
|
+
- Checks if the record already exists locally by ID.
|
|
105
|
+
- If not found, marshals the record and creates it with identity insert enabled and all auto-stamps disabled.
|
|
106
|
+
- Updates the progress tracker after each insert.
|
|
107
|
+
8. **Early termination** -- If an empty page is returned, stops downloading.
|
|
108
|
+
|
|
109
|
+
**Progress tracking:** Uses `MeadowOperation` with tracker hash `FullSync-{TableName}`.
|
|
110
|
+
|
|
111
|
+
## Usage Examples
|
|
112
|
+
|
|
113
|
+
### Direct Instantiation and Sync
|
|
114
|
+
|
|
115
|
+
```js
|
|
116
|
+
const libFable = require('fable');
|
|
117
|
+
const libSyncEntityInitial = require('meadow-integration/source/services/clone/Meadow-Service-Sync-Entity-Initial');
|
|
118
|
+
|
|
119
|
+
const fable = new libFable({ Product: 'CloneApp' });
|
|
120
|
+
|
|
121
|
+
fable.serviceManager.addServiceType('MeadowSyncEntityInitial', libSyncEntityInitial);
|
|
122
|
+
|
|
123
|
+
const entitySchema = {
|
|
124
|
+
TableName: 'Customer',
|
|
125
|
+
Columns:
|
|
126
|
+
[
|
|
127
|
+
{ Column: 'IDCustomer', DataType: 'AutoIdentity' },
|
|
128
|
+
{ Column: 'GUIDCustomer', DataType: 'GUID' },
|
|
129
|
+
{ Column: 'Name', DataType: 'String' },
|
|
130
|
+
{ Column: 'Email', DataType: 'String' },
|
|
131
|
+
{ Column: 'CreateDate', DataType: 'DateTime' },
|
|
132
|
+
{ Column: 'UpdateDate', DataType: 'DateTime' },
|
|
133
|
+
{ Column: 'Deleted', DataType: 'Boolean' }
|
|
134
|
+
],
|
|
135
|
+
MeadowSchema:
|
|
136
|
+
{
|
|
137
|
+
DefaultIdentifier: 'IDCustomer',
|
|
138
|
+
Schema: [
|
|
139
|
+
{ Column: 'IDCustomer' },
|
|
140
|
+
{ Column: 'GUIDCustomer' },
|
|
141
|
+
{ Column: 'Name' },
|
|
142
|
+
{ Column: 'Email' },
|
|
143
|
+
{ Column: 'CreateDate' },
|
|
144
|
+
{ Column: 'UpdateDate' },
|
|
145
|
+
{ Column: 'Deleted' }
|
|
146
|
+
]
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
const syncEntity = fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityInitial',
|
|
151
|
+
{
|
|
152
|
+
MeadowEntitySchema: entitySchema,
|
|
153
|
+
ConnectionPool: connectionPool,
|
|
154
|
+
PageSize: 500
|
|
155
|
+
},
|
|
156
|
+
'SyncEntity-Customer');
|
|
157
|
+
|
|
158
|
+
syncEntity.initialize(
|
|
159
|
+
(pError) =>
|
|
160
|
+
{
|
|
161
|
+
if (pError)
|
|
162
|
+
{
|
|
163
|
+
console.error('Initialization failed:', pError);
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
syncEntity.sync(
|
|
168
|
+
() =>
|
|
169
|
+
{
|
|
170
|
+
console.log('Initial sync of Customer complete.');
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
### Marshaling a Record Manually
|
|
176
|
+
|
|
177
|
+
```js
|
|
178
|
+
const serverRecord = {
|
|
179
|
+
IDCustomer: 42,
|
|
180
|
+
GUIDCustomer: 'abc-123-def',
|
|
181
|
+
Name: 'Alice',
|
|
182
|
+
Email: 'alice@example.com',
|
|
183
|
+
CreateDate: '2024-01-15T10:30:00.000Z',
|
|
184
|
+
UpdateDate: '2024-06-20T14:00:00.000Z',
|
|
185
|
+
Deleted: 0,
|
|
186
|
+
Metadata: { tier: 'premium' }
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
const localRecord = syncEntity.marshalRecord(serverRecord);
|
|
190
|
+
// localRecord =>
|
|
191
|
+
// {
|
|
192
|
+
// IDCustomer: 42,
|
|
193
|
+
// GUIDCustomer: 'abc-123-def',
|
|
194
|
+
// Name: 'Alice',
|
|
195
|
+
// Email: 'alice@example.com',
|
|
196
|
+
// CreateDate: '2024-01-15 10:30:00.000',
|
|
197
|
+
// UpdateDate: '2024-06-20 14:00:00.000',
|
|
198
|
+
// Deleted: 0
|
|
199
|
+
// }
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
## Sync Flow Diagram
|
|
203
|
+
|
|
204
|
+
```
|
|
205
|
+
Local DB Remote Server
|
|
206
|
+
-------- -------------
|
|
207
|
+
1. Get max ID (local)
|
|
208
|
+
2. Get count (local)
|
|
209
|
+
3. GET /Entity/Max/IDEntity
|
|
210
|
+
4. GET /Entitys/Count
|
|
211
|
+
5. Calculate estimated new records
|
|
212
|
+
6. Generate paginated URL list
|
|
213
|
+
7. GET /Entitys/FilteredTo/FBV~ID~GT~{localMax}~/0/{pageSize}
|
|
214
|
+
8. For each record:
|
|
215
|
+
- Read by ID locally
|
|
216
|
+
- If not found: marshal + create
|
|
217
|
+
9. GET next page...
|
|
218
|
+
10. Repeat until pages exhausted
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
## Related Services
|
|
222
|
+
|
|
223
|
+
- [MeadowSync](./sync.md) -- The orchestrator that creates and manages instances of this class.
|
|
224
|
+
- [MeadowSyncEntityOngoing](./sync-entity-ongoing.md) -- The ongoing/differential variant of entity sync.
|
|
225
|
+
- [MeadowCloneRestClient](./clone-rest-client.md) -- Used via `fable.MeadowCloneRestClient` to download records.
|
|
226
|
+
- [MeadowOperation](./operation.md) -- Provides timestamp and progress tracking utilities used during sync.
|
|
227
|
+
- [MeadowConnectionManager](./connection-manager.md) -- Provides the connection pool and index creation.
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
# MeadowSyncEntityOngoing
|
|
2
|
+
|
|
3
|
+
Performs ongoing (differential) synchronization of a single entity from a remote Meadow API server to a local database. Uses UpdateDate-based comparison to detect changed records, creating new records or updating existing ones as needed.
|
|
4
|
+
|
|
5
|
+
**Source:** `source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js`
|
|
6
|
+
|
|
7
|
+
**Extends:** `fable-serviceproviderbase`
|
|
8
|
+
|
|
9
|
+
**Service Type:** `MeadowSyncEntityOngoing`
|
|
10
|
+
|
|
11
|
+
## Constructor
|
|
12
|
+
|
|
13
|
+
```js
|
|
14
|
+
const syncEntity = fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityOngoing', pOptions, pServiceHash);
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
### Options
|
|
18
|
+
|
|
19
|
+
| Option | Type | Required | Default | Description |
|
|
20
|
+
|--------|------|----------|---------|-------------|
|
|
21
|
+
| `MeadowEntitySchema` | `object` | Yes | -- | Entity schema object containing `TableName`, `Columns` (array), and `MeadowSchema`. |
|
|
22
|
+
| `ConnectionPool` | `object` | No | -- | Database connection pool (used for table/index creation). |
|
|
23
|
+
| `PageSize` | `number` | No | `100` | Number of records per paginated download request. |
|
|
24
|
+
|
|
25
|
+
**Validation:** Same as `MeadowSyncEntityInitial` -- throws errors for missing or invalid schema properties.
|
|
26
|
+
|
|
27
|
+
## Properties
|
|
28
|
+
|
|
29
|
+
### `EntitySchema`
|
|
30
|
+
|
|
31
|
+
`object` -- Deep copy of the provided `MeadowEntitySchema` option.
|
|
32
|
+
|
|
33
|
+
### `DefaultIdentifier`
|
|
34
|
+
|
|
35
|
+
`string` -- The default identifier column name from `EntitySchema.MeadowSchema.DefaultIdentifier`.
|
|
36
|
+
|
|
37
|
+
### `PageSize`
|
|
38
|
+
|
|
39
|
+
`number` -- Records per download page.
|
|
40
|
+
|
|
41
|
+
### `Meadow`
|
|
42
|
+
|
|
43
|
+
The Meadow ORM instance for this entity. Set by `initialize()`.
|
|
44
|
+
|
|
45
|
+
### `operation`
|
|
46
|
+
|
|
47
|
+
`MeadowOperation` -- Instance of the operation utility for timestamps and progress tracking.
|
|
48
|
+
|
|
49
|
+
## Methods
|
|
50
|
+
|
|
51
|
+
### `initialize(fCallback)`
|
|
52
|
+
|
|
53
|
+
Prepares the entity for synchronization. Identical behavior to `MeadowSyncEntityInitial.initialize()`: creates the local table if needed and sets up GUID (unique) and Deleted (non-unique) indexes.
|
|
54
|
+
|
|
55
|
+
| Parameter | Type | Description |
|
|
56
|
+
|-----------|------|-------------|
|
|
57
|
+
| `fCallback` | `function(pError)` | Callback invoked when initialization is complete. |
|
|
58
|
+
|
|
59
|
+
### `marshalRecord(pSourceRecord)`
|
|
60
|
+
|
|
61
|
+
Transforms a server-side record into a local record suitable for insertion or update.
|
|
62
|
+
|
|
63
|
+
| Parameter | Type | Description |
|
|
64
|
+
|-----------|------|-------------|
|
|
65
|
+
| `pSourceRecord` | `object` | The record object from the server. |
|
|
66
|
+
|
|
67
|
+
**Returns:** `object` -- The marshaled record.
|
|
68
|
+
|
|
69
|
+
**Differences from Initial sync marshaling:**
|
|
70
|
+
- Does **not** perform UTC DateTime formatting. Values are copied as-is (unlike Initial sync which formats DateTime columns to `'YYYY-MM-DD HH:mm:ss.SSS'`).
|
|
71
|
+
- Otherwise follows the same column-mapping logic: skips null/undefined, stringifies objects, handles `*JSON` columns.
|
|
72
|
+
|
|
73
|
+
### `addSyncAnticipateEntry(tmpSyncState, tmpAnticipate)`
|
|
74
|
+
|
|
75
|
+
Adds a recursive anticipate entry for paginated sync. This is an internal method that implements the core download-and-sync loop for ongoing sync.
|
|
76
|
+
|
|
77
|
+
| Parameter | Type | Description |
|
|
78
|
+
|-----------|------|-------------|
|
|
79
|
+
| `tmpSyncState` | `object` | Mutable sync state tracking `LastRequestedID`, `RequestsPerformed`, `EstimatedRequestCount`. |
|
|
80
|
+
| `tmpAnticipate` | `object` | Fable Anticipate instance for managing asynchronous operations. |
|
|
81
|
+
|
|
82
|
+
**Recursive pagination pattern:**
|
|
83
|
+
1. Downloads a page of records filtered to `ID > LastRequestedID`, sorted ascending.
|
|
84
|
+
2. For each record in the page, adds an anticipate entry that:
|
|
85
|
+
- Updates `LastRequestedID` to track progress.
|
|
86
|
+
- Reads the local record by ID.
|
|
87
|
+
- If found: compares `UpdateDate` between server and local. Skips if difference is less than 5ms.
|
|
88
|
+
- If the record exists and dates differ: marshals and updates via `Meadow.doUpdate()`.
|
|
89
|
+
- If the record does not exist: marshals and creates via `Meadow.doCreate()` with identity insert enabled.
|
|
90
|
+
3. After processing the page, if `RequestsPerformed < EstimatedRequestCount`, recursively calls `addSyncAnticipateEntry` to fetch the next page.
|
|
91
|
+
|
|
92
|
+
### `sync(fCallback)`
|
|
93
|
+
|
|
94
|
+
Executes the full ongoing sync algorithm for this entity.
|
|
95
|
+
|
|
96
|
+
| Parameter | Type | Description |
|
|
97
|
+
|-----------|------|-------------|
|
|
98
|
+
| `fCallback` | `function()` | Callback invoked when sync completes (errors are logged, not passed). |
|
|
99
|
+
|
|
100
|
+
**Algorithm:**
|
|
101
|
+
|
|
102
|
+
1. **Detect UpdateDate column** -- Checks if the entity schema has an `UpdateDate` column.
|
|
103
|
+
2. **Get local max ID** -- Reads the highest-ID record from the local database.
|
|
104
|
+
3. **Get local max UpdateDate** -- Reads the record with the most recent `UpdateDate`.
|
|
105
|
+
4. **Get local count** -- Counts all local records.
|
|
106
|
+
5. **Get server max ID** -- Requests from `{Entity}/Max/{DefaultIdentifier}`.
|
|
107
|
+
6. **Get server max UpdateDate** -- Requests from `{Entity}/Max/UpdateDate`.
|
|
108
|
+
7. **Get server count** -- Requests from `{Entity}s/Count`.
|
|
109
|
+
8. **Calculate estimated requests** -- `ceil(Server.RecordCount / PageSize)`.
|
|
110
|
+
9. **Begin recursive sync** -- Calls `addSyncAnticipateEntry()` to start the paginated download-and-sync loop. All records on the server are scanned (not just those newer than the local max).
|
|
111
|
+
|
|
112
|
+
**Progress tracking:** Uses `MeadowOperation` with tracker hash `UpdateSync-{TableName}`.
|
|
113
|
+
|
|
114
|
+
## Differences from Initial Sync
|
|
115
|
+
|
|
116
|
+
| Aspect | Initial | Ongoing |
|
|
117
|
+
|--------|---------|---------|
|
|
118
|
+
| **Strategy** | ID-based gap fill | Full scan with UpdateDate comparison |
|
|
119
|
+
| **Filter** | Only records with `ID > LocalMaxID` | All records, paginated by ascending ID |
|
|
120
|
+
| **Existing records** | Skips (already present) | Compares `UpdateDate`, updates if changed |
|
|
121
|
+
| **DateTime marshaling** | Formats to `YYYY-MM-DD HH:mm:ss.SSS` UTC | Copies values as-is |
|
|
122
|
+
| **Pagination** | Pre-computed URL list | Recursive anticipate pattern |
|
|
123
|
+
| **UpdateDate threshold** | N/A | Skips updates if difference < 5ms |
|
|
124
|
+
| **Progress tracker hash** | `FullSync-{TableName}` | `UpdateSync-{TableName}` |
|
|
125
|
+
|
|
126
|
+
## Usage Examples
|
|
127
|
+
|
|
128
|
+
### Direct Instantiation and Ongoing Sync
|
|
129
|
+
|
|
130
|
+
```js
|
|
131
|
+
const libFable = require('fable');
|
|
132
|
+
const libSyncEntityOngoing = require('meadow-integration/source/services/clone/Meadow-Service-Sync-Entity-Ongoing');
|
|
133
|
+
|
|
134
|
+
const fable = new libFable({ Product: 'CloneApp' });
|
|
135
|
+
|
|
136
|
+
fable.serviceManager.addServiceType('MeadowSyncEntityOngoing', libSyncEntityOngoing);
|
|
137
|
+
|
|
138
|
+
const entitySchema = {
|
|
139
|
+
TableName: 'Order',
|
|
140
|
+
Columns:
|
|
141
|
+
[
|
|
142
|
+
{ Column: 'IDOrder', DataType: 'AutoIdentity' },
|
|
143
|
+
{ Column: 'GUIDOrder', DataType: 'GUID' },
|
|
144
|
+
{ Column: 'CustomerID', DataType: 'Integer' },
|
|
145
|
+
{ Column: 'Total', DataType: 'Decimal' },
|
|
146
|
+
{ Column: 'CreateDate', DataType: 'DateTime' },
|
|
147
|
+
{ Column: 'UpdateDate', DataType: 'DateTime' },
|
|
148
|
+
{ Column: 'Deleted', DataType: 'Boolean' }
|
|
149
|
+
],
|
|
150
|
+
MeadowSchema:
|
|
151
|
+
{
|
|
152
|
+
DefaultIdentifier: 'IDOrder',
|
|
153
|
+
Schema:
|
|
154
|
+
[
|
|
155
|
+
{ Column: 'IDOrder' },
|
|
156
|
+
{ Column: 'GUIDOrder' },
|
|
157
|
+
{ Column: 'CustomerID' },
|
|
158
|
+
{ Column: 'Total' },
|
|
159
|
+
{ Column: 'CreateDate' },
|
|
160
|
+
{ Column: 'UpdateDate' },
|
|
161
|
+
{ Column: 'Deleted' }
|
|
162
|
+
]
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const syncEntity = fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityOngoing',
|
|
167
|
+
{
|
|
168
|
+
MeadowEntitySchema: entitySchema,
|
|
169
|
+
ConnectionPool: connectionPool,
|
|
170
|
+
PageSize: 250
|
|
171
|
+
},
|
|
172
|
+
'SyncEntity-Order');
|
|
173
|
+
|
|
174
|
+
syncEntity.initialize(
|
|
175
|
+
(pError) =>
|
|
176
|
+
{
|
|
177
|
+
if (pError)
|
|
178
|
+
{
|
|
179
|
+
console.error('Initialization failed:', pError);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
syncEntity.sync(
|
|
184
|
+
() =>
|
|
185
|
+
{
|
|
186
|
+
console.log('Ongoing sync of Order complete.');
|
|
187
|
+
});
|
|
188
|
+
});
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
### Using via MeadowSync Orchestrator
|
|
192
|
+
|
|
193
|
+
```js
|
|
194
|
+
const sync = fable.serviceManager.instantiateServiceProvider('MeadowSync',
|
|
195
|
+
{
|
|
196
|
+
ConnectionPool: connectionPool,
|
|
197
|
+
PageSize: 200,
|
|
198
|
+
SyncEntityList: ['Customer', 'Order', 'Product']
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
// Set ongoing mode before loading schema
|
|
202
|
+
sync.SyncMode = 'Ongoing';
|
|
203
|
+
|
|
204
|
+
sync.loadMeadowSchema(compiledSchema,
|
|
205
|
+
(pLoadError) =>
|
|
206
|
+
{
|
|
207
|
+
sync.syncAll(
|
|
208
|
+
(pSyncError) =>
|
|
209
|
+
{
|
|
210
|
+
console.log('Ongoing sync of all entities complete.');
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
## Sync Flow Diagram
|
|
216
|
+
|
|
217
|
+
```
|
|
218
|
+
Local DB Remote Server
|
|
219
|
+
-------- -------------
|
|
220
|
+
1. Check for UpdateDate column
|
|
221
|
+
2. Get max ID (local)
|
|
222
|
+
3. Get max UpdateDate (local)
|
|
223
|
+
4. Get count (local)
|
|
224
|
+
5. GET /Entity/Max/IDEntity
|
|
225
|
+
6. GET /Entity/Max/UpdateDate
|
|
226
|
+
7. GET /Entitys/Count
|
|
227
|
+
8. Calculate estimated request count
|
|
228
|
+
9. GET page (ID > LastRequestedID, ascending)
|
|
229
|
+
10. For each record:
|
|
230
|
+
- Read by ID locally
|
|
231
|
+
- If found & UpdateDate diff >= 5ms: marshal + update
|
|
232
|
+
- If not found: marshal + create
|
|
233
|
+
- Update LastRequestedID
|
|
234
|
+
11. Recursively fetch next page...
|
|
235
|
+
12. Repeat until all pages processed
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
## Related Services
|
|
239
|
+
|
|
240
|
+
- [MeadowSync](./sync.md) -- The orchestrator that creates and manages instances of this class.
|
|
241
|
+
- [MeadowSyncEntityInitial](./sync-entity-initial.md) -- The initial full-sync variant.
|
|
242
|
+
- [MeadowCloneRestClient](./clone-rest-client.md) -- Used via `fable.MeadowCloneRestClient` to download records.
|
|
243
|
+
- [MeadowOperation](./operation.md) -- Provides timestamp and progress tracking utilities.
|
|
244
|
+
- [MeadowConnectionManager](./connection-manager.md) -- Provides the connection pool and index creation.
|
package/docs/api/sync.md
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# MeadowSync
|
|
2
|
+
|
|
3
|
+
Top-level synchronization orchestrator for the data-clone pipeline. Loads a Meadow schema, creates per-entity sync objects (Initial or Ongoing), and coordinates syncing all entities in sequence.
|
|
4
|
+
|
|
5
|
+
**Source:** `source/services/clone/Meadow-Service-Sync.js`
|
|
6
|
+
|
|
7
|
+
**Extends:** `fable-serviceproviderbase`
|
|
8
|
+
|
|
9
|
+
**Service Type:** `MeadowSync`
|
|
10
|
+
|
|
11
|
+
## Constructor
|
|
12
|
+
|
|
13
|
+
```js
|
|
14
|
+
const sync = fable.serviceManager.instantiateServiceProvider('MeadowSync', pOptions);
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
### Options
|
|
18
|
+
|
|
19
|
+
| Option | Type | Default | Description |
|
|
20
|
+
|--------|------|---------|-------------|
|
|
21
|
+
| `SyncEntityList` | `Array<string>` | `[]` | Ordered list of entity table names to sync. If empty, all entities in the schema are synced. Also read from `fable.ProgramConfiguration.SyncEntityList`. |
|
|
22
|
+
| `SyncEntityOptions` | `object` | `{}` | Per-entity sync options keyed by table name. Also read from `fable.ProgramConfiguration.SyncEntityOptions`. |
|
|
23
|
+
| `ConnectionPool` | `object` | *(none)* | Database connection pool passed through to per-entity sync objects. |
|
|
24
|
+
| `PageSize` | `number` | `100` | Number of records per download page, passed through to per-entity sync objects. |
|
|
25
|
+
|
|
26
|
+
## Properties
|
|
27
|
+
|
|
28
|
+
### `SyncMode`
|
|
29
|
+
|
|
30
|
+
`string` -- Controls which sync strategy is used when `loadMeadowSchema()` creates entity sync objects.
|
|
31
|
+
|
|
32
|
+
| Value | Behavior |
|
|
33
|
+
|-------|----------|
|
|
34
|
+
| `'Initial'` *(default)* | Creates `MeadowSyncEntityInitial` instances. Performs a full ID-based sync of all missing records. |
|
|
35
|
+
| `'Ongoing'` | Creates `MeadowSyncEntityOngoing` instances. Performs UpdateDate-based differential sync, creating or updating records. |
|
|
36
|
+
|
|
37
|
+
Set this property **before** calling `loadMeadowSchema()`.
|
|
38
|
+
|
|
39
|
+
### `SyncEntityList`
|
|
40
|
+
|
|
41
|
+
`Array<string>` -- The list of entity table names to synchronize, in order. Populated from options, `ProgramConfiguration`, or automatically from the loaded schema.
|
|
42
|
+
|
|
43
|
+
### `SyncEntityOptions`
|
|
44
|
+
|
|
45
|
+
`object` -- Per-entity options map. Keys are table names.
|
|
46
|
+
|
|
47
|
+
### `MeadowSyncEntities`
|
|
48
|
+
|
|
49
|
+
`object` -- Map of table name to instantiated `MeadowSyncEntityInitial` or `MeadowSyncEntityOngoing` service instances. Populated by `loadMeadowSchema()`.
|
|
50
|
+
|
|
51
|
+
### `MeadowSchemaTableList`
|
|
52
|
+
|
|
53
|
+
`Array<string>` -- List of all table names found in the loaded Meadow schema. Set by `loadMeadowSchema()`.
|
|
54
|
+
|
|
55
|
+
## Methods
|
|
56
|
+
|
|
57
|
+
### `loadMeadowSchema(pSchema, fCallback)`
|
|
58
|
+
|
|
59
|
+
Loads a compiled Meadow schema and creates per-entity sync objects.
|
|
60
|
+
|
|
61
|
+
| Parameter | Type | Description |
|
|
62
|
+
|-----------|------|-------------|
|
|
63
|
+
| `pSchema` | `object` | A compiled Meadow schema with a `Tables` property. Each key is a table name; each value is an entity schema object with `TableName`, `Columns`, and `MeadowSchema`. |
|
|
64
|
+
| `fCallback` | `function(pError)` | Callback invoked after all entity sync objects are initialized (tables created, indexes set up). |
|
|
65
|
+
|
|
66
|
+
**Behavior:**
|
|
67
|
+
1. Iterates through every table in the schema.
|
|
68
|
+
2. For tables that are in `SyncEntityList` (or all tables if the list is empty), creates a sync entity object according to `SyncMode`.
|
|
69
|
+
3. Calls `initialize()` on each entity sync object to ensure the local table and indexes exist.
|
|
70
|
+
4. If `SyncEntityList` was empty, populates it with all initialized entity names.
|
|
71
|
+
|
|
72
|
+
### `syncEntity(pEntityHash, fCallback)`
|
|
73
|
+
|
|
74
|
+
Syncs a single entity by its table name.
|
|
75
|
+
|
|
76
|
+
| Parameter | Type | Description |
|
|
77
|
+
|-----------|------|-------------|
|
|
78
|
+
| `pEntityHash` | `string` | The table name of the entity to sync (e.g. `'Animal'`). |
|
|
79
|
+
| `fCallback` | `function(pError)` | Callback invoked when the entity sync completes. |
|
|
80
|
+
|
|
81
|
+
Logs a warning and returns immediately if the entity does not exist in `MeadowSyncEntities`.
|
|
82
|
+
|
|
83
|
+
### `syncAll(fCallback)`
|
|
84
|
+
|
|
85
|
+
Syncs all entities in `SyncEntityList` sequentially.
|
|
86
|
+
|
|
87
|
+
| Parameter | Type | Description |
|
|
88
|
+
|-----------|------|-------------|
|
|
89
|
+
| `fCallback` | `function(pError)` | Callback invoked when all entities have been synced. |
|
|
90
|
+
|
|
91
|
+
Iterates through `SyncEntityList` with a concurrency of 1, calling `syncEntity()` for each.
|
|
92
|
+
|
|
93
|
+
## Usage Examples
|
|
94
|
+
|
|
95
|
+
### Full Initial Sync Workflow
|
|
96
|
+
|
|
97
|
+
```js
|
|
98
|
+
const libFable = require('fable');
|
|
99
|
+
const libConnectionManager = require('meadow-integration/source/services/clone/Meadow-Service-ConnectionManager');
|
|
100
|
+
const libRestClient = require('meadow-integration/source/services/clone/Meadow-Service-RestClient');
|
|
101
|
+
const libSync = require('meadow-integration/source/services/clone/Meadow-Service-Sync');
|
|
102
|
+
|
|
103
|
+
const fable = new libFable({ Product: 'DataClone' });
|
|
104
|
+
|
|
105
|
+
// Register services
|
|
106
|
+
fable.serviceManager.addServiceType('MeadowConnectionManager', libConnectionManager);
|
|
107
|
+
fable.serviceManager.addServiceType('MeadowCloneRestClient', libRestClient);
|
|
108
|
+
fable.serviceManager.addServiceType('MeadowSync', libSync);
|
|
109
|
+
|
|
110
|
+
// Instantiate connection manager
|
|
111
|
+
const connectionManager = fable.serviceManager.instantiateServiceProvider('MeadowConnectionManager',
|
|
112
|
+
{
|
|
113
|
+
Provider: 'MySQL',
|
|
114
|
+
MySQL: { server: '127.0.0.1', user: 'root', password: '', database: 'clone_db' }
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
// Instantiate REST client
|
|
118
|
+
const restClient = fable.serviceManager.instantiateServiceProvider('MeadowCloneRestClient',
|
|
119
|
+
{
|
|
120
|
+
ServerURL: 'https://api.example.com/1.0/',
|
|
121
|
+
UserID: 'sync_user',
|
|
122
|
+
Password: 'sync_password'
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
// Connect, authenticate, then sync
|
|
126
|
+
connectionManager.connect(
|
|
127
|
+
(pConnError) =>
|
|
128
|
+
{
|
|
129
|
+
if (pConnError) { throw pConnError; }
|
|
130
|
+
|
|
131
|
+
restClient.authenticate(
|
|
132
|
+
(pAuthError) =>
|
|
133
|
+
{
|
|
134
|
+
if (pAuthError) { throw pAuthError; }
|
|
135
|
+
|
|
136
|
+
const sync = fable.serviceManager.instantiateServiceProvider('MeadowSync',
|
|
137
|
+
{
|
|
138
|
+
ConnectionPool: connectionManager.ConnectionPool,
|
|
139
|
+
PageSize: 200,
|
|
140
|
+
SyncEntityList: ['Customer', 'Order', 'Product']
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// Use initial sync mode (the default)
|
|
144
|
+
sync.SyncMode = 'Initial';
|
|
145
|
+
|
|
146
|
+
const meadowSchema = require('./my-compiled-schema.json');
|
|
147
|
+
sync.loadMeadowSchema(meadowSchema,
|
|
148
|
+
(pLoadError) =>
|
|
149
|
+
{
|
|
150
|
+
if (pLoadError) { throw pLoadError; }
|
|
151
|
+
|
|
152
|
+
sync.syncAll(
|
|
153
|
+
(pSyncError) =>
|
|
154
|
+
{
|
|
155
|
+
if (pSyncError)
|
|
156
|
+
{
|
|
157
|
+
console.error('Sync failed:', pSyncError);
|
|
158
|
+
}
|
|
159
|
+
else
|
|
160
|
+
{
|
|
161
|
+
console.log('Initial sync complete!');
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
restClient.deauthenticate(() => { process.exit(0); });
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### Ongoing (Differential) Sync
|
|
172
|
+
|
|
173
|
+
```js
|
|
174
|
+
const sync = fable.serviceManager.instantiateServiceProvider('MeadowSync',
|
|
175
|
+
{
|
|
176
|
+
ConnectionPool: connectionManager.ConnectionPool,
|
|
177
|
+
PageSize: 100
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
// Switch to ongoing mode before loading schema
|
|
181
|
+
sync.SyncMode = 'Ongoing';
|
|
182
|
+
|
|
183
|
+
sync.loadMeadowSchema(meadowSchema,
|
|
184
|
+
(pLoadError) =>
|
|
185
|
+
{
|
|
186
|
+
sync.syncAll(
|
|
187
|
+
(pSyncError) =>
|
|
188
|
+
{
|
|
189
|
+
console.log('Ongoing sync complete!');
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
### Syncing a Single Entity
|
|
195
|
+
|
|
196
|
+
```js
|
|
197
|
+
sync.loadMeadowSchema(meadowSchema,
|
|
198
|
+
(pLoadError) =>
|
|
199
|
+
{
|
|
200
|
+
sync.syncEntity('Customer',
|
|
201
|
+
(pSyncError) =>
|
|
202
|
+
{
|
|
203
|
+
console.log('Customer sync complete!');
|
|
204
|
+
});
|
|
205
|
+
});
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
## Related Services
|
|
209
|
+
|
|
210
|
+
- [MeadowConnectionManager](./connection-manager.md) -- Provides the database connection pool.
|
|
211
|
+
- [MeadowCloneRestClient](./clone-rest-client.md) -- REST client used by entity sync objects to download records.
|
|
212
|
+
- [MeadowSyncEntityInitial](./sync-entity-initial.md) -- Created by MeadowSync when `SyncMode` is `'Initial'`.
|
|
213
|
+
- [MeadowSyncEntityOngoing](./sync-entity-ongoing.md) -- Created by MeadowSync when `SyncMode` is `'Ongoing'`.
|