meadow-integration 1.0.4 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/.dockerignore +11 -0
  2. package/Docker-Build.sh +2 -0
  3. package/Docker-Compose.sh +2 -0
  4. package/Docker-Push.sh +2 -0
  5. package/Docker-Tag.sh +2 -0
  6. package/Dockerfile +28 -0
  7. package/Dockerfile_LUXURYCode +23 -0
  8. package/README.md +139 -25
  9. package/docker-compose.yml +16 -0
  10. package/docs/README.md +65 -18
  11. package/docs/{cover.md → _cover.md} +3 -2
  12. package/docs/_sidebar.md +52 -7
  13. package/docs/_topbar.md +2 -0
  14. package/docs/api/clone-rest-client.md +278 -0
  15. package/docs/api/connection-manager.md +179 -0
  16. package/docs/api/guid-map.md +234 -0
  17. package/docs/api/integration-adapter.md +283 -0
  18. package/docs/api/operation.md +241 -0
  19. package/docs/api/sync-entity-initial.md +227 -0
  20. package/docs/api/sync-entity-ongoing.md +244 -0
  21. package/docs/api/sync.md +213 -0
  22. package/docs/api/tabular-check.md +213 -0
  23. package/docs/api/tabular-transform.md +316 -0
  24. package/docs/architecture.md +423 -0
  25. package/docs/cli/comprehensionarray.md +111 -0
  26. package/docs/cli/comprehensionintersect.md +132 -0
  27. package/docs/cli/csvcheck.md +111 -0
  28. package/docs/cli/csvtransform.md +170 -0
  29. package/docs/cli/data-clone.md +277 -0
  30. package/docs/cli/jsonarraytransform.md +166 -0
  31. package/docs/cli/load-comprehension.md +129 -0
  32. package/docs/cli/objectarraytocsv.md +159 -0
  33. package/docs/cli/overview.md +96 -0
  34. package/docs/cli/serve.md +102 -0
  35. package/docs/cli/tsvtransform.md +144 -0
  36. package/docs/data-clone/configuration.md +357 -0
  37. package/docs/data-clone/connection-manager.md +206 -0
  38. package/docs/data-clone/docker.md +290 -0
  39. package/docs/data-clone/overview.md +173 -0
  40. package/docs/data-clone/sync-modes.md +186 -0
  41. package/docs/implementation-reference.md +311 -0
  42. package/docs/overview.md +156 -0
  43. package/docs/quickstart.md +233 -0
  44. package/docs/rest/comprehension-push.md +209 -0
  45. package/docs/rest/comprehension.md +506 -0
  46. package/docs/rest/csv.md +255 -0
  47. package/docs/rest/entity-generation.md +158 -0
  48. package/docs/rest/json-array.md +243 -0
  49. package/docs/rest/overview.md +120 -0
  50. package/docs/rest/status.md +63 -0
  51. package/docs/rest/tsv.md +241 -0
  52. package/docs/retold-catalog.json +93 -3
  53. package/docs/retold-keyword-index.json +23683 -1901
  54. package/package.json +13 -10
  55. package/scripts/run.sh +18 -0
  56. package/source/Meadow-Integration.js +15 -1
  57. package/source/cli/Default-Meadow-Integration-Configuration.json +37 -2
  58. package/source/cli/Meadow-Integration-CLI-Program.js +4 -1
  59. package/source/cli/commands/Meadow-Integration-Command-DataClone.js +284 -0
  60. package/source/services/clone/Meadow-Service-ConnectionManager.js +251 -0
  61. package/source/services/clone/Meadow-Service-Operation.js +196 -0
  62. package/source/services/clone/Meadow-Service-RestClient.js +364 -0
  63. package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js +367 -0
  64. package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js +457 -0
  65. package/source/services/clone/Meadow-Service-Sync.js +142 -0
  66. /package/docs/examples/bookstore/{mapping_books_Author.json → mapping_books_author.json} +0 -0
  67. /package/docs/examples/bookstore/{mapping_books_Book.json → mapping_books_book.json} +0 -0
@@ -0,0 +1,227 @@
1
+ # MeadowSyncEntityInitial
2
+
3
+ Performs a full initial synchronization of a single entity from a remote Meadow API server to a local database. Downloads all records that do not yet exist locally by comparing max IDs, then inserts missing records in paginated batches.
4
+
5
+ **Source:** `source/services/clone/Meadow-Service-Sync-Entity-Initial.js`
6
+
7
+ **Extends:** `fable-serviceproviderbase`
8
+
9
+ **Service Type:** `MeadowSyncEntityInitial`
10
+
11
+ ## Constructor
12
+
13
+ ```js
14
+ const syncEntity = fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityInitial', pOptions, pServiceHash);
15
+ ```
16
+
17
+ ### Options
18
+
19
+ | Option | Type | Required | Default | Description |
20
+ |--------|------|----------|---------|-------------|
21
+ | `MeadowEntitySchema` | `object` | Yes | -- | Entity schema object containing `TableName`, `Columns` (array), and `MeadowSchema`. |
22
+ | `ConnectionPool` | `object` | No | -- | Database connection pool (used for table/index creation). |
23
+ | `PageSize` | `number` | No | `100` | Number of records per paginated download request. |
24
+
25
+ **Validation:** The constructor throws an `Error` if:
26
+ - `MeadowEntitySchema` is missing or not an object.
27
+ - `MeadowEntitySchema.TableName` is missing or empty.
28
+ - `MeadowEntitySchema.Columns` is missing or not a non-empty array.
29
+ - `MeadowEntitySchema.MeadowSchema` is missing.
30
+
31
+ ## Properties
32
+
33
+ ### `EntitySchema`
34
+
35
+ `object` -- Deep copy of the provided `MeadowEntitySchema` option.
36
+
37
+ ### `DefaultIdentifier`
38
+
39
+ `string` -- The default identifier column name from `EntitySchema.MeadowSchema.DefaultIdentifier` (typically `'IDAnimal'`, `'IDCustomer'`, etc.).
40
+
41
+ ### `PageSize`
42
+
43
+ `number` -- Records per download page.
44
+
45
+ ### `Meadow`
46
+
47
+ The Meadow ORM instance for this entity, loaded from the entity's `MeadowSchema` package object. Set by `initialize()`.
48
+
49
+ ### `operation`
50
+
51
+ `MeadowOperation` -- Instance of the operation utility for timestamps and progress tracking.
52
+
53
+ ## Methods
54
+
55
+ ### `initialize(fCallback)`
56
+
57
+ Prepares the entity for synchronization by creating the local database table (if it does not exist) and setting up indexes on GUID and Deleted columns.
58
+
59
+ | Parameter | Type | Description |
60
+ |-----------|------|-------------|
61
+ | `fCallback` | `function(pError)` | Callback invoked when initialization is complete. |
62
+
63
+ **Behavior:**
64
+ 1. Loads the Meadow ORM instance from the entity's schema package.
65
+ 2. Calls the provider's `createTable()` to ensure the table exists.
66
+ 3. If the schema has a GUID column (`DataType == 'GUID'`), creates a unique index on it.
67
+ 4. If the schema has a `Deleted` column, creates a non-unique index on it.
68
+ 5. Index creation requires `fable.MeadowConnectionManager` to be available.
69
+
70
+ ### `marshalRecord(pSourceRecord)`
71
+
72
+ Transforms a server-side record into a local record suitable for insertion, based on the entity schema columns.
73
+
74
+ | Parameter | Type | Description |
75
+ |-----------|------|-------------|
76
+ | `pSourceRecord` | `object` | The record object from the server. |
77
+
78
+ **Returns:** `object` -- The marshaled record with only schema-defined columns.
79
+
80
+ **Transformation rules:**
81
+ - `null`/`undefined` values are skipped.
82
+ - Object values are JSON-stringified.
83
+ - `DateTime` columns are formatted to `'YYYY-MM-DD HH:mm:ss.SSS'` in UTC.
84
+ - Empty string values are skipped (except for DateTime).
85
+ - Columns ending in `JSON` auto-stringify the corresponding non-JSON property from the source (e.g., `MetadataJSON` from `Metadata`).
86
+
87
+ ### `sync(fCallback)`
88
+
89
+ Executes the full initial sync algorithm for this entity.
90
+
91
+ | Parameter | Type | Description |
92
+ |-----------|------|-------------|
93
+ | `fCallback` | `function()` | Callback invoked when sync completes (errors are logged, not passed). |
94
+
95
+ **Algorithm:**
96
+
97
+ 1. **Get local max ID** -- Reads the highest-ID record from the local database.
98
+ 2. **Get local count** -- Counts all local records.
99
+ 3. **Get server max ID** -- Requests max ID from `{Entity}/Max/{DefaultIdentifier}`.
100
+ 4. **Get server count** -- Requests count from `{Entity}s/Count`.
101
+ 5. **Calculate estimated records** -- `Server.RecordCount - Local.RecordCount`.
102
+ 6. **Generate paginated URLs** -- Creates URL partials filtered to `{DefaultIdentifier} > {LocalMaxID}`, sorted ascending, paginated by `PageSize`.
103
+ 7. **Download and insert** -- For each page, downloads records and for each record:
104
+ - Checks if the record already exists locally by ID.
105
+ - If not found, marshals the record and creates it with identity insert enabled and all auto-stamps disabled.
106
+ - Updates the progress tracker after each insert.
107
+ 8. **Early termination** -- If an empty page is returned, stops downloading.
108
+
109
+ **Progress tracking:** Uses `MeadowOperation` with tracker hash `FullSync-{TableName}`.
110
+
111
+ ## Usage Examples
112
+
113
+ ### Direct Instantiation and Sync
114
+
115
+ ```js
116
+ const libFable = require('fable');
117
+ const libSyncEntityInitial = require('meadow-integration/source/services/clone/Meadow-Service-Sync-Entity-Initial');
118
+
119
+ const fable = new libFable({ Product: 'CloneApp' });
120
+
121
+ fable.serviceManager.addServiceType('MeadowSyncEntityInitial', libSyncEntityInitial);
122
+
123
+ const entitySchema = {
124
+ TableName: 'Customer',
125
+ Columns:
126
+ [
127
+ { Column: 'IDCustomer', DataType: 'AutoIdentity' },
128
+ { Column: 'GUIDCustomer', DataType: 'GUID' },
129
+ { Column: 'Name', DataType: 'String' },
130
+ { Column: 'Email', DataType: 'String' },
131
+ { Column: 'CreateDate', DataType: 'DateTime' },
132
+ { Column: 'UpdateDate', DataType: 'DateTime' },
133
+ { Column: 'Deleted', DataType: 'Boolean' }
134
+ ],
135
+ MeadowSchema:
136
+ {
137
+ DefaultIdentifier: 'IDCustomer',
138
+ Schema: [
139
+ { Column: 'IDCustomer' },
140
+ { Column: 'GUIDCustomer' },
141
+ { Column: 'Name' },
142
+ { Column: 'Email' },
143
+ { Column: 'CreateDate' },
144
+ { Column: 'UpdateDate' },
145
+ { Column: 'Deleted' }
146
+ ]
147
+ }
148
+ };
149
+
150
+ const syncEntity = fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityInitial',
151
+ {
152
+ MeadowEntitySchema: entitySchema,
153
+ ConnectionPool: connectionPool,
154
+ PageSize: 500
155
+ },
156
+ 'SyncEntity-Customer');
157
+
158
+ syncEntity.initialize(
159
+ (pError) =>
160
+ {
161
+ if (pError)
162
+ {
163
+ console.error('Initialization failed:', pError);
164
+ return;
165
+ }
166
+
167
+ syncEntity.sync(
168
+ () =>
169
+ {
170
+ console.log('Initial sync of Customer complete.');
171
+ });
172
+ });
173
+ ```
174
+
175
+ ### Marshaling a Record Manually
176
+
177
+ ```js
178
+ const serverRecord = {
179
+ IDCustomer: 42,
180
+ GUIDCustomer: 'abc-123-def',
181
+ Name: 'Alice',
182
+ Email: 'alice@example.com',
183
+ CreateDate: '2024-01-15T10:30:00.000Z',
184
+ UpdateDate: '2024-06-20T14:00:00.000Z',
185
+ Deleted: 0,
186
+ Metadata: { tier: 'premium' }
187
+ };
188
+
189
+ const localRecord = syncEntity.marshalRecord(serverRecord);
190
+ // localRecord =>
191
+ // {
192
+ // IDCustomer: 42,
193
+ // GUIDCustomer: 'abc-123-def',
194
+ // Name: 'Alice',
195
+ // Email: 'alice@example.com',
196
+ // CreateDate: '2024-01-15 10:30:00.000',
197
+ // UpdateDate: '2024-06-20 14:00:00.000',
198
+ // Deleted: 0
199
+ // }
200
+ ```
201
+
202
+ ## Sync Flow Diagram
203
+
204
+ ```
205
+ Local DB Remote Server
206
+ -------- -------------
207
+ 1. Get max ID (local)
208
+ 2. Get count (local)
209
+ 3. GET /Entity/Max/IDEntity
210
+ 4. GET /Entitys/Count
211
+ 5. Calculate estimated new records
212
+ 6. Generate paginated URL list
213
+ 7. GET /Entitys/FilteredTo/FBV~ID~GT~{localMax}~/0/{pageSize}
214
+ 8. For each record:
215
+ - Read by ID locally
216
+ - If not found: marshal + create
217
+ 9. GET next page...
218
+ 10. Repeat until pages exhausted
219
+ ```
220
+
221
+ ## Related Services
222
+
223
+ - [MeadowSync](./sync.md) -- The orchestrator that creates and manages instances of this class.
224
+ - [MeadowSyncEntityOngoing](./sync-entity-ongoing.md) -- The ongoing/differential variant of entity sync.
225
+ - [MeadowCloneRestClient](./clone-rest-client.md) -- Used via `fable.MeadowCloneRestClient` to download records.
226
+ - [MeadowOperation](./operation.md) -- Provides timestamp and progress tracking utilities used during sync.
227
+ - [MeadowConnectionManager](./connection-manager.md) -- Provides the connection pool and index creation.
@@ -0,0 +1,244 @@
1
+ # MeadowSyncEntityOngoing
2
+
3
+ Performs ongoing (differential) synchronization of a single entity from a remote Meadow API server to a local database. Uses UpdateDate-based comparison to detect changed records, creating new records or updating existing ones as needed.
4
+
5
+ **Source:** `source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js`
6
+
7
+ **Extends:** `fable-serviceproviderbase`
8
+
9
+ **Service Type:** `MeadowSyncEntityOngoing`
10
+
11
+ ## Constructor
12
+
13
+ ```js
14
+ const syncEntity = fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityOngoing', pOptions, pServiceHash);
15
+ ```
16
+
17
+ ### Options
18
+
19
+ | Option | Type | Required | Default | Description |
20
+ |--------|------|----------|---------|-------------|
21
+ | `MeadowEntitySchema` | `object` | Yes | -- | Entity schema object containing `TableName`, `Columns` (array), and `MeadowSchema`. |
22
+ | `ConnectionPool` | `object` | No | -- | Database connection pool (used for table/index creation). |
23
+ | `PageSize` | `number` | No | `100` | Number of records per paginated download request. |
24
+
25
+ **Validation:** Same as `MeadowSyncEntityInitial` -- throws errors for missing or invalid schema properties.
26
+
27
+ ## Properties
28
+
29
+ ### `EntitySchema`
30
+
31
+ `object` -- Deep copy of the provided `MeadowEntitySchema` option.
32
+
33
+ ### `DefaultIdentifier`
34
+
35
+ `string` -- The default identifier column name from `EntitySchema.MeadowSchema.DefaultIdentifier`.
36
+
37
+ ### `PageSize`
38
+
39
+ `number` -- Records per download page.
40
+
41
+ ### `Meadow`
42
+
43
+ The Meadow ORM instance for this entity. Set by `initialize()`.
44
+
45
+ ### `operation`
46
+
47
+ `MeadowOperation` -- Instance of the operation utility for timestamps and progress tracking.
48
+
49
+ ## Methods
50
+
51
+ ### `initialize(fCallback)`
52
+
53
+ Prepares the entity for synchronization. Identical behavior to `MeadowSyncEntityInitial.initialize()`: creates the local table if needed and sets up GUID (unique) and Deleted (non-unique) indexes.
54
+
55
+ | Parameter | Type | Description |
56
+ |-----------|------|-------------|
57
+ | `fCallback` | `function(pError)` | Callback invoked when initialization is complete. |
58
+
59
+ ### `marshalRecord(pSourceRecord)`
60
+
61
+ Transforms a server-side record into a local record suitable for insertion or update.
62
+
63
+ | Parameter | Type | Description |
64
+ |-----------|------|-------------|
65
+ | `pSourceRecord` | `object` | The record object from the server. |
66
+
67
+ **Returns:** `object` -- The marshaled record.
68
+
69
+ **Differences from Initial sync marshaling:**
70
+ - Does **not** perform UTC DateTime formatting. Values are copied as-is (unlike Initial sync which formats DateTime columns to `'YYYY-MM-DD HH:mm:ss.SSS'`).
71
+ - Otherwise follows the same column-mapping logic: skips null/undefined, stringifies objects, handles `*JSON` columns.
72
+
73
+ ### `addSyncAnticipateEntry(tmpSyncState, tmpAnticipate)`
74
+
75
+ Adds a recursive anticipate entry for paginated sync. This is an internal method that implements the core download-and-sync loop for ongoing sync.
76
+
77
+ | Parameter | Type | Description |
78
+ |-----------|------|-------------|
79
+ | `tmpSyncState` | `object` | Mutable sync state tracking `LastRequestedID`, `RequestsPerformed`, `EstimatedRequestCount`. |
80
+ | `tmpAnticipate` | `object` | Fable Anticipate instance for managing asynchronous operations. |
81
+
82
+ **Recursive pagination pattern:**
83
+ 1. Downloads a page of records filtered to `ID > LastRequestedID`, sorted ascending.
84
+ 2. For each record in the page, adds an anticipate entry that:
85
+ - Updates `LastRequestedID` to track progress.
86
+ - Reads the local record by ID.
87
+ - If found: compares `UpdateDate` between server and local. Skips if difference is less than 5ms.
88
+ - If the record exists and dates differ: marshals and updates via `Meadow.doUpdate()`.
89
+ - If the record does not exist: marshals and creates via `Meadow.doCreate()` with identity insert enabled.
90
+ 3. After processing the page, if `RequestsPerformed < EstimatedRequestCount`, recursively calls `addSyncAnticipateEntry` to fetch the next page.
91
+
92
+ ### `sync(fCallback)`
93
+
94
+ Executes the full ongoing sync algorithm for this entity.
95
+
96
+ | Parameter | Type | Description |
97
+ |-----------|------|-------------|
98
+ | `fCallback` | `function()` | Callback invoked when sync completes (errors are logged, not passed). |
99
+
100
+ **Algorithm:**
101
+
102
+ 1. **Detect UpdateDate column** -- Checks if the entity schema has an `UpdateDate` column.
103
+ 2. **Get local max ID** -- Reads the highest-ID record from the local database.
104
+ 3. **Get local max UpdateDate** -- Reads the record with the most recent `UpdateDate`.
105
+ 4. **Get local count** -- Counts all local records.
106
+ 5. **Get server max ID** -- Requests from `{Entity}/Max/{DefaultIdentifier}`.
107
+ 6. **Get server max UpdateDate** -- Requests from `{Entity}/Max/UpdateDate`.
108
+ 7. **Get server count** -- Requests from `{Entity}s/Count`.
109
+ 8. **Calculate estimated requests** -- `ceil(Server.RecordCount / PageSize)`.
110
+ 9. **Begin recursive sync** -- Calls `addSyncAnticipateEntry()` to start the paginated download-and-sync loop. All records on the server are scanned (not just those newer than the local max).
111
+
112
+ **Progress tracking:** Uses `MeadowOperation` with tracker hash `UpdateSync-{TableName}`.
113
+
114
+ ## Differences from Initial Sync
115
+
116
+ | Aspect | Initial | Ongoing |
117
+ |--------|---------|---------|
118
+ | **Strategy** | ID-based gap fill | Full scan with UpdateDate comparison |
119
+ | **Filter** | Only records with `ID > LocalMaxID` | All records, paginated by ascending ID |
120
+ | **Existing records** | Skips (already present) | Compares `UpdateDate`, updates if changed |
121
+ | **DateTime marshaling** | Formats to `YYYY-MM-DD HH:mm:ss.SSS` UTC | Copies values as-is |
122
+ | **Pagination** | Pre-computed URL list | Recursive anticipate pattern |
123
+ | **UpdateDate threshold** | N/A | Skips updates if difference < 5ms |
124
+ | **Progress tracker hash** | `FullSync-{TableName}` | `UpdateSync-{TableName}` |
125
+
126
+ ## Usage Examples
127
+
128
+ ### Direct Instantiation and Ongoing Sync
129
+
130
+ ```js
131
+ const libFable = require('fable');
132
+ const libSyncEntityOngoing = require('meadow-integration/source/services/clone/Meadow-Service-Sync-Entity-Ongoing');
133
+
134
+ const fable = new libFable({ Product: 'CloneApp' });
135
+
136
+ fable.serviceManager.addServiceType('MeadowSyncEntityOngoing', libSyncEntityOngoing);
137
+
138
+ const entitySchema = {
139
+ TableName: 'Order',
140
+ Columns:
141
+ [
142
+ { Column: 'IDOrder', DataType: 'AutoIdentity' },
143
+ { Column: 'GUIDOrder', DataType: 'GUID' },
144
+ { Column: 'CustomerID', DataType: 'Integer' },
145
+ { Column: 'Total', DataType: 'Decimal' },
146
+ { Column: 'CreateDate', DataType: 'DateTime' },
147
+ { Column: 'UpdateDate', DataType: 'DateTime' },
148
+ { Column: 'Deleted', DataType: 'Boolean' }
149
+ ],
150
+ MeadowSchema:
151
+ {
152
+ DefaultIdentifier: 'IDOrder',
153
+ Schema:
154
+ [
155
+ { Column: 'IDOrder' },
156
+ { Column: 'GUIDOrder' },
157
+ { Column: 'CustomerID' },
158
+ { Column: 'Total' },
159
+ { Column: 'CreateDate' },
160
+ { Column: 'UpdateDate' },
161
+ { Column: 'Deleted' }
162
+ ]
163
+ }
164
+ };
165
+
166
+ const syncEntity = fable.serviceManager.instantiateServiceProvider('MeadowSyncEntityOngoing',
167
+ {
168
+ MeadowEntitySchema: entitySchema,
169
+ ConnectionPool: connectionPool,
170
+ PageSize: 250
171
+ },
172
+ 'SyncEntity-Order');
173
+
174
+ syncEntity.initialize(
175
+ (pError) =>
176
+ {
177
+ if (pError)
178
+ {
179
+ console.error('Initialization failed:', pError);
180
+ return;
181
+ }
182
+
183
+ syncEntity.sync(
184
+ () =>
185
+ {
186
+ console.log('Ongoing sync of Order complete.');
187
+ });
188
+ });
189
+ ```
190
+
191
+ ### Using via MeadowSync Orchestrator
192
+
193
+ ```js
194
+ const sync = fable.serviceManager.instantiateServiceProvider('MeadowSync',
195
+ {
196
+ ConnectionPool: connectionPool,
197
+ PageSize: 200,
198
+ SyncEntityList: ['Customer', 'Order', 'Product']
199
+ });
200
+
201
+ // Set ongoing mode before loading schema
202
+ sync.SyncMode = 'Ongoing';
203
+
204
+ sync.loadMeadowSchema(compiledSchema,
205
+ (pLoadError) =>
206
+ {
207
+ sync.syncAll(
208
+ (pSyncError) =>
209
+ {
210
+ console.log('Ongoing sync of all entities complete.');
211
+ });
212
+ });
213
+ ```
214
+
215
+ ## Sync Flow Diagram
216
+
217
+ ```
218
+ Local DB Remote Server
219
+ -------- -------------
220
+ 1. Check for UpdateDate column
221
+ 2. Get max ID (local)
222
+ 3. Get max UpdateDate (local)
223
+ 4. Get count (local)
224
+ 5. GET /Entity/Max/IDEntity
225
+ 6. GET /Entity/Max/UpdateDate
226
+ 7. GET /Entitys/Count
227
+ 8. Calculate estimated request count
228
+ 9. GET page (ID > LastRequestedID, ascending)
229
+ 10. For each record:
230
+ - Read by ID locally
231
+ - If found & UpdateDate diff >= 5ms: marshal + update
232
+ - If not found: marshal + create
233
+ - Update LastRequestedID
234
+ 11. Recursively fetch next page...
235
+ 12. Repeat until all pages processed
236
+ ```
237
+
238
+ ## Related Services
239
+
240
+ - [MeadowSync](./sync.md) -- The orchestrator that creates and manages instances of this class.
241
+ - [MeadowSyncEntityInitial](./sync-entity-initial.md) -- The initial full-sync variant.
242
+ - [MeadowCloneRestClient](./clone-rest-client.md) -- Used via `fable.MeadowCloneRestClient` to download records.
243
+ - [MeadowOperation](./operation.md) -- Provides timestamp and progress tracking utilities.
244
+ - [MeadowConnectionManager](./connection-manager.md) -- Provides the connection pool and index creation.
@@ -0,0 +1,213 @@
1
+ # MeadowSync
2
+
3
+ Top-level synchronization orchestrator for the data-clone pipeline. Loads a Meadow schema, creates per-entity sync objects (Initial or Ongoing), and coordinates syncing all entities in sequence.
4
+
5
+ **Source:** `source/services/clone/Meadow-Service-Sync.js`
6
+
7
+ **Extends:** `fable-serviceproviderbase`
8
+
9
+ **Service Type:** `MeadowSync`
10
+
11
+ ## Constructor
12
+
13
+ ```js
14
+ const sync = fable.serviceManager.instantiateServiceProvider('MeadowSync', pOptions);
15
+ ```
16
+
17
+ ### Options
18
+
19
+ | Option | Type | Default | Description |
20
+ |--------|------|---------|-------------|
21
+ | `SyncEntityList` | `Array<string>` | `[]` | Ordered list of entity table names to sync. If empty, all entities in the schema are synced. Also read from `fable.ProgramConfiguration.SyncEntityList`. |
22
+ | `SyncEntityOptions` | `object` | `{}` | Per-entity sync options keyed by table name. Also read from `fable.ProgramConfiguration.SyncEntityOptions`. |
23
+ | `ConnectionPool` | `object` | *(none)* | Database connection pool passed through to per-entity sync objects. |
24
+ | `PageSize` | `number` | `100` | Number of records per download page, passed through to per-entity sync objects. |
25
+
26
+ ## Properties
27
+
28
+ ### `SyncMode`
29
+
30
+ `string` -- Controls which sync strategy is used when `loadMeadowSchema()` creates entity sync objects.
31
+
32
+ | Value | Behavior |
33
+ |-------|----------|
34
+ | `'Initial'` *(default)* | Creates `MeadowSyncEntityInitial` instances. Performs a full ID-based sync of all missing records. |
35
+ | `'Ongoing'` | Creates `MeadowSyncEntityOngoing` instances. Performs UpdateDate-based differential sync, creating or updating records. |
36
+
37
+ Set this property **before** calling `loadMeadowSchema()`.
38
+
39
+ ### `SyncEntityList`
40
+
41
+ `Array<string>` -- The list of entity table names to synchronize, in order. Populated from options, `ProgramConfiguration`, or automatically from the loaded schema.
42
+
43
+ ### `SyncEntityOptions`
44
+
45
+ `object` -- Per-entity options map. Keys are table names.
46
+
47
+ ### `MeadowSyncEntities`
48
+
49
+ `object` -- Map of table name to instantiated `MeadowSyncEntityInitial` or `MeadowSyncEntityOngoing` service instances. Populated by `loadMeadowSchema()`.
50
+
51
+ ### `MeadowSchemaTableList`
52
+
53
+ `Array<string>` -- List of all table names found in the loaded Meadow schema. Set by `loadMeadowSchema()`.
54
+
55
+ ## Methods
56
+
57
+ ### `loadMeadowSchema(pSchema, fCallback)`
58
+
59
+ Loads a compiled Meadow schema and creates per-entity sync objects.
60
+
61
+ | Parameter | Type | Description |
62
+ |-----------|------|-------------|
63
+ | `pSchema` | `object` | A compiled Meadow schema with a `Tables` property. Each key is a table name; each value is an entity schema object with `TableName`, `Columns`, and `MeadowSchema`. |
64
+ | `fCallback` | `function(pError)` | Callback invoked after all entity sync objects are initialized (tables created, indexes set up). |
65
+
66
+ **Behavior:**
67
+ 1. Iterates through every table in the schema.
68
+ 2. For tables that are in `SyncEntityList` (or all tables if the list is empty), creates a sync entity object according to `SyncMode`.
69
+ 3. Calls `initialize()` on each entity sync object to ensure the local table and indexes exist.
70
+ 4. If `SyncEntityList` was empty, populates it with all initialized entity names.
71
+
72
+ ### `syncEntity(pEntityHash, fCallback)`
73
+
74
+ Syncs a single entity by its table name.
75
+
76
+ | Parameter | Type | Description |
77
+ |-----------|------|-------------|
78
+ | `pEntityHash` | `string` | The table name of the entity to sync (e.g. `'Animal'`). |
79
+ | `fCallback` | `function(pError)` | Callback invoked when the entity sync completes. |
80
+
81
+ Logs a warning and returns immediately if the entity does not exist in `MeadowSyncEntities`.
82
+
83
+ ### `syncAll(fCallback)`
84
+
85
+ Syncs all entities in `SyncEntityList` sequentially.
86
+
87
+ | Parameter | Type | Description |
88
+ |-----------|------|-------------|
89
+ | `fCallback` | `function(pError)` | Callback invoked when all entities have been synced. |
90
+
91
+ Iterates through `SyncEntityList` with a concurrency of 1, calling `syncEntity()` for each.
92
+
93
+ ## Usage Examples
94
+
95
+ ### Full Initial Sync Workflow
96
+
97
+ ```js
98
+ const libFable = require('fable');
99
+ const libConnectionManager = require('meadow-integration/source/services/clone/Meadow-Service-ConnectionManager');
100
+ const libRestClient = require('meadow-integration/source/services/clone/Meadow-Service-RestClient');
101
+ const libSync = require('meadow-integration/source/services/clone/Meadow-Service-Sync');
102
+
103
+ const fable = new libFable({ Product: 'DataClone' });
104
+
105
+ // Register services
106
+ fable.serviceManager.addServiceType('MeadowConnectionManager', libConnectionManager);
107
+ fable.serviceManager.addServiceType('MeadowCloneRestClient', libRestClient);
108
+ fable.serviceManager.addServiceType('MeadowSync', libSync);
109
+
110
+ // Instantiate connection manager
111
+ const connectionManager = fable.serviceManager.instantiateServiceProvider('MeadowConnectionManager',
112
+ {
113
+ Provider: 'MySQL',
114
+ MySQL: { server: '127.0.0.1', user: 'root', password: '', database: 'clone_db' }
115
+ });
116
+
117
+ // Instantiate REST client
118
+ const restClient = fable.serviceManager.instantiateServiceProvider('MeadowCloneRestClient',
119
+ {
120
+ ServerURL: 'https://api.example.com/1.0/',
121
+ UserID: 'sync_user',
122
+ Password: 'sync_password'
123
+ });
124
+
125
+ // Connect, authenticate, then sync
126
+ connectionManager.connect(
127
+ (pConnError) =>
128
+ {
129
+ if (pConnError) { throw pConnError; }
130
+
131
+ restClient.authenticate(
132
+ (pAuthError) =>
133
+ {
134
+ if (pAuthError) { throw pAuthError; }
135
+
136
+ const sync = fable.serviceManager.instantiateServiceProvider('MeadowSync',
137
+ {
138
+ ConnectionPool: connectionManager.ConnectionPool,
139
+ PageSize: 200,
140
+ SyncEntityList: ['Customer', 'Order', 'Product']
141
+ });
142
+
143
+ // Use initial sync mode (the default)
144
+ sync.SyncMode = 'Initial';
145
+
146
+ const meadowSchema = require('./my-compiled-schema.json');
147
+ sync.loadMeadowSchema(meadowSchema,
148
+ (pLoadError) =>
149
+ {
150
+ if (pLoadError) { throw pLoadError; }
151
+
152
+ sync.syncAll(
153
+ (pSyncError) =>
154
+ {
155
+ if (pSyncError)
156
+ {
157
+ console.error('Sync failed:', pSyncError);
158
+ }
159
+ else
160
+ {
161
+ console.log('Initial sync complete!');
162
+ }
163
+
164
+ restClient.deauthenticate(() => { process.exit(0); });
165
+ });
166
+ });
167
+ });
168
+ });
169
+ ```
170
+
171
+ ### Ongoing (Differential) Sync
172
+
173
+ ```js
174
+ const sync = fable.serviceManager.instantiateServiceProvider('MeadowSync',
175
+ {
176
+ ConnectionPool: connectionManager.ConnectionPool,
177
+ PageSize: 100
178
+ });
179
+
180
+ // Switch to ongoing mode before loading schema
181
+ sync.SyncMode = 'Ongoing';
182
+
183
+ sync.loadMeadowSchema(meadowSchema,
184
+ (pLoadError) =>
185
+ {
186
+ sync.syncAll(
187
+ (pSyncError) =>
188
+ {
189
+ console.log('Ongoing sync complete!');
190
+ });
191
+ });
192
+ ```
193
+
194
+ ### Syncing a Single Entity
195
+
196
+ ```js
197
+ sync.loadMeadowSchema(meadowSchema,
198
+ (pLoadError) =>
199
+ {
200
+ sync.syncEntity('Customer',
201
+ (pSyncError) =>
202
+ {
203
+ console.log('Customer sync complete!');
204
+ });
205
+ });
206
+ ```
207
+
208
+ ## Related Services
209
+
210
+ - [MeadowConnectionManager](./connection-manager.md) -- Provides the database connection pool.
211
+ - [MeadowCloneRestClient](./clone-rest-client.md) -- REST client used by entity sync objects to download records.
212
+ - [MeadowSyncEntityInitial](./sync-entity-initial.md) -- Created by MeadowSync when `SyncMode` is `'Initial'`.
213
+ - [MeadowSyncEntityOngoing](./sync-entity-ongoing.md) -- Created by MeadowSync when `SyncMode` is `'Ongoing'`.