meadow-integration 1.0.20 → 1.0.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/example-applications/mapping-demo/.quackage.json +10 -0
  2. package/example-applications/mapping-demo/README.md +99 -0
  3. package/example-applications/mapping-demo/data/books-sample.csv +21 -0
  4. package/example-applications/mapping-demo/generate-build-config.js +44 -0
  5. package/example-applications/mapping-demo/mappings/books-to-book.json +14 -0
  6. package/example-applications/mapping-demo/package.json +14 -0
  7. package/example-applications/mapping-demo/server.js +814 -0
  8. package/example-applications/mapping-demo/source/MappingDemoApp.js +52 -0
  9. package/example-applications/mapping-demo/source/views/MappingDemoEditorView.js +186 -0
  10. package/example-applications/mapping-demo/web/index.html +892 -0
  11. package/example-applications/mapping-demo/web/mapping-demo-editor.js +3195 -0
  12. package/example-applications/mapping-demo/web/mapping-demo-editor.js.map +1 -0
  13. package/example-applications/mapping-demo/web/mapping-demo-editor.min.js +2 -0
  14. package/example-applications/mapping-demo/web/mapping-demo-editor.min.js.map +1 -0
  15. package/example-applications/mapping-demo/web/pict.min.js +12 -0
  16. package/package.json +8 -4
  17. package/source/Meadow-Integration-Browser.js +31 -0
  18. package/source/Meadow-Integration.js +16 -1
  19. package/source/services/certainty/Service-CertaintyAccumulator.js +402 -0
  20. package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js +16 -3
  21. package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js +15 -2
  22. package/source/services/clone/Meadow-Service-Sync.js +21 -0
  23. package/source/views/MappingEditor-SchemaUtils.js +71 -0
  24. package/source/views/PictView-MeadowMappingEditor.js +1299 -0
  25. package/source/views/flow-cards/FlowCard-MappingSource.js +50 -0
  26. package/source/views/flow-cards/FlowCard-MappingTarget.js +49 -0
  27. package/source/views/flow-cards/FlowCard-SolverExpression.js +78 -0
  28. package/source/views/flow-cards/FlowCard-TemplateExpression.js +77 -0
  29. package/test/Meadow-Integration-BisectionSync_test.js +1601 -0
  30. package/test/Meadow-Integration-CloneDeleteSync_test.js +809 -0
package/package.json CHANGED
@@ -1,11 +1,12 @@
1
1
  {
2
2
  "name": "meadow-integration",
3
- "version": "1.0.20",
3
+ "version": "1.0.23",
4
4
  "description": "Meadow Data Integration",
5
5
  "bin": {
6
6
  "mdwint": "source/cli/Meadow-Integration-CLI-Run.js"
7
7
  },
8
8
  "main": "source/Meadow-Integration.js",
9
+ "browser": "source/Meadow-Integration-Browser.js",
9
10
  "scripts": {
10
11
  "test": "npx quack test",
11
12
  "start": "node source/cli/Meadow-Integration-CLI-Run.js",
@@ -16,7 +17,8 @@
16
17
  "author": "steven velozo <steven@velozo.com>",
17
18
  "license": "MIT",
18
19
  "devDependencies": {
19
- "quackage": "^1.0.64"
20
+ "meadow-connection-sqlite": "^1.0.18",
21
+ "quackage": "^1.0.65"
20
22
  },
21
23
  "mocha": {
22
24
  "diff": true,
@@ -37,16 +39,18 @@
37
39
  ]
38
40
  },
39
41
  "dependencies": {
40
- "fable": "^3.1.63",
42
+ "fable": "^3.1.67",
41
43
  "fable-serviceproviderbase": "^3.0.19",
42
44
  "fast-xml-parser": "^4.4.1",
43
45
  "meadow": "^2.0.33",
44
46
  "meadow-connection-mssql": "^1.0.16",
45
47
  "meadow-connection-mysql": "^1.0.14",
46
48
  "orator": "^6.0.4",
47
- "orator-serviceserver-restify": "^2.0.9",
49
+ "orator-serviceserver-restify": "^2.0.10",
50
+ "pict-section-flow": "^0.0.17",
48
51
  "pict-service-commandlineutility": "^1.0.19",
49
52
  "pict-sessionmanager": "^1.0.2",
53
+ "pict-view": "^1.0.67",
50
54
  "xlsx": "^0.18.5"
51
55
  }
52
56
  }
@@ -0,0 +1,31 @@
1
+ /**
2
+ * Meadow-Integration Browser Entry Point
3
+ *
4
+ * Browser-safe exports only — no database drivers, no server-side
5
+ * connection managers, no file system access.
6
+ *
7
+ * Use this entry point in browser bundles:
8
+ * const libMI = require('meadow-integration/source/Meadow-Integration-Browser.js');
9
+ *
10
+ * Or add to package.json "browser" field for automatic resolution.
11
+ */
12
+
13
+ 'use strict';
14
+
15
+ const libMeadowMappingEditorView = require(`./views/PictView-MeadowMappingEditor.js`);
16
+ const libMappingEditorSchemaUtils = require(`./views/MappingEditor-SchemaUtils.js`);
17
+ const libFlowCardMappingSource = require(`./views/flow-cards/FlowCard-MappingSource.js`);
18
+ const libFlowCardMappingTarget = require(`./views/flow-cards/FlowCard-MappingTarget.js`);
19
+ const libFlowCardTemplateExpression = require(`./views/flow-cards/FlowCard-TemplateExpression.js`);
20
+ const libFlowCardSolverExpression = require(`./views/flow-cards/FlowCard-SolverExpression.js`);
21
+
22
+ module.exports =
23
+ {
24
+ // Visual mapping editor components
25
+ MeadowMappingEditorView: libMeadowMappingEditorView,
26
+ MappingEditorSchemaUtils: libMappingEditorSchemaUtils,
27
+ FlowCardMappingSource: libFlowCardMappingSource,
28
+ FlowCardMappingTarget: libFlowCardMappingTarget,
29
+ FlowCardTemplateExpression: libFlowCardTemplateExpression,
30
+ FlowCardSolverExpression: libFlowCardSolverExpression,
31
+ };
@@ -18,6 +18,13 @@ const libFileParserXLSX = require(`./services/parser/Service-FileParser-XLSX.js`
18
18
  const libFileParserXML = require(`./services/parser/Service-FileParser-XML.js`);
19
19
  const libFileParserFixedWidth = require(`./services/parser/Service-FileParser-FixedWidth.js`);
20
20
 
21
+ const libMeadowMappingEditorView = require(`./views/PictView-MeadowMappingEditor.js`);
22
+ const libMappingEditorSchemaUtils = require(`./views/MappingEditor-SchemaUtils.js`);
23
+ const libFlowCardMappingSource = require(`./views/flow-cards/FlowCard-MappingSource.js`);
24
+ const libFlowCardMappingTarget = require(`./views/flow-cards/FlowCard-MappingTarget.js`);
25
+ const libFlowCardTemplateExpression = require(`./views/flow-cards/FlowCard-TemplateExpression.js`);
26
+ const libFlowCardSolverExpression = require(`./views/flow-cards/FlowCard-SolverExpression.js`);
27
+
21
28
  module.exports = (
22
29
  {
23
30
  TabularCheck: libTabularCheck,
@@ -38,5 +45,13 @@ module.exports = (
38
45
  FileParserJSON: libFileParserJSON,
39
46
  FileParserXLSX: libFileParserXLSX,
40
47
  FileParserXML: libFileParserXML,
41
- FileParserFixedWidth: libFileParserFixedWidth
48
+ FileParserFixedWidth: libFileParserFixedWidth,
49
+
50
+ // Visual mapping editor components (for browser bundles)
51
+ MeadowMappingEditorView: libMeadowMappingEditorView,
52
+ MappingEditorSchemaUtils: libMappingEditorSchemaUtils,
53
+ FlowCardMappingSource: libFlowCardMappingSource,
54
+ FlowCardMappingTarget: libFlowCardMappingTarget,
55
+ FlowCardTemplateExpression: libFlowCardTemplateExpression,
56
+ FlowCardSolverExpression: libFlowCardSolverExpression
42
57
  });
@@ -0,0 +1,402 @@
1
+ /**
2
+ * Service-CertaintyAccumulator
3
+ *
4
+ * Evidence accumulation engine for data integration certainty scoring.
5
+ *
6
+ * Tracks two certainty dimensions per field per record (GUID):
7
+ *
8
+ * 1. Cross-dataset presence: 1 - ∏(1 - weight_i)
9
+ * Strong signal — independent datasets corroborating a value.
10
+ *
11
+ * 2. Within-dataset ratio: log(count + 1) / log(datasetSize + 1)
12
+ * Weak signal — repetition within a single dataset has diminishing returns.
13
+ *
14
+ * 3. Composite: (w_presence × presence) + (w_ratio × ratio)
15
+ * Configurable weights; default 70% presence, 30% ratio.
16
+ *
17
+ * Evidence is tracked at the FIELD level. Record-level certainty is
18
+ * derived on the fly as the mean of its field composites.
19
+ *
20
+ * Designed to plug into Facto's MultiSet projection pipeline at the
21
+ * merge decision point, replacing the simple linear confidence tracker.
22
+ *
23
+ * @module Service-CertaintyAccumulator
24
+ */
25
+
26
+ 'use strict';
27
+
28
+ const libFableServiceProviderBase = require('fable-serviceproviderbase');
29
+
30
+ class MeadowIntegrationCertaintyAccumulator extends libFableServiceProviderBase
31
+ {
32
+ constructor(pFable, pOptions, pServiceHash)
33
+ {
34
+ super(pFable, pOptions, pServiceHash);
35
+
36
+ this.serviceType = 'CertaintyAccumulator';
37
+ }
38
+
39
+ // ─────────────────────────────────────────────
40
+ // Context lifecycle
41
+ // ─────────────────────────────────────────────
42
+
43
+ /**
44
+ * Create a new accumulation context for a merge pipeline.
45
+ *
46
+ * @param {object} [pConfig] — optional overrides
47
+ * @param {number} [pConfig.PresenceWeight=0.7]
48
+ * @param {number} [pConfig.RatioWeight=0.3]
49
+ * @returns {object} AccumulationContext
50
+ */
51
+ newAccumulationContext(pConfig)
52
+ {
53
+ let tmpConfig = (typeof pConfig === 'object' && pConfig !== null) ? pConfig : {};
54
+
55
+ let tmpPresenceWeight = (typeof tmpConfig.PresenceWeight === 'number') ? tmpConfig.PresenceWeight : 0.7;
56
+ let tmpRatioWeight = (typeof tmpConfig.RatioWeight === 'number') ? tmpConfig.RatioWeight : 0.3;
57
+
58
+ // Normalize so they sum to 1.0
59
+ let tmpSum = tmpPresenceWeight + tmpRatioWeight;
60
+ if (tmpSum > 0 && tmpSum !== 1.0)
61
+ {
62
+ tmpPresenceWeight = tmpPresenceWeight / tmpSum;
63
+ tmpRatioWeight = tmpRatioWeight / tmpSum;
64
+ }
65
+
66
+ return {
67
+ Weights:
68
+ {
69
+ presence: tmpPresenceWeight,
70
+ ratio: tmpRatioWeight,
71
+ },
72
+
73
+ // Per-GUID, per-field evidence
74
+ // { [GUID]: { [fieldName]: { value, sources: [...] } } }
75
+ FieldEvidence: {},
76
+
77
+ // Per-dataset metadata
78
+ // { [stepLabel]: { weight, totalRecords, uniqueGUIDs } }
79
+ DatasetContributions: {},
80
+ };
81
+ }
82
+
83
+ // ─────────────────────────────────────────────
84
+ // Evidence accumulation
85
+ // ─────────────────────────────────────────────
86
+
87
+ /**
88
+ * Record evidence from a single dataset/step for a specific GUID.
89
+ *
90
+ * Call this once per GUID per step, after the merge strategy has
91
+ * decided the action but before updating the accumulated comprehension.
92
+ *
93
+ * @param {object} pContext — the AccumulationContext
94
+ * @param {string} pGUID — record GUID
95
+ * @param {object} pNewRecord — incoming record from this step
96
+ * @param {object|null} pExistingRecord — previously merged record (null if first seen)
97
+ * @param {string} pMergeAction — e.g. 'Created', 'Merged', 'Merged_Reinforced', 'Skipped_*'
98
+ * @param {object} pStepInfo — source metadata
99
+ * @param {number} pStepInfo.ReliabilityWeight — DatasetSource weight (0–1)
100
+ * @param {number} pStepInfo.DatasetSize — total records in this source dataset
101
+ * @param {number} [pStepInfo.RecordCountInDataset=1] — how many times this GUID appeared in the dataset
102
+ * @param {string} pStepInfo.StepLabel — human-readable source label
103
+ * @param {number} pStepInfo.StepOrdinal — sequential step number
104
+ */
105
+ accumulateEvidence(pContext, pGUID, pNewRecord, pExistingRecord, pMergeAction, pStepInfo)
106
+ {
107
+ if (!pContext || !pGUID || !pNewRecord)
108
+ {
109
+ return;
110
+ }
111
+
112
+ let tmpStepInfo = pStepInfo || {};
113
+ let tmpWeight = (typeof tmpStepInfo.ReliabilityWeight === 'number') ? tmpStepInfo.ReliabilityWeight : 0.5;
114
+ let tmpDatasetSize = tmpStepInfo.DatasetSize || 1;
115
+ let tmpCount = tmpStepInfo.RecordCountInDataset || 1;
116
+ let tmpStepLabel = tmpStepInfo.StepLabel || 'unknown';
117
+ let tmpStepOrdinal = tmpStepInfo.StepOrdinal || 0;
118
+
119
+ // Track dataset contributions
120
+ if (!pContext.DatasetContributions[tmpStepLabel])
121
+ {
122
+ pContext.DatasetContributions[tmpStepLabel] =
123
+ {
124
+ weight: tmpWeight,
125
+ totalRecords: tmpDatasetSize,
126
+ uniqueGUIDs: 0,
127
+ };
128
+ }
129
+ pContext.DatasetContributions[tmpStepLabel].uniqueGUIDs++;
130
+
131
+ // Initialize GUID evidence if needed
132
+ if (!pContext.FieldEvidence[pGUID])
133
+ {
134
+ pContext.FieldEvidence[pGUID] = {};
135
+ }
136
+ let tmpGUIDEvidence = pContext.FieldEvidence[pGUID];
137
+
138
+ // Record evidence for each field in the new record
139
+ let tmpFieldNames = Object.keys(pNewRecord);
140
+ for (let i = 0; i < tmpFieldNames.length; i++)
141
+ {
142
+ let tmpFieldName = tmpFieldNames[i];
143
+ let tmpValue = pNewRecord[tmpFieldName];
144
+
145
+ // Skip GUID fields and internal metadata
146
+ if (tmpFieldName.startsWith('GUID') || tmpFieldName.startsWith('_'))
147
+ {
148
+ continue;
149
+ }
150
+
151
+ if (!tmpGUIDEvidence[tmpFieldName])
152
+ {
153
+ tmpGUIDEvidence[tmpFieldName] =
154
+ {
155
+ value: tmpValue,
156
+ sources: [],
157
+ };
158
+ }
159
+
160
+ tmpGUIDEvidence[tmpFieldName].sources.push(
161
+ {
162
+ weight: tmpWeight,
163
+ count: tmpCount,
164
+ datasetSize: tmpDatasetSize,
165
+ stepLabel: tmpStepLabel,
166
+ stepOrdinal: tmpStepOrdinal,
167
+ value: tmpValue,
168
+ });
169
+ }
170
+ }
171
+
172
+ // ─────────────────────────────────────────────
173
+ // Certainty computation
174
+ // ─────────────────────────────────────────────
175
+
176
+ /**
177
+ * Compute field-level certainty scores for a single GUID.
178
+ *
179
+ * @param {object} pContext — the AccumulationContext
180
+ * @param {string} pGUID — record GUID
181
+ * @returns {object} — { fields: { fieldName: { presence, ratio, composite, agreeing, conflicting } }, recordComposite }
182
+ */
183
+ computeCertainty(pContext, pGUID)
184
+ {
185
+ if (!pContext || !pContext.FieldEvidence[pGUID])
186
+ {
187
+ return { fields: {}, recordComposite: 0.5 };
188
+ }
189
+
190
+ let tmpGUIDEvidence = pContext.FieldEvidence[pGUID];
191
+ let tmpFieldNames = Object.keys(tmpGUIDEvidence);
192
+ let tmpFields = {};
193
+ let tmpCompositeSum = 0;
194
+ let tmpFieldCount = 0;
195
+
196
+ for (let i = 0; i < tmpFieldNames.length; i++)
197
+ {
198
+ let tmpFieldName = tmpFieldNames[i];
199
+ let tmpFieldData = tmpGUIDEvidence[tmpFieldName];
200
+ let tmpSources = tmpFieldData.sources;
201
+ let tmpCurrentValue = tmpFieldData.value;
202
+
203
+ // ── Cross-dataset presence ──────────────────────────
204
+ // Only count sources where the value AGREES with the current value.
205
+ // Conflicting sources don't contribute to presence.
206
+ let tmpAgreeingSources = [];
207
+ let tmpConflictingSources = [];
208
+
209
+ for (let j = 0; j < tmpSources.length; j++)
210
+ {
211
+ if (this._valuesMatch(tmpSources[j].value, tmpCurrentValue))
212
+ {
213
+ tmpAgreeingSources.push(tmpSources[j]);
214
+ }
215
+ else
216
+ {
217
+ tmpConflictingSources.push(tmpSources[j]);
218
+ }
219
+ }
220
+
221
+ // presence = 1 - ∏(1 - weight_i) for agreeing sources
222
+ let tmpUncertainty = 1.0;
223
+ for (let j = 0; j < tmpAgreeingSources.length; j++)
224
+ {
225
+ tmpUncertainty *= (1.0 - tmpAgreeingSources[j].weight);
226
+ }
227
+ let tmpPresence = 1.0 - tmpUncertainty;
228
+
229
+ // Penalize if there are conflicting sources
230
+ if (tmpConflictingSources.length > 0)
231
+ {
232
+ let tmpConflictPenalty = tmpConflictingSources.length / (tmpAgreeingSources.length + tmpConflictingSources.length);
233
+ tmpPresence *= (1.0 - (tmpConflictPenalty * 0.5));
234
+ }
235
+
236
+ // ── Within-dataset ratio (best source) ──────────────
237
+ // Use the strongest ratio from any agreeing source
238
+ let tmpBestRatio = 0;
239
+ for (let j = 0; j < tmpAgreeingSources.length; j++)
240
+ {
241
+ let tmpSrc = tmpAgreeingSources[j];
242
+ let tmpRatio = Math.log(tmpSrc.count + 1) / Math.log(tmpSrc.datasetSize + 1);
243
+ if (tmpRatio > tmpBestRatio)
244
+ {
245
+ tmpBestRatio = tmpRatio;
246
+ }
247
+ }
248
+
249
+ // ── Composite ───────────────────────────────────────
250
+ let tmpComposite = (pContext.Weights.presence * tmpPresence) +
251
+ (pContext.Weights.ratio * tmpBestRatio);
252
+
253
+ // Clamp to [0, 1]
254
+ tmpComposite = Math.max(0, Math.min(1, tmpComposite));
255
+
256
+ tmpFields[tmpFieldName] =
257
+ {
258
+ presence: Math.round(tmpPresence * 10000) / 10000,
259
+ ratio: Math.round(tmpBestRatio * 10000) / 10000,
260
+ composite: Math.round(tmpComposite * 10000) / 10000,
261
+ agreeing: tmpAgreeingSources.length,
262
+ conflicting: tmpConflictingSources.length,
263
+ };
264
+
265
+ tmpCompositeSum += tmpComposite;
266
+ tmpFieldCount++;
267
+ }
268
+
269
+ let tmpRecordComposite = tmpFieldCount > 0
270
+ ? Math.round((tmpCompositeSum / tmpFieldCount) * 10000) / 10000
271
+ : 0.5;
272
+
273
+ return {
274
+ fields: tmpFields,
275
+ recordComposite: tmpRecordComposite,
276
+ };
277
+ }
278
+
279
+ /**
280
+ * Bulk compute certainty for all GUIDs in the context.
281
+ *
282
+ * @param {object} pContext — the AccumulationContext
283
+ * @returns {object} — { [GUID]: CertaintyResult }
284
+ */
285
+ computeAllCertainties(pContext)
286
+ {
287
+ if (!pContext || !pContext.FieldEvidence)
288
+ {
289
+ return {};
290
+ }
291
+
292
+ let tmpResults = {};
293
+ let tmpGUIDs = Object.keys(pContext.FieldEvidence);
294
+
295
+ for (let i = 0; i < tmpGUIDs.length; i++)
296
+ {
297
+ tmpResults[tmpGUIDs[i]] = this.computeCertainty(pContext, tmpGUIDs[i]);
298
+ }
299
+
300
+ return tmpResults;
301
+ }
302
+
303
+ // ─────────────────────────────────────────────
304
+ // CertaintyIndex entry generation
305
+ // ─────────────────────────────────────────────
306
+
307
+ /**
308
+ * Generate CertaintyIndex entries for a single GUID.
309
+ * Creates one entry per field per dimension (presence, ratio, composite).
310
+ *
311
+ * @param {object} pContext — the AccumulationContext
312
+ * @param {string} pGUID — record GUID
313
+ * @param {number} pIDRecord — Facto Record ID to link entries to
314
+ * @returns {Array} — array of { IDRecord, CertaintyValue, Dimension, Justification }
315
+ */
316
+ generateCertaintyEntries(pContext, pGUID, pIDRecord)
317
+ {
318
+ let tmpCertainty = this.computeCertainty(pContext, pGUID);
319
+ let tmpEntries = [];
320
+
321
+ let tmpFieldNames = Object.keys(tmpCertainty.fields);
322
+ for (let i = 0; i < tmpFieldNames.length; i++)
323
+ {
324
+ let tmpFieldName = tmpFieldNames[i];
325
+ let tmpField = tmpCertainty.fields[tmpFieldName];
326
+
327
+ let tmpJustification = JSON.stringify(
328
+ {
329
+ agreeing: tmpField.agreeing,
330
+ conflicting: tmpField.conflicting,
331
+ weights: pContext.Weights,
332
+ });
333
+
334
+ tmpEntries.push(
335
+ {
336
+ IDRecord: pIDRecord,
337
+ CertaintyValue: tmpField.presence,
338
+ Dimension: `field:${tmpFieldName}:presence`,
339
+ Justification: tmpJustification,
340
+ });
341
+
342
+ tmpEntries.push(
343
+ {
344
+ IDRecord: pIDRecord,
345
+ CertaintyValue: tmpField.ratio,
346
+ Dimension: `field:${tmpFieldName}:ratio`,
347
+ Justification: tmpJustification,
348
+ });
349
+
350
+ tmpEntries.push(
351
+ {
352
+ IDRecord: pIDRecord,
353
+ CertaintyValue: tmpField.composite,
354
+ Dimension: `field:${tmpFieldName}:composite`,
355
+ Justification: tmpJustification,
356
+ });
357
+ }
358
+
359
+ return tmpEntries;
360
+ }
361
+
362
+ // ─────────────────────────────────────────────
363
+ // Internal helpers
364
+ // ─────────────────────────────────────────────
365
+
366
+ /**
367
+ * Compare two values for equality.
368
+ * Handles strings, numbers, nulls, and undefined.
369
+ * String comparisons are case-insensitive and trimmed.
370
+ */
371
+ _valuesMatch(pValueA, pValueB)
372
+ {
373
+ // Both null/undefined
374
+ if (pValueA == null && pValueB == null)
375
+ {
376
+ return true;
377
+ }
378
+
379
+ // One null, other not
380
+ if (pValueA == null || pValueB == null)
381
+ {
382
+ return false;
383
+ }
384
+
385
+ // Both strings — case-insensitive, trimmed
386
+ if (typeof pValueA === 'string' && typeof pValueB === 'string')
387
+ {
388
+ return pValueA.trim().toLowerCase() === pValueB.trim().toLowerCase();
389
+ }
390
+
391
+ // Numeric comparison with tolerance
392
+ if (typeof pValueA === 'number' && typeof pValueB === 'number')
393
+ {
394
+ return Math.abs(pValueA - pValueB) < 0.0001;
395
+ }
396
+
397
+ // Fall back to strict equality
398
+ return pValueA === pValueB;
399
+ }
400
+ }
401
+
402
+ module.exports = MeadowIntegrationCertaintyAccumulator;
@@ -43,6 +43,11 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
43
43
  this.MaxRecordsPerEntity = this.options.MaxRecordsPerEntity || 0;
44
44
  this.UseAdvancedIDPagination = this.options.UseAdvancedIDPagination || false;
45
45
 
46
+ // Optional query string appended to deleted record API requests.
47
+ // Used to work around older APIs where FBV~Deleted~EQ~1 does not
48
+ // override the automatic Deleted=0 filter (e.g. "includeDeleted=true").
49
+ this.SyncDeletedRecordsQueryString = this.options.SyncDeletedRecordsQueryString || '';
50
+
46
51
  this.Meadow = false;
47
52
 
48
53
  this.operation = new libMeadowOperation(this.fable);
@@ -119,7 +124,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
119
124
  {
120
125
  this.log.warn(`${this.EntitySchema.TableName}: Index creation error: ${pIndexError}`);
121
126
  }
122
- return fValidateAndCallback(pIndexError || pCreateError);
127
+ return fCallback(pIndexError || pCreateError);
123
128
  });
124
129
  });
125
130
  }
@@ -170,6 +175,12 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
170
175
  return tmpRecordToCommit;
171
176
  }
172
177
 
178
+ _appendDeletedQueryString(pURL)
179
+ {
180
+ if (!this.SyncDeletedRecordsQueryString) return pURL;
181
+ return pURL + (pURL.indexOf('?') > -1 ? '&' : '?') + this.SyncDeletedRecordsQueryString;
182
+ }
183
+
173
184
  syncDeletedRecords(fCallback)
174
185
  {
175
186
  const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
@@ -183,7 +194,9 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
183
194
 
184
195
  // Get the count of deleted records from the server.
185
196
  // The explicit FBV~Deleted~EQ~1 filter overrides foxhound's automatic Deleted=0 filter.
186
- this.fable.MeadowCloneRestClient.getJSON(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`,
197
+ // On older APIs where this doesn't work, SyncDeletedRecordsQueryString provides a
198
+ // fallback (e.g. "includeDeleted=true") that disables server-side delete tracking.
199
+ this.fable.MeadowCloneRestClient.getJSON(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`),
187
200
  (pError, pResponse, pBody) =>
188
201
  {
189
202
  if (pError || !pBody || !pBody.hasOwnProperty('Count'))
@@ -208,7 +221,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
208
221
  const tmpDeleteURLPartials = [];
209
222
  for (let i = 0; i < tmpDeleteCap; i += this.PageSize)
210
223
  {
211
- tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
224
+ tmpDeleteURLPartials.push(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`));
212
225
  }
213
226
 
214
227
  this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
@@ -42,6 +42,11 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
42
42
  this.SyncDeletedRecords = this.options.SyncDeletedRecords || false;
43
43
  this.MaxRecordsPerEntity = this.options.MaxRecordsPerEntity || 0;
44
44
 
45
+ // Optional query string appended to deleted record API requests.
46
+ // Used to work around older APIs where FBV~Deleted~EQ~1 does not
47
+ // override the automatic Deleted=0 filter (e.g. "includeDeleted=true").
48
+ this.SyncDeletedRecordsQueryString = this.options.SyncDeletedRecordsQueryString || '';
49
+
45
50
  // Minimum range size for bisection -- when a range is this small or smaller,
46
51
  // pull all records in the range from the server instead of subdividing further.
47
52
  this.BisectMinRangeSize = this.options.BisectMinRangeSize || 1000;
@@ -645,6 +650,12 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
645
650
 
646
651
  // ---- Deleted records sync ----
647
652
 
653
+ _appendDeletedQueryString(pURL)
654
+ {
655
+ if (!this.SyncDeletedRecordsQueryString) return pURL;
656
+ return pURL + (pURL.indexOf('?') > -1 ? '&' : '?') + this.SyncDeletedRecordsQueryString;
657
+ }
658
+
648
659
  syncDeletedRecords(fCallback)
649
660
  {
650
661
  const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
@@ -657,7 +668,9 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
657
668
  this.fable.log.info(`Checking for deleted records on server for ${this.EntitySchema.TableName}...`);
658
669
 
659
670
  // The explicit FBV~Deleted~EQ~1 filter overrides foxhound's automatic Deleted=0 filter.
660
- this.fable.MeadowCloneRestClient.getJSON(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`,
671
+ // On older APIs where this doesn't work, SyncDeletedRecordsQueryString provides a
672
+ // fallback (e.g. "includeDeleted=true") that disables server-side delete tracking.
673
+ this.fable.MeadowCloneRestClient.getJSON(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`),
661
674
  (pError, pResponse, pBody) =>
662
675
  {
663
676
  if (pError || !pBody || !pBody.hasOwnProperty('Count'))
@@ -682,7 +695,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
682
695
  const tmpDeleteURLPartials = [];
683
696
  for (let i = 0; i < tmpDeleteCap; i += this.PageSize)
684
697
  {
685
- tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
698
+ tmpDeleteURLPartials.push(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`));
686
699
  }
687
700
 
688
701
  this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
@@ -89,6 +89,20 @@ class MeadowSync extends libFableServiceProviderBase
89
89
  this.DateTimePrecisionMS = parseInt(this.options.DateTimePrecisionMS, 10) || 1000;
90
90
  }
91
91
 
92
+ // Optional query string appended to deleted record API requests for all entities.
93
+ // Used to work around older APIs where FBV~Deleted~EQ~1 does not override the
94
+ // automatic Deleted=0 filter (e.g. "includeDeleted=true").
95
+ // Can be overridden per-entity via SyncEntityOptions.
96
+ this.SyncDeletedRecordsQueryString = '';
97
+ if (this.fable.ProgramConfiguration.hasOwnProperty('SyncDeletedRecordsQueryString'))
98
+ {
99
+ this.SyncDeletedRecordsQueryString = this.fable.ProgramConfiguration.SyncDeletedRecordsQueryString;
100
+ }
101
+ else if (this.options.hasOwnProperty('SyncDeletedRecordsQueryString'))
102
+ {
103
+ this.SyncDeletedRecordsQueryString = this.options.SyncDeletedRecordsQueryString;
104
+ }
105
+
92
106
  this.MeadowSchema = false;
93
107
  this.MeadowSchemaTableList = false;
94
108
 
@@ -124,11 +138,18 @@ class MeadowSync extends libFableServiceProviderBase
124
138
  ConnectionPool: this.options.ConnectionPool,
125
139
  PageSize: this.options.PageSize || 100,
126
140
  SyncDeletedRecords: this.SyncDeletedRecords,
141
+ SyncDeletedRecordsQueryString: this.SyncDeletedRecordsQueryString,
127
142
  MaxRecordsPerEntity: this.MaxRecordsPerEntity,
128
143
  DateTimePrecisionMS: this.DateTimePrecisionMS,
129
144
  UseAdvancedIDPagination: this.UseAdvancedIDPagination,
130
145
  };
131
146
 
147
+ // Apply per-entity option overrides if configured
148
+ if (this.SyncEntityOptions.hasOwnProperty(tmpEntitySchema.TableName))
149
+ {
150
+ Object.assign(tmpSyncEntityOptions, this.SyncEntityOptions[tmpEntitySchema.TableName]);
151
+ }
152
+
132
153
  let tmpSyncEntity;
133
154
 
134
155
  if (this.SyncMode == 'Ongoing')