meadow-integration 1.0.20 → 1.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/example-applications/mapping-demo/.quackage.json +10 -0
- package/example-applications/mapping-demo/README.md +99 -0
- package/example-applications/mapping-demo/data/books-sample.csv +21 -0
- package/example-applications/mapping-demo/generate-build-config.js +44 -0
- package/example-applications/mapping-demo/mappings/books-to-book.json +14 -0
- package/example-applications/mapping-demo/package.json +14 -0
- package/example-applications/mapping-demo/server.js +814 -0
- package/example-applications/mapping-demo/source/MappingDemoApp.js +52 -0
- package/example-applications/mapping-demo/source/views/MappingDemoEditorView.js +186 -0
- package/example-applications/mapping-demo/web/index.html +892 -0
- package/example-applications/mapping-demo/web/mapping-demo-editor.js +3195 -0
- package/example-applications/mapping-demo/web/mapping-demo-editor.js.map +1 -0
- package/example-applications/mapping-demo/web/mapping-demo-editor.min.js +2 -0
- package/example-applications/mapping-demo/web/mapping-demo-editor.min.js.map +1 -0
- package/example-applications/mapping-demo/web/pict.min.js +12 -0
- package/package.json +8 -4
- package/source/Meadow-Integration-Browser.js +31 -0
- package/source/Meadow-Integration.js +16 -1
- package/source/services/certainty/Service-CertaintyAccumulator.js +402 -0
- package/source/services/clone/Meadow-Service-Sync-Entity-Initial.js +16 -3
- package/source/services/clone/Meadow-Service-Sync-Entity-Ongoing.js +15 -2
- package/source/services/clone/Meadow-Service-Sync.js +21 -0
- package/source/views/MappingEditor-SchemaUtils.js +71 -0
- package/source/views/PictView-MeadowMappingEditor.js +1299 -0
- package/source/views/flow-cards/FlowCard-MappingSource.js +50 -0
- package/source/views/flow-cards/FlowCard-MappingTarget.js +49 -0
- package/source/views/flow-cards/FlowCard-SolverExpression.js +78 -0
- package/source/views/flow-cards/FlowCard-TemplateExpression.js +77 -0
- package/test/Meadow-Integration-CloneDeleteSync_test.js +809 -0
package/package.json
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "meadow-integration",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.21",
|
|
4
4
|
"description": "Meadow Data Integration",
|
|
5
5
|
"bin": {
|
|
6
6
|
"mdwint": "source/cli/Meadow-Integration-CLI-Run.js"
|
|
7
7
|
},
|
|
8
8
|
"main": "source/Meadow-Integration.js",
|
|
9
|
+
"browser": "source/Meadow-Integration-Browser.js",
|
|
9
10
|
"scripts": {
|
|
10
11
|
"test": "npx quack test",
|
|
11
12
|
"start": "node source/cli/Meadow-Integration-CLI-Run.js",
|
|
@@ -16,7 +17,8 @@
|
|
|
16
17
|
"author": "steven velozo <steven@velozo.com>",
|
|
17
18
|
"license": "MIT",
|
|
18
19
|
"devDependencies": {
|
|
19
|
-
"
|
|
20
|
+
"meadow-connection-sqlite": "^1.0.18",
|
|
21
|
+
"quackage": "^1.0.65"
|
|
20
22
|
},
|
|
21
23
|
"mocha": {
|
|
22
24
|
"diff": true,
|
|
@@ -37,16 +39,18 @@
|
|
|
37
39
|
]
|
|
38
40
|
},
|
|
39
41
|
"dependencies": {
|
|
40
|
-
"fable": "^3.1.
|
|
42
|
+
"fable": "^3.1.67",
|
|
41
43
|
"fable-serviceproviderbase": "^3.0.19",
|
|
42
44
|
"fast-xml-parser": "^4.4.1",
|
|
43
45
|
"meadow": "^2.0.33",
|
|
44
46
|
"meadow-connection-mssql": "^1.0.16",
|
|
45
47
|
"meadow-connection-mysql": "^1.0.14",
|
|
46
48
|
"orator": "^6.0.4",
|
|
47
|
-
"orator-serviceserver-restify": "^2.0.
|
|
49
|
+
"orator-serviceserver-restify": "^2.0.10",
|
|
50
|
+
"pict-section-flow": "^0.0.17",
|
|
48
51
|
"pict-service-commandlineutility": "^1.0.19",
|
|
49
52
|
"pict-sessionmanager": "^1.0.2",
|
|
53
|
+
"pict-view": "^1.0.67",
|
|
50
54
|
"xlsx": "^0.18.5"
|
|
51
55
|
}
|
|
52
56
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Meadow-Integration Browser Entry Point
|
|
3
|
+
*
|
|
4
|
+
* Browser-safe exports only — no database drivers, no server-side
|
|
5
|
+
* connection managers, no file system access.
|
|
6
|
+
*
|
|
7
|
+
* Use this entry point in browser bundles:
|
|
8
|
+
* const libMI = require('meadow-integration/source/Meadow-Integration-Browser.js');
|
|
9
|
+
*
|
|
10
|
+
* Or add to package.json "browser" field for automatic resolution.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
'use strict';
|
|
14
|
+
|
|
15
|
+
const libMeadowMappingEditorView = require(`./views/PictView-MeadowMappingEditor.js`);
|
|
16
|
+
const libMappingEditorSchemaUtils = require(`./views/MappingEditor-SchemaUtils.js`);
|
|
17
|
+
const libFlowCardMappingSource = require(`./views/flow-cards/FlowCard-MappingSource.js`);
|
|
18
|
+
const libFlowCardMappingTarget = require(`./views/flow-cards/FlowCard-MappingTarget.js`);
|
|
19
|
+
const libFlowCardTemplateExpression = require(`./views/flow-cards/FlowCard-TemplateExpression.js`);
|
|
20
|
+
const libFlowCardSolverExpression = require(`./views/flow-cards/FlowCard-SolverExpression.js`);
|
|
21
|
+
|
|
22
|
+
module.exports =
|
|
23
|
+
{
|
|
24
|
+
// Visual mapping editor components
|
|
25
|
+
MeadowMappingEditorView: libMeadowMappingEditorView,
|
|
26
|
+
MappingEditorSchemaUtils: libMappingEditorSchemaUtils,
|
|
27
|
+
FlowCardMappingSource: libFlowCardMappingSource,
|
|
28
|
+
FlowCardMappingTarget: libFlowCardMappingTarget,
|
|
29
|
+
FlowCardTemplateExpression: libFlowCardTemplateExpression,
|
|
30
|
+
FlowCardSolverExpression: libFlowCardSolverExpression,
|
|
31
|
+
};
|
|
@@ -18,6 +18,13 @@ const libFileParserXLSX = require(`./services/parser/Service-FileParser-XLSX.js`
|
|
|
18
18
|
const libFileParserXML = require(`./services/parser/Service-FileParser-XML.js`);
|
|
19
19
|
const libFileParserFixedWidth = require(`./services/parser/Service-FileParser-FixedWidth.js`);
|
|
20
20
|
|
|
21
|
+
const libMeadowMappingEditorView = require(`./views/PictView-MeadowMappingEditor.js`);
|
|
22
|
+
const libMappingEditorSchemaUtils = require(`./views/MappingEditor-SchemaUtils.js`);
|
|
23
|
+
const libFlowCardMappingSource = require(`./views/flow-cards/FlowCard-MappingSource.js`);
|
|
24
|
+
const libFlowCardMappingTarget = require(`./views/flow-cards/FlowCard-MappingTarget.js`);
|
|
25
|
+
const libFlowCardTemplateExpression = require(`./views/flow-cards/FlowCard-TemplateExpression.js`);
|
|
26
|
+
const libFlowCardSolverExpression = require(`./views/flow-cards/FlowCard-SolverExpression.js`);
|
|
27
|
+
|
|
21
28
|
module.exports = (
|
|
22
29
|
{
|
|
23
30
|
TabularCheck: libTabularCheck,
|
|
@@ -38,5 +45,13 @@ module.exports = (
|
|
|
38
45
|
FileParserJSON: libFileParserJSON,
|
|
39
46
|
FileParserXLSX: libFileParserXLSX,
|
|
40
47
|
FileParserXML: libFileParserXML,
|
|
41
|
-
FileParserFixedWidth: libFileParserFixedWidth
|
|
48
|
+
FileParserFixedWidth: libFileParserFixedWidth,
|
|
49
|
+
|
|
50
|
+
// Visual mapping editor components (for browser bundles)
|
|
51
|
+
MeadowMappingEditorView: libMeadowMappingEditorView,
|
|
52
|
+
MappingEditorSchemaUtils: libMappingEditorSchemaUtils,
|
|
53
|
+
FlowCardMappingSource: libFlowCardMappingSource,
|
|
54
|
+
FlowCardMappingTarget: libFlowCardMappingTarget,
|
|
55
|
+
FlowCardTemplateExpression: libFlowCardTemplateExpression,
|
|
56
|
+
FlowCardSolverExpression: libFlowCardSolverExpression
|
|
42
57
|
});
|
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Service-CertaintyAccumulator
|
|
3
|
+
*
|
|
4
|
+
* Evidence accumulation engine for data integration certainty scoring.
|
|
5
|
+
*
|
|
6
|
+
* Tracks two certainty dimensions per field per record (GUID):
|
|
7
|
+
*
|
|
8
|
+
* 1. Cross-dataset presence: 1 - ∏(1 - weight_i)
|
|
9
|
+
* Strong signal — independent datasets corroborating a value.
|
|
10
|
+
*
|
|
11
|
+
* 2. Within-dataset ratio: log(count + 1) / log(datasetSize + 1)
|
|
12
|
+
* Weak signal — repetition within a single dataset has diminishing returns.
|
|
13
|
+
*
|
|
14
|
+
* 3. Composite: (w_presence × presence) + (w_ratio × ratio)
|
|
15
|
+
* Configurable weights; default 70% presence, 30% ratio.
|
|
16
|
+
*
|
|
17
|
+
* Evidence is tracked at the FIELD level. Record-level certainty is
|
|
18
|
+
* derived on the fly as the mean of its field composites.
|
|
19
|
+
*
|
|
20
|
+
* Designed to plug into Facto's MultiSet projection pipeline at the
|
|
21
|
+
* merge decision point, replacing the simple linear confidence tracker.
|
|
22
|
+
*
|
|
23
|
+
* @module Service-CertaintyAccumulator
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
'use strict';
|
|
27
|
+
|
|
28
|
+
const libFableServiceProviderBase = require('fable-serviceproviderbase');
|
|
29
|
+
|
|
30
|
+
class MeadowIntegrationCertaintyAccumulator extends libFableServiceProviderBase
|
|
31
|
+
{
|
|
32
|
+
constructor(pFable, pOptions, pServiceHash)
|
|
33
|
+
{
|
|
34
|
+
super(pFable, pOptions, pServiceHash);
|
|
35
|
+
|
|
36
|
+
this.serviceType = 'CertaintyAccumulator';
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// ─────────────────────────────────────────────
|
|
40
|
+
// Context lifecycle
|
|
41
|
+
// ─────────────────────────────────────────────
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Create a new accumulation context for a merge pipeline.
|
|
45
|
+
*
|
|
46
|
+
* @param {object} [pConfig] — optional overrides
|
|
47
|
+
* @param {number} [pConfig.PresenceWeight=0.7]
|
|
48
|
+
* @param {number} [pConfig.RatioWeight=0.3]
|
|
49
|
+
* @returns {object} AccumulationContext
|
|
50
|
+
*/
|
|
51
|
+
newAccumulationContext(pConfig)
|
|
52
|
+
{
|
|
53
|
+
let tmpConfig = (typeof pConfig === 'object' && pConfig !== null) ? pConfig : {};
|
|
54
|
+
|
|
55
|
+
let tmpPresenceWeight = (typeof tmpConfig.PresenceWeight === 'number') ? tmpConfig.PresenceWeight : 0.7;
|
|
56
|
+
let tmpRatioWeight = (typeof tmpConfig.RatioWeight === 'number') ? tmpConfig.RatioWeight : 0.3;
|
|
57
|
+
|
|
58
|
+
// Normalize so they sum to 1.0
|
|
59
|
+
let tmpSum = tmpPresenceWeight + tmpRatioWeight;
|
|
60
|
+
if (tmpSum > 0 && tmpSum !== 1.0)
|
|
61
|
+
{
|
|
62
|
+
tmpPresenceWeight = tmpPresenceWeight / tmpSum;
|
|
63
|
+
tmpRatioWeight = tmpRatioWeight / tmpSum;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return {
|
|
67
|
+
Weights:
|
|
68
|
+
{
|
|
69
|
+
presence: tmpPresenceWeight,
|
|
70
|
+
ratio: tmpRatioWeight,
|
|
71
|
+
},
|
|
72
|
+
|
|
73
|
+
// Per-GUID, per-field evidence
|
|
74
|
+
// { [GUID]: { [fieldName]: { value, sources: [...] } } }
|
|
75
|
+
FieldEvidence: {},
|
|
76
|
+
|
|
77
|
+
// Per-dataset metadata
|
|
78
|
+
// { [stepLabel]: { weight, totalRecords, uniqueGUIDs } }
|
|
79
|
+
DatasetContributions: {},
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// ─────────────────────────────────────────────
|
|
84
|
+
// Evidence accumulation
|
|
85
|
+
// ─────────────────────────────────────────────
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Record evidence from a single dataset/step for a specific GUID.
|
|
89
|
+
*
|
|
90
|
+
* Call this once per GUID per step, after the merge strategy has
|
|
91
|
+
* decided the action but before updating the accumulated comprehension.
|
|
92
|
+
*
|
|
93
|
+
* @param {object} pContext — the AccumulationContext
|
|
94
|
+
* @param {string} pGUID — record GUID
|
|
95
|
+
* @param {object} pNewRecord — incoming record from this step
|
|
96
|
+
* @param {object|null} pExistingRecord — previously merged record (null if first seen)
|
|
97
|
+
* @param {string} pMergeAction — e.g. 'Created', 'Merged', 'Merged_Reinforced', 'Skipped_*'
|
|
98
|
+
* @param {object} pStepInfo — source metadata
|
|
99
|
+
* @param {number} pStepInfo.ReliabilityWeight — DatasetSource weight (0–1)
|
|
100
|
+
* @param {number} pStepInfo.DatasetSize — total records in this source dataset
|
|
101
|
+
* @param {number} [pStepInfo.RecordCountInDataset=1] — how many times this GUID appeared in the dataset
|
|
102
|
+
* @param {string} pStepInfo.StepLabel — human-readable source label
|
|
103
|
+
* @param {number} pStepInfo.StepOrdinal — sequential step number
|
|
104
|
+
*/
|
|
105
|
+
accumulateEvidence(pContext, pGUID, pNewRecord, pExistingRecord, pMergeAction, pStepInfo)
|
|
106
|
+
{
|
|
107
|
+
if (!pContext || !pGUID || !pNewRecord)
|
|
108
|
+
{
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
let tmpStepInfo = pStepInfo || {};
|
|
113
|
+
let tmpWeight = (typeof tmpStepInfo.ReliabilityWeight === 'number') ? tmpStepInfo.ReliabilityWeight : 0.5;
|
|
114
|
+
let tmpDatasetSize = tmpStepInfo.DatasetSize || 1;
|
|
115
|
+
let tmpCount = tmpStepInfo.RecordCountInDataset || 1;
|
|
116
|
+
let tmpStepLabel = tmpStepInfo.StepLabel || 'unknown';
|
|
117
|
+
let tmpStepOrdinal = tmpStepInfo.StepOrdinal || 0;
|
|
118
|
+
|
|
119
|
+
// Track dataset contributions
|
|
120
|
+
if (!pContext.DatasetContributions[tmpStepLabel])
|
|
121
|
+
{
|
|
122
|
+
pContext.DatasetContributions[tmpStepLabel] =
|
|
123
|
+
{
|
|
124
|
+
weight: tmpWeight,
|
|
125
|
+
totalRecords: tmpDatasetSize,
|
|
126
|
+
uniqueGUIDs: 0,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
pContext.DatasetContributions[tmpStepLabel].uniqueGUIDs++;
|
|
130
|
+
|
|
131
|
+
// Initialize GUID evidence if needed
|
|
132
|
+
if (!pContext.FieldEvidence[pGUID])
|
|
133
|
+
{
|
|
134
|
+
pContext.FieldEvidence[pGUID] = {};
|
|
135
|
+
}
|
|
136
|
+
let tmpGUIDEvidence = pContext.FieldEvidence[pGUID];
|
|
137
|
+
|
|
138
|
+
// Record evidence for each field in the new record
|
|
139
|
+
let tmpFieldNames = Object.keys(pNewRecord);
|
|
140
|
+
for (let i = 0; i < tmpFieldNames.length; i++)
|
|
141
|
+
{
|
|
142
|
+
let tmpFieldName = tmpFieldNames[i];
|
|
143
|
+
let tmpValue = pNewRecord[tmpFieldName];
|
|
144
|
+
|
|
145
|
+
// Skip GUID fields and internal metadata
|
|
146
|
+
if (tmpFieldName.startsWith('GUID') || tmpFieldName.startsWith('_'))
|
|
147
|
+
{
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (!tmpGUIDEvidence[tmpFieldName])
|
|
152
|
+
{
|
|
153
|
+
tmpGUIDEvidence[tmpFieldName] =
|
|
154
|
+
{
|
|
155
|
+
value: tmpValue,
|
|
156
|
+
sources: [],
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
tmpGUIDEvidence[tmpFieldName].sources.push(
|
|
161
|
+
{
|
|
162
|
+
weight: tmpWeight,
|
|
163
|
+
count: tmpCount,
|
|
164
|
+
datasetSize: tmpDatasetSize,
|
|
165
|
+
stepLabel: tmpStepLabel,
|
|
166
|
+
stepOrdinal: tmpStepOrdinal,
|
|
167
|
+
value: tmpValue,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ─────────────────────────────────────────────
|
|
173
|
+
// Certainty computation
|
|
174
|
+
// ─────────────────────────────────────────────
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Compute field-level certainty scores for a single GUID.
|
|
178
|
+
*
|
|
179
|
+
* @param {object} pContext — the AccumulationContext
|
|
180
|
+
* @param {string} pGUID — record GUID
|
|
181
|
+
* @returns {object} — { fields: { fieldName: { presence, ratio, composite, agreeing, conflicting } }, recordComposite }
|
|
182
|
+
*/
|
|
183
|
+
computeCertainty(pContext, pGUID)
|
|
184
|
+
{
|
|
185
|
+
if (!pContext || !pContext.FieldEvidence[pGUID])
|
|
186
|
+
{
|
|
187
|
+
return { fields: {}, recordComposite: 0.5 };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
let tmpGUIDEvidence = pContext.FieldEvidence[pGUID];
|
|
191
|
+
let tmpFieldNames = Object.keys(tmpGUIDEvidence);
|
|
192
|
+
let tmpFields = {};
|
|
193
|
+
let tmpCompositeSum = 0;
|
|
194
|
+
let tmpFieldCount = 0;
|
|
195
|
+
|
|
196
|
+
for (let i = 0; i < tmpFieldNames.length; i++)
|
|
197
|
+
{
|
|
198
|
+
let tmpFieldName = tmpFieldNames[i];
|
|
199
|
+
let tmpFieldData = tmpGUIDEvidence[tmpFieldName];
|
|
200
|
+
let tmpSources = tmpFieldData.sources;
|
|
201
|
+
let tmpCurrentValue = tmpFieldData.value;
|
|
202
|
+
|
|
203
|
+
// ── Cross-dataset presence ──────────────────────────
|
|
204
|
+
// Only count sources where the value AGREES with the current value.
|
|
205
|
+
// Conflicting sources don't contribute to presence.
|
|
206
|
+
let tmpAgreeingSources = [];
|
|
207
|
+
let tmpConflictingSources = [];
|
|
208
|
+
|
|
209
|
+
for (let j = 0; j < tmpSources.length; j++)
|
|
210
|
+
{
|
|
211
|
+
if (this._valuesMatch(tmpSources[j].value, tmpCurrentValue))
|
|
212
|
+
{
|
|
213
|
+
tmpAgreeingSources.push(tmpSources[j]);
|
|
214
|
+
}
|
|
215
|
+
else
|
|
216
|
+
{
|
|
217
|
+
tmpConflictingSources.push(tmpSources[j]);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// presence = 1 - ∏(1 - weight_i) for agreeing sources
|
|
222
|
+
let tmpUncertainty = 1.0;
|
|
223
|
+
for (let j = 0; j < tmpAgreeingSources.length; j++)
|
|
224
|
+
{
|
|
225
|
+
tmpUncertainty *= (1.0 - tmpAgreeingSources[j].weight);
|
|
226
|
+
}
|
|
227
|
+
let tmpPresence = 1.0 - tmpUncertainty;
|
|
228
|
+
|
|
229
|
+
// Penalize if there are conflicting sources
|
|
230
|
+
if (tmpConflictingSources.length > 0)
|
|
231
|
+
{
|
|
232
|
+
let tmpConflictPenalty = tmpConflictingSources.length / (tmpAgreeingSources.length + tmpConflictingSources.length);
|
|
233
|
+
tmpPresence *= (1.0 - (tmpConflictPenalty * 0.5));
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// ── Within-dataset ratio (best source) ──────────────
|
|
237
|
+
// Use the strongest ratio from any agreeing source
|
|
238
|
+
let tmpBestRatio = 0;
|
|
239
|
+
for (let j = 0; j < tmpAgreeingSources.length; j++)
|
|
240
|
+
{
|
|
241
|
+
let tmpSrc = tmpAgreeingSources[j];
|
|
242
|
+
let tmpRatio = Math.log(tmpSrc.count + 1) / Math.log(tmpSrc.datasetSize + 1);
|
|
243
|
+
if (tmpRatio > tmpBestRatio)
|
|
244
|
+
{
|
|
245
|
+
tmpBestRatio = tmpRatio;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// ── Composite ───────────────────────────────────────
|
|
250
|
+
let tmpComposite = (pContext.Weights.presence * tmpPresence) +
|
|
251
|
+
(pContext.Weights.ratio * tmpBestRatio);
|
|
252
|
+
|
|
253
|
+
// Clamp to [0, 1]
|
|
254
|
+
tmpComposite = Math.max(0, Math.min(1, tmpComposite));
|
|
255
|
+
|
|
256
|
+
tmpFields[tmpFieldName] =
|
|
257
|
+
{
|
|
258
|
+
presence: Math.round(tmpPresence * 10000) / 10000,
|
|
259
|
+
ratio: Math.round(tmpBestRatio * 10000) / 10000,
|
|
260
|
+
composite: Math.round(tmpComposite * 10000) / 10000,
|
|
261
|
+
agreeing: tmpAgreeingSources.length,
|
|
262
|
+
conflicting: tmpConflictingSources.length,
|
|
263
|
+
};
|
|
264
|
+
|
|
265
|
+
tmpCompositeSum += tmpComposite;
|
|
266
|
+
tmpFieldCount++;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
let tmpRecordComposite = tmpFieldCount > 0
|
|
270
|
+
? Math.round((tmpCompositeSum / tmpFieldCount) * 10000) / 10000
|
|
271
|
+
: 0.5;
|
|
272
|
+
|
|
273
|
+
return {
|
|
274
|
+
fields: tmpFields,
|
|
275
|
+
recordComposite: tmpRecordComposite,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Bulk compute certainty for all GUIDs in the context.
|
|
281
|
+
*
|
|
282
|
+
* @param {object} pContext — the AccumulationContext
|
|
283
|
+
* @returns {object} — { [GUID]: CertaintyResult }
|
|
284
|
+
*/
|
|
285
|
+
computeAllCertainties(pContext)
|
|
286
|
+
{
|
|
287
|
+
if (!pContext || !pContext.FieldEvidence)
|
|
288
|
+
{
|
|
289
|
+
return {};
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
let tmpResults = {};
|
|
293
|
+
let tmpGUIDs = Object.keys(pContext.FieldEvidence);
|
|
294
|
+
|
|
295
|
+
for (let i = 0; i < tmpGUIDs.length; i++)
|
|
296
|
+
{
|
|
297
|
+
tmpResults[tmpGUIDs[i]] = this.computeCertainty(pContext, tmpGUIDs[i]);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return tmpResults;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// ─────────────────────────────────────────────
|
|
304
|
+
// CertaintyIndex entry generation
|
|
305
|
+
// ─────────────────────────────────────────────
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Generate CertaintyIndex entries for a single GUID.
|
|
309
|
+
* Creates one entry per field per dimension (presence, ratio, composite).
|
|
310
|
+
*
|
|
311
|
+
* @param {object} pContext — the AccumulationContext
|
|
312
|
+
* @param {string} pGUID — record GUID
|
|
313
|
+
* @param {number} pIDRecord — Facto Record ID to link entries to
|
|
314
|
+
* @returns {Array} — array of { IDRecord, CertaintyValue, Dimension, Justification }
|
|
315
|
+
*/
|
|
316
|
+
generateCertaintyEntries(pContext, pGUID, pIDRecord)
|
|
317
|
+
{
|
|
318
|
+
let tmpCertainty = this.computeCertainty(pContext, pGUID);
|
|
319
|
+
let tmpEntries = [];
|
|
320
|
+
|
|
321
|
+
let tmpFieldNames = Object.keys(tmpCertainty.fields);
|
|
322
|
+
for (let i = 0; i < tmpFieldNames.length; i++)
|
|
323
|
+
{
|
|
324
|
+
let tmpFieldName = tmpFieldNames[i];
|
|
325
|
+
let tmpField = tmpCertainty.fields[tmpFieldName];
|
|
326
|
+
|
|
327
|
+
let tmpJustification = JSON.stringify(
|
|
328
|
+
{
|
|
329
|
+
agreeing: tmpField.agreeing,
|
|
330
|
+
conflicting: tmpField.conflicting,
|
|
331
|
+
weights: pContext.Weights,
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
tmpEntries.push(
|
|
335
|
+
{
|
|
336
|
+
IDRecord: pIDRecord,
|
|
337
|
+
CertaintyValue: tmpField.presence,
|
|
338
|
+
Dimension: `field:${tmpFieldName}:presence`,
|
|
339
|
+
Justification: tmpJustification,
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
tmpEntries.push(
|
|
343
|
+
{
|
|
344
|
+
IDRecord: pIDRecord,
|
|
345
|
+
CertaintyValue: tmpField.ratio,
|
|
346
|
+
Dimension: `field:${tmpFieldName}:ratio`,
|
|
347
|
+
Justification: tmpJustification,
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
tmpEntries.push(
|
|
351
|
+
{
|
|
352
|
+
IDRecord: pIDRecord,
|
|
353
|
+
CertaintyValue: tmpField.composite,
|
|
354
|
+
Dimension: `field:${tmpFieldName}:composite`,
|
|
355
|
+
Justification: tmpJustification,
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
return tmpEntries;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// ─────────────────────────────────────────────
|
|
363
|
+
// Internal helpers
|
|
364
|
+
// ─────────────────────────────────────────────
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Compare two values for equality.
|
|
368
|
+
* Handles strings, numbers, nulls, and undefined.
|
|
369
|
+
* String comparisons are case-insensitive and trimmed.
|
|
370
|
+
*/
|
|
371
|
+
_valuesMatch(pValueA, pValueB)
|
|
372
|
+
{
|
|
373
|
+
// Both null/undefined
|
|
374
|
+
if (pValueA == null && pValueB == null)
|
|
375
|
+
{
|
|
376
|
+
return true;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// One null, other not
|
|
380
|
+
if (pValueA == null || pValueB == null)
|
|
381
|
+
{
|
|
382
|
+
return false;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
// Both strings — case-insensitive, trimmed
|
|
386
|
+
if (typeof pValueA === 'string' && typeof pValueB === 'string')
|
|
387
|
+
{
|
|
388
|
+
return pValueA.trim().toLowerCase() === pValueB.trim().toLowerCase();
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// Numeric comparison with tolerance
|
|
392
|
+
if (typeof pValueA === 'number' && typeof pValueB === 'number')
|
|
393
|
+
{
|
|
394
|
+
return Math.abs(pValueA - pValueB) < 0.0001;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// Fall back to strict equality
|
|
398
|
+
return pValueA === pValueB;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
module.exports = MeadowIntegrationCertaintyAccumulator;
|
|
@@ -43,6 +43,11 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
|
|
|
43
43
|
this.MaxRecordsPerEntity = this.options.MaxRecordsPerEntity || 0;
|
|
44
44
|
this.UseAdvancedIDPagination = this.options.UseAdvancedIDPagination || false;
|
|
45
45
|
|
|
46
|
+
// Optional query string appended to deleted record API requests.
|
|
47
|
+
// Used to work around older APIs where FBV~Deleted~EQ~1 does not
|
|
48
|
+
// override the automatic Deleted=0 filter (e.g. "includeDeleted=true").
|
|
49
|
+
this.SyncDeletedRecordsQueryString = this.options.SyncDeletedRecordsQueryString || '';
|
|
50
|
+
|
|
46
51
|
this.Meadow = false;
|
|
47
52
|
|
|
48
53
|
this.operation = new libMeadowOperation(this.fable);
|
|
@@ -119,7 +124,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
|
|
|
119
124
|
{
|
|
120
125
|
this.log.warn(`${this.EntitySchema.TableName}: Index creation error: ${pIndexError}`);
|
|
121
126
|
}
|
|
122
|
-
return
|
|
127
|
+
return fCallback(pIndexError || pCreateError);
|
|
123
128
|
});
|
|
124
129
|
});
|
|
125
130
|
}
|
|
@@ -170,6 +175,12 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
|
|
|
170
175
|
return tmpRecordToCommit;
|
|
171
176
|
}
|
|
172
177
|
|
|
178
|
+
_appendDeletedQueryString(pURL)
|
|
179
|
+
{
|
|
180
|
+
if (!this.SyncDeletedRecordsQueryString) return pURL;
|
|
181
|
+
return pURL + (pURL.indexOf('?') > -1 ? '&' : '?') + this.SyncDeletedRecordsQueryString;
|
|
182
|
+
}
|
|
183
|
+
|
|
173
184
|
syncDeletedRecords(fCallback)
|
|
174
185
|
{
|
|
175
186
|
const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
|
|
@@ -183,7 +194,9 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
|
|
|
183
194
|
|
|
184
195
|
// Get the count of deleted records from the server.
|
|
185
196
|
// The explicit FBV~Deleted~EQ~1 filter overrides foxhound's automatic Deleted=0 filter.
|
|
186
|
-
this
|
|
197
|
+
// On older APIs where this doesn't work, SyncDeletedRecordsQueryString provides a
|
|
198
|
+
// fallback (e.g. "includeDeleted=true") that disables server-side delete tracking.
|
|
199
|
+
this.fable.MeadowCloneRestClient.getJSON(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`),
|
|
187
200
|
(pError, pResponse, pBody) =>
|
|
188
201
|
{
|
|
189
202
|
if (pError || !pBody || !pBody.hasOwnProperty('Count'))
|
|
@@ -208,7 +221,7 @@ class MeadowSyncEntityInitial extends libFableServiceProviderBase
|
|
|
208
221
|
const tmpDeleteURLPartials = [];
|
|
209
222
|
for (let i = 0; i < tmpDeleteCap; i += this.PageSize)
|
|
210
223
|
{
|
|
211
|
-
tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
|
|
224
|
+
tmpDeleteURLPartials.push(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`));
|
|
212
225
|
}
|
|
213
226
|
|
|
214
227
|
this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
|
|
@@ -42,6 +42,11 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
|
42
42
|
this.SyncDeletedRecords = this.options.SyncDeletedRecords || false;
|
|
43
43
|
this.MaxRecordsPerEntity = this.options.MaxRecordsPerEntity || 0;
|
|
44
44
|
|
|
45
|
+
// Optional query string appended to deleted record API requests.
|
|
46
|
+
// Used to work around older APIs where FBV~Deleted~EQ~1 does not
|
|
47
|
+
// override the automatic Deleted=0 filter (e.g. "includeDeleted=true").
|
|
48
|
+
this.SyncDeletedRecordsQueryString = this.options.SyncDeletedRecordsQueryString || '';
|
|
49
|
+
|
|
45
50
|
// Minimum range size for bisection -- when a range is this small or smaller,
|
|
46
51
|
// pull all records in the range from the server instead of subdividing further.
|
|
47
52
|
this.BisectMinRangeSize = this.options.BisectMinRangeSize || 1000;
|
|
@@ -645,6 +650,12 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
|
645
650
|
|
|
646
651
|
// ---- Deleted records sync ----
|
|
647
652
|
|
|
653
|
+
_appendDeletedQueryString(pURL)
|
|
654
|
+
{
|
|
655
|
+
if (!this.SyncDeletedRecordsQueryString) return pURL;
|
|
656
|
+
return pURL + (pURL.indexOf('?') > -1 ? '&' : '?') + this.SyncDeletedRecordsQueryString;
|
|
657
|
+
}
|
|
658
|
+
|
|
648
659
|
syncDeletedRecords(fCallback)
|
|
649
660
|
{
|
|
650
661
|
const tmpDeletedColumn = this.EntitySchema.Columns.find((c) => c.Column == 'Deleted');
|
|
@@ -657,7 +668,9 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
|
657
668
|
this.fable.log.info(`Checking for deleted records on server for ${this.EntitySchema.TableName}...`);
|
|
658
669
|
|
|
659
670
|
// The explicit FBV~Deleted~EQ~1 filter overrides foxhound's automatic Deleted=0 filter.
|
|
660
|
-
this
|
|
671
|
+
// On older APIs where this doesn't work, SyncDeletedRecordsQueryString provides a
|
|
672
|
+
// fallback (e.g. "includeDeleted=true") that disables server-side delete tracking.
|
|
673
|
+
this.fable.MeadowCloneRestClient.getJSON(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/Count/FilteredTo/FBV~Deleted~EQ~1`),
|
|
661
674
|
(pError, pResponse, pBody) =>
|
|
662
675
|
{
|
|
663
676
|
if (pError || !pBody || !pBody.hasOwnProperty('Count'))
|
|
@@ -682,7 +695,7 @@ class MeadowSyncEntityOngoing extends libFableServiceProviderBase
|
|
|
682
695
|
const tmpDeleteURLPartials = [];
|
|
683
696
|
for (let i = 0; i < tmpDeleteCap; i += this.PageSize)
|
|
684
697
|
{
|
|
685
|
-
tmpDeleteURLPartials.push(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`);
|
|
698
|
+
tmpDeleteURLPartials.push(this._appendDeletedQueryString(`${this.EntitySchema.TableName}s/FilteredTo/FBV~Deleted~EQ~1~FSF~${this.DefaultIdentifier}~ASC~ASC/${i}/${this.PageSize}`));
|
|
686
699
|
}
|
|
687
700
|
|
|
688
701
|
this.fable.Utility.eachLimit(tmpDeleteURLPartials, 1,
|
|
@@ -89,6 +89,20 @@ class MeadowSync extends libFableServiceProviderBase
|
|
|
89
89
|
this.DateTimePrecisionMS = parseInt(this.options.DateTimePrecisionMS, 10) || 1000;
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
+
// Optional query string appended to deleted record API requests for all entities.
|
|
93
|
+
// Used to work around older APIs where FBV~Deleted~EQ~1 does not override the
|
|
94
|
+
// automatic Deleted=0 filter (e.g. "includeDeleted=true").
|
|
95
|
+
// Can be overridden per-entity via SyncEntityOptions.
|
|
96
|
+
this.SyncDeletedRecordsQueryString = '';
|
|
97
|
+
if (this.fable.ProgramConfiguration.hasOwnProperty('SyncDeletedRecordsQueryString'))
|
|
98
|
+
{
|
|
99
|
+
this.SyncDeletedRecordsQueryString = this.fable.ProgramConfiguration.SyncDeletedRecordsQueryString;
|
|
100
|
+
}
|
|
101
|
+
else if (this.options.hasOwnProperty('SyncDeletedRecordsQueryString'))
|
|
102
|
+
{
|
|
103
|
+
this.SyncDeletedRecordsQueryString = this.options.SyncDeletedRecordsQueryString;
|
|
104
|
+
}
|
|
105
|
+
|
|
92
106
|
this.MeadowSchema = false;
|
|
93
107
|
this.MeadowSchemaTableList = false;
|
|
94
108
|
|
|
@@ -124,11 +138,18 @@ class MeadowSync extends libFableServiceProviderBase
|
|
|
124
138
|
ConnectionPool: this.options.ConnectionPool,
|
|
125
139
|
PageSize: this.options.PageSize || 100,
|
|
126
140
|
SyncDeletedRecords: this.SyncDeletedRecords,
|
|
141
|
+
SyncDeletedRecordsQueryString: this.SyncDeletedRecordsQueryString,
|
|
127
142
|
MaxRecordsPerEntity: this.MaxRecordsPerEntity,
|
|
128
143
|
DateTimePrecisionMS: this.DateTimePrecisionMS,
|
|
129
144
|
UseAdvancedIDPagination: this.UseAdvancedIDPagination,
|
|
130
145
|
};
|
|
131
146
|
|
|
147
|
+
// Apply per-entity option overrides if configured
|
|
148
|
+
if (this.SyncEntityOptions.hasOwnProperty(tmpEntitySchema.TableName))
|
|
149
|
+
{
|
|
150
|
+
Object.assign(tmpSyncEntityOptions, this.SyncEntityOptions[tmpEntitySchema.TableName]);
|
|
151
|
+
}
|
|
152
|
+
|
|
132
153
|
let tmpSyncEntity;
|
|
133
154
|
|
|
134
155
|
if (this.SyncMode == 'Ongoing')
|