@memberjunction/integration-engine 0.0.1 → 5.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -28
- package/dist/BaseIntegrationConnector.d.ts +142 -0
- package/dist/BaseIntegrationConnector.d.ts.map +1 -0
- package/dist/BaseIntegrationConnector.js +91 -0
- package/dist/BaseIntegrationConnector.js.map +1 -0
- package/dist/ConnectorFactory.d.ts +30 -0
- package/dist/ConnectorFactory.d.ts.map +1 -0
- package/dist/ConnectorFactory.js +50 -0
- package/dist/ConnectorFactory.js.map +1 -0
- package/dist/FieldMappingEngine.d.ts +94 -0
- package/dist/FieldMappingEngine.d.ts.map +1 -0
- package/dist/FieldMappingEngine.js +250 -0
- package/dist/FieldMappingEngine.js.map +1 -0
- package/dist/IntegrationOrchestrator.d.ts +126 -0
- package/dist/IntegrationOrchestrator.d.ts.map +1 -0
- package/dist/IntegrationOrchestrator.js +548 -0
- package/dist/IntegrationOrchestrator.js.map +1 -0
- package/dist/MatchEngine.d.ts +50 -0
- package/dist/MatchEngine.d.ts.map +1 -0
- package/dist/MatchEngine.js +124 -0
- package/dist/MatchEngine.js.map +1 -0
- package/dist/RetryRunner.d.ts +27 -0
- package/dist/RetryRunner.d.ts.map +1 -0
- package/dist/RetryRunner.js +59 -0
- package/dist/RetryRunner.js.map +1 -0
- package/dist/WatermarkService.d.ts +44 -0
- package/dist/WatermarkService.d.ts.map +1 -0
- package/dist/WatermarkService.js +97 -0
- package/dist/WatermarkService.js.map +1 -0
- package/dist/entity-types.d.ts +90 -0
- package/dist/entity-types.d.ts.map +1 -0
- package/dist/entity-types.js +13 -0
- package/dist/entity-types.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/transforms.d.ts +75 -0
- package/dist/transforms.d.ts.map +1 -0
- package/dist/transforms.js +2 -0
- package/dist/transforms.js.map +1 -0
- package/dist/types.d.ts +192 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +44 -0
- package/dist/types.js.map +1 -0
- package/package.json +29 -7
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Engine responsible for applying field-level mappings and transformations
|
|
3
|
+
* from external records to MJ entity fields.
|
|
4
|
+
*/
|
|
5
|
+
export class FieldMappingEngine {
|
|
6
|
+
/**
|
|
7
|
+
* Applies field mappings to a batch of external records, producing mapped records
|
|
8
|
+
* ready for match resolution and persistence.
|
|
9
|
+
*
|
|
10
|
+
* @param records - External records to map
|
|
11
|
+
* @param fieldMaps - Active field map entities defining source→destination mappings
|
|
12
|
+
* @param entityName - Target MJ entity name
|
|
13
|
+
* @returns Array of mapped records with transformed field values
|
|
14
|
+
*/
|
|
15
|
+
Apply(records, fieldMaps, entityName) {
|
|
16
|
+
const activeMaps = fieldMaps.filter(fm => fm.Status === 'Active');
|
|
17
|
+
return records.map(record => this.MapSingleRecord(record, activeMaps, entityName));
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Maps a single external record through all active field mappings.
|
|
21
|
+
*/
|
|
22
|
+
MapSingleRecord(record, fieldMaps, entityName) {
|
|
23
|
+
const mappedFields = {};
|
|
24
|
+
for (const fieldMap of fieldMaps) {
|
|
25
|
+
const value = this.ApplyFieldMapping(record, fieldMap);
|
|
26
|
+
if (value !== undefined) {
|
|
27
|
+
mappedFields[fieldMap.DestinationFieldName] = value;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return {
|
|
31
|
+
ExternalRecord: record,
|
|
32
|
+
MJEntityName: entityName,
|
|
33
|
+
MappedFields: mappedFields,
|
|
34
|
+
ChangeType: record.IsDeleted ? 'Delete' : 'Create',
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Applies a single field mapping, including the full transform pipeline.
|
|
39
|
+
* Returns undefined if the field should be skipped (OnError: Skip).
|
|
40
|
+
*/
|
|
41
|
+
ApplyFieldMapping(record, fieldMap) {
|
|
42
|
+
let value = record.Fields[fieldMap.SourceFieldName];
|
|
43
|
+
const pipeline = this.ParseTransformPipeline(fieldMap.TransformPipeline);
|
|
44
|
+
for (const step of pipeline) {
|
|
45
|
+
const result = this.ExecuteTransformStep(step, value, record.Fields);
|
|
46
|
+
if (result.Skipped)
|
|
47
|
+
return undefined;
|
|
48
|
+
value = result.Value;
|
|
49
|
+
}
|
|
50
|
+
return value;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Parses the JSON transform pipeline string into typed TransformStep objects.
|
|
54
|
+
*/
|
|
55
|
+
ParseTransformPipeline(pipelineJson) {
|
|
56
|
+
if (!pipelineJson || pipelineJson.trim() === '')
|
|
57
|
+
return [];
|
|
58
|
+
try {
|
|
59
|
+
const parsed = JSON.parse(pipelineJson);
|
|
60
|
+
if (!Array.isArray(parsed))
|
|
61
|
+
return [];
|
|
62
|
+
return parsed;
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Executes a single transform step, applying the configured transformation
|
|
70
|
+
* and handling errors according to the step's OnError strategy.
|
|
71
|
+
*/
|
|
72
|
+
ExecuteTransformStep(step, value, allFields) {
|
|
73
|
+
const onError = step.OnError ?? 'Fail';
|
|
74
|
+
try {
|
|
75
|
+
const transformed = this.DispatchTransform(step, value, allFields);
|
|
76
|
+
return { Value: transformed, Skipped: false };
|
|
77
|
+
}
|
|
78
|
+
catch (err) {
|
|
79
|
+
return this.HandleTransformError(onError, err);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Dispatches to the appropriate transform handler based on step type.
|
|
84
|
+
*/
|
|
85
|
+
DispatchTransform(step, value, allFields) {
|
|
86
|
+
switch (step.Type) {
|
|
87
|
+
case 'direct':
|
|
88
|
+
return this.ApplyDirect(value, step.Config);
|
|
89
|
+
case 'regex':
|
|
90
|
+
return this.ApplyRegex(value, step.Config);
|
|
91
|
+
case 'split':
|
|
92
|
+
return this.ApplySplit(value, step.Config);
|
|
93
|
+
case 'combine':
|
|
94
|
+
return this.ApplyCombine(allFields, step.Config);
|
|
95
|
+
case 'lookup':
|
|
96
|
+
return this.ApplyLookup(value, step.Config);
|
|
97
|
+
case 'format':
|
|
98
|
+
return this.ApplyFormat(value, step.Config);
|
|
99
|
+
case 'coerce':
|
|
100
|
+
return this.ApplyCoerce(value, step.Config);
|
|
101
|
+
case 'substring':
|
|
102
|
+
return this.ApplySubstring(value, step.Config);
|
|
103
|
+
case 'custom':
|
|
104
|
+
return this.ApplyCustom(value, allFields, step.Config);
|
|
105
|
+
default:
|
|
106
|
+
throw new Error(`Unknown transform type: ${step.Type}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Handles a transform error according to the OnError strategy.
|
|
111
|
+
*/
|
|
112
|
+
HandleTransformError(onError, err) {
|
|
113
|
+
switch (onError) {
|
|
114
|
+
case 'Skip':
|
|
115
|
+
return { Value: undefined, Skipped: true };
|
|
116
|
+
case 'Null':
|
|
117
|
+
return { Value: null, Skipped: false };
|
|
118
|
+
case 'Fail':
|
|
119
|
+
throw err;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Direct pass-through, applying default value if source is null/undefined.
|
|
124
|
+
*/
|
|
125
|
+
ApplyDirect(value, config) {
|
|
126
|
+
if (value == null && config.DefaultValue !== undefined) {
|
|
127
|
+
return config.DefaultValue;
|
|
128
|
+
}
|
|
129
|
+
return value;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Applies a regex replacement to a string value.
|
|
133
|
+
*/
|
|
134
|
+
ApplyRegex(value, config) {
|
|
135
|
+
const strValue = String(value ?? '');
|
|
136
|
+
const regex = new RegExp(config.Pattern, config.Flags ?? '');
|
|
137
|
+
return strValue.replace(regex, config.Replacement);
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Splits a string value and extracts a part by index.
|
|
141
|
+
*/
|
|
142
|
+
ApplySplit(value, config) {
|
|
143
|
+
const strValue = String(value ?? '');
|
|
144
|
+
const parts = strValue.split(config.Delimiter);
|
|
145
|
+
return parts[config.Index] ?? null;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Combines multiple source fields with a separator.
|
|
149
|
+
*/
|
|
150
|
+
ApplyCombine(allFields, config) {
|
|
151
|
+
const values = config.SourceFields.map(f => String(allFields[f] ?? ''));
|
|
152
|
+
return values.join(config.Separator);
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Performs a case-insensitive value lookup/mapping.
|
|
156
|
+
*/
|
|
157
|
+
ApplyLookup(value, config) {
|
|
158
|
+
const strValue = String(value ?? '');
|
|
159
|
+
const lowerKey = strValue.toLowerCase();
|
|
160
|
+
const entry = Object.entries(config.Map).find(([key]) => key.toLowerCase() === lowerKey);
|
|
161
|
+
return entry ? entry[1] : (config.Default ?? null);
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Applies date/number/string formatting.
|
|
165
|
+
*/
|
|
166
|
+
ApplyFormat(value, config) {
|
|
167
|
+
if (config.FormatType === 'date') {
|
|
168
|
+
const dateVal = value instanceof Date ? value : new Date(String(value));
|
|
169
|
+
return this.FormatDate(dateVal, config.FormatString);
|
|
170
|
+
}
|
|
171
|
+
if (config.FormatType === 'number') {
|
|
172
|
+
const numVal = Number(value);
|
|
173
|
+
return numVal.toFixed(Number(config.FormatString) || 0);
|
|
174
|
+
}
|
|
175
|
+
return String(value ?? '');
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Basic date formatting supporting ISO and common format tokens.
|
|
179
|
+
*/
|
|
180
|
+
FormatDate(date, formatString) {
|
|
181
|
+
if (formatString.toLowerCase() === 'iso' || formatString === 'ISO8601') {
|
|
182
|
+
return date.toISOString();
|
|
183
|
+
}
|
|
184
|
+
return date.toLocaleDateString('en-US');
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Coerces a value to the specified target type.
|
|
188
|
+
*/
|
|
189
|
+
ApplyCoerce(value, config) {
|
|
190
|
+
switch (config.TargetType) {
|
|
191
|
+
case 'string':
|
|
192
|
+
return String(value ?? '');
|
|
193
|
+
case 'number':
|
|
194
|
+
return this.CoerceToNumber(value);
|
|
195
|
+
case 'boolean':
|
|
196
|
+
return this.CoerceToBoolean(value);
|
|
197
|
+
case 'date':
|
|
198
|
+
return new Date(String(value));
|
|
199
|
+
default:
|
|
200
|
+
throw new Error(`Unknown coerce target type: ${config.TargetType}`);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Coerces a value to a number, throwing on NaN.
|
|
205
|
+
*/
|
|
206
|
+
CoerceToNumber(value) {
|
|
207
|
+
const num = Number(value);
|
|
208
|
+
if (isNaN(num)) {
|
|
209
|
+
throw new Error(`Cannot coerce "${String(value)}" to number`);
|
|
210
|
+
}
|
|
211
|
+
return num;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Coerces a value to boolean, supporting common truthy/falsy strings.
|
|
215
|
+
*/
|
|
216
|
+
CoerceToBoolean(value) {
|
|
217
|
+
if (typeof value === 'boolean')
|
|
218
|
+
return value;
|
|
219
|
+
if (typeof value === 'number')
|
|
220
|
+
return value !== 0;
|
|
221
|
+
const strVal = String(value).toLowerCase().trim();
|
|
222
|
+
return strVal === 'true' || strVal === '1' || strVal === 'yes';
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Extracts a substring from a string value.
|
|
226
|
+
*/
|
|
227
|
+
ApplySubstring(value, config) {
|
|
228
|
+
const strValue = String(value ?? '');
|
|
229
|
+
if (config.Length !== undefined) {
|
|
230
|
+
return strValue.substring(config.Start, config.Start + config.Length);
|
|
231
|
+
}
|
|
232
|
+
return strValue.substring(config.Start);
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Evaluates a custom JavaScript expression.
|
|
236
|
+
* The expression has access to `value` (current field value) and `fields` (all record fields).
|
|
237
|
+
*/
|
|
238
|
+
ApplyCustom(value, allFields, config) {
|
|
239
|
+
try {
|
|
240
|
+
// eslint-disable-next-line @typescript-eslint/no-implied-eval
|
|
241
|
+
const fn = new Function('value', 'fields', `return (${config.Expression});`);
|
|
242
|
+
return fn(value, allFields);
|
|
243
|
+
}
|
|
244
|
+
catch (err) {
|
|
245
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
246
|
+
throw new Error(`Custom transform expression failed: ${message}`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
//# sourceMappingURL=FieldMappingEngine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FieldMappingEngine.js","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAgBA;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IAC3B;;;;;;;;OAQG;IACI,KAAK,CACR,OAAyB,EACzB,SAAwC,EACxC,UAAkB;QAElB,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACK,eAAe,CACnB,MAAsB,EACtB,SAAwC,EACxC,UAAkB;QAElB,MAAM,YAAY,GAA4B,EAAE,CAAC;QAEjD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC,GAAG,KAAK,CAAC;YACxD,CAAC;QACL,CAAC;QAED,OAAO;YACH,cAAc,EAAE,MAAM;YACtB,YAAY,EAAE,UAAU;YACxB,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;SACrD,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,iBAAiB,CACrB,MAAsB,EACtB,QAAqC;QAErC,IAAI,KAAK,GAAY,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAEzE,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YACrE,IAAI,MAAM,CAAC,OAAO;gBAAE,OAAO,SAAS,CAAC;YACrC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACzB,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,YAA2B;QACtD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAE3D,IAAI,CAAC;YACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,OAAO,EAAE,CAAC;YACtC,OAAO,MAAyB,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,oBAAoB,CACxB,IAAmB,EACnB,KAAc,EACd,SAAkC;QAElC,MAAM,OAAO,GAAqB,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC;QAEzD,IAAI,CAAC;YACD,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;YACnE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAClD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,CACrB,IAAmB,EACnB,KAAc,EACd,SAAkC;QAElC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAChB,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,OAAO;gBACR,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAqB,CAAC,CAAC;YAC9D,KAAK,OAAO;gBACR,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAqB,CAAC,CAAC;YAC9D,KAAK,SAAS;gBACV,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,MAAuB,CAAC,CAAC;YACtE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,WAAW;gBACZ,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,MAAyB,CAAC,CAAC;YACtE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAC3E;gBACI,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,OAAyB,EAAE,GAAY;QAChE,QAAQ,OAAO,EAAE,CAAC;YACd,KAAK,MAAM;gBACP,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YAC/C,KAAK,MAAM;gBACP,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAC3C,KAAK,MAAM;gBACP,MAAM,GAAG,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,IAAI,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YACrD,OAAO,MAAM,CAAC,YAAY,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,KAAc,EAAE,MAAmB;QAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,KAAc,EAAE,MAAmB;QAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,SAAkC,EAAE,MAAqB;QAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CACzC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,QAAQ,CAC5C,CAAC;QACF,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,IAAI,MAAM,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,KAAK,YAAY,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,IAAU,EAAE,YAAoB;QAC/C,IAAI,YAAY,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YACrE,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;YACxB,KAAK,QAAQ;gBACT,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAC/B,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YACtC,KAAK,SAAS;gBACV,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACvC,KAAK,MAAM;gBACP,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACnC;gBACI,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAC5E,CAAC;IACL,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAc;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,KAAK,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAClD,OAAO,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,KAAK,CAAC;IACnE,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAc,EAAE,MAAuB;QAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACK,WAAW,CACf,KAAc,EACd,SAAkC,EAClC,MAAoB;QAEpB,IAAI,CAAC;YACD,8DAA8D;YAC9D,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;YAC7E,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAY,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { type UserInfo } from '@memberjunction/core';
|
|
2
|
+
import type { SyncResult, SyncTriggerType, OnProgressCallback, OnNotificationCallback } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Top-level orchestrator that runs an end-to-end integration sync.
|
|
5
|
+
* Coordinates connector, field mapping, match resolution, and entity persistence.
|
|
6
|
+
*/
|
|
7
|
+
export declare class IntegrationOrchestrator {
|
|
8
|
+
private readonly fieldMappingEngine;
|
|
9
|
+
private readonly matchEngine;
|
|
10
|
+
private readonly watermarkService;
|
|
11
|
+
/** In-process lock map to prevent concurrent syncs for the same CompanyIntegration */
|
|
12
|
+
private static readonly activeSyncs;
|
|
13
|
+
/** Configurable maximum batch size. Connector batches exceeding this are truncated. */
|
|
14
|
+
MaxBatchSize: number;
|
|
15
|
+
/**
|
|
16
|
+
* Executes a full sync run for a company integration.
|
|
17
|
+
* If a sync is already running for the same integration, waits for it to finish
|
|
18
|
+
* and returns the existing result (concurrency lock).
|
|
19
|
+
*
|
|
20
|
+
* @param companyIntegrationID - ID of the CompanyIntegration to sync
|
|
21
|
+
* @param contextUser - User context for all data operations
|
|
22
|
+
* @param triggerType - What triggered this sync (defaults to 'Manual')
|
|
23
|
+
* @param onProgress - Optional callback invoked with progress updates during sync
|
|
24
|
+
* @param onNotification - Optional callback invoked with a notification when the sync completes or fails
|
|
25
|
+
* @returns Aggregate sync result with record counts and errors
|
|
26
|
+
*/
|
|
27
|
+
RunSync(companyIntegrationID: string, contextUser: UserInfo, triggerType?: SyncTriggerType, onProgress?: OnProgressCallback, onNotification?: OnNotificationCallback): Promise<SyncResult>;
|
|
28
|
+
/**
|
|
29
|
+
* Internal sync execution method. Contains the full orchestration logic.
|
|
30
|
+
*/
|
|
31
|
+
private executeSyncInternal;
|
|
32
|
+
/**
|
|
33
|
+
* Loads all configuration needed for a sync run.
|
|
34
|
+
*/
|
|
35
|
+
private LoadRunConfiguration;
|
|
36
|
+
/**
|
|
37
|
+
* Creates a new CompanyIntegrationRun record to track this sync.
|
|
38
|
+
*/
|
|
39
|
+
private CreateRunRecord;
|
|
40
|
+
/**
|
|
41
|
+
* Processes all entity maps, aggregating results with progress tracking.
|
|
42
|
+
*/
|
|
43
|
+
private ExecuteEntityMaps;
|
|
44
|
+
/**
|
|
45
|
+
* Processes a single entity map: fetch → map → match → validate → apply.
|
|
46
|
+
*/
|
|
47
|
+
private ProcessSingleEntityMap;
|
|
48
|
+
/**
|
|
49
|
+
* Emits a progress update via the callback.
|
|
50
|
+
*/
|
|
51
|
+
private emitProgress;
|
|
52
|
+
/**
|
|
53
|
+
* Loads active field maps for a given entity map.
|
|
54
|
+
*/
|
|
55
|
+
private LoadFieldMaps;
|
|
56
|
+
/**
|
|
57
|
+
* Applies resolved records to MJ, handling each individually for error isolation.
|
|
58
|
+
*/
|
|
59
|
+
private ApplyRecords;
|
|
60
|
+
/**
|
|
61
|
+
* Applies a single record change (Create, Update, Delete, or Skip).
|
|
62
|
+
*/
|
|
63
|
+
private ApplySingleRecord;
|
|
64
|
+
/**
|
|
65
|
+
* Creates a new MJ record with pre-write validation and saves a record map entry.
|
|
66
|
+
*/
|
|
67
|
+
private CreateRecord;
|
|
68
|
+
/**
|
|
69
|
+
* Updates an existing MJ record with pre-write validation.
|
|
70
|
+
*/
|
|
71
|
+
private UpdateRecord;
|
|
72
|
+
/**
|
|
73
|
+
* Runs Validate() on an entity if the method exists.
|
|
74
|
+
* Throws a validation error with details if validation fails.
|
|
75
|
+
*/
|
|
76
|
+
private validateEntity;
|
|
77
|
+
/**
|
|
78
|
+
* Deletes (or soft-deletes) an MJ record based on the entity map's DeleteBehavior.
|
|
79
|
+
*/
|
|
80
|
+
private DeleteRecord;
|
|
81
|
+
/**
|
|
82
|
+
* Builds a CompositeKey for a single ID field.
|
|
83
|
+
*/
|
|
84
|
+
private BuildCompositeKey;
|
|
85
|
+
/**
|
|
86
|
+
* Sets fields on a BaseEntity instance from a field value map.
|
|
87
|
+
*/
|
|
88
|
+
private SetEntityFields;
|
|
89
|
+
/**
|
|
90
|
+
* Creates or updates a CompanyIntegrationRecordMap entry to track the external↔MJ mapping.
|
|
91
|
+
*/
|
|
92
|
+
private SaveRecordMap;
|
|
93
|
+
/**
|
|
94
|
+
* Creates a CompanyIntegrationRunDetail record for audit/reporting.
|
|
95
|
+
*/
|
|
96
|
+
private CreateRunDetail;
|
|
97
|
+
/**
|
|
98
|
+
* Merges an entity-map-level result into the aggregate result.
|
|
99
|
+
*/
|
|
100
|
+
private MergeResult;
|
|
101
|
+
/**
|
|
102
|
+
* Finalizes a successful run with aggregate totals and emits a completion notification.
|
|
103
|
+
*/
|
|
104
|
+
private FinalizeRun;
|
|
105
|
+
/**
|
|
106
|
+
* Marks a run as failed after an unrecoverable error and emits a failure notification.
|
|
107
|
+
*/
|
|
108
|
+
private FailRun;
|
|
109
|
+
/**
|
|
110
|
+
* Builds a SyncNotification for a completed run (success or completed-with-errors).
|
|
111
|
+
*/
|
|
112
|
+
private buildCompletionNotification;
|
|
113
|
+
/**
|
|
114
|
+
* Builds a SyncNotification for a catastrophically failed run.
|
|
115
|
+
*/
|
|
116
|
+
private buildFailureNotification;
|
|
117
|
+
/**
|
|
118
|
+
* Formats a human-readable sync result body for notification messages.
|
|
119
|
+
*/
|
|
120
|
+
private buildSyncResultBody;
|
|
121
|
+
/**
|
|
122
|
+
* Invokes the notification callback without propagating exceptions.
|
|
123
|
+
*/
|
|
124
|
+
private safeNotify;
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=IntegrationOrchestrator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IntegrationOrchestrator.d.ts","sourceRoot":"","sources":["../src/IntegrationOrchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmC,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAatF,OAAO,KAAK,EACR,UAAU,EAEV,eAAe,EAEf,kBAAkB,EAClB,sBAAsB,EAGzB,MAAM,YAAY,CAAC;AAWpB;;;GAGG;AACH,qBAAa,uBAAuB;IAChC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA4B;IAC/D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA0B;IAE3D,sFAAsF;IACtF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAA0C;IAE7E,uFAAuF;IAChF,YAAY,EAAE,MAAM,CAAsB;IAEjD;;;;;;;;;;;OAWG;IACU,OAAO,CAChB,oBAAoB,EAAE,MAAM,EAC5B,WAAW,EAAE,QAAQ,EACrB,WAAW,GAAE,eAA0B,EACvC,UAAU,CAAC,EAAE,kBAAkB,EAC/B,cAAc,CAAC,EAAE,sBAAsB,GACxC,OAAO,CAAC,UAAU,CAAC;IAmBtB;;OAEG;YACW,mBAAmB;IAoBjC;;OAEG;YACW,oBAAoB;IAuDlC;;OAEG;YACW,eAAe;IAyB7B;;OAEG;YACW,iBAAiB;IA8B/B;;OAEG;YACW,sBAAsB;IA2FpC;;OAEG;IACH,OAAO,CAAC,YAAY;IAqBpB;;OAEG;YACW,aAAa;IAe3B;;OAEG;YACW,YAAY;IA0B1B;;OAEG;YACW,iBAAiB;IA0B/B;;OAEG;YACW,YAAY;IA4B1B;;OAEG;YACW,YAAY;IA0B1B;;;OAGG;IACH,OAAO,CAAC,cAAc;IAatB;;OAEG;YACW,YAAY;IAwB1B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAMzB;;OAEG;IACH,OAAO,CAAC,eAAe;IASvB;;OAEG;YACW,aAAa;IAwB3B;;OAEG;YACW,eAAe;IAqB7B;;OAEG;IACH,OAAO,CAAC,WAAW;IAcnB;;OAEG;YACW,WAAW;IAoBzB;;OAEG;YACW,OAAO;IA2BrB;;OAEG;IACH,OAAO,CAAC,2BAA2B;IA0BnC;;OAEG;IACH,OAAO,CAAC,wBAAwB;IA2BhC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,OAAO,CAAC,UAAU;CAOrB"}
|