@memberjunction/integration-engine 5.42.0 → 5.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/dist/BaseRESTIntegrationConnector.d.ts +4 -0
- package/dist/BaseRESTIntegrationConnector.d.ts.map +1 -1
- package/dist/BaseRESTIntegrationConnector.js +46 -0
- package/dist/BaseRESTIntegrationConnector.js.map +1 -1
- package/dist/FieldMappingEngine.d.ts +16 -90
- package/dist/FieldMappingEngine.d.ts.map +1 -1
- package/dist/FieldMappingEngine.js +20 -254
- package/dist/FieldMappingEngine.js.map +1 -1
- package/dist/IntegrationEngine.d.ts +14 -0
- package/dist/IntegrationEngine.d.ts.map +1 -1
- package/dist/IntegrationEngine.js +20 -0
- package/dist/IntegrationEngine.js.map +1 -1
- package/package.json +7 -7
|
@@ -1,27 +1,26 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { FieldTransformEngine } from '@memberjunction/global';
|
|
2
2
|
import { computeUnmappedFields } from './CustomOverflow.js';
|
|
3
3
|
import { flattenRecord, hasNestedObject } from './RecordFlatten.js';
|
|
4
4
|
/**
|
|
5
|
-
* Engine responsible for applying field-level mappings and transformations
|
|
6
|
-
*
|
|
5
|
+
* Engine responsible for applying field-level mappings and transformations from external records to
|
|
6
|
+
* MJ entity fields.
|
|
7
|
+
*
|
|
8
|
+
* The per-value transform pipeline (direct / regex / split / combine / lookup / format / coerce /
|
|
9
|
+
* substring / custom) is owned by the shared {@link FieldTransformEngine} in `@memberjunction/global`
|
|
10
|
+
* and reused here, so integration sync and the rules-based bulk-update processor run the *exact same*
|
|
11
|
+
* transform implementation — one place to fix, one place to extend. This engine keeps only the
|
|
12
|
+
* integration-specific concerns: source flattening, field-map iteration, and unmapped-field overflow
|
|
13
|
+
* capture. See the Field Rules guide in `@memberjunction/global` for the shared engine, and
|
|
14
|
+
* `EntityFieldRules` in `@memberjunction/core` for the metadata-aware entity-update sibling.
|
|
7
15
|
*/
|
|
8
16
|
export class FieldMappingEngine {
|
|
9
17
|
constructor() {
|
|
10
|
-
/**
|
|
11
|
-
|
|
12
|
-
* identical expression is compiled once instead of once per record in a mapping batch.
|
|
13
|
-
*
|
|
14
|
-
* A failed compile is cached as the `Error` it threw — a malformed expression is therefore
|
|
15
|
-
* compiled a single time (then re-thrown from cache on every record) rather than recompiled
|
|
16
|
-
* on the per-record error path. Bounded via {@link MJLruCache} because the owning
|
|
17
|
-
* `IntegrationEngine` is a process-lifetime singleton; expression cardinality is normally
|
|
18
|
-
* tiny (it tracks configured field maps), so the 1000-entry default is comfortably ample.
|
|
19
|
-
*/
|
|
20
|
-
this.customFunctionCache = new MJLruCache();
|
|
18
|
+
/** The shared transform pipeline. Holds its own LRU cache of compiled custom expressions. */
|
|
19
|
+
this.transformEngine = new FieldTransformEngine();
|
|
21
20
|
}
|
|
22
21
|
/**
|
|
23
|
-
* Applies field mappings to a batch of external records, producing mapped records
|
|
24
|
-
*
|
|
22
|
+
* Applies field mappings to a batch of external records, producing mapped records ready for match
|
|
23
|
+
* resolution and persistence.
|
|
25
24
|
*
|
|
26
25
|
* @param records - External records to map
|
|
27
26
|
* @param fieldMaps - Active field map entities defining source→destination mappings
|
|
@@ -68,19 +67,14 @@ export class FieldMappingEngine {
|
|
|
68
67
|
};
|
|
69
68
|
}
|
|
70
69
|
/**
|
|
71
|
-
* Applies a single field mapping, including the full transform pipeline
|
|
72
|
-
* Returns undefined if the field should be skipped (OnError: Skip).
|
|
70
|
+
* Applies a single field mapping, including the full transform pipeline (delegated to the shared
|
|
71
|
+
* {@link FieldTransformEngine}). Returns undefined if the field should be skipped (OnError: Skip).
|
|
73
72
|
*/
|
|
74
73
|
ApplyFieldMapping(record, fieldMap) {
|
|
75
|
-
|
|
74
|
+
const value = record.Fields[fieldMap.SourceFieldName];
|
|
76
75
|
const pipeline = this.ParseTransformPipeline(fieldMap.TransformPipeline);
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
if (result.Skipped)
|
|
80
|
-
return undefined;
|
|
81
|
-
value = result.Value;
|
|
82
|
-
}
|
|
83
|
-
return value;
|
|
76
|
+
const result = this.transformEngine.ExecutePipeline(value, record.Fields, pipeline);
|
|
77
|
+
return result.Skipped ? undefined : result.Value;
|
|
84
78
|
}
|
|
85
79
|
/**
|
|
86
80
|
* Parses the JSON transform pipeline string into typed TransformStep objects.
|
|
@@ -98,233 +92,5 @@ export class FieldMappingEngine {
|
|
|
98
92
|
return [];
|
|
99
93
|
}
|
|
100
94
|
}
|
|
101
|
-
/**
|
|
102
|
-
* Executes a single transform step, applying the configured transformation
|
|
103
|
-
* and handling errors according to the step's OnError strategy.
|
|
104
|
-
*/
|
|
105
|
-
ExecuteTransformStep(step, value, allFields) {
|
|
106
|
-
// Grace by default (§8 "still get as much data"): an unconfigured transform that errors on one
|
|
107
|
-
// record's value nulls that FIELD and lets the record sync, rather than failing the record/batch.
|
|
108
|
-
// An author can still opt into strict 'Fail' or 'Skip' explicitly per step.
|
|
109
|
-
const onError = step.OnError ?? 'Null';
|
|
110
|
-
try {
|
|
111
|
-
const transformed = this.DispatchTransform(step, value, allFields);
|
|
112
|
-
return { Value: transformed, Skipped: false };
|
|
113
|
-
}
|
|
114
|
-
catch (err) {
|
|
115
|
-
return this.HandleTransformError(onError, err);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
/**
|
|
119
|
-
* Dispatches to the appropriate transform handler based on step type.
|
|
120
|
-
*/
|
|
121
|
-
DispatchTransform(step, value, allFields) {
|
|
122
|
-
switch (step.Type) {
|
|
123
|
-
case 'direct':
|
|
124
|
-
return this.ApplyDirect(value, step.Config);
|
|
125
|
-
case 'regex':
|
|
126
|
-
return this.ApplyRegex(value, step.Config);
|
|
127
|
-
case 'split':
|
|
128
|
-
return this.ApplySplit(value, step.Config);
|
|
129
|
-
case 'combine':
|
|
130
|
-
return this.ApplyCombine(allFields, step.Config);
|
|
131
|
-
case 'lookup':
|
|
132
|
-
return this.ApplyLookup(value, step.Config);
|
|
133
|
-
case 'format':
|
|
134
|
-
return this.ApplyFormat(value, step.Config);
|
|
135
|
-
case 'coerce':
|
|
136
|
-
return this.ApplyCoerce(value, step.Config);
|
|
137
|
-
case 'substring':
|
|
138
|
-
return this.ApplySubstring(value, step.Config);
|
|
139
|
-
case 'custom':
|
|
140
|
-
return this.ApplyCustom(value, allFields, step.Config);
|
|
141
|
-
default:
|
|
142
|
-
throw new Error(`Unknown transform type: ${step.Type}`);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
/**
|
|
146
|
-
* Handles a transform error according to the OnError strategy.
|
|
147
|
-
*/
|
|
148
|
-
HandleTransformError(onError, err) {
|
|
149
|
-
switch (onError) {
|
|
150
|
-
case 'Skip':
|
|
151
|
-
return { Value: undefined, Skipped: true };
|
|
152
|
-
case 'Null':
|
|
153
|
-
return { Value: null, Skipped: false };
|
|
154
|
-
case 'Fail':
|
|
155
|
-
throw err;
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
/**
|
|
159
|
-
* Direct pass-through, applying default value if source is null/undefined.
|
|
160
|
-
*/
|
|
161
|
-
ApplyDirect(value, config) {
|
|
162
|
-
if (value == null && config.DefaultValue !== undefined) {
|
|
163
|
-
return config.DefaultValue;
|
|
164
|
-
}
|
|
165
|
-
return value;
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* Applies a regex replacement to a string value.
|
|
169
|
-
*/
|
|
170
|
-
ApplyRegex(value, config) {
|
|
171
|
-
const strValue = String(value ?? '');
|
|
172
|
-
const regex = new RegExp(config.Pattern, config.Flags ?? '');
|
|
173
|
-
return strValue.replace(regex, config.Replacement);
|
|
174
|
-
}
|
|
175
|
-
/**
|
|
176
|
-
* Splits a string value and extracts a part by index.
|
|
177
|
-
*/
|
|
178
|
-
ApplySplit(value, config) {
|
|
179
|
-
const strValue = String(value ?? '');
|
|
180
|
-
const parts = strValue.split(config.Delimiter);
|
|
181
|
-
return parts[config.Index] ?? null;
|
|
182
|
-
}
|
|
183
|
-
/**
|
|
184
|
-
* Combines multiple source fields with a separator.
|
|
185
|
-
*/
|
|
186
|
-
ApplyCombine(allFields, config) {
|
|
187
|
-
const values = config.SourceFields.map(f => String(allFields[f] ?? ''));
|
|
188
|
-
return values.join(config.Separator);
|
|
189
|
-
}
|
|
190
|
-
/**
|
|
191
|
-
* Performs a case-insensitive value lookup/mapping.
|
|
192
|
-
*/
|
|
193
|
-
ApplyLookup(value, config) {
|
|
194
|
-
const strValue = String(value ?? '');
|
|
195
|
-
const lowerKey = strValue.toLowerCase();
|
|
196
|
-
const entry = Object.entries(config.Map).find(([key]) => key.toLowerCase() === lowerKey);
|
|
197
|
-
return entry ? entry[1] : (config.Default ?? null);
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Applies date/number/string formatting.
|
|
201
|
-
*/
|
|
202
|
-
ApplyFormat(value, config) {
|
|
203
|
-
if (config.FormatType === 'date') {
|
|
204
|
-
const dateVal = value instanceof Date ? value : new Date(String(value));
|
|
205
|
-
// Throw a CONTROLLED error on an unparseable date — vs the raw RangeError FormatDate→toISOString()
|
|
206
|
-
// would throw — so ExecuteTransformStep's OnError strategy handles it uniformly (default 'Null').
|
|
207
|
-
if (isNaN(dateVal.getTime()))
|
|
208
|
-
throw new Error(`Cannot format "${String(value)}" as a date`);
|
|
209
|
-
return this.FormatDate(dateVal, config.FormatString);
|
|
210
|
-
}
|
|
211
|
-
if (config.FormatType === 'number') {
|
|
212
|
-
const numVal = Number(value);
|
|
213
|
-
// Throw on non-numeric — was: emitted the literal string "NaN", which BYPASSED OnError and then
|
|
214
|
-
// failed SQL numeric binding (corrupting the write). OnError now handles it (default 'Null').
|
|
215
|
-
if (!Number.isFinite(numVal))
|
|
216
|
-
throw new Error(`Cannot format "${String(value)}" as a number`);
|
|
217
|
-
return numVal.toFixed(Number(config.FormatString) || 0);
|
|
218
|
-
}
|
|
219
|
-
return String(value ?? '');
|
|
220
|
-
}
|
|
221
|
-
/**
|
|
222
|
-
* Basic date formatting supporting ISO and common format tokens.
|
|
223
|
-
*/
|
|
224
|
-
FormatDate(date, formatString) {
|
|
225
|
-
if (formatString.toLowerCase() === 'iso' || formatString === 'ISO8601') {
|
|
226
|
-
return date.toISOString();
|
|
227
|
-
}
|
|
228
|
-
return date.toLocaleDateString('en-US');
|
|
229
|
-
}
|
|
230
|
-
/**
|
|
231
|
-
* Coerces a value to the specified target type.
|
|
232
|
-
*/
|
|
233
|
-
ApplyCoerce(value, config) {
|
|
234
|
-
switch (config.TargetType) {
|
|
235
|
-
case 'string':
|
|
236
|
-
return String(value ?? '');
|
|
237
|
-
case 'number':
|
|
238
|
-
return this.CoerceToNumber(value);
|
|
239
|
-
case 'boolean':
|
|
240
|
-
return this.CoerceToBoolean(value);
|
|
241
|
-
case 'date': {
|
|
242
|
-
// Throw on unparseable — was: returned an Invalid Date that corrupted the write, bypassing
|
|
243
|
-
// OnError. OnError handles it now (default 'Null' → field nulled, record syncs).
|
|
244
|
-
const d = new Date(String(value));
|
|
245
|
-
if (isNaN(d.getTime()))
|
|
246
|
-
throw new Error(`Cannot coerce "${String(value)}" to date`);
|
|
247
|
-
return d;
|
|
248
|
-
}
|
|
249
|
-
default:
|
|
250
|
-
throw new Error(`Unknown coerce target type: ${config.TargetType}`);
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
/**
|
|
254
|
-
* Coerces a value to a number, throwing on NaN.
|
|
255
|
-
*/
|
|
256
|
-
CoerceToNumber(value) {
|
|
257
|
-
const num = Number(value);
|
|
258
|
-
// Throw on unparseable so OnError decides (default 'Null' → field nulled, record syncs; 'Fail' → strict).
|
|
259
|
-
// The throw is caught by ExecuteTransformStep and never escapes to fail the record/batch.
|
|
260
|
-
if (isNaN(num)) {
|
|
261
|
-
throw new Error(`Cannot coerce "${String(value)}" to number`);
|
|
262
|
-
}
|
|
263
|
-
return num;
|
|
264
|
-
}
|
|
265
|
-
/**
|
|
266
|
-
* Coerces a value to boolean, supporting common truthy/falsy strings.
|
|
267
|
-
*/
|
|
268
|
-
CoerceToBoolean(value) {
|
|
269
|
-
if (typeof value === 'boolean')
|
|
270
|
-
return value;
|
|
271
|
-
if (typeof value === 'number')
|
|
272
|
-
return value !== 0;
|
|
273
|
-
const strVal = String(value).toLowerCase().trim();
|
|
274
|
-
return strVal === 'true' || strVal === '1' || strVal === 'yes';
|
|
275
|
-
}
|
|
276
|
-
/**
|
|
277
|
-
* Extracts a substring from a string value.
|
|
278
|
-
*/
|
|
279
|
-
ApplySubstring(value, config) {
|
|
280
|
-
const strValue = String(value ?? '');
|
|
281
|
-
if (config.Length !== undefined) {
|
|
282
|
-
return strValue.substring(config.Start, config.Start + config.Length);
|
|
283
|
-
}
|
|
284
|
-
return strValue.substring(config.Start);
|
|
285
|
-
}
|
|
286
|
-
/**
|
|
287
|
-
* Evaluates a custom JavaScript expression.
|
|
288
|
-
* The expression has access to `value` (current field value) and `fields` (all record fields).
|
|
289
|
-
*/
|
|
290
|
-
ApplyCustom(value, allFields, config) {
|
|
291
|
-
try {
|
|
292
|
-
const fn = this.GetCompiledExpression(config.Expression);
|
|
293
|
-
return fn(value, allFields);
|
|
294
|
-
}
|
|
295
|
-
catch (err) {
|
|
296
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
297
|
-
throw new Error(`Custom transform expression failed: ${message}`);
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
/**
|
|
301
|
-
* Returns the compiled function for an expression, compiling and caching on first use.
|
|
302
|
-
* If the expression failed to compile previously, the cached error is re-thrown — a
|
|
303
|
-
* malformed expression is compiled exactly once across an entire batch.
|
|
304
|
-
*/
|
|
305
|
-
GetCompiledExpression(expression) {
|
|
306
|
-
let cached = this.customFunctionCache.Get(expression);
|
|
307
|
-
if (cached === undefined) {
|
|
308
|
-
cached = this.compileExpression(expression);
|
|
309
|
-
this.customFunctionCache.Set(expression, cached);
|
|
310
|
-
}
|
|
311
|
-
if (cached instanceof Error) {
|
|
312
|
-
throw cached;
|
|
313
|
-
}
|
|
314
|
-
return cached;
|
|
315
|
-
}
|
|
316
|
-
/**
|
|
317
|
-
* Compiles a custom expression into a callable, returning the thrown `Error` instead of
|
|
318
|
-
* propagating it so the caller can cache compile failures alongside successes.
|
|
319
|
-
*/
|
|
320
|
-
compileExpression(expression) {
|
|
321
|
-
try {
|
|
322
|
-
// eslint-disable-next-line @typescript-eslint/no-implied-eval
|
|
323
|
-
return new Function('value', 'fields', `return (${expression});`);
|
|
324
|
-
}
|
|
325
|
-
catch (err) {
|
|
326
|
-
return err instanceof Error ? err : new Error(String(err));
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
95
|
}
|
|
330
96
|
//# sourceMappingURL=FieldMappingEngine.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FieldMappingEngine.js","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"FieldMappingEngine.js","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAKpE;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,kBAAkB;IAA/B;QACI,6FAA6F;QAC5E,oBAAe,GAAG,IAAI,oBAAoB,EAAE,CAAC;IA4FlE,CAAC;IA1FG;;;;;;;;OAQG;IACI,KAAK,CACR,OAAyB,EACzB,SAAwC,EACxC,UAAkB;QAElB,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACK,eAAe,CACnB,MAAsB,EACtB,SAAwC,EACxC,UAAkB;QAElB,8EAA8E;QAC9E,6EAA6E;QAC7E,kFAAkF;QAClF,wFAAwF;QACxF,uFAAuF;QACvF,0EAA0E;QAC1E,MAAM,GAAG,GAAmB,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC;YACtD,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;YACrD,CAAC,CAAC,MAAM,CAAC;QAEb,MAAM,YAAY,GAA4B,EAAE,CAAC;QACjD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE5C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;YAChD,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YACpD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC,GAAG,KAAK,CAAC;YACxD,CAAC;QACL,CAAC;QAED,iFAAiF;QACjF,mFAAmF;QACnF,sFAAsF;QACtF,qFAAqF;QACrF,MAAM,cAAc,GAAG,qBAAqB,CAAC,GAAG,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;QAE5E,OAAO;YACH,cAAc,EAAE,GAAG;YACnB,YAAY,EAAE,UAAU;YACxB,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAC/C,cAAc,EAAE,cAAc;SACjC,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,iBAAiB,CACrB,MAAsB,EACtB,QAAqC;QAErC,MAAM,KAAK,GAAY,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpF,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IACrD,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,YAA2B;QACtD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAE3D,IAAI,CAAC;YACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,OAAO,EAAE,CAAC;YACtC,OAAO,MAAyB,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -67,6 +67,20 @@ export declare class NumericOverflowError extends ValueFitError {
|
|
|
67
67
|
}
|
|
68
68
|
export declare class IntegrationEngine extends BaseSingleton<IntegrationEngine> {
|
|
69
69
|
constructor();
|
|
70
|
+
/**
|
|
71
|
+
* SQL Server's hard per-table column limit is 1024; the framework reserves headroom for its own
|
|
72
|
+
* sync/system columns (the __mj_integration_* set + ID + timestamps), so the effective ceiling is 1000.
|
|
73
|
+
* MJ_INTEGRATION_MAX_COLUMNS_PER_TABLE can only LOWER this, never raise it past 1000.
|
|
74
|
+
*/
|
|
75
|
+
static readonly MAX_COLUMNS_CEILING = 1000;
|
|
76
|
+
/**
|
|
77
|
+
* Effective per-table column limit — read from MJ_INTEGRATION_MAX_COLUMNS_PER_TABLE at startup and
|
|
78
|
+
* clamped to [1, MAX_COLUMNS_CEILING] (defaults to the ceiling when unset). Objects whose column count
|
|
79
|
+
* exceeds this are auto-disabled at apply time (reversible) instead of failing CREATE TABLE.
|
|
80
|
+
*/
|
|
81
|
+
readonly MaxColumnsPerTable: number;
|
|
82
|
+
/** Reads + clamps the column limit from env at startup; logs once when a configured value is clamped down. */
|
|
83
|
+
private static computeMaxColumnsPerTable;
|
|
70
84
|
private readonly fieldMappingEngine;
|
|
71
85
|
private readonly matchEngine;
|
|
72
86
|
private readonly watermarkService;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IntegrationEngine.d.ts","sourceRoot":"","sources":["../src/IntegrationEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsC,iBAAiB,EAAkC,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC5I,OAAO,EAAE,aAAa,EAAc,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,KAAK,EACR,0BAA0B,EAC1B,mCAAmC,EACnC,kCAAkC,EAIlC,uCAAuC,EACvC,mBAAmB,EACnB,6BAA6B,EAC7B,yBAAyB,EACzB,8BAA8B,EACjC,MAAM,+BAA+B,CAAC;AAKvC,OAAO,KAAK,EACR,UAAU,EAGV,eAAe,EAEf,kBAAkB,EAClB,sBAAsB,EAGtB,sBAAsB,EAEtB,oBAAoB,EAEpB,+BAA+B,EAClC,MAAM,YAAY,CAAC;AAyDpB;;;;;;;;;;;GAWG;AAEH;;;;;;;;;;;;GAYG;AACH,qBAAa,uBAAwB,SAAQ,KAAK;IAC9C,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,mBAAmB,EAAE,MAAM,CAAC;gBAEhC,UAAU,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM;CAU9D;AAED;;;;;;;GAOG;AACH,8BAAsB,aAAc,SAAQ,KAAK;IAC7C,SAAgB,SAAS,EAAE,MAAM,CAAC;IAClC,mFAAmF;IACnF,kBAAyB,WAAW,EAAE,MAAM,CAAC;gBACjC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAK9C,yEAAyE;aACzD,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CACrD;AAED,6DAA6D;AAC7D,qBAAa,mBAAoB,SAAQ,aAAa;IAClD,SAAgB,WAAW,6BAA6B;IACxD,SAAgB,WAAW,EAAE,MAAM,CAAC;IACpC,SAAgB,SAAS,EAAE,MAAM,CAAC;gBACtB,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAK9D,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAC5C;AAED,6HAA6H;AAC7H,qBAAa,oBAAqB,SAAQ,aAAa;IACnD,SAAgB,WAAW,8BAA8B;IACzD,SAAgB,KAAK,EAAE,MAAM,CAAC;IAC9B,SAAgB,OAAO,EAAE,MAAM,CAAC;gBACpB,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAKtD,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAC5C;AA+BD,qBAAa,iBAAkB,SAAQ,aAAa,CAAC,iBAAiB,CAAC;;
|
|
1
|
+
{"version":3,"file":"IntegrationEngine.d.ts","sourceRoot":"","sources":["../src/IntegrationEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsC,iBAAiB,EAAkC,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC5I,OAAO,EAAE,aAAa,EAAc,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,KAAK,EACR,0BAA0B,EAC1B,mCAAmC,EACnC,kCAAkC,EAIlC,uCAAuC,EACvC,mBAAmB,EACnB,6BAA6B,EAC7B,yBAAyB,EACzB,8BAA8B,EACjC,MAAM,+BAA+B,CAAC;AAKvC,OAAO,KAAK,EACR,UAAU,EAGV,eAAe,EAEf,kBAAkB,EAClB,sBAAsB,EAGtB,sBAAsB,EAEtB,oBAAoB,EAEpB,+BAA+B,EAClC,MAAM,YAAY,CAAC;AAyDpB;;;;;;;;;;;GAWG;AAEH;;;;;;;;;;;;GAYG;AACH,qBAAa,uBAAwB,SAAQ,KAAK;IAC9C,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,mBAAmB,EAAE,MAAM,CAAC;gBAEhC,UAAU,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM;CAU9D;AAED;;;;;;;GAOG;AACH,8BAAsB,aAAc,SAAQ,KAAK;IAC7C,SAAgB,SAAS,EAAE,MAAM,CAAC;IAClC,mFAAmF;IACnF,kBAAyB,WAAW,EAAE,MAAM,CAAC;gBACjC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAK9C,yEAAyE;aACzD,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CACrD;AAED,6DAA6D;AAC7D,qBAAa,mBAAoB,SAAQ,aAAa;IAClD,SAAgB,WAAW,6BAA6B;IACxD,SAAgB,WAAW,EAAE,MAAM,CAAC;IACpC,SAAgB,SAAS,EAAE,MAAM,CAAC;gBACtB,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAK9D,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAC5C;AAED,6HAA6H;AAC7H,qBAAa,oBAAqB,SAAQ,aAAa;IACnD,SAAgB,WAAW,8BAA8B;IACzD,SAAgB,KAAK,EAAE,MAAM,CAAC;IAC9B,SAAgB,OAAO,EAAE,MAAM,CAAC;gBACpB,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAKtD,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAC5C;AA+BD,qBAAa,iBAAkB,SAAQ,aAAa,CAAC,iBAAiB,CAAC;;IAMnE;;;;OAIG;IACH,gBAAuB,mBAAmB,QAAQ;IAElD;;;;OAIG;IACH,SAAgB,kBAAkB,EAAE,MAAM,CAAC;IAE3C,8GAA8G;IAC9G,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA4B;IAC/D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA0B;IAE3D,gFAAgF;IAChF,OAAO,CAAC,SAAS,CAAC,CAAoB;IAEtC;;;;;OAKG;IACH,OAAO,CAAC,+BAA+B,CAAC,CAAkC;IAE1E,wFAAwF;IACjF,kCAAkC,CAAC,QAAQ,EAAE,+BAA+B,GAAG,SAAS,GAAG,IAAI;IAItG,4FAA4F;IAC5F,SAAS,KAAK,aAAa,IAAI,iBAAiB,CAE/C;IAED,sFAAsF;IACtF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAA0C;IAE7E;;;;;;;;OAQG;IACH,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,iBAAiB;IAQzB,qDAAqD;IACrD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAsC;IAE/E,qFAAqF;IACrF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAA2C;IAEhF,2FAA2F;WAC7E,eAAe,CAAC,oBAAoB,EAAE,MAAM,GAAG,oBAAoB,GAAG,SAAS;IAI7F,6FAA6F;WAC/E,UAAU,CAAC,oBAAoB,EAAE,MAAM,GAAG,OAAO;IAW/D,2CAA2C;WAC7B,kBAAkB,IAAI,GAAG,CAAC,MAAM,EAAE,oBAAoB,CAAC;IAIrE,uFAAuF;IAChF,YAAY,EAAE,MAAM,CAAsB;IAEjD;;;;;;;OAOG;IACU,mBAAmB,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2HpG;;;;;;;;;;;OAWG;IACU,OAAO,CAChB,oBAAoB,EAAE,MAAM,EAC5B,WAAW,EAAE,QAAQ,EACrB,WAAW,GAAE,eAA0B,EACvC,UAAU,CAAC,EAAE,kBAAkB,EAC/B,cAAc,CAAC,EAAE,sBAAsB,EACvC,OAAO,CAAC,EAAE,sBAAsB,EAChC,QAAQ,CAAC,EAAE,iBAAiB,GAC7B,OAAO,CAAC,UAAU,CAAC;IAgDtB;;OAEG;YACW,mBAAmB;IAoIjC;;;;OAIG;IACH,OAAO,CAAC,yBAAyB;IA2BjC,gFAAgF;IAChF,OAAO,CAAC,yBAAyB;IAQjC,iFAAiF;YACnE,oBAAoB;IA4BlC;;OAEG;YACW,oBAAoB;IAsElC;;;;;;;;OAQG;YACW,wBAAwB;IA4BtC;;OAEG;YACW,eAAe;IAiC7B;;OAEG;YACW,iBAAiB;IAoK/B;;;;;;;OAOG;IACH,OAAO,CAAC,8BAA8B;IAgDtC;;;;;;;OAOG;IACH,OAAO,CAAC,8BAA8B;IA6CtC;;;;;;;;OAQG;IACH,OAAO,CAAC,qBAAqB;IA0C7B,6GAA6G;IAC7G,OAAO,CAAC,kBAAkB;IAY1B;;;;;;;OAOG;IACH,OAAO,CAAC,4BAA4B;IAQpC;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IA0B5B;;;;;OAKG;YACW,eAAe;IAyC7B,sGAAsG;YACxF,UAAU;IAgBxB,iHAAiH;IACjH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAEhE,kHAAkH;IAClH,OAAO,CAAC,mBAAmB;IAU3B;;;;;;OAMG;YACW,SAAS;IAIvB;;;;;;;OAOG;IACH;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAiB1B,OAAO,CAAC,cAAc;IAyBtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAQzB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAKzB;;;;;OAKG;YACW,sBAAsB;IA4BpC;;OAEG;YACW,eAAe;IAwhB7B;;;;;;OAMG;YACW,eAAe;IAkI7B,oGAAoG;YACtF,oBAAoB;IAyDlC;;;;OAIG;YACW,gBAAgB;IA8D9B,gEAAgE;YAClD,gBAAgB;IA2I9B;;;;;OAKG;IACH,OAAO,CAAC,YAAY;IAepB;;;;;;;;;OASG;YACW,kBAAkB;IAmFhC,+FAA+F;IAC/F,OAAO,CAAC,WAAW;IAOnB;;;;;;OAMG;IACH,OAAO,CAAC,uBAAuB;IAI/B,4GAA4G;YAC9F,sBAAsB;IAyCpC;;;;OAIG;YACW,qBAAqB;IA+DnC;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;IAQ/B,mCAAmC;IACnC,OAAO,CAAC,WAAW;IAOnB;;OAEG;IACH,OAAO,CAAC,YAAY;IAqBpB;;OAEG;YACW,aAAa;IAgB3B,gGAAgG;IAChG,OAAO,CAAC,oBAAoB;IAM5B,gGAAgG;IAChG,OAAO,CAAC,2BAA2B;IAInC,iGAAiG;IACjG,OAAO,CAAC,uBAAuB;IAK/B;;;;;;;;;;;;;;OAcG;YACW,0BAA0B;IAgFxC;;OAEG;YACW,YAAY;IA0I1B;;;;;;;OAOG;YACW,qBAAqB;IAsCnC;;;;;;;;;;;;;;OAcG;YACW,wBAAwB;IAgFtC;;OAEG;YACW,iBAAiB;IAkF/B;;;;;;;;;;;;;;;OAeG;YACW,YAAY;IAsF1B;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;IAiB/B;;;;OAIG;YACW,YAAY;IAkH1B;;;;;;;;OAQG;YACW,qBAAqB;IA2DnC;;;OAGG;IACH,OAAO,CAAC,cAAc;IAatB;;OAEG;YACW,YAAY;IAkD1B;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAgB7B;;OAEG;IACH,OAAO,CAAC,eAAe;IA6BvB;;;;;;;;OAQG;IACH,OAAO,CAAC,eAAe;IAavB;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IAgE3B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,aAAa;IAIrB;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAyEpC;;OAEG;YACW,aAAa;IA6C3B;;OAEG;YACW,eAAe;IA2C7B;;OAEG;IACH,OAAO,CAAC,WAAW;IAoBnB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAoB5B;;OAEG;YACW,WAAW;IAsDzB;;;;;;;OAOG;YACW,kBAAkB;IA6ChC;;OAEG;YACW,OAAO;IA2BrB;;OAEG;IACH,OAAO,CAAC,2BAA2B;IA0BnC;;OAEG;IACH,OAAO,CAAC,wBAAwB;IA2BhC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,OAAO,CAAC,UAAU;IAQlB;;;;;OAKG;YACW,2BAA2B;IAwBzC,SAAS,KAAK,IAAI,IAAI,qBAAqB,CAE1C;IAED;;;OAGG;IACU,MAAM,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhH,IAAW,YAAY,IAAI,mBAAmB,EAAE,CAE/C;IAED,IAAW,WAAW,IAAI,6BAA6B,EAAE,CAExD;IAED,IAAW,mBAAmB,IAAI,0BAA0B,EAAE,CAE7D;IAED,IAAW,UAAU,IAAI,mCAAmC,EAAE,CAE7D;IAED,IAAW,SAAS,IAAI,kCAAkC,EAAE,CAE3D;IAED,IAAW,UAAU,IAAI,uCAAuC,EAAE,CAEjE;IAIM,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,mBAAmB,GAAG,SAAS;IAI/D,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,SAAS;IAInE,yBAAyB,CAAC,EAAE,EAAE,MAAM,GAAG,0BAA0B,GAAG,SAAS;IAI7E,qCAAqC,CAAC,aAAa,EAAE,MAAM,GAAG,0BAA0B,EAAE;IAI1F,kCAAkC,CAAC,oBAAoB,EAAE,MAAM,GAAG,mCAAmC,EAAE;IAIvG,wBAAwB,CAAC,WAAW,EAAE,MAAM,GAAG,kCAAkC,EAAE;IAInF,oBAAoB,CAAC,oBAAoB,EAAE,MAAM,GAAG,mCAAmC,EAAE;IAIzF,mCAAmC,CAAC,oBAAoB,EAAE,MAAM,GAAG,mBAAmB,GAAG,SAAS;IAMzG,IAAW,kBAAkB,IAAI,yBAAyB,EAAE,CAE3D;IAED,IAAW,uBAAuB,IAAI,8BAA8B,EAAE,CAErE;IAEM,oCAAoC,CAAC,aAAa,EAAE,MAAM,GAAG,yBAAyB,EAAE;IAIxF,oBAAoB,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,yBAAyB,GAAG,SAAS;IAItG,wBAAwB,CAAC,QAAQ,EAAE,MAAM,GAAG,yBAAyB,GAAG,SAAS;IAIjF,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG,8BAA8B,EAAE;IAI9E,2BAA2B,CAAC,aAAa,EAAE,MAAM,GAAG,yBAAyB,EAAE;IAI/E,2BAA2B,CAAC,aAAa,EAAE,MAAM,GAAG,yBAAyB,EAAE;IAMtF,WAAkB,QAAQ,IAAI,iBAAiB,CAE9C;CACJ"}
|
|
@@ -169,6 +169,26 @@ export class IntegrationEngine extends BaseSingleton {
|
|
|
169
169
|
this.MaxBatchSize = DEFAULT_BATCH_SIZE;
|
|
170
170
|
/** Per-integration request-spacing chain for the rate limiter (keyed by IntegrationID → last scheduled time). */
|
|
171
171
|
this._rateLimiters = new Map();
|
|
172
|
+
this.MaxColumnsPerTable = IntegrationEngine.computeMaxColumnsPerTable();
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* SQL Server's hard per-table column limit is 1024; the framework reserves headroom for its own
|
|
176
|
+
* sync/system columns (the __mj_integration_* set + ID + timestamps), so the effective ceiling is 1000.
|
|
177
|
+
* MJ_INTEGRATION_MAX_COLUMNS_PER_TABLE can only LOWER this, never raise it past 1000.
|
|
178
|
+
*/
|
|
179
|
+
static { this.MAX_COLUMNS_CEILING = 1000; }
|
|
180
|
+
/** Reads + clamps the column limit from env at startup; logs once when a configured value is clamped down. */
|
|
181
|
+
static computeMaxColumnsPerTable() {
|
|
182
|
+
const ceiling = IntegrationEngine.MAX_COLUMNS_CEILING;
|
|
183
|
+
const raw = parseInt(process.env.MJ_INTEGRATION_MAX_COLUMNS_PER_TABLE ?? '', 10);
|
|
184
|
+
if (Number.isFinite(raw) && raw > 0) {
|
|
185
|
+
if (raw > ceiling) {
|
|
186
|
+
LogStatusEx({ message: `[IntegrationEngine] MJ_INTEGRATION_MAX_COLUMNS_PER_TABLE=${raw} exceeds the maximum ${ceiling} (SQL Server's 1024-column limit minus framework column headroom) — clamped to ${ceiling}.` });
|
|
187
|
+
return ceiling;
|
|
188
|
+
}
|
|
189
|
+
return raw;
|
|
190
|
+
}
|
|
191
|
+
return ceiling;
|
|
172
192
|
}
|
|
173
193
|
/** Registers (or clears, with undefined) the post-sync custom-column promotion hook. */
|
|
174
194
|
SetPostSyncSchemaPromotionCallback(callback) {
|