@memberjunction/integration-engine 5.41.0 → 5.43.0

package/dist/FieldMappingEngine.js

@@ -1,25 +1,26 @@
-import { MJLruCache } from '@memberjunction/global';
+import { FieldTransformEngine } from '@memberjunction/global';
+import { computeUnmappedFields } from './CustomOverflow.js';
+import { flattenRecord, hasNestedObject } from './RecordFlatten.js';
 /**
- * Engine responsible for applying field-level mappings and transformations
- * from external records to MJ entity fields.
+ * Engine responsible for applying field-level mappings and transformations from external records to
+ * MJ entity fields.
+ *
+ * The per-value transform pipeline (direct / regex / split / combine / lookup / format / coerce /
+ * substring / custom) is owned by the shared {@link FieldTransformEngine} in `@memberjunction/global`
+ * and reused here, so integration sync and the rules-based bulk-update processor run the *exact same*
+ * transform implementation — one place to fix, one place to extend. This engine keeps only the
+ * integration-specific concerns: source flattening, field-map iteration, and unmapped-field overflow
+ * capture. See the Field Rules guide in `@memberjunction/global` for the shared engine, and
+ * `EntityFieldRules` in `@memberjunction/core` for the metadata-aware entity-update sibling.
  */
 export class FieldMappingEngine {
     constructor() {
-        /**
-         * Cache of compiled custom-expression functions, keyed by the expression string, so an
-         * identical expression is compiled once instead of once per record in a mapping batch.
-         *
-         * A failed compile is cached as the `Error` it threw — a malformed expression is therefore
-         * compiled a single time (then re-thrown from cache on every record) rather than recompiled
-         * on the per-record error path. Bounded via {@link MJLruCache} because the owning
-         * `IntegrationEngine` is a process-lifetime singleton; expression cardinality is normally
-         * tiny (it tracks configured field maps), so the 1000-entry default is comfortably ample.
-         */
-        this.customFunctionCache = new MJLruCache();
+        /** The shared transform pipeline. Holds its own LRU cache of compiled custom expressions. */
+        this.transformEngine = new FieldTransformEngine();
     }
     /**
-     * Applies field mappings to a batch of external records, producing mapped records
-     * ready for match resolution and persistence.
+     * Applies field mappings to a batch of external records, producing mapped records ready for match
+     * resolution and persistence.
      *
      * @param records - External records to map
      * @param fieldMaps - Active field map entities defining source→destination mappings
@@ -34,34 +35,46 @@ export class FieldMappingEngine {
      * Maps a single external record through all active field mappings.
      */
     MapSingleRecord(record, fieldMaps, entityName) {
+        // Flatten nested source objects to scalar columns (e.g. checkin_question.id →
+        // checkin_question_id) BEFORE mapping. The match keys on the mapped PK VALUE
+        // (MatchEngine.FindByKeyFields), so an object-valued source field would produce a
+        // per-occurrence / per-version key → duplicate rows. Discovery flattens identically, so
+        // the field maps reference the flattened scalar names. A record with no nested objects
+        // passes through unchanged — every flat-record connector is a no-op here.
+        const ext = hasNestedObject(record.Fields)
+            ? { ...record, Fields: flattenRecord(record.Fields) }
+            : record;
         const mappedFields = {};
+        const mappedSourceNames = new Set();
         for (const fieldMap of fieldMaps) {
-            const value = this.ApplyFieldMapping(record, fieldMap);
+            mappedSourceNames.add(fieldMap.SourceFieldName);
+            const value = this.ApplyFieldMapping(ext, fieldMap);
             if (value !== undefined) {
                 mappedFields[fieldMap.DestinationFieldName] = value;
             }
         }
+        // Capture the source keys with no field map (gaps.md §2). Cheap O(columns) diff;
+        // the result is empty (and discarded by the writer) in the common all-mapped case,
+        // so this adds no measurable cost to a customs-free sync. The engine parks any extras
+        // in the __mj_integration_CustomOverflow system column — see {@link CustomOverflow}.
+        const unmappedFields = computeUnmappedFields(ext.Fields, mappedSourceNames);
         return {
-            ExternalRecord: record,
+            ExternalRecord: ext,
             MJEntityName: entityName,
             MappedFields: mappedFields,
-            ChangeType: record.IsDeleted ? 'Delete' : 'Create',
+            ChangeType: ext.IsDeleted ? 'Delete' : 'Create',
+            UnmappedFields: unmappedFields,
         };
     }
     /**
-     * Applies a single field mapping, including the full transform pipeline.
-     * Returns undefined if the field should be skipped (OnError: Skip).
+     * Applies a single field mapping, including the full transform pipeline (delegated to the shared
+     * {@link FieldTransformEngine}). Returns undefined if the field should be skipped (OnError: Skip).
      */
     ApplyFieldMapping(record, fieldMap) {
-        let value = record.Fields[fieldMap.SourceFieldName];
+        const value = record.Fields[fieldMap.SourceFieldName];
         const pipeline = this.ParseTransformPipeline(fieldMap.TransformPipeline);
-        for (const step of pipeline) {
-            const result = this.ExecuteTransformStep(step, value, record.Fields);
-            if (result.Skipped)
-                return undefined;
-            value = result.Value;
-        }
-        return value;
+        const result = this.transformEngine.ExecutePipeline(value, record.Fields, pipeline);
+        return result.Skipped ? undefined : result.Value;
     }
     /**
      * Parses the JSON transform pipeline string into typed TransformStep objects.
@@ -79,233 +92,5 @@ export class FieldMappingEngine {
             return [];
         }
     }
-    /**
-     * Executes a single transform step, applying the configured transformation
-     * and handling errors according to the step's OnError strategy.
-     */
-    ExecuteTransformStep(step, value, allFields) {
-        // Grace by default (§8 "still get as much data"): an unconfigured transform that errors on one
-        // record's value nulls that FIELD and lets the record sync, rather than failing the record/batch.
-        // An author can still opt into strict 'Fail' or 'Skip' explicitly per step.
-        const onError = step.OnError ?? 'Null';
-        try {
-            const transformed = this.DispatchTransform(step, value, allFields);
-            return { Value: transformed, Skipped: false };
-        }
-        catch (err) {
-            return this.HandleTransformError(onError, err);
-        }
-    }
-    /**
-     * Dispatches to the appropriate transform handler based on step type.
-     */
-    DispatchTransform(step, value, allFields) {
-        switch (step.Type) {
-            case 'direct':
-                return this.ApplyDirect(value, step.Config);
-            case 'regex':
-                return this.ApplyRegex(value, step.Config);
-            case 'split':
-                return this.ApplySplit(value, step.Config);
-            case 'combine':
-                return this.ApplyCombine(allFields, step.Config);
-            case 'lookup':
-                return this.ApplyLookup(value, step.Config);
-            case 'format':
-                return this.ApplyFormat(value, step.Config);
-            case 'coerce':
-                return this.ApplyCoerce(value, step.Config);
-            case 'substring':
-                return this.ApplySubstring(value, step.Config);
-            case 'custom':
-                return this.ApplyCustom(value, allFields, step.Config);
-            default:
-                throw new Error(`Unknown transform type: ${step.Type}`);
-        }
-    }
-    /**
-     * Handles a transform error according to the OnError strategy.
-     */
-    HandleTransformError(onError, err) {
-        switch (onError) {
-            case 'Skip':
-                return { Value: undefined, Skipped: true };
-            case 'Null':
-                return { Value: null, Skipped: false };
-            case 'Fail':
-                throw err;
-        }
-    }
-    /**
-     * Direct pass-through, applying default value if source is null/undefined.
-     */
-    ApplyDirect(value, config) {
-        if (value == null && config.DefaultValue !== undefined) {
-            return config.DefaultValue;
-        }
-        return value;
-    }
-    /**
-     * Applies a regex replacement to a string value.
-     */
-    ApplyRegex(value, config) {
-        const strValue = String(value ?? '');
-        const regex = new RegExp(config.Pattern, config.Flags ?? '');
-        return strValue.replace(regex, config.Replacement);
-    }
-    /**
-     * Splits a string value and extracts a part by index.
-     */
-    ApplySplit(value, config) {
-        const strValue = String(value ?? '');
-        const parts = strValue.split(config.Delimiter);
-        return parts[config.Index] ?? null;
-    }
-    /**
-     * Combines multiple source fields with a separator.
-     */
-    ApplyCombine(allFields, config) {
-        const values = config.SourceFields.map(f => String(allFields[f] ?? ''));
-        return values.join(config.Separator);
-    }
-    /**
-     * Performs a case-insensitive value lookup/mapping.
-     */
-    ApplyLookup(value, config) {
-        const strValue = String(value ?? '');
-        const lowerKey = strValue.toLowerCase();
-        const entry = Object.entries(config.Map).find(([key]) => key.toLowerCase() === lowerKey);
-        return entry ? entry[1] : (config.Default ?? null);
-    }
-    /**
-     * Applies date/number/string formatting.
-     */
-    ApplyFormat(value, config) {
-        if (config.FormatType === 'date') {
-            const dateVal = value instanceof Date ? value : new Date(String(value));
-            // Throw a CONTROLLED error on an unparseable date — vs the raw RangeError FormatDate→toISOString()
-            // would throw — so ExecuteTransformStep's OnError strategy handles it uniformly (default 'Null').
-            if (isNaN(dateVal.getTime()))
-                throw new Error(`Cannot format "${String(value)}" as a date`);
-            return this.FormatDate(dateVal, config.FormatString);
-        }
-        if (config.FormatType === 'number') {
-            const numVal = Number(value);
-            // Throw on non-numeric — was: emitted the literal string "NaN", which BYPASSED OnError and then
-            // failed SQL numeric binding (corrupting the write). OnError now handles it (default 'Null').
-            if (!Number.isFinite(numVal))
-                throw new Error(`Cannot format "${String(value)}" as a number`);
-            return numVal.toFixed(Number(config.FormatString) || 0);
-        }
-        return String(value ?? '');
-    }
-    /**
-     * Basic date formatting supporting ISO and common format tokens.
-     */
-    FormatDate(date, formatString) {
-        if (formatString.toLowerCase() === 'iso' || formatString === 'ISO8601') {
-            return date.toISOString();
-        }
-        return date.toLocaleDateString('en-US');
-    }
-    /**
-     * Coerces a value to the specified target type.
-     */
-    ApplyCoerce(value, config) {
-        switch (config.TargetType) {
-            case 'string':
-                return String(value ?? '');
-            case 'number':
-                return this.CoerceToNumber(value);
-            case 'boolean':
-                return this.CoerceToBoolean(value);
-            case 'date': {
-                // Throw on unparseable — was: returned an Invalid Date that corrupted the write, bypassing
-                // OnError. OnError handles it now (default 'Null' → field nulled, record syncs).
-                const d = new Date(String(value));
-                if (isNaN(d.getTime()))
-                    throw new Error(`Cannot coerce "${String(value)}" to date`);
-                return d;
-            }
-            default:
-                throw new Error(`Unknown coerce target type: ${config.TargetType}`);
-        }
-    }
-    /**
-     * Coerces a value to a number, throwing on NaN.
-     */
-    CoerceToNumber(value) {
-        const num = Number(value);
-        // Throw on unparseable so OnError decides (default 'Null' → field nulled, record syncs; 'Fail' → strict).
-        // The throw is caught by ExecuteTransformStep and never escapes to fail the record/batch.
-        if (isNaN(num)) {
-            throw new Error(`Cannot coerce "${String(value)}" to number`);
-        }
-        return num;
-    }
-    /**
-     * Coerces a value to boolean, supporting common truthy/falsy strings.
-     */
-    CoerceToBoolean(value) {
-        if (typeof value === 'boolean')
-            return value;
-        if (typeof value === 'number')
-            return value !== 0;
-        const strVal = String(value).toLowerCase().trim();
-        return strVal === 'true' || strVal === '1' || strVal === 'yes';
-    }
-    /**
-     * Extracts a substring from a string value.
-     */
-    ApplySubstring(value, config) {
-        const strValue = String(value ?? '');
-        if (config.Length !== undefined) {
-            return strValue.substring(config.Start, config.Start + config.Length);
-        }
-        return strValue.substring(config.Start);
-    }
-    /**
-     * Evaluates a custom JavaScript expression.
-     * The expression has access to `value` (current field value) and `fields` (all record fields).
-     */
-    ApplyCustom(value, allFields, config) {
-        try {
-            const fn = this.GetCompiledExpression(config.Expression);
-            return fn(value, allFields);
-        }
-        catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            throw new Error(`Custom transform expression failed: ${message}`);
-        }
-    }
-    /**
-     * Returns the compiled function for an expression, compiling and caching on first use.
-     * If the expression failed to compile previously, the cached error is re-thrown — a
-     * malformed expression is compiled exactly once across an entire batch.
-     */
-    GetCompiledExpression(expression) {
-        let cached = this.customFunctionCache.Get(expression);
-        if (cached === undefined) {
-            cached = this.compileExpression(expression);
-            this.customFunctionCache.Set(expression, cached);
-        }
-        if (cached instanceof Error) {
-            throw cached;
-        }
-        return cached;
-    }
-    /**
-     * Compiles a custom expression into a callable, returning the thrown `Error` instead of
-     * propagating it so the caller can cache compile failures alongside successes.
-     */
-    compileExpression(expression) {
-        try {
-            // eslint-disable-next-line @typescript-eslint/no-implied-eval
-            return new Function('value', 'fields', `return (${expression});`);
-        }
-        catch (err) {
-            return err instanceof Error ? err : new Error(String(err));
-        }
-    }
 }
 //# sourceMappingURL=FieldMappingEngine.js.map

package/dist/FieldMappingEngine.js.map

@@ -1 +1 @@
-{"version":3,"file":"FieldMappingEngine.js","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAiBpD;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IAA/B;QACI;;;;;;;;;WASG;QACc,wBAAmB,GAAG,IAAI,UAAU,EAAsC,CAAC;IA2UhG,CAAC;IAzUG;;;;;;;;OAQG;IACI,KAAK,CACR,OAAyB,EACzB,SAAwC,EACxC,UAAkB;QAElB,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACK,eAAe,CACnB,MAAsB,EACtB,SAAwC,EACxC,UAAkB;QAElB,MAAM,YAAY,GAA4B,EAAE,CAAC;QAEjD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC,GAAG,KAAK,CAAC;YACxD,CAAC;QACL,CAAC;QAED,OAAO;YACH,cAAc,EAAE,MAAM;YACtB,YAAY,EAAE,UAAU;YACxB,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;SACrD,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,iBAAiB,CACrB,MAAsB,EACtB,QAAqC;QAErC,IAAI,KAAK,GAAY,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAEzE,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YACrE,IAAI,MAAM,CAAC,OAAO;gBAAE,OAAO,SAAS,CAAC;YACrC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACzB,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,YAA2B;QACtD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAE3D,IAAI,CAAC;YACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,OAAO,EAAE,CAAC;YACtC,OAAO,MAAyB,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,oBAAoB,CACxB,IAAmB,EACnB,KAAc,EACd,SAAkC;QAElC,+FAA+F;QAC/F,kGAAkG;QAClG,4EAA4E;QAC5E,MAAM,OAAO,GAAqB,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC;QAEzD,IAAI,CAAC;YACD,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;YACnE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAClD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,CACrB,IAAmB,EACnB,KAAc,EACd,SAAkC;QAElC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAChB,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,OAAO;gBACR,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAqB,CAAC,CAAC;YAC9D,KAAK,OAAO;gBACR,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAqB,CAAC,CAAC;YAC9D,KAAK,SAAS;gBACV,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,MAAuB,CAAC,CAAC;YACtE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,WAAW;gBACZ,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,MAAyB,CAAC,CAAC;YACtE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAC3E;gBACI,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,OAAyB,EAAE,GAAY;QAChE,QAAQ,OAAO,EAAE,CAAC;YACd,KAAK,MAAM;gBACP,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YAC/C,KAAK,MAAM;gBACP,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAC3C,KAAK,MAAM;gBACP,MAAM,GAAG,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,IAAI,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YACrD,OAAO,MAAM,CAAC,YAAY,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,KAAc,EAAE,MAAmB;QAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,KAAc,EAAE,MAAmB;QAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,SAAkC,EAAE,MAAqB;QAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CACzC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,QAAQ,CAC5C,CAAC;QACF,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,IAAI,MAAM,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,KAAK,YAAY,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACxE,mGAAmG;YACnG,kGAAkG;YAClG,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC5F,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,gGAAgG;YAChG,8FAA8F;YAC9F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9F,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,IAAU,EAAE,YAAoB;QAC/C,IAAI,YAAY,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YACrE,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;YACxB,KAAK,QAAQ;gBACT,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAC/B,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YACtC,KAAK,SAAS;gBACV,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACvC,KAAK,MAAM,CAAC,CAAC,CAAC;gBACV,2FAA2F;gBAC3F,iFAAiF;gBACjF,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAClC,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACpF,OAAO,CAAC,CAAC;YACb,CAAC;YACD;gBACI,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAC5E,CAAC;IACL,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAc;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,0GAA0G;QAC1G,0FAA0F;QAC1F,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,KAAK,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAClD,OAAO,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,KAAK,CAAC;IACnE,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAc,EAAE,MAAuB;QAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACK,WAAW,CACf,KAAc,EACd,SAAkC,EAClC,MAAoB;QAEpB,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACzD,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,UAAkB;QAC5C,IAAI,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAC5C,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,MAAM,YAAY,KAAK,EAAE,CAAC;YAC1B,MAAM,MAAM,CAAC;QACjB,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,iBAAiB,CAAC,UAAkB;QACxC,IAAI,CAAC;YACD,8DAA8D;YAC9D,OAAO,IAAI,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,UAAU,IAAI,CAAuB,CAAC;QAC5F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;CACJ"}
+{"version":3,"file":"FieldMappingEngine.js","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAKpE;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,kBAAkB;IAA/B;QACI,6FAA6F;QAC5E,oBAAe,GAAG,IAAI,oBAAoB,EAAE,CAAC;IA4FlE,CAAC;IA1FG;;;;;;;;OAQG;IACI,KAAK,CACR,OAAyB,EACzB,SAAwC,EACxC,UAAkB;QAElB,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACK,eAAe,CACnB,MAAsB,EACtB,SAAwC,EACxC,UAAkB;QAElB,8EAA8E;QAC9E,6EAA6E;QAC7E,kFAAkF;QAClF,wFAAwF;QACxF,uFAAuF;QACvF,0EAA0E;QAC1E,MAAM,GAAG,GAAmB,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC;YACtD,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;YACrD,CAAC,CAAC,MAAM,CAAC;QAEb,MAAM,YAAY,GAA4B,EAAE,CAAC;QACjD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE5C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;YAChD,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YACpD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC,GAAG,KAAK,CAAC;YACxD,CAAC;QACL,CAAC;QAED,iFAAiF;QACjF,mFAAmF;QACnF,sFAAsF;QACtF,qFAAqF;QACrF,MAAM,cAAc,GAAG,qBAAqB,CAAC,GAAG,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;QAE5E,OAAO;YACH,cAAc,EAAE,GAAG;YACnB,YAAY,EAAE,UAAU;YACxB,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAC/C,cAAc,EAAE,cAAc;SACjC,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,iBAAiB,CACrB,MAAsB,EACtB,QAAqC;QAErC,MAAM,KAAK,GAAY,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpF,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IACrD,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,YAA2B;QACtD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAE3D,IAAI,CAAC;YACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,OAAO,EAAE,CAAC;YACtC,OAAO,MAAyB,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}

package/dist/IntegrationConnectorCreationPipeline.d.ts

@@ -38,6 +38,13 @@ export interface ConnectorCreationPipelineOptions {
     RunID?: string;
     /** Trigger reason recorded in the manifest. */
     TriggerType?: 'Manual' | 'Scheduled' | 'Webhook' | 'Pipeline' | 'Restart';
+    /**
+     * §7 — when true, this is a COMPREHENSIVE re-discovery: declared/discovered objects (and their
+     * fields) ABSENT from this run are deactivated (Status='Disabled', never deleted; reversible on a
+     * later rediscovery). Only set on a full-surface refresh — a scoped/partial discovery must leave it
+     * false so it never disables what it didn't look at. Threaded to PersistDiscoveredSchema.
+     */
+    DeactivateAbsent?: boolean;
 }
 /** Outcome of a single pipeline invocation. */
 export interface ConnectorCreationPipelineResult {
@@ -77,7 +84,33 @@ export interface ConnectorCreationPipelineResult {
  * `__mj.Entity`. The pipeline emits `entity.skipped-no-pk` events for visibility.
  */
 export declare class IntegrationConnectorCreationPipeline {
+    /** In-flight runs by CompanyIntegrationID — coalesces a concurrent duplicate onto the same promise. */
+    private static readonly inFlightRuns;
+    /** Just-completed runs by CompanyIntegrationID — coalesces a *sequential* duplicate within the window. */
+    private static readonly recentRuns;
+    /** Default coalesce window (ms) when the env override is unset/invalid. */
+    private static readonly DEFAULT_COALESCE_WINDOW_MS;
+    /**
+     * How long a just-completed run is reused for a duplicate invocation of the SAME CompanyIntegration.
+     * Sized only to absorb the create-time double-fire (the `IsActive` Save-hook runs the pipeline, then
+     * `IntegrationCreateConnection` calls it again milliseconds later) — short enough that a genuine,
+     * later operator-initiated re-refresh always runs fresh. Override via the
+     * `MJ_CONNECTOR_PIPELINE_COALESCE_WINDOW_MS` env var (a positive integer of milliseconds; set 0/unset
+     * to use the default). Env-based, matching the RSU env-var convention (RSU_WORK_DIR, etc.).
+     */
+    private static get COALESCE_WINDOW_MS();
+    /**
+     * Public entry. De-dups the known create-time double-invocation: the connection's `IsActive`
+     * false→true Save fires the entity-server hook (which runs this pipeline WITH LLM PK inference),
+     * and the create resolver then calls it again. Both converge here, so we run the pipeline ONCE
+     * per CompanyIntegration and hand both callers the same result — no double introspect/persist/
+     * classify, no double live API calls, and the resolver still gets a real summary. A legitimate
+     * re-refresh later (outside the window) runs fresh.
+     */
     Run(opts: ConnectorCreationPipelineOptions): Promise<ConnectorCreationPipelineResult>;
+    /** Drops recent-run entries older than the coalesce window so the map stays bounded. */
+    private static pruneRecentRuns;
+    private runInternal;
     private StageConnectionTest;
     private StageIntrospect;
     private StagePersist;

package/dist/IntegrationConnectorCreationPipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"IntegrationConnectorCreationPipeline.d.ts","sourceRoot":"","sources":["../src/IntegrationConnectorCreationPipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,KAAK,EACR,0BAA0B,EAG7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEH,KAAK,sBAAsB,EAC9B,MAAM,gDAAgD,CAAC;AACxD,OAAO,EAEH,KAAK,kBAAkB,EAC1B,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,OAAO,EAAyB,KAAK,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAC7F,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAE1D,qDAAqD;AACrD,MAAM,WAAW,gCAAgC;IAC7C,uEAAuE;IACvE,SAAS,EAAE,wBAAwB,CAAC;IACpC,mDAAmD;IACnD,kBAAkB,EAAE,0BAA0B,CAAC;IAC/C,8CAA8C;IAC9C,WAAW,EAAE,QAAQ,CAAC;IACtB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,+EAA+E;IAC/E,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,+EAA+E;IAC/E,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,gFAAgF;IAChF,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACpE;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kDAAkD;IAClD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,WAAW,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;CAC7E;AAED,+CAA+C;AAC/C,MAAM,WAAW,+BAA+B;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,EAAE,KAAK,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,+EAA+E;IAC/E,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,wEAAwE;IACxE,QAAQ,EAAE,sBAAsB,CAAC;IACjC,0CAA0C;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,oCAAoC;IAChC,GAAG,CAAC,IAAI,EAAE,gCAAgC,GAAG,OAAO,CAAC,+BAA+B,CAAC;YA4DpF,mBAAmB;YAiBnB,eAAe;YAiCf,YAAY;YAuCZ,eAAe;CAoFhC"}
+{"version":3,"file":"IntegrationConnectorCreationPipeline.d.ts","sourceRoot":"","sources":["../src/IntegrationConnectorCreationPipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,KAAK,EACR,0BAA0B,EAG7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEH,KAAK,sBAAsB,EAC9B,MAAM,gDAAgD,CAAC;AACxD,OAAO,EAEH,KAAK,kBAAkB,EAC1B,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,wBAAwB,EAAuD,MAAM,+BAA+B,CAAC;AAE9H,OAAO,EAAyB,KAAK,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAC7F,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAE1D,qDAAqD;AACrD,MAAM,WAAW,gCAAgC;IAC7C,uEAAuE;IACvE,SAAS,EAAE,wBAAwB,CAAC;IACpC,mDAAmD;IACnD,kBAAkB,EAAE,0BAA0B,CAAC;IAC/C,8CAA8C;IAC9C,WAAW,EAAE,QAAQ,CAAC;IACtB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,+EAA+E;IAC/E,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,+EAA+E;IAC/E,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,gFAAgF;IAChF,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACpE;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kDAAkD;IAClD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,WAAW,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;IAC1E;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,+CAA+C;AAC/C,MAAM,WAAW,+BAA+B;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,EAAE,KAAK,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,+EAA+E;IAC/E,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,wEAAwE;IACxE,QAAQ,EAAE,sBAAsB,CAAC;IACjC,0CAA0C;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,oCAAoC;IAC7C,uGAAuG;IACvG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAA+D;IACnG,0GAA0G;IAC1G,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA8E;IAChH,2EAA2E;IAC3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAQ;IAE1D;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM,KAAK,kBAAkB,GAGpC;IAED;;;;;;;OAOG;IACU,GAAG,CAAC,IAAI,EAAE,gCAAgC,GAAG,OAAO,CAAC,+BAA+B,CAAC;IAuBlG,wFAAwF;IACxF,OAAO,CAAC,MAAM,CAAC,eAAe;YAShB,WAAW;YA4DX,mBAAmB;YAiBnB,eAAe;YAiIf,YAAY;YAyCZ,eAAe;CAoFhC"}

package/dist/IntegrationConnectorCreationPipeline.js

@@ -21,7 +21,65 @@ import { IntegrationSchemaSync } from './IntegrationSchemaSync.js';
  * `__mj.Entity`. The pipeline emits `entity.skipped-no-pk` events for visibility.
  */
 export class IntegrationConnectorCreationPipeline {
+    /** In-flight runs by CompanyIntegrationID — coalesces a concurrent duplicate onto the same promise. */
+    static { this.inFlightRuns = new Map(); }
+    /** Just-completed runs by CompanyIntegrationID — coalesces a *sequential* duplicate within the window. */
+    static { this.recentRuns = new Map(); }
+    /** Default coalesce window (ms) when the env override is unset/invalid. */
+    static { this.DEFAULT_COALESCE_WINDOW_MS = 5000; }
+    /**
+     * How long a just-completed run is reused for a duplicate invocation of the SAME CompanyIntegration.
+     * Sized only to absorb the create-time double-fire (the `IsActive` Save-hook runs the pipeline, then
+     * `IntegrationCreateConnection` calls it again milliseconds later) — short enough that a genuine,
+     * later operator-initiated re-refresh always runs fresh. Override via the
+     * `MJ_CONNECTOR_PIPELINE_COALESCE_WINDOW_MS` env var (a positive integer of milliseconds; set 0/unset
+     * to use the default). Env-based, matching the RSU env-var convention (RSU_WORK_DIR, etc.).
+     */
+    static get COALESCE_WINDOW_MS() {
+        const override = Number(process.env.MJ_CONNECTOR_PIPELINE_COALESCE_WINDOW_MS);
+        return Number.isFinite(override) && override > 0 ? override : IntegrationConnectorCreationPipeline.DEFAULT_COALESCE_WINDOW_MS;
+    }
+    /**
+     * Public entry. De-dups the known create-time double-invocation: the connection's `IsActive`
+     * false→true Save fires the entity-server hook (which runs this pipeline WITH LLM PK inference),
+     * and the create resolver then calls it again. Both converge here, so we run the pipeline ONCE
+     * per CompanyIntegration and hand both callers the same result — no double introspect/persist/
+     * classify, no double live API calls, and the resolver still gets a real summary. A legitimate
+     * re-refresh later (outside the window) runs fresh.
+     */
     async Run(opts) {
+        const ciID = opts.CompanyIntegration?.ID;
+        if (!ciID)
+            return this.runInternal(opts); // no key to de-dup on — run directly
+        const cls = IntegrationConnectorCreationPipeline;
+        const inFlight = cls.inFlightRuns.get(ciID);
+        if (inFlight)
+            return inFlight; // a concurrent run is already going — share it
+        cls.pruneRecentRuns();
+        const recent = cls.recentRuns.get(ciID);
+        if (recent)
+            return recent.result; // a run just completed (within the window) — reuse it
+        const promise = this.runInternal(opts);
+        cls.inFlightRuns.set(ciID, promise);
+        try {
+            const result = await promise;
+            cls.recentRuns.set(ciID, { result, at: Date.now() });
+            return result;
+        }
+        finally {
+            cls.inFlightRuns.delete(ciID);
+        }
+    }
+    /** Drops recent-run entries older than the coalesce window so the map stays bounded. */
+    static pruneRecentRuns() {
+        const now = Date.now();
+        for (const [key, entry] of IntegrationConnectorCreationPipeline.recentRuns) {
+            if (now - entry.at >= IntegrationConnectorCreationPipeline.COALESCE_WINDOW_MS) {
+                IntegrationConnectorCreationPipeline.recentRuns.delete(key);
+            }
+        }
+    }
+    async runInternal(opts) {
         const runID = opts.RunID ?? IntegrationProgressEmitter.newRunID('connector');
         const manifest = {
             runID,
@@ -95,6 +153,103 @@ export class IntegrationConnectorCreationPipeline {
         const startMs = Date.now();
         try {
             const schema = await opts.Connector.IntrospectSchema(opts.CompanyIntegration, opts.ContextUser, opts.IntrospectOptions);
+            // UNIVERSAL additive runtime-object discovery — the single chokepoint EVERY connector
+            // funnels through, regardless of which base it extends or whether that base's
+            // IntrospectSchema only reflects already-declared metadata. IntrospectSchema gives the
+            // rich declared/persisted catalog; we then ADD any objects the connector surfaces at
+            // runtime that aren't already in it, via the abstract DiscoverObjects/DiscoverFields
+            // primitives that EVERY connector implements (so a future connector on a different base
+            // can't silently lose runtime discovery). PersistDiscoveredSchema is additive, so
+            // declared objects are preserved and runtime-only objects (e.g. an auth-gated file
+            // feed's streams) get created as Discovered. Errors are SURFACED, never swallowed.
+            const seen = new Set(schema.Objects.map(o => o.ExternalName.toLowerCase()));
+            let runtimeObjects = [];
+            try {
+                runtimeObjects = await opts.Connector.DiscoverObjects(opts.CompanyIntegration, opts.ContextUser);
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                emitter.stageError('Introspect', `DiscoverObjects failed: ${msg}`, { code: 'discover-objects-failed' });
+                console.error(`[IntrospectPipeline] DiscoverObjects failed: ${msg}`);
+            }
+            let runtimeAdded = 0;
+            for (const d of runtimeObjects) {
+                const key = d.Name.toLowerCase();
+                if (seen.has(key)) {
+                    // §case-3 (data-only-discoverable): a DECLARED object the connector ALSO surfaces at
+                    // runtime, but the declared form carries NO fields (e.g. a file-feed stream declared by
+                    // NAME only — its columns are knowable solely from the records). Without this, the loop
+                    // skipped it, it stayed field-less + PK-less, and ApplyAll dropped it — so the user's data
+                    // for that object never landed. Discover its fields over the read path and populate the
+                    // existing declared object IN PLACE so it becomes syncable.
+                    const existing = schema.Objects.find(o => o.ExternalName.toLowerCase() === key);
+                    if (existing && existing.Fields.length === 0) {
+                        try {
+                            const dfields = await opts.Connector.DiscoverFieldsViaFetch(opts.CompanyIntegration, d.Name, opts.ContextUser);
+                            existing.Fields = dfields.map(f => ({
+                                Name: f.Name, Label: f.Label, Description: f.Description, SourceType: f.DataType,
+                                IsRequired: f.IsRequired, AllowsNull: f.AllowsNull, MaxLength: f.MaxLength ?? null,
+                                Precision: f.Precision ?? null, Scale: f.Scale ?? null, DefaultValue: f.DefaultValue ?? null,
+                                IsPrimaryKey: f.IsPrimaryKey ?? false, IsUniqueKey: f.IsUniqueKey, IsReadOnly: f.IsReadOnly,
+                                IsForeignKey: f.IsForeignKey ?? false, ForeignKeyTarget: f.ForeignKeyTarget ?? null,
+                            }));
+                            existing.PrimaryKeyFields = dfields.filter(f => f.IsPrimaryKey).map(f => f.Name);
+                            existing.Relationships = dfields
+                                .filter(f => (f.IsForeignKey ?? false) && f.ForeignKeyTarget)
+                                .map(f => ({ FieldName: f.Name, TargetObject: f.ForeignKeyTarget, TargetField: 'ID' }));
+                            console.log(`[IntrospectPipeline] declared field-less object "${d.Name}" → discovered ${dfields.length} fields via read path`);
+                        }
+                        catch (err) {
+                            const msg = err instanceof Error ? err.message : String(err);
+                            emitter.stageError('Introspect', `DiscoverFieldsViaFetch failed for declared field-less "${d.Name}": ${msg}`, { code: 'discover-fields-failed' });
+                            console.error(`[IntrospectPipeline] DiscoverFieldsViaFetch failed for declared "${d.Name}": ${msg}`);
+                        }
+                    }
+                    continue;
+                }
+                seen.add(key);
+                let fields = [];
+                try {
+                    // Discover fields + PROVABLE PK over the READ PATH (FetchChanges), time-bounded and
+                    // SAVE-LESS — stream as much of the feed as the budget allows so the PK decision is
+                    // made on a statistically-significant sample, not a single (possibly tiny) file. No
+                    // DB write happens here; the real save is the later ApplyAll → StartSync.
+                    fields = await opts.Connector.DiscoverFieldsViaFetch(opts.CompanyIntegration, d.Name, opts.ContextUser);
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    emitter.stageError('Introspect', `DiscoverFieldsViaFetch failed for "${d.Name}": ${msg}`, { code: 'discover-fields-failed' });
+                    console.error(`[IntrospectPipeline] DiscoverFieldsViaFetch failed for "${d.Name}": ${msg}`);
+                }
+                schema.Objects.push({
+                    ExternalName: d.Name,
+                    ExternalLabel: d.Label,
+                    Description: d.Description,
+                    Fields: fields.map(f => ({
+                        Name: f.Name,
+                        Label: f.Label,
+                        Description: f.Description,
+                        SourceType: f.DataType,
+                        IsRequired: f.IsRequired,
+                        AllowsNull: f.AllowsNull,
+                        MaxLength: f.MaxLength ?? null,
+                        Precision: f.Precision ?? null,
+                        Scale: f.Scale ?? null,
+                        DefaultValue: f.DefaultValue ?? null,
+                        IsPrimaryKey: f.IsPrimaryKey ?? false,
+                        IsUniqueKey: f.IsUniqueKey,
+                        IsReadOnly: f.IsReadOnly,
+                        IsForeignKey: f.IsForeignKey ?? false,
+                        ForeignKeyTarget: f.ForeignKeyTarget ?? null,
+                    })),
+                    PrimaryKeyFields: fields.filter(f => f.IsPrimaryKey).map(f => f.Name),
+                    Relationships: fields
+                        .filter(f => (f.IsForeignKey ?? false) && f.ForeignKeyTarget)
+                        .map(f => ({ FieldName: f.Name, TargetObject: f.ForeignKeyTarget, TargetField: 'ID' })),
+                });
+                runtimeAdded++;
+            }
+            console.log(`[IntrospectPipeline] declared=${seen.size - runtimeAdded} runtime-added=${runtimeAdded} total=${schema.Objects.length}`);
             const fieldCount = schema.Objects.reduce((acc, o) => acc + o.Fields.length, 0);
             emitter.stageComplete('Introspect', {
                 processed: schema.Objects.length,
@@ -124,6 +279,8 @@ export class IntegrationConnectorCreationPipeline {
             ContextUser: opts.ContextUser,
             Provider: opts.Provider,
             UseTransactionGroup: true,
+            // §7 comprehensive-refresh deactivation (objects + fields absent from this discovery).
+            DeactivateAbsent: opts.DeactivateAbsent ?? false,
         });
         // Emit per-object + per-field structural-transparency events so the UI/audit
         // trail captures WHO won each attribute.