@memberjunction/integration-engine 5.40.2 → 5.42.0

package/dist/CustomColumnPromotion.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CustomColumnPromotion.d.ts","sourceRoot":"","sources":["../src/CustomColumnPromotion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,0FAA0F;AAC1F,MAAM,WAAW,kBAAkB;IAC/B,4FAA4F;IAC5F,eAAe,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,CAAC;IAC9D,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;IACtB,qFAAqF;IACrF,YAAY,EAAE,MAAM,CAAC;IACrB,iFAAiF;IACjF,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,2GAA2G;AAC3G,MAAM,WAAW,gBAAgB;IAC7B,yDAAyD;IACzD,GAAG,EAAE,MAAM,CAAC;IACZ,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,YAAY,EAAE,OAAO,EAAE,CAAC;CAC3B;AAED,uEAAuE;AACvE,MAAM,WAAW,kBAAkB;IAC/B,yFAAyF;IACzF,GAAG,EAAE,MAAM,CAAC;IACZ,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,QAAQ,EAAE,kBAAkB,CAAC;CAChC;AAED,mDAAmD;AACnD,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAC7C;AAmBD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC1B,KAAK,EAAE,gBAAgB,EAAE,EACzB,IAAI,GAAE,oBAAyB,GAChC,kBAAkB,EAAE,CAsBtB;AAED;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,kBAAkB,CA4BjF;AAwCD;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAC9B,mBAAmB,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,EACrD,SAAS,GAAE,MAA2B,GACvC,gBAAgB,EAAE,CA0BpB;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKtD"}

package/dist/CustomColumnPromotion.js

@@ -0,0 +1,202 @@
+/**
+ * Promotion planner — the brain of the post-sync custom-column step (gaps.md §2, M2).
+ *
+ * After a sync, the values a source returned with no field map are parked as JSON in the
+ * {@link CUSTOM_OVERFLOW_COLUMN} system column (M1 capture). This module decides, from a
+ * coverage scan of that column, WHICH of those keys earn a real column and WHAT bounded
+ * type each should get. It is PURE — no DB, no RSU, no I/O — so the genuinely new logic is
+ * fully unit-testable. The server-side orchestrator (M2b) feeds the resulting plan into the
+ * EXISTING refresh pipeline (SchemaEvolution ADD COLUMN → RSU → IntegrationSchemaSync IOF →
+ * field map); this module never touches schema itself.
+ *
+ * Design rules (from gaps.md §2 + the connector-code conventions):
+ * - Provable-only / never fabricate: a key earns a column only on PERVASIVENESS evidence
+ *   (coverage ≥ threshold), never on a single malformed row. PK/FK are NEVER inferred here —
+ *   customs are always emitted nullable, non-PK, non-FK (classification is deferred to D4).
+ * - Generous bounded typing (the NVARCHAR(MAX) problem): size columns comfortably from the
+ *   observed samples and err LARGER — a roomy bounded column beats a truncating tight one,
+ *   and both beat MAX. Only fall back to MAX/TEXT when the observed length genuinely can't be
+ *   bounded. Narrow to number/boolean/datetime ONLY when every non-null sample unambiguously
+ *   supports it; otherwise default to a (generously bounded) string — that is the safe choice
+ *   that can hold anything the source later returns.
+ * - Terminate / never re-promote: a key whose column already exists is skipped, so the
+ *   capture→promote loop converges instead of re-promoting forever.
+ */
+const DEFAULT_COVERAGE_THRESHOLD = 0; // §23 — presence-based: a key seen in even one row earns a column.
+/** SQL Server's largest bounded NVARCHAR before MAX; above this we must use MAX/TEXT. */
+const MAX_BOUNDED_STRING = 4000;
+/** Floor for a generous string bound — never size a custom string column tighter than this. */
+const MIN_STRING_BOUND = 255;
+/**
+ * Minimum all-integer sample before we trust the field is INTEGRAL and narrow to BIGINT (#A9). Below this
+ * a handful of integer samples does NOT prove the unscanned tail is integer-only — a later decimal would be
+ * silently rejected by BIGINT (dead-lettered). Under the threshold we use wide fixed-point (which holds
+ * integers too), so a later decimal never fails; the column can still re-promote to BIGINT once enough
+ * integer evidence accrues.
+ */
+const MIN_SAMPLE_FOR_INTEGER_NARROW = 12;
+/** Matches an ISO-8601-ish date / datetime so we don't mistake "5" or "2020" for a date. */
+const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;
+/**
+ * Plans which overflow keys to promote to real columns. Pure; deterministic (sorted by key).
+ *
+ * @param stats - per-key coverage statistics from the overflow scan
+ * @param opts  - coverage threshold + the set of already-existing column names
+ * @returns the promotion candidates, sorted by key for stable, replayable output
+ */
+export function planPromotions(stats, opts = {}) {
+    const threshold = opts.CoverageThreshold ?? DEFAULT_COVERAGE_THRESHOLD;
+    const existing = lowercaseSet(opts.ExistingColumnNames);
+    const candidates = [];
+    for (const stat of stats) {
+        // Terminate / never re-promote: the column already exists.
+        if (existing.has(stat.Key.toLowerCase()))
+            continue;
+        // Guard div-by-zero only; §23 — presence is enough (default threshold 0), so a key seen in
+        // even one row earns a column. A higher bar is opt-in via CoverageThreshold.
+        if (stat.TotalRows <= 0)
+            continue;
+        const coverage = stat.Occurrences / stat.TotalRows;
+        if (coverage < threshold)
+            continue;
+        candidates.push({
+            Key: stat.Key,
+            Coverage: coverage,
+            Inferred: inferColumnTypeFromSamples(stat.SampleValues),
+        });
+    }
+    return candidates.sort((a, b) => a.Key.localeCompare(b.Key));
+}
+/**
+ * Infers a generously-bounded column type from observed sample values. Narrows to
+ * boolean/number/datetime ONLY when EVERY non-null sample unambiguously supports it;
+ * otherwise defaults to a comfortably-bounded string. Never returns MAX/TEXT unless the
+ * observed string length genuinely can't be bounded.
+ */
+export function inferColumnTypeFromSamples(samples) {
+    const nonNull = samples.filter(v => v !== null && v !== undefined);
+    // No evidence → a safe, generous default string.
+    if (nonNull.length === 0)
+        return stringType(MIN_STRING_BOUND);
+    if (nonNull.every(isBoolean)) {
+        return { SchemaFieldType: 'boolean', SqlServerType: 'BIT', PostgresType: 'BOOLEAN', MaxLength: null };
+    }
+    if (nonNull.every(isFiniteNumber)) {
+        // Narrow to BIGINT only with an ADEQUATE all-integer sample (#A9). A small all-integer sample
+        // doesn't prove the field is always integral — a decimal in the unscanned tail would be silently
+        // rejected by BIGINT. Below the threshold, use wide fixed-point (which holds integers too).
+        const confidentInteger = nonNull.length >= MIN_SAMPLE_FOR_INTEGER_NARROW && nonNull.every(v => Number.isInteger(v));
+        return confidentInteger
+            ? { SchemaFieldType: 'number', SqlServerType: 'BIGINT', PostgresType: 'BIGINT', MaxLength: null }
+            // Generous fixed-point — wide precision so we don't truncate or reject decimals.
+            : { SchemaFieldType: 'number', SqlServerType: 'DECIMAL(38,10)', PostgresType: 'DECIMAL(38,10)', MaxLength: null };
+    }
+    if (nonNull.every(isIsoDateString)) {
+        return { SchemaFieldType: 'datetime', SqlServerType: 'DATETIMEOFFSET', PostgresType: 'TIMESTAMPTZ', MaxLength: null };
+    }
+    // Default: a generously-bounded string sized to comfortably hold the longest observed value.
+    const longest = Math.max(...nonNull.map(v => String(v).length));
+    return stringType(generousStringBound(longest));
+}
+/** Builds a string {@link InferredColumnType}; bound===null ⇒ unbounded (MAX/TEXT). */
+function stringType(bound) {
+    if (bound === null) {
+        return { SchemaFieldType: 'string', SqlServerType: 'NVARCHAR(MAX)', PostgresType: 'TEXT', MaxLength: null };
+    }
+    return { SchemaFieldType: 'string', SqlServerType: `NVARCHAR(${bound})`, PostgresType: `VARCHAR(${bound})`, MaxLength: bound };
+}
+/**
+ * Generous bound for a string column: double the longest observed length (room to grow),
+ * floored at {@link MIN_STRING_BOUND}; if that would exceed the bounded limit, fall back to
+ * unbounded (null → MAX/TEXT) since it genuinely can't be bounded safely.
+ */
+function generousStringBound(longestObserved) {
+    const doubled = Math.max(longestObserved * 2, MIN_STRING_BOUND);
+    return doubled > MAX_BOUNDED_STRING ? null : doubled;
+}
+function isBoolean(v) {
+    return typeof v === 'boolean';
+}
+function isFiniteNumber(v) {
+    return typeof v === 'number' && Number.isFinite(v);
+}
+function isIsoDateString(v) {
+    return typeof v === 'string' && ISO_DATE_RE.test(v) && !Number.isNaN(Date.parse(v));
+}
+function lowercaseSet(set) {
+    if (!set || set.size === 0)
+        return new Set();
+    return new Set([...set].map(s => s.toLowerCase()));
+}
+/** Max distinct sample values retained per key for type inference (bounded memory). */
+const DEFAULT_SAMPLE_CAP = 20;
+/**
+ * Builds per-key coverage statistics from a SAMPLE of rows that carried overflow JSON.
+ * Each row's overflow column is a JSON object string (what M1 parked); this tallies, per key,
+ * how many sampled rows had it non-null and collects a bounded sample of its values for type
+ * inference. Robust to malformed/empty JSON (such rows contribute to TotalRows but no keys).
+ *
+ * Coverage is therefore computed over the SAMPLE (bounded memory at any table size) — a
+ * statistically sound basis for the pervasiveness decision; the exactness the threshold needs
+ * (a key on ~half the rows vs. a junk key on one) survives sampling.
+ *
+ * @param overflowJsonStrings - the raw overflow-column value from each sampled row
+ * @param sampleCap - max values retained per key (default {@link DEFAULT_SAMPLE_CAP})
+ */
+export function buildOverflowStats(overflowJsonStrings, sampleCap = DEFAULT_SAMPLE_CAP) {
+    const byKey = new Map();
+    let totalRows = 0;
+    for (const raw of overflowJsonStrings) {
+        totalRows++;
+        const parsed = safeParseObject(raw);
+        if (!parsed)
+            continue;
+        for (const [key, value] of Object.entries(parsed)) {
+            if (value === null || value === undefined)
+                continue;
+            let entry = byKey.get(key);
+            if (!entry) {
+                entry = { occurrences: 0, samples: [] };
+                byKey.set(key, entry);
+            }
+            entry.occurrences++;
+            if (entry.samples.length < sampleCap)
+                entry.samples.push(value);
+        }
+    }
+    return [...byKey.entries()].map(([key, e]) => ({
+        Key: key,
+        Occurrences: e.occurrences,
+        TotalRows: totalRows,
+        SampleValues: e.samples,
+    }));
+}
+/**
+ * Sanitizes a source key into a safe SQL/MJ column identifier (letters, digits, underscore;
+ * never leading-digit; bounded length). The original key is kept as the field map's
+ * SourceFieldName; the sanitized form becomes the column + DestinationFieldName. Returns the
+ * BASE name only — collision resolution (suffixing) is the orchestrator's job since it needs
+ * the existing-column context.
+ */
+export function sanitizeColumnName(key) {
+    let name = key.replace(/[^A-Za-z0-9_]/g, '_').replace(/_+/g, '_').replace(/^_+|_+$/g, '');
+    if (name.length === 0)
+        name = 'Custom';
+    if (/^[0-9]/.test(name))
+        name = `c_${name}`;
+    return name.length > 120 ? name.slice(0, 120) : name;
+}
+function safeParseObject(raw) {
+    if (!raw || typeof raw !== 'string')
+        return null;
+    try {
+        const parsed = JSON.parse(raw);
+        return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+            ? parsed
+            : null;
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=CustomColumnPromotion.js.map

package/dist/CustomColumnPromotion.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CustomColumnPromotion.js","sourceRoot":"","sources":["../src/CustomColumnPromotion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAoDH,MAAM,0BAA0B,GAAG,CAAC,CAAC,CAAC,mEAAmE;AACzG,yFAAyF;AACzF,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,+FAA+F;AAC/F,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B;;;;;;GAMG;AACH,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEzC,4FAA4F;AAC5F,MAAM,WAAW,GAAG,6EAA6E,CAAC;AAElG;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC1B,KAAyB,EACzB,OAA6B,EAAE;IAE/B,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,IAAI,0BAA0B,CAAC;IACvE,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACxD,MAAM,UAAU,GAAyB,EAAE,CAAC;IAE5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,2DAA2D;QAC3D,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACnD,2FAA2F;QAC3F,6EAA6E;QAC7E,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC;YAAE,SAAS;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;QACnD,IAAI,QAAQ,GAAG,SAAS;YAAE,SAAS;QAEnC,UAAU,CAAC,IAAI,CAAC;YACZ,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,QAAQ,EAAE,QAAQ;YAClB,QAAQ,EAAE,0BAA0B,CAAC,IAAI,CAAC,YAAY,CAAC;SAC1D,CAAC,CAAC;IACP,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAAkB;IACzD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC;IAEnE,iDAAiD;IACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,UAAU,CAAC,gBAAgB,CAAC,CAAC;IAE9D,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1G,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAChC,8FAA8F;QAC9F,iGAAiG;QACjG,4FAA4F;QAC5F,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,IAAI,6BAA6B,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAW,CAAC,CAAC,CAAC;QAC9H,OAAO,gBAAgB;YACnB,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE;YACjG,iFAAiF;YACjF,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1H,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;QACjC,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1H,CAAC;IAED,6FAA6F;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,OAAO,UAAU,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,uFAAuF;AACvF,SAAS,UAAU,CAAC,KAAoB;IACpC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACjB,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAChH,CAAC;IACD,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,YAAY,KAAK,GAAG,EAAE,YAAY,EAAE,WAAW,KAAK,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACnI,CAAC;AAED;;;;GAIG;AACH,SAAS,mBAAmB,CAAC,eAAuB;IAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAChE,OAAO,OAAO,GAAG,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;AACzD,CAAC;AAED,SAAS,SAAS,CAAC,CAAU;IACzB,OAAO,OAAO,CAAC,KAAK,SAAS,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CAAC,CAAU;IAC9B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,eAAe,CAAC,CAAU;IAC/B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,YAAY,CAAC,GAAoC;IACtD,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,GAAG,EAAU,CAAC;IACrD,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,uFAAuF;AACvF,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,kBAAkB,CAC9B,mBAAqD,EACrD,YAAoB,kBAAkB;IAEtC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuD,CAAC;IAC7E,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;QACpC,SAAS,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS;YACpD,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,KAAK,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;gBACxC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC1B,CAAC;YACD,KAAK,CAAC,WAAW,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS;gBAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpE,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,GAAG,EAAE,GAAG;QACR,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,SAAS,EAAE,SAAS;QACpB,YAAY,EAAE,CAAC,CAAC,OAAO;KAC1B,CAAC,CAAC,CAAC;AACR,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC1C,IAAI,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC1F,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,IAAI,GAAG,QAAQ,CAAC;IACvC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;IAC5C,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACzD,CAAC;AAED,SAAS,eAAe,CAAC,GAA8B;IACnD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACjD,IAAI,CAAC;QACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YACjE,CAAC,CAAE,MAAkC;YACrC,CAAC,CAAC,IAAI,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC"}

package/dist/CustomOverflow.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Name of the per-record custom-overflow column. Written on every integration table
+ * alongside the other `__mj_integration_*` columns when present. Holds a JSON object of
+ * the source fields a record returned that have NO field map — the "extra" keys the
+ * target table doesn't (yet) have a column for.
+ *
+ * This is the capture half of framework-level custom-column support (gaps.md §2). A
+ * source with no schema/describe endpoint (the PropFuel class) can only learn its full
+ * field set by reading data; a static catalog then silently drops any extra key at
+ * {@link MapSingleRecord} (only mapped fields get persisted). Rather than lose them, the
+ * unmapped keys are parked here, on the row, in the SAME write as the mapped fields — so
+ * capture costs no extra round-trip. A post-sync Runtime-Schema-Updation (RSU) pass later
+ * reads this column back out and promotes pervasive keys to real columns (ADD COLUMN +
+ * CodeGen + a field map). It does NOT backfill historical values: once the field map exists
+ * the key is no longer "unmapped", so the NEXT sync maps it natively and populates the new
+ * column with the LIVE source value (a full sync repopulates every row; an incremental fills
+ * rows as they next change) — and stops re-capturing it here. The loop converges go-forward.
+ *
+ * Shape mirrors {@link CONTENT_HASH_COLUMN}: a JSON-serialized string sidecar, identical
+ * in kind to `__mj_integration_LastSyncedSnapshot`. SS `NVARCHAR(MAX)` / PG `TEXT`; the
+ * post-sync coverage scan + spread cast `::jsonb` on Postgres when they query it.
+ *
+ * This is BACKEND PLUMBING, not user-facing metadata. It is a SYSTEM field — in the same
+ * hidden/internal class as `__mj_integration_ContentHash` & the other `__mj_integration_*`
+ * columns: metadata-registered only so the engine's `entity.Set(...)`/`entity.Fields` write
+ * path is permitted to touch it (so it is NOT an out-of-band "phantom" column the engine
+ * writes blind), but NOT surfaced or managed through the metadata-driven framework — not in
+ * forms, not user-mappable, not "a field" anyone works with. It is machinery.
+ *
+ * The things that genuinely ENTER the metadata-driven framework are the columns this gets
+ * PROMOTED into: a pervasive key here is turned into a real user-facing, mappable EntityField
+ * (IntegrationSchemaSync IOF → RSU ADD COLUMN → CodeGen → md.Refresh) BEFORE the engine maps
+ * to it. So: backend staging until promotion, first-class metadata-driven schema after. Only
+ * the JSON *values* parked here are ever transient.
+ *
+ * Gated everywhere by EntityInfo field presence, so it is a no-op on tables that predate
+ * the column. CRUCIAL invariant: when a record has NO unmapped keys, NOTHING is written
+ * here — empty across all rows is the signal that no RSU pass is needed, which is what
+ * keeps a customs-free sync byte-identical to today (single-stage, zero overhead).
+ */
+export declare const CUSTOM_OVERFLOW_COLUMN = "__mj_integration_CustomOverflow";
+/**
+ * Computes the unmapped subset of a record's external fields: every key the source
+ * returned that is NOT the SourceFieldName of any active field map.
+ *
+ * Pure + O(columns) — no per-row allocation beyond the result object, which is empty
+ * (and discarded by the caller) for the common no-customs case.
+ *
+ * Honest limit (documented in gaps.md §2): a field consumed only indirectly — e.g. as a
+ * `combine`/`custom` transform input rather than as a map's own SourceFieldName — will
+ * also appear here. That is harmless (the value is still in `Fields`); the post-sync
+ * promotion stage applies a coverage + novelty filter before it ever creates a column.
+ *
+ * @param externalFields - the full source record (`ExternalRecord.Fields`)
+ * @param mappedSourceFieldNames - SourceFieldName of every active field map
+ * @returns the extra keys + values, or an empty object when everything was mapped
+ */
+export declare function computeUnmappedFields(externalFields: Record<string, unknown>, mappedSourceFieldNames: ReadonlySet<string>): Record<string, unknown>;
+/**
+ * Whether a computed unmapped-field object carries anything worth persisting. Used by
+ * both the per-record write gate and (later) the post-sync RSU trigger gate.
+ */
+export declare function hasUnmappedFields(unmapped: Record<string, unknown> | undefined | null): boolean;
+//# sourceMappingURL=CustomOverflow.d.ts.map

package/dist/CustomOverflow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CustomOverflow.d.ts","sourceRoot":"","sources":["../src/CustomOverflow.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,eAAO,MAAM,sBAAsB,oCAAoC,CAAC;AAExE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CACjC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACvC,sBAAsB,EAAE,WAAW,CAAC,MAAM,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAQzB;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI,GAAG,OAAO,CAE/F"}

package/dist/CustomOverflow.js

@@ -0,0 +1,74 @@
+/**
+ * Name of the per-record custom-overflow column. Written on every integration table
+ * alongside the other `__mj_integration_*` columns when present. Holds a JSON object of
+ * the source fields a record returned that have NO field map — the "extra" keys the
+ * target table doesn't (yet) have a column for.
+ *
+ * This is the capture half of framework-level custom-column support (gaps.md §2). A
+ * source with no schema/describe endpoint (the PropFuel class) can only learn its full
+ * field set by reading data; a static catalog then silently drops any extra key at
+ * {@link MapSingleRecord} (only mapped fields get persisted). Rather than lose them, the
+ * unmapped keys are parked here, on the row, in the SAME write as the mapped fields — so
+ * capture costs no extra round-trip. A post-sync Runtime-Schema-Updation (RSU) pass later
+ * reads this column back out and promotes pervasive keys to real columns (ADD COLUMN +
+ * CodeGen + a field map). It does NOT backfill historical values: once the field map exists
+ * the key is no longer "unmapped", so the NEXT sync maps it natively and populates the new
+ * column with the LIVE source value (a full sync repopulates every row; an incremental fills
+ * rows as they next change) — and stops re-capturing it here. The loop converges go-forward.
+ *
+ * Shape mirrors {@link CONTENT_HASH_COLUMN}: a JSON-serialized string sidecar, identical
+ * in kind to `__mj_integration_LastSyncedSnapshot`. SS `NVARCHAR(MAX)` / PG `TEXT`; the
+ * post-sync coverage scan + spread cast `::jsonb` on Postgres when they query it.
+ *
+ * This is BACKEND PLUMBING, not user-facing metadata. It is a SYSTEM field — in the same
+ * hidden/internal class as `__mj_integration_ContentHash` & the other `__mj_integration_*`
+ * columns: metadata-registered only so the engine's `entity.Set(...)`/`entity.Fields` write
+ * path is permitted to touch it (so it is NOT an out-of-band "phantom" column the engine
+ * writes blind), but NOT surfaced or managed through the metadata-driven framework — not in
+ * forms, not user-mappable, not "a field" anyone works with. It is machinery.
+ *
+ * The things that genuinely ENTER the metadata-driven framework are the columns this gets
+ * PROMOTED into: a pervasive key here is turned into a real user-facing, mappable EntityField
+ * (IntegrationSchemaSync IOF → RSU ADD COLUMN → CodeGen → md.Refresh) BEFORE the engine maps
+ * to it. So: backend staging until promotion, first-class metadata-driven schema after. Only
+ * the JSON *values* parked here are ever transient.
+ *
+ * Gated everywhere by EntityInfo field presence, so it is a no-op on tables that predate
+ * the column. CRUCIAL invariant: when a record has NO unmapped keys, NOTHING is written
+ * here — empty across all rows is the signal that no RSU pass is needed, which is what
+ * keeps a customs-free sync byte-identical to today (single-stage, zero overhead).
+ */
+export const CUSTOM_OVERFLOW_COLUMN = '__mj_integration_CustomOverflow';
+/**
+ * Computes the unmapped subset of a record's external fields: every key the source
+ * returned that is NOT the SourceFieldName of any active field map.
+ *
+ * Pure + O(columns) — no per-row allocation beyond the result object, which is empty
+ * (and discarded by the caller) for the common no-customs case.
+ *
+ * Honest limit (documented in gaps.md §2): a field consumed only indirectly — e.g. as a
+ * `combine`/`custom` transform input rather than as a map's own SourceFieldName — will
+ * also appear here. That is harmless (the value is still in `Fields`); the post-sync
+ * promotion stage applies a coverage + novelty filter before it ever creates a column.
+ *
+ * @param externalFields - the full source record (`ExternalRecord.Fields`)
+ * @param mappedSourceFieldNames - SourceFieldName of every active field map
+ * @returns the extra keys + values, or an empty object when everything was mapped
+ */
+export function computeUnmappedFields(externalFields, mappedSourceFieldNames) {
+    const unmapped = {};
+    for (const key of Object.keys(externalFields)) {
+        if (!mappedSourceFieldNames.has(key)) {
+            unmapped[key] = externalFields[key];
+        }
+    }
+    return unmapped;
+}
+/**
+ * Whether a computed unmapped-field object carries anything worth persisting. Used by
+ * both the per-record write gate and (later) the post-sync RSU trigger gate.
+ */
+export function hasUnmappedFields(unmapped) {
+    return unmapped != null && Object.keys(unmapped).length > 0;
+}
+//# sourceMappingURL=CustomOverflow.js.map

package/dist/CustomOverflow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CustomOverflow.js","sourceRoot":"","sources":["../src/CustomOverflow.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,iCAAiC,CAAC;AAExE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,qBAAqB,CACjC,cAAuC,EACvC,sBAA2C;IAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,QAAQ,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAoD;IAClF,OAAO,QAAQ,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AAChE,CAAC"}

package/dist/FieldMappingEngine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"FieldMappingEngine.d.ts","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAe/D;;;GAGG;AACH,qBAAa,kBAAkB;IAC3B;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwD;IAE5F;;;;;;;;OAQG;IACI,KAAK,CACR,OAAO,EAAE,cAAc,EAAE,EACzB,SAAS,EAAE,2BAA2B,EAAE,EACxC,UAAU,EAAE,MAAM,GACnB,YAAY,EAAE;IAKjB;;OAEG;IACH,OAAO,CAAC,eAAe;IAsBvB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAgBzB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAY9B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA6BzB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAW5B;;OAEG;IACH,OAAO,CAAC,WAAW;IAOnB;;OAEG;IACH,OAAO,CAAC,UAAU;IAMlB;;OAEG;IACH,OAAO,CAAC,UAAU;IAMlB;;OAEG;IACH,OAAO,CAAC,YAAY;IAKpB;;OAEG;IACH,OAAO,CAAC,WAAW;IASnB;;OAEG;IACH,OAAO,CAAC,WAAW;IAkBnB;;OAEG;IACH,OAAO,CAAC,UAAU;IAOlB;;OAEG;IACH,OAAO,CAAC,WAAW;IAoBnB;;OAEG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAOvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAY7B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;CAQ5B"}
+{"version":3,"file":"FieldMappingEngine.d.ts","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAe/D;;;GAGG;AACH,qBAAa,kBAAkB;IAC3B;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwD;IAE5F;;;;;;;;OAQG;IACI,KAAK,CACR,OAAO,EAAE,cAAc,EAAE,EACzB,SAAS,EAAE,2BAA2B,EAAE,EACxC,UAAU,EAAE,MAAM,GACnB,YAAY,EAAE;IAKjB;;OAEG;IACH,OAAO,CAAC,eAAe;IAyCvB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAgBzB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAY9B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA6BzB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAW5B;;OAEG;IACH,OAAO,CAAC,WAAW;IAOnB;;OAEG;IACH,OAAO,CAAC,UAAU;IAMlB;;OAEG;IACH,OAAO,CAAC,UAAU;IAMlB;;OAEG;IACH,OAAO,CAAC,YAAY;IAKpB;;OAEG;IACH,OAAO,CAAC,WAAW;IASnB;;OAEG;IACH,OAAO,CAAC,WAAW;IAkBnB;;OAEG;IACH,OAAO,CAAC,UAAU;IAOlB;;OAEG;IACH,OAAO,CAAC,WAAW;IAoBnB;;OAEG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAOvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAY7B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;CAQ5B"}

package/dist/FieldMappingEngine.js

@@ -1,4 +1,6 @@
 import { MJLruCache } from '@memberjunction/global';
+import { computeUnmappedFields } from './CustomOverflow.js';
+import { flattenRecord, hasNestedObject } from './RecordFlatten.js';
 /**
  * Engine responsible for applying field-level mappings and transformations
  * from external records to MJ entity fields.
@@ -34,18 +36,35 @@ export class FieldMappingEngine {
      * Maps a single external record through all active field mappings.
      */
     MapSingleRecord(record, fieldMaps, entityName) {
+        // Flatten nested source objects to scalar columns (e.g. checkin_question.id →
+        // checkin_question_id) BEFORE mapping. The match keys on the mapped PK VALUE
+        // (MatchEngine.FindByKeyFields), so an object-valued source field would produce a
+        // per-occurrence / per-version key → duplicate rows. Discovery flattens identically, so
+        // the field maps reference the flattened scalar names. A record with no nested objects
+        // passes through unchanged — every flat-record connector is a no-op here.
+        const ext = hasNestedObject(record.Fields)
+            ? { ...record, Fields: flattenRecord(record.Fields) }
+            : record;
         const mappedFields = {};
+        const mappedSourceNames = new Set();
         for (const fieldMap of fieldMaps) {
-            const value = this.ApplyFieldMapping(record, fieldMap);
+            mappedSourceNames.add(fieldMap.SourceFieldName);
+            const value = this.ApplyFieldMapping(ext, fieldMap);
             if (value !== undefined) {
                 mappedFields[fieldMap.DestinationFieldName] = value;
             }
         }
+        // Capture the source keys with no field map (gaps.md §2). Cheap O(columns) diff;
+        // the result is empty (and discarded by the writer) in the common all-mapped case,
+        // so this adds no measurable cost to a customs-free sync. The engine parks any extras
+        // in the __mj_integration_CustomOverflow system column — see {@link CustomOverflow}.
+        const unmappedFields = computeUnmappedFields(ext.Fields, mappedSourceNames);
         return {
-            ExternalRecord: record,
+            ExternalRecord: ext,
             MJEntityName: entityName,
             MappedFields: mappedFields,
-            ChangeType: record.IsDeleted ? 'Delete' : 'Create',
+            ChangeType: ext.IsDeleted ? 'Delete' : 'Create',
+            UnmappedFields: unmappedFields,
         };
     }
     /**

package/dist/FieldMappingEngine.js.map

@@ -1 +1 @@
-{"version":3,"file":"FieldMappingEngine.js","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAiBpD;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IAA/B;QACI;;;;;;;;;WASG;QACc,wBAAmB,GAAG,IAAI,UAAU,EAAsC,CAAC;IA2UhG,CAAC;IAzUG;;;;;;;;OAQG;IACI,KAAK,CACR,OAAyB,EACzB,SAAwC,EACxC,UAAkB;QAElB,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACK,eAAe,CACnB,MAAsB,EACtB,SAAwC,EACxC,UAAkB;QAElB,MAAM,YAAY,GAA4B,EAAE,CAAC;QAEjD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC,GAAG,KAAK,CAAC;YACxD,CAAC;QACL,CAAC;QAED,OAAO;YACH,cAAc,EAAE,MAAM;YACtB,YAAY,EAAE,UAAU;YACxB,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;SACrD,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,iBAAiB,CACrB,MAAsB,EACtB,QAAqC;QAErC,IAAI,KAAK,GAAY,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAEzE,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YACrE,IAAI,MAAM,CAAC,OAAO;gBAAE,OAAO,SAAS,CAAC;YACrC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACzB,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,YAA2B;QACtD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAE3D,IAAI,CAAC;YACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,OAAO,EAAE,CAAC;YACtC,OAAO,MAAyB,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,oBAAoB,CACxB,IAAmB,EACnB,KAAc,EACd,SAAkC;QAElC,+FAA+F;QAC/F,kGAAkG;QAClG,4EAA4E;QAC5E,MAAM,OAAO,GAAqB,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC;QAEzD,IAAI,CAAC;YACD,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;YACnE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAClD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,CACrB,IAAmB,EACnB,KAAc,EACd,SAAkC;QAElC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAChB,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,OAAO;gBACR,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAqB,CAAC,CAAC;YAC9D,KAAK,OAAO;gBACR,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAqB,CAAC,CAAC;YAC9D,KAAK,SAAS;gBACV,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,MAAuB,CAAC,CAAC;YACtE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,WAAW;gBACZ,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,MAAyB,CAAC,CAAC;YACtE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAC3E;gBACI,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,OAAyB,EAAE,GAAY;QAChE,QAAQ,OAAO,EAAE,CAAC;YACd,KAAK,MAAM;gBACP,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YAC/C,KAAK,MAAM;gBACP,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAC3C,KAAK,MAAM;gBACP,MAAM,GAAG,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,IAAI,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YACrD,OAAO,MAAM,CAAC,YAAY,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,KAAc,EAAE,MAAmB;QAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,KAAc,EAAE,MAAmB;QAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,SAAkC,EAAE,MAAqB;QAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CACzC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,QAAQ,CAC5C,CAAC;QACF,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,IAAI,MAAM,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,KAAK,YAAY,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACxE,mGAAmG;YACnG,kGAAkG;YAClG,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC5F,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,gGAAgG;YAChG,8FAA8F;YAC9F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9F,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,IAAU,EAAE,YAAoB;QAC/C,IAAI,YAAY,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YACrE,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;YACxB,KAAK,QAAQ;gBACT,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAC/B,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YACtC,KAAK,SAAS;gBACV,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACvC,KAAK,MAAM,CAAC,CAAC,CAAC;gBACV,2FAA2F;gBAC3F,iFAAiF;gBACjF,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAClC,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACpF,OAAO,CAAC,CAAC;YACb,CAAC;YACD;gBACI,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAC5E,CAAC;IACL,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAc;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,0GAA0G;QAC1G,0FAA0F;QAC1F,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,KAAK,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAClD,OAAO,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,KAAK,CAAC;IACnE,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAc,EAAE,MAAuB;QAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACK,WAAW,CACf,KAAc,EACd,SAAkC,EAClC,MAAoB;QAEpB,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACzD,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,UAAkB;QAC5C,IAAI,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAC5C,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,MAAM,YAAY,KAAK,EAAE,CAAC;YAC1B,MAAM,MAAM,CAAC;QACjB,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,iBAAiB,CAAC,UAAkB;QACxC,IAAI,CAAC;YACD,8DAA8D;YAC9D,OAAO,IAAI,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,UAAU,IAAI,CAAuB,CAAC;QAC5F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;CACJ"}
+{"version":3,"file":"FieldMappingEngine.js","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAiBpE;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IAA/B;QACI;;;;;;;;;WASG;QACc,wBAAmB,GAAG,IAAI,UAAU,EAAsC,CAAC;IA8VhG,CAAC;IA5VG;;;;;;;;OAQG;IACI,KAAK,CACR,OAAyB,EACzB,SAAwC,EACxC,UAAkB;QAElB,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACK,eAAe,CACnB,MAAsB,EACtB,SAAwC,EACxC,UAAkB;QAElB,8EAA8E;QAC9E,6EAA6E;QAC7E,kFAAkF;QAClF,wFAAwF;QACxF,uFAAuF;QACvF,0EAA0E;QAC1E,MAAM,GAAG,GAAmB,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC;YACtD,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;YACrD,CAAC,CAAC,MAAM,CAAC;QAEb,MAAM,YAAY,GAA4B,EAAE,CAAC;QACjD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE5C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;YAChD,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YACpD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC,GAAG,KAAK,CAAC;YACxD,CAAC;QACL,CAAC;QAED,iFAAiF;QACjF,mFAAmF;QACnF,sFAAsF;QACtF,qFAAqF;QACrF,MAAM,cAAc,GAAG,qBAAqB,CAAC,GAAG,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;QAE5E,OAAO;YACH,cAAc,EAAE,GAAG;YACnB,YAAY,EAAE,UAAU;YACxB,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAC/C,cAAc,EAAE,cAAc;SACjC,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,iBAAiB,CACrB,MAAsB,EACtB,QAAqC;QAErC,IAAI,KAAK,GAAY,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAEzE,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YACrE,IAAI,MAAM,CAAC,OAAO;gBAAE,OAAO,SAAS,CAAC;YACrC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACzB,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,YAA2B;QACtD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAE3D,IAAI,CAAC;YACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,OAAO,EAAE,CAAC;YACtC,OAAO,MAAyB,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,oBAAoB,CACxB,IAAmB,EACnB,KAAc,EACd,SAAkC;QAElC,+FAA+F;QAC/F,kGAAkG;QAClG,4EAA4E;QAC5E,MAAM,OAAO,GAAqB,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC;QAEzD,IAAI,CAAC;YACD,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;YACnE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAClD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,CACrB,IAAmB,EACnB,KAAc,EACd,SAAkC;QAElC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAChB,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,OAAO;gBACR,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAqB,CAAC,CAAC;YAC9D,KAAK,OAAO;gBACR,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAqB,CAAC,CAAC;YAC9D,KAAK,SAAS;gBACV,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,MAAuB,CAAC,CAAC;YACtE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAChE,KAAK,WAAW;gBACZ,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,MAAyB,CAAC,CAAC;YACtE,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,MAAsB,CAAC,CAAC;YAC3E;gBACI,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,OAAyB,EAAE,GAAY;QAChE,QAAQ,OAAO,EAAE,CAAC;YACd,KAAK,MAAM;gBACP,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YAC/C,KAAK,MAAM;gBACP,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAC3C,KAAK,MAAM;gBACP,MAAM,GAAG,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,IAAI,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YACrD,OAAO,MAAM,CAAC,YAAY,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,KAAc,EAAE,MAAmB;QAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,KAAc,EAAE,MAAmB;QAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,SAAkC,EAAE,MAAqB;QAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CACzC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,QAAQ,CAC5C,CAAC;QACF,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,IAAI,MAAM,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,KAAK,YAAY,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACxE,mGAAmG;YACnG,kGAAkG;YAClG,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC5F,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,gGAAgG;YAChG,8FAA8F;YAC9F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9F,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,IAAU,EAAE,YAAoB;QAC/C,IAAI,YAAY,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YACrE,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,MAAoB;QACpD,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;YACxB,KAAK,QAAQ;gBACT,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAC/B,KAAK,QAAQ;gBACT,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YACtC,KAAK,SAAS;gBACV,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACvC,KAAK,MAAM,CAAC,CAAC,CAAC;gBACV,2FAA2F;gBAC3F,iFAAiF;gBACjF,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAClC,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACpF,OAAO,CAAC,CAAC;YACb,CAAC;YACD;gBACI,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAC5E,CAAC;IACL,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAc;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,0GAA0G;QAC1G,0FAA0F;QAC1F,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,KAAK,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAClD,OAAO,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,KAAK,CAAC;IACnE,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAc,EAAE,MAAuB;QAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACK,WAAW,CACf,KAAc,EACd,SAAkC,EAClC,MAAoB;QAEpB,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACzD,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,UAAkB;QAC5C,IAAI,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAC5C,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,MAAM,YAAY,KAAK,EAAE,CAAC;YAC1B,MAAM,MAAM,CAAC;QACjB,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,iBAAiB,CAAC,UAAkB;QACxC,IAAI,CAAC;YACD,8DAA8D;YAC9D,OAAO,IAAI,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,UAAU,IAAI,CAAuB,CAAC;QAC5F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;CACJ"}

package/dist/IntegrationConnectorCreationPipeline.d.ts

@@ -38,6 +38,13 @@ export interface ConnectorCreationPipelineOptions {
     RunID?: string;
     /** Trigger reason recorded in the manifest. */
     TriggerType?: 'Manual' | 'Scheduled' | 'Webhook' | 'Pipeline' | 'Restart';
+    /**
+     * §7 — when true, this is a COMPREHENSIVE re-discovery: declared/discovered objects (and their
+     * fields) ABSENT from this run are deactivated (Status='Disabled', never deleted; reversible on a
+     * later rediscovery). Only set on a full-surface refresh — a scoped/partial discovery must leave it
+     * false so it never disables what it didn't look at. Threaded to PersistDiscoveredSchema.
+     */
+    DeactivateAbsent?: boolean;
 }
 /** Outcome of a single pipeline invocation. */
 export interface ConnectorCreationPipelineResult {
@@ -77,7 +84,33 @@ export interface ConnectorCreationPipelineResult {
  * `__mj.Entity`. The pipeline emits `entity.skipped-no-pk` events for visibility.
  */
 export declare class IntegrationConnectorCreationPipeline {
+    /** In-flight runs by CompanyIntegrationID — coalesces a concurrent duplicate onto the same promise. */
+    private static readonly inFlightRuns;
+    /** Just-completed runs by CompanyIntegrationID — coalesces a *sequential* duplicate within the window. */
+    private static readonly recentRuns;
+    /** Default coalesce window (ms) when the env override is unset/invalid. */
+    private static readonly DEFAULT_COALESCE_WINDOW_MS;
+    /**
+     * How long a just-completed run is reused for a duplicate invocation of the SAME CompanyIntegration.
+     * Sized only to absorb the create-time double-fire (the `IsActive` Save-hook runs the pipeline, then
+     * `IntegrationCreateConnection` calls it again milliseconds later) — short enough that a genuine,
+     * later operator-initiated re-refresh always runs fresh. Override via the
+     * `MJ_CONNECTOR_PIPELINE_COALESCE_WINDOW_MS` env var (a positive integer of milliseconds; set 0/unset
+     * to use the default). Env-based, matching the RSU env-var convention (RSU_WORK_DIR, etc.).
+     */
+    private static get COALESCE_WINDOW_MS();
+    /**
+     * Public entry. De-dups the known create-time double-invocation: the connection's `IsActive`
+     * false→true Save fires the entity-server hook (which runs this pipeline WITH LLM PK inference),
+     * and the create resolver then calls it again. Both converge here, so we run the pipeline ONCE
+     * per CompanyIntegration and hand both callers the same result — no double introspect/persist/
+     * classify, no double live API calls, and the resolver still gets a real summary. A legitimate
+     * re-refresh later (outside the window) runs fresh.
+     */
     Run(opts: ConnectorCreationPipelineOptions): Promise<ConnectorCreationPipelineResult>;
+    /** Drops recent-run entries older than the coalesce window so the map stays bounded. */
+    private static pruneRecentRuns;
+    private runInternal;
     private StageConnectionTest;
     private StageIntrospect;
     private StagePersist;

package/dist/IntegrationConnectorCreationPipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"IntegrationConnectorCreationPipeline.d.ts","sourceRoot":"","sources":["../src/IntegrationConnectorCreationPipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,KAAK,EACR,0BAA0B,EAG7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEH,KAAK,sBAAsB,EAC9B,MAAM,gDAAgD,CAAC;AACxD,OAAO,EAEH,KAAK,kBAAkB,EAC1B,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,OAAO,EAAyB,KAAK,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAC7F,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAE1D,qDAAqD;AACrD,MAAM,WAAW,gCAAgC;IAC7C,uEAAuE;IACvE,SAAS,EAAE,wBAAwB,CAAC;IACpC,mDAAmD;IACnD,kBAAkB,EAAE,0BAA0B,CAAC;IAC/C,8CAA8C;IAC9C,WAAW,EAAE,QAAQ,CAAC;IACtB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,+EAA+E;IAC/E,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,+EAA+E;IAC/E,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,gFAAgF;IAChF,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACpE;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kDAAkD;IAClD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,WAAW,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;CAC7E;AAED,+CAA+C;AAC/C,MAAM,WAAW,+BAA+B;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,EAAE,KAAK,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,+EAA+E;IAC/E,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,wEAAwE;IACxE,QAAQ,EAAE,sBAAsB,CAAC;IACjC,0CAA0C;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,oCAAoC;IAChC,GAAG,CAAC,IAAI,EAAE,gCAAgC,GAAG,OAAO,CAAC,+BAA+B,CAAC;YA4DpF,mBAAmB;YAiBnB,eAAe;YAiCf,YAAY;YAuCZ,eAAe;CAoFhC"}
+{"version":3,"file":"IntegrationConnectorCreationPipeline.d.ts","sourceRoot":"","sources":["../src/IntegrationConnectorCreationPipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,KAAK,EACR,0BAA0B,EAG7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEH,KAAK,sBAAsB,EAC9B,MAAM,gDAAgD,CAAC;AACxD,OAAO,EAEH,KAAK,kBAAkB,EAC1B,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,wBAAwB,EAAuD,MAAM,+BAA+B,CAAC;AAE9H,OAAO,EAAyB,KAAK,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAC7F,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAE1D,qDAAqD;AACrD,MAAM,WAAW,gCAAgC;IAC7C,uEAAuE;IACvE,SAAS,EAAE,wBAAwB,CAAC;IACpC,mDAAmD;IACnD,kBAAkB,EAAE,0BAA0B,CAAC;IAC/C,8CAA8C;IAC9C,WAAW,EAAE,QAAQ,CAAC;IACtB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,+EAA+E;IAC/E,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,+EAA+E;IAC/E,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,gFAAgF;IAChF,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACpE;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kDAAkD;IAClD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,WAAW,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;IAC1E;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,+CAA+C;AAC/C,MAAM,WAAW,+BAA+B;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,EAAE,KAAK,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,+EAA+E;IAC/E,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,wEAAwE;IACxE,QAAQ,EAAE,sBAAsB,CAAC;IACjC,0CAA0C;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,oCAAoC;IAC7C,uGAAuG;IACvG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAA+D;IACnG,0GAA0G;IAC1G,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA8E;IAChH,2EAA2E;IAC3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAQ;IAE1D;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM,KAAK,kBAAkB,GAGpC;IAED;;;;;;;OAOG;IACU,GAAG,CAAC,IAAI,EAAE,gCAAgC,GAAG,OAAO,CAAC,+BAA+B,CAAC;IAuBlG,wFAAwF;IACxF,OAAO,CAAC,MAAM,CAAC,eAAe;YAShB,WAAW;YA4DX,mBAAmB;YAiBnB,eAAe;YAiIf,YAAY;YAyCZ,eAAe;CAoFhC"}