@memberjunction/integration-engine 5.41.0 → 5.43.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +51 -0
  2. package/dist/BaseIntegrationConnector.d.ts +109 -0
  3. package/dist/BaseIntegrationConnector.d.ts.map +1 -1
  4. package/dist/BaseIntegrationConnector.js +308 -93
  5. package/dist/BaseIntegrationConnector.js.map +1 -1
  6. package/dist/BaseRESTIntegrationConnector.d.ts +36 -0
  7. package/dist/BaseRESTIntegrationConnector.d.ts.map +1 -1
  8. package/dist/BaseRESTIntegrationConnector.js +228 -28
  9. package/dist/BaseRESTIntegrationConnector.js.map +1 -1
  10. package/dist/ContentHash.d.ts +22 -0
  11. package/dist/ContentHash.d.ts.map +1 -1
  12. package/dist/ContentHash.js +35 -0
  13. package/dist/ContentHash.js.map +1 -1
  14. package/dist/CustomColumnPromotion.d.ts +108 -0
  15. package/dist/CustomColumnPromotion.d.ts.map +1 -0
  16. package/dist/CustomColumnPromotion.js +202 -0
  17. package/dist/CustomColumnPromotion.js.map +1 -0
  18. package/dist/CustomOverflow.d.ts +64 -0
  19. package/dist/CustomOverflow.d.ts.map +1 -0
  20. package/dist/CustomOverflow.js +74 -0
  21. package/dist/CustomOverflow.js.map +1 -0
  22. package/dist/FieldMappingEngine.d.ts +16 -90
  23. package/dist/FieldMappingEngine.d.ts.map +1 -1
  24. package/dist/FieldMappingEngine.js +42 -257
  25. package/dist/FieldMappingEngine.js.map +1 -1
  26. package/dist/IntegrationConnectorCreationPipeline.d.ts +33 -0
  27. package/dist/IntegrationConnectorCreationPipeline.d.ts.map +1 -1
  28. package/dist/IntegrationConnectorCreationPipeline.js +157 -0
  29. package/dist/IntegrationConnectorCreationPipeline.js.map +1 -1
  30. package/dist/IntegrationEngine.d.ts +120 -6
  31. package/dist/IntegrationEngine.d.ts.map +1 -1
  32. package/dist/IntegrationEngine.js +890 -168
  33. package/dist/IntegrationEngine.js.map +1 -1
  34. package/dist/IntegrationSchemaSync.d.ts +84 -13
  35. package/dist/IntegrationSchemaSync.d.ts.map +1 -1
  36. package/dist/IntegrationSchemaSync.js +198 -36
  37. package/dist/IntegrationSchemaSync.js.map +1 -1
  38. package/dist/KeySerialization.d.ts +21 -0
  39. package/dist/KeySerialization.d.ts.map +1 -0
  40. package/dist/KeySerialization.js +29 -0
  41. package/dist/KeySerialization.js.map +1 -0
  42. package/dist/MatchEngine.d.ts.map +1 -1
  43. package/dist/MatchEngine.js +12 -4
  44. package/dist/MatchEngine.js.map +1 -1
  45. package/dist/RecordFlatten.d.ts +49 -0
  46. package/dist/RecordFlatten.d.ts.map +1 -0
  47. package/dist/RecordFlatten.js +50 -0
  48. package/dist/RecordFlatten.js.map +1 -0
  49. package/dist/StreamingDiscovery.d.ts +145 -0
  50. package/dist/StreamingDiscovery.d.ts.map +1 -0
  51. package/dist/StreamingDiscovery.js +306 -0
  52. package/dist/StreamingDiscovery.js.map +1 -0
  53. package/dist/SyncLogger.d.ts +1 -1
  54. package/dist/SyncLogger.d.ts.map +1 -1
  55. package/dist/SyncLogger.js +19 -1
  56. package/dist/SyncLogger.js.map +1 -1
  57. package/dist/auth-helpers/BasicAuthHeaderBuilder.d.ts +36 -0
  58. package/dist/auth-helpers/BasicAuthHeaderBuilder.d.ts.map +1 -0
  59. package/dist/auth-helpers/BasicAuthHeaderBuilder.js +38 -0
  60. package/dist/auth-helpers/BasicAuthHeaderBuilder.js.map +1 -0
  61. package/dist/auth-helpers/OAuth1aSigner.d.ts +56 -0
  62. package/dist/auth-helpers/OAuth1aSigner.d.ts.map +1 -0
  63. package/dist/auth-helpers/OAuth1aSigner.js +91 -0
  64. package/dist/auth-helpers/OAuth1aSigner.js.map +1 -0
  65. package/dist/auth-helpers/OAuth2TokenManager.d.ts +103 -0
  66. package/dist/auth-helpers/OAuth2TokenManager.d.ts.map +1 -0
  67. package/dist/auth-helpers/OAuth2TokenManager.js +143 -0
  68. package/dist/auth-helpers/OAuth2TokenManager.js.map +1 -0
  69. package/dist/auth-helpers/index.d.ts +11 -0
  70. package/dist/auth-helpers/index.d.ts.map +1 -0
  71. package/dist/auth-helpers/index.js +8 -0
  72. package/dist/auth-helpers/index.js.map +1 -0
  73. package/dist/index.d.ts +15 -1
  74. package/dist/index.d.ts.map +1 -1
  75. package/dist/index.js +9 -0
  76. package/dist/index.js.map +1 -1
  77. package/dist/types.d.ts +73 -0
  78. package/dist/types.d.ts.map +1 -1
  79. package/package.json +7 -7
@@ -0,0 +1,108 @@
1
+ /**
2
+ * Promotion planner — the brain of the post-sync custom-column step (gaps.md §2, M2).
3
+ *
4
+ * After a sync, the values a source returned with no field map are parked as JSON in the
5
+ * {@link CUSTOM_OVERFLOW_COLUMN} system column (M1 capture). This module decides, from a
6
+ * coverage scan of that column, WHICH of those keys earn a real column and WHAT bounded
7
+ * type each should get. It is PURE — no DB, no RSU, no I/O — so the genuinely new logic is
8
+ * fully unit-testable. The server-side orchestrator (M2b) feeds the resulting plan into the
9
+ * EXISTING refresh pipeline (SchemaEvolution ADD COLUMN → RSU → IntegrationSchemaSync IOF →
10
+ * field map); this module never touches schema itself.
11
+ *
12
+ * Design rules (from gaps.md §2 + the connector-code conventions):
13
+ * - Provable-only / never fabricate: a key earns a column only on PERVASIVENESS evidence
14
+ * (coverage ≥ threshold), never on a single malformed row. PK/FK are NEVER inferred here —
15
+ * customs are always emitted nullable, non-PK, non-FK (classification is deferred to D4).
16
+ * - Generous bounded typing (the NVARCHAR(MAX) problem): size columns comfortably from the
17
+ * observed samples and err LARGER — a roomy bounded column beats a truncating tight one,
18
+ * and both beat MAX. Only fall back to MAX/TEXT when the observed length genuinely can't be
19
+ * bounded. Narrow to number/boolean/datetime ONLY when every non-null sample unambiguously
20
+ * supports it; otherwise default to a (generously bounded) string — that is the safe choice
21
+ * that can hold anything the source later returns.
22
+ * - Terminate / never re-promote: a key whose column already exists is skipped, so the
23
+ * capture→promote loop converges instead of re-promoting forever.
24
+ */
25
+ /** The schema-builder field-type family + the concrete per-dialect SQL types we infer. */
26
+ export interface InferredColumnType {
27
+ /** Schema-builder SchemaFieldType family ('string' | 'number' | 'boolean' | 'datetime'). */
28
+ SchemaFieldType: 'string' | 'number' | 'boolean' | 'datetime';
29
+ /** Concrete SQL Server column type, e.g. `NVARCHAR(255)`, `BIGINT`, `DATETIMEOFFSET`. */
30
+ SqlServerType: string;
31
+ /** Concrete PostgreSQL column type, e.g. `VARCHAR(255)`, `BIGINT`, `TIMESTAMPTZ`. */
32
+ PostgresType: string;
33
+ /** Bound for string types (null for non-string or when unbounded → MAX/TEXT). */
34
+ MaxLength: number | null;
35
+ }
36
+ /** Per-key statistics from a coverage scan of the overflow column for one (CompanyIntegration, entity). */
37
+ export interface OverflowKeyStats {
38
+ /** The source key as it appears in the overflow JSON. */
39
+ Key: string;
40
+ /** Rows in which the key was present with a non-null value. */
41
+ Occurrences: number;
42
+ /** Total rows scanned (rows that had any overflow JSON). */
43
+ TotalRows: number;
44
+ /** A bounded sample of observed values for this key, for type inference. */
45
+ SampleValues: unknown[];
46
+ }
47
+ /** A key that earned a real column, with its inferred bounded type. */
48
+ export interface PromotionCandidate {
49
+ /** The source key → becomes both the SourceFieldName (field map) and the column name. */
50
+ Key: string;
51
+ /** Pervasiveness in [0,1] = Occurrences / TotalRows. */
52
+ Coverage: number;
53
+ /** The inferred, generously-bounded column type. */
54
+ Inferred: InferredColumnType;
55
+ }
56
+ /** Tuning + context for {@link planPromotions}. */
57
+ export interface PromotionPlanOptions {
58
+ /**
59
+ * Minimum coverage (in [0,1]) a key must clear to earn a column. Default 0 — §23: a custom key
60
+ * earns a column on FIRST occurrence (presence, not prevalence); appearing in even one row is
61
+ * enough. Raise this only if a caller deliberately wants to suppress sparse keys. The captured
62
+ * values are never lost regardless — they stay in the overflow JSON until promoted.
63
+ */
64
+ CoverageThreshold?: number;
65
+ /**
66
+ * Column names that already exist on the target entity (case-insensitive). A key whose
67
+ * column already exists is NOT re-promoted — this is what makes the loop terminate.
68
+ */
69
+ ExistingColumnNames?: ReadonlySet<string>;
70
+ }
71
+ /**
72
+ * Plans which overflow keys to promote to real columns. Pure; deterministic (sorted by key).
73
+ *
74
+ * @param stats - per-key coverage statistics from the overflow scan
75
+ * @param opts - coverage threshold + the set of already-existing column names
76
+ * @returns the promotion candidates, sorted by key for stable, replayable output
77
+ */
78
+ export declare function planPromotions(stats: OverflowKeyStats[], opts?: PromotionPlanOptions): PromotionCandidate[];
79
+ /**
80
+ * Infers a generously-bounded column type from observed sample values. Narrows to
81
+ * boolean/number/datetime ONLY when EVERY non-null sample unambiguously supports it;
82
+ * otherwise defaults to a comfortably-bounded string. Never returns MAX/TEXT unless the
83
+ * observed string length genuinely can't be bounded.
84
+ */
85
+ export declare function inferColumnTypeFromSamples(samples: unknown[]): InferredColumnType;
86
+ /**
87
+ * Builds per-key coverage statistics from a SAMPLE of rows that carried overflow JSON.
88
+ * Each row's overflow column is a JSON object string (what M1 parked); this tallies, per key,
89
+ * how many sampled rows had it non-null and collects a bounded sample of its values for type
90
+ * inference. Robust to malformed/empty JSON (such rows contribute to TotalRows but no keys).
91
+ *
92
+ * Coverage is therefore computed over the SAMPLE (bounded memory at any table size) — a
93
+ * statistically sound basis for the pervasiveness decision; the exactness the threshold needs
94
+ * (a key on ~half the rows vs. a junk key on one) survives sampling.
95
+ *
96
+ * @param overflowJsonStrings - the raw overflow-column value from each sampled row
97
+ * @param sampleCap - max values retained per key (default {@link DEFAULT_SAMPLE_CAP})
98
+ */
99
+ export declare function buildOverflowStats(overflowJsonStrings: Array<string | null | undefined>, sampleCap?: number): OverflowKeyStats[];
100
+ /**
101
+ * Sanitizes a source key into a safe SQL/MJ column identifier (letters, digits, underscore;
102
+ * never leading-digit; bounded length). The original key is kept as the field map's
103
+ * SourceFieldName; the sanitized form becomes the column + DestinationFieldName. Returns the
104
+ * BASE name only — collision resolution (suffixing) is the orchestrator's job since it needs
105
+ * the existing-column context.
106
+ */
107
+ export declare function sanitizeColumnName(key: string): string;
108
+ //# sourceMappingURL=CustomColumnPromotion.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CustomColumnPromotion.d.ts","sourceRoot":"","sources":["../src/CustomColumnPromotion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,0FAA0F;AAC1F,MAAM,WAAW,kBAAkB;IAC/B,4FAA4F;IAC5F,eAAe,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,CAAC;IAC9D,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;IACtB,qFAAqF;IACrF,YAAY,EAAE,MAAM,CAAC;IACrB,iFAAiF;IACjF,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,2GAA2G;AAC3G,MAAM,WAAW,gBAAgB;IAC7B,yDAAyD;IACzD,GAAG,EAAE,MAAM,CAAC;IACZ,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,YAAY,EAAE,OAAO,EAAE,CAAC;CAC3B;AAED,uEAAuE;AACvE,MAAM,WAAW,kBAAkB;IAC/B,yFAAyF;IACzF,GAAG,EAAE,MAAM,CAAC;IACZ,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,QAAQ,EAAE,kBAAkB,CAAC;CAChC;AAED,mDAAmD;AACnD,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAC7C;AAmBD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC1B,KAAK,EAAE,gBAAgB,EAAE,EACzB,IAAI,GAAE,oBAAyB,GAChC,kBAAkB,EAAE,CAsBtB;AAED;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,kBAAkB,CA4BjF;AAwCD;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAC9B,mBAAmB,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,EACrD,SAAS,GAAE,MAA2B,GACvC,gBAAgB,EAAE,CA0BpB;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKtD"}
@@ -0,0 +1,202 @@
1
+ /**
2
+ * Promotion planner — the brain of the post-sync custom-column step (gaps.md §2, M2).
3
+ *
4
+ * After a sync, the values a source returned with no field map are parked as JSON in the
5
+ * {@link CUSTOM_OVERFLOW_COLUMN} system column (M1 capture). This module decides, from a
6
+ * coverage scan of that column, WHICH of those keys earn a real column and WHAT bounded
7
+ * type each should get. It is PURE — no DB, no RSU, no I/O — so the genuinely new logic is
8
+ * fully unit-testable. The server-side orchestrator (M2b) feeds the resulting plan into the
9
+ * EXISTING refresh pipeline (SchemaEvolution ADD COLUMN → RSU → IntegrationSchemaSync IOF →
10
+ * field map); this module never touches schema itself.
11
+ *
12
+ * Design rules (from gaps.md §2 + the connector-code conventions):
13
+ * - Provable-only / never fabricate: a key earns a column only on PERVASIVENESS evidence
14
+ * (coverage ≥ threshold), never on a single malformed row. PK/FK are NEVER inferred here —
15
+ * customs are always emitted nullable, non-PK, non-FK (classification is deferred to D4).
16
+ * - Generous bounded typing (the NVARCHAR(MAX) problem): size columns comfortably from the
17
+ * observed samples and err LARGER — a roomy bounded column beats a truncating tight one,
18
+ * and both beat MAX. Only fall back to MAX/TEXT when the observed length genuinely can't be
19
+ * bounded. Narrow to number/boolean/datetime ONLY when every non-null sample unambiguously
20
+ * supports it; otherwise default to a (generously bounded) string — that is the safe choice
21
+ * that can hold anything the source later returns.
22
+ * - Terminate / never re-promote: a key whose column already exists is skipped, so the
23
+ * capture→promote loop converges instead of re-promoting forever.
24
+ */
25
+ const DEFAULT_COVERAGE_THRESHOLD = 0; // §23 — presence-based: a key seen in even one row earns a column.
26
+ /** SQL Server's largest bounded NVARCHAR before MAX; above this we must use MAX/TEXT. */
27
+ const MAX_BOUNDED_STRING = 4000;
28
+ /** Floor for a generous string bound — never size a custom string column tighter than this. */
29
+ const MIN_STRING_BOUND = 255;
30
+ /**
31
+ * Minimum all-integer sample before we trust the field is INTEGRAL and narrow to BIGINT (#A9). Below this
32
+ * a handful of integer samples does NOT prove the unscanned tail is integer-only — a later decimal would be
33
+ * silently rejected by BIGINT (dead-lettered). Under the threshold we use wide fixed-point (which holds
34
+ * integers too), so a later decimal never fails; the column can still re-promote to BIGINT once enough
35
+ * integer evidence accrues.
36
+ */
37
+ const MIN_SAMPLE_FOR_INTEGER_NARROW = 12;
38
+ /** Matches an ISO-8601-ish date / datetime so we don't mistake "5" or "2020" for a date. */
39
+ const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;
40
+ /**
41
+ * Plans which overflow keys to promote to real columns. Pure; deterministic (sorted by key).
42
+ *
43
+ * @param stats - per-key coverage statistics from the overflow scan
44
+ * @param opts - coverage threshold + the set of already-existing column names
45
+ * @returns the promotion candidates, sorted by key for stable, replayable output
46
+ */
47
+ export function planPromotions(stats, opts = {}) {
48
+ const threshold = opts.CoverageThreshold ?? DEFAULT_COVERAGE_THRESHOLD;
49
+ const existing = lowercaseSet(opts.ExistingColumnNames);
50
+ const candidates = [];
51
+ for (const stat of stats) {
52
+ // Terminate / never re-promote: the column already exists.
53
+ if (existing.has(stat.Key.toLowerCase()))
54
+ continue;
55
+ // Guard div-by-zero only; §23 — presence is enough (default threshold 0), so a key seen in
56
+ // even one row earns a column. A higher bar is opt-in via CoverageThreshold.
57
+ if (stat.TotalRows <= 0)
58
+ continue;
59
+ const coverage = stat.Occurrences / stat.TotalRows;
60
+ if (coverage < threshold)
61
+ continue;
62
+ candidates.push({
63
+ Key: stat.Key,
64
+ Coverage: coverage,
65
+ Inferred: inferColumnTypeFromSamples(stat.SampleValues),
66
+ });
67
+ }
68
+ return candidates.sort((a, b) => a.Key.localeCompare(b.Key));
69
+ }
70
+ /**
71
+ * Infers a generously-bounded column type from observed sample values. Narrows to
72
+ * boolean/number/datetime ONLY when EVERY non-null sample unambiguously supports it;
73
+ * otherwise defaults to a comfortably-bounded string. Never returns MAX/TEXT unless the
74
+ * observed string length genuinely can't be bounded.
75
+ */
76
+ export function inferColumnTypeFromSamples(samples) {
77
+ const nonNull = samples.filter(v => v !== null && v !== undefined);
78
+ // No evidence → a safe, generous default string.
79
+ if (nonNull.length === 0)
80
+ return stringType(MIN_STRING_BOUND);
81
+ if (nonNull.every(isBoolean)) {
82
+ return { SchemaFieldType: 'boolean', SqlServerType: 'BIT', PostgresType: 'BOOLEAN', MaxLength: null };
83
+ }
84
+ if (nonNull.every(isFiniteNumber)) {
85
+ // Narrow to BIGINT only with an ADEQUATE all-integer sample (#A9). A small all-integer sample
86
+ // doesn't prove the field is always integral — a decimal in the unscanned tail would be silently
87
+ // rejected by BIGINT. Below the threshold, use wide fixed-point (which holds integers too).
88
+ const confidentInteger = nonNull.length >= MIN_SAMPLE_FOR_INTEGER_NARROW && nonNull.every(v => Number.isInteger(v));
89
+ return confidentInteger
90
+ ? { SchemaFieldType: 'number', SqlServerType: 'BIGINT', PostgresType: 'BIGINT', MaxLength: null }
91
+ // Generous fixed-point — wide precision so we don't truncate or reject decimals.
92
+ : { SchemaFieldType: 'number', SqlServerType: 'DECIMAL(38,10)', PostgresType: 'DECIMAL(38,10)', MaxLength: null };
93
+ }
94
+ if (nonNull.every(isIsoDateString)) {
95
+ return { SchemaFieldType: 'datetime', SqlServerType: 'DATETIMEOFFSET', PostgresType: 'TIMESTAMPTZ', MaxLength: null };
96
+ }
97
+ // Default: a generously-bounded string sized to comfortably hold the longest observed value.
98
+ const longest = Math.max(...nonNull.map(v => String(v).length));
99
+ return stringType(generousStringBound(longest));
100
+ }
101
+ /** Builds a string {@link InferredColumnType}; bound===null ⇒ unbounded (MAX/TEXT). */
102
+ function stringType(bound) {
103
+ if (bound === null) {
104
+ return { SchemaFieldType: 'string', SqlServerType: 'NVARCHAR(MAX)', PostgresType: 'TEXT', MaxLength: null };
105
+ }
106
+ return { SchemaFieldType: 'string', SqlServerType: `NVARCHAR(${bound})`, PostgresType: `VARCHAR(${bound})`, MaxLength: bound };
107
+ }
108
+ /**
109
+ * Generous bound for a string column: double the longest observed length (room to grow),
110
+ * floored at {@link MIN_STRING_BOUND}; if that would exceed the bounded limit, fall back to
111
+ * unbounded (null → MAX/TEXT) since it genuinely can't be bounded safely.
112
+ */
113
+ function generousStringBound(longestObserved) {
114
+ const doubled = Math.max(longestObserved * 2, MIN_STRING_BOUND);
115
+ return doubled > MAX_BOUNDED_STRING ? null : doubled;
116
+ }
117
+ function isBoolean(v) {
118
+ return typeof v === 'boolean';
119
+ }
120
+ function isFiniteNumber(v) {
121
+ return typeof v === 'number' && Number.isFinite(v);
122
+ }
123
+ function isIsoDateString(v) {
124
+ return typeof v === 'string' && ISO_DATE_RE.test(v) && !Number.isNaN(Date.parse(v));
125
+ }
126
+ function lowercaseSet(set) {
127
+ if (!set || set.size === 0)
128
+ return new Set();
129
+ return new Set([...set].map(s => s.toLowerCase()));
130
+ }
131
+ /** Max distinct sample values retained per key for type inference (bounded memory). */
132
+ const DEFAULT_SAMPLE_CAP = 20;
133
+ /**
134
+ * Builds per-key coverage statistics from a SAMPLE of rows that carried overflow JSON.
135
+ * Each row's overflow column is a JSON object string (what M1 parked); this tallies, per key,
136
+ * how many sampled rows had it non-null and collects a bounded sample of its values for type
137
+ * inference. Robust to malformed/empty JSON (such rows contribute to TotalRows but no keys).
138
+ *
139
+ * Coverage is therefore computed over the SAMPLE (bounded memory at any table size) — a
140
+ * statistically sound basis for the pervasiveness decision; the exactness the threshold needs
141
+ * (a key on ~half the rows vs. a junk key on one) survives sampling.
142
+ *
143
+ * @param overflowJsonStrings - the raw overflow-column value from each sampled row
144
+ * @param sampleCap - max values retained per key (default {@link DEFAULT_SAMPLE_CAP})
145
+ */
146
+ export function buildOverflowStats(overflowJsonStrings, sampleCap = DEFAULT_SAMPLE_CAP) {
147
+ const byKey = new Map();
148
+ let totalRows = 0;
149
+ for (const raw of overflowJsonStrings) {
150
+ totalRows++;
151
+ const parsed = safeParseObject(raw);
152
+ if (!parsed)
153
+ continue;
154
+ for (const [key, value] of Object.entries(parsed)) {
155
+ if (value === null || value === undefined)
156
+ continue;
157
+ let entry = byKey.get(key);
158
+ if (!entry) {
159
+ entry = { occurrences: 0, samples: [] };
160
+ byKey.set(key, entry);
161
+ }
162
+ entry.occurrences++;
163
+ if (entry.samples.length < sampleCap)
164
+ entry.samples.push(value);
165
+ }
166
+ }
167
+ return [...byKey.entries()].map(([key, e]) => ({
168
+ Key: key,
169
+ Occurrences: e.occurrences,
170
+ TotalRows: totalRows,
171
+ SampleValues: e.samples,
172
+ }));
173
+ }
174
+ /**
175
+ * Sanitizes a source key into a safe SQL/MJ column identifier (letters, digits, underscore;
176
+ * never leading-digit; bounded length). The original key is kept as the field map's
177
+ * SourceFieldName; the sanitized form becomes the column + DestinationFieldName. Returns the
178
+ * BASE name only — collision resolution (suffixing) is the orchestrator's job since it needs
179
+ * the existing-column context.
180
+ */
181
+ export function sanitizeColumnName(key) {
182
+ let name = key.replace(/[^A-Za-z0-9_]/g, '_').replace(/_+/g, '_').replace(/^_+|_+$/g, '');
183
+ if (name.length === 0)
184
+ name = 'Custom';
185
+ if (/^[0-9]/.test(name))
186
+ name = `c_${name}`;
187
+ return name.length > 120 ? name.slice(0, 120) : name;
188
+ }
189
+ function safeParseObject(raw) {
190
+ if (!raw || typeof raw !== 'string')
191
+ return null;
192
+ try {
193
+ const parsed = JSON.parse(raw);
194
+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
195
+ ? parsed
196
+ : null;
197
+ }
198
+ catch {
199
+ return null;
200
+ }
201
+ }
202
+ //# sourceMappingURL=CustomColumnPromotion.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CustomColumnPromotion.js","sourceRoot":"","sources":["../src/CustomColumnPromotion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAoDH,MAAM,0BAA0B,GAAG,CAAC,CAAC,CAAC,mEAAmE;AACzG,yFAAyF;AACzF,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,+FAA+F;AAC/F,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B;;;;;;GAMG;AACH,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEzC,4FAA4F;AAC5F,MAAM,WAAW,GAAG,6EAA6E,CAAC;AAElG;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC1B,KAAyB,EACzB,OAA6B,EAAE;IAE/B,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,IAAI,0BAA0B,CAAC;IACvE,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACxD,MAAM,UAAU,GAAyB,EAAE,CAAC;IAE5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,2DAA2D;QAC3D,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACnD,2FAA2F;QAC3F,6EAA6E;QAC7E,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC;YAAE,SAAS;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;QACnD,IAAI,QAAQ,GAAG,SAAS;YAAE,SAAS;QAEnC,UAAU,CAAC,IAAI,CAAC;YACZ,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,QAAQ,EAAE,QAAQ;YAClB,QAAQ,EAAE,0BAA0B,CAAC,IAAI,CAAC,YAAY,CAAC;SAC1D,CAAC,CAAC;IACP,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAAkB;IACzD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC;IAEnE,iDAAiD;IACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,UAAU,CAAC,gBAAgB,CAAC,CAAC;IAE9D,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1G,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAChC,8FAA8F;QAC9F,iGAAiG;QACjG,4FAA4F;QAC5F,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,IAAI,6BAA6B,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAW,CAAC,CAAC,CAAC;QAC9H,OAAO,gBAAgB;YACnB,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE;YACjG,iFAAiF;YACjF,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1H,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;QACjC,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1H,CAAC;IAED,6FAA6F;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,OAAO,UAAU,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,uFAAuF;AACvF,SAAS,UAAU,CAAC,KAAoB;IACpC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACjB,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAChH,CAAC;IACD,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,YAAY,KAAK,GAAG,EAAE,YAAY,EAAE,WAAW,KAAK,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACnI,CAAC;AAED;;;;GAIG;AACH,SAAS,mBAAmB,CAAC,eAAuB;IAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAChE,OAAO,OAAO,GAAG,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;AACzD,CAAC;AAED,SAAS,SAAS,CAAC,CAAU;IACzB,OAAO,OAAO,CAAC,KAAK,SAAS,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CAAC,CAAU;IAC9B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,eAAe,CAAC,CAAU;IAC/B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,YAAY,CAAC,GAAoC;IACtD,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,GAAG,EAAU,CAAC;IACrD,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,uFAAuF;AACvF,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,kBAAkB,CAC9B,mBAAqD,EACrD,YAAoB,kBAAkB;IAEtC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuD,CAAC;IAC7E,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;QACpC,SAAS,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS;YACpD,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,KAAK,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;gBACxC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC1B,CAAC;YACD,KAAK,CAAC,WAAW,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS;gBAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpE,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,GAAG,EAAE,GAAG;QACR,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,SAAS,EAAE,SAAS;QACpB,YAAY,EAAE,CAAC,CAAC,OAAO;KAC1B,CAAC,CAAC,CAAC;AACR,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC1C,IAAI,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC1F,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,IAAI,GAAG,QAAQ,CAAC;IACvC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;IAC5C,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACzD,CAAC;AAED,SAAS,eAAe,CAAC,GAA8B;IACnD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACjD,IAAI,CAAC;QACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YACjE,CAAC,CAAE,MAAkC;YACrC,CAAC,CAAC,IAAI,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC"}
@@ -0,0 +1,64 @@
1
+ /**
2
+ * Name of the per-record custom-overflow column. Written on every integration table
3
+ * alongside the other `__mj_integration_*` columns when present. Holds a JSON object of
4
+ * the source fields a record returned that have NO field map — the "extra" keys the
5
+ * target table doesn't (yet) have a column for.
6
+ *
7
+ * This is the capture half of framework-level custom-column support (gaps.md §2). A
8
+ * source with no schema/describe endpoint (the PropFuel class) can only learn its full
9
+ * field set by reading data; a static catalog then silently drops any extra key at
10
+ * {@link MapSingleRecord} (only mapped fields get persisted). Rather than lose them, the
11
+ * unmapped keys are parked here, on the row, in the SAME write as the mapped fields — so
12
+ * capture costs no extra round-trip. A post-sync Runtime-Schema-Updation (RSU) pass later
13
+ * reads this column back out and promotes pervasive keys to real columns (ADD COLUMN +
14
+ * CodeGen + a field map). It does NOT backfill historical values: once the field map exists
15
+ * the key is no longer "unmapped", so the NEXT sync maps it natively and populates the new
16
+ * column with the LIVE source value (a full sync repopulates every row; an incremental fills
17
+ * rows as they next change) — and stops re-capturing it here. The loop converges go-forward.
18
+ *
19
+ * Shape mirrors {@link CONTENT_HASH_COLUMN}: a JSON-serialized string sidecar, identical
20
+ * in kind to `__mj_integration_LastSyncedSnapshot`. SS `NVARCHAR(MAX)` / PG `TEXT`; the
21
+ * post-sync coverage scan + spread cast `::jsonb` on Postgres when they query it.
22
+ *
23
+ * This is BACKEND PLUMBING, not user-facing metadata. It is a SYSTEM field — in the same
24
+ * hidden/internal class as `__mj_integration_ContentHash` & the other `__mj_integration_*`
25
+ * columns: metadata-registered only so the engine's `entity.Set(...)`/`entity.Fields` write
26
+ * path is permitted to touch it (so it is NOT an out-of-band "phantom" column the engine
27
+ * writes blind), but NOT surfaced or managed through the metadata-driven framework — not in
28
+ * forms, not user-mappable, not "a field" anyone works with. It is machinery.
29
+ *
30
+ * The things that genuinely ENTER the metadata-driven framework are the columns this gets
31
+ * PROMOTED into: a pervasive key here is turned into a real user-facing, mappable EntityField
32
+ * (IntegrationSchemaSync IOF → RSU ADD COLUMN → CodeGen → md.Refresh) BEFORE the engine maps
33
+ * to it. So: backend staging until promotion, first-class metadata-driven schema after. Only
34
+ * the JSON *values* parked here are ever transient.
35
+ *
36
+ * Gated everywhere by EntityInfo field presence, so it is a no-op on tables that predate
37
+ * the column. CRUCIAL invariant: when a record has NO unmapped keys, NOTHING is written
38
+ * here — empty across all rows is the signal that no RSU pass is needed, which is what
39
+ * keeps a customs-free sync byte-identical to today (single-stage, zero overhead).
40
+ */
41
+ export declare const CUSTOM_OVERFLOW_COLUMN = "__mj_integration_CustomOverflow";
42
+ /**
43
+ * Computes the unmapped subset of a record's external fields: every key the source
44
+ * returned that is NOT the SourceFieldName of any active field map.
45
+ *
46
+ * Pure + O(columns) — no per-row allocation beyond the result object, which is empty
47
+ * (and discarded by the caller) for the common no-customs case.
48
+ *
49
+ * Honest limit (documented in gaps.md §2): a field consumed only indirectly — e.g. as a
50
+ * `combine`/`custom` transform input rather than as a map's own SourceFieldName — will
51
+ * also appear here. That is harmless (the value is still in `Fields`); the post-sync
52
+ * promotion stage applies a coverage + novelty filter before it ever creates a column.
53
+ *
54
+ * @param externalFields - the full source record (`ExternalRecord.Fields`)
55
+ * @param mappedSourceFieldNames - SourceFieldName of every active field map
56
+ * @returns the extra keys + values, or an empty object when everything was mapped
57
+ */
58
+ export declare function computeUnmappedFields(externalFields: Record<string, unknown>, mappedSourceFieldNames: ReadonlySet<string>): Record<string, unknown>;
59
+ /**
60
+ * Whether a computed unmapped-field object carries anything worth persisting. Used by
61
+ * both the per-record write gate and (later) the post-sync RSU trigger gate.
62
+ */
63
+ export declare function hasUnmappedFields(unmapped: Record<string, unknown> | undefined | null): boolean;
64
+ //# sourceMappingURL=CustomOverflow.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CustomOverflow.d.ts","sourceRoot":"","sources":["../src/CustomOverflow.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,eAAO,MAAM,sBAAsB,oCAAoC,CAAC;AAExE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CACjC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACvC,sBAAsB,EAAE,WAAW,CAAC,MAAM,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAQzB;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI,GAAG,OAAO,CAE/F"}
@@ -0,0 +1,74 @@
1
+ /**
2
+ * Name of the per-record custom-overflow column. Written on every integration table
3
+ * alongside the other `__mj_integration_*` columns when present. Holds a JSON object of
4
+ * the source fields a record returned that have NO field map — the "extra" keys the
5
+ * target table doesn't (yet) have a column for.
6
+ *
7
+ * This is the capture half of framework-level custom-column support (gaps.md §2). A
8
+ * source with no schema/describe endpoint (the PropFuel class) can only learn its full
9
+ * field set by reading data; a static catalog then silently drops any extra key at
10
+ * {@link MapSingleRecord} (only mapped fields get persisted). Rather than lose them, the
11
+ * unmapped keys are parked here, on the row, in the SAME write as the mapped fields — so
12
+ * capture costs no extra round-trip. A post-sync Runtime-Schema-Updation (RSU) pass later
13
+ * reads this column back out and promotes pervasive keys to real columns (ADD COLUMN +
14
+ * CodeGen + a field map). It does NOT backfill historical values: once the field map exists
15
+ * the key is no longer "unmapped", so the NEXT sync maps it natively and populates the new
16
+ * column with the LIVE source value (a full sync repopulates every row; an incremental fills
17
+ * rows as they next change) — and stops re-capturing it here. The loop converges go-forward.
18
+ *
19
+ * Shape mirrors {@link CONTENT_HASH_COLUMN}: a JSON-serialized string sidecar, identical
20
+ * in kind to `__mj_integration_LastSyncedSnapshot`. SS `NVARCHAR(MAX)` / PG `TEXT`; the
21
+ * post-sync coverage scan + spread cast `::jsonb` on Postgres when they query it.
22
+ *
23
+ * This is BACKEND PLUMBING, not user-facing metadata. It is a SYSTEM field — in the same
24
+ * hidden/internal class as `__mj_integration_ContentHash` & the other `__mj_integration_*`
25
+ * columns: metadata-registered only so the engine's `entity.Set(...)`/`entity.Fields` write
26
+ * path is permitted to touch it (so it is NOT an out-of-band "phantom" column the engine
27
+ * writes blind), but NOT surfaced or managed through the metadata-driven framework — not in
28
+ * forms, not user-mappable, not "a field" anyone works with. It is machinery.
29
+ *
30
+ * The things that genuinely ENTER the metadata-driven framework are the columns this gets
31
+ * PROMOTED into: a pervasive key here is turned into a real user-facing, mappable EntityField
32
+ * (IntegrationSchemaSync IOF → RSU ADD COLUMN → CodeGen → md.Refresh) BEFORE the engine maps
33
+ * to it. So: backend staging until promotion, first-class metadata-driven schema after. Only
34
+ * the JSON *values* parked here are ever transient.
35
+ *
36
+ * Gated everywhere by EntityInfo field presence, so it is a no-op on tables that predate
37
+ * the column. CRUCIAL invariant: when a record has NO unmapped keys, NOTHING is written
38
+ * here — empty across all rows is the signal that no RSU pass is needed, which is what
39
+ * keeps a customs-free sync byte-identical to today (single-stage, zero overhead).
40
+ */
41
+ export const CUSTOM_OVERFLOW_COLUMN = '__mj_integration_CustomOverflow';
42
+ /**
43
+ * Computes the unmapped subset of a record's external fields: every key the source
44
+ * returned that is NOT the SourceFieldName of any active field map.
45
+ *
46
+ * Pure + O(columns) — no per-row allocation beyond the result object, which is empty
47
+ * (and discarded by the caller) for the common no-customs case.
48
+ *
49
+ * Honest limit (documented in gaps.md §2): a field consumed only indirectly — e.g. as a
50
+ * `combine`/`custom` transform input rather than as a map's own SourceFieldName — will
51
+ * also appear here. That is harmless (the value is still in `Fields`); the post-sync
52
+ * promotion stage applies a coverage + novelty filter before it ever creates a column.
53
+ *
54
+ * @param externalFields - the full source record (`ExternalRecord.Fields`)
55
+ * @param mappedSourceFieldNames - SourceFieldName of every active field map
56
+ * @returns the extra keys + values, or an empty object when everything was mapped
57
+ */
58
+ export function computeUnmappedFields(externalFields, mappedSourceFieldNames) {
59
+ const unmapped = {};
60
+ for (const key of Object.keys(externalFields)) {
61
+ if (!mappedSourceFieldNames.has(key)) {
62
+ unmapped[key] = externalFields[key];
63
+ }
64
+ }
65
+ return unmapped;
66
+ }
67
+ /**
68
+ * Whether a computed unmapped-field object carries anything worth persisting. Used by
69
+ * both the per-record write gate and (later) the post-sync RSU trigger gate.
70
+ */
71
+ export function hasUnmappedFields(unmapped) {
72
+ return unmapped != null && Object.keys(unmapped).length > 0;
73
+ }
74
+ //# sourceMappingURL=CustomOverflow.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CustomOverflow.js","sourceRoot":"","sources":["../src/CustomOverflow.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,iCAAiC,CAAC;AAExE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,qBAAqB,CACjC,cAAuC,EACvC,sBAA2C;IAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,QAAQ,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAoD;IAClF,OAAO,QAAQ,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AAChE,CAAC"}
@@ -1,24 +1,23 @@
1
1
  import type { ICompanyIntegrationFieldMap } from './entity-types.js';
2
2
  import type { ExternalRecord, MappedRecord } from './types.js';
3
3
  /**
4
- * Engine responsible for applying field-level mappings and transformations
5
- * from external records to MJ entity fields.
4
+ * Engine responsible for applying field-level mappings and transformations from external records to
5
+ * MJ entity fields.
6
+ *
7
+ * The per-value transform pipeline (direct / regex / split / combine / lookup / format / coerce /
8
+ * substring / custom) is owned by the shared {@link FieldTransformEngine} in `@memberjunction/global`
9
+ * and reused here, so integration sync and the rules-based bulk-update processor run the *exact same*
10
+ * transform implementation — one place to fix, one place to extend. This engine keeps only the
11
+ * integration-specific concerns: source flattening, field-map iteration, and unmapped-field overflow
12
+ * capture. See the Field Rules guide in `@memberjunction/global` for the shared engine, and
13
+ * `EntityFieldRules` in `@memberjunction/core` for the metadata-aware entity-update sibling.
6
14
  */
7
15
  export declare class FieldMappingEngine {
16
+ /** The shared transform pipeline. Holds its own LRU cache of compiled custom expressions. */
17
+ private readonly transformEngine;
8
18
  /**
9
- * Cache of compiled custom-expression functions, keyed by the expression string, so an
10
- * identical expression is compiled once instead of once per record in a mapping batch.
11
- *
12
- * A failed compile is cached as the `Error` it threw — a malformed expression is therefore
13
- * compiled a single time (then re-thrown from cache on every record) rather than recompiled
14
- * on the per-record error path. Bounded via {@link MJLruCache} because the owning
15
- * `IntegrationEngine` is a process-lifetime singleton; expression cardinality is normally
16
- * tiny (it tracks configured field maps), so the 1000-entry default is comfortably ample.
17
- */
18
- private readonly customFunctionCache;
19
- /**
20
- * Applies field mappings to a batch of external records, producing mapped records
21
- * ready for match resolution and persistence.
19
+ * Applies field mappings to a batch of external records, producing mapped records ready for match
20
+ * resolution and persistence.
22
21
  *
23
22
  * @param records - External records to map
24
23
  * @param fieldMaps - Active field map entities defining source→destination mappings
@@ -31,86 +30,13 @@ export declare class FieldMappingEngine {
31
30
  */
32
31
  private MapSingleRecord;
33
32
  /**
34
- * Applies a single field mapping, including the full transform pipeline.
35
- * Returns undefined if the field should be skipped (OnError: Skip).
33
+ * Applies a single field mapping, including the full transform pipeline (delegated to the shared
34
+ * {@link FieldTransformEngine}). Returns undefined if the field should be skipped (OnError: Skip).
36
35
  */
37
36
  private ApplyFieldMapping;
38
37
  /**
39
38
  * Parses the JSON transform pipeline string into typed TransformStep objects.
40
39
  */
41
40
  private ParseTransformPipeline;
42
- /**
43
- * Executes a single transform step, applying the configured transformation
44
- * and handling errors according to the step's OnError strategy.
45
- */
46
- private ExecuteTransformStep;
47
- /**
48
- * Dispatches to the appropriate transform handler based on step type.
49
- */
50
- private DispatchTransform;
51
- /**
52
- * Handles a transform error according to the OnError strategy.
53
- */
54
- private HandleTransformError;
55
- /**
56
- * Direct pass-through, applying default value if source is null/undefined.
57
- */
58
- private ApplyDirect;
59
- /**
60
- * Applies a regex replacement to a string value.
61
- */
62
- private ApplyRegex;
63
- /**
64
- * Splits a string value and extracts a part by index.
65
- */
66
- private ApplySplit;
67
- /**
68
- * Combines multiple source fields with a separator.
69
- */
70
- private ApplyCombine;
71
- /**
72
- * Performs a case-insensitive value lookup/mapping.
73
- */
74
- private ApplyLookup;
75
- /**
76
- * Applies date/number/string formatting.
77
- */
78
- private ApplyFormat;
79
- /**
80
- * Basic date formatting supporting ISO and common format tokens.
81
- */
82
- private FormatDate;
83
- /**
84
- * Coerces a value to the specified target type.
85
- */
86
- private ApplyCoerce;
87
- /**
88
- * Coerces a value to a number, throwing on NaN.
89
- */
90
- private CoerceToNumber;
91
- /**
92
- * Coerces a value to boolean, supporting common truthy/falsy strings.
93
- */
94
- private CoerceToBoolean;
95
- /**
96
- * Extracts a substring from a string value.
97
- */
98
- private ApplySubstring;
99
- /**
100
- * Evaluates a custom JavaScript expression.
101
- * The expression has access to `value` (current field value) and `fields` (all record fields).
102
- */
103
- private ApplyCustom;
104
- /**
105
- * Returns the compiled function for an expression, compiling and caching on first use.
106
- * If the expression failed to compile previously, the cached error is re-thrown — a
107
- * malformed expression is compiled exactly once across an entire batch.
108
- */
109
- private GetCompiledExpression;
110
- /**
111
- * Compiles a custom expression into a callable, returning the thrown `Error` instead of
112
- * propagating it so the caller can cache compile failures alongside successes.
113
- */
114
- private compileExpression;
115
41
  }
116
42
  //# sourceMappingURL=FieldMappingEngine.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"FieldMappingEngine.d.ts","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAe/D;;;GAGG;AACH,qBAAa,kBAAkB;IAC3B;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwD;IAE5F;;;;;;;;OAQG;IACI,KAAK,CACR,OAAO,EAAE,cAAc,EAAE,EACzB,SAAS,EAAE,2BAA2B,EAAE,EACxC,UAAU,EAAE,MAAM,GACnB,YAAY,EAAE;IAKjB;;OAEG;IACH,OAAO,CAAC,eAAe;IAsBvB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAgBzB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAY9B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA6BzB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAW5B;;OAEG;IACH,OAAO,CAAC,WAAW;IAOnB;;OAEG;IACH,OAAO,CAAC,UAAU;IAMlB;;OAEG;IACH,OAAO,CAAC,UAAU;IAMlB;;OAEG;IACH,OAAO,CAAC,YAAY;IAKpB;;OAEG;IACH,OAAO,CAAC,WAAW;IASnB;;OAEG;IACH,OAAO,CAAC,WAAW;IAkBnB;;OAEG;IACH,OAAO,CAAC,UAAU;IAOlB;;OAEG;IACH,OAAO,CAAC,WAAW;IAoBnB;;OAEG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAOvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAY7B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;CAQ5B"}
1
+ {"version":3,"file":"FieldMappingEngine.d.ts","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAG/D;;;;;;;;;;;GAWG;AACH,qBAAa,kBAAkB;IAC3B,6FAA6F;IAC7F,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA8B;IAE9D;;;;;;;;OAQG;IACI,KAAK,CACR,OAAO,EAAE,cAAc,EAAE,EACzB,SAAS,EAAE,2BAA2B,EAAE,EACxC,UAAU,EAAE,MAAM,GACnB,YAAY,EAAE;IAKjB;;OAEG;IACH,OAAO,CAAC,eAAe;IAyCvB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAUzB;;OAEG;IACH,OAAO,CAAC,sBAAsB;CAWjC"}