@memberjunction/integration-engine 5.41.0 → 5.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -0
- package/dist/BaseIntegrationConnector.d.ts +109 -0
- package/dist/BaseIntegrationConnector.d.ts.map +1 -1
- package/dist/BaseIntegrationConnector.js +308 -93
- package/dist/BaseIntegrationConnector.js.map +1 -1
- package/dist/BaseRESTIntegrationConnector.d.ts +36 -0
- package/dist/BaseRESTIntegrationConnector.d.ts.map +1 -1
- package/dist/BaseRESTIntegrationConnector.js +228 -28
- package/dist/BaseRESTIntegrationConnector.js.map +1 -1
- package/dist/ContentHash.d.ts +22 -0
- package/dist/ContentHash.d.ts.map +1 -1
- package/dist/ContentHash.js +35 -0
- package/dist/ContentHash.js.map +1 -1
- package/dist/CustomColumnPromotion.d.ts +108 -0
- package/dist/CustomColumnPromotion.d.ts.map +1 -0
- package/dist/CustomColumnPromotion.js +202 -0
- package/dist/CustomColumnPromotion.js.map +1 -0
- package/dist/CustomOverflow.d.ts +64 -0
- package/dist/CustomOverflow.d.ts.map +1 -0
- package/dist/CustomOverflow.js +74 -0
- package/dist/CustomOverflow.js.map +1 -0
- package/dist/FieldMappingEngine.d.ts +16 -90
- package/dist/FieldMappingEngine.d.ts.map +1 -1
- package/dist/FieldMappingEngine.js +42 -257
- package/dist/FieldMappingEngine.js.map +1 -1
- package/dist/IntegrationConnectorCreationPipeline.d.ts +33 -0
- package/dist/IntegrationConnectorCreationPipeline.d.ts.map +1 -1
- package/dist/IntegrationConnectorCreationPipeline.js +157 -0
- package/dist/IntegrationConnectorCreationPipeline.js.map +1 -1
- package/dist/IntegrationEngine.d.ts +120 -6
- package/dist/IntegrationEngine.d.ts.map +1 -1
- package/dist/IntegrationEngine.js +890 -168
- package/dist/IntegrationEngine.js.map +1 -1
- package/dist/IntegrationSchemaSync.d.ts +84 -13
- package/dist/IntegrationSchemaSync.d.ts.map +1 -1
- package/dist/IntegrationSchemaSync.js +198 -36
- package/dist/IntegrationSchemaSync.js.map +1 -1
- package/dist/KeySerialization.d.ts +21 -0
- package/dist/KeySerialization.d.ts.map +1 -0
- package/dist/KeySerialization.js +29 -0
- package/dist/KeySerialization.js.map +1 -0
- package/dist/MatchEngine.d.ts.map +1 -1
- package/dist/MatchEngine.js +12 -4
- package/dist/MatchEngine.js.map +1 -1
- package/dist/RecordFlatten.d.ts +49 -0
- package/dist/RecordFlatten.d.ts.map +1 -0
- package/dist/RecordFlatten.js +50 -0
- package/dist/RecordFlatten.js.map +1 -0
- package/dist/StreamingDiscovery.d.ts +145 -0
- package/dist/StreamingDiscovery.d.ts.map +1 -0
- package/dist/StreamingDiscovery.js +306 -0
- package/dist/StreamingDiscovery.js.map +1 -0
- package/dist/SyncLogger.d.ts +1 -1
- package/dist/SyncLogger.d.ts.map +1 -1
- package/dist/SyncLogger.js +19 -1
- package/dist/SyncLogger.js.map +1 -1
- package/dist/auth-helpers/BasicAuthHeaderBuilder.d.ts +36 -0
- package/dist/auth-helpers/BasicAuthHeaderBuilder.d.ts.map +1 -0
- package/dist/auth-helpers/BasicAuthHeaderBuilder.js +38 -0
- package/dist/auth-helpers/BasicAuthHeaderBuilder.js.map +1 -0
- package/dist/auth-helpers/OAuth1aSigner.d.ts +56 -0
- package/dist/auth-helpers/OAuth1aSigner.d.ts.map +1 -0
- package/dist/auth-helpers/OAuth1aSigner.js +91 -0
- package/dist/auth-helpers/OAuth1aSigner.js.map +1 -0
- package/dist/auth-helpers/OAuth2TokenManager.d.ts +103 -0
- package/dist/auth-helpers/OAuth2TokenManager.d.ts.map +1 -0
- package/dist/auth-helpers/OAuth2TokenManager.js +143 -0
- package/dist/auth-helpers/OAuth2TokenManager.js.map +1 -0
- package/dist/auth-helpers/index.d.ts +11 -0
- package/dist/auth-helpers/index.d.ts.map +1 -0
- package/dist/auth-helpers/index.js +8 -0
- package/dist/auth-helpers/index.js.map +1 -0
- package/dist/index.d.ts +15 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +73 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +7 -7
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promotion planner — the brain of the post-sync custom-column step (gaps.md §2, M2).
|
|
3
|
+
*
|
|
4
|
+
* After a sync, the values a source returned with no field map are parked as JSON in the
|
|
5
|
+
* {@link CUSTOM_OVERFLOW_COLUMN} system column (M1 capture). This module decides, from a
|
|
6
|
+
* coverage scan of that column, WHICH of those keys earn a real column and WHAT bounded
|
|
7
|
+
* type each should get. It is PURE — no DB, no RSU, no I/O — so the genuinely new logic is
|
|
8
|
+
* fully unit-testable. The server-side orchestrator (M2b) feeds the resulting plan into the
|
|
9
|
+
* EXISTING refresh pipeline (SchemaEvolution ADD COLUMN → RSU → IntegrationSchemaSync IOF →
|
|
10
|
+
* field map); this module never touches schema itself.
|
|
11
|
+
*
|
|
12
|
+
* Design rules (from gaps.md §2 + the connector-code conventions):
|
|
13
|
+
* - Provable-only / never fabricate: a key earns a column only on PERVASIVENESS evidence
|
|
14
|
+
* (coverage ≥ threshold), never on a single malformed row. PK/FK are NEVER inferred here —
|
|
15
|
+
* customs are always emitted nullable, non-PK, non-FK (classification is deferred to D4).
|
|
16
|
+
* - Generous bounded typing (the NVARCHAR(MAX) problem): size columns comfortably from the
|
|
17
|
+
* observed samples and err LARGER — a roomy bounded column beats a truncating tight one,
|
|
18
|
+
* and both beat MAX. Only fall back to MAX/TEXT when the observed length genuinely can't be
|
|
19
|
+
* bounded. Narrow to number/boolean/datetime ONLY when every non-null sample unambiguously
|
|
20
|
+
* supports it; otherwise default to a (generously bounded) string — that is the safe choice
|
|
21
|
+
* that can hold anything the source later returns.
|
|
22
|
+
* - Terminate / never re-promote: a key whose column already exists is skipped, so the
|
|
23
|
+
* capture→promote loop converges instead of re-promoting forever.
|
|
24
|
+
*/
|
|
25
|
+
/** The schema-builder field-type family + the concrete per-dialect SQL types we infer. */
|
|
26
|
+
export interface InferredColumnType {
|
|
27
|
+
/** Schema-builder SchemaFieldType family ('string' | 'number' | 'boolean' | 'datetime'). */
|
|
28
|
+
SchemaFieldType: 'string' | 'number' | 'boolean' | 'datetime';
|
|
29
|
+
/** Concrete SQL Server column type, e.g. `NVARCHAR(255)`, `BIGINT`, `DATETIMEOFFSET`. */
|
|
30
|
+
SqlServerType: string;
|
|
31
|
+
/** Concrete PostgreSQL column type, e.g. `VARCHAR(255)`, `BIGINT`, `TIMESTAMPTZ`. */
|
|
32
|
+
PostgresType: string;
|
|
33
|
+
/** Bound for string types (null for non-string or when unbounded → MAX/TEXT). */
|
|
34
|
+
MaxLength: number | null;
|
|
35
|
+
}
|
|
36
|
+
/** Per-key statistics from a coverage scan of the overflow column for one (CompanyIntegration, entity). */
|
|
37
|
+
export interface OverflowKeyStats {
|
|
38
|
+
/** The source key as it appears in the overflow JSON. */
|
|
39
|
+
Key: string;
|
|
40
|
+
/** Rows in which the key was present with a non-null value. */
|
|
41
|
+
Occurrences: number;
|
|
42
|
+
/** Total rows scanned (rows that had any overflow JSON). */
|
|
43
|
+
TotalRows: number;
|
|
44
|
+
/** A bounded sample of observed values for this key, for type inference. */
|
|
45
|
+
SampleValues: unknown[];
|
|
46
|
+
}
|
|
47
|
+
/** A key that earned a real column, with its inferred bounded type. */
|
|
48
|
+
export interface PromotionCandidate {
|
|
49
|
+
/** The source key → becomes both the SourceFieldName (field map) and the column name. */
|
|
50
|
+
Key: string;
|
|
51
|
+
/** Pervasiveness in [0,1] = Occurrences / TotalRows. */
|
|
52
|
+
Coverage: number;
|
|
53
|
+
/** The inferred, generously-bounded column type. */
|
|
54
|
+
Inferred: InferredColumnType;
|
|
55
|
+
}
|
|
56
|
+
/** Tuning + context for {@link planPromotions}. */
|
|
57
|
+
export interface PromotionPlanOptions {
|
|
58
|
+
/**
|
|
59
|
+
* Minimum coverage (in [0,1]) a key must clear to earn a column. Default 0 — §23: a custom key
|
|
60
|
+
* earns a column on FIRST occurrence (presence, not prevalence); appearing in even one row is
|
|
61
|
+
* enough. Raise this only if a caller deliberately wants to suppress sparse keys. The captured
|
|
62
|
+
* values are never lost regardless — they stay in the overflow JSON until promoted.
|
|
63
|
+
*/
|
|
64
|
+
CoverageThreshold?: number;
|
|
65
|
+
/**
|
|
66
|
+
* Column names that already exist on the target entity (case-insensitive). A key whose
|
|
67
|
+
* column already exists is NOT re-promoted — this is what makes the loop terminate.
|
|
68
|
+
*/
|
|
69
|
+
ExistingColumnNames?: ReadonlySet<string>;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Plans which overflow keys to promote to real columns. Pure; deterministic (sorted by key).
|
|
73
|
+
*
|
|
74
|
+
* @param stats - per-key coverage statistics from the overflow scan
|
|
75
|
+
* @param opts - coverage threshold + the set of already-existing column names
|
|
76
|
+
* @returns the promotion candidates, sorted by key for stable, replayable output
|
|
77
|
+
*/
|
|
78
|
+
export declare function planPromotions(stats: OverflowKeyStats[], opts?: PromotionPlanOptions): PromotionCandidate[];
|
|
79
|
+
/**
|
|
80
|
+
* Infers a generously-bounded column type from observed sample values. Narrows to
|
|
81
|
+
* boolean/number/datetime ONLY when EVERY non-null sample unambiguously supports it;
|
|
82
|
+
* otherwise defaults to a comfortably-bounded string. Never returns MAX/TEXT unless the
|
|
83
|
+
* observed string length genuinely can't be bounded.
|
|
84
|
+
*/
|
|
85
|
+
export declare function inferColumnTypeFromSamples(samples: unknown[]): InferredColumnType;
|
|
86
|
+
/**
|
|
87
|
+
* Builds per-key coverage statistics from a SAMPLE of rows that carried overflow JSON.
|
|
88
|
+
* Each row's overflow column is a JSON object string (what M1 parked); this tallies, per key,
|
|
89
|
+
* how many sampled rows had it non-null and collects a bounded sample of its values for type
|
|
90
|
+
* inference. Robust to malformed/empty JSON (such rows contribute to TotalRows but no keys).
|
|
91
|
+
*
|
|
92
|
+
* Coverage is therefore computed over the SAMPLE (bounded memory at any table size) — a
|
|
93
|
+
* statistically sound basis for the pervasiveness decision; the exactness the threshold needs
|
|
94
|
+
* (a key on ~half the rows vs. a junk key on one) survives sampling.
|
|
95
|
+
*
|
|
96
|
+
* @param overflowJsonStrings - the raw overflow-column value from each sampled row
|
|
97
|
+
* @param sampleCap - max values retained per key (default {@link DEFAULT_SAMPLE_CAP})
|
|
98
|
+
*/
|
|
99
|
+
export declare function buildOverflowStats(overflowJsonStrings: Array<string | null | undefined>, sampleCap?: number): OverflowKeyStats[];
|
|
100
|
+
/**
|
|
101
|
+
* Sanitizes a source key into a safe SQL/MJ column identifier (letters, digits, underscore;
|
|
102
|
+
* never leading-digit; bounded length). The original key is kept as the field map's
|
|
103
|
+
* SourceFieldName; the sanitized form becomes the column + DestinationFieldName. Returns the
|
|
104
|
+
* BASE name only — collision resolution (suffixing) is the orchestrator's job since it needs
|
|
105
|
+
* the existing-column context.
|
|
106
|
+
*/
|
|
107
|
+
export declare function sanitizeColumnName(key: string): string;
|
|
108
|
+
//# sourceMappingURL=CustomColumnPromotion.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CustomColumnPromotion.d.ts","sourceRoot":"","sources":["../src/CustomColumnPromotion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,0FAA0F;AAC1F,MAAM,WAAW,kBAAkB;IAC/B,4FAA4F;IAC5F,eAAe,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,CAAC;IAC9D,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;IACtB,qFAAqF;IACrF,YAAY,EAAE,MAAM,CAAC;IACrB,iFAAiF;IACjF,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,2GAA2G;AAC3G,MAAM,WAAW,gBAAgB;IAC7B,yDAAyD;IACzD,GAAG,EAAE,MAAM,CAAC;IACZ,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,YAAY,EAAE,OAAO,EAAE,CAAC;CAC3B;AAED,uEAAuE;AACvE,MAAM,WAAW,kBAAkB;IAC/B,yFAAyF;IACzF,GAAG,EAAE,MAAM,CAAC;IACZ,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,QAAQ,EAAE,kBAAkB,CAAC;CAChC;AAED,mDAAmD;AACnD,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAC7C;AAmBD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC1B,KAAK,EAAE,gBAAgB,EAAE,EACzB,IAAI,GAAE,oBAAyB,GAChC,kBAAkB,EAAE,CAsBtB;AAED;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,kBAAkB,CA4BjF;AAwCD;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAC9B,mBAAmB,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,EACrD,SAAS,GAAE,MAA2B,GACvC,gBAAgB,EAAE,CA0BpB;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKtD"}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promotion planner — the brain of the post-sync custom-column step (gaps.md §2, M2).
|
|
3
|
+
*
|
|
4
|
+
* After a sync, the values a source returned with no field map are parked as JSON in the
|
|
5
|
+
* {@link CUSTOM_OVERFLOW_COLUMN} system column (M1 capture). This module decides, from a
|
|
6
|
+
* coverage scan of that column, WHICH of those keys earn a real column and WHAT bounded
|
|
7
|
+
* type each should get. It is PURE — no DB, no RSU, no I/O — so the genuinely new logic is
|
|
8
|
+
* fully unit-testable. The server-side orchestrator (M2b) feeds the resulting plan into the
|
|
9
|
+
* EXISTING refresh pipeline (SchemaEvolution ADD COLUMN → RSU → IntegrationSchemaSync IOF →
|
|
10
|
+
* field map); this module never touches schema itself.
|
|
11
|
+
*
|
|
12
|
+
* Design rules (from gaps.md §2 + the connector-code conventions):
|
|
13
|
+
* - Provable-only / never fabricate: a key earns a column only on PERVASIVENESS evidence
|
|
14
|
+
* (coverage ≥ threshold), never on a single malformed row. PK/FK are NEVER inferred here —
|
|
15
|
+
* customs are always emitted nullable, non-PK, non-FK (classification is deferred to D4).
|
|
16
|
+
* - Generous bounded typing (the NVARCHAR(MAX) problem): size columns comfortably from the
|
|
17
|
+
* observed samples and err LARGER — a roomy bounded column beats a truncating tight one,
|
|
18
|
+
* and both beat MAX. Only fall back to MAX/TEXT when the observed length genuinely can't be
|
|
19
|
+
* bounded. Narrow to number/boolean/datetime ONLY when every non-null sample unambiguously
|
|
20
|
+
* supports it; otherwise default to a (generously bounded) string — that is the safe choice
|
|
21
|
+
* that can hold anything the source later returns.
|
|
22
|
+
* - Terminate / never re-promote: a key whose column already exists is skipped, so the
|
|
23
|
+
* capture→promote loop converges instead of re-promoting forever.
|
|
24
|
+
*/
|
|
25
|
+
const DEFAULT_COVERAGE_THRESHOLD = 0; // §23 — presence-based: a key seen in even one row earns a column.
|
|
26
|
+
/** SQL Server's largest bounded NVARCHAR before MAX; above this we must use MAX/TEXT. */
|
|
27
|
+
const MAX_BOUNDED_STRING = 4000;
|
|
28
|
+
/** Floor for a generous string bound — never size a custom string column tighter than this. */
|
|
29
|
+
const MIN_STRING_BOUND = 255;
|
|
30
|
+
/**
|
|
31
|
+
* Minimum all-integer sample before we trust the field is INTEGRAL and narrow to BIGINT (#A9). Below this
|
|
32
|
+
* a handful of integer samples does NOT prove the unscanned tail is integer-only — a later decimal would be
|
|
33
|
+
* silently rejected by BIGINT (dead-lettered). Under the threshold we use wide fixed-point (which holds
|
|
34
|
+
* integers too), so a later decimal never fails; the column can still re-promote to BIGINT once enough
|
|
35
|
+
* integer evidence accrues.
|
|
36
|
+
*/
|
|
37
|
+
const MIN_SAMPLE_FOR_INTEGER_NARROW = 12;
|
|
38
|
+
/** Matches an ISO-8601-ish date / datetime so we don't mistake "5" or "2020" for a date. */
|
|
39
|
+
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;
|
|
40
|
+
/**
|
|
41
|
+
* Plans which overflow keys to promote to real columns. Pure; deterministic (sorted by key).
|
|
42
|
+
*
|
|
43
|
+
* @param stats - per-key coverage statistics from the overflow scan
|
|
44
|
+
* @param opts - coverage threshold + the set of already-existing column names
|
|
45
|
+
* @returns the promotion candidates, sorted by key for stable, replayable output
|
|
46
|
+
*/
|
|
47
|
+
export function planPromotions(stats, opts = {}) {
|
|
48
|
+
const threshold = opts.CoverageThreshold ?? DEFAULT_COVERAGE_THRESHOLD;
|
|
49
|
+
const existing = lowercaseSet(opts.ExistingColumnNames);
|
|
50
|
+
const candidates = [];
|
|
51
|
+
for (const stat of stats) {
|
|
52
|
+
// Terminate / never re-promote: the column already exists.
|
|
53
|
+
if (existing.has(stat.Key.toLowerCase()))
|
|
54
|
+
continue;
|
|
55
|
+
// Guard div-by-zero only; §23 — presence is enough (default threshold 0), so a key seen in
|
|
56
|
+
// even one row earns a column. A higher bar is opt-in via CoverageThreshold.
|
|
57
|
+
if (stat.TotalRows <= 0)
|
|
58
|
+
continue;
|
|
59
|
+
const coverage = stat.Occurrences / stat.TotalRows;
|
|
60
|
+
if (coverage < threshold)
|
|
61
|
+
continue;
|
|
62
|
+
candidates.push({
|
|
63
|
+
Key: stat.Key,
|
|
64
|
+
Coverage: coverage,
|
|
65
|
+
Inferred: inferColumnTypeFromSamples(stat.SampleValues),
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
return candidates.sort((a, b) => a.Key.localeCompare(b.Key));
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Infers a generously-bounded column type from observed sample values. Narrows to
|
|
72
|
+
* boolean/number/datetime ONLY when EVERY non-null sample unambiguously supports it;
|
|
73
|
+
* otherwise defaults to a comfortably-bounded string. Never returns MAX/TEXT unless the
|
|
74
|
+
* observed string length genuinely can't be bounded.
|
|
75
|
+
*/
|
|
76
|
+
export function inferColumnTypeFromSamples(samples) {
|
|
77
|
+
const nonNull = samples.filter(v => v !== null && v !== undefined);
|
|
78
|
+
// No evidence → a safe, generous default string.
|
|
79
|
+
if (nonNull.length === 0)
|
|
80
|
+
return stringType(MIN_STRING_BOUND);
|
|
81
|
+
if (nonNull.every(isBoolean)) {
|
|
82
|
+
return { SchemaFieldType: 'boolean', SqlServerType: 'BIT', PostgresType: 'BOOLEAN', MaxLength: null };
|
|
83
|
+
}
|
|
84
|
+
if (nonNull.every(isFiniteNumber)) {
|
|
85
|
+
// Narrow to BIGINT only with an ADEQUATE all-integer sample (#A9). A small all-integer sample
|
|
86
|
+
// doesn't prove the field is always integral — a decimal in the unscanned tail would be silently
|
|
87
|
+
// rejected by BIGINT. Below the threshold, use wide fixed-point (which holds integers too).
|
|
88
|
+
const confidentInteger = nonNull.length >= MIN_SAMPLE_FOR_INTEGER_NARROW && nonNull.every(v => Number.isInteger(v));
|
|
89
|
+
return confidentInteger
|
|
90
|
+
? { SchemaFieldType: 'number', SqlServerType: 'BIGINT', PostgresType: 'BIGINT', MaxLength: null }
|
|
91
|
+
// Generous fixed-point — wide precision so we don't truncate or reject decimals.
|
|
92
|
+
: { SchemaFieldType: 'number', SqlServerType: 'DECIMAL(38,10)', PostgresType: 'DECIMAL(38,10)', MaxLength: null };
|
|
93
|
+
}
|
|
94
|
+
if (nonNull.every(isIsoDateString)) {
|
|
95
|
+
return { SchemaFieldType: 'datetime', SqlServerType: 'DATETIMEOFFSET', PostgresType: 'TIMESTAMPTZ', MaxLength: null };
|
|
96
|
+
}
|
|
97
|
+
// Default: a generously-bounded string sized to comfortably hold the longest observed value.
|
|
98
|
+
const longest = Math.max(...nonNull.map(v => String(v).length));
|
|
99
|
+
return stringType(generousStringBound(longest));
|
|
100
|
+
}
|
|
101
|
+
/** Builds a string {@link InferredColumnType}; bound===null ⇒ unbounded (MAX/TEXT). */
|
|
102
|
+
function stringType(bound) {
|
|
103
|
+
if (bound === null) {
|
|
104
|
+
return { SchemaFieldType: 'string', SqlServerType: 'NVARCHAR(MAX)', PostgresType: 'TEXT', MaxLength: null };
|
|
105
|
+
}
|
|
106
|
+
return { SchemaFieldType: 'string', SqlServerType: `NVARCHAR(${bound})`, PostgresType: `VARCHAR(${bound})`, MaxLength: bound };
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Generous bound for a string column: double the longest observed length (room to grow),
|
|
110
|
+
* floored at {@link MIN_STRING_BOUND}; if that would exceed the bounded limit, fall back to
|
|
111
|
+
* unbounded (null → MAX/TEXT) since it genuinely can't be bounded safely.
|
|
112
|
+
*/
|
|
113
|
+
function generousStringBound(longestObserved) {
|
|
114
|
+
const doubled = Math.max(longestObserved * 2, MIN_STRING_BOUND);
|
|
115
|
+
return doubled > MAX_BOUNDED_STRING ? null : doubled;
|
|
116
|
+
}
|
|
117
|
+
function isBoolean(v) {
|
|
118
|
+
return typeof v === 'boolean';
|
|
119
|
+
}
|
|
120
|
+
function isFiniteNumber(v) {
|
|
121
|
+
return typeof v === 'number' && Number.isFinite(v);
|
|
122
|
+
}
|
|
123
|
+
function isIsoDateString(v) {
|
|
124
|
+
return typeof v === 'string' && ISO_DATE_RE.test(v) && !Number.isNaN(Date.parse(v));
|
|
125
|
+
}
|
|
126
|
+
function lowercaseSet(set) {
|
|
127
|
+
if (!set || set.size === 0)
|
|
128
|
+
return new Set();
|
|
129
|
+
return new Set([...set].map(s => s.toLowerCase()));
|
|
130
|
+
}
|
|
131
|
+
/** Max distinct sample values retained per key for type inference (bounded memory). */
|
|
132
|
+
const DEFAULT_SAMPLE_CAP = 20;
|
|
133
|
+
/**
|
|
134
|
+
* Builds per-key coverage statistics from a SAMPLE of rows that carried overflow JSON.
|
|
135
|
+
* Each row's overflow column is a JSON object string (what M1 parked); this tallies, per key,
|
|
136
|
+
* how many sampled rows had it non-null and collects a bounded sample of its values for type
|
|
137
|
+
* inference. Robust to malformed/empty JSON (such rows contribute to TotalRows but no keys).
|
|
138
|
+
*
|
|
139
|
+
* Coverage is therefore computed over the SAMPLE (bounded memory at any table size) — a
|
|
140
|
+
* statistically sound basis for the pervasiveness decision; the exactness the threshold needs
|
|
141
|
+
* (a key on ~half the rows vs. a junk key on one) survives sampling.
|
|
142
|
+
*
|
|
143
|
+
* @param overflowJsonStrings - the raw overflow-column value from each sampled row
|
|
144
|
+
* @param sampleCap - max values retained per key (default {@link DEFAULT_SAMPLE_CAP})
|
|
145
|
+
*/
|
|
146
|
+
export function buildOverflowStats(overflowJsonStrings, sampleCap = DEFAULT_SAMPLE_CAP) {
|
|
147
|
+
const byKey = new Map();
|
|
148
|
+
let totalRows = 0;
|
|
149
|
+
for (const raw of overflowJsonStrings) {
|
|
150
|
+
totalRows++;
|
|
151
|
+
const parsed = safeParseObject(raw);
|
|
152
|
+
if (!parsed)
|
|
153
|
+
continue;
|
|
154
|
+
for (const [key, value] of Object.entries(parsed)) {
|
|
155
|
+
if (value === null || value === undefined)
|
|
156
|
+
continue;
|
|
157
|
+
let entry = byKey.get(key);
|
|
158
|
+
if (!entry) {
|
|
159
|
+
entry = { occurrences: 0, samples: [] };
|
|
160
|
+
byKey.set(key, entry);
|
|
161
|
+
}
|
|
162
|
+
entry.occurrences++;
|
|
163
|
+
if (entry.samples.length < sampleCap)
|
|
164
|
+
entry.samples.push(value);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return [...byKey.entries()].map(([key, e]) => ({
|
|
168
|
+
Key: key,
|
|
169
|
+
Occurrences: e.occurrences,
|
|
170
|
+
TotalRows: totalRows,
|
|
171
|
+
SampleValues: e.samples,
|
|
172
|
+
}));
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Sanitizes a source key into a safe SQL/MJ column identifier (letters, digits, underscore;
|
|
176
|
+
* never leading-digit; bounded length). The original key is kept as the field map's
|
|
177
|
+
* SourceFieldName; the sanitized form becomes the column + DestinationFieldName. Returns the
|
|
178
|
+
* BASE name only — collision resolution (suffixing) is the orchestrator's job since it needs
|
|
179
|
+
* the existing-column context.
|
|
180
|
+
*/
|
|
181
|
+
export function sanitizeColumnName(key) {
|
|
182
|
+
let name = key.replace(/[^A-Za-z0-9_]/g, '_').replace(/_+/g, '_').replace(/^_+|_+$/g, '');
|
|
183
|
+
if (name.length === 0)
|
|
184
|
+
name = 'Custom';
|
|
185
|
+
if (/^[0-9]/.test(name))
|
|
186
|
+
name = `c_${name}`;
|
|
187
|
+
return name.length > 120 ? name.slice(0, 120) : name;
|
|
188
|
+
}
|
|
189
|
+
function safeParseObject(raw) {
|
|
190
|
+
if (!raw || typeof raw !== 'string')
|
|
191
|
+
return null;
|
|
192
|
+
try {
|
|
193
|
+
const parsed = JSON.parse(raw);
|
|
194
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
195
|
+
? parsed
|
|
196
|
+
: null;
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
return null;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
//# sourceMappingURL=CustomColumnPromotion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CustomColumnPromotion.js","sourceRoot":"","sources":["../src/CustomColumnPromotion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAoDH,MAAM,0BAA0B,GAAG,CAAC,CAAC,CAAC,mEAAmE;AACzG,yFAAyF;AACzF,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,+FAA+F;AAC/F,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B;;;;;;GAMG;AACH,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEzC,4FAA4F;AAC5F,MAAM,WAAW,GAAG,6EAA6E,CAAC;AAElG;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC1B,KAAyB,EACzB,OAA6B,EAAE;IAE/B,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,IAAI,0BAA0B,CAAC;IACvE,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACxD,MAAM,UAAU,GAAyB,EAAE,CAAC;IAE5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,2DAA2D;QAC3D,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACnD,2FAA2F;QAC3F,6EAA6E;QAC7E,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC;YAAE,SAAS;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;QACnD,IAAI,QAAQ,GAAG,SAAS;YAAE,SAAS;QAEnC,UAAU,CAAC,IAAI,CAAC;YACZ,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,QAAQ,EAAE,QAAQ;YAClB,QAAQ,EAAE,0BAA0B,CAAC,IAAI,CAAC,YAAY,CAAC;SAC1D,CAAC,CAAC;IACP,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAAkB;IACzD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC;IAEnE,iDAAiD;IACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,UAAU,CAAC,gBAAgB,CAAC,CAAC;IAE9D,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1G,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAChC,8FAA8F;QAC9F,iGAAiG;QACjG,4FAA4F;QAC5F,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,IAAI,6BAA6B,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAW,CAAC,CAAC,CAAC;QAC9H,OAAO,gBAAgB;YACnB,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE;YACjG,iFAAiF;YACjF,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1H,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;QACjC,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1H,CAAC;IAED,6FAA6F;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,OAAO,UAAU,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,uFAAuF;AACvF,SAAS,UAAU,CAAC,KAAoB;IACpC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACjB,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAChH,CAAC;IACD,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,YAAY,KAAK,GAAG,EAAE,YAAY,EAAE,WAAW,KAAK,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACnI,CAAC;AAED;;;;GAIG;AACH,SAAS,mBAAmB,CAAC,eAAuB;IAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAChE,OAAO,OAAO,GAAG,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;AACzD,CAAC;AAED,SAAS,SAAS,CAAC,CAAU;IACzB,OAAO,OAAO,CAAC,KAAK,SAAS,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CAAC,CAAU;IAC9B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,eAAe,CAAC,CAAU;IAC/B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,YAAY,CAAC,GAAoC;IACtD,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,GAAG,EAAU,CAAC;IACrD,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,uFAAuF;AACvF,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,kBAAkB,CAC9B,mBAAqD,EACrD,YAAoB,kBAAkB;IAEtC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuD,CAAC;IAC7E,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;QACpC,SAAS,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS;YACpD,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,KAAK,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;gBACxC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC1B,CAAC;YACD,KAAK,CAAC,WAAW,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS;gBAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpE,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,GAAG,EAAE,GAAG;QACR,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,SAAS,EAAE,SAAS;QACpB,YAAY,EAAE,CAAC,CAAC,OAAO;KAC1B,CAAC,CAAC,CAAC;AACR,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC1C,IAAI,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC1F,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,IAAI,GAAG,QAAQ,CAAC;IACvC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;IAC5C,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACzD,CAAC;AAED,SAAS,eAAe,CAAC,GAA8B;IACnD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACjD,IAAI,CAAC;QACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YACjE,CAAC,CAAE,MAAkC;YACrC,CAAC,CAAC,IAAI,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Name of the per-record custom-overflow column. Written on every integration table
|
|
3
|
+
* alongside the other `__mj_integration_*` columns when present. Holds a JSON object of
|
|
4
|
+
* the source fields a record returned that have NO field map — the "extra" keys the
|
|
5
|
+
* target table doesn't (yet) have a column for.
|
|
6
|
+
*
|
|
7
|
+
* This is the capture half of framework-level custom-column support (gaps.md §2). A
|
|
8
|
+
* source with no schema/describe endpoint (the PropFuel class) can only learn its full
|
|
9
|
+
* field set by reading data; a static catalog then silently drops any extra key at
|
|
10
|
+
* {@link MapSingleRecord} (only mapped fields get persisted). Rather than lose them, the
|
|
11
|
+
* unmapped keys are parked here, on the row, in the SAME write as the mapped fields — so
|
|
12
|
+
* capture costs no extra round-trip. A post-sync Runtime-Schema-Updation (RSU) pass later
|
|
13
|
+
* reads this column back out and promotes pervasive keys to real columns (ADD COLUMN +
|
|
14
|
+
* CodeGen + a field map). It does NOT backfill historical values: once the field map exists
|
|
15
|
+
* the key is no longer "unmapped", so the NEXT sync maps it natively and populates the new
|
|
16
|
+
* column with the LIVE source value (a full sync repopulates every row; an incremental fills
|
|
17
|
+
* rows as they next change) — and stops re-capturing it here. The loop converges go-forward.
|
|
18
|
+
*
|
|
19
|
+
* Shape mirrors {@link CONTENT_HASH_COLUMN}: a JSON-serialized string sidecar, identical
|
|
20
|
+
* in kind to `__mj_integration_LastSyncedSnapshot`. SS `NVARCHAR(MAX)` / PG `TEXT`; the
|
|
21
|
+
* post-sync coverage scan + spread cast `::jsonb` on Postgres when they query it.
|
|
22
|
+
*
|
|
23
|
+
* This is BACKEND PLUMBING, not user-facing metadata. It is a SYSTEM field — in the same
|
|
24
|
+
* hidden/internal class as `__mj_integration_ContentHash` & the other `__mj_integration_*`
|
|
25
|
+
* columns: metadata-registered only so the engine's `entity.Set(...)`/`entity.Fields` write
|
|
26
|
+
* path is permitted to touch it (so it is NOT an out-of-band "phantom" column the engine
|
|
27
|
+
* writes blind), but NOT surfaced or managed through the metadata-driven framework — not in
|
|
28
|
+
* forms, not user-mappable, not "a field" anyone works with. It is machinery.
|
|
29
|
+
*
|
|
30
|
+
* The things that genuinely ENTER the metadata-driven framework are the columns this gets
|
|
31
|
+
* PROMOTED into: a pervasive key here is turned into a real user-facing, mappable EntityField
|
|
32
|
+
* (IntegrationSchemaSync IOF → RSU ADD COLUMN → CodeGen → md.Refresh) BEFORE the engine maps
|
|
33
|
+
* to it. So: backend staging until promotion, first-class metadata-driven schema after. Only
|
|
34
|
+
* the JSON *values* parked here are ever transient.
|
|
35
|
+
*
|
|
36
|
+
* Gated everywhere by EntityInfo field presence, so it is a no-op on tables that predate
|
|
37
|
+
* the column. CRUCIAL invariant: when a record has NO unmapped keys, NOTHING is written
|
|
38
|
+
* here — empty across all rows is the signal that no RSU pass is needed, which is what
|
|
39
|
+
* keeps a customs-free sync byte-identical to today (single-stage, zero overhead).
|
|
40
|
+
*/
|
|
41
|
+
export declare const CUSTOM_OVERFLOW_COLUMN = "__mj_integration_CustomOverflow";
|
|
42
|
+
/**
|
|
43
|
+
* Computes the unmapped subset of a record's external fields: every key the source
|
|
44
|
+
* returned that is NOT the SourceFieldName of any active field map.
|
|
45
|
+
*
|
|
46
|
+
* Pure + O(columns) — no per-row allocation beyond the result object, which is empty
|
|
47
|
+
* (and discarded by the caller) for the common no-customs case.
|
|
48
|
+
*
|
|
49
|
+
* Honest limit (documented in gaps.md §2): a field consumed only indirectly — e.g. as a
|
|
50
|
+
* `combine`/`custom` transform input rather than as a map's own SourceFieldName — will
|
|
51
|
+
* also appear here. That is harmless (the value is still in `Fields`); the post-sync
|
|
52
|
+
* promotion stage applies a coverage + novelty filter before it ever creates a column.
|
|
53
|
+
*
|
|
54
|
+
* @param externalFields - the full source record (`ExternalRecord.Fields`)
|
|
55
|
+
* @param mappedSourceFieldNames - SourceFieldName of every active field map
|
|
56
|
+
* @returns the extra keys + values, or an empty object when everything was mapped
|
|
57
|
+
*/
|
|
58
|
+
export declare function computeUnmappedFields(externalFields: Record<string, unknown>, mappedSourceFieldNames: ReadonlySet<string>): Record<string, unknown>;
|
|
59
|
+
/**
|
|
60
|
+
* Whether a computed unmapped-field object carries anything worth persisting. Used by
|
|
61
|
+
* both the per-record write gate and (later) the post-sync RSU trigger gate.
|
|
62
|
+
*/
|
|
63
|
+
export declare function hasUnmappedFields(unmapped: Record<string, unknown> | undefined | null): boolean;
|
|
64
|
+
//# sourceMappingURL=CustomOverflow.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CustomOverflow.d.ts","sourceRoot":"","sources":["../src/CustomOverflow.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,eAAO,MAAM,sBAAsB,oCAAoC,CAAC;AAExE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CACjC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACvC,sBAAsB,EAAE,WAAW,CAAC,MAAM,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAQzB;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI,GAAG,OAAO,CAE/F"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Name of the per-record custom-overflow column. Written on every integration table
|
|
3
|
+
* alongside the other `__mj_integration_*` columns when present. Holds a JSON object of
|
|
4
|
+
* the source fields a record returned that have NO field map — the "extra" keys the
|
|
5
|
+
* target table doesn't (yet) have a column for.
|
|
6
|
+
*
|
|
7
|
+
* This is the capture half of framework-level custom-column support (gaps.md §2). A
|
|
8
|
+
* source with no schema/describe endpoint (the PropFuel class) can only learn its full
|
|
9
|
+
* field set by reading data; a static catalog then silently drops any extra key at
|
|
10
|
+
* {@link MapSingleRecord} (only mapped fields get persisted). Rather than lose them, the
|
|
11
|
+
* unmapped keys are parked here, on the row, in the SAME write as the mapped fields — so
|
|
12
|
+
* capture costs no extra round-trip. A post-sync Runtime-Schema-Updation (RSU) pass later
|
|
13
|
+
* reads this column back out and promotes pervasive keys to real columns (ADD COLUMN +
|
|
14
|
+
* CodeGen + a field map). It does NOT backfill historical values: once the field map exists
|
|
15
|
+
* the key is no longer "unmapped", so the NEXT sync maps it natively and populates the new
|
|
16
|
+
* column with the LIVE source value (a full sync repopulates every row; an incremental fills
|
|
17
|
+
* rows as they next change) — and stops re-capturing it here. The loop converges go-forward.
|
|
18
|
+
*
|
|
19
|
+
* Shape mirrors {@link CONTENT_HASH_COLUMN}: a JSON-serialized string sidecar, identical
|
|
20
|
+
* in kind to `__mj_integration_LastSyncedSnapshot`. SS `NVARCHAR(MAX)` / PG `TEXT`; the
|
|
21
|
+
* post-sync coverage scan + spread cast `::jsonb` on Postgres when they query it.
|
|
22
|
+
*
|
|
23
|
+
* This is BACKEND PLUMBING, not user-facing metadata. It is a SYSTEM field — in the same
|
|
24
|
+
* hidden/internal class as `__mj_integration_ContentHash` & the other `__mj_integration_*`
|
|
25
|
+
* columns: metadata-registered only so the engine's `entity.Set(...)`/`entity.Fields` write
|
|
26
|
+
* path is permitted to touch it (so it is NOT an out-of-band "phantom" column the engine
|
|
27
|
+
* writes blind), but NOT surfaced or managed through the metadata-driven framework — not in
|
|
28
|
+
* forms, not user-mappable, not "a field" anyone works with. It is machinery.
|
|
29
|
+
*
|
|
30
|
+
* The things that genuinely ENTER the metadata-driven framework are the columns this gets
|
|
31
|
+
* PROMOTED into: a pervasive key here is turned into a real user-facing, mappable EntityField
|
|
32
|
+
* (IntegrationSchemaSync IOF → RSU ADD COLUMN → CodeGen → md.Refresh) BEFORE the engine maps
|
|
33
|
+
* to it. So: backend staging until promotion, first-class metadata-driven schema after. Only
|
|
34
|
+
* the JSON *values* parked here are ever transient.
|
|
35
|
+
*
|
|
36
|
+
* Gated everywhere by EntityInfo field presence, so it is a no-op on tables that predate
|
|
37
|
+
* the column. CRUCIAL invariant: when a record has NO unmapped keys, NOTHING is written
|
|
38
|
+
* here — empty across all rows is the signal that no RSU pass is needed, which is what
|
|
39
|
+
* keeps a customs-free sync byte-identical to today (single-stage, zero overhead).
|
|
40
|
+
*/
|
|
41
|
+
export const CUSTOM_OVERFLOW_COLUMN = '__mj_integration_CustomOverflow';
|
|
42
|
+
/**
|
|
43
|
+
* Computes the unmapped subset of a record's external fields: every key the source
|
|
44
|
+
* returned that is NOT the SourceFieldName of any active field map.
|
|
45
|
+
*
|
|
46
|
+
* Pure + O(columns) — no per-row allocation beyond the result object, which is empty
|
|
47
|
+
* (and discarded by the caller) for the common no-customs case.
|
|
48
|
+
*
|
|
49
|
+
* Honest limit (documented in gaps.md §2): a field consumed only indirectly — e.g. as a
|
|
50
|
+
* `combine`/`custom` transform input rather than as a map's own SourceFieldName — will
|
|
51
|
+
* also appear here. That is harmless (the value is still in `Fields`); the post-sync
|
|
52
|
+
* promotion stage applies a coverage + novelty filter before it ever creates a column.
|
|
53
|
+
*
|
|
54
|
+
* @param externalFields - the full source record (`ExternalRecord.Fields`)
|
|
55
|
+
* @param mappedSourceFieldNames - SourceFieldName of every active field map
|
|
56
|
+
* @returns the extra keys + values, or an empty object when everything was mapped
|
|
57
|
+
*/
|
|
58
|
+
export function computeUnmappedFields(externalFields, mappedSourceFieldNames) {
|
|
59
|
+
const unmapped = {};
|
|
60
|
+
for (const key of Object.keys(externalFields)) {
|
|
61
|
+
if (!mappedSourceFieldNames.has(key)) {
|
|
62
|
+
unmapped[key] = externalFields[key];
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return unmapped;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Whether a computed unmapped-field object carries anything worth persisting. Used by
|
|
69
|
+
* both the per-record write gate and (later) the post-sync RSU trigger gate.
|
|
70
|
+
*/
|
|
71
|
+
export function hasUnmappedFields(unmapped) {
|
|
72
|
+
return unmapped != null && Object.keys(unmapped).length > 0;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=CustomOverflow.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CustomOverflow.js","sourceRoot":"","sources":["../src/CustomOverflow.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,iCAAiC,CAAC;AAExE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,qBAAqB,CACjC,cAAuC,EACvC,sBAA2C;IAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,QAAQ,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAoD;IAClF,OAAO,QAAQ,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -1,24 +1,23 @@
|
|
|
1
1
|
import type { ICompanyIntegrationFieldMap } from './entity-types.js';
|
|
2
2
|
import type { ExternalRecord, MappedRecord } from './types.js';
|
|
3
3
|
/**
|
|
4
|
-
* Engine responsible for applying field-level mappings and transformations
|
|
5
|
-
*
|
|
4
|
+
* Engine responsible for applying field-level mappings and transformations from external records to
|
|
5
|
+
* MJ entity fields.
|
|
6
|
+
*
|
|
7
|
+
* The per-value transform pipeline (direct / regex / split / combine / lookup / format / coerce /
|
|
8
|
+
* substring / custom) is owned by the shared {@link FieldTransformEngine} in `@memberjunction/global`
|
|
9
|
+
* and reused here, so integration sync and the rules-based bulk-update processor run the *exact same*
|
|
10
|
+
* transform implementation — one place to fix, one place to extend. This engine keeps only the
|
|
11
|
+
* integration-specific concerns: source flattening, field-map iteration, and unmapped-field overflow
|
|
12
|
+
* capture. See the Field Rules guide in `@memberjunction/global` for the shared engine, and
|
|
13
|
+
* `EntityFieldRules` in `@memberjunction/core` for the metadata-aware entity-update sibling.
|
|
6
14
|
*/
|
|
7
15
|
export declare class FieldMappingEngine {
|
|
16
|
+
/** The shared transform pipeline. Holds its own LRU cache of compiled custom expressions. */
|
|
17
|
+
private readonly transformEngine;
|
|
8
18
|
/**
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* A failed compile is cached as the `Error` it threw — a malformed expression is therefore
|
|
13
|
-
* compiled a single time (then re-thrown from cache on every record) rather than recompiled
|
|
14
|
-
* on the per-record error path. Bounded via {@link MJLruCache} because the owning
|
|
15
|
-
* `IntegrationEngine` is a process-lifetime singleton; expression cardinality is normally
|
|
16
|
-
* tiny (it tracks configured field maps), so the 1000-entry default is comfortably ample.
|
|
17
|
-
*/
|
|
18
|
-
private readonly customFunctionCache;
|
|
19
|
-
/**
|
|
20
|
-
* Applies field mappings to a batch of external records, producing mapped records
|
|
21
|
-
* ready for match resolution and persistence.
|
|
19
|
+
* Applies field mappings to a batch of external records, producing mapped records ready for match
|
|
20
|
+
* resolution and persistence.
|
|
22
21
|
*
|
|
23
22
|
* @param records - External records to map
|
|
24
23
|
* @param fieldMaps - Active field map entities defining source→destination mappings
|
|
@@ -31,86 +30,13 @@ export declare class FieldMappingEngine {
|
|
|
31
30
|
*/
|
|
32
31
|
private MapSingleRecord;
|
|
33
32
|
/**
|
|
34
|
-
* Applies a single field mapping, including the full transform pipeline
|
|
35
|
-
* Returns undefined if the field should be skipped (OnError: Skip).
|
|
33
|
+
* Applies a single field mapping, including the full transform pipeline (delegated to the shared
|
|
34
|
+
* {@link FieldTransformEngine}). Returns undefined if the field should be skipped (OnError: Skip).
|
|
36
35
|
*/
|
|
37
36
|
private ApplyFieldMapping;
|
|
38
37
|
/**
|
|
39
38
|
* Parses the JSON transform pipeline string into typed TransformStep objects.
|
|
40
39
|
*/
|
|
41
40
|
private ParseTransformPipeline;
|
|
42
|
-
/**
|
|
43
|
-
* Executes a single transform step, applying the configured transformation
|
|
44
|
-
* and handling errors according to the step's OnError strategy.
|
|
45
|
-
*/
|
|
46
|
-
private ExecuteTransformStep;
|
|
47
|
-
/**
|
|
48
|
-
* Dispatches to the appropriate transform handler based on step type.
|
|
49
|
-
*/
|
|
50
|
-
private DispatchTransform;
|
|
51
|
-
/**
|
|
52
|
-
* Handles a transform error according to the OnError strategy.
|
|
53
|
-
*/
|
|
54
|
-
private HandleTransformError;
|
|
55
|
-
/**
|
|
56
|
-
* Direct pass-through, applying default value if source is null/undefined.
|
|
57
|
-
*/
|
|
58
|
-
private ApplyDirect;
|
|
59
|
-
/**
|
|
60
|
-
* Applies a regex replacement to a string value.
|
|
61
|
-
*/
|
|
62
|
-
private ApplyRegex;
|
|
63
|
-
/**
|
|
64
|
-
* Splits a string value and extracts a part by index.
|
|
65
|
-
*/
|
|
66
|
-
private ApplySplit;
|
|
67
|
-
/**
|
|
68
|
-
* Combines multiple source fields with a separator.
|
|
69
|
-
*/
|
|
70
|
-
private ApplyCombine;
|
|
71
|
-
/**
|
|
72
|
-
* Performs a case-insensitive value lookup/mapping.
|
|
73
|
-
*/
|
|
74
|
-
private ApplyLookup;
|
|
75
|
-
/**
|
|
76
|
-
* Applies date/number/string formatting.
|
|
77
|
-
*/
|
|
78
|
-
private ApplyFormat;
|
|
79
|
-
/**
|
|
80
|
-
* Basic date formatting supporting ISO and common format tokens.
|
|
81
|
-
*/
|
|
82
|
-
private FormatDate;
|
|
83
|
-
/**
|
|
84
|
-
* Coerces a value to the specified target type.
|
|
85
|
-
*/
|
|
86
|
-
private ApplyCoerce;
|
|
87
|
-
/**
|
|
88
|
-
* Coerces a value to a number, throwing on NaN.
|
|
89
|
-
*/
|
|
90
|
-
private CoerceToNumber;
|
|
91
|
-
/**
|
|
92
|
-
* Coerces a value to boolean, supporting common truthy/falsy strings.
|
|
93
|
-
*/
|
|
94
|
-
private CoerceToBoolean;
|
|
95
|
-
/**
|
|
96
|
-
* Extracts a substring from a string value.
|
|
97
|
-
*/
|
|
98
|
-
private ApplySubstring;
|
|
99
|
-
/**
|
|
100
|
-
* Evaluates a custom JavaScript expression.
|
|
101
|
-
* The expression has access to `value` (current field value) and `fields` (all record fields).
|
|
102
|
-
*/
|
|
103
|
-
private ApplyCustom;
|
|
104
|
-
/**
|
|
105
|
-
* Returns the compiled function for an expression, compiling and caching on first use.
|
|
106
|
-
* If the expression failed to compile previously, the cached error is re-thrown — a
|
|
107
|
-
* malformed expression is compiled exactly once across an entire batch.
|
|
108
|
-
*/
|
|
109
|
-
private GetCompiledExpression;
|
|
110
|
-
/**
|
|
111
|
-
* Compiles a custom expression into a callable, returning the thrown `Error` instead of
|
|
112
|
-
* propagating it so the caller can cache compile failures alongside successes.
|
|
113
|
-
*/
|
|
114
|
-
private compileExpression;
|
|
115
41
|
}
|
|
116
42
|
//# sourceMappingURL=FieldMappingEngine.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FieldMappingEngine.d.ts","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"FieldMappingEngine.d.ts","sourceRoot":"","sources":["../src/FieldMappingEngine.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAG/D;;;;;;;;;;;GAWG;AACH,qBAAa,kBAAkB;IAC3B,6FAA6F;IAC7F,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA8B;IAE9D;;;;;;;;OAQG;IACI,KAAK,CACR,OAAO,EAAE,cAAc,EAAE,EACzB,SAAS,EAAE,2BAA2B,EAAE,EACxC,UAAU,EAAE,MAAM,GACnB,YAAY,EAAE;IAKjB;;OAEG;IACH,OAAO,CAAC,eAAe;IAyCvB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAUzB;;OAEG;IACH,OAAO,CAAC,sBAAsB;CAWjC"}
|