@memberjunction/integration-engine 5.49.0 → 5.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/IntegrationEngine.d.ts +77 -0
- package/dist/IntegrationEngine.d.ts.map +1 -1
- package/dist/IntegrationEngine.js +247 -51
- package/dist/IntegrationEngine.js.map +1 -1
- package/dist/MatchEngine.d.ts +156 -14
- package/dist/MatchEngine.d.ts.map +1 -1
- package/dist/MatchEngine.js +438 -68
- package/dist/MatchEngine.js.map +1 -1
- package/dist/RecordMapBatch.d.ts +162 -0
- package/dist/RecordMapBatch.d.ts.map +1 -0
- package/dist/RecordMapBatch.js +283 -0
- package/dist/RecordMapBatch.js.map +1 -0
- package/dist/prefetchFilter.d.ts +18 -0
- package/dist/prefetchFilter.d.ts.map +1 -1
- package/dist/prefetchFilter.js +15 -10
- package/dist/prefetchFilter.js.map +1 -1
- package/package.json +7 -7
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import { IMetadataProvider, type UserInfo } from '@memberjunction/core';
|
|
2
|
+
/** One external↔MJ mapping awaiting write. */
|
|
3
|
+
export interface PendingRecordMap {
|
|
4
|
+
/** The MJ entity the mapping is for. Batches are keyed on this + CompanyIntegrationID. */
|
|
5
|
+
EntityID: string;
|
|
6
|
+
/** The external system's ID for the record. */
|
|
7
|
+
ExternalID: string;
|
|
8
|
+
/** The MJ record's primary key, '|'-joined for composite PKs. */
|
|
9
|
+
EntityRecordID: string;
|
|
10
|
+
}
|
|
11
|
+
/** A mapping that did not make it to the database, with the reason. */
|
|
12
|
+
export interface RecordMapFailure {
|
|
13
|
+
ExternalID: string;
|
|
14
|
+
EntityID: string;
|
|
15
|
+
ErrorMessage: string;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Batched writer for `MJ: Company Integration Record Maps`.
|
|
19
|
+
*
|
|
20
|
+
* **Why this exists.** The per-record `SaveRecordMap` costs three round trips — a `RunView` to
|
|
21
|
+
* find an existing mapping, a `Load`, and a `Save` — for *every* record the sync touches,
|
|
22
|
+
* including records it decided not to change. On a no-change full sync that is roughly 60% of
|
|
23
|
+
* the total cost: we pay full price to discover there is nothing to do.
|
|
24
|
+
*
|
|
25
|
+
* **What it does instead.** It queues mappings and resolves a whole chunk against the database in
|
|
26
|
+
* ONE `RunView`, then writes only the rows that are actually new or actually point somewhere
|
|
27
|
+
* different. On an incremental sync where mappings are stable that is one read and zero writes
|
|
28
|
+
* per chunk, against 3N round trips before.
|
|
29
|
+
*
|
|
30
|
+
* **Every write goes through the provider's entity layer, never hand-written SQL.** `RunView` and
|
|
31
|
+
* `GetEntityObject`/`Save()` are dialect-agnostic by construction — `Save()` lands in the
|
|
32
|
+
* CodeGen-generated `spCreate`/`spUpdate` for the entity, on whichever platform the provider is —
|
|
33
|
+
* so this file contains no SQL text, no `ExecuteSQL`, and no platform branch that could be right
|
|
34
|
+
* on SQL Server and wrong on Postgres. The one place SQL text is unavoidable is a `RunView`
|
|
35
|
+
* `ExtraFilter`, and both its identifiers and its literals are quoted through the provider's own
|
|
36
|
+
* dialect, the same seam `buildContentHashPrefetchFilter` uses.
|
|
37
|
+
*
|
|
38
|
+
* Going through the entity layer also means the writes get everything a direct statement would
|
|
39
|
+
* have skipped: field validation, `__mj_CreatedAt`/`__mj_UpdatedAt`, Record Changes tracking, and
|
|
40
|
+
* — the one that bites silently — the save event that drives `LocalCacheManager` invalidation.
|
|
41
|
+
* This entity is read by `RunView` elsewhere in this engine, so a raw `UPDATE` here would leave
|
|
42
|
+
* those cached reads stale until their TTL (see `ScheduledJobEngine.updateJobStatistics`, which
|
|
43
|
+
* has to invalidate by hand for exactly that reason).
|
|
44
|
+
*
|
|
45
|
+
* **Per-row error attribution is the hard requirement, not an extra.** Batching without it trades
|
|
46
|
+
* a slow correct sync for a fast wrong one: one malformed row would silently take out its 499
|
|
47
|
+
* neighbours. Here attribution is structural rather than reconstructed — each mapping is its own
|
|
48
|
+
* `Save()` returning its own boolean, so a failure names the row that failed and the other rows in
|
|
49
|
+
* the chunk are unaffected. There is no read-back-and-diff step because there is nothing to
|
|
50
|
+
* reconstruct.
|
|
51
|
+
*
|
|
52
|
+
* **Durability note.** Mappings are written slightly later than the record they describe, so a
|
|
53
|
+
* process killed between the two loses that window's map rows. That is recoverable by design:
|
|
54
|
+
* the next sync re-matches those records by primary key (see `MatchEngine.FindByKeyFields`) and
|
|
55
|
+
* re-establishes the mapping. It cannot produce duplicates — the write is an upsert keyed on
|
|
56
|
+
* (CompanyIntegration, Entity, ExternalID).
|
|
57
|
+
*/
|
|
58
|
+
export declare class RecordMapBatch {
|
|
59
|
+
private readonly provider;
|
|
60
|
+
private readonly companyIntegrationID;
|
|
61
|
+
private readonly contextUser;
|
|
62
|
+
private readonly saveSingle;
|
|
63
|
+
/**
|
|
64
|
+
* Hard ceiling on the chunk size. A chunk becomes one `IN (…)` list in a `RunView` filter, so
|
|
65
|
+
* the filter text grows linearly with the chunk; past 5,000 IDs it starts approaching filter
|
|
66
|
+
* length and query-plan limits on both platforms.
|
|
67
|
+
*/
|
|
68
|
+
static readonly CHUNK_SIZE_CEILING = 5000;
|
|
69
|
+
/**
|
|
70
|
+
* Chunk size. 500 keeps the generated filter well inside length limits.
|
|
71
|
+
* Read from MJ_INTEGRATION_RECORD_MAP_CHUNK_SIZE at class-init and clamped to
|
|
72
|
+
* [1, CHUNK_SIZE_CEILING].
|
|
73
|
+
*/
|
|
74
|
+
static readonly ChunkSize: number;
|
|
75
|
+
/**
|
|
76
|
+
* How many map rows are saved concurrently within a chunk.
|
|
77
|
+
*
|
|
78
|
+
* The writes are independent — each is its own row, keyed on its own external ID — so they do
|
|
79
|
+
* not have to be serialized. Bounded rather than unbounded so a first-ever sync of a large
|
|
80
|
+
* entity cannot open 500 simultaneous requests against the connection pool the rest of the
|
|
81
|
+
* sync is also using.
|
|
82
|
+
*/
|
|
83
|
+
static readonly WriteConcurrency = 10;
|
|
84
|
+
/** Reads + clamps the chunk size from env; warns once when a configured value is clamped down. */
|
|
85
|
+
private static computeChunkSize;
|
|
86
|
+
/**
|
|
87
|
+
* Queued mappings, keyed `EntityID|ExternalID` — the upsert's own key, so the map both
|
|
88
|
+
* dedups and preserves queue order (JS Maps iterate in insertion order). A Map rather than
|
|
89
|
+
* an array because dedup on every Queue() would otherwise be a scan of the whole batch.
|
|
90
|
+
*/
|
|
91
|
+
private pending;
|
|
92
|
+
private failures;
|
|
93
|
+
/** The dedup key. EntityID is a UUID, so the separator cannot appear in the left half. */
|
|
94
|
+
private static pendingKey;
|
|
95
|
+
/**
|
|
96
|
+
* @param provider the provider that owns this sync's connection
|
|
97
|
+
* @param companyIntegrationID the CompanyIntegration all queued mappings belong to
|
|
98
|
+
* @param contextUser user context for the writes
|
|
99
|
+
* @param saveSingle the original one-row upsert, used when the chunk read fails
|
|
100
|
+
*/
|
|
101
|
+
constructor(provider: IMetadataProvider, companyIntegrationID: string, contextUser: UserInfo, saveSingle: (companyIntegrationID: string, externalID: string, entityID: string, entityRecordID: string, contextUser: UserInfo) => Promise<void>);
|
|
102
|
+
/** Mappings that failed to write, with the external ID that owns each failure. */
|
|
103
|
+
get Failures(): readonly RecordMapFailure[];
|
|
104
|
+
/** Returns the failures accumulated so far and clears them, so a caller reports each once. */
|
|
105
|
+
TakeFailures(): RecordMapFailure[];
|
|
106
|
+
/**
|
|
107
|
+
* Drops everything queued but not yet written.
|
|
108
|
+
*
|
|
109
|
+
* Called when the batch transaction that produced these mappings rolled back: the records
|
|
110
|
+
* they point at do not exist, so writing the mappings would leave the map pointing at
|
|
111
|
+
* nothing. The per-record retry path re-queues whatever actually commits.
|
|
112
|
+
*/
|
|
113
|
+
Discard(): void;
|
|
114
|
+
/**
|
|
115
|
+
* Queues a mapping. **Never writes** — the caller decides when a flush is safe.
|
|
116
|
+
*
|
|
117
|
+
* Queuing deliberately does not auto-flush on a full chunk. Queue() is called from inside the
|
|
118
|
+
* apply pass's batch transaction, and a flush fired from there would write map rows within
|
|
119
|
+
* that transaction, which `Discard()` could no longer un-queue on rollback. The apply loop
|
|
120
|
+
* calls {@link Flush} once per batch, after commit — which also bounds `pending` to one
|
|
121
|
+
* batch's worth of mappings.
|
|
122
|
+
*
|
|
123
|
+
* Queuing the same external ID twice within a window keeps only the latest — the mapping is
|
|
124
|
+
* an upsert, so the last value written would win anyway.
|
|
125
|
+
*/
|
|
126
|
+
Queue(mapping: PendingRecordMap): void;
|
|
127
|
+
/** Writes everything queued. Safe to call repeatedly; a no-op when the queue is empty. */
|
|
128
|
+
Flush(): Promise<void>;
|
|
129
|
+
/**
|
|
130
|
+
* Resolves one chunk against the database in a single read, then writes only what differs.
|
|
131
|
+
*
|
|
132
|
+
* A mapping whose row already points at the same MJ record needs no write at all — that is
|
|
133
|
+
* the common case on an incremental sync and the reason this is cheaper than the per-record
|
|
134
|
+
* path even though the writes themselves are per-row.
|
|
135
|
+
*/
|
|
136
|
+
private flushChunk;
|
|
137
|
+
/**
|
|
138
|
+
* Reads the chunk's existing map rows in one query, keyed by external ID. Returns null when
|
|
139
|
+
* the read fails, which is deliberately distinct from an empty map (read succeeded, nothing
|
|
140
|
+
* there) — the two lead to different write paths.
|
|
141
|
+
*/
|
|
142
|
+
private readExisting;
|
|
143
|
+
/**
|
|
144
|
+
* Writes one mapping through the provider's entity layer — update in place when a row already
|
|
145
|
+
* exists, insert otherwise. A failure is attributed to this external ID and nothing else.
|
|
146
|
+
*/
|
|
147
|
+
private writeOne;
|
|
148
|
+
/** Fallback when the chunk read failed: one row at a time through the original upsert. */
|
|
149
|
+
private writeIndividually;
|
|
150
|
+
/**
|
|
151
|
+
* The provider's dialect, for the one place SQL text is unavoidable — a `RunView`
|
|
152
|
+
* `ExtraFilter`. Same seam `buildContentHashPrefetchFilter` uses, so both filters this engine
|
|
153
|
+
* builds quote by the platform's own rule rather than a hand-rolled `replace(/'/g, "''")`
|
|
154
|
+
* that happens to match it today.
|
|
155
|
+
*
|
|
156
|
+
* Narrowed with `instanceof` rather than cast: `IMetadataProvider` in general carries no
|
|
157
|
+
* dialect (a client-side provider has none), and null routes to the per-row path instead of
|
|
158
|
+
* guessing an escaping rule.
|
|
159
|
+
*/
|
|
160
|
+
private get quoter();
|
|
161
|
+
}
|
|
162
|
+
//# sourceMappingURL=RecordMapBatch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RecordMapBatch.d.ts","sourceRoot":"","sources":["../src/RecordMapBatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAwB,iBAAiB,EAAW,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAIvG,8CAA8C;AAC9C,MAAM,WAAW,gBAAgB;IAC7B,0FAA0F;IAC1F,QAAQ,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,EAAE,MAAM,CAAC;CAC1B;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACxB;AAQD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,qBAAa,cAAc;IAyDnB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,oBAAoB;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,UAAU;IA3D/B;;;;OAIG;IACH,gBAAuB,kBAAkB,QAAS;IAElD;;;;OAIG;IACH,gBAAuB,SAAS,SAAqC;IAErE;;;;;;;OAOG;IACH,gBAAuB,gBAAgB,MAAM;IAE7C,kGAAkG;IAClG,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAW/B;;;;OAIG;IACH,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,QAAQ,CAA0B;IAE1C,0FAA0F;IAC1F,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB;;;;;OAKG;gBAEkB,QAAQ,EAAE,iBAAiB,EAC3B,oBAAoB,EAAE,MAAM,EAC5B,WAAW,EAAE,QAAQ,EACrB,UAAU,EAAE,CACzB,oBAAoB,EAAE,MAAM,EAC5B,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,QAAQ,KACpB,OAAO,CAAC,IAAI,CAAC;IAGtB,kFAAkF;IAClF,IAAW,QAAQ,IAAI,SAAS,gBAAgB,EAAE,CAEjD;IAED,8FAA8F;IACvF,YAAY,IAAI,gBAAgB,EAAE;IAMzC;;;;;;OAMG;IACI,OAAO,IAAI,IAAI;IAItB;;;;;;;;;;;OAWG;IACI,KAAK,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI7C,0FAA0F;IAC7E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBnC;;;;;;OAMG;YACW,UAAU;IAiBxB;;;;OAIG;YACW,YAAY;IAoC1B;;;OAGG;YACW,QAAQ;IAqCtB,0FAA0F;YAC5E,iBAAiB;IAmB/B;;;;;;;;;OASG;IACH,OAAO,KAAK,MAAM,GAEjB;CACJ"}
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
import { DatabaseProviderBase, RunView } from '@memberjunction/core';
|
|
2
|
+
import { quoteTextLiteral } from './prefetchFilter.js';
|
|
3
|
+
/**
|
|
4
|
+
* Batched writer for `MJ: Company Integration Record Maps`.
|
|
5
|
+
*
|
|
6
|
+
* **Why this exists.** The per-record `SaveRecordMap` costs three round trips — a `RunView` to
|
|
7
|
+
* find an existing mapping, a `Load`, and a `Save` — for *every* record the sync touches,
|
|
8
|
+
* including records it decided not to change. On a no-change full sync that is roughly 60% of
|
|
9
|
+
* the total cost: we pay full price to discover there is nothing to do.
|
|
10
|
+
*
|
|
11
|
+
* **What it does instead.** It queues mappings and resolves a whole chunk against the database in
|
|
12
|
+
* ONE `RunView`, then writes only the rows that are actually new or actually point somewhere
|
|
13
|
+
* different. On an incremental sync where mappings are stable that is one read and zero writes
|
|
14
|
+
* per chunk, against 3N round trips before.
|
|
15
|
+
*
|
|
16
|
+
* **Every write goes through the provider's entity layer, never hand-written SQL.** `RunView` and
|
|
17
|
+
* `GetEntityObject`/`Save()` are dialect-agnostic by construction — `Save()` lands in the
|
|
18
|
+
* CodeGen-generated `spCreate`/`spUpdate` for the entity, on whichever platform the provider is —
|
|
19
|
+
* so this file contains no SQL text, no `ExecuteSQL`, and no platform branch that could be right
|
|
20
|
+
* on SQL Server and wrong on Postgres. The one place SQL text is unavoidable is a `RunView`
|
|
21
|
+
* `ExtraFilter`, and both its identifiers and its literals are quoted through the provider's own
|
|
22
|
+
* dialect, the same seam `buildContentHashPrefetchFilter` uses.
|
|
23
|
+
*
|
|
24
|
+
* Going through the entity layer also means the writes get everything a direct statement would
|
|
25
|
+
* have skipped: field validation, `__mj_CreatedAt`/`__mj_UpdatedAt`, Record Changes tracking, and
|
|
26
|
+
* — the one that bites silently — the save event that drives `LocalCacheManager` invalidation.
|
|
27
|
+
* This entity is read by `RunView` elsewhere in this engine, so a raw `UPDATE` here would leave
|
|
28
|
+
* those cached reads stale until their TTL (see `ScheduledJobEngine.updateJobStatistics`, which
|
|
29
|
+
* has to invalidate by hand for exactly that reason).
|
|
30
|
+
*
|
|
31
|
+
* **Per-row error attribution is the hard requirement, not an extra.** Batching without it trades
|
|
32
|
+
* a slow correct sync for a fast wrong one: one malformed row would silently take out its 499
|
|
33
|
+
* neighbours. Here attribution is structural rather than reconstructed — each mapping is its own
|
|
34
|
+
* `Save()` returning its own boolean, so a failure names the row that failed and the other rows in
|
|
35
|
+
* the chunk are unaffected. There is no read-back-and-diff step because there is nothing to
|
|
36
|
+
* reconstruct.
|
|
37
|
+
*
|
|
38
|
+
* **Durability note.** Mappings are written slightly later than the record they describe, so a
|
|
39
|
+
* process killed between the two loses that window's map rows. That is recoverable by design:
|
|
40
|
+
* the next sync re-matches those records by primary key (see `MatchEngine.FindByKeyFields`) and
|
|
41
|
+
* re-establishes the mapping. It cannot produce duplicates — the write is an upsert keyed on
|
|
42
|
+
* (CompanyIntegration, Entity, ExternalID).
|
|
43
|
+
*/
|
|
44
|
+
export class RecordMapBatch {
|
|
45
|
+
/**
|
|
46
|
+
* Hard ceiling on the chunk size. A chunk becomes one `IN (…)` list in a `RunView` filter, so
|
|
47
|
+
* the filter text grows linearly with the chunk; past 5,000 IDs it starts approaching filter
|
|
48
|
+
* length and query-plan limits on both platforms.
|
|
49
|
+
*/
|
|
50
|
+
static { this.CHUNK_SIZE_CEILING = 5_000; }
|
|
51
|
+
/**
|
|
52
|
+
* Chunk size. 500 keeps the generated filter well inside length limits.
|
|
53
|
+
* Read from MJ_INTEGRATION_RECORD_MAP_CHUNK_SIZE at class-init and clamped to
|
|
54
|
+
* [1, CHUNK_SIZE_CEILING].
|
|
55
|
+
*/
|
|
56
|
+
static { this.ChunkSize = RecordMapBatch.computeChunkSize(); }
|
|
57
|
+
/**
|
|
58
|
+
* How many map rows are saved concurrently within a chunk.
|
|
59
|
+
*
|
|
60
|
+
* The writes are independent — each is its own row, keyed on its own external ID — so they do
|
|
61
|
+
* not have to be serialized. Bounded rather than unbounded so a first-ever sync of a large
|
|
62
|
+
* entity cannot open 500 simultaneous requests against the connection pool the rest of the
|
|
63
|
+
* sync is also using.
|
|
64
|
+
*/
|
|
65
|
+
static { this.WriteConcurrency = 10; }
|
|
66
|
+
/** Reads + clamps the chunk size from env; warns once when a configured value is clamped down. */
|
|
67
|
+
static computeChunkSize() {
|
|
68
|
+
const ceiling = RecordMapBatch.CHUNK_SIZE_CEILING;
|
|
69
|
+
const raw = parseInt(process.env.MJ_INTEGRATION_RECORD_MAP_CHUNK_SIZE ?? '', 10);
|
|
70
|
+
if (!Number.isFinite(raw) || raw <= 0)
|
|
71
|
+
return 500;
|
|
72
|
+
if (raw > ceiling) {
|
|
73
|
+
console.warn(`[RecordMapBatch] MJ_INTEGRATION_RECORD_MAP_CHUNK_SIZE=${raw} exceeds the maximum ${ceiling} — clamped to ${ceiling}.`);
|
|
74
|
+
return ceiling;
|
|
75
|
+
}
|
|
76
|
+
return raw;
|
|
77
|
+
}
|
|
78
|
+
/** The dedup key. EntityID is a UUID, so the separator cannot appear in the left half. */
|
|
79
|
+
static pendingKey(entityID, externalID) {
|
|
80
|
+
return `${entityID}|${externalID}`;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* @param provider the provider that owns this sync's connection
|
|
84
|
+
* @param companyIntegrationID the CompanyIntegration all queued mappings belong to
|
|
85
|
+
* @param contextUser user context for the writes
|
|
86
|
+
* @param saveSingle the original one-row upsert, used when the chunk read fails
|
|
87
|
+
*/
|
|
88
|
+
constructor(provider, companyIntegrationID, contextUser, saveSingle) {
|
|
89
|
+
this.provider = provider;
|
|
90
|
+
this.companyIntegrationID = companyIntegrationID;
|
|
91
|
+
this.contextUser = contextUser;
|
|
92
|
+
this.saveSingle = saveSingle;
|
|
93
|
+
/**
|
|
94
|
+
* Queued mappings, keyed `EntityID|ExternalID` — the upsert's own key, so the map both
|
|
95
|
+
* dedups and preserves queue order (JS Maps iterate in insertion order). A Map rather than
|
|
96
|
+
* an array because dedup on every Queue() would otherwise be a scan of the whole batch.
|
|
97
|
+
*/
|
|
98
|
+
this.pending = new Map();
|
|
99
|
+
this.failures = [];
|
|
100
|
+
}
|
|
101
|
+
/** Mappings that failed to write, with the external ID that owns each failure. */
|
|
102
|
+
get Failures() {
|
|
103
|
+
return this.failures;
|
|
104
|
+
}
|
|
105
|
+
/** Returns the failures accumulated so far and clears them, so a caller reports each once. */
|
|
106
|
+
TakeFailures() {
|
|
107
|
+
const taken = this.failures;
|
|
108
|
+
this.failures = [];
|
|
109
|
+
return taken;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Drops everything queued but not yet written.
|
|
113
|
+
*
|
|
114
|
+
* Called when the batch transaction that produced these mappings rolled back: the records
|
|
115
|
+
* they point at do not exist, so writing the mappings would leave the map pointing at
|
|
116
|
+
* nothing. The per-record retry path re-queues whatever actually commits.
|
|
117
|
+
*/
|
|
118
|
+
Discard() {
|
|
119
|
+
this.pending.clear();
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Queues a mapping. **Never writes** — the caller decides when a flush is safe.
|
|
123
|
+
*
|
|
124
|
+
* Queuing deliberately does not auto-flush on a full chunk. Queue() is called from inside the
|
|
125
|
+
* apply pass's batch transaction, and a flush fired from there would write map rows within
|
|
126
|
+
* that transaction, which `Discard()` could no longer un-queue on rollback. The apply loop
|
|
127
|
+
* calls {@link Flush} once per batch, after commit — which also bounds `pending` to one
|
|
128
|
+
* batch's worth of mappings.
|
|
129
|
+
*
|
|
130
|
+
* Queuing the same external ID twice within a window keeps only the latest — the mapping is
|
|
131
|
+
* an upsert, so the last value written would win anyway.
|
|
132
|
+
*/
|
|
133
|
+
Queue(mapping) {
|
|
134
|
+
this.pending.set(RecordMapBatch.pendingKey(mapping.EntityID, mapping.ExternalID), mapping);
|
|
135
|
+
}
|
|
136
|
+
/** Writes everything queued. Safe to call repeatedly; a no-op when the queue is empty. */
|
|
137
|
+
async Flush() {
|
|
138
|
+
if (this.pending.size === 0)
|
|
139
|
+
return;
|
|
140
|
+
const queued = Array.from(this.pending.values());
|
|
141
|
+
this.pending.clear();
|
|
142
|
+
// Group by entity: the mapping's identity includes EntityID, so a chunk read has to be
|
|
143
|
+
// scoped to one entity to be answerable in a single filter.
|
|
144
|
+
const byEntity = new Map();
|
|
145
|
+
for (const m of queued) {
|
|
146
|
+
const bucket = byEntity.get(m.EntityID);
|
|
147
|
+
if (bucket)
|
|
148
|
+
bucket.push(m);
|
|
149
|
+
else
|
|
150
|
+
byEntity.set(m.EntityID, [m]);
|
|
151
|
+
}
|
|
152
|
+
for (const [entityID, mappings] of byEntity) {
|
|
153
|
+
for (let i = 0; i < mappings.length; i += RecordMapBatch.ChunkSize) {
|
|
154
|
+
await this.flushChunk(entityID, mappings.slice(i, i + RecordMapBatch.ChunkSize));
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Resolves one chunk against the database in a single read, then writes only what differs.
|
|
160
|
+
*
|
|
161
|
+
* A mapping whose row already points at the same MJ record needs no write at all — that is
|
|
162
|
+
* the common case on an incremental sync and the reason this is cheaper than the per-record
|
|
163
|
+
* path even though the writes themselves are per-row.
|
|
164
|
+
*/
|
|
165
|
+
async flushChunk(entityID, chunk) {
|
|
166
|
+
const existing = await this.readExisting(entityID, chunk);
|
|
167
|
+
if (!existing) {
|
|
168
|
+
// The read is what makes the batched path safe; without it we cannot tell an insert
|
|
169
|
+
// from an update, and guessing either way produces a duplicate map row. Fall back to
|
|
170
|
+
// the per-record upsert, which does its own read.
|
|
171
|
+
await this.writeIndividually(entityID, chunk);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const work = chunk.filter(m => existing.get(m.ExternalID)?.EntityRecordID !== m.EntityRecordID);
|
|
175
|
+
for (let i = 0; i < work.length; i += RecordMapBatch.WriteConcurrency) {
|
|
176
|
+
const slice = work.slice(i, i + RecordMapBatch.WriteConcurrency);
|
|
177
|
+
await Promise.all(slice.map(m => this.writeOne(entityID, m, existing.get(m.ExternalID)?.ID)));
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Reads the chunk's existing map rows in one query, keyed by external ID. Returns null when
|
|
182
|
+
* the read fails, which is deliberately distinct from an empty map (read succeeded, nothing
|
|
183
|
+
* there) — the two lead to different write paths.
|
|
184
|
+
*/
|
|
185
|
+
async readExisting(entityID, chunk) {
|
|
186
|
+
const q = this.quoter;
|
|
187
|
+
if (!q)
|
|
188
|
+
return null; // no dialect to quote with — the per-row path resolves each row itself
|
|
189
|
+
// The external IDs are the only free-text values here (the other two are UUIDs), and they
|
|
190
|
+
// are compared against an nvarchar column — see `quoteTextLiteral`.
|
|
191
|
+
const inList = chunk.map(m => quoteTextLiteral(m.ExternalID, q)).join(',');
|
|
192
|
+
const rv = new RunView();
|
|
193
|
+
const result = await rv.RunView({
|
|
194
|
+
EntityName: 'MJ: Company Integration Record Maps',
|
|
195
|
+
ExtraFilter: `${q.QuoteIdentifier('CompanyIntegrationID')}=${q.QuoteStringLiteral(this.companyIntegrationID)} ` +
|
|
196
|
+
`AND ${q.QuoteIdentifier('EntityID')}=${q.QuoteStringLiteral(entityID)} ` +
|
|
197
|
+
`AND ${q.QuoteIdentifier('ExternalSystemRecordID')} IN (${inList})`,
|
|
198
|
+
Fields: ['ID', 'ExternalSystemRecordID', 'EntityRecordID'],
|
|
199
|
+
IgnoreMaxRows: true, // a chunk can exceed the entity's default row cap
|
|
200
|
+
ResultType: 'simple',
|
|
201
|
+
BypassCache: true, // this decides INSERT vs UPDATE; a stale miss duplicates the row
|
|
202
|
+
}, this.contextUser);
|
|
203
|
+
if (!result.Success)
|
|
204
|
+
return null;
|
|
205
|
+
const byExternalID = new Map();
|
|
206
|
+
for (const row of result.Results) {
|
|
207
|
+
// First row wins if the table somehow carries a duplicate for this key: writing to
|
|
208
|
+
// whichever we saw last would flip the mapping between syncs for no reason.
|
|
209
|
+
if (!byExternalID.has(row.ExternalSystemRecordID)) {
|
|
210
|
+
byExternalID.set(row.ExternalSystemRecordID, { ID: row.ID, EntityRecordID: row.EntityRecordID });
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
return byExternalID;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Writes one mapping through the provider's entity layer — update in place when a row already
|
|
217
|
+
* exists, insert otherwise. A failure is attributed to this external ID and nothing else.
|
|
218
|
+
*/
|
|
219
|
+
async writeOne(entityID, mapping, existingID) {
|
|
220
|
+
try {
|
|
221
|
+
const recordMap = await this.provider.GetEntityObject('MJ: Company Integration Record Maps', this.contextUser);
|
|
222
|
+
if (existingID) {
|
|
223
|
+
const loaded = await recordMap.Load(existingID);
|
|
224
|
+
// The row was read a moment ago, so a failure to load means it has since been
|
|
225
|
+
// deleted. Insert rather than abandon the mapping.
|
|
226
|
+
if (!loaded)
|
|
227
|
+
recordMap.NewRecord();
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
recordMap.NewRecord();
|
|
231
|
+
}
|
|
232
|
+
recordMap.CompanyIntegrationID = this.companyIntegrationID;
|
|
233
|
+
recordMap.EntityID = entityID;
|
|
234
|
+
recordMap.ExternalSystemRecordID = mapping.ExternalID;
|
|
235
|
+
recordMap.EntityRecordID = mapping.EntityRecordID;
|
|
236
|
+
if (!await recordMap.Save()) {
|
|
237
|
+
this.failures.push({
|
|
238
|
+
ExternalID: mapping.ExternalID,
|
|
239
|
+
EntityID: entityID,
|
|
240
|
+
ErrorMessage: `Record map write failed: ${recordMap.LatestResult?.CompleteMessage ?? 'unknown error'}`,
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
catch (err) {
|
|
245
|
+
this.failures.push({
|
|
246
|
+
ExternalID: mapping.ExternalID,
|
|
247
|
+
EntityID: entityID,
|
|
248
|
+
ErrorMessage: err instanceof Error ? err.message : String(err),
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
/** Fallback when the chunk read failed: one row at a time through the original upsert. */
|
|
253
|
+
async writeIndividually(entityID, chunk) {
|
|
254
|
+
console.warn(`[RecordMapBatch] Could not read existing record maps for ${chunk.length} mapping(s); ` +
|
|
255
|
+
`falling back to per-row upserts so each row resolves itself.`);
|
|
256
|
+
for (const m of chunk) {
|
|
257
|
+
try {
|
|
258
|
+
await this.saveSingle(this.companyIntegrationID, m.ExternalID, entityID, m.EntityRecordID, this.contextUser);
|
|
259
|
+
}
|
|
260
|
+
catch (err) {
|
|
261
|
+
this.failures.push({
|
|
262
|
+
ExternalID: m.ExternalID,
|
|
263
|
+
EntityID: entityID,
|
|
264
|
+
ErrorMessage: err instanceof Error ? err.message : String(err),
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* The provider's dialect, for the one place SQL text is unavoidable — a `RunView`
|
|
271
|
+
* `ExtraFilter`. Same seam `buildContentHashPrefetchFilter` uses, so both filters this engine
|
|
272
|
+
* builds quote by the platform's own rule rather than a hand-rolled `replace(/'/g, "''")`
|
|
273
|
+
* that happens to match it today.
|
|
274
|
+
*
|
|
275
|
+
* Narrowed with `instanceof` rather than cast: `IMetadataProvider` in general carries no
|
|
276
|
+
* dialect (a client-side provider has none), and null routes to the per-row path instead of
|
|
277
|
+
* guessing an escaping rule.
|
|
278
|
+
*/
|
|
279
|
+
get quoter() {
|
|
280
|
+
return this.provider instanceof DatabaseProviderBase ? this.provider.Dialect : null;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
//# sourceMappingURL=RecordMapBatch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RecordMapBatch.js","sourceRoot":"","sources":["../src/RecordMapBatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAqB,OAAO,EAAiB,MAAM,sBAAsB,CAAC;AAEvG,OAAO,EAAE,gBAAgB,EAAkB,MAAM,qBAAqB,CAAC;AAyBvE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,MAAM,OAAO,cAAc;IACvB;;;;OAIG;aACoB,uBAAkB,GAAG,KAAK,AAAR,CAAS;IAElD;;;;OAIG;aACoB,cAAS,GAAG,cAAc,CAAC,gBAAgB,EAAE,AAApC,CAAqC;IAErE;;;;;;;OAOG;aACoB,qBAAgB,GAAG,EAAE,AAAL,CAAM;IAE7C,kGAAkG;IAC1F,MAAM,CAAC,gBAAgB;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,kBAAkB,CAAC;QAClD,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAoC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACjF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;QAClD,IAAI,GAAG,GAAG,OAAO,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,yDAAyD,GAAG,wBAAwB,OAAO,iBAAiB,OAAO,GAAG,CAAC,CAAC;YACrI,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;IAUD,0FAA0F;IAClF,MAAM,CAAC,UAAU,CAAC,QAAgB,EAAE,UAAkB;QAC1D,OAAO,GAAG,QAAQ,IAAI,UAAU,EAAE,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACH,YACqB,QAA2B,EAC3B,oBAA4B,EAC5B,WAAqB,EACrB,UAMC;QATD,aAAQ,GAAR,QAAQ,CAAmB;QAC3B,yBAAoB,GAApB,oBAAoB,CAAQ;QAC5B,gBAAW,GAAX,WAAW,CAAU;QACrB,eAAU,GAAV,UAAU,CAMT;QA7BtB;;;;WAIG;QACK,YAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;QAC9C,aAAQ,GAAuB,EAAE,CAAC;IAwBvC,CAAC;IAEJ,kFAAkF;IAClF,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED,8FAA8F;IACvF,YAAY;QACf,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACI,OAAO;QACV,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAED;;;;;;;;;;;OAWG;IACI,KAAK,CAAC,OAAyB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC;IAC/F,CAAC;IAED,0FAA0F;IACnF,KAAK,CAAC,KAAK;QACd,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO;QAEpC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAErB,uFAAuF;QACvF,4DAA4D;QAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA8B,CAAC;QACvD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;gBACtB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QAED,KAAK,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,QAAQ,EAAE,CAAC;YAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC;gBACjE,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;YACrF,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,UAAU,CAAC,QAAgB,EAAE,KAAyB;QAChE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,oFAAoF;YACpF,qFAAqF;YACrF,kDAAkD;YAClD,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC9C,OAAO;QACX,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,cAAc,KAAK,CAAC,CAAC,cAAc,CAAC,CAAC;QAChG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;YACpE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,gBAAgB,CAAC,CAAC;YACjE,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAClG,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,YAAY,CACtB,QAAgB,EAChB,KAAyB;QAEzB,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QACtB,IAAI,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,uEAAuE;QAE5F,0FAA0F;QAC1F,oEAAoE;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3E,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAyE;YACpG,UAAU,EAAE,qCAAqC;YACjD,WAAW,EACP,GAAG,CAAC,CAAC,eAAe,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG;gBAClG,OAAO,CAAC,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,GAAG;gBACzE,OAAO,CAAC,CAAC,eAAe,CAAC,wBAAwB,CAAC,QAAQ,MAAM,GAAG;YACvE,MAAM,EAAE,CAAC,IAAI,EAAE,wBAAwB,EAAE,gBAAgB,CAAC;YAC1D,aAAa,EAAE,IAAI,EAAE,kDAAkD;YACvE,UAAU,EAAE,QAAQ;YACpB,WAAW,EAAE,IAAI,EAAE,iEAAiE;SACvF,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAErB,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAEjC,MAAM,YAAY,GAAG,IAAI,GAAG,EAA6B,CAAC;QAC1D,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,mFAAmF;YACnF,4EAA4E;YAC5E,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC,EAAE,CAAC;gBAChD,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,sBAAsB,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC;YACrG,CAAC;QACL,CAAC;QACD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,QAAQ,CAAC,QAAgB,EAAE,OAAyB,EAAE,UAA8B;QAC9F,IAAI,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,CACjD,qCAAqC,EACrC,IAAI,CAAC,WAAW,CACnB,CAAC;YAEF,IAAI,UAAU,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAChD,8EAA8E;gBAC9E,mDAAmD;gBACnD,IAAI,CAAC,MAAM;oBAAE,SAAS,CAAC,SAAS,EAAE,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACJ,SAAS,CAAC,SAAS,EAAE,CAAC;YAC1B,CAAC;YAED,SAAS,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC;YAC3D,SAAS,CAAC,QAAQ,GAAG,QAAQ,CAAC;YAC9B,SAAS,CAAC,sBAAsB,GAAG,OAAO,CAAC,UAAU,CAAC;YACtD,SAAS,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;YAElD,IAAI,CAAC,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACf,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,QAAQ,EAAE,QAAQ;oBAClB,YAAY,EAAE,4BAA4B,SAAS,CAAC,YAAY,EAAE,eAAe,IAAI,eAAe,EAAE;iBACzG,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACf,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,QAAQ,EAAE,QAAQ;gBAClB,YAAY,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACjE,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,0FAA0F;IAClF,KAAK,CAAC,iBAAiB,CAAC,QAAgB,EAAE,KAAyB;QACvE,OAAO,CAAC,IAAI,CACR,4DAA4D,KAAK,CAAC,MAAM,eAAe;YACvF,8DAA8D,CACjE,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACpB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACjH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACf,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,QAAQ,EAAE,QAAQ;oBAClB,YAAY,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACjE,CAAC,CAAC;YACP,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,IAAY,MAAM;QACd,OAAO,IAAI,CAAC,QAAQ,YAAY,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACxF,CAAC"}
|
package/dist/prefetchFilter.d.ts
CHANGED
|
@@ -11,11 +11,29 @@
|
|
|
11
11
|
* targets, so it does NOT reintroduce the "SS brackets break Postgres" problem that motivated the
|
|
12
12
|
* previous (unsafe) unquoted form.
|
|
13
13
|
*/
|
|
14
|
+
import type { DatabasePlatform } from '@memberjunction/core';
|
|
14
15
|
/** The minimal dialect surface this builder needs; satisfied by `DatabaseProviderBase.Dialect` (SQLDialect). */
|
|
15
16
|
export interface SqlQuoter {
|
|
16
17
|
QuoteIdentifier(name: string): string;
|
|
17
18
|
QuoteStringLiteral(value: string): string;
|
|
19
|
+
/** Which platform the quoter is for. Always present on a real `SQLDialect`. */
|
|
20
|
+
PlatformKey?: DatabasePlatform;
|
|
18
21
|
}
|
|
22
|
+
/**
|
|
23
|
+
* Quotes a value as a string literal that a **Unicode text column** comparison will actually match.
|
|
24
|
+
*
|
|
25
|
+
* On SQL Server a bare `'…'` is a *varchar* literal: every character outside the database's
|
|
26
|
+
* collation codepage is replaced with `?` before the comparison runs. On the default
|
|
27
|
+
* `SQL_Latin1_General_CP1_CI_AS`, an external ID of `ünïcödé-Ω-日本語` becomes `ünïcödé-O-???`, so
|
|
28
|
+
* `ExternalSystemRecordID = '…'` matches the (nvarchar) row zero times — the record reads as
|
|
29
|
+
* unmapped and the sync creates it a second time in the customer's external system. `N'…'` makes
|
|
30
|
+
* the literal nvarchar, which is what every MJ string column already is, so there is no implicit
|
|
31
|
+
* conversion on the column and the index seek is unaffected.
|
|
32
|
+
*
|
|
33
|
+
* PostgreSQL literals are already Unicode and it has no `N` prefix, hence the platform check
|
|
34
|
+
* rather than an unconditional prefix.
|
|
35
|
+
*/
|
|
36
|
+
export declare function quoteTextLiteral(value: string, q: SqlQuoter): string;
|
|
19
37
|
/**
|
|
20
38
|
* @param pkNames the entity's primary-key field name(s), in `PrimaryKeys` order.
|
|
21
39
|
* @param ids the `MatchedMJRecordID` values — a single value per record for a single PK, or the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prefetchFilter.d.ts","sourceRoot":"","sources":["../src/prefetchFilter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;
|
|
1
|
+
{"version":3,"file":"prefetchFilter.d.ts","sourceRoot":"","sources":["../src/prefetchFilter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,gHAAgH;AAChH,MAAM,WAAW,SAAS;IACtB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACtC,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1C,+EAA+E;IAC/E,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAClC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,GAAG,MAAM,CAGpE;AAED;;;;;;GAMG;AACH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,GAAG,MAAM,CAarG"}
|
package/dist/prefetchFilter.js
CHANGED
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
* — the bulk stored-hash lookup that powers the idempotent-skip fast path.
|
|
2
|
+
* Quotes a value as a string literal that a **Unicode text column** comparison will actually match.
|
|
4
3
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
4
|
+
* On SQL Server a bare `'…'` is a *varchar* literal: every character outside the database's
|
|
5
|
+
* collation codepage is replaced with `?` before the comparison runs. On the default
|
|
6
|
+
* `SQL_Latin1_General_CP1_CI_AS`, an external ID of `ünïcödé-Ω-日本語` becomes `ünïcödé-O-???`, so
|
|
7
|
+
* `ExternalSystemRecordID = '…'` matches the (nvarchar) row zero times — the record reads as
|
|
8
|
+
* unmapped and the sync creates it a second time in the customer's external system. `N'…'` makes
|
|
9
|
+
* the literal nvarchar, which is what every MJ string column already is, so there is no implicit
|
|
10
|
+
* conversion on the column and the index seek is unaffected.
|
|
11
|
+
*
|
|
12
|
+
* PostgreSQL literals are already Unicode and it has no `N` prefix, hence the platform check
|
|
13
|
+
* rather than an unconditional prefix.
|
|
13
14
|
*/
|
|
15
|
+
export function quoteTextLiteral(value, q) {
|
|
16
|
+
const quoted = q.QuoteStringLiteral(value);
|
|
17
|
+
return q.PlatformKey === 'sqlserver' ? `N${quoted}` : quoted;
|
|
18
|
+
}
|
|
14
19
|
/**
|
|
15
20
|
* @param pkNames the entity's primary-key field name(s), in `PrimaryKeys` order.
|
|
16
21
|
* @param ids the `MatchedMJRecordID` values — a single value per record for a single PK, or the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prefetchFilter.js","sourceRoot":"","sources":["../src/prefetchFilter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"prefetchFilter.js","sourceRoot":"","sources":["../src/prefetchFilter.ts"],"names":[],"mappings":"AAuBA;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,CAAY;IACxD,MAAM,MAAM,GAAG,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO,CAAC,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AACjE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,8BAA8B,CAAC,OAAiB,EAAE,GAAa,EAAE,CAAY;IACzF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,4CAA4C;QAC5C,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzE,OAAO,GAAG,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAM,GAAG,CAAC;IAC7D,CAAC;IACD,2FAA2F;IAC3F,mFAAmF;IACnF,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QACjB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrC,OAAO,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACjC,GAAG,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC;IAC5G,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACpB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memberjunction/integration-engine",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.51.0",
|
|
5
5
|
"description": "MemberJunction Integration Engine - orchestration, field mapping, and connector framework",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"types": "dist/index.d.ts",
|
|
@@ -14,12 +14,12 @@
|
|
|
14
14
|
"test:watch": "vitest"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@memberjunction/core": "5.
|
|
18
|
-
"@memberjunction/core-entities": "5.
|
|
19
|
-
"@memberjunction/global": "5.
|
|
20
|
-
"@memberjunction/integration-engine-base": "5.
|
|
21
|
-
"@memberjunction/integration-pk-classifier": "5.
|
|
22
|
-
"@memberjunction/integration-progress-artifacts": "5.
|
|
17
|
+
"@memberjunction/core": "5.51.0",
|
|
18
|
+
"@memberjunction/core-entities": "5.51.0",
|
|
19
|
+
"@memberjunction/global": "5.51.0",
|
|
20
|
+
"@memberjunction/integration-engine-base": "5.51.0",
|
|
21
|
+
"@memberjunction/integration-pk-classifier": "5.51.0",
|
|
22
|
+
"@memberjunction/integration-progress-artifacts": "5.51.0"
|
|
23
23
|
},
|
|
24
24
|
"devDependencies": {
|
|
25
25
|
"tsc-alias": "^1.8.16",
|