@memberjunction/integration-engine 6.1.0-edge.2 → 6.1.0-edge.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +180 -4
- package/dist/BaseIntegrationConnector.d.ts +115 -4
- package/dist/BaseIntegrationConnector.d.ts.map +1 -1
- package/dist/BaseIntegrationConnector.js +191 -24
- package/dist/BaseIntegrationConnector.js.map +1 -1
- package/dist/BaseRESTIntegrationConnector.d.ts +14 -1
- package/dist/BaseRESTIntegrationConnector.d.ts.map +1 -1
- package/dist/BaseRESTIntegrationConnector.js +50 -3
- package/dist/BaseRESTIntegrationConnector.js.map +1 -1
- package/dist/BatchIdentity.d.ts +47 -0
- package/dist/BatchIdentity.d.ts.map +1 -0
- package/dist/BatchIdentity.js +78 -0
- package/dist/BatchIdentity.js.map +1 -0
- package/dist/DiscoveryWatchdog.d.ts +103 -0
- package/dist/DiscoveryWatchdog.d.ts.map +1 -0
- package/dist/DiscoveryWatchdog.js +166 -0
- package/dist/DiscoveryWatchdog.js.map +1 -0
- package/dist/FieldMapValidation.d.ts +40 -0
- package/dist/FieldMapValidation.d.ts.map +1 -0
- package/dist/FieldMapValidation.js +58 -0
- package/dist/FieldMapValidation.js.map +1 -0
- package/dist/FieldMappingEngine.d.ts +1 -13
- package/dist/FieldMappingEngine.d.ts.map +1 -1
- package/dist/FieldMappingEngine.js +18 -3
- package/dist/FieldMappingEngine.js.map +1 -1
- package/dist/IntegrationConnectorCreationPipeline.d.ts +92 -1
- package/dist/IntegrationConnectorCreationPipeline.d.ts.map +1 -1
- package/dist/IntegrationConnectorCreationPipeline.js +186 -11
- package/dist/IntegrationConnectorCreationPipeline.js.map +1 -1
- package/dist/IntegrationEngine.d.ts +198 -21
- package/dist/IntegrationEngine.d.ts.map +1 -1
- package/dist/IntegrationEngine.js +1092 -165
- package/dist/IntegrationEngine.js.map +1 -1
- package/dist/IntegrationSchemaSync.d.ts +8 -0
- package/dist/IntegrationSchemaSync.d.ts.map +1 -1
- package/dist/IntegrationSchemaSync.js +39 -2
- package/dist/IntegrationSchemaSync.js.map +1 -1
- package/dist/KeylessRecordGuard.d.ts +56 -0
- package/dist/KeylessRecordGuard.d.ts.map +1 -0
- package/dist/KeylessRecordGuard.js +80 -0
- package/dist/KeylessRecordGuard.js.map +1 -0
- package/dist/ResumeConcurrency.d.ts +46 -0
- package/dist/ResumeConcurrency.d.ts.map +1 -0
- package/dist/ResumeConcurrency.js +74 -0
- package/dist/ResumeConcurrency.js.map +1 -0
- package/dist/RetryAfter.d.ts +50 -0
- package/dist/RetryAfter.d.ts.map +1 -0
- package/dist/RetryAfter.js +127 -0
- package/dist/RetryAfter.js.map +1 -0
- package/dist/RetryRunner.d.ts +26 -2
- package/dist/RetryRunner.d.ts.map +1 -1
- package/dist/RetryRunner.js +11 -3
- package/dist/RetryRunner.js.map +1 -1
- package/dist/RunOwnershipService.d.ts +177 -0
- package/dist/RunOwnershipService.d.ts.map +1 -0
- package/dist/RunOwnershipService.js +279 -0
- package/dist/RunOwnershipService.js.map +1 -0
- package/dist/SyncDirectives.d.ts +66 -0
- package/dist/SyncDirectives.d.ts.map +1 -0
- package/dist/SyncDirectives.js +123 -0
- package/dist/SyncDirectives.js.map +1 -0
- package/dist/SyncLogger.d.ts +1 -1
- package/dist/SyncLogger.d.ts.map +1 -1
- package/dist/SyncLogger.js.map +1 -1
- package/dist/WatermarkService.d.ts +8 -0
- package/dist/WatermarkService.d.ts.map +1 -1
- package/dist/WatermarkService.js +18 -0
- package/dist/WatermarkService.js.map +1 -1
- package/dist/index.d.ts +12 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +28 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +66 -1
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
package/LICENSE
CHANGED
|
@@ -1,7 +1,183 @@
|
|
|
1
|
-
|
|
1
|
+
Business Source License 1.1
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
License text copyright (c) 2024 MariaDB plc, All Rights Reserved.
|
|
4
|
+
"Business Source License" is a trademark of MariaDB plc.
|
|
4
5
|
|
|
5
|
-
|
|
6
|
+
-----------------------------------------------------------------------------
|
|
6
7
|
|
|
7
|
-
|
|
8
|
+
Parameters
|
|
9
|
+
|
|
10
|
+
Licensor: Blue Cypress, Inc.
|
|
11
|
+
|
|
12
|
+
Licensed Work: MemberJunction.
|
|
13
|
+
The Licensed Work is (c) 2023-2026 Blue Cypress, Inc.
|
|
14
|
+
|
|
15
|
+
Additional Use Grant: Subject to the terms of this License, Licensor grants
|
|
16
|
+
you the following additional rights to make Production
|
|
17
|
+
Use of the Licensed Work.
|
|
18
|
+
|
|
19
|
+
1. Internal Use
|
|
20
|
+
|
|
21
|
+
You may make production use of the Licensed Work for
|
|
22
|
+
your own internal business or organizational operations.
|
|
23
|
+
|
|
24
|
+
2. Nonprofit Use
|
|
25
|
+
|
|
26
|
+
If you are a Nonprofit, you may make production use of
|
|
27
|
+
the Licensed Work for the operations and activities of
|
|
28
|
+
your Organizational Family.
|
|
29
|
+
|
|
30
|
+
3. MemberJunction Certified Program Use
|
|
31
|
+
|
|
32
|
+
If you are authorized by Licensor under the
|
|
33
|
+
MemberJunction Certified Program to provide professional
|
|
34
|
+
services using the Licensed Work, you may make
|
|
35
|
+
production use of the Licensed Work in providing such
|
|
36
|
+
professional services to a client, provided that:
|
|
37
|
+
|
|
38
|
+
(a) the Licensed Work is deployed in, and the applicable
|
|
39
|
+
production use occurs within, an environment owned,
|
|
40
|
+
leased, licensed, subscribed to, or otherwise controlled
|
|
41
|
+
by that client; and
|
|
42
|
+
|
|
43
|
+
(b) the production use is for that client's own internal
|
|
44
|
+
business or organizational operations or is otherwise
|
|
45
|
+
independently permitted to that client under this
|
|
46
|
+
Additional Use Grant.
|
|
47
|
+
|
|
48
|
+
4. Definitions Applicable to the Additional Use Grant
|
|
49
|
+
|
|
50
|
+
"Affiliate" means, with respect to a specified Person,
|
|
51
|
+
any other Person that directly or indirectly Controls,
|
|
52
|
+
is Controlled by, or is under common Control with such
|
|
53
|
+
specified Person.
|
|
54
|
+
|
|
55
|
+
"Control" (including the terms "Controls," "Controlled
|
|
56
|
+
by," and "under common Control with") means the direct
|
|
57
|
+
or indirect possession of the power to direct or cause
|
|
58
|
+
the direction of the management and policies of a
|
|
59
|
+
Person, whether through ownership of voting interests,
|
|
60
|
+
by contract, or otherwise.
|
|
61
|
+
|
|
62
|
+
"Organizational Family" means, with respect to a Person,
|
|
63
|
+
(a) such Person and its Affiliates, and (b) any
|
|
64
|
+
nonprofit organization, governmental entity, chapter,
|
|
65
|
+
division, local affiliate, regional affiliate, state
|
|
66
|
+
affiliate, national affiliate, or other entity that is
|
|
67
|
+
formally affiliated with such Person through governing
|
|
68
|
+
documents, a charter, bylaws, a membership agreement, or
|
|
69
|
+
another written organizational instrument, and is
|
|
70
|
+
recognized under such documents as part of the same
|
|
71
|
+
organizational structure.
|
|
72
|
+
|
|
73
|
+
"Nonprofit" means a Person recognized by the Internal
|
|
74
|
+
Revenue Service as exempt from federal income taxation
|
|
75
|
+
under Section 501(c)(3), 501(c)(4), 501(c)(5), or
|
|
76
|
+
501(c)(6) of the Internal Revenue Code, or a foreign
|
|
77
|
+
organization recognized under substantially equivalent
|
|
78
|
+
laws.
|
|
79
|
+
|
|
80
|
+
A Person claiming eligibility as a Nonprofit shall, upon
|
|
81
|
+
Licensor's reasonable request, provide documentation
|
|
82
|
+
reasonably sufficient to demonstrate that it qualifies
|
|
83
|
+
as a Nonprofit. If such Person materially misrepresents,
|
|
84
|
+
or is unable to demonstrate, its qualification as a
|
|
85
|
+
Nonprofit, the rights granted to such Person under
|
|
86
|
+
Section 2 of this Additional Use Grant shall terminate.
|
|
87
|
+
|
|
88
|
+
"MemberJunction Certified Program" means Licensor's
|
|
89
|
+
then-current program for certifying and authorizing a
|
|
90
|
+
Person to provide professional services using the
|
|
91
|
+
Licensed Work.
|
|
92
|
+
|
|
93
|
+
"Person" means any individual, corporation, limited
|
|
94
|
+
liability company, partnership, association, nonprofit
|
|
95
|
+
organization, governmental entity, or other legal or
|
|
96
|
+
organizational entity.
|
|
97
|
+
|
|
98
|
+
Change Date: Four (4) years from the date the Licensed Work is first
|
|
99
|
+
made available.
|
|
100
|
+
|
|
101
|
+
Change License: MIT License.
|
|
102
|
+
|
|
103
|
+
For information about alternative licensing arrangements for the Licensed
|
|
104
|
+
Work, please contact Blue Cypress, Inc.
|
|
105
|
+
|
|
106
|
+
-----------------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
Terms
|
|
109
|
+
|
|
110
|
+
The Licensor hereby grants you the right to copy, modify, create derivative
|
|
111
|
+
works, redistribute, and make non-production use of the Licensed Work. The
|
|
112
|
+
Licensor may make an Additional Use Grant, above, permitting limited
|
|
113
|
+
production use.
|
|
114
|
+
|
|
115
|
+
Effective on the Change Date, or the fourth anniversary of the first publicly
|
|
116
|
+
available distribution of a specific version of the Licensed Work under this
|
|
117
|
+
License, whichever comes first, the Licensor hereby grants you rights under
|
|
118
|
+
the terms of the Change License, and the rights granted in the paragraph
|
|
119
|
+
above terminate.
|
|
120
|
+
|
|
121
|
+
If your use of the Licensed Work does not comply with the requirements
|
|
122
|
+
currently in effect as described in this License, you must purchase a
|
|
123
|
+
commercial license from the Licensor, its affiliated entities, or authorized
|
|
124
|
+
resellers, or you must refrain from using the Licensed Work.
|
|
125
|
+
|
|
126
|
+
All copies of the original and modified Licensed Work, and derivative works
|
|
127
|
+
of the Licensed Work, are subject to this License. This License applies
|
|
128
|
+
separately for each version of the Licensed Work and the Change Date may vary
|
|
129
|
+
for each version of the Licensed Work released by Licensor.
|
|
130
|
+
|
|
131
|
+
You must conspicuously display this License on each original or modified copy
|
|
132
|
+
of the Licensed Work. If you receive the Licensed Work in original or
|
|
133
|
+
modified form from a third party, the terms and conditions set forth in this
|
|
134
|
+
License apply to your use of that work.
|
|
135
|
+
|
|
136
|
+
Any use of the Licensed Work in violation of this License will automatically
|
|
137
|
+
terminate your rights under this License for the current and all other
|
|
138
|
+
versions of the Licensed Work.
|
|
139
|
+
|
|
140
|
+
This License does not grant you any right in any trademark or logo of
|
|
141
|
+
Licensor or its affiliates (provided that you may use a trademark or logo of
|
|
142
|
+
Licensor as expressly required by this License).
|
|
143
|
+
|
|
144
|
+
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
|
|
145
|
+
AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
|
|
146
|
+
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
|
|
147
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
|
|
148
|
+
TITLE.
|
|
149
|
+
|
|
150
|
+
MariaDB hereby grants you permission to use this License's text to license
|
|
151
|
+
your works, and to refer to it using the trademark "Business Source License",
|
|
152
|
+
as long as you comply with the Covenants of Licensor below.
|
|
153
|
+
|
|
154
|
+
-----------------------------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
Covenants of Licensor
|
|
157
|
+
|
|
158
|
+
In consideration of the right to use this License's text and the "Business
|
|
159
|
+
Source License" name and trademark, Licensor covenants to MariaDB, and to all
|
|
160
|
+
other recipients of the licensed work to be provided by Licensor:
|
|
161
|
+
|
|
162
|
+
1. To specify as the Change License the GPL Version 2.0 or any later version,
|
|
163
|
+
or a license that is compatible with GPL Version 2.0 or a later version,
|
|
164
|
+
where "compatible" means that software provided under the Change License
|
|
165
|
+
can be included in a program with software provided under GPL Version 2.0
|
|
166
|
+
or a later version. Licensor may specify additional Change Licenses without
|
|
167
|
+
limitation.
|
|
168
|
+
|
|
169
|
+
2. To either: (a) specify an additional grant of rights to use that does not
|
|
170
|
+
impose any additional restriction on the right granted in this License, as
|
|
171
|
+
the Additional Use Grant; or (b) insert the text "None".
|
|
172
|
+
|
|
173
|
+
3. To specify a Change Date.
|
|
174
|
+
|
|
175
|
+
4. Not to modify this License in any other way.
|
|
176
|
+
|
|
177
|
+
-----------------------------------------------------------------------------
|
|
178
|
+
|
|
179
|
+
Notice
|
|
180
|
+
|
|
181
|
+
The Business Source License (this document, or the "License") is not an Open
|
|
182
|
+
Source license. However, the Licensed Work will eventually be made available
|
|
183
|
+
under an Open Source License, as stated in this License.
|
|
@@ -120,6 +120,45 @@ export interface FetchContext {
|
|
|
120
120
|
RateLimitAcquire?: () => Promise<void>;
|
|
121
121
|
RateLimitReport?: (throttledErr?: unknown) => void;
|
|
122
122
|
MaxConcurrency?: number;
|
|
123
|
+
/**
|
|
124
|
+
* SAMPLING, NOT SYNCING — and the wall-clock this call must not outlive.
|
|
125
|
+
*
|
|
126
|
+
* Discovery wants a corpus, not a corpus of everything: ~50 records is enough to infer columns,
|
|
127
|
+
* types, string widths and a provable primary key. `DiscoverFieldsViaFetch` already knows that
|
|
128
|
+
* and already computes a budget — but it hands that budget to the code CONSUMING the record
|
|
129
|
+
* stream, and the consumer only regains control BETWEEN `FetchChanges` calls. Nothing was ever
|
|
130
|
+
* passed to the connector itself, so a connector could not honour a budget even if it wanted to:
|
|
131
|
+
* it had no way to know it was being sampled rather than synced.
|
|
132
|
+
*
|
|
133
|
+
* That is survivable while one `FetchChanges` is one HTTP page — the consumer stops after 50
|
|
134
|
+
* records and the gap never shows. It is NOT survivable for a parent-scoped object, where a
|
|
135
|
+
* single call fans out internally into one request per parent. There the consumer cannot
|
|
136
|
+
* interrupt anything, because control does not come back until every parent has been walked.
|
|
137
|
+
*
|
|
138
|
+
* Observed live 2026-08-12: a Totara discovery spent 28 minutes inside ONE `FetchChanges` call,
|
|
139
|
+
* walking every parent, and returned `rows=0` — half an hour of correct, pointless work to
|
|
140
|
+
* collect a sample it could never have found there. A sampling operation had silently become an
|
|
141
|
+
* exhaustive one.
|
|
142
|
+
*
|
|
143
|
+
* So the intent now travels with the call. A connector that ignores these behaves exactly as
|
|
144
|
+
* before; one that fans out internally can stop early and return what it has.
|
|
145
|
+
*
|
|
146
|
+
* STOP ON RECORDS, NOT ON PARENTS. A child object only yields through its parents, so capping the
|
|
147
|
+
* number of parents visited would be wrong — if the first three courses have no enrolments you
|
|
148
|
+
* genuinely must keep walking to find fifty rows. `SampleTargetRecords` is therefore the primary
|
|
149
|
+
* stop and the walk should honour it the moment it is met, whichever parent it happens to be on.
|
|
150
|
+
* `DeadlineMs` is the BACKSTOP for the other case: parents that will never yield anything, where
|
|
151
|
+
* no record count can ever be reached and only the clock can end it.
|
|
152
|
+
*
|
|
153
|
+
* - `IsDiscoverySample` — this call exists to characterise the shape of the data, not to move it.
|
|
154
|
+
* - `SampleTargetRecords` — stop as soon as this many records have been collected. Enough is enough.
|
|
155
|
+
* - `DeadlineMs` — epoch ms after which the connector should stop and return what it has.
|
|
156
|
+
* A partial sample is the CORRECT result here: discovery infers from
|
|
157
|
+
* whatever it gets, and returning little beats half an hour of silence.
|
|
158
|
+
*/
|
|
159
|
+
IsDiscoverySample?: boolean;
|
|
160
|
+
SampleTargetRecords?: number;
|
|
161
|
+
DeadlineMs?: number;
|
|
123
162
|
}
|
|
124
163
|
/**
|
|
125
164
|
* A non-fatal diagnostic a connector attaches to a fetch result so the engine surfaces it in the
|
|
@@ -172,14 +211,41 @@ export interface OperationTimeouts {
|
|
|
172
211
|
/** Default timeout values for connector operations */
|
|
173
212
|
export declare const DEFAULT_OPERATION_TIMEOUTS: OperationTimeouts;
|
|
174
213
|
/**
|
|
175
|
-
*
|
|
214
|
+
* The error {@link WithTimeout} rejects with when ITS OWN budget expires — as distinct from an error
|
|
215
|
+
* the wrapped operation itself produced.
|
|
216
|
+
*
|
|
217
|
+
* That distinction is the whole point of the class. `ClassifyError` maps anything matching "timeout"
|
|
218
|
+
* to `NETWORK_TIMEOUT`, which `IsRetryableError` treats as transient — correct for a socket that
|
|
219
|
+
* dropped, wrong for "we gave this operation N ms and it wanted more." Retrying the latter re-runs
|
|
220
|
+
* the same work under the same budget, so it fails the same way, having spent the budget again.
|
|
221
|
+
*
|
|
222
|
+
* The message is deliberately UNCHANGED from the plain `Error` this replaced: `ClassifyError` reads
|
|
223
|
+
* message text, so existing classification, logging and the run-event stream all behave exactly as
|
|
224
|
+
* before. Only callers that explicitly check `instanceof` see any difference.
|
|
225
|
+
*/
|
|
226
|
+
export declare class OperationTimeoutError extends Error {
|
|
227
|
+
/** The `operationName` passed to {@link WithTimeout}. */
|
|
228
|
+
readonly OperationName: string;
|
|
229
|
+
/** The budget that expired, in milliseconds. */
|
|
230
|
+
readonly TimeoutMs: number;
|
|
231
|
+
constructor(operationName: string, timeoutMs: number);
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Wraps a promise with a timeout. Rejects with {@link OperationTimeoutError} if the
|
|
176
235
|
* promise does not resolve within the specified duration.
|
|
177
236
|
*
|
|
237
|
+
* CAVEAT — this does NOT cancel the wrapped operation. It is a `Promise.race`, so on timeout the
|
|
238
|
+
* underlying work keeps running to completion (or its own failure) with nobody awaiting it. A caller
|
|
239
|
+
* that retries a timed-out operation therefore stacks a second copy on top of the first, still
|
|
240
|
+
* in-flight — which is why `IntegrationEngine`'s fetch path does not retry
|
|
241
|
+
* {@link OperationTimeoutError}. Real cancellation needs an `AbortSignal` threaded into the
|
|
242
|
+
* connector contract; until then, treat a timeout as terminal for that attempt.
|
|
243
|
+
*
|
|
178
244
|
* @param promise - The promise to wrap
|
|
179
245
|
* @param timeoutMs - Timeout in milliseconds
|
|
180
246
|
* @param operationName - Name of the operation for error messaging
|
|
181
247
|
* @returns The result of the promise
|
|
182
|
-
* @throws
|
|
248
|
+
* @throws OperationTimeoutError if the operation times out
|
|
183
249
|
*/
|
|
184
250
|
export declare function WithTimeout<T>(promise: Promise<T>, timeoutMs: number, operationName: string): Promise<T>;
|
|
185
251
|
/** Proposed default configuration for a quick-start setup */
|
|
@@ -327,13 +393,51 @@ export declare abstract class BaseIntegrationConnector {
|
|
|
327
393
|
* Parse a Retry-After / rate-limit signal out of a failed response or thrown error into
|
|
328
394
|
* milliseconds so the engine can back off precisely. Return `undefined` when the error is not a
|
|
329
395
|
* throttle (or carries no hint).
|
|
396
|
+
*
|
|
397
|
+
* The default reads the standard `Retry-After` header (RFC 9110 §10.2.3 — the header a 429 and a
|
|
398
|
+
* 503 carry), in both its delay-seconds and HTTP-date forms, from wherever the HTTP client put
|
|
399
|
+
* it. That is not a heuristic and not vendor-specific: there is one correct reading of it, and
|
|
400
|
+
* every HTTP connector benefits from having it read.
|
|
401
|
+
*
|
|
402
|
+
* This used to return `undefined` unconditionally, and no connector in this repo overrode it —
|
|
403
|
+
* so the engine's limiter never learned a delay any vendor had actually stated, and discovery's
|
|
404
|
+
* throttle check (which asked this and only this) concluded "not a throttle" for every 429 MJ
|
|
405
|
+
* ever received.
|
|
406
|
+
*
|
|
407
|
+
* Override when the vendor signals its delay somewhere non-standard — in the response body, or
|
|
408
|
+
* in prose (PheedLoop's "Expected available in N second"). Deliberately not parsed here:
|
|
409
|
+
* guessing a duration out of message text risks inventing one, and a wrong Retry-After is worse
|
|
410
|
+
* than none, since it freezes the token bucket for a made-up interval.
|
|
330
411
|
*/
|
|
331
|
-
ExtractRetryAfterMs(
|
|
412
|
+
ExtractRetryAfterMs(error: unknown): number | undefined;
|
|
332
413
|
/**
|
|
333
414
|
* Highest SAFE per-layer concurrency the source tolerates (plan.md §7 peak parallelization) — the
|
|
334
415
|
* ceiling the engine's adaptive controller ramps toward. `null` → use configured syncConcurrency.
|
|
335
416
|
*/
|
|
336
417
|
get MaxConcurrencyHint(): number | null;
|
|
418
|
+
/**
|
|
419
|
+
* Per-connector override for the `FetchChanges` timeout, in milliseconds. `null` → use
|
|
420
|
+
* `DEFAULT_OPERATION_TIMEOUTS.FetchChangesMs` (30s).
|
|
421
|
+
*
|
|
422
|
+
* Raise this when a single page is legitimately slow: a connector that fans out one request per
|
|
423
|
+
* parent (ORCID's per-iD `/record`, any second-layer object) does N requests inside ONE
|
|
424
|
+
* `FetchChanges` call, so its page time scales with `BatchSize` — and with how much concurrency
|
|
425
|
+
* the engine's adaptive controller currently allows. Under the fixed 30s, a page that comfortably
|
|
426
|
+
* fit when parallel no longer fit once the controller had cut concurrency, which forced connector
|
|
427
|
+
* authors to shrink `BatchSize` for the sequential worst case and waste the parallel headroom the
|
|
428
|
+
* rest of the time.
|
|
429
|
+
*
|
|
430
|
+
* Note the timeout does NOT itself cut concurrency: `ClassifyError` gives it `NETWORK_TIMEOUT`,
|
|
431
|
+
* and only `RATE_LIMIT_EXCEEDED` feeds the adaptive limiter. A timing-out page simply never ramps
|
|
432
|
+
* the rate UP (the ramp needs a clean fetch), and the object ends incomplete — reported as
|
|
433
|
+
* `FETCH_ABORTED_INCOMPLETE`. Earlier revisions of this comment described a self-reinforcing
|
|
434
|
+
* timeout→concurrency-cut spiral; that is not what the code does.
|
|
435
|
+
*
|
|
436
|
+
* Precedence (highest first): `CompanyIntegration.Configuration.fetchTimeoutMs` → this property
|
|
437
|
+
* → `DEFAULT_OPERATION_TIMEOUTS.FetchChangesMs`. Deployments therefore keep the last word without
|
|
438
|
+
* a code change, while a connector that KNOWS it is slow ships a sane default.
|
|
439
|
+
*/
|
|
440
|
+
get FetchChangesTimeoutMs(): number | null;
|
|
337
441
|
/**
|
|
338
442
|
* Name of a stable, monotonic ordering key (PK/identity) usable for KEYSET/seek resume on
|
|
339
443
|
* watermark-less objects (plan.md §7 — resume from last-seen key, robust to mid-stream
|
|
@@ -421,7 +525,14 @@ export declare abstract class BaseIntegrationConnector {
|
|
|
421
525
|
* (e.g. REST) overrides this to sample a template-var CHILD with the correct record-constrained,
|
|
422
526
|
* recursive stream. Yields plain field maps; the caller stops it at `maxRecords`.
|
|
423
527
|
*/
|
|
424
|
-
protected DiscoverySampleRecordStream(companyIntegration: MJCompanyIntegrationEntity, objectName: string, contextUser: UserInfo, batchSize: number, maxRecords: number
|
|
528
|
+
protected DiscoverySampleRecordStream(companyIntegration: MJCompanyIntegrationEntity, objectName: string, contextUser: UserInfo, batchSize: number, maxRecords: number,
|
|
529
|
+
/**
|
|
530
|
+
* Epoch ms this sample must not outlive, forwarded to the connector as `FetchContext.DeadlineMs`.
|
|
531
|
+
* Optional so existing overrides of this method keep compiling and keep their current behaviour.
|
|
532
|
+
*/
|
|
533
|
+
deadlineMs?: number,
|
|
534
|
+
/** Watchdog key for this sample, when the caller registered one. Diagnostics only. */
|
|
535
|
+
watchKey?: string): AsyncGenerator<Record<string, unknown>>;
|
|
425
536
|
/**
|
|
426
537
|
* Tests connectivity to the external system.
|
|
427
538
|
* @param companyIntegration - The company integration entity with connection credentials
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BaseIntegrationConnector.d.ts","sourceRoot":"","sources":["../src/BaseIntegrationConnector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,KAAK,EACR,qBAAqB,EACrB,qBAAqB,EACxB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,EACR,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,uBAAuB,EACvB,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,UAAU,EACV,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACb,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"BaseIntegrationConnector.d.ts","sourceRoot":"","sources":["../src/BaseIntegrationConnector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,KAAK,EACR,qBAAqB,EACrB,qBAAqB,EACxB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,EACR,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,uBAAuB,EACvB,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,UAAU,EACV,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACb,MAAM,YAAY,CAAC;AAGpB,OAAO,EAIH,KAAK,sBAAsB,EAC3B,KAAK,aAAa,EACrB,MAAM,yBAAyB,CAAC;AAKjC,2DAA2D;AAC3D,MAAM,WAAW,oBAAoB;IACjC,4CAA4C;IAC5C,OAAO,EAAE,OAAO,CAAC;IACjB,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,kEAAkE;AAClE,MAAM,WAAW,oBAAoB;IACjC,gDAAgD;IAChD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,0DAA0D;IAC1D,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,uBAAuB,EAAE,OAAO,CAAC;IACjC,gEAAgE;IAChE,aAAa,EAAE,OAAO,CAAC;CAC1B;AAED,iEAAiE;AACjE,MAAM,WAAW,mBAAmB;IAChC,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,UAAU,EAAE,OAAO,CAAC;IACpB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,0EAA0E;IAC1E,WAAW,EAAE,OAAO,CAAC;IACrB,qCAAqC;IACrC,UAAU,EAAE,OAAO,CAAC;IACpB,0CAA0C;IAC1C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,+CAA+C;IAC/C,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,oFAAoF;IACpF,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,gFAAgF;IAChF,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED,oEAAoE;AACpE,MAAM,WAAW,YAAY;IACzB,kEAAkE;IAClE,kBAAkB,EAAE,0BAA0B,CAAC;IAC/C,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,4EAA4E;IAC5E,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,WAAW,EAAE,QAAQ,CAAC;IACtB,qGAAqG;IACrG,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wFAAwF;IACxF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,wFAAwF;IACxF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,uJAAuJ;IACvJ,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;IACjC;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,eAAe,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IACnD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IACzB,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,sEAAsE;IACtE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,mEAAmE;AACnE,MAAM,WAAW,gBAAgB;IAC7B,sCAAsC;IACtC,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,YAAY,EAAE,CAAC;IAC1B,+CAA+C;IAC/C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,0GAA0G;IAC1G,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yGAAyG;IACzG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8JAA8J;IAC9J,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yGAAyG;IACzG,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,2DAA2D;AAC3D,MAAM,WAAW,iBAAiB;IAC9B,gEAAgE;IAChE,gBAAgB,EAAE,MAAM,CAAC;IACzB,kEAAkE;IAClE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iEAAiE;IACjE,gBAAgB,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,cAAc,EAAE,MAAM,CAAC;CAC1B;AAED,sDAAsD;AACtD,eAAO,MAAM,0BAA0B,EAAE,iBAKxC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,qBAAa,qBAAsB,SAAQ,KAAK;IAC5C,yDAAyD;IACzD,SAAgB,aAAa,EAAE,MAAM,CAAC;IACtC,gDAAgD;IAChD,SAAgB,SAAS,EAAE,MAAM,CAAC;gBAEtB,aAAa,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;CAMvD;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAC/B,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EACnB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,GACtB,OAAO,CAAC,CAAC,CAAC,CAiBZ;AAED,6DAA6D;AAC7D,MAAM,WAAW,mBAAmB;IAChC,gDAAgD;IAChD,gBAAgB,EAAE,MAAM,CAAC;IACzB,oDAAoD;IACpD,eAAe,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,WAAW,EAAE,OAAO,CAAC;IACrB,8BAA8B;IAC9B,aAAa,EAAE,mBAAmB,EAAE,CAAC;CACxC;AAED,yEAAyE;AACzE,MAAM,WAAW,wBAAwB;IACrC,iFAAiF;IACjF,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kEAAkE;IAClE,cAAc,EAAE,mBAAmB,EAAE,CAAC;CACzC;AAED;;;;;;;;GAQG;AACH;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC5B,0DAA0D;IAC1D,YAAY,EAAE,MAAM,CAAC;IACrB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,8BAAsB,wBAAwB;IAM1C,6EAA6E;IAC7E,IAAW,WAAW,IAAI,OAAO,CAAiB;IAElD,mFAAmF;IACnF,IAAW,cAAc,IAAI,OAAO,CAAkB;IAEtD,wFAAwF;IACxF,IAAW,cAAc,IAAI,OAAO,CAAkB;IAEtD;;;OAGG;IACH,IAAW,cAAc,IAAI,OAAO,CAAkB;IAEtD,iFAAiF;IACjF,IAAW,cAAc,IAAI,OAAO,CAAkB;IAEtD,+EAA+E;IAC/E,IAAW,cAAc,IAAI,OAAO,CAAkB;IAEtD,oEAAoE;IACpE,IAAW,eAAe,IAAI,OAAO,CAAkB;IAMvD;;;;OAIG;IACU,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,UAAU,CAAC;IAIzE;;;;OAIG;IACU,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,UAAU,CAAC;IAIzE;;;;;OAKG;IACU,MAAM,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,UAAU,CAAC;IAInE;;;;OAIG;IACU,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,UAAU,CAAC;IAIzE;;;OAGG;IACU,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAI9E;;;;OAIG;IACU,aAAa,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAItE;;;;OAIG;IACU,WAAW,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC;IAIhE;;;;;;OAMG;IACH,SAAS,CAAC,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,UAAU;IAkBvH;;;;OAIG;IACH,IAAW,eAAe,IAAI,eAAe,GAAG,IAAI,CAAiB;IAErE;;;;;;;;;;;;OAYG;IACH,IAAW,kBAAkB,IAAI,OAAO,CAAkB;IAE1D;;;;;;;;;;;;;;;;;;;OAmBG;IACI,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS;IAI9D;;;OAGG;IACH,IAAW,kBAAkB,IAAI,MAAM,GAAG,IAAI,CAAiB;IAE/D;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,IAAW,qBAAqB,IAAI,MAAM,GAAG,IAAI,CAAiB;IAElE;;;;OAIG;IACI,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAE5D,8FAA8F;IAC9F,IAAW,kBAAkB,IAAI,OAAO,CAAkB;IAE1D,4GAA4G;IAC/F,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAGnF,8DAA8D;IACjD,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAGnF,8DAA8D;IACjD,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;YAGrE,kBAAkB;IAMhC;;;;;OAKG;IACI,iBAAiB,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc;IAEhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiCG;cACa,uBAAuB,CACnC,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EACnF,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,sBAAsB,CAAC;QAAC,EAAE,CAAC,EAAE,aAAa,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,GAC1F,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAwEjC;;;;;;;;;;;;;;OAcG;IACU,sBAAsB,CAC/B,kBAAkB,EAAE,0BAA0B,EAC9C,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,QAAQ,EACrB,IAAI,GAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAO,GAC9E,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAyEjC;;;;;OAKG;cACc,2BAA2B,CACxC,kBAAkB,EAAE,0BAA0B,EAC9C,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,QAAQ,EACrB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM;IAClB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM;IACnB,sFAAsF;IACtF,QAAQ,CAAC,EAAE,MAAM,GAClB,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAgE1C;;;;;OAKG;aACa,cAAc,CAC1B,kBAAkB,EAAE,0BAA0B,EAC9C,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,oBAAoB,CAAC;IAEhC;;;;;OAKG;aACa,eAAe,CAC3B,kBAAkB,EAAE,0BAA0B,EAC9C,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAElC;;;;;;OAMG;aACa,cAAc,CAC1B,kBAAkB,EAAE,0BAA0B,EAC9C,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAEjC;;;;;OAKG;aACa,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAE1E;;;;;;OAMG;IACI,uBAAuB,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,mBAAmB,EAAE;IAI/F;;;;;OAKG;IACI,uBAAuB,IAAI,wBAAwB,GAAG,IAAI;IAMjE;;;;;;;;OAQG;IACI,qBAAqB,IAAI,qBAAqB,EAAE;IAIvD;;;;;;;OAOG;IACI,wBAAwB,IAAI,qBAAqB,GAAG,IAAI;IAgB/D;;;;;;OAMG;IACH,IAAW,eAAe,IAAI,MAAM,CAEnC;IAID;;;;;;;;;;;;OAYG;IACH;;;;;;;;OAQG;IACH,IAAW,wBAAwB,IAAI,OAAO,CAE7C;IAEY,gBAAgB,CACzB,kBAAkB,EAAE,0BAA0B,EAC9C,WAAW,EAAE,QAAQ,EACrB,OAAO,CAAC,EAAE,uBAAuB,GAClC,OAAO,CAAC,gBAAgB,CAAC;CAwI/B"}
|