@memberjunction/integration-engine 6.1.0-edge.3 → 6.1.0-edge.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/BaseIntegrationConnector.d.ts +26 -2
- package/dist/BaseIntegrationConnector.d.ts.map +1 -1
- package/dist/BaseIntegrationConnector.js +145 -23
- package/dist/BaseIntegrationConnector.js.map +1 -1
- package/dist/BaseRESTIntegrationConnector.d.ts +62 -8
- package/dist/BaseRESTIntegrationConnector.d.ts.map +1 -1
- package/dist/BaseRESTIntegrationConnector.js +200 -20
- package/dist/BaseRESTIntegrationConnector.js.map +1 -1
- package/dist/BatchIdentity.d.ts +47 -0
- package/dist/BatchIdentity.d.ts.map +1 -0
- package/dist/BatchIdentity.js +78 -0
- package/dist/BatchIdentity.js.map +1 -0
- package/dist/DeclaredSampleMerge.d.ts +74 -0
- package/dist/DeclaredSampleMerge.d.ts.map +1 -0
- package/dist/DeclaredSampleMerge.js +104 -0
- package/dist/DeclaredSampleMerge.js.map +1 -0
- package/dist/DiscoveryWatchdog.d.ts +103 -0
- package/dist/DiscoveryWatchdog.d.ts.map +1 -0
- package/dist/DiscoveryWatchdog.js +166 -0
- package/dist/DiscoveryWatchdog.js.map +1 -0
- package/dist/FieldMapValidation.d.ts +40 -0
- package/dist/FieldMapValidation.d.ts.map +1 -0
- package/dist/FieldMapValidation.js +58 -0
- package/dist/FieldMapValidation.js.map +1 -0
- package/dist/FieldMappingEngine.d.ts +1 -13
- package/dist/FieldMappingEngine.d.ts.map +1 -1
- package/dist/FieldMappingEngine.js +18 -3
- package/dist/FieldMappingEngine.js.map +1 -1
- package/dist/IntegrationConnectorCreationPipeline.d.ts +20 -0
- package/dist/IntegrationConnectorCreationPipeline.d.ts.map +1 -1
- package/dist/IntegrationConnectorCreationPipeline.js +203 -30
- package/dist/IntegrationConnectorCreationPipeline.js.map +1 -1
- package/dist/IntegrationEngine.d.ts +204 -1
- package/dist/IntegrationEngine.d.ts.map +1 -1
- package/dist/IntegrationEngine.js +1224 -262
- package/dist/IntegrationEngine.js.map +1 -1
- package/dist/IntegrationSchemaSync.d.ts +54 -2
- package/dist/IntegrationSchemaSync.d.ts.map +1 -1
- package/dist/IntegrationSchemaSync.js +122 -16
- package/dist/IntegrationSchemaSync.js.map +1 -1
- package/dist/KeylessRecordGuard.d.ts +56 -0
- package/dist/KeylessRecordGuard.d.ts.map +1 -0
- package/dist/KeylessRecordGuard.js +80 -0
- package/dist/KeylessRecordGuard.js.map +1 -0
- package/dist/ResumeConcurrency.d.ts +46 -0
- package/dist/ResumeConcurrency.d.ts.map +1 -0
- package/dist/ResumeConcurrency.js +74 -0
- package/dist/ResumeConcurrency.js.map +1 -0
- package/dist/RetryAfter.d.ts +50 -0
- package/dist/RetryAfter.d.ts.map +1 -0
- package/dist/RetryAfter.js +127 -0
- package/dist/RetryAfter.js.map +1 -0
- package/dist/RetryRunner.d.ts +26 -2
- package/dist/RetryRunner.d.ts.map +1 -1
- package/dist/RetryRunner.js +11 -3
- package/dist/RetryRunner.js.map +1 -1
- package/dist/StreamingDiscovery.d.ts +11 -0
- package/dist/StreamingDiscovery.d.ts.map +1 -1
- package/dist/StreamingDiscovery.js +13 -3
- package/dist/StreamingDiscovery.js.map +1 -1
- package/dist/SyncDirectives.d.ts +66 -0
- package/dist/SyncDirectives.d.ts.map +1 -0
- package/dist/SyncDirectives.js +123 -0
- package/dist/SyncDirectives.js.map +1 -0
- package/dist/SyncLogger.d.ts +2 -1
- package/dist/SyncLogger.d.ts.map +1 -1
- package/dist/SyncLogger.js +1 -0
- package/dist/SyncLogger.js.map +1 -1
- package/dist/WatermarkService.d.ts +15 -0
- package/dist/WatermarkService.d.ts.map +1 -1
- package/dist/WatermarkService.js +26 -0
- package/dist/WatermarkService.js.map +1 -1
- package/dist/WriteSerializer.d.ts +44 -0
- package/dist/WriteSerializer.d.ts.map +1 -0
- package/dist/WriteSerializer.js +70 -0
- package/dist/WriteSerializer.js.map +1 -0
- package/dist/index.d.ts +10 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -1
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +45 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +66 -1
- package/dist/types.js.map +1 -1
- package/package.json +7 -7
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Field-level sync directives.
|
|
3
|
+
*
|
|
4
|
+
* A connector can now declare that a source field should not be synced at all
|
|
5
|
+
* (`SourceFieldInfo.SyncDirective: 'Exclude'`). The directive is persisted into
|
|
6
|
+
* `IntegrationObjectField.Configuration` (a JSON column that already exists, so no
|
|
7
|
+
* migration is required) and honoured by the mapping stage, which strips excluded
|
|
8
|
+
* keys from the record BEFORE field mapping runs.
|
|
9
|
+
*
|
|
10
|
+
* Why stripping must happen there and not by deactivating a field map: an unmapped
|
|
11
|
+
* source key does not disappear — `computeUnmappedFields` captures it and the writer
|
|
12
|
+
* parks it in `__mj_integration_CustomOverflow`. Deactivating the map therefore
|
|
13
|
+
* REROUTES the value instead of excluding it, and the custom-column promoter will
|
|
14
|
+
* happily resurrect it as a real column later. Stripping before mapping keeps the
|
|
15
|
+
* key out of MappedFields, out of the overflow, and out of the content hash (whose
|
|
16
|
+
* basis is MappedFields), so an excluded field stops influencing change detection
|
|
17
|
+
* entirely.
|
|
18
|
+
*
|
|
19
|
+
* What this is for, concretely (measured on a live tenant, 2026-08-19):
|
|
20
|
+
* - `preferences` on Totara Users / Enrolled_Users: Moodle UI widget state
|
|
21
|
+
* (file-picker recents, user-selector toggles), identical nine keys on 100% of
|
|
22
|
+
* rows — ~2.5M expanded values of nothing.
|
|
23
|
+
* - `courseformatoptions` on Courses: course theming (header colours, tile icons).
|
|
24
|
+
* - `enrolledcourses` on Enrolled_Users: re-derives the Enrolled_Users table
|
|
25
|
+
* itself — ~7.6M expanded values that duplicate an object already synced.
|
|
26
|
+
*
|
|
27
|
+
* The directive lives in Configuration rather than a new column so that existing
|
|
28
|
+
* installs pick it up on their next schema sync with no DDL. Absence of a directive
|
|
29
|
+
* means Sync — connectors that never heard of this feature behave exactly as before.
|
|
30
|
+
*/
|
|
31
|
+
/** The Configuration key under which a field's sync directive is stored. */
|
|
32
|
+
export const SYNC_DIRECTIVE_CONFIG_KEY = 'syncDirective';
|
|
33
|
+
/**
|
|
34
|
+
* Reads the sync directive out of an IntegrationObjectField.Configuration JSON blob.
|
|
35
|
+
* Malformed JSON, a missing key, or an unrecognised value all mean 'Sync' — the
|
|
36
|
+
* directive can only ever narrow what is synced, never break a sync by its absence.
|
|
37
|
+
*/
|
|
38
|
+
export function ReadFieldSyncDirective(configuration) {
|
|
39
|
+
if (!configuration)
|
|
40
|
+
return 'Sync';
|
|
41
|
+
try {
|
|
42
|
+
const parsed = JSON.parse(configuration);
|
|
43
|
+
if (parsed && typeof parsed === 'object') {
|
|
44
|
+
const v = parsed[SYNC_DIRECTIVE_CONFIG_KEY];
|
|
45
|
+
if (typeof v === 'string' && v.toLowerCase() === 'exclude')
|
|
46
|
+
return 'Exclude';
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
// Malformed Configuration is not this feature's problem to surface.
|
|
51
|
+
}
|
|
52
|
+
return 'Sync';
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Merges a directive into an existing Configuration JSON string, preserving every
|
|
56
|
+
* other key. Returns the new JSON, or the original string when nothing changes
|
|
57
|
+
* (so callers can cheaply detect no-op writes). Setting 'Sync' REMOVES the key
|
|
58
|
+
* rather than storing the default.
|
|
59
|
+
*/
|
|
60
|
+
export function WriteFieldSyncDirective(configuration, directive) {
|
|
61
|
+
let obj = {};
|
|
62
|
+
if (configuration) {
|
|
63
|
+
try {
|
|
64
|
+
const parsed = JSON.parse(configuration);
|
|
65
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
66
|
+
obj = parsed;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
// Unparseable existing Configuration: preserve it untouched rather than
|
|
71
|
+
// destroy whatever it held. The directive is not worth data loss.
|
|
72
|
+
return configuration;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (directive === 'Exclude') {
|
|
76
|
+
obj[SYNC_DIRECTIVE_CONFIG_KEY] = 'Exclude';
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
delete obj[SYNC_DIRECTIVE_CONFIG_KEY];
|
|
80
|
+
}
|
|
81
|
+
const out = Object.keys(obj).length > 0 ? JSON.stringify(obj) : null;
|
|
82
|
+
return out;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* The set of source field names to strip for one integration object, derived from
|
|
86
|
+
* its IntegrationObjectField rows. Empty set when nothing is excluded — the
|
|
87
|
+
* mapping engine's fast path.
|
|
88
|
+
*/
|
|
89
|
+
export function ComputeExcludedSourceNames(fields) {
|
|
90
|
+
const out = new Set();
|
|
91
|
+
for (const f of fields) {
|
|
92
|
+
if (ReadFieldSyncDirective(f.Configuration) === 'Exclude') {
|
|
93
|
+
out.add(f.Name);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Returns a copy of `fields` without the excluded keys, or the ORIGINAL object when
|
|
100
|
+
* no key matches (so the no-exclusions path allocates nothing). Matching is exact:
|
|
101
|
+
* source field names are the connector's own spelling, already used verbatim as
|
|
102
|
+
* field-map SourceFieldName and overflow keys.
|
|
103
|
+
*/
|
|
104
|
+
export function StripExcludedFields(fields, excluded) {
|
|
105
|
+
if (excluded.size === 0)
|
|
106
|
+
return fields;
|
|
107
|
+
let hit = false;
|
|
108
|
+
for (const k of Object.keys(fields)) {
|
|
109
|
+
if (excluded.has(k)) {
|
|
110
|
+
hit = true;
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (!hit)
|
|
115
|
+
return fields;
|
|
116
|
+
const out = {};
|
|
117
|
+
for (const [k, v] of Object.entries(fields)) {
|
|
118
|
+
if (!excluded.has(k))
|
|
119
|
+
out[k] = v;
|
|
120
|
+
}
|
|
121
|
+
return out;
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=SyncDirectives.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SyncDirectives.js","sourceRoot":"","sources":["../src/SyncDirectives.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,4EAA4E;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,eAAe,CAAC;AAWzD;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,aAAwC;IAC3E,IAAI,CAAC,aAAa;QAAE,OAAO,MAAM,CAAC;IAClC,IAAI,CAAC;QACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAClD,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACvC,MAAM,CAAC,GAAI,MAAkC,CAAC,yBAAyB,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,SAAS;gBAAE,OAAO,SAAS,CAAC;QACjF,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACL,oEAAoE;IACxE,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CACnC,aAAwC,EACxC,SAA6B;IAE7B,IAAI,GAAG,GAA4B,EAAE,CAAC;IACtC,IAAI,aAAa,EAAE,CAAC;QAChB,IAAI,CAAC;YACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAClD,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjE,GAAG,GAAG,MAAiC,CAAC;YAC5C,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACL,wEAAwE;YACxE,kEAAkE;YAClE,OAAO,aAAa,CAAC;QACzB,CAAC;IACL,CAAC;IACD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC1B,GAAG,CAAC,yBAAyB,CAAC,GAAG,SAAS,CAAC;IAC/C,CAAC;SAAM,CAAC;QACJ,OAAO,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,OAAO,GAAG,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CAAC,MAA6C;IACpF,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACrB,IAAI,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,SAAS,EAAE,CAAC;YACxD,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAC/B,MAA+B,EAC/B,QAA6B;IAE7B,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IACvC,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAAC,GAAG,GAAG,IAAI,CAAC;YAAC,MAAM;QAAC,CAAC;IAC/C,CAAC;IACD,IAAI,CAAC,GAAG;QAAE,OAAO,MAAM,CAAC;IACxB,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC"}
|
package/dist/SyncLogger.d.ts
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* sync.connector.built — connector instance constructed (class + import path)
|
|
13
13
|
* sync.connector.test — TestConnection result (success + duration)
|
|
14
14
|
* sync.entity-map.start — per-IO sync starting (direction, watermark, externalObjectName)
|
|
15
|
+
* sync.entity-map.skipped — per-IO sync skipped without contacting the source (skippedReason)
|
|
15
16
|
* sync.fetch.batch.start — outbound API fetch about to fire (page/offset/cursor + URL hints)
|
|
16
17
|
* sync.fetch.batch.complete — fetch returned (record count, duration, hasMore)
|
|
17
18
|
* sync.record.decision — per-record action (Create/Update/Skip/Delete) + reason + matchKey
|
|
@@ -35,7 +36,7 @@
|
|
|
35
36
|
* tail -f /tmp/mjapi.log | grep '"event":"sync\.push\.' # only outbound writes
|
|
36
37
|
* tail -f /tmp/mjapi.log | grep '"event":"sync\..*\.error"' # only errors
|
|
37
38
|
*/
|
|
38
|
-
export type SyncLogEvent = 'sync.run.start' | 'sync.config.loaded' | 'sync.connector.built' | 'sync.connector.test' | 'sync.entity-map.start' | 'sync.resume.keyset' | 'sync.partition.reconcile' | 'sync.fetch.batch.start' | 'sync.fetch.batch.complete' | 'sync.fetch.retry' | 'sync.record.decision' | 'sync.record.saved' | 'sync.record.error' | 'sync.record.retry' | 'sync.record.archived' | 'sync.record.conflict' | 'sync.push.candidates' | 'sync.push.record' | 'sync.push.response' | 'sync.entity-map.complete' | 'sync.run.complete' | 'sync.run.fail' | 'sync.run.cancelled' | 'sync.schema_update' | 'sync.warning';
|
|
39
|
+
export type SyncLogEvent = 'sync.run.start' | 'sync.config.loaded' | 'sync.connector.built' | 'sync.connector.test' | 'sync.entity-map.start' | 'sync.entity-map.exclusions' | 'sync.entity-map.skipped' | 'sync.resume.keyset' | 'sync.partition.reconcile' | 'sync.fetch.batch.start' | 'sync.fetch.batch.complete' | 'sync.fetch.retry' | 'sync.record.decision' | 'sync.record.saved' | 'sync.record.error' | 'sync.record.retry' | 'sync.record.archived' | 'sync.record.conflict' | 'sync.push.candidates' | 'sync.push.record' | 'sync.push.response' | 'sync.entity-map.complete' | 'sync.run.complete' | 'sync.run.fail' | 'sync.run.cancelled' | 'sync.schema_update' | 'sync.warning';
|
|
39
40
|
import type { IntegrationProgressEmitter } from '@memberjunction/integration-progress-artifacts';
|
|
40
41
|
export interface SyncLoggerContext {
|
|
41
42
|
/** CompanyIntegration ID — the per-run anchor for filtering. */
|
package/dist/SyncLogger.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SyncLogger.d.ts","sourceRoot":"","sources":["../src/SyncLogger.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"SyncLogger.d.ts","sourceRoot":"","sources":["../src/SyncLogger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,MAAM,MAAM,YAAY,GAClB,gBAAgB,GAChB,oBAAoB,GACpB,sBAAsB,GACtB,qBAAqB,GACrB,uBAAuB,GACvB,4BAA4B,GAC5B,yBAAyB,GACzB,oBAAoB,GACpB,0BAA0B,GAC1B,wBAAwB,GACxB,2BAA2B,GAC3B,kBAAkB,GAClB,sBAAsB,GACtB,mBAAmB,GACnB,mBAAmB,GACnB,mBAAmB,GACnB,sBAAsB,GACtB,sBAAsB,GACtB,sBAAsB,GACtB,kBAAkB,GAClB,oBAAoB,GACpB,0BAA0B,GAC1B,mBAAmB,GACnB,eAAe,GACf,oBAAoB,GACpB,oBAAoB,GACpB,cAAc,CAAC;AAErB,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,gDAAgD,CAAC;AAEjG,MAAM,WAAW,iBAAiB;IAC9B,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,WAAW,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,iDAAiD;IACjD,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAkBD;;;;GAIG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;IACxC,OAAO,CAAC,OAAO,CAAC,CAA6B;gBAEjC,GAAG,EAAE,iBAAiB;IAIlC,6DAA6D;IACtD,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAIvC,6EAA6E;IACtE,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI;IAInE;;;;;;;OAOG;IACI,aAAa,CAAC,OAAO,EAAE,0BAA0B,GAAG,IAAI;IAI/D;;;;;OAKG;IACI,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI/E;;;;;;;OAOG;IACI,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3F,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GAAG,IAAI;IAuB1E;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAyDxB,OAAO,CAAC,UAAU;IAKlB,OAAO,CAAC,GAAG;IAIX,qFAAqF;IACrF,OAAO,CAAC,QAAQ;CAGnB"}
|
package/dist/SyncLogger.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* sync.connector.built — connector instance constructed (class + import path)
|
|
13
13
|
* sync.connector.test — TestConnection result (success + duration)
|
|
14
14
|
* sync.entity-map.start — per-IO sync starting (direction, watermark, externalObjectName)
|
|
15
|
+
* sync.entity-map.skipped — per-IO sync skipped without contacting the source (skippedReason)
|
|
15
16
|
* sync.fetch.batch.start — outbound API fetch about to fire (page/offset/cursor + URL hints)
|
|
16
17
|
* sync.fetch.batch.complete — fetch returned (record count, duration, hasMore)
|
|
17
18
|
* sync.record.decision — per-record action (Create/Update/Skip/Delete) + reason + matchKey
|
package/dist/SyncLogger.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SyncLogger.js","sourceRoot":"","sources":["../src/SyncLogger.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"SyncLogger.js","sourceRoot":"","sources":["../src/SyncLogger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAoDH;;;;;;GAMG;AACH,MAAM,qBAAqB,GAA8B,IAAI,GAAG,CAAe;IAC3E,sBAAsB;IACtB,mBAAmB;IACnB,sBAAsB;IACtB,kBAAkB;IAClB,oBAAoB;CACvB,CAAC,CAAC;AACH,MAAM,2BAA2B,GAAG,OAAO,CAAC,GAAG,CAAC,kCAAkC,KAAK,MAAM,CAAC;AAE9F;;;;GAIG;AACH,MAAM,OAAO,UAAU;IAInB,YAAY,GAAsB;QAC9B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACnB,CAAC;IAED,6DAA6D;IACtD,WAAW,CAAC,KAAa;QAC5B,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,6EAA6E;IACtE,qBAAqB,CAAC,IAA+B;QACxD,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC;IAChC,CAAC;IAED;;;;;;;OAOG;IACI,aAAa,CAAC,OAAmC;QACpD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACI,UAAU,CAAC,KAAa,EAAE,cAAuC;QACpE,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IACpD,CAAC;IAED;;;;;;;OAOG;IACI,OAAO,CAAC,KAAa,EAAE,IAAY,EAAE,OAAe,EAAE,IAA8B;QACvF,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC;IAEM,IAAI,CAAC,KAAmB,EAAE,OAAgC,EAAE;QAC/D,MAAM,KAAK,GAAiB;YACxB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,KAAK;YACL,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;YACnB,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI;YACzC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI;YAC7B,GAAG,IAAI;SACV,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,KAAK,KAAK,eAAe,IAAI,KAAK,KAAK,mBAAmB,EAAE,CAAC;YAC7D,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,KAAK,KAAK,cAAc,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,2BAA2B,EAAE,CAAC;YAC1E,oFAAoF;YACpF,oFAAoF;YACpF,wFAAwF;YACxF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACK,gBAAgB,CAAC,KAAmB,EAAE,IAA6B;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACpC,QAAQ,KAAK,EAAE,CAAC;gBACZ,KAAK,uBAAuB;oBACxB,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,KAAK,EAAE,CAAC,CAAC;oBAC9C,MAAM;gBACV,KAAK,2BAA2B;oBAC5B,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE;wBACnC,KAAK;wBACL,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE;wBACjD,IAAI;qBACP,CAAC,CAAC;oBACH,MAAM;gBACV,KAAK,mBAAmB;oBACpB,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE;wBACzB,KAAK;wBACL,KAAK,EAAE,OAAO;wBACd,OAAO,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc;wBACrE,IAAI;qBACP,CAAC,CAAC;oBACH,MAAM;gBACV,KAAK,sBAAsB;oBACvB,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;oBAClF,MAAM;gBACV,KAAK,cAAc;oBACf,OAAO,CAAC,OAAO,CACX,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EACnD,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EACrD,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EACpD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAClC,CAAC;oBACF,MAAM;gBACV,KAAK,0BAA0B;oBAC3B,OAAO,CAAC,aAAa,CAAC,KAAK,EAAE;wBACzB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC;wBAC1C,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC;wBACxE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC;wBACrC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC;qBACzC,CAAC,CAAC;oBACH,MAAM;gBACV,KAAK,oBAAoB;oBACrB,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;oBACxE,MAAM;gBACV;oBACI,kFAAkF;oBAClF,gFAAgF;oBAChF,2EAA2E;oBAC3E,MAAM;YACd,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACL,uEAAuE;QAC3E,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAA6B;QAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC;QACtF,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9D,CAAC;IAEO,GAAG,CAAC,CAAU;QAClB,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,qFAAqF;IAC7E,QAAQ,CAAC,CAAU;QACvB,OAAO,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,CAA6B,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5F,CAAC;CACJ"}
|
|
@@ -24,6 +24,14 @@ export declare class WatermarkService {
|
|
|
24
24
|
* @param contextUser - User context for data access
|
|
25
25
|
*/
|
|
26
26
|
Update(entityMapID: string, newValue: string, contextUser: UserInfo, direction?: 'Pull' | 'Push'): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Puts the watermark VALUE back to a prior state — the undo for a mid-run durability floor
|
|
29
|
+
* (IntegrationEngine §8a) after a page-skip gap makes that floor unsafe. Accepts null because
|
|
30
|
+
* the prior state may be "no watermark yet" (first run / the row was created BY the floor);
|
|
31
|
+
* a null value re-triggers a full window fetch on the next run, which is exactly the hold
|
|
32
|
+
* semantics the gap requires. A missing row means no floor was ever written — nothing to do.
|
|
33
|
+
*/
|
|
34
|
+
RestoreValue(entityMapID: string, value: string | null, contextUser: UserInfo, direction?: 'Pull' | 'Push'): Promise<void>;
|
|
27
35
|
/**
|
|
28
36
|
* Updates the progress fields on an existing watermark mid-sync without changing
|
|
29
37
|
* the WatermarkValue. Updates RecordsSynced to the cumulative total written so far
|
|
@@ -74,6 +82,13 @@ export declare class WatermarkService {
|
|
|
74
82
|
LoadPartitionRollups(entityMapID: string, contextUser: UserInfo): Promise<Map<string, string>>;
|
|
75
83
|
/**
|
|
76
84
|
* Updates an existing watermark record with a new value and timestamp.
|
|
85
|
+
*
|
|
86
|
+
* The type is restored, not just the value. This row is shared with the keyset resume position
|
|
87
|
+
* ({@link SaveKeysetPosition}), which flips it to `WatermarkType='Cursor'` mid-run. Creating a
|
|
88
|
+
* watermark stamps `'Timestamp'` but updating one used to leave the type alone, so a map that
|
|
89
|
+
* saved a cursor mid-run and then completed cleanly ended up holding a TIMESTAMP value typed as
|
|
90
|
+
* a CURSOR — and {@link Load}'s consumers read the type to decide what the value means, feeding
|
|
91
|
+
* it back to the connector as a seek key on the next run.
|
|
77
92
|
*/
|
|
78
93
|
private UpdateExistingWatermark;
|
|
79
94
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WatermarkService.d.ts","sourceRoot":"","sources":["../src/WatermarkService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,KAAK,EAAE,gCAAgC,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD;;;GAGG;AACH,qBAAa,gBAAgB;IACzB;;;;;;OAMG;IACU,IAAI,CACb,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,QAAQ,EACrB,SAAS,GAAE,MAAM,GAAG,MAAe,GACpC,OAAO,CAAC,gCAAgC,GAAG,IAAI,CAAC;IAsBnD;;;;;;;;OAQG;IACU,MAAM,CACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,QAAQ,EACrB,SAAS,GAAE,MAAM,GAAG,MAAe,GACpC,OAAO,CAAC,IAAI,CAAC;IAShB;;;;;;;;;OASG;IACU,cAAc,CACvB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAShB;;;;;;OAMG;IACI,iBAAiB,CAAC,cAAc,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO;IAkBvF;;;;;;;;OAQG;IACU,kBAAkB,CAC3B,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAehB;;;;;OAKG;IACU,mBAAmB,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3F;;;;;OAKG;IACU,oBAAoB,CAC7B,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC5B,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAchB;;;OAGG;IACU,oBAAoB,CAC7B,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAa/B
|
|
1
|
+
{"version":3,"file":"WatermarkService.d.ts","sourceRoot":"","sources":["../src/WatermarkService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,KAAK,EAAE,gCAAgC,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD;;;GAGG;AACH,qBAAa,gBAAgB;IACzB;;;;;;OAMG;IACU,IAAI,CACb,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,QAAQ,EACrB,SAAS,GAAE,MAAM,GAAG,MAAe,GACpC,OAAO,CAAC,gCAAgC,GAAG,IAAI,CAAC;IAsBnD;;;;;;;;OAQG;IACU,MAAM,CACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,QAAQ,EACrB,SAAS,GAAE,MAAM,GAAG,MAAe,GACpC,OAAO,CAAC,IAAI,CAAC;IAShB;;;;;;OAMG;IACU,YAAY,CACrB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,WAAW,EAAE,QAAQ,EACrB,SAAS,GAAE,MAAM,GAAG,MAAe,GACpC,OAAO,CAAC,IAAI,CAAC;IAWhB;;;;;;;;;OASG;IACU,cAAc,CACvB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAShB;;;;;;OAMG;IACI,iBAAiB,CAAC,cAAc,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO;IAkBvF;;;;;;;;OAQG;IACU,kBAAkB,CAC3B,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAehB;;;;;OAKG;IACU,mBAAmB,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3F;;;;;OAKG;IACU,oBAAoB,CAC7B,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC5B,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAchB;;;OAGG;IACU,oBAAoB,CAC7B,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAa/B;;;;;;;;;OASG;YACW,uBAAuB;IAarC;;;OAGG;YACW,kBAAkB;IAShC;;;OAGG;YACW,eAAe;CAsChC"}
|
package/dist/WatermarkService.js
CHANGED
|
@@ -50,6 +50,24 @@ export class WatermarkService {
|
|
|
50
50
|
await this.CreateNewWatermark(entityMapID, newValue, contextUser, direction);
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Puts the watermark VALUE back to a prior state — the undo for a mid-run durability floor
|
|
55
|
+
* (IntegrationEngine §8a) after a page-skip gap makes that floor unsafe. Accepts null because
|
|
56
|
+
* the prior state may be "no watermark yet" (first run / the row was created BY the floor);
|
|
57
|
+
* a null value re-triggers a full window fetch on the next run, which is exactly the hold
|
|
58
|
+
* semantics the gap requires. A missing row means no floor was ever written — nothing to do.
|
|
59
|
+
*/
|
|
60
|
+
async RestoreValue(entityMapID, value, contextUser, direction = 'Pull') {
|
|
61
|
+
const existing = await this.Load(entityMapID, contextUser, direction);
|
|
62
|
+
if (!existing)
|
|
63
|
+
return;
|
|
64
|
+
existing.WatermarkValue = value;
|
|
65
|
+
existing.LastSyncAt = new Date();
|
|
66
|
+
const saved = await existing.Save();
|
|
67
|
+
if (!saved) {
|
|
68
|
+
throw new Error(`Failed to restore watermark for EntityMapID=${entityMapID}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
53
71
|
/**
|
|
54
72
|
* Updates the progress fields on an existing watermark mid-sync without changing
|
|
55
73
|
* the WatermarkValue. Updates RecordsSynced to the cumulative total written so far
|
|
@@ -174,9 +192,17 @@ export class WatermarkService {
|
|
|
174
192
|
}
|
|
175
193
|
/**
|
|
176
194
|
* Updates an existing watermark record with a new value and timestamp.
|
|
195
|
+
*
|
|
196
|
+
* The type is restored, not just the value. This row is shared with the keyset resume position
|
|
197
|
+
* ({@link SaveKeysetPosition}), which flips it to `WatermarkType='Cursor'` mid-run. Creating a
|
|
198
|
+
* watermark stamps `'Timestamp'` but updating one used to leave the type alone, so a map that
|
|
199
|
+
* saved a cursor mid-run and then completed cleanly ended up holding a TIMESTAMP value typed as
|
|
200
|
+
* a CURSOR — and {@link Load}'s consumers read the type to decide what the value means, feeding
|
|
201
|
+
* it back to the connector as a seek key on the next run.
|
|
177
202
|
*/
|
|
178
203
|
async UpdateExistingWatermark(watermark, newValue) {
|
|
179
204
|
watermark.WatermarkValue = newValue;
|
|
205
|
+
watermark.WatermarkType = 'Timestamp';
|
|
180
206
|
watermark.LastSyncAt = new Date();
|
|
181
207
|
const saved = await watermark.Save();
|
|
182
208
|
if (!saved) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WatermarkService.js","sourceRoot":"","sources":["../src/WatermarkService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAiB,MAAM,sBAAsB,CAAC;AAKxE;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IACzB;;;;;;OAMG;IACI,KAAK,CAAC,IAAI,CACb,WAAmB,EACnB,WAAqB,EACrB,YAA6B,MAAM;QAEnC,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAmC;YAC9D,UAAU,EAAE,yCAAyC;YACrD,WAAW,EAAE,gBAAgB,WAAW,oBAAoB,SAAS,GAAG;YACxE,OAAO,EAAE,iBAAiB;YAC1B,OAAO,EAAE,CAAC;YACV,UAAU,EAAE,eAAe;YAC3B,8FAA8F;YAC9F,uFAAuF;YACvF,gGAAgG;YAChG,kGAAkG;YAClG,mGAAmG;YACnG,kGAAkG;YAClG,wEAAwE;YACxE,WAAW,EAAE,IAAI;SACpB,EAAE,WAAW,CAAC,CAAC;QAEhB,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAChE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED;;;;;;;;OAQG;IACI,KAAK,CAAC,MAAM,CACf,WAAmB,EACnB,QAAgB,EAChB,WAAqB,EACrB,YAA6B,MAAM;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACtE,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC3D,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACjF,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,cAAc,CACvB,WAAmB,EACnB,YAAoB,EACpB,WAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAC3D,IAAI,CAAC,QAAQ;YAAE,OAAO;QAEtB,QAAQ,CAAC,aAAa,GAAG,YAAY,CAAC;QACtC,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QACjC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACI,iBAAiB,CAAC,cAAsB,EAAE,aAA4B;QACzE,IAAI,CAAC,cAAc,IAAI,cAAc,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,QAAQ,aAAa,EAAE,CAAC;YACpB,KAAK,WAAW;gBACZ,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAC9C,KAAK,SAAS;gBACV,OAAO,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACxC,KAAK,QAAQ,CAAC;YACd,KAAK,aAAa;gBACd,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;YACrC;gBACI,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;QACzC,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACI,KAAK,CAAC,kBAAkB,CAC3B,WAAmB,EACnB,QAAgB,EAChB,WAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,aAAa,GAAG,QAAQ,CAAC;YAClC,QAAQ,CAAC,cAAc,GAAG,QAAQ,CAAC;YACnC,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,kDAAkD,WAAW,EAAE,CAAC,CAAC;YACrF,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QACrF,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,mBAAmB,CAAC,WAAmB,EAAE,WAAqB;QACvE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,QAAQ,CAAC,aAAa,GAAG,QAAQ,CAAC;QAClC,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;QAC/B,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,EAAE,CAAC,CAAC;QACtF,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,oBAAoB,CAC7B,WAAmB,EACnB,OAA4B,EAC5B,WAAqB;QAErB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,aAAa,GAAG,aAAa,CAAC;YACvC,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;YAC/B,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,WAAW,EAAE,CAAC,CAAC;QACnG,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QACtF,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,oBAAoB,CAC7B,WAAmB,EACnB,WAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,QAAQ,EAAE,aAAa,KAAK,aAAa,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;YACvE,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAA2B,CAAC;gBAC1E,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;oBAAE,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5E,CAAC;YAAC,MAAM,CAAC;gBACL,0FAA0F;YAC9F,CAAC;QACL,CAAC;QACD,OAAO,IAAI,GAAG,EAAE,CAAC;IACrB,CAAC;IAED
|
|
1
|
+
{"version":3,"file":"WatermarkService.js","sourceRoot":"","sources":["../src/WatermarkService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAiB,MAAM,sBAAsB,CAAC;AAKxE;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IACzB;;;;;;OAMG;IACI,KAAK,CAAC,IAAI,CACb,WAAmB,EACnB,WAAqB,EACrB,YAA6B,MAAM;QAEnC,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAmC;YAC9D,UAAU,EAAE,yCAAyC;YACrD,WAAW,EAAE,gBAAgB,WAAW,oBAAoB,SAAS,GAAG;YACxE,OAAO,EAAE,iBAAiB;YAC1B,OAAO,EAAE,CAAC;YACV,UAAU,EAAE,eAAe;YAC3B,8FAA8F;YAC9F,uFAAuF;YACvF,gGAAgG;YAChG,kGAAkG;YAClG,mGAAmG;YACnG,kGAAkG;YAClG,wEAAwE;YACxE,WAAW,EAAE,IAAI;SACpB,EAAE,WAAW,CAAC,CAAC;QAEhB,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAChE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED;;;;;;;;OAQG;IACI,KAAK,CAAC,MAAM,CACf,WAAmB,EACnB,QAAgB,EAChB,WAAqB,EACrB,YAA6B,MAAM;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACtE,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC3D,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACjF,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,YAAY,CACrB,WAAmB,EACnB,KAAoB,EACpB,WAAqB,EACrB,YAA6B,MAAM;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACtE,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,QAAQ,CAAC,cAAc,GAAG,KAAK,CAAC;QAChC,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,+CAA+C,WAAW,EAAE,CAAC,CAAC;QAClF,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,cAAc,CACvB,WAAmB,EACnB,YAAoB,EACpB,WAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAC3D,IAAI,CAAC,QAAQ;YAAE,OAAO;QAEtB,QAAQ,CAAC,aAAa,GAAG,YAAY,CAAC;QACtC,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QACjC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACI,iBAAiB,CAAC,cAAsB,EAAE,aAA4B;QACzE,IAAI,CAAC,cAAc,IAAI,cAAc,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,QAAQ,aAAa,EAAE,CAAC;YACpB,KAAK,WAAW;gBACZ,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAC9C,KAAK,SAAS;gBACV,OAAO,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACxC,KAAK,QAAQ,CAAC;YACd,KAAK,aAAa;gBACd,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;YACrC;gBACI,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;QACzC,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACI,KAAK,CAAC,kBAAkB,CAC3B,WAAmB,EACnB,QAAgB,EAChB,WAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,aAAa,GAAG,QAAQ,CAAC;YAClC,QAAQ,CAAC,cAAc,GAAG,QAAQ,CAAC;YACnC,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,kDAAkD,WAAW,EAAE,CAAC,CAAC;YACrF,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QACrF,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,mBAAmB,CAAC,WAAmB,EAAE,WAAqB;QACvE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,QAAQ,CAAC,aAAa,GAAG,QAAQ,CAAC;QAClC,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;QAC/B,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,EAAE,CAAC,CAAC;QACtF,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,oBAAoB,CAC7B,WAAmB,EACnB,OAA4B,EAC5B,WAAqB;QAErB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,aAAa,GAAG,aAAa,CAAC;YACvC,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;YAC/B,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,WAAW,EAAE,CAAC,CAAC;QACnG,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QACtF,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,oBAAoB,CAC7B,WAAmB,EACnB,WAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,QAAQ,EAAE,aAAa,KAAK,aAAa,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;YACvE,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAA2B,CAAC;gBAC1E,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;oBAAE,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5E,CAAC;YAAC,MAAM,CAAC;gBACL,0FAA0F;YAC9F,CAAC;QACL,CAAC;QACD,OAAO,IAAI,GAAG,EAAE,CAAC;IACrB,CAAC;IAED;;;;;;;;;OASG;IACK,KAAK,CAAC,uBAAuB,CACjC,SAA2C,EAC3C,QAAgB;QAEhB,SAAS,CAAC,cAAc,GAAG,QAAQ,CAAC;QACpC,SAAS,CAAC,aAAa,GAAG,WAAW,CAAC;QACtC,SAAS,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,8CAA8C,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3F,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB,CAC5B,WAAmB,EACnB,QAAgB,EAChB,WAAqB,EACrB,YAA6B,MAAM;QAEnC,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAC3F,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,eAAe,CACzB,WAAmB,EACnB,KAAoB,EACpB,aAA4B,EAC5B,SAA0B,EAC1B,WAAqB;QAErB,MAAM,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC,CAAE,kEAAkE;QAC9F,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,eAAe,CACtC,yCAAyC,EACzC,WAAW,CACd,CAAC;QACF,SAAS,CAAC,SAAS,EAAE,CAAC;QACtB,SAAS,CAAC,WAAW,GAAG,WAAW,CAAC;QACpC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;QACjC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC;QAChC,SAAS,CAAC,aAAa,GAAG,aAAa,CAAC;QACxC,SAAS,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAClC,SAAS,CAAC,aAAa,GAAG,CAAC,CAAC;QAE5B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,gGAAgG;YAChG,gGAAgG;YAChG,mGAAmG;YACnG,+FAA+F;YAC/F,+EAA+E;YAC/E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;YACtE,IAAI,QAAQ,EAAE,CAAC;gBACX,QAAQ,CAAC,cAAc,GAAG,KAAK,CAAC;gBAChC,QAAQ,CAAC,aAAa,GAAG,aAAa,CAAC;gBACvC,QAAQ,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;gBACjC,IAAI,MAAM,QAAQ,CAAC,IAAI,EAAE;oBAAE,OAAO;YACtC,CAAC;YACD,MAAM,GAAG,GAAG,SAAS,CAAC,YAAY,EAAE,OAAO,IAAI,eAAe,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,8CAA8C,WAAW,KAAK,GAAG,EAAE,CAAC,CAAC;QACzF,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Orders the engine's writes against one provider.
|
|
3
|
+
*
|
|
4
|
+
* The provider holds a SINGLE transaction bound to a single connection, so any write issued while
|
|
5
|
+
* that transaction is open joins it. That is why every engine write was funnelled through one
|
|
6
|
+
* provider-wide chain: it made it impossible for a batch's transaction to swallow an unrelated
|
|
7
|
+
* entity map's watermark save.
|
|
8
|
+
*
|
|
9
|
+
* The cost was that writes belonging to DIFFERENT entity maps also queued behind each other, even
|
|
10
|
+
* when no transaction was involved at all — which is the common case once a connection uses batched
|
|
11
|
+
* writes, since those carry their own TransactionGroup and never open the provider's.
|
|
12
|
+
*
|
|
13
|
+
* So this is a two-mode lock rather than a single queue:
|
|
14
|
+
*
|
|
15
|
+
* - {@link RunExclusive} is for work that opens the provider transaction. Nothing else writes
|
|
16
|
+
* while it runs, and it waits for everything already in flight.
|
|
17
|
+
* - {@link RunKeyed} is for work scoped to one entity map that opens no transaction. Different
|
|
18
|
+
* keys overlap; the SAME key stays ordered, so one map never races its own writes.
|
|
19
|
+
*
|
|
20
|
+
* DEADLOCK SAFETY: each call waits only on work that existed BEFORE it. An exclusive section
|
|
21
|
+
* snapshots the in-flight keyed work at call time; a keyed call captures the barrier at call time.
|
|
22
|
+
* Neither ever waits on something created after it, so the wait graph cannot contain a cycle.
|
|
23
|
+
*
|
|
24
|
+
* A rejected operation never breaks ordering for anyone else: every chain continues past failures,
|
|
25
|
+
* so one errored batch cannot deadlock later writers.
|
|
26
|
+
*/
|
|
27
|
+
export declare class WriteSerializer {
|
|
28
|
+
/** Resolves when the most recently queued exclusive section has finished. */
|
|
29
|
+
private barrier;
|
|
30
|
+
/** Tail of in-flight work per key, so same-key calls stay ordered. */
|
|
31
|
+
private readonly keyed;
|
|
32
|
+
/** Keyed work not yet settled — what an exclusive section must wait out. */
|
|
33
|
+
private readonly active;
|
|
34
|
+
/** Runs `fn` with no other write in flight against this provider. */
|
|
35
|
+
RunExclusive<T>(fn: () => Promise<T>): Promise<T>;
|
|
36
|
+
/**
|
|
37
|
+
* Runs `fn` ordered against other work for the same `key`, concurrently with other keys, and
|
|
38
|
+
* never overlapping an exclusive section queued before it.
|
|
39
|
+
*/
|
|
40
|
+
RunKeyed<T>(key: string, fn: () => Promise<T>): Promise<T>;
|
|
41
|
+
/** Keys currently tracked — for tests and diagnostics only. */
|
|
42
|
+
get TrackedKeyCount(): number;
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=WriteSerializer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"WriteSerializer.d.ts","sourceRoot":"","sources":["../src/WriteSerializer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,qBAAa,eAAe;IACxB,6EAA6E;IAC7E,OAAO,CAAC,OAAO,CAAuC;IACtD,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuC;IAC7D,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA+B;IAEtD,qEAAqE;IAC9D,YAAY,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IASxD;;;OAGG;IACI,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAgBjE,+DAA+D;IAC/D,IAAW,eAAe,IAAI,MAAM,CAEnC;CACJ"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Orders the engine's writes against one provider.
|
|
3
|
+
*
|
|
4
|
+
* The provider holds a SINGLE transaction bound to a single connection, so any write issued while
|
|
5
|
+
* that transaction is open joins it. That is why every engine write was funnelled through one
|
|
6
|
+
* provider-wide chain: it made it impossible for a batch's transaction to swallow an unrelated
|
|
7
|
+
* entity map's watermark save.
|
|
8
|
+
*
|
|
9
|
+
* The cost was that writes belonging to DIFFERENT entity maps also queued behind each other, even
|
|
10
|
+
* when no transaction was involved at all — which is the common case once a connection uses batched
|
|
11
|
+
* writes, since those carry their own TransactionGroup and never open the provider's.
|
|
12
|
+
*
|
|
13
|
+
* So this is a two-mode lock rather than a single queue:
|
|
14
|
+
*
|
|
15
|
+
* - {@link RunExclusive} is for work that opens the provider transaction. Nothing else writes
|
|
16
|
+
* while it runs, and it waits for everything already in flight.
|
|
17
|
+
* - {@link RunKeyed} is for work scoped to one entity map that opens no transaction. Different
|
|
18
|
+
* keys overlap; the SAME key stays ordered, so one map never races its own writes.
|
|
19
|
+
*
|
|
20
|
+
* DEADLOCK SAFETY: each call waits only on work that existed BEFORE it. An exclusive section
|
|
21
|
+
* snapshots the in-flight keyed work at call time; a keyed call captures the barrier at call time.
|
|
22
|
+
* Neither ever waits on something created after it, so the wait graph cannot contain a cycle.
|
|
23
|
+
*
|
|
24
|
+
* A rejected operation never breaks ordering for anyone else: every chain continues past failures,
|
|
25
|
+
* so one errored batch cannot deadlock later writers.
|
|
26
|
+
*/
|
|
27
|
+
export class WriteSerializer {
|
|
28
|
+
constructor() {
|
|
29
|
+
/** Resolves when the most recently queued exclusive section has finished. */
|
|
30
|
+
this.barrier = Promise.resolve();
|
|
31
|
+
/** Tail of in-flight work per key, so same-key calls stay ordered. */
|
|
32
|
+
this.keyed = new Map();
|
|
33
|
+
/** Keyed work not yet settled — what an exclusive section must wait out. */
|
|
34
|
+
this.active = new Set();
|
|
35
|
+
}
|
|
36
|
+
/** Runs `fn` with no other write in flight against this provider. */
|
|
37
|
+
RunExclusive(fn) {
|
|
38
|
+
// Snapshot: everything queued before this call. Work queued AFTER waits on our barrier
|
|
39
|
+
// instead, which is what keeps the wait graph acyclic.
|
|
40
|
+
const gate = Promise.allSettled([this.barrier, ...this.active]);
|
|
41
|
+
const run = gate.then(() => fn());
|
|
42
|
+
this.barrier = run.then(() => undefined, () => undefined);
|
|
43
|
+
return run;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Runs `fn` ordered against other work for the same `key`, concurrently with other keys, and
|
|
47
|
+
* never overlapping an exclusive section queued before it.
|
|
48
|
+
*/
|
|
49
|
+
RunKeyed(key, fn) {
|
|
50
|
+
const priorSameKey = this.keyed.get(key) ?? Promise.resolve();
|
|
51
|
+
const gate = Promise.allSettled([this.barrier, priorSameKey]);
|
|
52
|
+
const run = gate.then(() => fn());
|
|
53
|
+
const settled = run.then(() => undefined, () => undefined);
|
|
54
|
+
this.keyed.set(key, settled);
|
|
55
|
+
this.active.add(settled);
|
|
56
|
+
void settled.then(() => {
|
|
57
|
+
this.active.delete(settled);
|
|
58
|
+
// Drop the key only if nothing newer claimed it, so a long run over many entity maps
|
|
59
|
+
// does not grow this map without bound.
|
|
60
|
+
if (this.keyed.get(key) === settled)
|
|
61
|
+
this.keyed.delete(key);
|
|
62
|
+
});
|
|
63
|
+
return run;
|
|
64
|
+
}
|
|
65
|
+
/** Keys currently tracked — for tests and diagnostics only. */
|
|
66
|
+
get TrackedKeyCount() {
|
|
67
|
+
return this.keyed.size;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=WriteSerializer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"WriteSerializer.js","sourceRoot":"","sources":["../src/WriteSerializer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,OAAO,eAAe;IAA5B;QACI,6EAA6E;QACrE,YAAO,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;QACtD,sEAAsE;QACrD,UAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;QAC7D,4EAA4E;QAC3D,WAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAoC1D,CAAC;IAlCG,qEAAqE;IAC9D,YAAY,CAAI,EAAoB;QACvC,uFAAuF;QACvF,uDAAuD;QACvD,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAChE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;;OAGG;IACI,QAAQ,CAAI,GAAW,EAAE,EAAoB;QAChD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAC9D,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACzB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YACnB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC5B,qFAAqF;YACrF,wCAAwC;YACxC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,OAAO;gBAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QACH,OAAO,GAAG,CAAC;IACf,CAAC;IAED,+DAA+D;IAC/D,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IAC3B,CAAC;CACJ"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export type { IIntegrationSourceType, ICompanyIntegrationEntityMap, ICompanyIntegrationFieldMap, ICompanyIntegrationSyncWatermark } from './entity-types.js';
|
|
2
2
|
export type { SyncDirection, SyncTriggerType, WatermarkType, ConflictResolution, DeleteBehavior, RecordChangeType, IntegrationRunStatus, ExternalRecord, MappedRecord, SyncResult, SyncRecordError, DefaultFieldMapping, SyncErrorCode, ErrorSeverity, SyncProgress, SyncProgressSnapshot, OnProgressCallback, SyncNotificationEvent, SyncNotificationSeverity, SyncNotification, OnNotificationCallback, IntegrationSyncOptions, EntityMapSyncResult, SourceSchemaInfo, SourceObjectInfo, SourceFieldInfo, SourceRelationshipInfo, IntrospectSchemaOptions, CRUDContext, CreateRecordContext, UpdateRecordContext, UpsertRecordContext, DeleteRecordContext, GetRecordContext, CRUDResult, SearchContext, SearchResult, ListContext, ListResult, SchemaPromotionResult, PostSyncSchemaPromotionCallback, CustomKeyStat, } from './types.js';
|
|
3
3
|
export { IsRetryableError, ClassifyError } from './types.js';
|
|
4
|
+
export { ExtractRetryAfterFromError, ParseRetryAfterValue, MAX_HONORED_RETRY_AFTER_MS } from './RetryAfter.js';
|
|
4
5
|
export type { TransformType, TransformOnError, TransformStep, TransformConfig, DirectConfig, RegexConfig, SplitConfig, CombineConfig, LookupConfig, FormatConfig, CoerceConfig, SubstringConfig, CustomConfig, } from './transforms.js';
|
|
5
6
|
export { BaseIntegrationConnector, WithTimeout, OperationTimeoutError, DEFAULT_OPERATION_TIMEOUTS } from './BaseIntegrationConnector.js';
|
|
6
7
|
export type { ConnectionTestResult, ExternalObjectSchema, ExternalFieldSchema, FetchContext, FetchBatchResult, OperationTimeouts, DefaultIntegrationConfig, DefaultObjectConfig, RateLimitPolicy, } from './BaseIntegrationConnector.js';
|
|
@@ -17,14 +18,20 @@ export { ActionMetadataGenerator } from './ActionMetadataGenerator.js';
|
|
|
17
18
|
export type { IntegrationObjectInfo, IntegrationFieldInfo, ActionGeneratorConfig, GeneratedActionMetadata } from './ActionMetadataGenerator.js';
|
|
18
19
|
export { IntegrationActionGenerator } from './IntegrationActionGenerator.js';
|
|
19
20
|
export type { IntegrationActionVerb, GenerateIntegrationActionResult } from './IntegrationActionGenerator.js';
|
|
20
|
-
export { IntegrationEngine } from './IntegrationEngine.js';
|
|
21
|
+
export { IntegrationEngine, ObjectUnavailableError, IsObjectUnavailable } from './IntegrationEngine.js';
|
|
21
22
|
export { RunOwnershipService, RunOwnershipLostError } from './RunOwnershipService.js';
|
|
22
23
|
export type { RenewResult, BoundaryCheckResult, HeartbeatOptions, TerminalRunStatus } from './RunOwnershipService.js';
|
|
23
24
|
export { IntegrationSchemaSync } from './IntegrationSchemaSync.js';
|
|
24
25
|
export type { PersistSchemaOptions, PersistSchemaResult } from './IntegrationSchemaSync.js';
|
|
25
26
|
export { computeContentHash, CONTENT_HASH_COLUMN } from './ContentHash.js';
|
|
27
|
+
export { CollapseDuplicateIdentities } from './BatchIdentity.js';
|
|
26
28
|
export { CUSTOM_OVERFLOW_COLUMN, computeUnmappedFields, hasUnmappedFields, foldCustomKeyStats, CUSTOM_KEY_SAMPLE_CAP } from './CustomOverflow.js';
|
|
29
|
+
export { SYNC_DIRECTIVE_CONFIG_KEY, ReadFieldSyncDirective, WriteFieldSyncDirective, ComputeExcludedSourceNames, StripExcludedFields } from './SyncDirectives.js';
|
|
30
|
+
export type { FieldSyncDirective, FieldWithConfiguration } from './SyncDirectives.js';
|
|
31
|
+
export { FindUnbindableFieldMaps, DescribeUnbindableFieldMaps } from './FieldMapValidation.js';
|
|
32
|
+
export type { ValidatableFieldMap, UnbindableFieldMap } from './FieldMapValidation.js';
|
|
27
33
|
export type { CustomKeyAccumulator } from './CustomOverflow.js';
|
|
34
|
+
export type { HasExternalID, BatchIdentityResult } from './BatchIdentity.js';
|
|
28
35
|
export { planPromotions, planColumnReclamations, inferColumnTypeFromSamples, inferColumnTypeFromStats, buildOverflowStats, sanitizeColumnName } from './CustomColumnPromotion.js';
|
|
29
36
|
export { discoverFromStream, pickPrimaryKeyFromStats } from './StreamingDiscovery.js';
|
|
30
37
|
export type { StreamDiscoveryOptions, DiscoveredColumnStat, StreamDiscoveryResult, PkPickOptions, PkStatVerdict } from './StreamingDiscovery.js';
|
|
@@ -39,6 +46,8 @@ export { RateLimiter } from './RateLimiter.js';
|
|
|
39
46
|
export type { RateLimiterOptions, RateLimiterKeyState, NowFn, SleepFn } from './RateLimiter.js';
|
|
40
47
|
export { IntegrationConnectorCreationPipeline } from './IntegrationConnectorCreationPipeline.js';
|
|
41
48
|
export type { ConnectorCreationPipelineOptions, ConnectorCreationPipelineResult } from './IntegrationConnectorCreationPipeline.js';
|
|
49
|
+
export { DiscoveryWatchdog, ResolveWatchdogIntervalMs } from './DiscoveryWatchdog.js';
|
|
50
|
+
export type { DiscoveryWatchEntry, DiscoveryWatchPatch, DiscoveryWatchdogOptions } from './DiscoveryWatchdog.js';
|
|
42
51
|
export { AdaptiveConcurrencyController, RunAdaptive } from './AdaptiveConcurrency.js';
|
|
43
52
|
export type { AdaptiveConcurrencyOptions, AdaptiveItemOutcome, AdaptiveRunResult } from './AdaptiveConcurrency.js';
|
|
44
53
|
export type { FetchWarning } from './BaseIntegrationConnector.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EAAE,sBAAsB,EAAE,4BAA4B,EAAE,2BAA2B,EAAE,gCAAgC,EAAE,MAAM,mBAAmB,CAAC;AAG7J,YAAY,EACV,aAAa,EACb,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,cAAc,EACd,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,YAAY,EACZ,UAAU,EACV,eAAe,EACf,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,YAAY,EACZ,oBAAoB,EACpB,kBAAkB,EAClB,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,sBAAsB,EACtB,uBAAuB,EACvB,WAAW,EACX,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,UAAU,EACV,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,qBAAqB,EACrB,+BAA+B,EAC/B,aAAa,GACd,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG7D,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,WAAW,EACX,WAAW,EACX,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,YAAY,GACb,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AACzI,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,wBAAwB,EACxB,mBAAmB,EACnB,eAAe,GAChB,MAAM,+BAA+B,CAAC;AAGvC,OAAO,EAAE,4BAA4B,EAAE,MAAM,mCAAmC,CAAC;AACjF,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAGxH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAGzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAG1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAGzD,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACnE,YAAY,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,YAAY,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAGhJ,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAC7E,YAAY,EAAE,qBAAqB,EAAE,+BAA+B,EAAE,MAAM,iCAAiC,CAAC;AAI9G,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EAAE,sBAAsB,EAAE,4BAA4B,EAAE,2BAA2B,EAAE,gCAAgC,EAAE,MAAM,mBAAmB,CAAC;AAG7J,YAAY,EACV,aAAa,EACb,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,cAAc,EACd,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,YAAY,EACZ,UAAU,EACV,eAAe,EACf,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,YAAY,EACZ,oBAAoB,EACpB,kBAAkB,EAClB,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,sBAAsB,EACtB,uBAAuB,EACvB,WAAW,EACX,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,UAAU,EACV,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,qBAAqB,EACrB,+BAA+B,EAC/B,aAAa,GACd,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG7D,OAAO,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAG/G,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,WAAW,EACX,WAAW,EACX,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,YAAY,GACb,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AACzI,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,wBAAwB,EACxB,mBAAmB,EACnB,eAAe,GAChB,MAAM,+BAA+B,CAAC;AAGvC,OAAO,EAAE,4BAA4B,EAAE,MAAM,mCAAmC,CAAC;AACjF,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAGxH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAGzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAG1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAGzD,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACnE,YAAY,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,YAAY,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAGhJ,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAC7E,YAAY,EAAE,qBAAqB,EAAE,+BAA+B,EAAE,MAAM,iCAAiC,CAAC;AAI9G,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAExG,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAItF,YAAY,EAAE,WAAW,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAGtH,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,YAAY,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAG5F,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAClJ,OAAO,EAAE,yBAAyB,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAClK,YAAY,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AACtF,OAAO,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAC;AAC/F,YAAY,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AACvF,YAAY,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChE,YAAY,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAClL,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACtF,YAAY,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACjJ,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC5L,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAC/G,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACxE,YAAY,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC5F,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAChG,OAAO,EAAE,oCAAoC,EAAE,MAAM,2CAA2C,CAAC;AACjG,YAAY,EAAE,gCAAgC,EAAE,+BAA+B,EAAE,MAAM,2CAA2C,CAAC;AACnI,OAAO,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACtF,YAAY,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AACjH,OAAO,EAAE,6BAA6B,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACtF,YAAY,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AACnH,YAAY,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAClE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AACzE,YAAY,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAGnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,YAAY,EAAE,eAAe,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC9E,YAAY,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AAC1F,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC1F,YAAY,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
// Error classification helpers
|
|
2
2
|
export { IsRetryableError, ClassifyError } from './types.js';
|
|
3
|
+
// Standard Retry-After extraction — exported so a connector overriding ExtractRetryAfterMs for a
|
|
4
|
+
// non-standard vendor signal can still fall back to the header when the vendor sends one.
|
|
5
|
+
export { ExtractRetryAfterFromError, ParseRetryAfterValue, MAX_HONORED_RETRY_AFTER_MS } from './RetryAfter.js';
|
|
3
6
|
// Connector
|
|
4
7
|
export { BaseIntegrationConnector, WithTimeout, OperationTimeoutError, DEFAULT_OPERATION_TIMEOUTS } from './BaseIntegrationConnector.js';
|
|
5
8
|
// REST Connector Base
|
|
@@ -20,13 +23,16 @@ export { ActionMetadataGenerator } from './ActionMetadataGenerator.js';
|
|
|
20
23
|
export { IntegrationActionGenerator } from './IntegrationActionGenerator.js';
|
|
21
24
|
// Integration Engine (server-side, wraps IntegrationEngineBase via composition)
|
|
22
25
|
// NOTE: For IntegrationEngineBase (client-safe metadata), import from @memberjunction/integration-engine-base
|
|
23
|
-
export { IntegrationEngine } from './IntegrationEngine.js';
|
|
26
|
+
export { IntegrationEngine, ObjectUnavailableError, IsObjectUnavailable } from './IntegrationEngine.js';
|
|
24
27
|
export { RunOwnershipService, RunOwnershipLostError } from './RunOwnershipService.js';
|
|
25
28
|
// Schema persistence — upserts dynamically discovered objects/fields to IntegrationObject/Field tables
|
|
26
29
|
export { IntegrationSchemaSync } from './IntegrationSchemaSync.js';
|
|
27
30
|
// ── Restored module exports dropped by the origin/next index.ts merge (union) ──
|
|
28
31
|
export { computeContentHash, CONTENT_HASH_COLUMN } from './ContentHash.js';
|
|
32
|
+
export { CollapseDuplicateIdentities } from './BatchIdentity.js';
|
|
29
33
|
export { CUSTOM_OVERFLOW_COLUMN, computeUnmappedFields, hasUnmappedFields, foldCustomKeyStats, CUSTOM_KEY_SAMPLE_CAP } from './CustomOverflow.js';
|
|
34
|
+
export { SYNC_DIRECTIVE_CONFIG_KEY, ReadFieldSyncDirective, WriteFieldSyncDirective, ComputeExcludedSourceNames, StripExcludedFields } from './SyncDirectives.js';
|
|
35
|
+
export { FindUnbindableFieldMaps, DescribeUnbindableFieldMaps } from './FieldMapValidation.js';
|
|
30
36
|
export { planPromotions, planColumnReclamations, inferColumnTypeFromSamples, inferColumnTypeFromStats, buildOverflowStats, sanitizeColumnName } from './CustomColumnPromotion.js';
|
|
31
37
|
export { discoverFromStream, pickPrimaryKeyFromStats } from './StreamingDiscovery.js';
|
|
32
38
|
export { partitionRecords, partitionRollupHash, diffPartitions, partitionKeyForIdentity } from './HashDiff.js';
|
|
@@ -34,6 +40,7 @@ export { mostRecentWinner, parseTimestamp } from './ConflictRecency.js';
|
|
|
34
40
|
export { EnrichSchemaConstraints } from './EnrichSchemaConstraints.js';
|
|
35
41
|
export { RateLimiter } from './RateLimiter.js';
|
|
36
42
|
export { IntegrationConnectorCreationPipeline } from './IntegrationConnectorCreationPipeline.js';
|
|
43
|
+
export { DiscoveryWatchdog, ResolveWatchdogIntervalMs } from './DiscoveryWatchdog.js';
|
|
37
44
|
export { AdaptiveConcurrencyController, RunAdaptive } from './AdaptiveConcurrency.js';
|
|
38
45
|
export { decideSchemaLimitViolations } from './IntegrationSchemaSync.js';
|
|
39
46
|
// ── Auth helpers (shared OAuth2/token primitives for connectors) ──
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAiDA,+BAA+B;AAC/B,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAiDA,+BAA+B;AAC/B,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC7D,iGAAiG;AACjG,0FAA0F;AAC1F,OAAO,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAmB/G,YAAY;AACZ,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAazI,sBAAsB;AACtB,OAAO,EAAE,4BAA4B,EAAE,MAAM,mCAAmC,CAAC;AAGjF,UAAU;AACV,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,UAAU;AACV,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE1D,WAAW;AACX,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,QAAQ;AACR,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAGnE,4BAA4B;AAC5B,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAGvE,qHAAqH;AACrH,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAG7E,gFAAgF;AAChF,8GAA8G;AAC9G,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAExG,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAMtF,uGAAuG;AACvG,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AAGnE,kFAAkF;AAClF,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAClJ,OAAO,EAAE,yBAAyB,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAElK,OAAO,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAC;AAI/F,OAAO,EAAE,cAAc,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAClL,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAGtF,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAE/G,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,oCAAoC,EAAE,MAAM,2CAA2C,CAAC;AAEjG,OAAO,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAEtF,OAAO,EAAE,6BAA6B,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAItF,OAAO,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAGzE,qEAAqE;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAE9E,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -14,7 +14,7 @@ export type RecordChangeType = 'Create' | 'Update' | 'Delete' | 'Skip';
|
|
|
14
14
|
/** Status of a running integration sync */
|
|
15
15
|
export type IntegrationRunStatus = 'Pending' | 'In Progress' | 'Success' | 'Failed';
|
|
16
16
|
/** Classification of sync errors for retry/alert decisions */
|
|
17
|
-
export type SyncErrorCode = 'VALIDATION_ERROR' | 'FK_CONSTRAINT_VIOLATION' | 'DUPLICATE_KEY' | 'NETWORK_TIMEOUT' | 'RATE_LIMIT_EXCEEDED' | 'CONNECTOR_ERROR' | 'TRANSFORM_ERROR' | 'MATCH_RESOLUTION_ERROR' | 'DATABASE_ERROR' | 'WRITE_VERIFICATION_ERROR' | 'WATERMARK_INVALID' | 'CONFIGURATION_ERROR' | 'UNKNOWN_ERROR';
|
|
17
|
+
export type SyncErrorCode = 'VALIDATION_ERROR' | 'FK_CONSTRAINT_VIOLATION' | 'DUPLICATE_KEY' | 'NETWORK_TIMEOUT' | 'RATE_LIMIT_EXCEEDED' | 'CONNECTOR_ERROR' | 'TRANSFORM_ERROR' | 'MATCH_RESOLUTION_ERROR' | 'DATABASE_ERROR' | 'WRITE_VERIFICATION_ERROR' | 'WATERMARK_INVALID' | 'CONFIGURATION_ERROR' | 'OBJECT_UNAVAILABLE' | 'UNKNOWN_ERROR';
|
|
18
18
|
/** Severity level of a sync error */
|
|
19
19
|
export type ErrorSeverity = 'Critical' | 'Warning' | 'Info';
|
|
20
20
|
/** Whether an error with the given code is retryable */
|
|
@@ -341,6 +341,30 @@ export interface SourceObjectInfo {
|
|
|
341
341
|
* when the source does not expose a documented watermark.
|
|
342
342
|
*/
|
|
343
343
|
IncrementalWatermarkField?: string;
|
|
344
|
+
/**
|
|
345
|
+
* Whether `Fields` is the object's COMPLETE column list for this account.
|
|
346
|
+
*
|
|
347
|
+
* Sources fall into three shapes and only the source knows which it is:
|
|
348
|
+
* - it describes no columns at all (declared metadata is the whole truth),
|
|
349
|
+
* - it returns only the account's CUSTOM columns (additive — the standard columns
|
|
350
|
+
* still exist, the source simply did not restate them),
|
|
351
|
+
* - it returns the full mapping (a column absent here is genuinely gone).
|
|
352
|
+
*
|
|
353
|
+
* Only the third shape may deactivate columns. This was previously INFERRED from
|
|
354
|
+
* "the field list came back non-empty", which cannot tell the second shape from the
|
|
355
|
+
* third — so a custom-only source looked authoritative and its standard columns were
|
|
356
|
+
* candidates for deactivation.
|
|
357
|
+
*
|
|
358
|
+
* Undefined means the object has not said, and the CONNECTOR's claim
|
|
359
|
+
* (`DiscoveryIsAuthoritative`, surfaced as `SourceSchemaInfo.IsAuthoritative`) is inherited:
|
|
360
|
+
* a connector affirming it returns the complete gamut is affirming it for the fields the same
|
|
361
|
+
* describe call returned. Since that claim defaults to false and a scoped introspection forces
|
|
362
|
+
* it false, an undeclared object under a non-affirming connector is still never deactivated.
|
|
363
|
+
*
|
|
364
|
+
* So set this `false` explicitly on an object whose describe is custom-only while the rest of
|
|
365
|
+
* the connector's discovery is complete — that is the one case the inherited claim gets wrong.
|
|
366
|
+
*/
|
|
367
|
+
FieldsAreAuthoritative?: boolean;
|
|
344
368
|
}
|
|
345
369
|
/** One field/column in a source object discovered during introspection. */
|
|
346
370
|
export interface SourceFieldInfo {
|
|
@@ -402,6 +426,26 @@ export interface SourceFieldInfo {
|
|
|
402
426
|
IsForeignKey?: boolean;
|
|
403
427
|
/** If FK, which source object it references (null if not a FK). */
|
|
404
428
|
ForeignKeyTarget: string | null;
|
|
429
|
+
/**
|
|
430
|
+
* How the sync engine should treat this field.
|
|
431
|
+
*
|
|
432
|
+
* - undefined / 'Sync' — normal behaviour (map, hash, persist).
|
|
433
|
+
* - 'Exclude' — the field is stripped from every fetched record BEFORE field
|
|
434
|
+
* mapping. It reaches neither the mapped row, nor the CustomOverflow column,
|
|
435
|
+
* nor the content-hash basis, so it stops influencing change detection.
|
|
436
|
+
*
|
|
437
|
+
* For payload a connector knows to be worthless or redundant at the field level:
|
|
438
|
+
* vendor UI state riding along on every record (Totara `preferences` — file-picker
|
|
439
|
+
* recents on 100% of rows), cosmetic configuration (`courseformatoptions`), or an
|
|
440
|
+
* embedded collection that re-derives an object already synced in its own right
|
|
441
|
+
* (`enrolledcourses` re-deriving Enrolled_Users). Exclusion — not field-map
|
|
442
|
+
* deactivation, which merely REROUTES the value into CustomOverflow, from which
|
|
443
|
+
* the custom-column promoter can resurrect it.
|
|
444
|
+
*
|
|
445
|
+
* Persisted into IntegrationObjectField.Configuration (JSON) by schema sync, so
|
|
446
|
+
* no migration is needed and existing installs pick it up on their next sync.
|
|
447
|
+
*/
|
|
448
|
+
SyncDirective?: 'Sync' | 'Exclude';
|
|
405
449
|
}
|
|
406
450
|
/** One foreign key relationship in a source object. */
|
|
407
451
|
export interface SourceRelationshipInfo {
|