@harperfast/harper-pro 5.0.16 → 5.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/DESIGN.md +32 -0
- package/core/bin/copyDb.ts +19 -0
- package/core/resources/replayLogs.ts +36 -3
- package/core/resources/replayLogsGuards.ts +42 -0
- package/core/resources/transactionBroadcast.ts +121 -66
- package/dist/core/bin/copyDb.js +16 -0
- package/dist/core/bin/copyDb.js.map +1 -1
- package/dist/core/resources/replayLogs.js +26 -2
- package/dist/core/resources/replayLogs.js.map +1 -1
- package/dist/core/resources/replayLogsGuards.js +43 -0
- package/dist/core/resources/replayLogsGuards.js.map +1 -0
- package/dist/core/resources/transactionBroadcast.js +129 -71
- package/dist/core/resources/transactionBroadcast.js.map +1 -1
- package/dist/replication/replicationConnection.js +111 -30
- package/dist/replication/replicationConnection.js.map +1 -1
- package/dist/replication/replicator.js +11 -2
- package/dist/replication/replicator.js.map +1 -1
- package/dist/replication/subscriptionManager.js +11 -1
- package/dist/replication/subscriptionManager.js.map +1 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/replication/replicationConnection.ts +110 -35
- package/replication/replicator.ts +11 -2
- package/replication/subscriptionManager.ts +11 -1
- package/studio/web/assets/{index-pr02wSIB.js → index-DhLu-DHX.js} +5 -5
- package/studio/web/assets/{index-pr02wSIB.js.map → index-DhLu-DHX.js.map} +1 -1
- package/studio/web/assets/{index.lazy-CorGZz3L.js → index.lazy-DBjOisCz.js} +2 -2
- package/studio/web/assets/{index.lazy-CorGZz3L.js.map → index.lazy-DBjOisCz.js.map} +1 -1
- package/studio/web/assets/{profile-SSvkzt9H.js → profile-DSL-499E.js} +2 -2
- package/studio/web/assets/{profile-SSvkzt9H.js.map → profile-DSL-499E.js.map} +1 -1
- package/studio/web/assets/{status-Xk93QrPQ.js → status-BRW5QtzY.js} +2 -2
- package/studio/web/assets/{status-Xk93QrPQ.js.map → status-BRW5QtzY.js.map} +1 -1
- package/studio/web/index.html +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replayLogs.js","sourceRoot":"","sources":["../../../core/resources/replayLogs.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"replayLogs.js","sourceRoot":"","sources":["../../../core/resources/replayLogs.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAWA,gCA0JC;AArKD,uDAAwF;AAGxF,4EAA8D;AAC9D,qEAA+D;AAE/D,6DAAmD;AACnD,yDAAmD;AACnD,+DAAoE;AAEpE,IAAI,qBAAqB,GAAG,KAAK,CAAC;AAClC,SAAgB,UAAU,CAAC,SAAwB,EAAE,MAAW;IAC/D,IAAI,CAAC,kCAAY;QAAE,OAAO,CAAC,oEAAoE;IAC/F,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE;YAC3D,OAAO,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,MAAM,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAC;QACrD,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;YAChC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACrC,CAAC;QACD,sBAAsB;QACtB,IAAI,WAAgC,CAAC;QACrC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,MAAM,MAAM,GAA6B,SAAS,CAAC,UAAU,CAAC;QAC9D,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAClG,MAAM,EACL,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,OAAO,EACP,WAAW,EACX,SAAS,EACT,oBAAoB,EACpB,QAAQ,EACR,YAAY,GACZ,GAAG,WAAW,CAAC;YAChB,IAAI,CAAC;gBACJ,IAAI,IAAA,iDAA2B,EAAC,YAAY,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,gBAAgB,EAAE,CAAC;oBACnF,OAAO,EAAE,CAAC;oBACV,SAAS;gBACV,CAAC;gBACD,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACrC,IAAI,CAAC,KAAK;oBAAE,SAAS;gBACrB,MAAM,OAAO,GAAY,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;gBACvG,MAAM,EAAE,YAAY,EAAE,GAAG,KAAK,CAAC;gBAC/B,MAAM,MAAM,GAAG,IAAI,gCAAa,EAAE,CAAC;gBACnC,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC;gBACjB,MAAM,aAAa,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC7D,qGAAqG;gBACrG,6FAA6F;gBAC7F,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBAC5B,qBAAqB,GAAG,IAAI,CAAC;oBAC7B,OAAO,CAAC,IAAI,CAAC,sFAAsF,CAAC,CAAC;gBACtG,CAAC;gBACD,IAAI,MAAW,CAAC;gBAChB,IAAI,CAAC;oBACJ,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;gBAC7C,CAAC;gBAAC,MAAM,CAAC;oBACR,2EAA2E;oBAC3E,4EAA4E;oBAC5E,6EAA6E;oBAC7E,4EAA4E;oBAC5E,6BAA6B;oBAC7B,OAAO,EAAE,CAAC;oBACV,SAAS;gBACV,CAAC;gBACD,IAAI,IAAA,iDAA2B,EAAC,YAAY,EAAE,OAAO,EAAE,MAAM,KAAK,SAAS,CAAC,KAAK,gBAAgB,EAAE,CAAC;oBACnG,OAAO,EAAE,CAAC;oBACV,SAAS;gBACV,CAAC;gBACD,IAAI,aAAa,KAAK,OAAO,EAAE,CAAC;oBAC/B,aAAa,GAAG,OAAO,CAAC;oBACxB,IAAI,CAAC;wBACJ,8DAA8D;wBAC9D,WAAW,EAAE,gBAAgB,EAAE,CAAC;oBACjC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBAChB,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;oBAC5D,CAAC;oBACD,WAAW,GAAG,IAAI,4CAAmB,EAAE,CAAC;oBACxC,WAAW,CAAC,EAAE,GAAG,YAAY,CAAC;oBAC9B,WAAW,CAAC,SAAS,GAAG,OAAO,CAAC;oBAChC,8GAA8G;oBAC9G,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC;gBACzB,CAAC;gBACD,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;gBAClC,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;gBACvE,MAAM,EAAE,CAAC;gBACT,QAAQ,IAAI,EAAE,CAAC;oBACd,KAAK,KAAK;wBACT,aAAa,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;wBAC5D,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,4BAA4B;wBAClD,MAAM;oBACP,KAAK,OAAO;wBACX,aAAa,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;wBAC7D,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,4BAA4B;wBAClD,MAAM;oBACP,KAAK,SAAS;wBACb,aAAa,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;wBACvD,MAAM;oBACP,KAAK,UAAU;wBACd,aAAa,CAAC,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBAChD,MAAM;oBACP,KAAK,QAAQ;wBACZ,aAAa,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBAC9C,MAAM;oBACP,KAAK,YAAY;wBAChB,aAAa,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;wBAC1D,MAAM;oBACP,KAAK,YAAY,CAAC,CAAC,CAAC;wBACnB,MAAM,gBAAgB,GAAG,IAAI,wBAAgB,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;wBAClE,MAAM,kBAAkB,GAAG,WAAW,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;wBACpE,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;wBAC3G,MAAM,kBAAkB,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;4BACzE,WAAW,EAAE,gBAAgB;yBAC7B,CAAC,CAAC;wBACH,IAAI,kBAAkB,EAAE,CAAC;4BACxB,IAAI,kBAAkB,YAAY,KAAK,EAAE,CAAC;gCACzC,IAAI,iBAAiB,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,CAAC;oCAC1D,MAAM,CAAC,IAAI,CACV,SAAS,kBAAkB,CAAC,MAAM,mCAAmC,iBAAiB,CAAC,MAAM,yBAAyB,iBAAiB,CAAC,MAAM,cAAc,CAC5J,CAAC;gCACH,CAAC;4BACF,CAAC;iCAAM,CAAC;gCACP,IAAI,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;oCACpF,MAAM,CAAC,IAAI,CACV,eAAe,kBAAkB,CAAC,MAAM,mCAAmC,iBAAiB,CAAC,MAAM,+BAA+B,iBAAiB,CAAC,MAAM,cAAc,CACxK,CAAC;gCACH,CAAC;gCACD,IAAI,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;oCACpF,MAAM,CAAC,IAAI,CACV,eAAe,kBAAkB,CAAC,MAAM,mCAAmC,iBAAiB,CAAC,MAAM,+BAA+B,iBAAiB,CAAC,MAAM,cAAc,CACxK,CAAC;gCACH,CAAC;4BACF,CAAC;wBACF,CAAC;wBACD,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,kBAAkB,CAAC,EAAE;4BAC5E,WAAW,EAAE,gBAAgB;yBAC7B,CAAC,CAAC;wBACH,gBAAgB,CAAC,UAAU,EAAE,CAAC;wBAC9B,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,iBAAiB,CAAC;oBACpD,CAAC;gBACF,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,GAAG,EAAE;oBACrD,OAAO;iBACP,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;QACD,IAAI,CAAC;YACJ,WAAW,EAAE,gBAAgB,EAAE,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;QAC5D,CAAC;QACD,IAAI,MAAM,GAAG,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,YAAY,MAAM,eAAe,SAAS,CAAC,YAAY,WAAW,CAAC,CAAC;QAChG,IAAI,OAAO,GAAG,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,mCAAmC,SAAS,CAAC,YAAY,yBAAyB,CAAC,CAAC;QACnH,+EAA+E;QAC/E,kCAAkC;IACnC,CAAC,CAAC,CAAC;AACJ,CAAC;AACD,SAAS,QAAQ,CAAC,MAAM;IACvB,OAAO,EAAE,CAAC,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;AAC5C,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Pure helpers for replayLogs (no Harper module dependencies, so unit tests can load
|
|
3
|
+
// them without bootstrapping the full Resource / RocksDB / DatabaseTransaction graph).
|
|
4
|
+
//
|
|
5
|
+
// Background: a node that crashed unclean re-runs replayLogs against the unflushed audit
|
|
6
|
+
// log on next boot. If any audit entry is corrupt or missing its record body, the loop
|
|
7
|
+
// hits a TypeError inside Table.validate() ("Cannot read properties of undefined
|
|
8
|
+
// (reading 'cacheKey')") and the per-iteration catch swallows it — but the loop keeps
|
|
9
|
+
// running over potentially millions of entries, pinning CPU. These guards classify each
|
|
10
|
+
// entry up front so the loop can skip cleanly.
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.RECORD_BEARING_FLAGS = void 0;
|
|
13
|
+
exports.classifyAuditEntryForReplay = classifyAuditEntryForReplay;
|
|
14
|
+
// Mirrors `HAS_RECORD` (16) | `HAS_PARTIAL_RECORD` (32) from auditStore.ts — the action
|
|
15
|
+
// bits the writer sets when an entry carries (or should carry) a record body. Redeclared
|
|
16
|
+
// here so this module stays free of the Harper module graph for unit testing; a lock
|
|
17
|
+
// test pins the value against auditStore so silent drift is caught.
|
|
18
|
+
exports.RECORD_BEARING_FLAGS = 16 | 32;
|
|
19
|
+
/**
|
|
20
|
+
* Decide whether an audit entry pulled from the unflushed log is safe to replay.
|
|
21
|
+
* Returns `null` if the entry should be replayed, or a short reason string if it should
|
|
22
|
+
* be skipped (the loop logs the aggregate skip count once at the end).
|
|
23
|
+
*
|
|
24
|
+
* Operates on the raw integer `action` field rather than the decoded type string: when
|
|
25
|
+
* `readAuditEntry` catches a header decode error it returns `{}`, so both `action` and
|
|
26
|
+
* `tableId` are `undefined` — the same signal — and matching the record-bearing flags
|
|
27
|
+
* directly against the action mirrors how the writer set them in `auditStore.ts`.
|
|
28
|
+
*
|
|
29
|
+
* @param action `auditRecord.extendedType` — the variable-length action field with
|
|
30
|
+
* the event type in the low nibble and HAS_* flags above it
|
|
31
|
+
* @param tableId `auditRecord.tableId`
|
|
32
|
+
* @param hasRecord `true` if `auditRecord.getValue(...)` produced a non-undefined value
|
|
33
|
+
*/
|
|
34
|
+
function classifyAuditEntryForReplay(action, tableId, hasRecord) {
|
|
35
|
+
if (action === undefined || tableId === undefined)
|
|
36
|
+
return 'corrupt-header';
|
|
37
|
+
// If the action advertises a record body but the decoded record is undefined, the
|
|
38
|
+
// downstream write path will crash inside validate() on the first attribute deref.
|
|
39
|
+
if ((action & exports.RECORD_BEARING_FLAGS) !== 0 && !hasRecord)
|
|
40
|
+
return 'missing-record';
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=replayLogsGuards.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replayLogsGuards.js","sourceRoot":"","sources":["../../../core/resources/replayLogsGuards.ts"],"names":[],"mappings":";AAAA,qFAAqF;AACrF,uFAAuF;AACvF,EAAE;AACF,yFAAyF;AACzF,uFAAuF;AACvF,iFAAiF;AACjF,sFAAsF;AACtF,wFAAwF;AACxF,+CAA+C;;;AAuB/C,kEAUC;AA/BD,wFAAwF;AACxF,yFAAyF;AACzF,qFAAqF;AACrF,oEAAoE;AACvD,QAAA,oBAAoB,GAAG,EAAE,GAAG,EAAE,CAAC;AAE5C;;;;;;;;;;;;;;GAcG;AACH,SAAgB,2BAA2B,CAC1C,MAA0B,EAC1B,OAA2B,EAC3B,SAAkB;IAElB,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,gBAAgB,CAAC;IAC3E,kFAAkF;IAClF,mFAAmF;IACnF,IAAI,CAAC,MAAM,GAAG,4BAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS;QAAE,OAAO,gBAAgB,CAAC;IACjF,OAAO,IAAI,CAAC;AACb,CAAC"}
|
|
@@ -44,8 +44,24 @@ function addSubscription(table, key, listener, startTime, options) {
|
|
|
44
44
|
auditLogIterator = auditStore.getRange({});
|
|
45
45
|
}
|
|
46
46
|
auditStore.hasSubscriptionCommitListener = true;
|
|
47
|
+
// Coalesce 'committed' bursts: instead of iterating the audit log synchronously inside the
|
|
48
|
+
// commit microtask (which pegs the event loop during replication backlog catch-up), defer
|
|
49
|
+
// to setImmediate. Multiple commits within the same turn collapse into one notify pass.
|
|
50
|
+
// notifyScheduled stays true for the full drain — including yield-and-resume — so new
|
|
51
|
+
// 'committed' events that fire mid-drain don't spawn an overlapping notify pass.
|
|
47
52
|
auditStore.on('committed', () => {
|
|
48
|
-
|
|
53
|
+
if (!databaseSubscriptions.activeCount) {
|
|
54
|
+
// No per-key listeners; skip the expensive audit-log iteration entirely. But still
|
|
55
|
+
// rotate the nextTransaction promise so any whenNextTransaction() waiter (used by
|
|
56
|
+
// outbound replication on databases with no local subscribers) wakes up.
|
|
57
|
+
if (auditStore.nextTransaction)
|
|
58
|
+
nextTransaction(auditStore);
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
if (databaseSubscriptions.notifyScheduled)
|
|
62
|
+
return;
|
|
63
|
+
databaseSubscriptions.notifyScheduled = true;
|
|
64
|
+
setImmediate(() => notifyFromTransactionData(databaseSubscriptions, auditLogIterator, true));
|
|
49
65
|
});
|
|
50
66
|
}
|
|
51
67
|
}
|
|
@@ -75,6 +91,7 @@ function addSubscription(table, key, listener, startTime, options) {
|
|
|
75
91
|
subscriptions.key = key;
|
|
76
92
|
}
|
|
77
93
|
subscription.subscriptions = subscriptions;
|
|
94
|
+
databaseSubscriptions.activeCount = (databaseSubscriptions.activeCount || 0) + 1;
|
|
78
95
|
return subscription;
|
|
79
96
|
}
|
|
80
97
|
/**
|
|
@@ -97,20 +114,21 @@ class Subscription extends IterableEventQueue_ts_1.IterableEventQueue {
|
|
|
97
114
|
// cleanup
|
|
98
115
|
if (!this.subscriptions)
|
|
99
116
|
return;
|
|
117
|
+
const tableSubscriptions = this.subscriptions.tables;
|
|
118
|
+
const envSubscriptions = tableSubscriptions?.envs;
|
|
100
119
|
this.subscriptions.splice(this.subscriptions.indexOf(this), 1);
|
|
101
120
|
if (this.subscriptions.length === 0) {
|
|
102
|
-
const tableSubscriptions = this.subscriptions.tables;
|
|
103
121
|
if (tableSubscriptions) {
|
|
104
122
|
// TODO: Handle cleanup of wildcard
|
|
105
123
|
const key = this.subscriptions.key;
|
|
106
124
|
tableSubscriptions.delete(key);
|
|
107
125
|
if (tableSubscriptions.size === 0) {
|
|
108
|
-
|
|
109
|
-
const dbi = tableSubscriptions.dbi;
|
|
110
|
-
delete envSubscriptions[dbi];
|
|
126
|
+
delete envSubscriptions[tableSubscriptions.tableId];
|
|
111
127
|
}
|
|
112
128
|
}
|
|
113
129
|
}
|
|
130
|
+
if (envSubscriptions?.activeCount > 0)
|
|
131
|
+
envSubscriptions.activeCount--;
|
|
114
132
|
this.subscriptions = null;
|
|
115
133
|
}
|
|
116
134
|
toJSON() {
|
|
@@ -118,13 +136,27 @@ class Subscription extends IterableEventQueue_ts_1.IterableEventQueue {
|
|
|
118
136
|
}
|
|
119
137
|
}
|
|
120
138
|
const ACTIONS_OF_INTEREST = ['put', 'patch', 'delete', 'message', 'invalidate'];
|
|
121
|
-
|
|
139
|
+
// Maximum audit records processed per synchronous turn before yielding back to the event loop.
|
|
140
|
+
// Sized to keep per-batch wall time within a few ms on commodity hardware while keeping the
|
|
141
|
+
// scheduling overhead amortized; tune if profiling shows different shapes.
|
|
142
|
+
const NOTIFY_BATCH_SIZE = 256;
|
|
143
|
+
function notifyFromTransactionData(subscriptions, auditLogIterable, allowYield = false) {
|
|
122
144
|
if (!subscriptions)
|
|
123
145
|
return; // if no subscriptions to this env path, don't need to read anything
|
|
146
|
+
// If no real subscribers are attached, skip the iteration. The reusable iterator preserves its
|
|
147
|
+
// position and will pick up from where we left it once a subscriber is added.
|
|
148
|
+
if (!subscriptions.activeCount) {
|
|
149
|
+
subscriptions.pendingTxnSubscribers = null; // discard any carry-over from a yielded run
|
|
150
|
+
if (allowYield)
|
|
151
|
+
subscriptions.notifyScheduled = false;
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
124
154
|
const auditStore = subscriptions.auditStore;
|
|
125
155
|
auditStore.resetReadTxn?.();
|
|
126
|
-
nextTransaction(
|
|
127
|
-
|
|
156
|
+
nextTransaction(auditStore);
|
|
157
|
+
// subscribersWithTxns is carried across batches so the end_txn signal fires only once the
|
|
158
|
+
// iterator truly drains, not at each yield point.
|
|
159
|
+
let subscribersWithTxns = subscriptions.pendingTxnSubscribers;
|
|
128
160
|
if (!auditLogIterable) {
|
|
129
161
|
// rocksdb will pass this in, but with lmdb, we have to re-create the iterable
|
|
130
162
|
auditLogIterable = auditStore.getRange({
|
|
@@ -132,77 +164,103 @@ function notifyFromTransactionData(subscriptions, auditLogIterable) {
|
|
|
132
164
|
exclusiveStart: true,
|
|
133
165
|
});
|
|
134
166
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
167
|
+
const iterator = auditLogIterable[Symbol.iterator]?.() ?? auditLogIterable;
|
|
168
|
+
let processed = 0;
|
|
169
|
+
let yielded = false;
|
|
170
|
+
try {
|
|
171
|
+
while (true) {
|
|
172
|
+
const result = iterator.next();
|
|
173
|
+
if (result.done)
|
|
174
|
+
break;
|
|
175
|
+
const auditRecord = result.value;
|
|
176
|
+
const timestamp = auditRecord.localTime ?? auditRecord.version;
|
|
177
|
+
subscriptions.lastTxnTime = timestamp;
|
|
178
|
+
if (ACTIONS_OF_INTEREST.includes(auditRecord.type)) {
|
|
179
|
+
const tableSubscriptions = subscriptions[auditRecord.tableId];
|
|
180
|
+
if (tableSubscriptions) {
|
|
181
|
+
const recordId = auditRecord.recordId;
|
|
182
|
+
// TODO: How to handle invalidation
|
|
183
|
+
let matchingKey = (0, Resources_ts_1.keyArrayToString)(recordId);
|
|
184
|
+
let ancestorLevel = 0;
|
|
185
|
+
do {
|
|
186
|
+
// we iterate through the key hierarchy, notifying all subscribers for each key,
|
|
187
|
+
// so for an id like resource/foo/bar, we notify subscribers for resource/foo/bar, resource/foo/, resource/foo, resource/, and resource
|
|
188
|
+
// this allows for efficient subscriptions to children ids/topics
|
|
189
|
+
const keySubscriptions = tableSubscriptions.get(matchingKey);
|
|
190
|
+
if (keySubscriptions) {
|
|
191
|
+
for (const subscription of keySubscriptions) {
|
|
192
|
+
if (ancestorLevel > 0 && // only ancestors if the subscription is for ancestors (and apply onlyChildren filtering as necessary)
|
|
193
|
+
!(subscription.includeDescendants && !(subscription.onlyChildren && ancestorLevel > 1)))
|
|
194
|
+
continue;
|
|
195
|
+
if (subscription.startTime >= timestamp) {
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
try {
|
|
199
|
+
let beginTxn;
|
|
200
|
+
if (subscription.supportsTransactions && subscription.txnInProgress !== auditRecord.version) {
|
|
201
|
+
// if the subscriber supports transactions, we mark this as the beginning of a new transaction
|
|
202
|
+
// tracking the subscription so that we can delimit the transaction on next transaction
|
|
203
|
+
// (with a beginTxn flag, which may be on an endTxn event)
|
|
204
|
+
beginTxn = true;
|
|
205
|
+
if (!subscription.txnInProgress) {
|
|
206
|
+
// if first txn for subscriber of this cycle, add to the transactional subscribers that we are tracking
|
|
207
|
+
if (!subscribersWithTxns)
|
|
208
|
+
subscribersWithTxns = [subscription];
|
|
209
|
+
else
|
|
210
|
+
subscribersWithTxns.push(subscription);
|
|
211
|
+
}
|
|
212
|
+
// the version defines the extent of a transaction, all audit records with the same version
|
|
213
|
+
// are part of the same transaction, and when the version changes, we know it is a new
|
|
214
|
+
// transaction
|
|
215
|
+
subscription.txnInProgress = auditRecord.version;
|
|
216
|
+
}
|
|
217
|
+
subscription.listener(recordId, auditRecord, timestamp, beginTxn);
|
|
218
|
+
}
|
|
219
|
+
catch (error) {
|
|
220
|
+
(0, harper_logger_js_1.warn)(error);
|
|
221
|
+
}
|
|
173
222
|
}
|
|
174
|
-
// the version defines the extent of a transaction, all audit records with the same version
|
|
175
|
-
// are part of the same transaction, and when the version changes, we know it is a new
|
|
176
|
-
// transaction
|
|
177
|
-
subscription.txnInProgress = auditRecord.version;
|
|
178
223
|
}
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
(
|
|
183
|
-
|
|
224
|
+
if (matchingKey == null)
|
|
225
|
+
break;
|
|
226
|
+
const lastSlash = matchingKey.lastIndexOf?.('/', matchingKey.length - 2);
|
|
227
|
+
if (lastSlash !== matchingKey.length - 1) {
|
|
228
|
+
ancestorLevel++; // don't increase the ancestor level for this going from resource/ to resource
|
|
229
|
+
}
|
|
230
|
+
if (lastSlash > -1) {
|
|
231
|
+
matchingKey = matchingKey.slice(0, lastSlash + 1);
|
|
232
|
+
}
|
|
233
|
+
else
|
|
234
|
+
matchingKey = null;
|
|
235
|
+
} while (true);
|
|
184
236
|
}
|
|
185
237
|
}
|
|
186
|
-
if (
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
238
|
+
if (allowYield && ++processed >= NOTIFY_BATCH_SIZE) {
|
|
239
|
+
// Yield the event loop. Save in-progress txn state so the next batch can resume.
|
|
240
|
+
// Reusable iterables (rocksdb) can be passed back in directly; LMDB-style iterables
|
|
241
|
+
// are recreated from the advanced lastTxnTime. The same-thread aftercommit path does not
|
|
242
|
+
// set allowYield because it holds an inter-thread lock that must not span event-loop turns.
|
|
243
|
+
subscriptions.pendingTxnSubscribers = subscribersWithTxns;
|
|
244
|
+
yielded = true;
|
|
245
|
+
setImmediate(() => notifyFromTransactionData(subscriptions, auditStore.reusableIterable ? auditLogIterable : null, true));
|
|
246
|
+
return;
|
|
191
247
|
}
|
|
192
|
-
|
|
193
|
-
|
|
248
|
+
}
|
|
249
|
+
subscriptions.pendingTxnSubscribers = null;
|
|
250
|
+
if (subscribersWithTxns) {
|
|
251
|
+
// any subscribers with open transactions need to have an event to indicate that their transaction has been ended
|
|
252
|
+
for (const subscription of subscribersWithTxns) {
|
|
253
|
+
subscription.txnInProgress = null; // clean up
|
|
254
|
+
subscription.listener(null, { type: 'end_txn' }, subscriptions.lastTxnTime, true);
|
|
194
255
|
}
|
|
195
|
-
else
|
|
196
|
-
matchingKey = null;
|
|
197
|
-
} while (true);
|
|
198
|
-
}
|
|
199
|
-
if (subscribersWithTxns) {
|
|
200
|
-
// any subscribers with open transactions need to have an event to indicate that their transaction has been ended
|
|
201
|
-
for (const subscription of subscribersWithTxns) {
|
|
202
|
-
subscription.txnInProgress = null; // clean up
|
|
203
|
-
subscription.listener(null, { type: 'end_txn' }, subscriptions.lastTxnTime, true);
|
|
204
256
|
}
|
|
205
257
|
}
|
|
258
|
+
finally {
|
|
259
|
+
// If we yielded, the continuation owns notifyScheduled; otherwise (drain or any throw) we
|
|
260
|
+
// must clear it here so a stuck flag doesn't permanently silence future commits.
|
|
261
|
+
if (allowYield && !yielded)
|
|
262
|
+
subscriptions.notifyScheduled = false;
|
|
263
|
+
}
|
|
206
264
|
}
|
|
207
265
|
/**
|
|
208
266
|
* Interface with database to listen for commits and traverse the audit log only on the same thread.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transactionBroadcast.js","sourceRoot":"","sources":["../../../core/resources/transactionBroadcast.ts"],"names":[],"mappings":";;AAiBA,
|
|
1
|
+
{"version":3,"file":"transactionBroadcast.js","sourceRoot":"","sources":["../../../core/resources/transactionBroadcast.ts"],"names":[],"mappings":";;AAiBA,0CAyEC;AAsKD,0CAkCC;AAUD,kDAeC;AA3TD,0EAA2D;AAC3D,mEAA6D;AAC7D,iDAAkD;AAGlD,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;AAC3F,MAAM,0BAA0B,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;AACrG;;;;;;;;;GASG;AACH,SAAgB,eAAe,CAAC,KAAK,EAAE,GAAG,EAAE,QAAuB,EAAE,SAAkB,EAAE,OAAa;IACrG,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC;IACrC,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC;IAC3C,sGAAsG;IACtG,iEAAiE;IACjE,IAAI,qBAAqB,CAAC;IAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,OAAO,EAAE,YAAY,KAAK,KAAK,EAAE,CAAC;QACrC,6GAA6G;QAC7G,qBAAqB,GAAG,0BAA0B,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACpG,eAAe,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IACvD,CAAC;SAAM,CAAC;QACP,qBAAqB,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,UAAU,CAAC,6BAA6B,EAAE,CAAC;YAC/C,IAAI,gBAAgB,CAAC;YACrB,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;gBACjC,iHAAiH;gBACjH,4FAA4F;gBAC5F,8CAA8C;gBAC9C,gBAAgB,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC5C,CAAC;YACD,UAAU,CAAC,6BAA6B,GAAG,IAAI,CAAC;YAChD,2FAA2F;YAC3F,0FAA0F;YAC1F,wFAAwF;YACxF,sFAAsF;YACtF,iFAAiF;YACjF,UAAU,CAAC,EAAE,CAAC,WAAW,EAAE,GAAG,EAAE;gBAC/B,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,CAAC;oBACxC,mFAAmF;oBACnF,kFAAkF;oBAClF,yEAAyE;oBACzE,IAAI,UAAU,CAAC,eAAe;wBAAE,eAAe,CAAC,UAAU,CAAC,CAAC;oBAC5D,OAAO;gBACR,CAAC;gBACD,IAAI,qBAAqB,CAAC,eAAe;oBAAE,OAAO;gBAClD,qBAAqB,CAAC,eAAe,GAAG,IAAI,CAAC;gBAC7C,YAAY,CAAC,GAAG,EAAE,CAAC,yBAAyB,CAAC,qBAAqB,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC;YAC9F,CAAC,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IACD,qBAAqB,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IACpD,IAAI,qBAAqB,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;QAC/C,qBAAqB,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChD,CAAC;IACD,IAAI,OAAO,EAAE,KAAK,KAAK,eAAe,EAAE,CAAC;QACxC,OAAO;IACR,CAAC;IACD,IAAI,kBAAkB,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACxD,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,qBAAqB,CAAC,OAAO,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC;QAChE,kBAAkB,CAAC,IAAI,GAAG,qBAAqB,CAAC;QAChD,kBAAkB,CAAC,OAAO,GAAG,OAAO,CAAC;QACrC,kBAAkB,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC;IAC/C,CAAC;IAED,GAAG,GAAG,IAAA,+BAAgB,EAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChD,YAAY,CAAC,SAAS,GAAG,SAAS,CAAC;IACnC,IAAI,aAAa,GAAU,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEvD,IAAI,aAAa;QAAE,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SAC/C,CAAC;QACL,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,aAAa,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9D,aAAa,CAAC,MAAM,GAAG,kBAAkB,CAAC;QAC1C,aAAa,CAAC,GAAG,GAAG,GAAG,CAAC;IACzB,CAAC;IACD,YAAY,CAAC,aAAa,GAAG,aAAa,CAAC;IAC3C,qBAAqB,CAAC,WAAW,GAAG,CAAC,qBAAqB,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACjF,OAAO,YAAY,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,YAAa,SAAQ,0CAAkB;IAC5C,QAAQ,CAAgF;IACxF,aAAa,CAAK;IAClB,SAAS,CAAU;IACnB,kBAAkB,CAAW;IAC7B,oBAAoB,CAAW;IAC/B,YAAY,CAAW;IACvB,YAAY,QAAQ;QACnB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpC,CAAC;IACD,GAAG;QACF,UAAU;QACV,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO;QAChC,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC;QACrD,MAAM,gBAAgB,GAAG,kBAAkB,EAAE,IAAI,CAAC;QAClD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,IAAI,kBAAkB,EAAE,CAAC;gBACxB,mCAAmC;gBACnC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;gBACnC,kBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC/B,IAAI,kBAAkB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBACnC,OAAO,gBAAgB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBACrD,CAAC;YACF,CAAC;QACF,CAAC;QACD,IAAI,gBAAgB,EAAE,WAAW,GAAG,CAAC;YAAE,gBAAgB,CAAC,WAAW,EAAE,CAAC;QACtE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC3B,CAAC;IACD,MAAM;QACL,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;IACjC,CAAC;CACD;AACD,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;AAChF,+FAA+F;AAC/F,4FAA4F;AAC5F,2EAA2E;AAC3E,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,SAAS,yBAAyB,CAAC,aAAa,EAAE,gBAAgB,EAAE,UAAU,GAAG,KAAK;IACrF,IAAI,CAAC,aAAa;QAAE,OAAO,CAAC,oEAAoE;IAChG,+FAA+F;IAC/F,8EAA8E;IAC9E,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAChC,aAAa,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAAC,4CAA4C;QACxF,IAAI,UAAU;YAAE,aAAa,CAAC,eAAe,GAAG,KAAK,CAAC;QACtD,OAAO;IACR,CAAC;IACD,MAAM,UAAU,GAAG,aAAa,CAAC,UAAU,CAAC;IAC5C,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;IAC5B,eAAe,CAAC,UAAU,CAAC,CAAC;IAC5B,0FAA0F;IAC1F,kDAAkD;IAClD,IAAI,mBAAmB,GAAG,aAAa,CAAC,qBAAqB,CAAC;IAC9D,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACvB,8EAA8E;QAC9E,gBAAgB,GAAG,UAAU,CAAC,QAAQ,CAAC;YACtC,KAAK,EAAE,aAAa,CAAC,WAAW;YAChC,cAAc,EAAE,IAAI;SACpB,CAAC,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,gBAAgB,CAAC;IAC3E,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,CAAC;QACJ,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,IAAI;gBAAE,MAAM;YACvB,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;YACjC,MAAM,SAAS,GAAW,WAAW,CAAC,SAAS,IAAI,WAAW,CAAC,OAAO,CAAC;YACvE,aAAa,CAAC,WAAW,GAAG,SAAS,CAAC;YACtC,IAAI,mBAAmB,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpD,MAAM,kBAAkB,GAAG,aAAa,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC9D,IAAI,kBAAkB,EAAE,CAAC;oBACxB,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC;oBACtC,mCAAmC;oBACnC,IAAI,WAAW,GAAG,IAAA,+BAAgB,EAAC,QAAQ,CAAC,CAAC;oBAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;oBACtB,GAAG,CAAC;wBACH,gFAAgF;wBAChF,uIAAuI;wBACvI,iEAAiE;wBACjE,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;wBAC7D,IAAI,gBAAgB,EAAE,CAAC;4BACtB,KAAK,MAAM,YAAY,IAAI,gBAAgB,EAAE,CAAC;gCAC7C,IACC,aAAa,GAAG,CAAC,IAAI,sGAAsG;oCAC3H,CAAC,CAAC,YAAY,CAAC,kBAAkB,IAAI,CAAC,CAAC,YAAY,CAAC,YAAY,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC;oCAEvF,SAAS;gCACV,IAAI,YAAY,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;oCACzC,SAAS;gCACV,CAAC;gCACD,IAAI,CAAC;oCACJ,IAAI,QAAQ,CAAC;oCACb,IAAI,YAAY,CAAC,oBAAoB,IAAI,YAAY,CAAC,aAAa,KAAK,WAAW,CAAC,OAAO,EAAE,CAAC;wCAC7F,8FAA8F;wCAC9F,uFAAuF;wCACvF,0DAA0D;wCAC1D,QAAQ,GAAG,IAAI,CAAC;wCAChB,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;4CACjC,uGAAuG;4CACvG,IAAI,CAAC,mBAAmB;gDAAE,mBAAmB,GAAG,CAAC,YAAY,CAAC,CAAC;;gDAC1D,mBAAmB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wCAC7C,CAAC;wCACD,2FAA2F;wCAC3F,sFAAsF;wCACtF,cAAc;wCACd,YAAY,CAAC,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC;oCAClD,CAAC;oCACD,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gCACnE,CAAC;gCAAC,OAAO,KAAK,EAAE,CAAC;oCAChB,IAAA,uBAAI,EAAC,KAAK,CAAC,CAAC;gCACb,CAAC;4BACF,CAAC;wBACF,CAAC;wBACD,IAAI,WAAW,IAAI,IAAI;4BAAE,MAAM;wBAC/B,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC,GAAG,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBACzE,IAAI,SAAS,KAAK,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC1C,aAAa,EAAE,CAAC,CAAC,8EAA8E;wBAChG,CAAC;wBACD,IAAI,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;4BACpB,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;wBACnD,CAAC;;4BAAM,WAAW,GAAG,IAAI,CAAC;oBAC3B,CAAC,QAAQ,IAAI,EAAE;gBAChB,CAAC;YACF,CAAC;YACD,IAAI,UAAU,IAAI,EAAE,SAAS,IAAI,iBAAiB,EAAE,CAAC;gBACpD,iFAAiF;gBACjF,oFAAoF;gBACpF,yFAAyF;gBACzF,4FAA4F;gBAC5F,aAAa,CAAC,qBAAqB,GAAG,mBAAmB,CAAC;gBAC1D,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,CAAC,GAAG,EAAE,CACjB,yBAAyB,CAAC,aAAa,EAAE,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CACrG,CAAC;gBACF,OAAO;YACR,CAAC;QACF,CAAC;QACD,aAAa,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAC3C,IAAI,mBAAmB,EAAE,CAAC;YACzB,iHAAiH;YACjH,KAAK,MAAM,YAAY,IAAI,mBAAmB,EAAE,CAAC;gBAChD,YAAY,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,WAAW;gBAC9C,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YACnF,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,0FAA0F;QAC1F,iFAAiF;QACjF,IAAI,UAAU,IAAI,CAAC,OAAO;YAAE,aAAa,CAAC,eAAe,GAAG,KAAK,CAAC;IACnE,CAAC;AACF,CAAC;AACD;;;;GAIG;AACH,SAAgB,eAAe,CAAC,YAAY,EAAE,UAAU;IACvD,MAAM,KAAK,GAAG,UAAU,IAAI,YAAY,CAAC;IACzC,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC;IAC/B,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC;IAC1B,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;QACrC,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACtC,KAAK,CAAC,EAAE,CAAC,aAAa,EAAE,CAAC,UAAU,EAAE,EAAE;YACtC,MAAM,aAAa,GAAG,0BAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,wEAAwE;YAChI,IAAI,CAAC,aAAa;gBAAE,OAAO;YAC3B,uFAAuF;YACvF,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/B,OAAO,yBAAyB,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;YAC7D,CAAC;YACD,0KAA0K;YAC1K,MAAM,YAAY,GAAG,GAAG,EAAE;gBACzB,wHAAwH;gBACxH,IAAI,CAAC,KAAK,CAAC,iBAAiB;oBAC3B,uCAAuC;oBACvC,KAAK,CAAC,iBAAiB,GAAG,IAAI,YAAY,CACzC,KAAK,CAAC,mBAAmB,CAAC,yBAAyB,EAAE,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,CACxE,CAAC;gBACH,aAAa,CAAC,OAAO,GAAG,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,sBAAsB;gBACxF,IAAI,CAAC;oBACJ,yBAAyB,CAAC,aAAa,CAAC,CAAC;gBAC1C,CAAC;wBAAS,CAAC;oBACV,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC,uBAAuB;oBAC/E,KAAK,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,uBAAuB;gBAC7D,CAAC;YACF,CAAC,CAAC;YACF,iCAAiC;YACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,YAAY,CAAC;gBAAE,OAAO;YAChE,YAAY,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;IACJ,CAAC;AACF,CAAC;AACD,SAAS,eAAe,CAAC,UAAU;IAClC,UAAU,CAAC,eAAe,EAAE,OAAO,EAAE,CAAC;IACtC,IAAI,WAAW,CAAC;IAChB,UAAU,CAAC,eAAe,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACpD,WAAW,GAAG,OAAO,CAAC;IACvB,CAAC,CAAC,CAAC;IACH,UAAU,CAAC,eAAe,CAAC,OAAO,GAAG,WAAW,CAAC;AAClD,CAAC;AAED,SAAgB,mBAAmB,CAAC,UAAU;IAC7C,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;QACjC,eAAe,CACd;YACC,YAAY,EAAE,UAAU;YACxB,UAAU;SACV,EACD,IAAI,EACJ,IAAI,EACJ,CAAC,EACD,EAAE,KAAK,EAAE,eAAe,EAAE,CAC1B,CAAC;QACF,eAAe,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,UAAU,CAAC,eAAe,CAAC;AACnC,CAAC"}
|
|
@@ -91,6 +91,11 @@ exports.BACK_PRESSURE_RATIO_POSITION = 6;
|
|
|
91
91
|
exports.RECEIVING_STATUS_WAITING = 0;
|
|
92
92
|
exports.RECEIVING_STATUS_RECEIVING = 1;
|
|
93
93
|
const MAX_PAYLOAD = environmentManager_js_1.default.get('replication_maxPayload') ?? 100_000_000;
|
|
94
|
+
// When receiving a replication message, we apply per-record backpressure to keep a single
|
|
95
|
+
// large batch from synchronously decoding thousands of records and ballooning the worker
|
|
96
|
+
// heap past its limit. If the local replicator queue grows beyond this threshold we pause
|
|
97
|
+
// the WS connection and wait for it to drain before continuing the decode loop.
|
|
98
|
+
const RECEIVE_EVENT_HIGH_WATER_MARK = environmentManager_js_1.default.get('replication_receiveEventHighWaterMark') ?? 100;
|
|
94
99
|
exports.tableUpdateListeners = new Map();
|
|
95
100
|
// This a map of the database name to the subscription object, for the subscriptions from our tables to the replication module
|
|
96
101
|
// when we receive messages from other nodes, we then forward them on to as a notification on these subscriptions
|
|
@@ -420,13 +425,37 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
420
425
|
const MAX_OUTSTANDING_BLOBS_BEING_SENT = environmentManager_js_1.default.get(hdbTerms_ts_1.CONFIG_PARAMS.REPLICATION_BLOBCONCURRENCY) ?? 5;
|
|
421
426
|
let outstandingCommits = 0;
|
|
422
427
|
let lastStructureLength = 0;
|
|
423
|
-
|
|
428
|
+
// Multiple independent conditions can ask to pause receive on this WS (commit backlog,
|
|
429
|
+
// consumer queue full, blob write backpressure). We refcount the reasons so that resuming
|
|
430
|
+
// one does not race ahead of another that still wants the WS paused.
|
|
431
|
+
let pauseReasons = 0;
|
|
432
|
+
let commitBacklogPaused = false;
|
|
433
|
+
function addPauseReason() {
|
|
434
|
+
if (pauseReasons === 0)
|
|
435
|
+
ws.pause();
|
|
436
|
+
pauseReasons++;
|
|
437
|
+
}
|
|
438
|
+
function removePauseReason() {
|
|
439
|
+
if (pauseReasons === 0)
|
|
440
|
+
return;
|
|
441
|
+
pauseReasons--;
|
|
442
|
+
if (pauseReasons === 0)
|
|
443
|
+
ws.resume();
|
|
444
|
+
}
|
|
424
445
|
let subscriptionRequest, auditSubscription;
|
|
425
446
|
let nodeSubscriptions;
|
|
426
447
|
let excludedNodes; // list of nodes to exclude from this subscription
|
|
427
448
|
let remoteShortIdToLocalId;
|
|
428
449
|
let subscribedNodeIds; // map of node IDs to their subscription time ranges
|
|
429
|
-
|
|
450
|
+
// Serialize message handling so that async backpressure inside onWSMessage doesn't allow
|
|
451
|
+
// the WS library to start processing the next frame before the current one is fully decoded.
|
|
452
|
+
// Without serialization, awaiting inside the handler would let concurrent message handlers
|
|
453
|
+
// share the consumer queue and defeat the per-record backpressure below.
|
|
454
|
+
let messageProcessing = Promise.resolve();
|
|
455
|
+
let wsClosed = false;
|
|
456
|
+
ws.on('message', (body) => {
|
|
457
|
+
messageProcessing = messageProcessing.then(() => (wsClosed ? undefined : onWSMessage(body)), () => (wsClosed ? undefined : onWSMessage(body)));
|
|
458
|
+
});
|
|
430
459
|
let authorizationFinished = false;
|
|
431
460
|
function checkAuthorization() {
|
|
432
461
|
authorizationFinished = true;
|
|
@@ -438,27 +467,26 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
438
467
|
}
|
|
439
468
|
return true;
|
|
440
469
|
}
|
|
441
|
-
function onWSMessage(body) {
|
|
470
|
+
async function onWSMessage(body) {
|
|
442
471
|
if (!authorizationFinished) {
|
|
443
472
|
if (authorization?.then) {
|
|
444
|
-
|
|
445
|
-
authorization =
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
}
|
|
449
|
-
}, (error) => {
|
|
473
|
+
try {
|
|
474
|
+
authorization = await authorization;
|
|
475
|
+
}
|
|
476
|
+
catch (error) {
|
|
450
477
|
authorizationFinished = true;
|
|
451
478
|
logger.error?.(connectionId, 'Authorization failed', error);
|
|
452
479
|
// don't send disconnect because we want the client to potentially retry
|
|
453
480
|
close(1008, 'Unauthorized');
|
|
454
|
-
|
|
455
|
-
}
|
|
456
|
-
else {
|
|
457
|
-
if (checkAuthorization()) {
|
|
458
|
-
onWSMessage(body); // continue on, now that authorization succeeded
|
|
481
|
+
return;
|
|
459
482
|
}
|
|
483
|
+
if (!checkAuthorization())
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
else if (!checkAuthorization()) {
|
|
487
|
+
return;
|
|
460
488
|
}
|
|
461
|
-
|
|
489
|
+
// fall through to handle this message now that authorization succeeded
|
|
462
490
|
}
|
|
463
491
|
if (!authorization)
|
|
464
492
|
return;
|
|
@@ -506,9 +534,18 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
506
534
|
if (checkDatabaseAccess(databaseName))
|
|
507
535
|
sendDBSchema(databaseName);
|
|
508
536
|
});
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
537
|
+
// onWSMessage is async, so the WS may have already closed by the time we get
|
|
538
|
+
// here — in that case 'close' has fired and adding the cleanup listener now
|
|
539
|
+
// would silently leak. Drop the registration immediately.
|
|
540
|
+
if (wsClosed) {
|
|
541
|
+
schemaUpdateListener.remove();
|
|
542
|
+
schemaUpdateListener = undefined;
|
|
543
|
+
}
|
|
544
|
+
else {
|
|
545
|
+
ws.on('close', () => {
|
|
546
|
+
schemaUpdateListener?.remove();
|
|
547
|
+
});
|
|
548
|
+
}
|
|
512
549
|
}
|
|
513
550
|
}
|
|
514
551
|
catch (error) {
|
|
@@ -695,8 +732,21 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
695
732
|
if (stream.connectedToBlob)
|
|
696
733
|
blobsInFlight.delete(fileId);
|
|
697
734
|
}
|
|
698
|
-
else
|
|
699
|
-
|
|
735
|
+
else if (!stream.write(blobBody)) {
|
|
736
|
+
// The PassThrough's internal queue is over its HWM, meaning the downstream
|
|
737
|
+
// file write (via pipeline in saveBlob) can't keep up. Pause the WS until the
|
|
738
|
+
// stream drains so blob chunks don't accumulate in memory faster than they
|
|
739
|
+
// can be flushed to disk. Also listen for 'close' so a destroyed stream
|
|
740
|
+
// (e.g. saveBlob error) doesn't strand the pause reason.
|
|
741
|
+
addPauseReason();
|
|
742
|
+
const release = () => {
|
|
743
|
+
stream.off('drain', release);
|
|
744
|
+
stream.off('close', release);
|
|
745
|
+
removePauseReason();
|
|
746
|
+
};
|
|
747
|
+
stream.on('drain', release);
|
|
748
|
+
stream.on('close', release);
|
|
749
|
+
}
|
|
700
750
|
}
|
|
701
751
|
catch (error) {
|
|
702
752
|
logger.error?.(`Error receiving blob for ${stream.recordId} from ${remoteNodeName} and streaming to storage`, error);
|
|
@@ -1186,6 +1236,16 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
1186
1236
|
close();
|
|
1187
1237
|
}
|
|
1188
1238
|
});
|
|
1239
|
+
// We are inside an async .then(); if the WS closed while waiting for it to
|
|
1240
|
+
// resolve, attaching a 'close' handler now will not fire and the listeners
|
|
1241
|
+
// above would stay subscribed on the global databaseEventsEmitter forever.
|
|
1242
|
+
if (wsClosed) {
|
|
1243
|
+
schemaUpdateListener.remove();
|
|
1244
|
+
dbRemovalListener.remove();
|
|
1245
|
+
schemaUpdateListener = undefined;
|
|
1246
|
+
dbRemovalListener = undefined;
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1189
1249
|
ws.on('close', () => {
|
|
1190
1250
|
schemaUpdateListener?.remove();
|
|
1191
1251
|
dbRemovalListener?.remove();
|
|
@@ -1382,6 +1442,20 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
1382
1442
|
// TODO: Once it is committed, also record the localtime in the table with symbol metadata, so we can resume from that point
|
|
1383
1443
|
logger.debug?.(connectionId, 'received replication message', auditRecord.type, 'id', event.id, 'version', new Date(auditRecord.version), 'nodeId', event.nodeId);
|
|
1384
1444
|
tableSubscriptionToReplicator.send(event);
|
|
1445
|
+
// Per-record backpressure: a single large WS message can synchronously decode
|
|
1446
|
+
// thousands of records, each holding a decoded value object and a closure over
|
|
1447
|
+
// the source buffer. Without yielding here the consumer can never drain the
|
|
1448
|
+
// queue mid-message and the worker heap balloons until it OOMs.
|
|
1449
|
+
const queueLength = tableSubscriptionToReplicator.queue?.length ?? 0;
|
|
1450
|
+
if (queueLength > RECEIVE_EVENT_HIGH_WATER_MARK) {
|
|
1451
|
+
addPauseReason();
|
|
1452
|
+
try {
|
|
1453
|
+
await tableSubscriptionToReplicator.waitForDrain();
|
|
1454
|
+
}
|
|
1455
|
+
finally {
|
|
1456
|
+
removePauseReason();
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1385
1459
|
}
|
|
1386
1460
|
decoder.position = start + eventLength;
|
|
1387
1461
|
} while (decoder.position < body.byteLength);
|
|
@@ -1389,9 +1463,9 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
1389
1463
|
if (databaseName !== 'system') {
|
|
1390
1464
|
(0, write_ts_1.recordAction)(body.byteLength, 'bytes-received', `${remoteNodeName}.${databaseName}.${event?.table || 'unknown_table'}`, 'replication', 'ingest');
|
|
1391
1465
|
}
|
|
1392
|
-
if (outstandingCommits > MAX_OUTSTANDING_COMMITS && !
|
|
1393
|
-
|
|
1394
|
-
|
|
1466
|
+
if (outstandingCommits > MAX_OUTSTANDING_COMMITS && !commitBacklogPaused) {
|
|
1467
|
+
commitBacklogPaused = true;
|
|
1468
|
+
addPauseReason();
|
|
1395
1469
|
logger.debug?.(`Commit backlog causing replication back-pressure, requesting that ${remoteNodeName} pause replication`);
|
|
1396
1470
|
}
|
|
1397
1471
|
tableSubscriptionToReplicator.send({
|
|
@@ -1406,9 +1480,9 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
1406
1480
|
}
|
|
1407
1481
|
}
|
|
1408
1482
|
outstandingCommits--;
|
|
1409
|
-
if (
|
|
1410
|
-
|
|
1411
|
-
|
|
1483
|
+
if (commitBacklogPaused) {
|
|
1484
|
+
commitBacklogPaused = false;
|
|
1485
|
+
removePauseReason();
|
|
1412
1486
|
logger.debug?.(`Replication resuming ${remoteNodeName}`);
|
|
1413
1487
|
}
|
|
1414
1488
|
// if there are outstanding blobs to finish writing, delay commit receipts until they are finished (so that if we are interrupting
|
|
@@ -1458,6 +1532,7 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
1458
1532
|
});
|
|
1459
1533
|
ws.on('close', (code, reasonBuffer) => {
|
|
1460
1534
|
// cleanup
|
|
1535
|
+
wsClosed = true;
|
|
1461
1536
|
clearInterval(sendPingInterval);
|
|
1462
1537
|
clearTimeout(receivePingTimer);
|
|
1463
1538
|
clearInterval(blobsTimer);
|
|
@@ -1576,16 +1651,22 @@ function replicateOverWS(ws, options, authorization) {
|
|
|
1576
1651
|
// we would skip this
|
|
1577
1652
|
const finished = (0, blob_ts_1.decodeFromDatabase)(() => (0, blob_ts_1.saveBlob)(localBlob).saving, tableSubscriptionToReplicator.auditStore?.rootStore);
|
|
1578
1653
|
if (finished) {
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
finished
|
|
1654
|
+
// We log the rejection via .catch() and also need the resulting promise — not the
|
|
1655
|
+
// raw `finished` — to be what we hand to `Promise.all(outstandingBlobsToFinish)` in
|
|
1656
|
+
// the end_txn onCommit path below. If we pushed `finished` directly, a save
|
|
1657
|
+
// rejection would surface to that `await Promise.all(...)` as an unhandled error
|
|
1658
|
+
// even though we already logged it here, and it would escape onCommit as an
|
|
1659
|
+
// uncaughtException — observed in prod as ~35/sec ENOENT spam during catch-up.
|
|
1660
|
+
const tracked = finished
|
|
1582
1661
|
.catch((err) => logger.error?.(`Blob save failed for ${blobId} from ${remoteNodeName}`, err))
|
|
1583
1662
|
.finally(() => {
|
|
1584
1663
|
logger.debug?.(`Finished receiving blob stream ${blobId}`);
|
|
1585
|
-
const index = outstandingBlobsToFinish.indexOf(
|
|
1664
|
+
const index = outstandingBlobsToFinish.indexOf(tracked);
|
|
1586
1665
|
if (index > -1)
|
|
1587
1666
|
outstandingBlobsToFinish.splice(index, 1);
|
|
1588
1667
|
});
|
|
1668
|
+
tracked.blobId = blobId;
|
|
1669
|
+
outstandingBlobsToFinish.push(tracked);
|
|
1589
1670
|
}
|
|
1590
1671
|
return localBlob;
|
|
1591
1672
|
}
|