@usherlabs/cex-broker 0.2.38 → 0.2.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/commands/cli.js +1794 -773
- package/dist/handlers/subscribe/handler.d.ts +2 -0
- package/dist/helpers/binance-user-data-stream.d.ts +12 -0
- package/dist/helpers/broker-execution-archive/index.d.ts +1 -1
- package/dist/helpers/broker-execution-archive/rows.d.ts +1 -0
- package/dist/helpers/broker-execution-archive/writer.d.ts +30 -0
- package/dist/helpers/deposit-archive-poller.d.ts +1 -0
- package/dist/helpers/market-data-archive/capture-context.d.ts +24 -3
- package/dist/helpers/market-data-archive/index.d.ts +1 -1
- package/dist/helpers/stream-health-publisher.d.ts +41 -0
- package/dist/helpers/user-data-stream-supervisor.d.ts +27 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1794 -773
- package/dist/index.js.map +18 -16
- package/dist/server.d.ts +2 -1
- package/package.json +10 -2
package/dist/commands/cli.js
CHANGED
|
@@ -6,25 +6,43 @@ var __getProtoOf = Object.getPrototypeOf;
|
|
|
6
6
|
var __defProp = Object.defineProperty;
|
|
7
7
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
8
8
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
function __accessProp(key) {
|
|
10
|
+
return this[key];
|
|
11
|
+
}
|
|
12
|
+
var __toESMCache_node;
|
|
13
|
+
var __toESMCache_esm;
|
|
9
14
|
var __toESM = (mod, isNodeMode, target) => {
|
|
15
|
+
var canCache = mod != null && typeof mod === "object";
|
|
16
|
+
if (canCache) {
|
|
17
|
+
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
18
|
+
var cached = cache.get(mod);
|
|
19
|
+
if (cached)
|
|
20
|
+
return cached;
|
|
21
|
+
}
|
|
10
22
|
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
11
23
|
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
12
24
|
for (let key of __getOwnPropNames(mod))
|
|
13
25
|
if (!__hasOwnProp.call(to, key))
|
|
14
26
|
__defProp(to, key, {
|
|
15
|
-
get: (
|
|
27
|
+
get: __accessProp.bind(mod, key),
|
|
16
28
|
enumerable: true
|
|
17
29
|
});
|
|
30
|
+
if (canCache)
|
|
31
|
+
cache.set(mod, to);
|
|
18
32
|
return to;
|
|
19
33
|
};
|
|
20
34
|
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
35
|
+
var __returnValue = (v) => v;
|
|
36
|
+
function __exportSetter(name, newValue) {
|
|
37
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
38
|
+
}
|
|
21
39
|
var __export = (target, all) => {
|
|
22
40
|
for (var name in all)
|
|
23
41
|
__defProp(target, name, {
|
|
24
42
|
get: all[name],
|
|
25
43
|
enumerable: true,
|
|
26
44
|
configurable: true,
|
|
27
|
-
set: (
|
|
45
|
+
set: __exportSetter.bind(all, name)
|
|
28
46
|
});
|
|
29
47
|
};
|
|
30
48
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
@@ -997,7 +1015,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
997
1015
|
this._exitCallback = (err) => {
|
|
998
1016
|
if (err.code !== "commander.executeSubCommandAsync") {
|
|
999
1017
|
throw err;
|
|
1000
|
-
}
|
|
1018
|
+
}
|
|
1001
1019
|
};
|
|
1002
1020
|
}
|
|
1003
1021
|
return this;
|
|
@@ -50030,7 +50048,7 @@ var require_tracestate_impl = __commonJS((exports) => {
|
|
|
50030
50048
|
const value = listMember.slice(i2 + 1, part.length);
|
|
50031
50049
|
if ((0, tracestate_validators_1.validateKey)(key) && (0, tracestate_validators_1.validateValue)(value)) {
|
|
50032
50050
|
agg.set(key, value);
|
|
50033
|
-
}
|
|
50051
|
+
}
|
|
50034
50052
|
}
|
|
50035
50053
|
return agg;
|
|
50036
50054
|
}, new Map);
|
|
@@ -59163,7 +59181,7 @@ var require_TraceState = __commonJS((exports) => {
|
|
|
59163
59181
|
const value = listMember.slice(i2 + 1, part.length);
|
|
59164
59182
|
if ((0, validators_1.validateKey)(key) && (0, validators_1.validateValue)(value)) {
|
|
59165
59183
|
agg.set(key, value);
|
|
59166
|
-
}
|
|
59184
|
+
}
|
|
59167
59185
|
}
|
|
59168
59186
|
return agg;
|
|
59169
59187
|
}, new Map);
|
|
@@ -315706,7 +315724,9 @@ function buildMarketMetadataSnapshotRow(input) {
|
|
|
315706
315724
|
// src/helpers/broker-execution-archive/writer.ts
|
|
315707
315725
|
var import_api_logs2 = __toESM(require_src7(), 1);
|
|
315708
315726
|
import { closeSync, fsyncSync, openSync, writeSync } from "node:fs";
|
|
315709
|
-
import {
|
|
315727
|
+
import {
|
|
315728
|
+
request as httpRequest2
|
|
315729
|
+
} from "node:http";
|
|
315710
315730
|
import { request as httpsRequest2 } from "node:https";
|
|
315711
315731
|
var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
|
|
315712
315732
|
"broker_execution.order_events",
|
|
@@ -315779,6 +315799,7 @@ class BrokerExecutionArchiver {
|
|
|
315779
315799
|
flushIntervalMs;
|
|
315780
315800
|
forwarderTimeoutMs;
|
|
315781
315801
|
queue = [];
|
|
315802
|
+
enqueueTimes = new WeakMap;
|
|
315782
315803
|
stats = {
|
|
315783
315804
|
enqueued: 0,
|
|
315784
315805
|
shed: 0,
|
|
@@ -315787,6 +315808,16 @@ class BrokerExecutionArchiver {
|
|
|
315787
315808
|
};
|
|
315788
315809
|
flushTimer = null;
|
|
315789
315810
|
flushInFlight = null;
|
|
315811
|
+
inFlightBatch = null;
|
|
315812
|
+
inFlightStartedAtMs = null;
|
|
315813
|
+
lastFailureAtMs = null;
|
|
315814
|
+
lastShedAtMs = null;
|
|
315815
|
+
lastSuccessAtMs = null;
|
|
315816
|
+
lastFailureEventSequence = null;
|
|
315817
|
+
lastShedEventSequence = null;
|
|
315818
|
+
lastSuccessEventSequence = null;
|
|
315819
|
+
lastSinkLatencyMs = null;
|
|
315820
|
+
healthEventSequence = 0;
|
|
315790
315821
|
lastShedWarnAtMs = 0;
|
|
315791
315822
|
closing = false;
|
|
315792
315823
|
closed = false;
|
|
@@ -315820,6 +315851,7 @@ class BrokerExecutionArchiver {
|
|
|
315820
315851
|
}
|
|
315821
315852
|
try {
|
|
315822
315853
|
this.flushTimer = setInterval(() => {
|
|
315854
|
+
this.emitArchiveHealthMetrics();
|
|
315823
315855
|
this.flush();
|
|
315824
315856
|
}, this.flushIntervalMs);
|
|
315825
315857
|
this.flushTimer.unref?.();
|
|
@@ -315863,32 +315895,34 @@ class BrokerExecutionArchiver {
|
|
|
315863
315895
|
row: { ...row.row, source: this.source }
|
|
315864
315896
|
};
|
|
315865
315897
|
if (this.queue.length >= this.maxQueueSize) {
|
|
315866
|
-
const
|
|
315867
|
-
if (
|
|
315868
|
-
this.appendLossRecords([
|
|
315898
|
+
const shedEntry = this.queue[0];
|
|
315899
|
+
if (shedEntry) {
|
|
315900
|
+
this.appendLossRecords([shedEntry], "queue_shed");
|
|
315869
315901
|
this.queue.shift();
|
|
315870
315902
|
}
|
|
315871
315903
|
this.stats.shed += 1;
|
|
315904
|
+
this.recordHealthEvent("shed");
|
|
315872
315905
|
this.recordArchiveMetric("cex_archive_rows_shed_total", {
|
|
315873
|
-
table:
|
|
315906
|
+
table: shedEntry?.table ?? "unknown",
|
|
315874
315907
|
source: this.source,
|
|
315875
|
-
feed:
|
|
315908
|
+
feed: shedEntry ? archiveFeed(shedEntry) : "NON_MARKET"
|
|
315876
315909
|
});
|
|
315877
315910
|
this.recordArchiveMetric("cex_archive_queue_saturated_rows_total", {
|
|
315878
|
-
table:
|
|
315911
|
+
table: shedEntry?.table ?? "unknown",
|
|
315879
315912
|
source: this.source,
|
|
315880
|
-
feed:
|
|
315913
|
+
feed: shedEntry ? archiveFeed(shedEntry) : "NON_MARKET"
|
|
315881
315914
|
});
|
|
315882
315915
|
const now3 = Date.now();
|
|
315883
315916
|
if (now3 - this.lastShedWarnAtMs >= SHED_WARN_INTERVAL_MS) {
|
|
315884
315917
|
log.warn("Archive queue full: shedding oldest rows", {
|
|
315885
315918
|
shed_total: this.stats.shed,
|
|
315886
315919
|
queue_max: this.maxQueueSize,
|
|
315887
|
-
table:
|
|
315920
|
+
table: shedEntry?.table ?? "unknown"
|
|
315888
315921
|
});
|
|
315889
315922
|
this.lastShedWarnAtMs = now3;
|
|
315890
315923
|
}
|
|
315891
315924
|
}
|
|
315925
|
+
this.enqueueTimes.set(archiveRow, Date.now());
|
|
315892
315926
|
this.queue.push(archiveRow);
|
|
315893
315927
|
this.stats.enqueued += 1;
|
|
315894
315928
|
this.recordArchiveMetric("cex_archive_rows_enqueued_total", {
|
|
@@ -315896,6 +315930,7 @@ class BrokerExecutionArchiver {
|
|
|
315896
315930
|
source: this.source,
|
|
315897
315931
|
feed: archiveFeed(archiveRow)
|
|
315898
315932
|
});
|
|
315933
|
+
this.emitArchiveHealthMetrics();
|
|
315899
315934
|
if (this.queue.length >= this.batchSize) {
|
|
315900
315935
|
this.flush();
|
|
315901
315936
|
}
|
|
@@ -315910,14 +315945,19 @@ class BrokerExecutionArchiver {
|
|
|
315910
315945
|
if (this.flushInFlight) {
|
|
315911
315946
|
return this.flushInFlight;
|
|
315912
315947
|
}
|
|
315948
|
+
const inFlightState = {};
|
|
315913
315949
|
const inFlight = this.flushBatch().then(() => {
|
|
315914
315950
|
return;
|
|
315915
315951
|
}).finally(() => {
|
|
315952
|
+
if (this.flushInFlight !== inFlightState.promise) {
|
|
315953
|
+
return;
|
|
315954
|
+
}
|
|
315916
315955
|
this.flushInFlight = null;
|
|
315917
315956
|
if (!this.closed && !this.closing && this.enabled && this.queue.length >= this.batchSize) {
|
|
315918
315957
|
queueMicrotask(() => void this.flush());
|
|
315919
315958
|
}
|
|
315920
315959
|
});
|
|
315960
|
+
inFlightState.promise = inFlight;
|
|
315921
315961
|
this.flushInFlight = inFlight;
|
|
315922
315962
|
return inFlight;
|
|
315923
315963
|
}
|
|
@@ -315964,6 +316004,79 @@ class BrokerExecutionArchiver {
|
|
|
315964
316004
|
getQueueDepth() {
|
|
315965
316005
|
return this.queue.length;
|
|
315966
316006
|
}
|
|
316007
|
+
getHealthSnapshot() {
|
|
316008
|
+
const now3 = Date.now();
|
|
316009
|
+
const oldestPendingAtMs = this.oldestPendingEnqueueAtMs();
|
|
316010
|
+
const oldestPendingAgeMs = oldestPendingAtMs === null ? 0 : Math.max(0, now3 - oldestPendingAtMs);
|
|
316011
|
+
const inFlightAgeMs = this.inFlightStartedAtMs === null ? 0 : Math.max(0, now3 - this.inFlightStartedAtMs);
|
|
316012
|
+
const lastUnhealthyEventSequence = Math.max(this.lastFailureEventSequence ?? Number.NEGATIVE_INFINITY, this.lastShedEventSequence ?? Number.NEGATIVE_INFINITY);
|
|
316013
|
+
const recoveredFromEvents = lastUnhealthyEventSequence === Number.NEGATIVE_INFINITY || this.lastSuccessEventSequence !== null && this.lastSuccessEventSequence > lastUnhealthyEventSequence;
|
|
316014
|
+
const healthy = this.enabled && !this.closing && !this.closed && this.queue.length < this.maxQueueSize && oldestPendingAgeMs <= this.forwarderTimeoutMs && inFlightAgeMs <= this.forwarderTimeoutMs && recoveredFromEvents;
|
|
316015
|
+
return {
|
|
316016
|
+
healthy,
|
|
316017
|
+
queue_depth: this.queue.length,
|
|
316018
|
+
oldest_pending_age_ms: oldestPendingAgeMs,
|
|
316019
|
+
shed_total: this.stats.shed,
|
|
316020
|
+
last_failure_at: this.lastFailureAtMs,
|
|
316021
|
+
last_shed_at: this.lastShedAtMs,
|
|
316022
|
+
last_success_at: this.lastSuccessAtMs,
|
|
316023
|
+
sink_latency_ms: this.lastSinkLatencyMs,
|
|
316024
|
+
sink_errors_total: this.stats.forwarderFailures,
|
|
316025
|
+
in_flight: this.inFlightBatch !== null,
|
|
316026
|
+
in_flight_age_ms: inFlightAgeMs
|
|
316027
|
+
};
|
|
316028
|
+
}
|
|
316029
|
+
oldestPendingEnqueueAtMs() {
|
|
316030
|
+
let oldest = null;
|
|
316031
|
+
for (const entry of this.queue) {
|
|
316032
|
+
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
316033
|
+
if (enqueuedAtMs === undefined) {
|
|
316034
|
+
continue;
|
|
316035
|
+
}
|
|
316036
|
+
oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
|
|
316037
|
+
}
|
|
316038
|
+
for (const entry of this.inFlightBatch ?? []) {
|
|
316039
|
+
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
316040
|
+
if (enqueuedAtMs === undefined) {
|
|
316041
|
+
continue;
|
|
316042
|
+
}
|
|
316043
|
+
oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
|
|
316044
|
+
}
|
|
316045
|
+
return oldest;
|
|
316046
|
+
}
|
|
316047
|
+
recordHealthEvent(event) {
|
|
316048
|
+
const eventSequence = ++this.healthEventSequence;
|
|
316049
|
+
const eventTimestampMs = Date.now();
|
|
316050
|
+
switch (event) {
|
|
316051
|
+
case "failure":
|
|
316052
|
+
this.lastFailureAtMs = eventTimestampMs;
|
|
316053
|
+
this.lastFailureEventSequence = eventSequence;
|
|
316054
|
+
return;
|
|
316055
|
+
case "shed":
|
|
316056
|
+
this.lastShedAtMs = eventTimestampMs;
|
|
316057
|
+
this.lastShedEventSequence = eventSequence;
|
|
316058
|
+
return;
|
|
316059
|
+
case "success":
|
|
316060
|
+
this.lastSuccessAtMs = eventTimestampMs;
|
|
316061
|
+
this.lastSuccessEventSequence = eventSequence;
|
|
316062
|
+
return;
|
|
316063
|
+
}
|
|
316064
|
+
}
|
|
316065
|
+
emitArchiveHealthMetrics() {
|
|
316066
|
+
const snapshot = this.getHealthSnapshot();
|
|
316067
|
+
const labels = { source: this.source };
|
|
316068
|
+
this.recordArchiveObservableGauge("cex_archive_health", snapshot.healthy ? 1 : 0, labels);
|
|
316069
|
+
this.recordArchiveGauge("cex_archive_queue_depth", snapshot.queue_depth, labels);
|
|
316070
|
+
this.recordArchiveGauge("cex_archive_oldest_pending_age_ms", snapshot.oldest_pending_age_ms, labels);
|
|
316071
|
+
this.recordArchiveGauge("cex_archive_shed_total", snapshot.shed_total, labels);
|
|
316072
|
+
this.recordArchiveGauge("cex_archive_last_failure_at", snapshot.last_failure_at ?? 0, labels);
|
|
316073
|
+
this.recordArchiveGauge("cex_archive_last_shed_at", snapshot.last_shed_at ?? 0, labels);
|
|
316074
|
+
this.recordArchiveGauge("cex_archive_last_success_at", snapshot.last_success_at ?? 0, labels);
|
|
316075
|
+
this.recordArchiveGauge("cex_archive_sink_latency_ms", snapshot.sink_latency_ms ?? 0, labels);
|
|
316076
|
+
this.recordArchiveGauge("cex_archive_sink_errors_total", snapshot.sink_errors_total, labels);
|
|
316077
|
+
this.recordArchiveGauge("cex_archive_in_flight", snapshot.in_flight ? 1 : 0, labels);
|
|
316078
|
+
this.recordArchiveGauge("cex_archive_in_flight_age_ms", snapshot.in_flight_age_ms, labels);
|
|
316079
|
+
}
|
|
315967
316080
|
closeLossJournal() {
|
|
315968
316081
|
if (this.deadLetterFd === undefined) {
|
|
315969
316082
|
return;
|
|
@@ -315986,8 +316099,9 @@ class BrokerExecutionArchiver {
|
|
|
315986
316099
|
this.appendLossRecords([dropped], "queue_shed");
|
|
315987
316100
|
this.queue.shift();
|
|
315988
316101
|
this.stats.shed += 1;
|
|
316102
|
+
this.recordHealthEvent("shed");
|
|
315989
316103
|
this.recordArchiveMetric("cex_archive_rows_shed_total", {
|
|
315990
|
-
table: dropped
|
|
316104
|
+
table: dropped.table,
|
|
315991
316105
|
source: this.source,
|
|
315992
316106
|
feed: archiveFeed(dropped)
|
|
315993
316107
|
});
|
|
@@ -316046,6 +316160,8 @@ class BrokerExecutionArchiver {
|
|
|
316046
316160
|
if (batch.length === 0) {
|
|
316047
316161
|
return true;
|
|
316048
316162
|
}
|
|
316163
|
+
this.inFlightBatch = batch;
|
|
316164
|
+
this.inFlightStartedAtMs = Date.now();
|
|
316049
316165
|
for (const entry of batch) {
|
|
316050
316166
|
if (isBrokerExecutionArchiveTable(entry.table)) {
|
|
316051
316167
|
this.emitOtelLog(entry);
|
|
@@ -316056,8 +316172,15 @@ class BrokerExecutionArchiver {
|
|
|
316056
316172
|
await this.postToForwarder(batch);
|
|
316057
316173
|
} catch (error) {
|
|
316058
316174
|
this.stats.forwarderFailures += 1;
|
|
316059
|
-
this.
|
|
316060
|
-
this.
|
|
316175
|
+
this.recordHealthEvent("failure");
|
|
316176
|
+
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
316177
|
+
this.queue.unshift(...batch);
|
|
316178
|
+
try {
|
|
316179
|
+
this.enforceQueueBound();
|
|
316180
|
+
} finally {
|
|
316181
|
+
this.clearInFlightBatch(batch);
|
|
316182
|
+
this.emitArchiveHealthMetrics();
|
|
316183
|
+
}
|
|
316061
316184
|
this.recordArchiveMetric("cex_archive_forwarder_failures_total", {
|
|
316062
316185
|
count: batch.length
|
|
316063
316186
|
});
|
|
@@ -316066,14 +316189,26 @@ class BrokerExecutionArchiver {
|
|
|
316066
316189
|
}
|
|
316067
316190
|
}
|
|
316068
316191
|
this.stats.flushed += batch.length;
|
|
316192
|
+
this.recordHealthEvent("success");
|
|
316193
|
+
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
316194
|
+
this.clearInFlightBatch(batch);
|
|
316069
316195
|
this.recordFlushHealth(batch);
|
|
316196
|
+
this.emitArchiveHealthMetrics();
|
|
316070
316197
|
return true;
|
|
316071
316198
|
}
|
|
316199
|
+
clearInFlightBatch(batch) {
|
|
316200
|
+
if (this.inFlightBatch !== batch) {
|
|
316201
|
+
return;
|
|
316202
|
+
}
|
|
316203
|
+
this.inFlightBatch = null;
|
|
316204
|
+
this.inFlightStartedAtMs = null;
|
|
316205
|
+
}
|
|
316072
316206
|
recordFlushHealth(batch) {
|
|
316073
316207
|
const countByTable = new Map;
|
|
316074
316208
|
for (const entry of batch) {
|
|
316075
|
-
const
|
|
316076
|
-
const
|
|
316209
|
+
const row = entry;
|
|
316210
|
+
const key = `${row.table}\x00${archiveFeed(row)}`;
|
|
316211
|
+
const grouped = countByTable.get(key) ?? { row, count: 0 };
|
|
316077
316212
|
grouped.count += 1;
|
|
316078
316213
|
countByTable.set(key, grouped);
|
|
316079
316214
|
}
|
|
@@ -316091,9 +316226,14 @@ class BrokerExecutionArchiver {
|
|
|
316091
316226
|
await this.otelMetrics?.recordCounter(metricName, value, labels);
|
|
316092
316227
|
} catch {}
|
|
316093
316228
|
}
|
|
316094
|
-
async recordArchiveGauge(metricName, value) {
|
|
316229
|
+
async recordArchiveGauge(metricName, value, labels = {}) {
|
|
316230
|
+
try {
|
|
316231
|
+
await this.otelMetrics?.recordGauge(metricName, value, labels);
|
|
316232
|
+
} catch {}
|
|
316233
|
+
}
|
|
316234
|
+
async recordArchiveObservableGauge(metricName, value, labels) {
|
|
316095
316235
|
try {
|
|
316096
|
-
await this.otelMetrics?.
|
|
316236
|
+
await this.otelMetrics?.setObservableGauge(metricName, value, labels);
|
|
316097
316237
|
} catch {}
|
|
316098
316238
|
}
|
|
316099
316239
|
emitOtelLog(entry) {
|
|
@@ -316130,27 +316270,136 @@ class BrokerExecutionArchiver {
|
|
|
316130
316270
|
headers.authorization = `Bearer ${this.forwarderAuthToken}`;
|
|
316131
316271
|
}
|
|
316132
316272
|
return new Promise((resolve, reject) => {
|
|
316133
|
-
|
|
316134
|
-
|
|
316135
|
-
|
|
316136
|
-
|
|
316137
|
-
|
|
316138
|
-
|
|
316139
|
-
|
|
316140
|
-
|
|
316141
|
-
|
|
316142
|
-
|
|
316143
|
-
|
|
316144
|
-
|
|
316273
|
+
let req;
|
|
316274
|
+
let response;
|
|
316275
|
+
let settled = false;
|
|
316276
|
+
let requestClosed = false;
|
|
316277
|
+
let requestCloseFailureScheduled = false;
|
|
316278
|
+
let responseClosed = false;
|
|
316279
|
+
let responseEnded = false;
|
|
316280
|
+
let outerTimer;
|
|
316281
|
+
function cleanupRequestListeners() {
|
|
316282
|
+
req?.removeListener("error", onRequestError);
|
|
316283
|
+
req?.removeListener("timeout", onRequestTimeout);
|
|
316284
|
+
req?.removeListener("close", onRequestClose);
|
|
316285
|
+
}
|
|
316286
|
+
function cleanupResponseListeners() {
|
|
316287
|
+
response?.removeListener("data", onResponseData);
|
|
316288
|
+
response?.removeListener("end", onResponseEnd);
|
|
316289
|
+
response?.removeListener("aborted", onResponseAborted);
|
|
316290
|
+
response?.removeListener("error", onResponseError);
|
|
316291
|
+
response?.removeListener("close", onResponseClose);
|
|
316292
|
+
}
|
|
316293
|
+
function onRequestClose() {
|
|
316294
|
+
requestClosed = true;
|
|
316295
|
+
if (!settled && !response && !requestCloseFailureScheduled) {
|
|
316296
|
+
requestCloseFailureScheduled = true;
|
|
316297
|
+
queueMicrotask(() => {
|
|
316298
|
+
if (!settled && !response) {
|
|
316299
|
+
settle2(new Error("Archive forwarder request closed before response headers"), false, true);
|
|
316300
|
+
}
|
|
316301
|
+
});
|
|
316302
|
+
}
|
|
316303
|
+
if (settled) {
|
|
316304
|
+
cleanupRequestListeners();
|
|
316305
|
+
}
|
|
316306
|
+
}
|
|
316307
|
+
function onRequestError(error) {
|
|
316308
|
+
if (!settled) {
|
|
316309
|
+
settle2(error, true);
|
|
316310
|
+
} else {
|
|
316311
|
+
cleanupRequestListeners();
|
|
316312
|
+
}
|
|
316313
|
+
}
|
|
316314
|
+
function onRequestTimeout() {
|
|
316315
|
+
settle2(new Error("Archive forwarder socket timed out"), true);
|
|
316316
|
+
}
|
|
316317
|
+
function onResponseData() {}
|
|
316318
|
+
function onResponseEnd() {
|
|
316319
|
+
responseEnded = true;
|
|
316320
|
+
if (!response?.complete) {
|
|
316321
|
+
settle2(new Error("Archive forwarder response was incomplete"), true);
|
|
316322
|
+
return;
|
|
316323
|
+
}
|
|
316324
|
+
const status = response.statusCode ?? 0;
|
|
316325
|
+
if (status < 200 || status >= 300) {
|
|
316326
|
+
settle2(new Error(`Archive forwarder returned ${status} ${response.statusMessage ?? ""}`), false);
|
|
316327
|
+
return;
|
|
316328
|
+
}
|
|
316329
|
+
settle2(undefined, false);
|
|
316330
|
+
}
|
|
316331
|
+
function onResponseAborted() {
|
|
316332
|
+
settle2(new Error("Archive forwarder response was aborted"), true);
|
|
316333
|
+
}
|
|
316334
|
+
function onResponseError(error) {
|
|
316335
|
+
if (!settled) {
|
|
316336
|
+
settle2(error, true);
|
|
316337
|
+
}
|
|
316338
|
+
}
|
|
316339
|
+
function onResponseClose() {
|
|
316340
|
+
responseClosed = true;
|
|
316341
|
+
if (!settled && !responseEnded) {
|
|
316342
|
+
settle2(new Error("Archive forwarder response closed before completion"), true);
|
|
316343
|
+
return;
|
|
316344
|
+
}
|
|
316345
|
+
cleanupResponseListeners();
|
|
316346
|
+
}
|
|
316347
|
+
function settle2(error, destroyRequest, retainClosedRequestError = false) {
|
|
316348
|
+
if (settled) {
|
|
316349
|
+
return;
|
|
316350
|
+
}
|
|
316351
|
+
settled = true;
|
|
316352
|
+
if (outerTimer) {
|
|
316353
|
+
clearTimeout(outerTimer);
|
|
316354
|
+
outerTimer = undefined;
|
|
316355
|
+
}
|
|
316356
|
+
response?.removeListener("data", onResponseData);
|
|
316357
|
+
response?.removeListener("end", onResponseEnd);
|
|
316358
|
+
response?.removeListener("aborted", onResponseAborted);
|
|
316359
|
+
if (responseClosed) {
|
|
316360
|
+
cleanupResponseListeners();
|
|
316361
|
+
}
|
|
316362
|
+
if (destroyRequest && req && !requestClosed) {
|
|
316363
|
+
req.destroy(error);
|
|
316364
|
+
} else if (retainClosedRequestError && req && requestClosed) {
|
|
316365
|
+
req.removeListener("timeout", onRequestTimeout);
|
|
316366
|
+
req.removeListener("close", onRequestClose);
|
|
316367
|
+
} else if (!req || requestClosed) {
|
|
316368
|
+
cleanupRequestListeners();
|
|
316369
|
+
}
|
|
316370
|
+
if (error) {
|
|
316371
|
+
reject(error);
|
|
316372
|
+
} else {
|
|
316145
316373
|
resolve();
|
|
316374
|
+
}
|
|
316375
|
+
}
|
|
316376
|
+
outerTimer = setTimeout(() => {
|
|
316377
|
+
settle2(new Error("Archive forwarder request deadline exceeded"), true);
|
|
316378
|
+
}, this.forwarderTimeoutMs);
|
|
316379
|
+
try {
|
|
316380
|
+
req = doRequest(url2, {
|
|
316381
|
+
method: "POST",
|
|
316382
|
+
headers,
|
|
316383
|
+
timeout: this.forwarderTimeoutMs
|
|
316384
|
+
}, (nextResponse) => {
|
|
316385
|
+
response = nextResponse;
|
|
316386
|
+
response.on("data", onResponseData);
|
|
316387
|
+
response.on("end", onResponseEnd);
|
|
316388
|
+
response.on("aborted", onResponseAborted);
|
|
316389
|
+
response.on("error", onResponseError);
|
|
316390
|
+
response.on("close", onResponseClose);
|
|
316391
|
+
if (settled) {
|
|
316392
|
+
response.resume();
|
|
316393
|
+
}
|
|
316146
316394
|
});
|
|
316147
|
-
|
|
316148
|
-
|
|
316149
|
-
|
|
316150
|
-
req.
|
|
316151
|
-
|
|
316152
|
-
|
|
316153
|
-
|
|
316395
|
+
req.on("error", onRequestError);
|
|
316396
|
+
req.on("timeout", onRequestTimeout);
|
|
316397
|
+
req.on("close", onRequestClose);
|
|
316398
|
+
req.write(body);
|
|
316399
|
+
req.end();
|
|
316400
|
+
} catch (error) {
|
|
316401
|
+
settle2(error instanceof Error ? error : new Error(String(error)), true);
|
|
316402
|
+
}
|
|
316154
316403
|
});
|
|
316155
316404
|
}
|
|
316156
316405
|
}
|
|
@@ -316623,9 +316872,18 @@ class AccountBalanceArchivePoller {
|
|
|
316623
316872
|
// src/helpers/deposit-archive-poller.ts
|
|
316624
316873
|
var DEFAULT_CONFIG2 = {
|
|
316625
316874
|
pollIntervalMs: 60000,
|
|
316875
|
+
fetchTimeoutMs: 30000,
|
|
316626
316876
|
lookbackMs: 24 * 60 * 60 * 1000,
|
|
316627
316877
|
depositsLimit: 50
|
|
316628
316878
|
};
|
|
316879
|
+
function withTimeout(promise, timeoutMs, label) {
|
|
316880
|
+
let timer;
|
|
316881
|
+
const expiry = new Promise((_resolve, reject) => {
|
|
316882
|
+
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
316883
|
+
timer.unref?.();
|
|
316884
|
+
});
|
|
316885
|
+
return Promise.race([promise, expiry]).finally(() => clearTimeout(timer));
|
|
316886
|
+
}
|
|
316629
316887
|
var ALL_CURRENCIES_CODE = "*";
|
|
316630
316888
|
function depositTimestamp(record) {
|
|
316631
316889
|
const observedAt = depositField(record, [
|
|
@@ -316738,6 +316996,14 @@ class DepositArchivePoller {
|
|
|
316738
316996
|
return true;
|
|
316739
316997
|
}
|
|
316740
316998
|
async#pollOne(target) {
|
|
316999
|
+
let outcome = "error";
|
|
317000
|
+
try {
|
|
317001
|
+
outcome = await this.#pollTarget(target);
|
|
317002
|
+
} finally {
|
|
317003
|
+
this.params.metrics?.recordCounter("cex_deposit_poller_polls_total", 1, { exchange: target.exchangeId, outcome });
|
|
317004
|
+
}
|
|
317005
|
+
}
|
|
317006
|
+
async#pollTarget(target) {
|
|
316741
317007
|
const exchange = target.account.exchange;
|
|
316742
317008
|
const key = this.#targetKey(target);
|
|
316743
317009
|
if (typeof exchange.fetchDeposits !== "function" || exchange.has?.fetchDeposits === false) {
|
|
@@ -316748,12 +317014,12 @@ class DepositArchivePoller {
|
|
|
316748
317014
|
account: target.account.label
|
|
316749
317015
|
});
|
|
316750
317016
|
}
|
|
316751
|
-
return;
|
|
317017
|
+
return "unsupported";
|
|
316752
317018
|
}
|
|
316753
317019
|
const since = this.#cursors.get(key) ?? Date.now() - this.#config.lookbackMs;
|
|
316754
317020
|
let deposits;
|
|
316755
317021
|
try {
|
|
316756
|
-
deposits = await exchange.fetchDeposits(undefined, since, this.#config.depositsLimit);
|
|
317022
|
+
deposits = await withTimeout(exchange.fetchDeposits(undefined, since, this.#config.depositsLimit), this.#config.fetchTimeoutMs, "fetchDeposits");
|
|
316757
317023
|
} catch (error) {
|
|
316758
317024
|
this.params.metrics?.recordCounter("cex_deposit_poller_errors_total", 1, { exchange: target.exchangeId });
|
|
316759
317025
|
log.warn("Deposit archive poll failed", {
|
|
@@ -316761,10 +317027,10 @@ class DepositArchivePoller {
|
|
|
316761
317027
|
account: target.account.label,
|
|
316762
317028
|
error
|
|
316763
317029
|
});
|
|
316764
|
-
return;
|
|
317030
|
+
return "error";
|
|
316765
317031
|
}
|
|
316766
317032
|
if (!Array.isArray(deposits) || deposits.length === 0) {
|
|
316767
|
-
return;
|
|
317033
|
+
return "ok";
|
|
316768
317034
|
}
|
|
316769
317035
|
let archived = 0;
|
|
316770
317036
|
for (const deposit of deposits) {
|
|
@@ -316812,7 +317078,7 @@ class DepositArchivePoller {
|
|
|
316812
317078
|
network: network === undefined ? undefined : String(network),
|
|
316813
317079
|
externalId: depositTxid,
|
|
316814
317080
|
txid: depositTxid,
|
|
316815
|
-
exchangeTimestamp:
|
|
317081
|
+
exchangeTimestamp: normalizeTimestamp2(creditedAt),
|
|
316816
317082
|
payload: record
|
|
316817
317083
|
}
|
|
316818
317084
|
}));
|
|
@@ -316845,6 +317111,7 @@ class DepositArchivePoller {
|
|
|
316845
317111
|
this.#lastArchivedByTarget.delete(key);
|
|
316846
317112
|
}
|
|
316847
317113
|
}
|
|
317114
|
+
return "ok";
|
|
316848
317115
|
}
|
|
316849
317116
|
#targetKey(target) {
|
|
316850
317117
|
return `${target.exchangeId}|${target.account.label}|${target.code}`;
|
|
@@ -316986,6 +317253,309 @@ class FillArchivePoller {
|
|
|
316986
317253
|
}
|
|
316987
317254
|
}
|
|
316988
317255
|
|
|
317256
|
+
// src/helpers/market-data-archive/capture-contract.ts
|
|
317257
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
317258
|
+
var MARKET_CAPTURE_SCHEMA_VERSION = "1.0.0";
|
|
317259
|
+
var CHECKSUM_ALGORITHM = "sha256-canonical-json-v1";
|
|
317260
|
+
var ARCHIVE_SOURCES = ["broker_read", "broker_write"];
|
|
317261
|
+
var CAPTURE_FEEDS = [
|
|
317262
|
+
"ORDERBOOK",
|
|
317263
|
+
"TICKER",
|
|
317264
|
+
"TRADES",
|
|
317265
|
+
"OHLCV"
|
|
317266
|
+
];
|
|
317267
|
+
var SOURCE_MODES = [
|
|
317268
|
+
"broker_live_stream_v1",
|
|
317269
|
+
"broker_live_sampling_v1",
|
|
317270
|
+
"broker_current_snapshot_v1",
|
|
317271
|
+
"broker_bootstrap_fetch_v1",
|
|
317272
|
+
"external_ccxt_fallback_v1",
|
|
317273
|
+
"external_hummingbot_fallback_v1",
|
|
317274
|
+
"legacy_migration_v1"
|
|
317275
|
+
];
|
|
317276
|
+
var RAW_CAPTURE_SCOPES = [
|
|
317277
|
+
"ccxt_normalized_object",
|
|
317278
|
+
"broker_visible_payload",
|
|
317279
|
+
"exchange_wire_frame"
|
|
317280
|
+
];
|
|
317281
|
+
var CHECKSUM_FIELDS = new Set([
|
|
317282
|
+
"normalized_row_checksum",
|
|
317283
|
+
"raw_checksum",
|
|
317284
|
+
"checksum"
|
|
317285
|
+
]);
|
|
317286
|
+
function canonicalDecimal(value) {
|
|
317287
|
+
if (!Number.isFinite(value)) {
|
|
317288
|
+
throw new Error("Canonical numbers must be finite");
|
|
317289
|
+
}
|
|
317290
|
+
if (Object.is(value, -0)) {
|
|
317291
|
+
return "0";
|
|
317292
|
+
}
|
|
317293
|
+
const rendered = String(value).toLowerCase();
|
|
317294
|
+
if (!rendered.includes("e")) {
|
|
317295
|
+
return rendered;
|
|
317296
|
+
}
|
|
317297
|
+
const [coefficient = "0", exponentText = "0"] = rendered.split("e");
|
|
317298
|
+
const exponent = Number.parseInt(exponentText, 10);
|
|
317299
|
+
const negative = coefficient.startsWith("-");
|
|
317300
|
+
const unsigned = negative ? coefficient.slice(1) : coefficient;
|
|
317301
|
+
const [integer = "0", fraction = ""] = unsigned.split(".");
|
|
317302
|
+
const digits = `${integer}${fraction}`;
|
|
317303
|
+
const decimalIndex = integer.length + exponent;
|
|
317304
|
+
let result;
|
|
317305
|
+
if (decimalIndex <= 0) {
|
|
317306
|
+
result = `0.${"0".repeat(-decimalIndex)}${digits}`;
|
|
317307
|
+
} else if (decimalIndex >= digits.length) {
|
|
317308
|
+
result = `${digits}${"0".repeat(decimalIndex - digits.length)}`;
|
|
317309
|
+
} else {
|
|
317310
|
+
result = `${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
|
|
317311
|
+
}
|
|
317312
|
+
return negative ? `-${result}` : result;
|
|
317313
|
+
}
|
|
317314
|
+
function serializeCanonical(value, stack) {
|
|
317315
|
+
if (value === null)
|
|
317316
|
+
return "null";
|
|
317317
|
+
if (typeof value === "string")
|
|
317318
|
+
return JSON.stringify(value);
|
|
317319
|
+
if (typeof value === "boolean")
|
|
317320
|
+
return value ? "true" : "false";
|
|
317321
|
+
if (typeof value === "number")
|
|
317322
|
+
return canonicalDecimal(value);
|
|
317323
|
+
if (typeof value === "bigint")
|
|
317324
|
+
return value.toString(10);
|
|
317325
|
+
if (value instanceof Date) {
|
|
317326
|
+
if (Number.isNaN(value.getTime())) {
|
|
317327
|
+
throw new Error("Canonical timestamps must be valid");
|
|
317328
|
+
}
|
|
317329
|
+
return value.getTime().toString(10);
|
|
317330
|
+
}
|
|
317331
|
+
if (Array.isArray(value)) {
|
|
317332
|
+
if (stack.has(value))
|
|
317333
|
+
throw new Error("Canonical values must be acyclic");
|
|
317334
|
+
stack.add(value);
|
|
317335
|
+
const result = `[${value.map((entry) => entry === undefined ? "null" : serializeCanonical(entry, stack)).join(",")}]`;
|
|
317336
|
+
stack.delete(value);
|
|
317337
|
+
return result;
|
|
317338
|
+
}
|
|
317339
|
+
if (typeof value === "object") {
|
|
317340
|
+
if (stack.has(value))
|
|
317341
|
+
throw new Error("Canonical values must be acyclic");
|
|
317342
|
+
stack.add(value);
|
|
317343
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([left], [right]) => left.localeCompare(right));
|
|
317344
|
+
const result = `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${serializeCanonical(entry, stack)}`).join(",")}}`;
|
|
317345
|
+
stack.delete(value);
|
|
317346
|
+
return result;
|
|
317347
|
+
}
|
|
317348
|
+
throw new Error(`Unsupported canonical value type: ${typeof value}`);
|
|
317349
|
+
}
|
|
317350
|
+
function canonicalSerialize(value) {
|
|
317351
|
+
return serializeCanonical(value, new Set);
|
|
317352
|
+
}
|
|
317353
|
+
function omitChecksumFields(value) {
|
|
317354
|
+
if (Array.isArray(value))
|
|
317355
|
+
return value.map(omitChecksumFields);
|
|
317356
|
+
if (value && typeof value === "object" && !(value instanceof Date)) {
|
|
317357
|
+
return Object.fromEntries(Object.entries(value).filter(([key]) => !CHECKSUM_FIELDS.has(key)).map(([key, entry]) => [key, omitChecksumFields(entry)]));
|
|
317358
|
+
}
|
|
317359
|
+
return value;
|
|
317360
|
+
}
|
|
317361
|
+
function sha256Canonical(value) {
|
|
317362
|
+
return createHash3("sha256").update(canonicalSerialize(omitChecksumFields(value))).digest("hex");
|
|
317363
|
+
}
|
|
317364
|
+
function normalizeTimestampMs(value, field) {
|
|
317365
|
+
let timestamp;
|
|
317366
|
+
if (value instanceof Date) {
|
|
317367
|
+
timestamp = value.getTime();
|
|
317368
|
+
} else if (typeof value === "number") {
|
|
317369
|
+
timestamp = value;
|
|
317370
|
+
} else if (typeof value === "string" && /^\d+$/.test(value.trim())) {
|
|
317371
|
+
timestamp = Number(value.trim());
|
|
317372
|
+
} else if (typeof value === "string") {
|
|
317373
|
+
timestamp = Date.parse(value);
|
|
317374
|
+
} else {
|
|
317375
|
+
timestamp = Number.NaN;
|
|
317376
|
+
}
|
|
317377
|
+
if (!Number.isSafeInteger(timestamp) || timestamp < 0) {
|
|
317378
|
+
throw new Error(`${field} must be a non-negative millisecond timestamp`);
|
|
317379
|
+
}
|
|
317380
|
+
return timestamp;
|
|
317381
|
+
}
|
|
317382
|
+
function assertCaptureContext(context2) {
|
|
317383
|
+
if (!ARCHIVE_SOURCES.includes(context2.source)) {
|
|
317384
|
+
throw new Error(`Unsupported archive source: ${context2.source}`);
|
|
317385
|
+
}
|
|
317386
|
+
if (!CAPTURE_FEEDS.includes(context2.feed)) {
|
|
317387
|
+
throw new Error(`Unsupported capture feed: ${context2.feed}`);
|
|
317388
|
+
}
|
|
317389
|
+
if (!SOURCE_MODES.includes(context2.sourceMode)) {
|
|
317390
|
+
throw new Error(`Unsupported source mode: ${context2.sourceMode}`);
|
|
317391
|
+
}
|
|
317392
|
+
for (const [field, value] of [
|
|
317393
|
+
["deployment_id", context2.deploymentId],
|
|
317394
|
+
["capture_bundle_id", context2.captureBundleId],
|
|
317395
|
+
["exchange", context2.exchange],
|
|
317396
|
+
["symbol", context2.symbol],
|
|
317397
|
+
["provider", context2.provider]
|
|
317398
|
+
]) {
|
|
317399
|
+
if (!value.trim())
|
|
317400
|
+
throw new Error(`${field} must not be empty`);
|
|
317401
|
+
}
|
|
317402
|
+
}
|
|
317403
|
+
function createRawCapture(context2, input) {
|
|
317404
|
+
assertCaptureContext(context2);
|
|
317405
|
+
if (!RAW_CAPTURE_SCOPES.includes(input.scope)) {
|
|
317406
|
+
throw new Error(`Unsupported raw capture scope: ${input.scope}`);
|
|
317407
|
+
}
|
|
317408
|
+
const eventTimeMs = normalizeTimestampMs(input.eventTimeMs, "event_time_ms");
|
|
317409
|
+
const receivedTimeMs = normalizeTimestampMs(input.receivedTimeMs, "received_time_ms");
|
|
317410
|
+
const redactedPayload = redactStreamPayload(input.payload);
|
|
317411
|
+
const rawChecksum = sha256Canonical(redactedPayload);
|
|
317412
|
+
const rawCaptureId = sha256Canonical({
|
|
317413
|
+
capture_bundle_id: context2.captureBundleId,
|
|
317414
|
+
exchange: context2.exchange.trim().toLowerCase(),
|
|
317415
|
+
feed: context2.feed,
|
|
317416
|
+
raw_capture_scope: input.scope,
|
|
317417
|
+
raw_payload_sha256: rawChecksum,
|
|
317418
|
+
schema_version: context2.schemaVersion,
|
|
317419
|
+
source_mode: context2.sourceMode,
|
|
317420
|
+
source_symbol: context2.symbol.trim(),
|
|
317421
|
+
source_time_ms: eventTimeMs
|
|
317422
|
+
});
|
|
317423
|
+
return {
|
|
317424
|
+
rawCaptureId,
|
|
317425
|
+
rawCaptureScope: input.scope,
|
|
317426
|
+
rawChecksum,
|
|
317427
|
+
redactedPayload,
|
|
317428
|
+
eventTimeMs,
|
|
317429
|
+
receivedTimeMs,
|
|
317430
|
+
checksumAlgorithm: context2.checksumAlgorithm
|
|
317431
|
+
};
|
|
317432
|
+
}
|
|
317433
|
+
function captureCoreFields(context2, rawCapture) {
|
|
317434
|
+
assertCaptureContext(context2);
|
|
317435
|
+
return {
|
|
317436
|
+
source: context2.source,
|
|
317437
|
+
deployment_id: context2.deploymentId,
|
|
317438
|
+
capture_bundle_id: context2.captureBundleId,
|
|
317439
|
+
exchange: context2.exchange.trim().toLowerCase(),
|
|
317440
|
+
symbol: context2.symbol.trim(),
|
|
317441
|
+
trading_pair: context2.symbol.trim().replace("/", "-"),
|
|
317442
|
+
source_symbol: context2.symbol.trim(),
|
|
317443
|
+
asset_type: context2.assetType,
|
|
317444
|
+
feed: context2.feed,
|
|
317445
|
+
provider: context2.provider,
|
|
317446
|
+
source_mode: context2.sourceMode,
|
|
317447
|
+
source_time_ms: rawCapture.eventTimeMs,
|
|
317448
|
+
received_time_ms: rawCapture.receivedTimeMs,
|
|
317449
|
+
raw_capture_id: rawCapture.rawCaptureId,
|
|
317450
|
+
raw_capture_scope: rawCapture.rawCaptureScope,
|
|
317451
|
+
schema_version: context2.schemaVersion,
|
|
317452
|
+
checksum_algorithm: context2.checksumAlgorithm,
|
|
317453
|
+
raw_checksum: rawCapture.rawChecksum,
|
|
317454
|
+
provenance_complete: context2.provenanceComplete ? 1 : 0
|
|
317455
|
+
};
|
|
317456
|
+
}
|
|
317457
|
+
|
|
317458
|
+
// src/helpers/market-data-archive/capture-context.ts
|
|
317459
|
+
function createMarketCaptureContext(input) {
|
|
317460
|
+
const environment = input.environment ?? "development";
|
|
317461
|
+
const deploymentId = input.deploymentId.trim();
|
|
317462
|
+
if (!deploymentId)
|
|
317463
|
+
throw new Error("deployment_id must not be empty");
|
|
317464
|
+
const configuredBundle = input.captureBundleId?.trim();
|
|
317465
|
+
if (environment === "production" && !configuredBundle) {
|
|
317466
|
+
throw new Error("capture_bundle_id is required for production market capture");
|
|
317467
|
+
}
|
|
317468
|
+
const exchange = input.exchange.trim().toLowerCase();
|
|
317469
|
+
const symbol = input.symbol.trim();
|
|
317470
|
+
if (!exchange || !symbol) {
|
|
317471
|
+
throw new Error("exchange and symbol are required for market capture");
|
|
317472
|
+
}
|
|
317473
|
+
return {
|
|
317474
|
+
source: input.source,
|
|
317475
|
+
deploymentId,
|
|
317476
|
+
captureBundleId: configuredBundle ?? `development:${deploymentId}`,
|
|
317477
|
+
exchange,
|
|
317478
|
+
symbol,
|
|
317479
|
+
assetType: input.assetType,
|
|
317480
|
+
feed: input.feed,
|
|
317481
|
+
provider: input.provider?.trim() || `ccxt:${exchange}`,
|
|
317482
|
+
sourceMode: input.sourceMode,
|
|
317483
|
+
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
317484
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
317485
|
+
provenanceComplete: true,
|
|
317486
|
+
timeframe: input.timeframe,
|
|
317487
|
+
accountSelector: input.accountSelector
|
|
317488
|
+
};
|
|
317489
|
+
}
|
|
317490
|
+
function resolveMarketCaptureArchiveState(input) {
|
|
317491
|
+
if (!input.archiveEnabled) {
|
|
317492
|
+
return { enabled: false, reason: "archive_disabled" };
|
|
317493
|
+
}
|
|
317494
|
+
if (!input.marketArchiveEnabled) {
|
|
317495
|
+
return { enabled: false, reason: "market_archive_disabled" };
|
|
317496
|
+
}
|
|
317497
|
+
const environment = input.environment?.trim() || "development";
|
|
317498
|
+
if (environment !== "development" && environment !== "production") {
|
|
317499
|
+
return { enabled: false, reason: "invalid_capture_environment" };
|
|
317500
|
+
}
|
|
317501
|
+
if (environment === "production") {
|
|
317502
|
+
const deploymentId = input.deploymentId?.trim();
|
|
317503
|
+
if (!deploymentId || deploymentId === "unknown") {
|
|
317504
|
+
return { enabled: false, reason: "missing_deployment_id" };
|
|
317505
|
+
}
|
|
317506
|
+
if (!input.captureBundleId?.trim()) {
|
|
317507
|
+
return { enabled: false, reason: "missing_capture_bundle_id" };
|
|
317508
|
+
}
|
|
317509
|
+
}
|
|
317510
|
+
return { enabled: true };
|
|
317511
|
+
}
|
|
317512
|
+
function assertMarketCaptureArchiveStartable(state) {
|
|
317513
|
+
if (state.enabled || state.reason === "archive_disabled" || state.reason === "market_archive_disabled") {
|
|
317514
|
+
return;
|
|
317515
|
+
}
|
|
317516
|
+
throw new Error(`Refusing to start: canonical market-data archival was requested but its capture identity is incomplete (${state.reason}). ` + "Set CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT, CEX_BROKER_DEPLOYMENT_ID and CEX_BROKER_CAPTURE_BUNDLE_ID, " + "or disable archival explicitly via CEX_BROKER_MARKET_ARCHIVE_ENABLED.");
|
|
317517
|
+
}
|
|
317518
|
+
function captureEnvironmentFromEnv(value = process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT) {
|
|
317519
|
+
const environment = value?.trim() || "development";
|
|
317520
|
+
if (environment !== "development" && environment !== "production") {
|
|
317521
|
+
throw new Error("CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT must be development or production");
|
|
317522
|
+
}
|
|
317523
|
+
return environment;
|
|
317524
|
+
}
|
|
317525
|
+
|
|
317526
|
+
// src/helpers/market-data-archive/orderbook-sampler.ts
|
|
317527
|
+
var DEFAULT_ORDERBOOK_INTERVAL_MS = 1000;
|
|
317528
|
+
function getOrderbookIntervalMs() {
|
|
317529
|
+
const raw = process.env.CEX_BROKER_ORDERBOOK_INTERVAL_MS ?? process.env.CEX_BROKER_ORDERBOOK_TOB_INTERVAL_MS;
|
|
317530
|
+
if (!raw) {
|
|
317531
|
+
return DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
317532
|
+
}
|
|
317533
|
+
const parsed = Number.parseInt(raw, 10);
|
|
317534
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
317535
|
+
}
|
|
317536
|
+
function isMarketArchiveEnabled() {
|
|
317537
|
+
return process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED !== "false";
|
|
317538
|
+
}
|
|
317539
|
+
|
|
317540
|
+
class OrderbookSampler {
|
|
317541
|
+
intervalMs;
|
|
317542
|
+
lastEmitMs = null;
|
|
317543
|
+
constructor(intervalMs = getOrderbookIntervalMs()) {
|
|
317544
|
+
this.intervalMs = intervalMs;
|
|
317545
|
+
}
|
|
317546
|
+
shouldEmit(nowMs = Date.now()) {
|
|
317547
|
+
if (this.lastEmitMs !== null && nowMs < this.lastEmitMs) {
|
|
317548
|
+
this.lastEmitMs = nowMs;
|
|
317549
|
+
return true;
|
|
317550
|
+
}
|
|
317551
|
+
if (this.lastEmitMs !== null && nowMs - this.lastEmitMs < this.intervalMs) {
|
|
317552
|
+
return false;
|
|
317553
|
+
}
|
|
317554
|
+
this.lastEmitMs = nowMs;
|
|
317555
|
+
return true;
|
|
317556
|
+
}
|
|
317557
|
+
}
|
|
317558
|
+
|
|
316989
317559
|
// src/helpers/order-activity-tracker.ts
|
|
316990
317560
|
var DEFAULT_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
|
316991
317561
|
|
|
@@ -317369,6 +317939,925 @@ function createOtelLogsFromEnv() {
|
|
|
317369
317939
|
return new OtelLogs(config);
|
|
317370
317940
|
}
|
|
317371
317941
|
|
|
317942
|
+
// src/helpers/stream-health-publisher.ts
|
|
317943
|
+
import { createHash as createHash4, randomUUID } from "node:crypto";
|
|
317944
|
+
import {
|
|
317945
|
+
closeSync as closeSync2,
|
|
317946
|
+
fsyncSync as fsyncSync2,
|
|
317947
|
+
openSync as openSync2,
|
|
317948
|
+
readFileSync,
|
|
317949
|
+
renameSync,
|
|
317950
|
+
statSync,
|
|
317951
|
+
unlinkSync,
|
|
317952
|
+
writeFileSync
|
|
317953
|
+
} from "node:fs";
|
|
317954
|
+
import { request as httpRequest3 } from "node:http";
|
|
317955
|
+
import { request as httpsRequest3 } from "node:https";
|
|
317956
|
+
import { dirname } from "node:path";
|
|
317957
|
+
var SOURCE = "broker_write";
|
|
317958
|
+
var TABLE = "broker_stream_health.snapshots";
|
|
317959
|
+
var PRODUCER_ID = "cex-broker-user-data";
|
|
317960
|
+
var STATE_VERSION = 1;
|
|
317961
|
+
var HEARTBEAT_MS = 30000;
|
|
317962
|
+
var FORWARDER_TIMEOUT_MS = 3000;
|
|
317963
|
+
var IDENTIFIER = /^[a-z0-9][a-z0-9:_-]{0,127}$/;
|
|
317964
|
+
function identifier(value, name) {
|
|
317965
|
+
const normalized = value.trim().toLowerCase();
|
|
317966
|
+
if (!IDENTIFIER.test(normalized)) {
|
|
317967
|
+
throw new Error(`${name} must be a lower-case stream-health identifier`);
|
|
317968
|
+
}
|
|
317969
|
+
return normalized;
|
|
317970
|
+
}
|
|
317971
|
+
function counter(value) {
|
|
317972
|
+
if (!/^(0|[1-9]\d*)$/.test(value)) {
|
|
317973
|
+
throw new Error("Invalid persisted stream-health counter");
|
|
317974
|
+
}
|
|
317975
|
+
return BigInt(value);
|
|
317976
|
+
}
|
|
317977
|
+
function next(value) {
|
|
317978
|
+
return (counter(value) + 1n).toString();
|
|
317979
|
+
}
|
|
317980
|
+
function key(snapshot) {
|
|
317981
|
+
return `exchange:${snapshot.exchange}|account:${snapshot.accountSelector}|stream:${snapshot.streamKind}|scope:${snapshot.accountScope}`;
|
|
317982
|
+
}
|
|
317983
|
+
function registryRevision(snapshots) {
|
|
317984
|
+
const rows = snapshots.map((snapshot) => ({
|
|
317985
|
+
exchange: snapshot.exchange,
|
|
317986
|
+
account_selector: snapshot.accountSelector,
|
|
317987
|
+
account_role: snapshot.accountRole ?? null,
|
|
317988
|
+
stream_kind: snapshot.streamKind,
|
|
317989
|
+
account_scope: snapshot.accountScope,
|
|
317990
|
+
registry_status: snapshot.registryStatus
|
|
317991
|
+
})).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
|
|
317992
|
+
return createHash4("sha256").update(JSON.stringify(rows)).digest("hex");
|
|
317993
|
+
}
|
|
317994
|
+
function validState(value) {
|
|
317995
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
317996
|
+
return false;
|
|
317997
|
+
const state = value;
|
|
317998
|
+
return state.version === STATE_VERSION && state.producerId === PRODUCER_ID && typeof state.producerEpoch === "string" && typeof state.runId === "string" && typeof state.nextBatchSequence === "string" && state.nextStreamSequences !== null && typeof state.nextStreamSequences === "object" && (state.pendingBody === undefined || typeof state.pendingBody === "string");
|
|
317999
|
+
}
|
|
318000
|
+
function forwarderPost(url2, body, authToken, timeoutMs) {
|
|
318001
|
+
const request = url2.protocol === "http:" ? httpRequest3 : httpsRequest3;
|
|
318002
|
+
const headers = {
|
|
318003
|
+
"content-type": "application/json",
|
|
318004
|
+
"content-length": Buffer.byteLength(body)
|
|
318005
|
+
};
|
|
318006
|
+
if (authToken)
|
|
318007
|
+
headers.authorization = `Bearer ${authToken}`;
|
|
318008
|
+
return new Promise((resolve, reject) => {
|
|
318009
|
+
const req = request(url2, { method: "POST", headers, timeout: timeoutMs }, (res) => {
|
|
318010
|
+
res.on("data", () => {});
|
|
318011
|
+
res.on("end", () => {
|
|
318012
|
+
const status = res.statusCode ?? 0;
|
|
318013
|
+
if (status < 200 || status >= 300) {
|
|
318014
|
+
reject(new Error(`Stream health forwarder returned ${status}`));
|
|
318015
|
+
return;
|
|
318016
|
+
}
|
|
318017
|
+
resolve();
|
|
318018
|
+
});
|
|
318019
|
+
});
|
|
318020
|
+
req.on("error", reject);
|
|
318021
|
+
req.on("timeout", () => req.destroy(new Error("Stream health forwarder timed out")));
|
|
318022
|
+
req.write(body);
|
|
318023
|
+
req.end();
|
|
318024
|
+
});
|
|
318025
|
+
}
|
|
318026
|
+
|
|
318027
|
+
class StreamHealthPublisher {
|
|
318028
|
+
#deploymentId;
|
|
318029
|
+
#statePath;
|
|
318030
|
+
#heartbeatMs;
|
|
318031
|
+
#post;
|
|
318032
|
+
#state;
|
|
318033
|
+
#advanceRun;
|
|
318034
|
+
#snapshots = [];
|
|
318035
|
+
#dirty = false;
|
|
318036
|
+
#closed = false;
|
|
318037
|
+
#pumping = null;
|
|
318038
|
+
#heartbeat = null;
|
|
318039
|
+
#retry = null;
|
|
318040
|
+
#retryAttempt = 0;
|
|
318041
|
+
constructor(options) {
|
|
318042
|
+
this.#deploymentId = identifier(options.deploymentId, "deployment_id");
|
|
318043
|
+
this.#statePath = options.statePath.trim();
|
|
318044
|
+
if (!this.#statePath) {
|
|
318045
|
+
throw new Error("CEX_BROKER_STREAM_HEALTH_STATE_PATH is required");
|
|
318046
|
+
}
|
|
318047
|
+
this.#heartbeatMs = options.heartbeatIntervalMs ?? HEARTBEAT_MS;
|
|
318048
|
+
if (!Number.isInteger(this.#heartbeatMs) || this.#heartbeatMs < 1 || this.#heartbeatMs > 60000) {
|
|
318049
|
+
throw new Error("Stream health heartbeat interval must be between 1 and 60000ms");
|
|
318050
|
+
}
|
|
318051
|
+
let url2;
|
|
318052
|
+
try {
|
|
318053
|
+
url2 = new URL(options.forwarderUrl.trim());
|
|
318054
|
+
} catch (error) {
|
|
318055
|
+
throw new Error("CEX_BROKER_ARCHIVE_FORWARDER_URL must be valid", {
|
|
318056
|
+
cause: error
|
|
318057
|
+
});
|
|
318058
|
+
}
|
|
318059
|
+
if (url2.protocol !== "http:" && url2.protocol !== "https:") {
|
|
318060
|
+
throw new Error("CEX_BROKER_ARCHIVE_FORWARDER_URL must use HTTP(S)");
|
|
318061
|
+
}
|
|
318062
|
+
this.#post = options.post ?? ((body) => forwarderPost(url2, body, options.forwarderAuthToken, options.forwarderTimeoutMs ?? FORWARDER_TIMEOUT_MS));
|
|
318063
|
+
const loaded = this.#read();
|
|
318064
|
+
this.#state = loaded ?? {
|
|
318065
|
+
version: STATE_VERSION,
|
|
318066
|
+
producerId: PRODUCER_ID,
|
|
318067
|
+
producerEpoch: "1",
|
|
318068
|
+
runId: randomUUID(),
|
|
318069
|
+
nextBatchSequence: "1",
|
|
318070
|
+
nextStreamSequences: {}
|
|
318071
|
+
};
|
|
318072
|
+
counter(this.#state.producerEpoch);
|
|
318073
|
+
counter(this.#state.nextBatchSequence);
|
|
318074
|
+
for (const value of Object.values(this.#state.nextStreamSequences))
|
|
318075
|
+
counter(value);
|
|
318076
|
+
this.#advanceRun = loaded !== null;
|
|
318077
|
+
if (!loaded)
|
|
318078
|
+
this.#persist();
|
|
318079
|
+
}
|
|
318080
|
+
start() {
|
|
318081
|
+
if (this.#closed || this.#heartbeat)
|
|
318082
|
+
return;
|
|
318083
|
+
this.#heartbeat = setInterval(() => {
|
|
318084
|
+
if (this.#snapshots.length > 0) {
|
|
318085
|
+
this.#dirty = true;
|
|
318086
|
+
this.#schedule();
|
|
318087
|
+
}
|
|
318088
|
+
}, this.#heartbeatMs);
|
|
318089
|
+
this.#heartbeat.unref?.();
|
|
318090
|
+
if (this.#state.pendingBody || this.#dirty)
|
|
318091
|
+
this.#schedule();
|
|
318092
|
+
}
|
|
318093
|
+
publish(snapshots) {
|
|
318094
|
+
if (this.#closed)
|
|
318095
|
+
return;
|
|
318096
|
+
this.#snapshots = snapshots.map((snapshot) => ({ ...snapshot }));
|
|
318097
|
+
this.#dirty = true;
|
|
318098
|
+
this.#schedule();
|
|
318099
|
+
}
|
|
318100
|
+
async close(snapshots, timeoutMs = FORWARDER_TIMEOUT_MS) {
|
|
318101
|
+
if (this.#closed)
|
|
318102
|
+
return;
|
|
318103
|
+
if (this.#heartbeat)
|
|
318104
|
+
clearInterval(this.#heartbeat);
|
|
318105
|
+
this.#heartbeat = null;
|
|
318106
|
+
if (this.#retry)
|
|
318107
|
+
clearTimeout(this.#retry);
|
|
318108
|
+
this.#retry = null;
|
|
318109
|
+
this.#snapshots = snapshots.map((snapshot) => ({ ...snapshot }));
|
|
318110
|
+
this.#dirty = this.#snapshots.length > 0;
|
|
318111
|
+
this.#schedule();
|
|
318112
|
+
await Promise.race([
|
|
318113
|
+
this.#waitForIdle(),
|
|
318114
|
+
new Promise((resolve) => setTimeout(resolve, timeoutMs))
|
|
318115
|
+
]);
|
|
318116
|
+
this.#closed = true;
|
|
318117
|
+
if (this.#retry)
|
|
318118
|
+
clearTimeout(this.#retry);
|
|
318119
|
+
this.#retry = null;
|
|
318120
|
+
}
|
|
318121
|
+
#schedule() {
|
|
318122
|
+
if (this.#closed || this.#pumping)
|
|
318123
|
+
return;
|
|
318124
|
+
this.#pumping = this.#pump().finally(() => {
|
|
318125
|
+
this.#pumping = null;
|
|
318126
|
+
if (!this.#closed && !this.#retry && (this.#state.pendingBody || this.#dirty))
|
|
318127
|
+
this.#schedule();
|
|
318128
|
+
});
|
|
318129
|
+
}
|
|
318130
|
+
async#pump() {
|
|
318131
|
+
if (this.#state.pendingBody && !await this.#deliver())
|
|
318132
|
+
return;
|
|
318133
|
+
if (!this.#dirty || this.#snapshots.length === 0)
|
|
318134
|
+
return;
|
|
318135
|
+
if (this.#advanceRun) {
|
|
318136
|
+
this.#state.producerEpoch = next(this.#state.producerEpoch);
|
|
318137
|
+
this.#state.runId = randomUUID();
|
|
318138
|
+
this.#state.nextBatchSequence = "1";
|
|
318139
|
+
this.#state.nextStreamSequences = {};
|
|
318140
|
+
this.#advanceRun = false;
|
|
318141
|
+
this.#persist();
|
|
318142
|
+
}
|
|
318143
|
+
this.#dirty = false;
|
|
318144
|
+
this.#state.pendingBody = this.#body(this.#snapshots);
|
|
318145
|
+
this.#persist();
|
|
318146
|
+
await this.#deliver();
|
|
318147
|
+
}
|
|
318148
|
+
async#deliver() {
|
|
318149
|
+
const body = this.#state.pendingBody;
|
|
318150
|
+
if (!body)
|
|
318151
|
+
return true;
|
|
318152
|
+
try {
|
|
318153
|
+
await this.#post(body);
|
|
318154
|
+
this.#state.pendingBody = undefined;
|
|
318155
|
+
this.#persist();
|
|
318156
|
+
this.#retryAttempt = 0;
|
|
318157
|
+
return true;
|
|
318158
|
+
} catch {
|
|
318159
|
+
this.#retryLater();
|
|
318160
|
+
return false;
|
|
318161
|
+
}
|
|
318162
|
+
}
|
|
318163
|
+
#body(snapshots) {
|
|
318164
|
+
const ordered3 = [...snapshots].sort((left, right) => key(left).localeCompare(key(right)));
|
|
318165
|
+
if (ordered3.length === 0 || ordered3.length > 1000) {
|
|
318166
|
+
throw new Error("Stream health requires between one and 1000 registry rows");
|
|
318167
|
+
}
|
|
318168
|
+
const batchSequence = this.#state.nextBatchSequence;
|
|
318169
|
+
this.#state.nextBatchSequence = next(batchSequence);
|
|
318170
|
+
const heartbeatAt = new Date().toISOString();
|
|
318171
|
+
const active = ordered3.filter((snapshot) => snapshot.registryStatus === "active").length;
|
|
318172
|
+
const rows = ordered3.map((snapshot) => {
|
|
318173
|
+
const streamKey = key(snapshot);
|
|
318174
|
+
const sequence = this.#state.nextStreamSequences[streamKey] ?? "1";
|
|
318175
|
+
this.#state.nextStreamSequences[streamKey] = next(sequence);
|
|
318176
|
+
return {
|
|
318177
|
+
table: TABLE,
|
|
318178
|
+
row: {
|
|
318179
|
+
producer_id: PRODUCER_ID,
|
|
318180
|
+
producer_epoch: this.#state.producerEpoch,
|
|
318181
|
+
run_id: this.#state.runId,
|
|
318182
|
+
batch_sequence: batchSequence,
|
|
318183
|
+
batch_snapshot_count: String(ordered3.length),
|
|
318184
|
+
batch_active_stream_count: String(active),
|
|
318185
|
+
registry_revision: registryRevision(ordered3),
|
|
318186
|
+
registry_status: snapshot.registryStatus,
|
|
318187
|
+
retired_at: snapshot.retiredAt,
|
|
318188
|
+
exchange: snapshot.exchange,
|
|
318189
|
+
account_selector: snapshot.accountSelector,
|
|
318190
|
+
account_role: snapshot.accountRole ?? null,
|
|
318191
|
+
stream_kind: snapshot.streamKind,
|
|
318192
|
+
account_scope: snapshot.accountScope,
|
|
318193
|
+
sequence,
|
|
318194
|
+
state: snapshot.state,
|
|
318195
|
+
state_changed_at: snapshot.stateChangedAt,
|
|
318196
|
+
last_connected_at: snapshot.lastConnectedAt,
|
|
318197
|
+
last_authenticated_at: snapshot.lastAuthenticatedAt,
|
|
318198
|
+
last_received_at: snapshot.lastReceivedAt,
|
|
318199
|
+
heartbeat_at: heartbeatAt,
|
|
318200
|
+
connect_attempt_count: snapshot.connectAttemptCount,
|
|
318201
|
+
reconnect_count: snapshot.reconnectCount,
|
|
318202
|
+
error_count: snapshot.errorCount,
|
|
318203
|
+
last_failure_kind: snapshot.lastFailureKind,
|
|
318204
|
+
last_failure_reason: snapshot.lastFailureReason,
|
|
318205
|
+
traffic_mode: snapshot.trafficMode,
|
|
318206
|
+
source_watermark: snapshot.sourceWatermark
|
|
318207
|
+
}
|
|
318208
|
+
};
|
|
318209
|
+
});
|
|
318210
|
+
return JSON.stringify({
|
|
318211
|
+
source: SOURCE,
|
|
318212
|
+
deployment_id: this.#deploymentId,
|
|
318213
|
+
rows
|
|
318214
|
+
});
|
|
318215
|
+
}
|
|
318216
|
+
#retryLater() {
|
|
318217
|
+
if (this.#closed || this.#retry)
|
|
318218
|
+
return;
|
|
318219
|
+
const delay = Math.min(1000 * 2 ** this.#retryAttempt, 30000);
|
|
318220
|
+
this.#retryAttempt += 1;
|
|
318221
|
+
this.#retry = setTimeout(() => {
|
|
318222
|
+
this.#retry = null;
|
|
318223
|
+
this.#schedule();
|
|
318224
|
+
}, delay);
|
|
318225
|
+
this.#retry.unref?.();
|
|
318226
|
+
}
|
|
318227
|
+
async#waitForIdle() {
|
|
318228
|
+
while (this.#pumping)
|
|
318229
|
+
await this.#pumping;
|
|
318230
|
+
}
|
|
318231
|
+
#read() {
|
|
318232
|
+
try {
|
|
318233
|
+
const parsed = JSON.parse(readFileSync(this.#statePath, "utf8"));
|
|
318234
|
+
if (!validState(parsed))
|
|
318235
|
+
throw new Error("invalid state shape");
|
|
318236
|
+
return parsed;
|
|
318237
|
+
} catch (error) {
|
|
318238
|
+
if (error.code === "ENOENT")
|
|
318239
|
+
return null;
|
|
318240
|
+
throw new Error("Stream health state cannot be read", { cause: error });
|
|
318241
|
+
}
|
|
318242
|
+
}
|
|
318243
|
+
#persist() {
|
|
318244
|
+
const parent = dirname(this.#statePath);
|
|
318245
|
+
try {
|
|
318246
|
+
if (!statSync(parent).isDirectory())
|
|
318247
|
+
throw new Error("state parent is not a directory");
|
|
318248
|
+
} catch (error) {
|
|
318249
|
+
throw new Error("Stream health state directory is unavailable", {
|
|
318250
|
+
cause: error
|
|
318251
|
+
});
|
|
318252
|
+
}
|
|
318253
|
+
const temporary = `${this.#statePath}.${process.pid}.${randomUUID()}.tmp`;
|
|
318254
|
+
let fd2;
|
|
318255
|
+
try {
|
|
318256
|
+
fd2 = openSync2(temporary, "wx", 384);
|
|
318257
|
+
writeFileSync(fd2, JSON.stringify(this.#state));
|
|
318258
|
+
fsyncSync2(fd2);
|
|
318259
|
+
closeSync2(fd2);
|
|
318260
|
+
fd2 = undefined;
|
|
318261
|
+
renameSync(temporary, this.#statePath);
|
|
318262
|
+
const parentFd = openSync2(parent, "r");
|
|
318263
|
+
try {
|
|
318264
|
+
fsyncSync2(parentFd);
|
|
318265
|
+
} finally {
|
|
318266
|
+
closeSync2(parentFd);
|
|
318267
|
+
}
|
|
318268
|
+
} catch (error) {
|
|
318269
|
+
if (fd2 !== undefined)
|
|
318270
|
+
closeSync2(fd2);
|
|
318271
|
+
try {
|
|
318272
|
+
unlinkSync(temporary);
|
|
318273
|
+
} catch {}
|
|
318274
|
+
throw new Error("Stream health state cannot be persisted", {
|
|
318275
|
+
cause: error
|
|
318276
|
+
});
|
|
318277
|
+
}
|
|
318278
|
+
}
|
|
318279
|
+
}
|
|
318280
|
+
function streamHealthPublisherConfigFromEnv(env = process.env) {
|
|
318281
|
+
if (env.CEX_BROKER_ARCHIVE_ENABLED !== "true") {
|
|
318282
|
+
throw new Error("Configured account user streams require CEX_BROKER_ARCHIVE_ENABLED=true");
|
|
318283
|
+
}
|
|
318284
|
+
const deploymentId = env.CEX_BROKER_DEPLOYMENT_ID?.trim();
|
|
318285
|
+
const forwarderUrl = env.CEX_BROKER_ARCHIVE_FORWARDER_URL?.trim();
|
|
318286
|
+
const statePath = env.CEX_BROKER_STREAM_HEALTH_STATE_PATH?.trim();
|
|
318287
|
+
if (!deploymentId || !forwarderUrl || !statePath) {
|
|
318288
|
+
throw new Error("Configured account user streams require deployment, forwarder, and persistent state configuration");
|
|
318289
|
+
}
|
|
318290
|
+
return {
|
|
318291
|
+
deploymentId,
|
|
318292
|
+
forwarderUrl,
|
|
318293
|
+
statePath,
|
|
318294
|
+
forwarderAuthToken: env.CEX_BROKER_ARCHIVE_FORWARDER_TOKEN?.trim() || undefined
|
|
318295
|
+
};
|
|
318296
|
+
}
|
|
318297
|
+
|
|
318298
|
+
// src/helpers/binance-user-data-stream.ts
|
|
318299
|
+
import { Buffer as Buffer2 } from "node:buffer";
|
|
318300
|
+
import { createHmac } from "node:crypto";
|
|
318301
|
+
var BINANCE_SPOT_WS_API_URL = "wss://ws-api.binance.com:443/ws-api/v3";
|
|
318302
|
+
var DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS = 16;
|
|
318303
|
+
var createWebSocket = (url2) => new wrapper_default(url2);
|
|
318304
|
+
var userDataRequestCounter = 0;
|
|
318305
|
+
function getExchangeString(exchange, key2) {
|
|
318306
|
+
const value = exchange[key2];
|
|
318307
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
318308
|
+
throw new Error(`Binance user-data stream requires exchange.${key2}`);
|
|
318309
|
+
}
|
|
318310
|
+
return value;
|
|
318311
|
+
}
|
|
318312
|
+
function sortedQuery(params) {
|
|
318313
|
+
return Object.entries(params).sort(([left], [right]) => left.localeCompare(right)).map(([key2, value]) => `${encodeURIComponent(key2)}=${encodeURIComponent(String(value))}`).join("&");
|
|
318314
|
+
}
|
|
318315
|
+
function signUserDataStreamParams(exchange, params) {
|
|
318316
|
+
const signParams = exchange.signParams;
|
|
318317
|
+
if (typeof signParams === "function") {
|
|
318318
|
+
return signParams.call(exchange, params);
|
|
318319
|
+
}
|
|
318320
|
+
const secret = getExchangeString(exchange, "secret");
|
|
318321
|
+
return {
|
|
318322
|
+
...params,
|
|
318323
|
+
signature: createHmac("sha256", secret).update(sortedQuery(params)).digest("hex")
|
|
318324
|
+
};
|
|
318325
|
+
}
|
|
318326
|
+
function getBinanceSpotWsApiUrl(exchange) {
|
|
318327
|
+
const urls = exchange.urls;
|
|
318328
|
+
return urls?.api?.ws?.["ws-api"]?.spot ?? BINANCE_SPOT_WS_API_URL;
|
|
318329
|
+
}
|
|
318330
|
+
function getRecord(value) {
|
|
318331
|
+
return typeof value === "object" && value !== null ? value : null;
|
|
318332
|
+
}
|
|
318333
|
+
function getMessage(value) {
|
|
318334
|
+
if (value instanceof Error) {
|
|
318335
|
+
return value.message;
|
|
318336
|
+
}
|
|
318337
|
+
if (typeof value === "string" && value.length > 0) {
|
|
318338
|
+
return value;
|
|
318339
|
+
}
|
|
318340
|
+
const record = getRecord(value);
|
|
318341
|
+
const message = record?.message;
|
|
318342
|
+
return typeof message === "string" && message.length > 0 ? message : null;
|
|
318343
|
+
}
|
|
318344
|
+
function getOptionalExchangeString(exchange, key2) {
|
|
318345
|
+
const value = exchange[key2];
|
|
318346
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
318347
|
+
}
|
|
318348
|
+
function redactDiagnosticMessage(message, secretValues) {
|
|
318349
|
+
let redacted = message;
|
|
318350
|
+
for (const value of secretValues) {
|
|
318351
|
+
if (value.length > 0) {
|
|
318352
|
+
redacted = redacted.split(value).join("[redacted]");
|
|
318353
|
+
}
|
|
318354
|
+
}
|
|
318355
|
+
return redacted.replace(/(\b(?:apiKey|secret|signature)\b\s*=\s*)[^\s&,;)]+/gi, "$1[redacted]").replace(/("(?:apiKey|secret|signature)"\s*:\s*")[^"]*(")/gi, "$1[redacted]$2");
|
|
318356
|
+
}
|
|
318357
|
+
function formatBinanceUserDataWebSocketError(event, secretValues) {
|
|
318358
|
+
const record = getRecord(event);
|
|
318359
|
+
const message = getMessage(record?.error) ?? getMessage(record?.message) ?? getMessage(event);
|
|
318360
|
+
const safeMessage = message === null ? null : redactDiagnosticMessage(message, secretValues);
|
|
318361
|
+
return new Error(safeMessage ? `Binance user-data WebSocket error: ${safeMessage}` : "Binance user-data WebSocket error");
|
|
318362
|
+
}
|
|
318363
|
+
function getCloseReason(value) {
|
|
318364
|
+
if (typeof value === "string") {
|
|
318365
|
+
return value.length > 0 ? value : null;
|
|
318366
|
+
}
|
|
318367
|
+
if (Buffer2.isBuffer(value)) {
|
|
318368
|
+
const reason = value.toString("utf8");
|
|
318369
|
+
return reason.length > 0 ? reason : null;
|
|
318370
|
+
}
|
|
318371
|
+
if (value instanceof Uint8Array) {
|
|
318372
|
+
const reason = Buffer2.from(value).toString("utf8");
|
|
318373
|
+
return reason.length > 0 ? reason : null;
|
|
318374
|
+
}
|
|
318375
|
+
return null;
|
|
318376
|
+
}
|
|
318377
|
+
function formatBinanceUserDataWebSocketClose(codeOrEvent, reasonOrUndefined, secretValues) {
|
|
318378
|
+
const record = getRecord(codeOrEvent);
|
|
318379
|
+
const code = record ? record.code : codeOrEvent;
|
|
318380
|
+
const reason = getCloseReason(record ? record.reason : reasonOrUndefined);
|
|
318381
|
+
const safeReason = reason === null ? null : redactDiagnosticMessage(reason, secretValues);
|
|
318382
|
+
const details = [
|
|
318383
|
+
typeof code === "number" || typeof code === "string" ? `code=${code}` : null,
|
|
318384
|
+
safeReason ? `reason=${safeReason}` : null
|
|
318385
|
+
].filter((detail) => detail !== null);
|
|
318386
|
+
return new Error(details.length > 0 ? `Binance user-data WebSocket closed unexpectedly (${details.join(", ")})` : "Binance user-data WebSocket closed unexpectedly");
|
|
318387
|
+
}
|
|
318388
|
+
function decodeMessageData(data) {
|
|
318389
|
+
if (typeof data === "string") {
|
|
318390
|
+
return data;
|
|
318391
|
+
}
|
|
318392
|
+
if (Buffer2.isBuffer(data)) {
|
|
318393
|
+
return data.toString("utf8");
|
|
318394
|
+
}
|
|
318395
|
+
if (data instanceof ArrayBuffer) {
|
|
318396
|
+
return Buffer2.from(data).toString("utf8");
|
|
318397
|
+
}
|
|
318398
|
+
if (ArrayBuffer.isView(data)) {
|
|
318399
|
+
return Buffer2.from(data.buffer, data.byteOffset, data.byteLength).toString("utf8");
|
|
318400
|
+
}
|
|
318401
|
+
if (Array.isArray(data) && data.every((item) => Buffer2.isBuffer(item))) {
|
|
318402
|
+
return Buffer2.concat(data).toString("utf8");
|
|
318403
|
+
}
|
|
318404
|
+
return data;
|
|
318405
|
+
}
|
|
318406
|
+
|
|
318407
|
+
class BinanceSpotUserDataStream {
|
|
318408
|
+
exchange;
|
|
318409
|
+
ws;
|
|
318410
|
+
secretValues;
|
|
318411
|
+
requestId = `user-data-${Date.now()}-${userDataRequestCounter++}`;
|
|
318412
|
+
maxBufferedEvents;
|
|
318413
|
+
observer;
|
|
318414
|
+
queue = [];
|
|
318415
|
+
waiters = [];
|
|
318416
|
+
closed = false;
|
|
318417
|
+
closeError = null;
|
|
318418
|
+
subscriptionId = null;
|
|
318419
|
+
constructor(exchange, options = {}) {
|
|
318420
|
+
this.exchange = exchange;
|
|
318421
|
+
this.maxBufferedEvents = options.maxBufferedEvents ?? DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS;
|
|
318422
|
+
this.observer = options.observer;
|
|
318423
|
+
this.secretValues = [
|
|
318424
|
+
getOptionalExchangeString(exchange, "apiKey"),
|
|
318425
|
+
getOptionalExchangeString(exchange, "secret")
|
|
318426
|
+
].filter((value) => value !== null);
|
|
318427
|
+
this.ws = createWebSocket(getBinanceSpotWsApiUrl(exchange));
|
|
318428
|
+
this.ws.on("open", () => {
|
|
318429
|
+
this.observer?.onConnected?.();
|
|
318430
|
+
this.subscribe();
|
|
318431
|
+
});
|
|
318432
|
+
this.ws.on("message", (data) => this.handleMessage(data));
|
|
318433
|
+
this.ws.on("error", (error) => this.fail(formatBinanceUserDataWebSocketError(error, this.secretValues), "transport_error"));
|
|
318434
|
+
this.ws.on("close", (code, reason) => this.handleClose(code, reason));
|
|
318435
|
+
}
|
|
318436
|
+
async* [Symbol.asyncIterator]() {
|
|
318437
|
+
while (true) {
|
|
318438
|
+
const event = await this.nextEvent();
|
|
318439
|
+
if (!event) {
|
|
318440
|
+
break;
|
|
318441
|
+
}
|
|
318442
|
+
yield event;
|
|
318443
|
+
}
|
|
318444
|
+
}
|
|
318445
|
+
close() {
|
|
318446
|
+
if (this.closed) {
|
|
318447
|
+
return;
|
|
318448
|
+
}
|
|
318449
|
+
this.closed = true;
|
|
318450
|
+
this.queue.length = 0;
|
|
318451
|
+
try {
|
|
318452
|
+
this.ws.close();
|
|
318453
|
+
} catch {}
|
|
318454
|
+
this.flushWaiters();
|
|
318455
|
+
}
|
|
318456
|
+
handleClose(code, reason) {
|
|
318457
|
+
if (this.closed) {
|
|
318458
|
+
return;
|
|
318459
|
+
}
|
|
318460
|
+
this.fail(formatBinanceUserDataWebSocketClose(code, reason, this.secretValues), "remote_closed");
|
|
318461
|
+
}
|
|
318462
|
+
subscribe() {
|
|
318463
|
+
const apiKey = getExchangeString(this.exchange, "apiKey");
|
|
318464
|
+
const signedParams = signUserDataStreamParams(this.exchange, {
|
|
318465
|
+
apiKey,
|
|
318466
|
+
timestamp: Date.now()
|
|
318467
|
+
});
|
|
318468
|
+
this.ws.send(JSON.stringify({
|
|
318469
|
+
id: this.requestId,
|
|
318470
|
+
method: "userDataStream.subscribe.signature",
|
|
318471
|
+
params: signedParams
|
|
318472
|
+
}));
|
|
318473
|
+
}
|
|
318474
|
+
handleMessage(data) {
|
|
318475
|
+
if (this.closed) {
|
|
318476
|
+
return;
|
|
318477
|
+
}
|
|
318478
|
+
let message;
|
|
318479
|
+
try {
|
|
318480
|
+
const decodedData = decodeMessageData(data);
|
|
318481
|
+
message = typeof decodedData === "string" ? JSON.parse(decodedData) : decodedData;
|
|
318482
|
+
} catch (error) {
|
|
318483
|
+
this.fail(error instanceof Error ? error : new Error("Invalid Binance user-data message"), "protocol_error");
|
|
318484
|
+
return;
|
|
318485
|
+
}
|
|
318486
|
+
if ("id" in message && message.id === this.requestId) {
|
|
318487
|
+
if (message.status !== 200) {
|
|
318488
|
+
this.fail(new Error(message.error?.msg ?? message.error?.message ?? `Binance user-data subscription failed with status ${message.status}`), "auth_failed");
|
|
318489
|
+
return;
|
|
318490
|
+
}
|
|
318491
|
+
this.subscriptionId = message.result?.subscriptionId ?? null;
|
|
318492
|
+
this.observer?.onAuthenticated?.();
|
|
318493
|
+
return;
|
|
318494
|
+
}
|
|
318495
|
+
if ("status" in message && typeof message.status === "number" && message.status !== 200) {
|
|
318496
|
+
const errorMessage = message.error?.msg ?? message.error?.message ?? `Binance user-data request failed with status ${message.status}`;
|
|
318497
|
+
const errorCode2 = message.error?.code;
|
|
318498
|
+
this.fail(new Error(typeof errorCode2 === "number" ? `${errorMessage} (code ${errorCode2})` : errorMessage), "protocol_error");
|
|
318499
|
+
return;
|
|
318500
|
+
}
|
|
318501
|
+
if (!("event" in message) || !message.event) {
|
|
318502
|
+
return;
|
|
318503
|
+
}
|
|
318504
|
+
const subscriptionId = message.subscriptionId ?? this.subscriptionId;
|
|
318505
|
+
if (subscriptionId === null || subscriptionId === undefined) {
|
|
318506
|
+
return;
|
|
318507
|
+
}
|
|
318508
|
+
this.push({ subscriptionId, event: message.event });
|
|
318509
|
+
}
|
|
318510
|
+
push(event) {
|
|
318511
|
+
if (this.closed) {
|
|
318512
|
+
return;
|
|
318513
|
+
}
|
|
318514
|
+
this.observer?.onEvent?.(event);
|
|
318515
|
+
const waiter = this.waiters.shift();
|
|
318516
|
+
if (waiter) {
|
|
318517
|
+
waiter.resolve(event);
|
|
318518
|
+
return;
|
|
318519
|
+
}
|
|
318520
|
+
if (this.queue.length >= this.maxBufferedEvents) {
|
|
318521
|
+
this.fail(new Error(`Binance user-data stream buffered event limit exceeded (${this.maxBufferedEvents}); downstream consumer is not keeping up`), "backpressure");
|
|
318522
|
+
return;
|
|
318523
|
+
}
|
|
318524
|
+
this.queue.push(event);
|
|
318525
|
+
}
|
|
318526
|
+
nextEvent() {
|
|
318527
|
+
const event = this.queue.shift();
|
|
318528
|
+
if (event) {
|
|
318529
|
+
return Promise.resolve(event);
|
|
318530
|
+
}
|
|
318531
|
+
if (this.closeError) {
|
|
318532
|
+
return Promise.reject(this.closeError);
|
|
318533
|
+
}
|
|
318534
|
+
if (this.closed) {
|
|
318535
|
+
return Promise.resolve(null);
|
|
318536
|
+
}
|
|
318537
|
+
return new Promise((resolve, reject) => {
|
|
318538
|
+
this.waiters.push({ resolve, reject });
|
|
318539
|
+
});
|
|
318540
|
+
}
|
|
318541
|
+
fail(error, kind) {
|
|
318542
|
+
if (this.closeError) {
|
|
318543
|
+
return;
|
|
318544
|
+
}
|
|
318545
|
+
this.closeError = error;
|
|
318546
|
+
this.observer?.onFailure?.({ kind, reason: error.message });
|
|
318547
|
+
this.closed = true;
|
|
318548
|
+
this.queue.length = 0;
|
|
318549
|
+
this.flushWaiters();
|
|
318550
|
+
try {
|
|
318551
|
+
this.ws.close();
|
|
318552
|
+
} catch {}
|
|
318553
|
+
}
|
|
318554
|
+
flushWaiters() {
|
|
318555
|
+
const error = this.closeError;
|
|
318556
|
+
for (const waiter of this.waiters.splice(0)) {
|
|
318557
|
+
if (error) {
|
|
318558
|
+
waiter.reject(error);
|
|
318559
|
+
} else {
|
|
318560
|
+
waiter.resolve(null);
|
|
318561
|
+
}
|
|
318562
|
+
}
|
|
318563
|
+
}
|
|
318564
|
+
}
|
|
318565
|
+
function isBinanceBalanceUserDataEvent(event) {
|
|
318566
|
+
return event.e === "outboundAccountPosition" || event.e === "balanceUpdate" || event.e === "externalLockUpdate";
|
|
318567
|
+
}
|
|
318568
|
+
function isBinanceOrderUserDataEvent(event) {
|
|
318569
|
+
return event.e === "executionReport" || event.e === "listStatus";
|
|
318570
|
+
}
|
|
318571
|
+
|
|
318572
|
+
// src/helpers/user-data-stream-supervisor.ts
|
|
318573
|
+
var MAX_SUBSCRIBER_EVENTS = 16;
|
|
318574
|
+
function now3() {
|
|
318575
|
+
return new Date().toISOString();
|
|
318576
|
+
}
|
|
318577
|
+
function retryDelay(attempt) {
|
|
318578
|
+
return Math.min(1000 * 2 ** attempt, 30000);
|
|
318579
|
+
}
|
|
318580
|
+
function safeFailureReason(exchange, reason) {
|
|
318581
|
+
const secrets = [exchange.apiKey, exchange.secret].filter((value) => typeof value === "string" && value.length > 0);
|
|
318582
|
+
return redactSecretLiterals(reason, secrets).replace(/\s+/g, " ").trim().slice(0, 256);
|
|
318583
|
+
}
|
|
318584
|
+
|
|
318585
|
+
class Subscriber {
|
|
318586
|
+
kind;
|
|
318587
|
+
marketId;
|
|
318588
|
+
onClose;
|
|
318589
|
+
#queue = [];
|
|
318590
|
+
#waiters = [];
|
|
318591
|
+
#closed = false;
|
|
318592
|
+
#error = null;
|
|
318593
|
+
constructor(kind, marketId, onClose) {
|
|
318594
|
+
this.kind = kind;
|
|
318595
|
+
this.marketId = marketId;
|
|
318596
|
+
this.onClose = onClose;
|
|
318597
|
+
}
|
|
318598
|
+
push(message) {
|
|
318599
|
+
if (this.#closed || !this.#matches(message.event))
|
|
318600
|
+
return;
|
|
318601
|
+
const waiter = this.#waiters.shift();
|
|
318602
|
+
if (waiter) {
|
|
318603
|
+
waiter.resolve(message);
|
|
318604
|
+
return;
|
|
318605
|
+
}
|
|
318606
|
+
if (this.#queue.length >= MAX_SUBSCRIBER_EVENTS) {
|
|
318607
|
+
this.#fail(new Error("Configured account user-data subscriber fell behind"));
|
|
318608
|
+
return;
|
|
318609
|
+
}
|
|
318610
|
+
this.#queue.push(message);
|
|
318611
|
+
}
|
|
318612
|
+
close() {
|
|
318613
|
+
if (this.#closed)
|
|
318614
|
+
return;
|
|
318615
|
+
this.#closed = true;
|
|
318616
|
+
this.#queue.length = 0;
|
|
318617
|
+
this.onClose();
|
|
318618
|
+
for (const waiter of this.#waiters.splice(0))
|
|
318619
|
+
waiter.resolve(null);
|
|
318620
|
+
}
|
|
318621
|
+
async* [Symbol.asyncIterator]() {
|
|
318622
|
+
while (true) {
|
|
318623
|
+
const event = await this.#next();
|
|
318624
|
+
if (!event)
|
|
318625
|
+
return;
|
|
318626
|
+
yield event;
|
|
318627
|
+
}
|
|
318628
|
+
}
|
|
318629
|
+
#matches(event) {
|
|
318630
|
+
if (this.kind === "balance")
|
|
318631
|
+
return isBinanceBalanceUserDataEvent(event);
|
|
318632
|
+
if (!isBinanceOrderUserDataEvent(event))
|
|
318633
|
+
return false;
|
|
318634
|
+
return !this.marketId || event.s === this.marketId;
|
|
318635
|
+
}
|
|
318636
|
+
#next() {
|
|
318637
|
+
const event = this.#queue.shift();
|
|
318638
|
+
if (event)
|
|
318639
|
+
return Promise.resolve(event);
|
|
318640
|
+
if (this.#error)
|
|
318641
|
+
return Promise.reject(this.#error);
|
|
318642
|
+
if (this.#closed)
|
|
318643
|
+
return Promise.resolve(null);
|
|
318644
|
+
return new Promise((resolve, reject) => this.#waiters.push({ resolve, reject }));
|
|
318645
|
+
}
|
|
318646
|
+
#fail(error) {
|
|
318647
|
+
if (this.#closed)
|
|
318648
|
+
return;
|
|
318649
|
+
this.#closed = true;
|
|
318650
|
+
this.#error = error;
|
|
318651
|
+
this.#queue.length = 0;
|
|
318652
|
+
this.onClose();
|
|
318653
|
+
for (const waiter of this.#waiters.splice(0))
|
|
318654
|
+
waiter.reject(error);
|
|
318655
|
+
}
|
|
318656
|
+
}
|
|
318657
|
+
|
|
318658
|
+
class AccountWorker {
|
|
318659
|
+
exchangeName;
|
|
318660
|
+
account;
|
|
318661
|
+
onChange;
|
|
318662
|
+
#subscribers = new Set;
|
|
318663
|
+
#snapshot;
|
|
318664
|
+
#stopping = false;
|
|
318665
|
+
#stream = null;
|
|
318666
|
+
#retryTimer = null;
|
|
318667
|
+
#retryResolve = null;
|
|
318668
|
+
#run = null;
|
|
318669
|
+
#failureObserved = false;
|
|
318670
|
+
#attempts = 0;
|
|
318671
|
+
constructor(exchangeName, account, onChange) {
|
|
318672
|
+
this.exchangeName = exchangeName;
|
|
318673
|
+
this.account = account;
|
|
318674
|
+
this.onChange = onChange;
|
|
318675
|
+
const timestamp = now3();
|
|
318676
|
+
this.#snapshot = {
|
|
318677
|
+
exchange: exchangeName,
|
|
318678
|
+
accountSelector: account.label,
|
|
318679
|
+
accountRole: account.role,
|
|
318680
|
+
streamKind: "user_data",
|
|
318681
|
+
accountScope: "spot",
|
|
318682
|
+
registryStatus: "active",
|
|
318683
|
+
retiredAt: null,
|
|
318684
|
+
state: "connecting",
|
|
318685
|
+
stateChangedAt: timestamp,
|
|
318686
|
+
lastConnectedAt: null,
|
|
318687
|
+
lastAuthenticatedAt: null,
|
|
318688
|
+
lastReceivedAt: null,
|
|
318689
|
+
connectAttemptCount: "0",
|
|
318690
|
+
reconnectCount: "0",
|
|
318691
|
+
errorCount: "0",
|
|
318692
|
+
lastFailureKind: "none",
|
|
318693
|
+
lastFailureReason: "",
|
|
318694
|
+
trafficMode: "event_driven",
|
|
318695
|
+
sourceWatermark: null
|
|
318696
|
+
};
|
|
318697
|
+
}
|
|
318698
|
+
start() {
|
|
318699
|
+
if (this.exchangeName !== "binance") {
|
|
318700
|
+
this.#fail("unsupported_connector", "Configured exchange has no user-data supervisor");
|
|
318701
|
+
return;
|
|
318702
|
+
}
|
|
318703
|
+
this.#run = this.#connectLoop();
|
|
318704
|
+
}
|
|
318705
|
+
subscribe(options) {
|
|
318706
|
+
if (this.#stopping)
|
|
318707
|
+
throw new Error("Configured account user-data supervisor is stopping");
|
|
318708
|
+
const subscriber = new Subscriber(options.kind, options.marketId, () => {
|
|
318709
|
+
this.#subscribers.delete(subscriber);
|
|
318710
|
+
});
|
|
318711
|
+
this.#subscribers.add(subscriber);
|
|
318712
|
+
return subscriber;
|
|
318713
|
+
}
|
|
318714
|
+
snapshot() {
|
|
318715
|
+
return { ...this.#snapshot };
|
|
318716
|
+
}
|
|
318717
|
+
async stop() {
|
|
318718
|
+
this.#stopping = true;
|
|
318719
|
+
if (this.#retryTimer)
|
|
318720
|
+
clearTimeout(this.#retryTimer);
|
|
318721
|
+
this.#retryTimer = null;
|
|
318722
|
+
this.#retryResolve?.();
|
|
318723
|
+
this.#retryResolve = null;
|
|
318724
|
+
this.#stream?.close();
|
|
318725
|
+
await this.#run;
|
|
318726
|
+
this.#transition("disconnected", "shutdown", "Broker shutdown");
|
|
318727
|
+
for (const subscriber of [...this.#subscribers])
|
|
318728
|
+
subscriber.close();
|
|
318729
|
+
}
|
|
318730
|
+
#transition(state, failureKind, failureReason) {
|
|
318731
|
+
const timestamp = now3();
|
|
318732
|
+
if (this.#snapshot.state !== state) {
|
|
318733
|
+
this.#snapshot.state = state;
|
|
318734
|
+
this.#snapshot.stateChangedAt = timestamp;
|
|
318735
|
+
}
|
|
318736
|
+
if (failureKind) {
|
|
318737
|
+
this.#snapshot.lastFailureKind = failureKind;
|
|
318738
|
+
this.#snapshot.lastFailureReason = failureReason ?? "";
|
|
318739
|
+
}
|
|
318740
|
+
this.onChange();
|
|
318741
|
+
}
|
|
318742
|
+
#connected() {
|
|
318743
|
+
this.#snapshot.lastConnectedAt = now3();
|
|
318744
|
+
this.#transition("connected");
|
|
318745
|
+
}
|
|
318746
|
+
#authenticated() {
|
|
318747
|
+
this.#snapshot.lastAuthenticatedAt = now3();
|
|
318748
|
+
this.onChange();
|
|
318749
|
+
}
|
|
318750
|
+
#received(message) {
|
|
318751
|
+
this.#snapshot.lastReceivedAt = now3();
|
|
318752
|
+
const eventTimestamp = message.event.E;
|
|
318753
|
+
this.#snapshot.sourceWatermark = typeof eventTimestamp === "number" || typeof eventTimestamp === "string" ? String(eventTimestamp).slice(0, 512) : null;
|
|
318754
|
+
for (const subscriber of this.#subscribers)
|
|
318755
|
+
subscriber.push(message);
|
|
318756
|
+
this.onChange();
|
|
318757
|
+
}
|
|
318758
|
+
#fail(kind, reason) {
|
|
318759
|
+
this.#failureObserved = true;
|
|
318760
|
+
this.#snapshot.errorCount = (BigInt(this.#snapshot.errorCount) + 1n).toString();
|
|
318761
|
+
this.#transition("error", kind, safeFailureReason(this.account.exchange, reason));
|
|
318762
|
+
}
|
|
318763
|
+
async#connectLoop() {
|
|
318764
|
+
while (!this.#stopping) {
|
|
318765
|
+
if (this.#attempts > 0) {
|
|
318766
|
+
this.#snapshot.reconnectCount = (BigInt(this.#snapshot.reconnectCount) + 1n).toString();
|
|
318767
|
+
}
|
|
318768
|
+
this.#attempts += 1;
|
|
318769
|
+
this.#snapshot.connectAttemptCount = String(this.#attempts);
|
|
318770
|
+
this.#failureObserved = false;
|
|
318771
|
+
this.#transition("connecting");
|
|
318772
|
+
const stream4 = new BinanceSpotUserDataStream(this.account.exchange, {
|
|
318773
|
+
observer: {
|
|
318774
|
+
onConnected: () => this.#connected(),
|
|
318775
|
+
onAuthenticated: () => this.#authenticated(),
|
|
318776
|
+
onEvent: (message) => this.#received(message),
|
|
318777
|
+
onFailure: (failure) => this.#handleStreamFailure(failure)
|
|
318778
|
+
}
|
|
318779
|
+
});
|
|
318780
|
+
this.#stream = stream4;
|
|
318781
|
+
try {
|
|
318782
|
+
for await (const _event of stream4) {}
|
|
318783
|
+
} catch (error) {
|
|
318784
|
+
if (!this.#stopping && !this.#failureObserved) {
|
|
318785
|
+
this.#fail("transport_error", error instanceof Error ? error.message : "User-data stream failed");
|
|
318786
|
+
}
|
|
318787
|
+
} finally {
|
|
318788
|
+
stream4.close();
|
|
318789
|
+
if (this.#stream === stream4)
|
|
318790
|
+
this.#stream = null;
|
|
318791
|
+
}
|
|
318792
|
+
if (!this.#stopping)
|
|
318793
|
+
await this.#waitForRetry();
|
|
318794
|
+
}
|
|
318795
|
+
}
|
|
318796
|
+
#handleStreamFailure(failure) {
|
|
318797
|
+
this.#fail(failure.kind, failure.reason);
|
|
318798
|
+
}
|
|
318799
|
+
#waitForRetry() {
|
|
318800
|
+
return new Promise((resolve) => {
|
|
318801
|
+
const delay = retryDelay(Math.max(this.#attempts - 1, 0));
|
|
318802
|
+
this.#retryResolve = resolve;
|
|
318803
|
+
this.#retryTimer = setTimeout(() => {
|
|
318804
|
+
this.#retryTimer = null;
|
|
318805
|
+
this.#retryResolve = null;
|
|
318806
|
+
resolve();
|
|
318807
|
+
}, delay);
|
|
318808
|
+
this.#retryTimer.unref?.();
|
|
318809
|
+
});
|
|
318810
|
+
}
|
|
318811
|
+
}
|
|
318812
|
+
|
|
318813
|
+
class UserDataStreamSupervisor {
|
|
318814
|
+
options;
|
|
318815
|
+
#workers = new Map;
|
|
318816
|
+
#started = false;
|
|
318817
|
+
constructor(options) {
|
|
318818
|
+
this.options = options;
|
|
318819
|
+
for (const [exchange, pool] of Object.entries(options.brokers)) {
|
|
318820
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
318821
|
+
const normalizedExchange = exchange.trim().toLowerCase();
|
|
318822
|
+
const worker = new AccountWorker(normalizedExchange, account, () => this.#publish());
|
|
318823
|
+
this.#workers.set(`${normalizedExchange}|${account.label}`, worker);
|
|
318824
|
+
}
|
|
318825
|
+
}
|
|
318826
|
+
if (this.#workers.size === 0) {
|
|
318827
|
+
throw new Error("User-data supervisor requires at least one configured account");
|
|
318828
|
+
}
|
|
318829
|
+
}
|
|
318830
|
+
start() {
|
|
318831
|
+
if (this.#started)
|
|
318832
|
+
return;
|
|
318833
|
+
this.#started = true;
|
|
318834
|
+
this.options.publisher.start();
|
|
318835
|
+
for (const worker of this.#workers.values())
|
|
318836
|
+
worker.start();
|
|
318837
|
+
this.#publish();
|
|
318838
|
+
}
|
|
318839
|
+
subscribe(options) {
|
|
318840
|
+
const exchange = options.exchange.trim().toLowerCase();
|
|
318841
|
+
const worker = this.#workers.get(`${exchange}|${options.accountSelector}`);
|
|
318842
|
+
if (!worker)
|
|
318843
|
+
throw new Error("Configured account user-data stream is unavailable");
|
|
318844
|
+
return worker.subscribe({ kind: options.kind, marketId: options.marketId });
|
|
318845
|
+
}
|
|
318846
|
+
async close() {
|
|
318847
|
+
for (const worker of this.#workers.values())
|
|
318848
|
+
await worker.stop();
|
|
318849
|
+
await this.options.publisher.close(this.#snapshots());
|
|
318850
|
+
}
|
|
318851
|
+
#snapshots() {
|
|
318852
|
+
return [...this.#workers.values()].map((worker) => worker.snapshot());
|
|
318853
|
+
}
|
|
318854
|
+
#publish() {
|
|
318855
|
+
if (!this.#started)
|
|
318856
|
+
return;
|
|
318857
|
+
this.options.publisher.publish(this.#snapshots());
|
|
318858
|
+
}
|
|
318859
|
+
}
|
|
318860
|
+
|
|
317372
318861
|
// src/server.ts
|
|
317373
318862
|
var grpc14 = __toESM(require_src3(), 1);
|
|
317374
318863
|
|
|
@@ -318209,9 +319698,9 @@ function floatSafeRemainder(val, step) {
|
|
|
318209
319698
|
return valInt % stepInt / 10 ** decCount;
|
|
318210
319699
|
}
|
|
318211
319700
|
var EVALUATING = Symbol("evaluating");
|
|
318212
|
-
function defineLazy(object,
|
|
319701
|
+
function defineLazy(object, key2, getter) {
|
|
318213
319702
|
let value = undefined;
|
|
318214
|
-
Object.defineProperty(object,
|
|
319703
|
+
Object.defineProperty(object, key2, {
|
|
318215
319704
|
get() {
|
|
318216
319705
|
if (value === EVALUATING) {
|
|
318217
319706
|
return;
|
|
@@ -318223,7 +319712,7 @@ function defineLazy(object, key, getter) {
|
|
|
318223
319712
|
return value;
|
|
318224
319713
|
},
|
|
318225
319714
|
set(v) {
|
|
318226
|
-
Object.defineProperty(object,
|
|
319715
|
+
Object.defineProperty(object, key2, {
|
|
318227
319716
|
value: v
|
|
318228
319717
|
});
|
|
318229
319718
|
},
|
|
@@ -318255,11 +319744,11 @@ function cloneDef(schema) {
|
|
|
318255
319744
|
function getElementAtPath(obj, path) {
|
|
318256
319745
|
if (!path)
|
|
318257
319746
|
return obj;
|
|
318258
|
-
return path.reduce((acc,
|
|
319747
|
+
return path.reduce((acc, key2) => acc?.[key2], obj);
|
|
318259
319748
|
}
|
|
318260
319749
|
function promiseAllObject(promisesObj) {
|
|
318261
319750
|
const keys2 = Object.keys(promisesObj);
|
|
318262
|
-
const promises = keys2.map((
|
|
319751
|
+
const promises = keys2.map((key2) => promisesObj[key2]);
|
|
318263
319752
|
return Promise.all(promises).then((results) => {
|
|
318264
319753
|
const resolvedObj = {};
|
|
318265
319754
|
for (let i2 = 0;i2 < keys2.length; i2++) {
|
|
@@ -318323,8 +319812,8 @@ function shallowClone(o) {
|
|
|
318323
319812
|
}
|
|
318324
319813
|
function numKeys(data) {
|
|
318325
319814
|
let keyCount = 0;
|
|
318326
|
-
for (const
|
|
318327
|
-
if (Object.prototype.hasOwnProperty.call(data,
|
|
319815
|
+
for (const key2 in data) {
|
|
319816
|
+
if (Object.prototype.hasOwnProperty.call(data, key2)) {
|
|
318328
319817
|
keyCount++;
|
|
318329
319818
|
}
|
|
318330
319819
|
}
|
|
@@ -318467,13 +319956,13 @@ function pick(schema, mask2) {
|
|
|
318467
319956
|
const def = mergeDefs(schema._zod.def, {
|
|
318468
319957
|
get shape() {
|
|
318469
319958
|
const newShape = {};
|
|
318470
|
-
for (const
|
|
318471
|
-
if (!(
|
|
318472
|
-
throw new Error(`Unrecognized key: "${
|
|
319959
|
+
for (const key2 in mask2) {
|
|
319960
|
+
if (!(key2 in currDef.shape)) {
|
|
319961
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
318473
319962
|
}
|
|
318474
|
-
if (!mask2[
|
|
319963
|
+
if (!mask2[key2])
|
|
318475
319964
|
continue;
|
|
318476
|
-
newShape[
|
|
319965
|
+
newShape[key2] = currDef.shape[key2];
|
|
318477
319966
|
}
|
|
318478
319967
|
assignProp(this, "shape", newShape);
|
|
318479
319968
|
return newShape;
|
|
@@ -318492,13 +319981,13 @@ function omit5(schema, mask2) {
|
|
|
318492
319981
|
const def = mergeDefs(schema._zod.def, {
|
|
318493
319982
|
get shape() {
|
|
318494
319983
|
const newShape = { ...schema._zod.def.shape };
|
|
318495
|
-
for (const
|
|
318496
|
-
if (!(
|
|
318497
|
-
throw new Error(`Unrecognized key: "${
|
|
319984
|
+
for (const key2 in mask2) {
|
|
319985
|
+
if (!(key2 in currDef.shape)) {
|
|
319986
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
318498
319987
|
}
|
|
318499
|
-
if (!mask2[
|
|
319988
|
+
if (!mask2[key2])
|
|
318500
319989
|
continue;
|
|
318501
|
-
delete newShape[
|
|
319990
|
+
delete newShape[key2];
|
|
318502
319991
|
}
|
|
318503
319992
|
assignProp(this, "shape", newShape);
|
|
318504
319993
|
return newShape;
|
|
@@ -318515,8 +320004,8 @@ function extend4(schema, shape) {
|
|
|
318515
320004
|
const hasChecks = checks && checks.length > 0;
|
|
318516
320005
|
if (hasChecks) {
|
|
318517
320006
|
const existingShape = schema._zod.def.shape;
|
|
318518
|
-
for (const
|
|
318519
|
-
if (Object.getOwnPropertyDescriptor(existingShape,
|
|
320007
|
+
for (const key2 in shape) {
|
|
320008
|
+
if (Object.getOwnPropertyDescriptor(existingShape, key2) !== undefined) {
|
|
318520
320009
|
throw new Error("Cannot overwrite keys on object schemas containing refinements. Use `.safeExtend()` instead.");
|
|
318521
320010
|
}
|
|
318522
320011
|
}
|
|
@@ -318569,23 +320058,23 @@ function partial(Class, schema, mask2) {
|
|
|
318569
320058
|
const oldShape = schema._zod.def.shape;
|
|
318570
320059
|
const shape = { ...oldShape };
|
|
318571
320060
|
if (mask2) {
|
|
318572
|
-
for (const
|
|
318573
|
-
if (!(
|
|
318574
|
-
throw new Error(`Unrecognized key: "${
|
|
320061
|
+
for (const key2 in mask2) {
|
|
320062
|
+
if (!(key2 in oldShape)) {
|
|
320063
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
318575
320064
|
}
|
|
318576
|
-
if (!mask2[
|
|
320065
|
+
if (!mask2[key2])
|
|
318577
320066
|
continue;
|
|
318578
|
-
shape[
|
|
320067
|
+
shape[key2] = Class ? new Class({
|
|
318579
320068
|
type: "optional",
|
|
318580
|
-
innerType: oldShape[
|
|
318581
|
-
}) : oldShape[
|
|
320069
|
+
innerType: oldShape[key2]
|
|
320070
|
+
}) : oldShape[key2];
|
|
318582
320071
|
}
|
|
318583
320072
|
} else {
|
|
318584
|
-
for (const
|
|
318585
|
-
shape[
|
|
320073
|
+
for (const key2 in oldShape) {
|
|
320074
|
+
shape[key2] = Class ? new Class({
|
|
318586
320075
|
type: "optional",
|
|
318587
|
-
innerType: oldShape[
|
|
318588
|
-
}) : oldShape[
|
|
320076
|
+
innerType: oldShape[key2]
|
|
320077
|
+
}) : oldShape[key2];
|
|
318589
320078
|
}
|
|
318590
320079
|
}
|
|
318591
320080
|
assignProp(this, "shape", shape);
|
|
@@ -318601,22 +320090,22 @@ function required(Class, schema, mask2) {
|
|
|
318601
320090
|
const oldShape = schema._zod.def.shape;
|
|
318602
320091
|
const shape = { ...oldShape };
|
|
318603
320092
|
if (mask2) {
|
|
318604
|
-
for (const
|
|
318605
|
-
if (!(
|
|
318606
|
-
throw new Error(`Unrecognized key: "${
|
|
320093
|
+
for (const key2 in mask2) {
|
|
320094
|
+
if (!(key2 in shape)) {
|
|
320095
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
318607
320096
|
}
|
|
318608
|
-
if (!mask2[
|
|
320097
|
+
if (!mask2[key2])
|
|
318609
320098
|
continue;
|
|
318610
|
-
shape[
|
|
320099
|
+
shape[key2] = new Class({
|
|
318611
320100
|
type: "nonoptional",
|
|
318612
|
-
innerType: oldShape[
|
|
320101
|
+
innerType: oldShape[key2]
|
|
318613
320102
|
});
|
|
318614
320103
|
}
|
|
318615
320104
|
} else {
|
|
318616
|
-
for (const
|
|
318617
|
-
shape[
|
|
320105
|
+
for (const key2 in oldShape) {
|
|
320106
|
+
shape[key2] = new Class({
|
|
318618
320107
|
type: "nonoptional",
|
|
318619
|
-
innerType: oldShape[
|
|
320108
|
+
innerType: oldShape[key2]
|
|
318620
320109
|
});
|
|
318621
320110
|
}
|
|
318622
320111
|
}
|
|
@@ -320366,19 +321855,19 @@ var $ZodArray = /* @__PURE__ */ $constructor("$ZodArray", (inst, def) => {
|
|
|
320366
321855
|
return payload;
|
|
320367
321856
|
};
|
|
320368
321857
|
});
|
|
320369
|
-
function handlePropertyResult(result, final,
|
|
321858
|
+
function handlePropertyResult(result, final, key2, input, isOptionalOut) {
|
|
320370
321859
|
if (result.issues.length) {
|
|
320371
|
-
if (isOptionalOut && !(
|
|
321860
|
+
if (isOptionalOut && !(key2 in input)) {
|
|
320372
321861
|
return;
|
|
320373
321862
|
}
|
|
320374
|
-
final.issues.push(...prefixIssues(
|
|
321863
|
+
final.issues.push(...prefixIssues(key2, result.issues));
|
|
320375
321864
|
}
|
|
320376
321865
|
if (result.value === undefined) {
|
|
320377
|
-
if (
|
|
320378
|
-
final.value[
|
|
321866
|
+
if (key2 in input) {
|
|
321867
|
+
final.value[key2] = undefined;
|
|
320379
321868
|
}
|
|
320380
321869
|
} else {
|
|
320381
|
-
final.value[
|
|
321870
|
+
final.value[key2] = result.value;
|
|
320382
321871
|
}
|
|
320383
321872
|
}
|
|
320384
321873
|
function normalizeDef(def) {
|
|
@@ -320403,18 +321892,18 @@ function handleCatchall(proms, input, payload, ctx, def, inst) {
|
|
|
320403
321892
|
const _catchall = def.catchall._zod;
|
|
320404
321893
|
const t = _catchall.def.type;
|
|
320405
321894
|
const isOptionalOut = _catchall.optout === "optional";
|
|
320406
|
-
for (const
|
|
320407
|
-
if (keySet.has(
|
|
321895
|
+
for (const key2 in input) {
|
|
321896
|
+
if (keySet.has(key2))
|
|
320408
321897
|
continue;
|
|
320409
321898
|
if (t === "never") {
|
|
320410
|
-
unrecognized.push(
|
|
321899
|
+
unrecognized.push(key2);
|
|
320411
321900
|
continue;
|
|
320412
321901
|
}
|
|
320413
|
-
const r = _catchall.run({ value: input[
|
|
321902
|
+
const r = _catchall.run({ value: input[key2], issues: [] }, ctx);
|
|
320414
321903
|
if (r instanceof Promise) {
|
|
320415
|
-
proms.push(r.then((r2) => handlePropertyResult(r2, payload,
|
|
321904
|
+
proms.push(r.then((r2) => handlePropertyResult(r2, payload, key2, input, isOptionalOut)));
|
|
320416
321905
|
} else {
|
|
320417
|
-
handlePropertyResult(r, payload,
|
|
321906
|
+
handlePropertyResult(r, payload, key2, input, isOptionalOut);
|
|
320418
321907
|
}
|
|
320419
321908
|
}
|
|
320420
321909
|
if (unrecognized.length) {
|
|
@@ -320450,12 +321939,12 @@ var $ZodObject = /* @__PURE__ */ $constructor("$ZodObject", (inst, def) => {
|
|
|
320450
321939
|
defineLazy(inst._zod, "propValues", () => {
|
|
320451
321940
|
const shape = def.shape;
|
|
320452
321941
|
const propValues = {};
|
|
320453
|
-
for (const
|
|
320454
|
-
const field = shape[
|
|
321942
|
+
for (const key2 in shape) {
|
|
321943
|
+
const field = shape[key2]._zod;
|
|
320455
321944
|
if (field.values) {
|
|
320456
|
-
propValues[
|
|
321945
|
+
propValues[key2] ?? (propValues[key2] = new Set);
|
|
320457
321946
|
for (const v of field.values)
|
|
320458
|
-
propValues[
|
|
321947
|
+
propValues[key2].add(v);
|
|
320459
321948
|
}
|
|
320460
321949
|
}
|
|
320461
321950
|
return propValues;
|
|
@@ -320478,14 +321967,14 @@ var $ZodObject = /* @__PURE__ */ $constructor("$ZodObject", (inst, def) => {
|
|
|
320478
321967
|
payload.value = {};
|
|
320479
321968
|
const proms = [];
|
|
320480
321969
|
const shape = value.shape;
|
|
320481
|
-
for (const
|
|
320482
|
-
const el = shape[
|
|
321970
|
+
for (const key2 of value.keys) {
|
|
321971
|
+
const el = shape[key2];
|
|
320483
321972
|
const isOptionalOut = el._zod.optout === "optional";
|
|
320484
|
-
const r = el._zod.run({ value: input[
|
|
321973
|
+
const r = el._zod.run({ value: input[key2], issues: [] }, ctx);
|
|
320485
321974
|
if (r instanceof Promise) {
|
|
320486
|
-
proms.push(r.then((r2) => handlePropertyResult(r2, payload,
|
|
321975
|
+
proms.push(r.then((r2) => handlePropertyResult(r2, payload, key2, input, isOptionalOut)));
|
|
320487
321976
|
} else {
|
|
320488
|
-
handlePropertyResult(r, payload,
|
|
321977
|
+
handlePropertyResult(r, payload, key2, input, isOptionalOut);
|
|
320489
321978
|
}
|
|
320490
321979
|
}
|
|
320491
321980
|
if (!catchall) {
|
|
@@ -320501,23 +321990,23 @@ var $ZodObjectJIT = /* @__PURE__ */ $constructor("$ZodObjectJIT", (inst, def) =>
|
|
|
320501
321990
|
const generateFastpass = (shape) => {
|
|
320502
321991
|
const doc = new Doc(["shape", "payload", "ctx"]);
|
|
320503
321992
|
const normalized = _normalized.value;
|
|
320504
|
-
const parseStr = (
|
|
320505
|
-
const k = esc(
|
|
321993
|
+
const parseStr = (key2) => {
|
|
321994
|
+
const k = esc(key2);
|
|
320506
321995
|
return `shape[${k}]._zod.run({ value: input[${k}], issues: [] }, ctx)`;
|
|
320507
321996
|
};
|
|
320508
321997
|
doc.write(`const input = payload.value;`);
|
|
320509
321998
|
const ids = Object.create(null);
|
|
320510
|
-
let
|
|
320511
|
-
for (const
|
|
320512
|
-
ids[
|
|
321999
|
+
let counter2 = 0;
|
|
322000
|
+
for (const key2 of normalized.keys) {
|
|
322001
|
+
ids[key2] = `key_${counter2++}`;
|
|
320513
322002
|
}
|
|
320514
322003
|
doc.write(`const newResult = {};`);
|
|
320515
|
-
for (const
|
|
320516
|
-
const id2 = ids[
|
|
320517
|
-
const k = esc(
|
|
320518
|
-
const schema = shape[
|
|
322004
|
+
for (const key2 of normalized.keys) {
|
|
322005
|
+
const id2 = ids[key2];
|
|
322006
|
+
const k = esc(key2);
|
|
322007
|
+
const schema = shape[key2];
|
|
320519
322008
|
const isOptionalOut = schema?._zod?.optout === "optional";
|
|
320520
|
-
doc.write(`const ${id2} = ${parseStr(
|
|
322009
|
+
doc.write(`const ${id2} = ${parseStr(key2)};`);
|
|
320521
322010
|
if (isOptionalOut) {
|
|
320522
322011
|
doc.write(`
|
|
320523
322012
|
if (${id2}.issues.length) {
|
|
@@ -320803,17 +322292,17 @@ function mergeValues(a, b2) {
|
|
|
320803
322292
|
}
|
|
320804
322293
|
if (isPlainObject2(a) && isPlainObject2(b2)) {
|
|
320805
322294
|
const bKeys = Object.keys(b2);
|
|
320806
|
-
const sharedKeys = Object.keys(a).filter((
|
|
322295
|
+
const sharedKeys = Object.keys(a).filter((key2) => bKeys.indexOf(key2) !== -1);
|
|
320807
322296
|
const newObj = { ...a, ...b2 };
|
|
320808
|
-
for (const
|
|
320809
|
-
const sharedValue = mergeValues(a[
|
|
322297
|
+
for (const key2 of sharedKeys) {
|
|
322298
|
+
const sharedValue = mergeValues(a[key2], b2[key2]);
|
|
320810
322299
|
if (!sharedValue.valid) {
|
|
320811
322300
|
return {
|
|
320812
322301
|
valid: false,
|
|
320813
|
-
mergeErrorPath: [
|
|
322302
|
+
mergeErrorPath: [key2, ...sharedValue.mergeErrorPath]
|
|
320814
322303
|
};
|
|
320815
322304
|
}
|
|
320816
|
-
newObj[
|
|
322305
|
+
newObj[key2] = sharedValue.data;
|
|
320817
322306
|
}
|
|
320818
322307
|
return { valid: true, data: newObj };
|
|
320819
322308
|
}
|
|
@@ -320969,30 +322458,30 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
|
|
|
320969
322458
|
if (values2) {
|
|
320970
322459
|
payload.value = {};
|
|
320971
322460
|
const recordKeys = new Set;
|
|
320972
|
-
for (const
|
|
320973
|
-
if (typeof
|
|
320974
|
-
recordKeys.add(typeof
|
|
320975
|
-
const result = def.valueType._zod.run({ value: input[
|
|
322461
|
+
for (const key2 of values2) {
|
|
322462
|
+
if (typeof key2 === "string" || typeof key2 === "number" || typeof key2 === "symbol") {
|
|
322463
|
+
recordKeys.add(typeof key2 === "number" ? key2.toString() : key2);
|
|
322464
|
+
const result = def.valueType._zod.run({ value: input[key2], issues: [] }, ctx);
|
|
320976
322465
|
if (result instanceof Promise) {
|
|
320977
322466
|
proms.push(result.then((result2) => {
|
|
320978
322467
|
if (result2.issues.length) {
|
|
320979
|
-
payload.issues.push(...prefixIssues(
|
|
322468
|
+
payload.issues.push(...prefixIssues(key2, result2.issues));
|
|
320980
322469
|
}
|
|
320981
|
-
payload.value[
|
|
322470
|
+
payload.value[key2] = result2.value;
|
|
320982
322471
|
}));
|
|
320983
322472
|
} else {
|
|
320984
322473
|
if (result.issues.length) {
|
|
320985
|
-
payload.issues.push(...prefixIssues(
|
|
322474
|
+
payload.issues.push(...prefixIssues(key2, result.issues));
|
|
320986
322475
|
}
|
|
320987
|
-
payload.value[
|
|
322476
|
+
payload.value[key2] = result.value;
|
|
320988
322477
|
}
|
|
320989
322478
|
}
|
|
320990
322479
|
}
|
|
320991
322480
|
let unrecognized;
|
|
320992
|
-
for (const
|
|
320993
|
-
if (!recordKeys.has(
|
|
322481
|
+
for (const key2 in input) {
|
|
322482
|
+
if (!recordKeys.has(key2)) {
|
|
320994
322483
|
unrecognized = unrecognized ?? [];
|
|
320995
|
-
unrecognized.push(
|
|
322484
|
+
unrecognized.push(key2);
|
|
320996
322485
|
}
|
|
320997
322486
|
}
|
|
320998
322487
|
if (unrecognized && unrecognized.length > 0) {
|
|
@@ -321005,16 +322494,16 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
|
|
|
321005
322494
|
}
|
|
321006
322495
|
} else {
|
|
321007
322496
|
payload.value = {};
|
|
321008
|
-
for (const
|
|
321009
|
-
if (
|
|
322497
|
+
for (const key2 of Reflect.ownKeys(input)) {
|
|
322498
|
+
if (key2 === "__proto__")
|
|
321010
322499
|
continue;
|
|
321011
|
-
let keyResult = def.keyType._zod.run({ value:
|
|
322500
|
+
let keyResult = def.keyType._zod.run({ value: key2, issues: [] }, ctx);
|
|
321012
322501
|
if (keyResult instanceof Promise) {
|
|
321013
322502
|
throw new Error("Async schemas not supported in object keys currently");
|
|
321014
322503
|
}
|
|
321015
|
-
const checkNumericKey = typeof
|
|
322504
|
+
const checkNumericKey = typeof key2 === "string" && number3.test(key2) && keyResult.issues.length;
|
|
321016
322505
|
if (checkNumericKey) {
|
|
321017
|
-
const retryResult = def.keyType._zod.run({ value: Number(
|
|
322506
|
+
const retryResult = def.keyType._zod.run({ value: Number(key2), issues: [] }, ctx);
|
|
321018
322507
|
if (retryResult instanceof Promise) {
|
|
321019
322508
|
throw new Error("Async schemas not supported in object keys currently");
|
|
321020
322509
|
}
|
|
@@ -321024,30 +322513,30 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
|
|
|
321024
322513
|
}
|
|
321025
322514
|
if (keyResult.issues.length) {
|
|
321026
322515
|
if (def.mode === "loose") {
|
|
321027
|
-
payload.value[
|
|
322516
|
+
payload.value[key2] = input[key2];
|
|
321028
322517
|
} else {
|
|
321029
322518
|
payload.issues.push({
|
|
321030
322519
|
code: "invalid_key",
|
|
321031
322520
|
origin: "record",
|
|
321032
322521
|
issues: keyResult.issues.map((iss) => finalizeIssue(iss, ctx, config())),
|
|
321033
|
-
input:
|
|
321034
|
-
path: [
|
|
322522
|
+
input: key2,
|
|
322523
|
+
path: [key2],
|
|
321035
322524
|
inst
|
|
321036
322525
|
});
|
|
321037
322526
|
}
|
|
321038
322527
|
continue;
|
|
321039
322528
|
}
|
|
321040
|
-
const result = def.valueType._zod.run({ value: input[
|
|
322529
|
+
const result = def.valueType._zod.run({ value: input[key2], issues: [] }, ctx);
|
|
321041
322530
|
if (result instanceof Promise) {
|
|
321042
322531
|
proms.push(result.then((result2) => {
|
|
321043
322532
|
if (result2.issues.length) {
|
|
321044
|
-
payload.issues.push(...prefixIssues(
|
|
322533
|
+
payload.issues.push(...prefixIssues(key2, result2.issues));
|
|
321045
322534
|
}
|
|
321046
322535
|
payload.value[keyResult.value] = result2.value;
|
|
321047
322536
|
}));
|
|
321048
322537
|
} else {
|
|
321049
322538
|
if (result.issues.length) {
|
|
321050
|
-
payload.issues.push(...prefixIssues(
|
|
322539
|
+
payload.issues.push(...prefixIssues(key2, result.issues));
|
|
321051
322540
|
}
|
|
321052
322541
|
payload.value[keyResult.value] = result.value;
|
|
321053
322542
|
}
|
|
@@ -321074,15 +322563,15 @@ var $ZodMap = /* @__PURE__ */ $constructor("$ZodMap", (inst, def) => {
|
|
|
321074
322563
|
}
|
|
321075
322564
|
const proms = [];
|
|
321076
322565
|
payload.value = new Map;
|
|
321077
|
-
for (const [
|
|
321078
|
-
const keyResult = def.keyType._zod.run({ value:
|
|
322566
|
+
for (const [key2, value] of input) {
|
|
322567
|
+
const keyResult = def.keyType._zod.run({ value: key2, issues: [] }, ctx);
|
|
321079
322568
|
const valueResult = def.valueType._zod.run({ value, issues: [] }, ctx);
|
|
321080
322569
|
if (keyResult instanceof Promise || valueResult instanceof Promise) {
|
|
321081
322570
|
proms.push(Promise.all([keyResult, valueResult]).then(([keyResult2, valueResult2]) => {
|
|
321082
|
-
handleMapResult(keyResult2, valueResult2, payload,
|
|
322571
|
+
handleMapResult(keyResult2, valueResult2, payload, key2, input, inst, ctx);
|
|
321083
322572
|
}));
|
|
321084
322573
|
} else {
|
|
321085
|
-
handleMapResult(keyResult, valueResult, payload,
|
|
322574
|
+
handleMapResult(keyResult, valueResult, payload, key2, input, inst, ctx);
|
|
321086
322575
|
}
|
|
321087
322576
|
}
|
|
321088
322577
|
if (proms.length)
|
|
@@ -321090,10 +322579,10 @@ var $ZodMap = /* @__PURE__ */ $constructor("$ZodMap", (inst, def) => {
|
|
|
321090
322579
|
return payload;
|
|
321091
322580
|
};
|
|
321092
322581
|
});
|
|
321093
|
-
function handleMapResult(keyResult, valueResult, final,
|
|
322582
|
+
function handleMapResult(keyResult, valueResult, final, key2, input, inst, ctx) {
|
|
321094
322583
|
if (keyResult.issues.length) {
|
|
321095
|
-
if (propertyKeyTypes.has(typeof
|
|
321096
|
-
final.issues.push(...prefixIssues(
|
|
322584
|
+
if (propertyKeyTypes.has(typeof key2)) {
|
|
322585
|
+
final.issues.push(...prefixIssues(key2, keyResult.issues));
|
|
321097
322586
|
} else {
|
|
321098
322587
|
final.issues.push({
|
|
321099
322588
|
code: "invalid_key",
|
|
@@ -321105,15 +322594,15 @@ function handleMapResult(keyResult, valueResult, final, key, input, inst, ctx) {
|
|
|
321105
322594
|
}
|
|
321106
322595
|
}
|
|
321107
322596
|
if (valueResult.issues.length) {
|
|
321108
|
-
if (propertyKeyTypes.has(typeof
|
|
321109
|
-
final.issues.push(...prefixIssues(
|
|
322597
|
+
if (propertyKeyTypes.has(typeof key2)) {
|
|
322598
|
+
final.issues.push(...prefixIssues(key2, valueResult.issues));
|
|
321110
322599
|
} else {
|
|
321111
322600
|
final.issues.push({
|
|
321112
322601
|
origin: "map",
|
|
321113
322602
|
code: "invalid_element",
|
|
321114
322603
|
input,
|
|
321115
322604
|
inst,
|
|
321116
|
-
key,
|
|
322605
|
+
key: key2,
|
|
321117
322606
|
issues: valueResult.issues.map((iss) => finalizeIssue(iss, ctx, config()))
|
|
321118
322607
|
});
|
|
321119
322608
|
}
|
|
@@ -321443,12 +322932,12 @@ var $ZodPipe = /* @__PURE__ */ $constructor("$ZodPipe", (inst, def) => {
|
|
|
321443
322932
|
return handlePipeResult(left, def.out, ctx);
|
|
321444
322933
|
};
|
|
321445
322934
|
});
|
|
321446
|
-
function handlePipeResult(left,
|
|
322935
|
+
function handlePipeResult(left, next2, ctx) {
|
|
321447
322936
|
if (left.issues.length) {
|
|
321448
322937
|
left.aborted = true;
|
|
321449
322938
|
return left;
|
|
321450
322939
|
}
|
|
321451
|
-
return
|
|
322940
|
+
return next2._zod.run({ value: left.value, issues: left.issues }, ctx);
|
|
321452
322941
|
}
|
|
321453
322942
|
var $ZodCodec = /* @__PURE__ */ $constructor("$ZodCodec", (inst, def) => {
|
|
321454
322943
|
$ZodType.init(inst, def);
|
|
@@ -328307,8 +329796,8 @@ function extractDefs(ctx, schema) {
|
|
|
328307
329796
|
if (defId)
|
|
328308
329797
|
seen.defId = defId;
|
|
328309
329798
|
const schema2 = seen.schema;
|
|
328310
|
-
for (const
|
|
328311
|
-
delete schema2[
|
|
329799
|
+
for (const key2 in schema2) {
|
|
329800
|
+
delete schema2[key2];
|
|
328312
329801
|
}
|
|
328313
329802
|
schema2.$ref = ref;
|
|
328314
329803
|
};
|
|
@@ -328375,20 +329864,20 @@ function finalize(ctx, schema) {
|
|
|
328375
329864
|
Object.assign(schema2, _cached);
|
|
328376
329865
|
const isParentRef = zodSchema._zod.parent === ref;
|
|
328377
329866
|
if (isParentRef) {
|
|
328378
|
-
for (const
|
|
328379
|
-
if (
|
|
329867
|
+
for (const key2 in schema2) {
|
|
329868
|
+
if (key2 === "$ref" || key2 === "allOf")
|
|
328380
329869
|
continue;
|
|
328381
|
-
if (!(
|
|
328382
|
-
delete schema2[
|
|
329870
|
+
if (!(key2 in _cached)) {
|
|
329871
|
+
delete schema2[key2];
|
|
328383
329872
|
}
|
|
328384
329873
|
}
|
|
328385
329874
|
}
|
|
328386
329875
|
if (refSchema.$ref && refSeen.def) {
|
|
328387
|
-
for (const
|
|
328388
|
-
if (
|
|
329876
|
+
for (const key2 in schema2) {
|
|
329877
|
+
if (key2 === "$ref" || key2 === "allOf")
|
|
328389
329878
|
continue;
|
|
328390
|
-
if (
|
|
328391
|
-
delete schema2[
|
|
329879
|
+
if (key2 in refSeen.def && JSON.stringify(schema2[key2]) === JSON.stringify(refSeen.def[key2])) {
|
|
329880
|
+
delete schema2[key2];
|
|
328392
329881
|
}
|
|
328393
329882
|
}
|
|
328394
329883
|
}
|
|
@@ -328400,11 +329889,11 @@ function finalize(ctx, schema) {
|
|
|
328400
329889
|
if (parentSeen?.schema.$ref) {
|
|
328401
329890
|
schema2.$ref = parentSeen.schema.$ref;
|
|
328402
329891
|
if (parentSeen.def) {
|
|
328403
|
-
for (const
|
|
328404
|
-
if (
|
|
329892
|
+
for (const key2 in schema2) {
|
|
329893
|
+
if (key2 === "$ref" || key2 === "allOf")
|
|
328405
329894
|
continue;
|
|
328406
|
-
if (
|
|
328407
|
-
delete schema2[
|
|
329895
|
+
if (key2 in parentSeen.def && JSON.stringify(schema2[key2]) === JSON.stringify(parentSeen.def[key2])) {
|
|
329896
|
+
delete schema2[key2];
|
|
328408
329897
|
}
|
|
328409
329898
|
}
|
|
328410
329899
|
}
|
|
@@ -328426,7 +329915,7 @@ function finalize(ctx, schema) {
|
|
|
328426
329915
|
result.$schema = "http://json-schema.org/draft-07/schema#";
|
|
328427
329916
|
} else if (ctx.target === "draft-04") {
|
|
328428
329917
|
result.$schema = "http://json-schema.org/draft-04/schema#";
|
|
328429
|
-
} else if (ctx.target === "openapi-3.0") {}
|
|
329918
|
+
} else if (ctx.target === "openapi-3.0") {}
|
|
328430
329919
|
if (ctx.external?.uri) {
|
|
328431
329920
|
const id2 = ctx.external.registry.get(schema)?.id;
|
|
328432
329921
|
if (!id2)
|
|
@@ -328495,8 +329984,8 @@ function isTransforming(_schema, _ctx) {
|
|
|
328495
329984
|
return isTransforming(def.in, ctx) || isTransforming(def.out, ctx);
|
|
328496
329985
|
}
|
|
328497
329986
|
if (def.type === "object") {
|
|
328498
|
-
for (const
|
|
328499
|
-
if (isTransforming(def.shape[
|
|
329987
|
+
for (const key2 in def.shape) {
|
|
329988
|
+
if (isTransforming(def.shape[key2], ctx))
|
|
328500
329989
|
return true;
|
|
328501
329990
|
}
|
|
328502
329991
|
return false;
|
|
@@ -328674,7 +330163,7 @@ var literalProcessor = (schema, ctx, json3, _params) => {
|
|
|
328674
330163
|
if (val === undefined) {
|
|
328675
330164
|
if (ctx.unrepresentable === "throw") {
|
|
328676
330165
|
throw new Error("Literal `undefined` cannot be represented in JSON Schema");
|
|
328677
|
-
}
|
|
330166
|
+
}
|
|
328678
330167
|
} else if (typeof val === "bigint") {
|
|
328679
330168
|
if (ctx.unrepresentable === "throw") {
|
|
328680
330169
|
throw new Error("BigInt literals cannot be represented in JSON Schema");
|
|
@@ -328787,15 +330276,15 @@ var objectProcessor = (schema, ctx, _json, params) => {
|
|
|
328787
330276
|
json3.type = "object";
|
|
328788
330277
|
json3.properties = {};
|
|
328789
330278
|
const shape = def.shape;
|
|
328790
|
-
for (const
|
|
328791
|
-
json3.properties[
|
|
330279
|
+
for (const key2 in shape) {
|
|
330280
|
+
json3.properties[key2] = process2(shape[key2], ctx, {
|
|
328792
330281
|
...params,
|
|
328793
|
-
path: [...params.path, "properties",
|
|
330282
|
+
path: [...params.path, "properties", key2]
|
|
328794
330283
|
});
|
|
328795
330284
|
}
|
|
328796
330285
|
const allKeys = new Set(Object.keys(shape));
|
|
328797
|
-
const requiredKeys = new Set([...allKeys].filter((
|
|
328798
|
-
const v = def.shape[
|
|
330286
|
+
const requiredKeys = new Set([...allKeys].filter((key2) => {
|
|
330287
|
+
const v = def.shape[key2]._zod;
|
|
328799
330288
|
if (ctx.io === "input") {
|
|
328800
330289
|
return v.optin === undefined;
|
|
328801
330290
|
} else {
|
|
@@ -329060,9 +330549,9 @@ function toJSONSchema(input, params) {
|
|
|
329060
330549
|
};
|
|
329061
330550
|
ctx2.external = external;
|
|
329062
330551
|
for (const entry of registry2._idmap.entries()) {
|
|
329063
|
-
const [
|
|
330552
|
+
const [key2, schema] = entry;
|
|
329064
330553
|
extractDefs(ctx2, schema);
|
|
329065
|
-
schemas[
|
|
330554
|
+
schemas[key2] = finalize(ctx2, schema);
|
|
329066
330555
|
}
|
|
329067
330556
|
if (Object.keys(defs).length > 0) {
|
|
329068
330557
|
const defsSegment = ctx2.target === "draft-2020-12" ? "$defs" : "definitions";
|
|
@@ -330619,11 +332108,11 @@ function resolveRef(ref, ctx) {
|
|
|
330619
332108
|
}
|
|
330620
332109
|
const defsKey = ctx.version === "draft-2020-12" ? "$defs" : "definitions";
|
|
330621
332110
|
if (path[0] === defsKey) {
|
|
330622
|
-
const
|
|
330623
|
-
if (!
|
|
332111
|
+
const key2 = path[1];
|
|
332112
|
+
if (!key2 || !ctx.defs[key2]) {
|
|
330624
332113
|
throw new Error(`Reference not found: ${ref}`);
|
|
330625
332114
|
}
|
|
330626
|
-
return ctx.defs[
|
|
332115
|
+
return ctx.defs[key2];
|
|
330627
332116
|
}
|
|
330628
332117
|
throw new Error(`Reference not found: ${ref}`);
|
|
330629
332118
|
}
|
|
@@ -330809,9 +332298,9 @@ function convertBaseSchema(schema, ctx) {
|
|
|
330809
332298
|
const shape = {};
|
|
330810
332299
|
const properties = schema.properties || {};
|
|
330811
332300
|
const requiredSet = new Set(schema.required || []);
|
|
330812
|
-
for (const [
|
|
332301
|
+
for (const [key2, propSchema] of Object.entries(properties)) {
|
|
330813
332302
|
const propZodSchema = convertSchema(propSchema, ctx);
|
|
330814
|
-
shape[
|
|
332303
|
+
shape[key2] = requiredSet.has(key2) ? propZodSchema : propZodSchema.optional();
|
|
330815
332304
|
}
|
|
330816
332305
|
if (schema.propertyNames) {
|
|
330817
332306
|
const keySchema = convertSchema(schema.propertyNames, ctx);
|
|
@@ -330955,20 +332444,20 @@ function convertSchema(schema, ctx) {
|
|
|
330955
332444
|
}
|
|
330956
332445
|
const extraMeta = {};
|
|
330957
332446
|
const coreMetadataKeys = ["$id", "id", "$comment", "$anchor", "$vocabulary", "$dynamicRef", "$dynamicAnchor"];
|
|
330958
|
-
for (const
|
|
330959
|
-
if (
|
|
330960
|
-
extraMeta[
|
|
332447
|
+
for (const key2 of coreMetadataKeys) {
|
|
332448
|
+
if (key2 in schema) {
|
|
332449
|
+
extraMeta[key2] = schema[key2];
|
|
330961
332450
|
}
|
|
330962
332451
|
}
|
|
330963
332452
|
const contentMetadataKeys = ["contentEncoding", "contentMediaType", "contentSchema"];
|
|
330964
|
-
for (const
|
|
330965
|
-
if (
|
|
330966
|
-
extraMeta[
|
|
332453
|
+
for (const key2 of contentMetadataKeys) {
|
|
332454
|
+
if (key2 in schema) {
|
|
332455
|
+
extraMeta[key2] = schema[key2];
|
|
330967
332456
|
}
|
|
330968
332457
|
}
|
|
330969
|
-
for (const
|
|
330970
|
-
if (!RECOGNIZED_KEYS.has(
|
|
330971
|
-
extraMeta[
|
|
332458
|
+
for (const key2 of Object.keys(schema)) {
|
|
332459
|
+
if (!RECOGNIZED_KEYS.has(key2)) {
|
|
332460
|
+
extraMeta[key2] = schema[key2];
|
|
330972
332461
|
}
|
|
330973
332462
|
}
|
|
330974
332463
|
if (Object.keys(extraMeta).length > 0) {
|
|
@@ -331333,7 +332822,7 @@ async function handleDeposit(ctx) {
|
|
|
331333
332822
|
network: depositNetwork?.exchangeNetworkId,
|
|
331334
332823
|
externalId: depositTxid,
|
|
331335
332824
|
txid: depositTxid,
|
|
331336
|
-
exchangeTimestamp:
|
|
332825
|
+
exchangeTimestamp: normalizeTimestamp2(creditedAt),
|
|
331337
332826
|
payload: deposit
|
|
331338
332827
|
}
|
|
331339
332828
|
});
|
|
@@ -331666,208 +333155,6 @@ function buildHistoricalOrderBookUnsupported(payload) {
|
|
|
331666
333155
|
// src/handlers/execute-action/order-book-call.ts
|
|
331667
333156
|
var grpc4 = __toESM(require_src3(), 1);
|
|
331668
333157
|
|
|
331669
|
-
// src/helpers/market-data-archive/capture-contract.ts
|
|
331670
|
-
import { createHash as createHash3 } from "node:crypto";
|
|
331671
|
-
var MARKET_CAPTURE_SCHEMA_VERSION = "1.0.0";
|
|
331672
|
-
var CHECKSUM_ALGORITHM = "sha256-canonical-json-v1";
|
|
331673
|
-
var ARCHIVE_SOURCES = ["broker_read", "broker_write"];
|
|
331674
|
-
var CAPTURE_FEEDS = [
|
|
331675
|
-
"ORDERBOOK",
|
|
331676
|
-
"TICKER",
|
|
331677
|
-
"TRADES",
|
|
331678
|
-
"OHLCV"
|
|
331679
|
-
];
|
|
331680
|
-
var SOURCE_MODES = [
|
|
331681
|
-
"broker_live_stream_v1",
|
|
331682
|
-
"broker_live_sampling_v1",
|
|
331683
|
-
"broker_current_snapshot_v1",
|
|
331684
|
-
"broker_bootstrap_fetch_v1",
|
|
331685
|
-
"external_ccxt_fallback_v1",
|
|
331686
|
-
"external_hummingbot_fallback_v1",
|
|
331687
|
-
"legacy_migration_v1"
|
|
331688
|
-
];
|
|
331689
|
-
var RAW_CAPTURE_SCOPES = [
|
|
331690
|
-
"ccxt_normalized_object",
|
|
331691
|
-
"broker_visible_payload",
|
|
331692
|
-
"exchange_wire_frame"
|
|
331693
|
-
];
|
|
331694
|
-
var CHECKSUM_FIELDS = new Set([
|
|
331695
|
-
"normalized_row_checksum",
|
|
331696
|
-
"raw_checksum",
|
|
331697
|
-
"checksum"
|
|
331698
|
-
]);
|
|
331699
|
-
function canonicalDecimal(value) {
|
|
331700
|
-
if (!Number.isFinite(value)) {
|
|
331701
|
-
throw new Error("Canonical numbers must be finite");
|
|
331702
|
-
}
|
|
331703
|
-
if (Object.is(value, -0)) {
|
|
331704
|
-
return "0";
|
|
331705
|
-
}
|
|
331706
|
-
const rendered = String(value).toLowerCase();
|
|
331707
|
-
if (!rendered.includes("e")) {
|
|
331708
|
-
return rendered;
|
|
331709
|
-
}
|
|
331710
|
-
const [coefficient = "0", exponentText = "0"] = rendered.split("e");
|
|
331711
|
-
const exponent = Number.parseInt(exponentText, 10);
|
|
331712
|
-
const negative = coefficient.startsWith("-");
|
|
331713
|
-
const unsigned = negative ? coefficient.slice(1) : coefficient;
|
|
331714
|
-
const [integer2 = "0", fraction = ""] = unsigned.split(".");
|
|
331715
|
-
const digits = `${integer2}${fraction}`;
|
|
331716
|
-
const decimalIndex = integer2.length + exponent;
|
|
331717
|
-
let result;
|
|
331718
|
-
if (decimalIndex <= 0) {
|
|
331719
|
-
result = `0.${"0".repeat(-decimalIndex)}${digits}`;
|
|
331720
|
-
} else if (decimalIndex >= digits.length) {
|
|
331721
|
-
result = `${digits}${"0".repeat(decimalIndex - digits.length)}`;
|
|
331722
|
-
} else {
|
|
331723
|
-
result = `${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
|
|
331724
|
-
}
|
|
331725
|
-
return negative ? `-${result}` : result;
|
|
331726
|
-
}
|
|
331727
|
-
function serializeCanonical(value, stack) {
|
|
331728
|
-
if (value === null)
|
|
331729
|
-
return "null";
|
|
331730
|
-
if (typeof value === "string")
|
|
331731
|
-
return JSON.stringify(value);
|
|
331732
|
-
if (typeof value === "boolean")
|
|
331733
|
-
return value ? "true" : "false";
|
|
331734
|
-
if (typeof value === "number")
|
|
331735
|
-
return canonicalDecimal(value);
|
|
331736
|
-
if (typeof value === "bigint")
|
|
331737
|
-
return value.toString(10);
|
|
331738
|
-
if (value instanceof Date) {
|
|
331739
|
-
if (Number.isNaN(value.getTime())) {
|
|
331740
|
-
throw new Error("Canonical timestamps must be valid");
|
|
331741
|
-
}
|
|
331742
|
-
return value.getTime().toString(10);
|
|
331743
|
-
}
|
|
331744
|
-
if (Array.isArray(value)) {
|
|
331745
|
-
if (stack.has(value))
|
|
331746
|
-
throw new Error("Canonical values must be acyclic");
|
|
331747
|
-
stack.add(value);
|
|
331748
|
-
const result = `[${value.map((entry) => entry === undefined ? "null" : serializeCanonical(entry, stack)).join(",")}]`;
|
|
331749
|
-
stack.delete(value);
|
|
331750
|
-
return result;
|
|
331751
|
-
}
|
|
331752
|
-
if (typeof value === "object") {
|
|
331753
|
-
if (stack.has(value))
|
|
331754
|
-
throw new Error("Canonical values must be acyclic");
|
|
331755
|
-
stack.add(value);
|
|
331756
|
-
const entries = Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([left], [right]) => left.localeCompare(right));
|
|
331757
|
-
const result = `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${serializeCanonical(entry, stack)}`).join(",")}}`;
|
|
331758
|
-
stack.delete(value);
|
|
331759
|
-
return result;
|
|
331760
|
-
}
|
|
331761
|
-
throw new Error(`Unsupported canonical value type: ${typeof value}`);
|
|
331762
|
-
}
|
|
331763
|
-
function canonicalSerialize(value) {
|
|
331764
|
-
return serializeCanonical(value, new Set);
|
|
331765
|
-
}
|
|
331766
|
-
function omitChecksumFields(value) {
|
|
331767
|
-
if (Array.isArray(value))
|
|
331768
|
-
return value.map(omitChecksumFields);
|
|
331769
|
-
if (value && typeof value === "object" && !(value instanceof Date)) {
|
|
331770
|
-
return Object.fromEntries(Object.entries(value).filter(([key]) => !CHECKSUM_FIELDS.has(key)).map(([key, entry]) => [key, omitChecksumFields(entry)]));
|
|
331771
|
-
}
|
|
331772
|
-
return value;
|
|
331773
|
-
}
|
|
331774
|
-
function sha256Canonical(value) {
|
|
331775
|
-
return createHash3("sha256").update(canonicalSerialize(omitChecksumFields(value))).digest("hex");
|
|
331776
|
-
}
|
|
331777
|
-
function normalizeTimestampMs(value, field) {
|
|
331778
|
-
let timestamp;
|
|
331779
|
-
if (value instanceof Date) {
|
|
331780
|
-
timestamp = value.getTime();
|
|
331781
|
-
} else if (typeof value === "number") {
|
|
331782
|
-
timestamp = value;
|
|
331783
|
-
} else if (typeof value === "string" && /^\d+$/.test(value.trim())) {
|
|
331784
|
-
timestamp = Number(value.trim());
|
|
331785
|
-
} else if (typeof value === "string") {
|
|
331786
|
-
timestamp = Date.parse(value);
|
|
331787
|
-
} else {
|
|
331788
|
-
timestamp = Number.NaN;
|
|
331789
|
-
}
|
|
331790
|
-
if (!Number.isSafeInteger(timestamp) || timestamp < 0) {
|
|
331791
|
-
throw new Error(`${field} must be a non-negative millisecond timestamp`);
|
|
331792
|
-
}
|
|
331793
|
-
return timestamp;
|
|
331794
|
-
}
|
|
331795
|
-
function assertCaptureContext(context2) {
|
|
331796
|
-
if (!ARCHIVE_SOURCES.includes(context2.source)) {
|
|
331797
|
-
throw new Error(`Unsupported archive source: ${context2.source}`);
|
|
331798
|
-
}
|
|
331799
|
-
if (!CAPTURE_FEEDS.includes(context2.feed)) {
|
|
331800
|
-
throw new Error(`Unsupported capture feed: ${context2.feed}`);
|
|
331801
|
-
}
|
|
331802
|
-
if (!SOURCE_MODES.includes(context2.sourceMode)) {
|
|
331803
|
-
throw new Error(`Unsupported source mode: ${context2.sourceMode}`);
|
|
331804
|
-
}
|
|
331805
|
-
for (const [field, value] of [
|
|
331806
|
-
["deployment_id", context2.deploymentId],
|
|
331807
|
-
["capture_bundle_id", context2.captureBundleId],
|
|
331808
|
-
["exchange", context2.exchange],
|
|
331809
|
-
["symbol", context2.symbol],
|
|
331810
|
-
["provider", context2.provider]
|
|
331811
|
-
]) {
|
|
331812
|
-
if (!value.trim())
|
|
331813
|
-
throw new Error(`${field} must not be empty`);
|
|
331814
|
-
}
|
|
331815
|
-
}
|
|
331816
|
-
function createRawCapture(context2, input) {
|
|
331817
|
-
assertCaptureContext(context2);
|
|
331818
|
-
if (!RAW_CAPTURE_SCOPES.includes(input.scope)) {
|
|
331819
|
-
throw new Error(`Unsupported raw capture scope: ${input.scope}`);
|
|
331820
|
-
}
|
|
331821
|
-
const eventTimeMs = normalizeTimestampMs(input.eventTimeMs, "event_time_ms");
|
|
331822
|
-
const receivedTimeMs = normalizeTimestampMs(input.receivedTimeMs, "received_time_ms");
|
|
331823
|
-
const redactedPayload = redactStreamPayload(input.payload);
|
|
331824
|
-
const rawChecksum = sha256Canonical(redactedPayload);
|
|
331825
|
-
const rawCaptureId = sha256Canonical({
|
|
331826
|
-
capture_bundle_id: context2.captureBundleId,
|
|
331827
|
-
exchange: context2.exchange.trim().toLowerCase(),
|
|
331828
|
-
feed: context2.feed,
|
|
331829
|
-
raw_capture_scope: input.scope,
|
|
331830
|
-
raw_payload_sha256: rawChecksum,
|
|
331831
|
-
schema_version: context2.schemaVersion,
|
|
331832
|
-
source_mode: context2.sourceMode,
|
|
331833
|
-
source_symbol: context2.symbol.trim(),
|
|
331834
|
-
source_time_ms: eventTimeMs
|
|
331835
|
-
});
|
|
331836
|
-
return {
|
|
331837
|
-
rawCaptureId,
|
|
331838
|
-
rawCaptureScope: input.scope,
|
|
331839
|
-
rawChecksum,
|
|
331840
|
-
redactedPayload,
|
|
331841
|
-
eventTimeMs,
|
|
331842
|
-
receivedTimeMs,
|
|
331843
|
-
checksumAlgorithm: context2.checksumAlgorithm
|
|
331844
|
-
};
|
|
331845
|
-
}
|
|
331846
|
-
function captureCoreFields(context2, rawCapture) {
|
|
331847
|
-
assertCaptureContext(context2);
|
|
331848
|
-
return {
|
|
331849
|
-
source: context2.source,
|
|
331850
|
-
deployment_id: context2.deploymentId,
|
|
331851
|
-
capture_bundle_id: context2.captureBundleId,
|
|
331852
|
-
exchange: context2.exchange.trim().toLowerCase(),
|
|
331853
|
-
symbol: context2.symbol.trim(),
|
|
331854
|
-
trading_pair: context2.symbol.trim().replace("/", "-"),
|
|
331855
|
-
source_symbol: context2.symbol.trim(),
|
|
331856
|
-
asset_type: context2.assetType,
|
|
331857
|
-
feed: context2.feed,
|
|
331858
|
-
provider: context2.provider,
|
|
331859
|
-
source_mode: context2.sourceMode,
|
|
331860
|
-
source_time_ms: rawCapture.eventTimeMs,
|
|
331861
|
-
received_time_ms: rawCapture.receivedTimeMs,
|
|
331862
|
-
raw_capture_id: rawCapture.rawCaptureId,
|
|
331863
|
-
raw_capture_scope: rawCapture.rawCaptureScope,
|
|
331864
|
-
schema_version: context2.schemaVersion,
|
|
331865
|
-
checksum_algorithm: context2.checksumAlgorithm,
|
|
331866
|
-
raw_checksum: rawCapture.rawChecksum,
|
|
331867
|
-
provenance_complete: context2.provenanceComplete ? 1 : 0
|
|
331868
|
-
};
|
|
331869
|
-
}
|
|
331870
|
-
|
|
331871
333158
|
// src/helpers/market-data-archive/canonical-orderbook.ts
|
|
331872
333159
|
class OrderBookValidationError extends Error {
|
|
331873
333160
|
reason;
|
|
@@ -332035,46 +333322,6 @@ function buildCanonicalOrderBookRows(input) {
|
|
|
332035
333322
|
};
|
|
332036
333323
|
}
|
|
332037
333324
|
|
|
332038
|
-
// src/helpers/market-data-archive/capture-context.ts
|
|
332039
|
-
function createMarketCaptureContext(input) {
|
|
332040
|
-
const environment = input.environment ?? "development";
|
|
332041
|
-
const deploymentId = input.deploymentId.trim();
|
|
332042
|
-
if (!deploymentId)
|
|
332043
|
-
throw new Error("deployment_id must not be empty");
|
|
332044
|
-
const configuredBundle = input.captureBundleId?.trim();
|
|
332045
|
-
if (environment === "production" && !configuredBundle) {
|
|
332046
|
-
throw new Error("capture_bundle_id is required for production market capture");
|
|
332047
|
-
}
|
|
332048
|
-
const exchange = input.exchange.trim().toLowerCase();
|
|
332049
|
-
const symbol2 = input.symbol.trim();
|
|
332050
|
-
if (!exchange || !symbol2) {
|
|
332051
|
-
throw new Error("exchange and symbol are required for market capture");
|
|
332052
|
-
}
|
|
332053
|
-
return {
|
|
332054
|
-
source: input.source,
|
|
332055
|
-
deploymentId,
|
|
332056
|
-
captureBundleId: configuredBundle ?? `development:${deploymentId}`,
|
|
332057
|
-
exchange,
|
|
332058
|
-
symbol: symbol2,
|
|
332059
|
-
assetType: input.assetType,
|
|
332060
|
-
feed: input.feed,
|
|
332061
|
-
provider: input.provider?.trim() || `ccxt:${exchange}`,
|
|
332062
|
-
sourceMode: input.sourceMode,
|
|
332063
|
-
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
332064
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
332065
|
-
provenanceComplete: true,
|
|
332066
|
-
timeframe: input.timeframe,
|
|
332067
|
-
accountSelector: input.accountSelector
|
|
332068
|
-
};
|
|
332069
|
-
}
|
|
332070
|
-
function captureEnvironmentFromEnv(value = process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT) {
|
|
332071
|
-
const environment = value?.trim() || "development";
|
|
332072
|
-
if (environment !== "development" && environment !== "production") {
|
|
332073
|
-
throw new Error("CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT must be development or production");
|
|
332074
|
-
}
|
|
332075
|
-
return environment;
|
|
332076
|
-
}
|
|
332077
|
-
|
|
332078
333325
|
// src/helpers/market-data-archive/ohlcv-bar-tracker.ts
|
|
332079
333326
|
function isFiniteNumber(value) {
|
|
332080
333327
|
return typeof value === "number" && Number.isFinite(value);
|
|
@@ -332209,39 +333456,6 @@ function getOrderbookArchiveDepthLimit() {
|
|
|
332209
333456
|
return Math.min(parsed, MAX_ORDERBOOK_ARCHIVE_DEPTH_LIMIT);
|
|
332210
333457
|
}
|
|
332211
333458
|
|
|
332212
|
-
// src/helpers/market-data-archive/orderbook-sampler.ts
|
|
332213
|
-
var DEFAULT_ORDERBOOK_INTERVAL_MS = 1000;
|
|
332214
|
-
function getOrderbookIntervalMs() {
|
|
332215
|
-
const raw = process.env.CEX_BROKER_ORDERBOOK_INTERVAL_MS ?? process.env.CEX_BROKER_ORDERBOOK_TOB_INTERVAL_MS;
|
|
332216
|
-
if (!raw) {
|
|
332217
|
-
return DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
332218
|
-
}
|
|
332219
|
-
const parsed = Number.parseInt(raw, 10);
|
|
332220
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
332221
|
-
}
|
|
332222
|
-
function isMarketArchiveEnabled() {
|
|
332223
|
-
return process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED !== "false";
|
|
332224
|
-
}
|
|
332225
|
-
|
|
332226
|
-
class OrderbookSampler {
|
|
332227
|
-
intervalMs;
|
|
332228
|
-
lastEmitMs = null;
|
|
332229
|
-
constructor(intervalMs = getOrderbookIntervalMs()) {
|
|
332230
|
-
this.intervalMs = intervalMs;
|
|
332231
|
-
}
|
|
332232
|
-
shouldEmit(nowMs = Date.now()) {
|
|
332233
|
-
if (this.lastEmitMs !== null && nowMs < this.lastEmitMs) {
|
|
332234
|
-
this.lastEmitMs = nowMs;
|
|
332235
|
-
return true;
|
|
332236
|
-
}
|
|
332237
|
-
if (this.lastEmitMs !== null && nowMs - this.lastEmitMs < this.intervalMs) {
|
|
332238
|
-
return false;
|
|
332239
|
-
}
|
|
332240
|
-
this.lastEmitMs = nowMs;
|
|
332241
|
-
return true;
|
|
332242
|
-
}
|
|
332243
|
-
}
|
|
332244
|
-
|
|
332245
333459
|
// src/helpers/market-data-archive/parse-stream.ts
|
|
332246
333460
|
function isFiniteNumber2(value) {
|
|
332247
333461
|
return typeof value === "number" && Number.isFinite(value);
|
|
@@ -332336,10 +333550,10 @@ function parseTicker(value, fallbackMs) {
|
|
|
332336
333550
|
["change", record2.change],
|
|
332337
333551
|
["percentage", record2.percentage]
|
|
332338
333552
|
];
|
|
332339
|
-
for (const [
|
|
333553
|
+
for (const [key2, rawValue] of fields) {
|
|
332340
333554
|
const numeric = toNumber2(rawValue);
|
|
332341
333555
|
if (numeric !== undefined) {
|
|
332342
|
-
parsed[
|
|
333556
|
+
parsed[key2] = numeric;
|
|
332343
333557
|
}
|
|
332344
333558
|
}
|
|
332345
333559
|
return parsed;
|
|
@@ -332356,9 +333570,19 @@ function withNormalizedChecksum(record2) {
|
|
|
332356
333570
|
normalized_row_checksum: sha256Canonical(compact)
|
|
332357
333571
|
};
|
|
332358
333572
|
}
|
|
333573
|
+
function legacyMarketFields(context2, rawCapture) {
|
|
333574
|
+
return {
|
|
333575
|
+
account_selector: context2.accountSelector,
|
|
333576
|
+
broker_observed_timestamp: new Date(rawCapture.receivedTimeMs).toISOString()
|
|
333577
|
+
};
|
|
333578
|
+
}
|
|
333579
|
+
function legacyDecimal8(value) {
|
|
333580
|
+
return value === undefined ? undefined : Number(value.toFixed(8));
|
|
333581
|
+
}
|
|
332359
333582
|
function buildCanonicalCexStreamEventRow(context2, rawCapture) {
|
|
332360
333583
|
const row = withNormalizedChecksum({
|
|
332361
333584
|
...captureCoreFields(context2, rawCapture),
|
|
333585
|
+
...legacyMarketFields(context2, rawCapture),
|
|
332362
333586
|
stream_type: context2.feed,
|
|
332363
333587
|
event_time_ms: rawCapture.eventTimeMs,
|
|
332364
333588
|
payload_encoding: "canonical_json_v1",
|
|
@@ -332372,19 +333596,21 @@ function buildCanonicalTickerEventRow(context2, rawCapture, ticker) {
|
|
|
332372
333596
|
}
|
|
332373
333597
|
const row = withNormalizedChecksum({
|
|
332374
333598
|
...captureCoreFields(context2, rawCapture),
|
|
333599
|
+
...legacyMarketFields(context2, rawCapture),
|
|
332375
333600
|
source_time_ms: ticker.eventTimeMs,
|
|
332376
333601
|
event_time_ms: ticker.eventTimeMs,
|
|
332377
|
-
last: ticker.last,
|
|
332378
|
-
bid: ticker.bid,
|
|
332379
|
-
ask: ticker.ask,
|
|
332380
|
-
high: ticker.high,
|
|
332381
|
-
low: ticker.low,
|
|
332382
|
-
open: ticker.open,
|
|
332383
|
-
close: ticker.close,
|
|
332384
|
-
base_volume: ticker.baseVolume,
|
|
332385
|
-
quote_volume: ticker.quoteVolume,
|
|
332386
|
-
change: ticker.change,
|
|
332387
|
-
percentage: ticker.percentage
|
|
333602
|
+
last: legacyDecimal8(ticker.last),
|
|
333603
|
+
bid: legacyDecimal8(ticker.bid),
|
|
333604
|
+
ask: legacyDecimal8(ticker.ask),
|
|
333605
|
+
high: legacyDecimal8(ticker.high),
|
|
333606
|
+
low: legacyDecimal8(ticker.low),
|
|
333607
|
+
open: legacyDecimal8(ticker.open),
|
|
333608
|
+
close: legacyDecimal8(ticker.close),
|
|
333609
|
+
base_volume: legacyDecimal8(ticker.baseVolume),
|
|
333610
|
+
quote_volume: legacyDecimal8(ticker.quoteVolume),
|
|
333611
|
+
change: legacyDecimal8(ticker.change),
|
|
333612
|
+
percentage: legacyDecimal8(ticker.percentage),
|
|
333613
|
+
payload_json: JSON.stringify(rawCapture.redactedPayload)
|
|
332388
333614
|
});
|
|
332389
333615
|
return { table: "market_data.cex_ticker_events", row };
|
|
332390
333616
|
}
|
|
@@ -332394,13 +333620,14 @@ function buildCanonicalTradeRow(context2, rawCapture, trade) {
|
|
|
332394
333620
|
}
|
|
332395
333621
|
const row = withNormalizedChecksum({
|
|
332396
333622
|
...captureCoreFields(context2, rawCapture),
|
|
333623
|
+
...legacyMarketFields(context2, rawCapture),
|
|
332397
333624
|
source_time_ms: trade.eventTimeMs,
|
|
332398
333625
|
trade_id: trade.tradeId,
|
|
332399
333626
|
event_time_ms: trade.eventTimeMs,
|
|
332400
333627
|
side: trade.side,
|
|
332401
|
-
price: trade.price,
|
|
332402
|
-
amount: trade.amount,
|
|
332403
|
-
cost: trade.cost,
|
|
333628
|
+
price: legacyDecimal8(trade.price),
|
|
333629
|
+
amount: legacyDecimal8(trade.amount),
|
|
333630
|
+
cost: legacyDecimal8(trade.cost),
|
|
332404
333631
|
taker_or_maker: trade.takerOrMaker
|
|
332405
333632
|
});
|
|
332406
333633
|
return { table: "market_data.cex_trades", row };
|
|
@@ -332480,10 +333707,19 @@ function resolveCaptureContext(archiver, input, feed, sourceMode) {
|
|
|
332480
333707
|
environment: captureEnvironmentFromEnv()
|
|
332481
333708
|
});
|
|
332482
333709
|
}
|
|
333710
|
+
function canArchiveMarketData(archiver) {
|
|
333711
|
+
return resolveMarketCaptureArchiveState({
|
|
333712
|
+
archiveEnabled: archiver?.isEnabled() ?? false,
|
|
333713
|
+
marketArchiveEnabled: isMarketArchiveEnabled(),
|
|
333714
|
+
environment: process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT,
|
|
333715
|
+
deploymentId: archiver?.getDeploymentId(),
|
|
333716
|
+
captureBundleId: process.env.CEX_BROKER_CAPTURE_BUNDLE_ID
|
|
333717
|
+
}).enabled;
|
|
333718
|
+
}
|
|
332483
333719
|
function archiveOrderbookInBackground(archiver, otelMetrics, input, options) {
|
|
332484
333720
|
const labels = watchLabels("orderbook", input, archiver, "ORDERBOOK");
|
|
332485
333721
|
recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
|
|
332486
|
-
if (!
|
|
333722
|
+
if (!canArchiveMarketData(archiver)) {
|
|
332487
333723
|
return;
|
|
332488
333724
|
}
|
|
332489
333725
|
if (options?.sampledOut) {
|
|
@@ -332525,7 +333761,7 @@ function archiveOrderbookInBackground(archiver, otelMetrics, input, options) {
|
|
|
332525
333761
|
function archiveOhlcvInBackground(archiver, otelMetrics, tracker, input) {
|
|
332526
333762
|
const labels = watchLabels("ohlcv", input, archiver, "OHLCV");
|
|
332527
333763
|
recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
|
|
332528
|
-
if (!
|
|
333764
|
+
if (!canArchiveMarketData(archiver)) {
|
|
332529
333765
|
return;
|
|
332530
333766
|
}
|
|
332531
333767
|
queueMicrotask(() => {
|
|
@@ -332570,7 +333806,7 @@ function createOhlcvBarTracker() {
|
|
|
332570
333806
|
function archiveMarketRowsInBackground(archiver, otelMetrics, stream4, input, feed, enqueueRows) {
|
|
332571
333807
|
const labels = watchLabels(stream4, input, archiver, feed);
|
|
332572
333808
|
recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
|
|
332573
|
-
if (!
|
|
333809
|
+
if (!canArchiveMarketData(archiver)) {
|
|
332574
333810
|
return;
|
|
332575
333811
|
}
|
|
332576
333812
|
queueMicrotask(() => {
|
|
@@ -334110,282 +335346,16 @@ class SubscribeBrokerLifecycle {
|
|
|
334110
335346
|
// src/handlers/subscribe/handler.ts
|
|
334111
335347
|
var grpc13 = __toESM(require_src3(), 1);
|
|
334112
335348
|
|
|
334113
|
-
// src/helpers/binance-user-data-stream.ts
|
|
334114
|
-
import { Buffer as Buffer2 } from "node:buffer";
|
|
334115
|
-
import { createHmac } from "node:crypto";
|
|
334116
|
-
var BINANCE_SPOT_WS_API_URL = "wss://ws-api.binance.com:443/ws-api/v3";
|
|
334117
|
-
var DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS = 16;
|
|
334118
|
-
var createWebSocket = (url3) => new wrapper_default(url3);
|
|
334119
|
-
var userDataRequestCounter = 0;
|
|
334120
|
-
function getExchangeString(exchange, key) {
|
|
334121
|
-
const value = exchange[key];
|
|
334122
|
-
if (typeof value !== "string" || value.length === 0) {
|
|
334123
|
-
throw new Error(`Binance user-data stream requires exchange.${key}`);
|
|
334124
|
-
}
|
|
334125
|
-
return value;
|
|
334126
|
-
}
|
|
334127
|
-
function sortedQuery(params) {
|
|
334128
|
-
return Object.entries(params).sort(([left], [right]) => left.localeCompare(right)).map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`).join("&");
|
|
334129
|
-
}
|
|
334130
|
-
function signUserDataStreamParams(exchange, params) {
|
|
334131
|
-
const signParams = exchange.signParams;
|
|
334132
|
-
if (typeof signParams === "function") {
|
|
334133
|
-
return signParams.call(exchange, params);
|
|
334134
|
-
}
|
|
334135
|
-
const secret = getExchangeString(exchange, "secret");
|
|
334136
|
-
return {
|
|
334137
|
-
...params,
|
|
334138
|
-
signature: createHmac("sha256", secret).update(sortedQuery(params)).digest("hex")
|
|
334139
|
-
};
|
|
334140
|
-
}
|
|
334141
|
-
function getBinanceSpotWsApiUrl(exchange) {
|
|
334142
|
-
const urls = exchange.urls;
|
|
334143
|
-
return urls?.api?.ws?.["ws-api"]?.spot ?? BINANCE_SPOT_WS_API_URL;
|
|
334144
|
-
}
|
|
334145
|
-
function getRecord(value) {
|
|
334146
|
-
return typeof value === "object" && value !== null ? value : null;
|
|
334147
|
-
}
|
|
334148
|
-
function getMessage(value) {
|
|
334149
|
-
if (value instanceof Error) {
|
|
334150
|
-
return value.message;
|
|
334151
|
-
}
|
|
334152
|
-
if (typeof value === "string" && value.length > 0) {
|
|
334153
|
-
return value;
|
|
334154
|
-
}
|
|
334155
|
-
const record2 = getRecord(value);
|
|
334156
|
-
const message = record2?.message;
|
|
334157
|
-
return typeof message === "string" && message.length > 0 ? message : null;
|
|
334158
|
-
}
|
|
334159
|
-
function getOptionalExchangeString(exchange, key) {
|
|
334160
|
-
const value = exchange[key];
|
|
334161
|
-
return typeof value === "string" && value.length > 0 ? value : null;
|
|
334162
|
-
}
|
|
334163
|
-
function redactDiagnosticMessage(message, secretValues) {
|
|
334164
|
-
let redacted = message;
|
|
334165
|
-
for (const value of secretValues) {
|
|
334166
|
-
if (value.length > 0) {
|
|
334167
|
-
redacted = redacted.split(value).join("[redacted]");
|
|
334168
|
-
}
|
|
334169
|
-
}
|
|
334170
|
-
return redacted.replace(/(\b(?:apiKey|secret|signature)\b\s*=\s*)[^\s&,;)]+/gi, "$1[redacted]").replace(/("(?:apiKey|secret|signature)"\s*:\s*")[^"]*(")/gi, "$1[redacted]$2");
|
|
334171
|
-
}
|
|
334172
|
-
function formatBinanceUserDataWebSocketError(event, secretValues) {
|
|
334173
|
-
const record2 = getRecord(event);
|
|
334174
|
-
const message = getMessage(record2?.error) ?? getMessage(record2?.message) ?? getMessage(event);
|
|
334175
|
-
const safeMessage = message === null ? null : redactDiagnosticMessage(message, secretValues);
|
|
334176
|
-
return new Error(safeMessage ? `Binance user-data WebSocket error: ${safeMessage}` : "Binance user-data WebSocket error");
|
|
334177
|
-
}
|
|
334178
|
-
function getCloseReason(value) {
|
|
334179
|
-
if (typeof value === "string") {
|
|
334180
|
-
return value.length > 0 ? value : null;
|
|
334181
|
-
}
|
|
334182
|
-
if (Buffer2.isBuffer(value)) {
|
|
334183
|
-
const reason = value.toString("utf8");
|
|
334184
|
-
return reason.length > 0 ? reason : null;
|
|
334185
|
-
}
|
|
334186
|
-
if (value instanceof Uint8Array) {
|
|
334187
|
-
const reason = Buffer2.from(value).toString("utf8");
|
|
334188
|
-
return reason.length > 0 ? reason : null;
|
|
334189
|
-
}
|
|
334190
|
-
return null;
|
|
334191
|
-
}
|
|
334192
|
-
function formatBinanceUserDataWebSocketClose(codeOrEvent, reasonOrUndefined, secretValues) {
|
|
334193
|
-
const record2 = getRecord(codeOrEvent);
|
|
334194
|
-
const code = record2 ? record2.code : codeOrEvent;
|
|
334195
|
-
const reason = getCloseReason(record2 ? record2.reason : reasonOrUndefined);
|
|
334196
|
-
const safeReason = reason === null ? null : redactDiagnosticMessage(reason, secretValues);
|
|
334197
|
-
const details = [
|
|
334198
|
-
typeof code === "number" || typeof code === "string" ? `code=${code}` : null,
|
|
334199
|
-
safeReason ? `reason=${safeReason}` : null
|
|
334200
|
-
].filter((detail) => detail !== null);
|
|
334201
|
-
return new Error(details.length > 0 ? `Binance user-data WebSocket closed unexpectedly (${details.join(", ")})` : "Binance user-data WebSocket closed unexpectedly");
|
|
334202
|
-
}
|
|
334203
|
-
function decodeMessageData(data) {
|
|
334204
|
-
if (typeof data === "string") {
|
|
334205
|
-
return data;
|
|
334206
|
-
}
|
|
334207
|
-
if (Buffer2.isBuffer(data)) {
|
|
334208
|
-
return data.toString("utf8");
|
|
334209
|
-
}
|
|
334210
|
-
if (data instanceof ArrayBuffer) {
|
|
334211
|
-
return Buffer2.from(data).toString("utf8");
|
|
334212
|
-
}
|
|
334213
|
-
if (ArrayBuffer.isView(data)) {
|
|
334214
|
-
return Buffer2.from(data.buffer, data.byteOffset, data.byteLength).toString("utf8");
|
|
334215
|
-
}
|
|
334216
|
-
if (Array.isArray(data) && data.every((item) => Buffer2.isBuffer(item))) {
|
|
334217
|
-
return Buffer2.concat(data).toString("utf8");
|
|
334218
|
-
}
|
|
334219
|
-
return data;
|
|
334220
|
-
}
|
|
334221
|
-
|
|
334222
|
-
class BinanceSpotUserDataStream {
|
|
334223
|
-
exchange;
|
|
334224
|
-
ws;
|
|
334225
|
-
secretValues;
|
|
334226
|
-
requestId = `user-data-${Date.now()}-${userDataRequestCounter++}`;
|
|
334227
|
-
maxBufferedEvents;
|
|
334228
|
-
queue = [];
|
|
334229
|
-
waiters = [];
|
|
334230
|
-
closed = false;
|
|
334231
|
-
closeError = null;
|
|
334232
|
-
subscriptionId = null;
|
|
334233
|
-
constructor(exchange, options = {}) {
|
|
334234
|
-
this.exchange = exchange;
|
|
334235
|
-
this.maxBufferedEvents = options.maxBufferedEvents ?? DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS;
|
|
334236
|
-
this.secretValues = [
|
|
334237
|
-
getOptionalExchangeString(exchange, "apiKey"),
|
|
334238
|
-
getOptionalExchangeString(exchange, "secret")
|
|
334239
|
-
].filter((value) => value !== null);
|
|
334240
|
-
this.ws = createWebSocket(getBinanceSpotWsApiUrl(exchange));
|
|
334241
|
-
this.ws.on("open", () => this.subscribe());
|
|
334242
|
-
this.ws.on("message", (data) => this.handleMessage(data));
|
|
334243
|
-
this.ws.on("error", (error48) => this.fail(formatBinanceUserDataWebSocketError(error48, this.secretValues)));
|
|
334244
|
-
this.ws.on("close", (code, reason) => this.handleClose(code, reason));
|
|
334245
|
-
}
|
|
334246
|
-
async* [Symbol.asyncIterator]() {
|
|
334247
|
-
while (true) {
|
|
334248
|
-
const event = await this.nextEvent();
|
|
334249
|
-
if (!event) {
|
|
334250
|
-
break;
|
|
334251
|
-
}
|
|
334252
|
-
yield event;
|
|
334253
|
-
}
|
|
334254
|
-
}
|
|
334255
|
-
close() {
|
|
334256
|
-
if (this.closed) {
|
|
334257
|
-
return;
|
|
334258
|
-
}
|
|
334259
|
-
this.closed = true;
|
|
334260
|
-
this.queue.length = 0;
|
|
334261
|
-
try {
|
|
334262
|
-
this.ws.close();
|
|
334263
|
-
} catch {}
|
|
334264
|
-
this.flushWaiters();
|
|
334265
|
-
}
|
|
334266
|
-
handleClose(code, reason) {
|
|
334267
|
-
if (this.closed) {
|
|
334268
|
-
return;
|
|
334269
|
-
}
|
|
334270
|
-
this.fail(formatBinanceUserDataWebSocketClose(code, reason, this.secretValues));
|
|
334271
|
-
}
|
|
334272
|
-
subscribe() {
|
|
334273
|
-
const apiKey = getExchangeString(this.exchange, "apiKey");
|
|
334274
|
-
const signedParams = signUserDataStreamParams(this.exchange, {
|
|
334275
|
-
apiKey,
|
|
334276
|
-
timestamp: Date.now()
|
|
334277
|
-
});
|
|
334278
|
-
this.ws.send(JSON.stringify({
|
|
334279
|
-
id: this.requestId,
|
|
334280
|
-
method: "userDataStream.subscribe.signature",
|
|
334281
|
-
params: signedParams
|
|
334282
|
-
}));
|
|
334283
|
-
}
|
|
334284
|
-
handleMessage(data) {
|
|
334285
|
-
if (this.closed) {
|
|
334286
|
-
return;
|
|
334287
|
-
}
|
|
334288
|
-
let message;
|
|
334289
|
-
try {
|
|
334290
|
-
const decodedData = decodeMessageData(data);
|
|
334291
|
-
message = typeof decodedData === "string" ? JSON.parse(decodedData) : decodedData;
|
|
334292
|
-
} catch (error48) {
|
|
334293
|
-
this.fail(error48 instanceof Error ? error48 : new Error("Invalid Binance user-data message"));
|
|
334294
|
-
return;
|
|
334295
|
-
}
|
|
334296
|
-
if ("id" in message && message.id === this.requestId) {
|
|
334297
|
-
if (message.status !== 200) {
|
|
334298
|
-
this.fail(new Error(message.error?.msg ?? message.error?.message ?? `Binance user-data subscription failed with status ${message.status}`));
|
|
334299
|
-
return;
|
|
334300
|
-
}
|
|
334301
|
-
this.subscriptionId = message.result?.subscriptionId ?? null;
|
|
334302
|
-
return;
|
|
334303
|
-
}
|
|
334304
|
-
if ("status" in message && typeof message.status === "number" && message.status !== 200) {
|
|
334305
|
-
const errorMessage = message.error?.msg ?? message.error?.message ?? `Binance user-data request failed with status ${message.status}`;
|
|
334306
|
-
const errorCode2 = message.error?.code;
|
|
334307
|
-
this.fail(new Error(typeof errorCode2 === "number" ? `${errorMessage} (code ${errorCode2})` : errorMessage));
|
|
334308
|
-
return;
|
|
334309
|
-
}
|
|
334310
|
-
if (!("event" in message) || !message.event) {
|
|
334311
|
-
return;
|
|
334312
|
-
}
|
|
334313
|
-
const subscriptionId = message.subscriptionId ?? this.subscriptionId;
|
|
334314
|
-
if (subscriptionId === null || subscriptionId === undefined) {
|
|
334315
|
-
return;
|
|
334316
|
-
}
|
|
334317
|
-
this.push({ subscriptionId, event: message.event });
|
|
334318
|
-
}
|
|
334319
|
-
push(event) {
|
|
334320
|
-
if (this.closed) {
|
|
334321
|
-
return;
|
|
334322
|
-
}
|
|
334323
|
-
const waiter = this.waiters.shift();
|
|
334324
|
-
if (waiter) {
|
|
334325
|
-
waiter.resolve(event);
|
|
334326
|
-
return;
|
|
334327
|
-
}
|
|
334328
|
-
if (this.queue.length >= this.maxBufferedEvents) {
|
|
334329
|
-
this.fail(new Error(`Binance user-data stream buffered event limit exceeded (${this.maxBufferedEvents}); downstream consumer is not keeping up`));
|
|
334330
|
-
return;
|
|
334331
|
-
}
|
|
334332
|
-
this.queue.push(event);
|
|
334333
|
-
}
|
|
334334
|
-
nextEvent() {
|
|
334335
|
-
const event = this.queue.shift();
|
|
334336
|
-
if (event) {
|
|
334337
|
-
return Promise.resolve(event);
|
|
334338
|
-
}
|
|
334339
|
-
if (this.closeError) {
|
|
334340
|
-
return Promise.reject(this.closeError);
|
|
334341
|
-
}
|
|
334342
|
-
if (this.closed) {
|
|
334343
|
-
return Promise.resolve(null);
|
|
334344
|
-
}
|
|
334345
|
-
return new Promise((resolve, reject) => {
|
|
334346
|
-
this.waiters.push({ resolve, reject });
|
|
334347
|
-
});
|
|
334348
|
-
}
|
|
334349
|
-
fail(error48) {
|
|
334350
|
-
if (this.closeError) {
|
|
334351
|
-
return;
|
|
334352
|
-
}
|
|
334353
|
-
this.closeError = error48;
|
|
334354
|
-
this.closed = true;
|
|
334355
|
-
this.queue.length = 0;
|
|
334356
|
-
this.flushWaiters();
|
|
334357
|
-
try {
|
|
334358
|
-
this.ws.close();
|
|
334359
|
-
} catch {}
|
|
334360
|
-
}
|
|
334361
|
-
flushWaiters() {
|
|
334362
|
-
const error48 = this.closeError;
|
|
334363
|
-
for (const waiter of this.waiters.splice(0)) {
|
|
334364
|
-
if (error48) {
|
|
334365
|
-
waiter.reject(error48);
|
|
334366
|
-
} else {
|
|
334367
|
-
waiter.resolve(null);
|
|
334368
|
-
}
|
|
334369
|
-
}
|
|
334370
|
-
}
|
|
334371
|
-
}
|
|
334372
|
-
function isBinanceBalanceUserDataEvent(event) {
|
|
334373
|
-
return event.e === "outboundAccountPosition" || event.e === "balanceUpdate" || event.e === "externalLockUpdate";
|
|
334374
|
-
}
|
|
334375
|
-
function isBinanceOrderUserDataEvent(event) {
|
|
334376
|
-
return event.e === "executionReport" || event.e === "listStatus";
|
|
334377
|
-
}
|
|
334378
|
-
|
|
334379
335349
|
// src/helpers/binance-user-data-normalization.ts
|
|
334380
|
-
function requireQuantity(entry,
|
|
334381
|
-
const value = entry[
|
|
335350
|
+
function requireQuantity(entry, key2) {
|
|
335351
|
+
const value = entry[key2];
|
|
334382
335352
|
if (typeof value === "string" && value.trim().length > 0) {
|
|
334383
335353
|
return value;
|
|
334384
335354
|
}
|
|
334385
335355
|
if (typeof value === "number" && Number.isFinite(value)) {
|
|
334386
335356
|
return String(value);
|
|
334387
335357
|
}
|
|
334388
|
-
throw new Error(`Invalid Binance balance quantity: ${
|
|
335358
|
+
throw new Error(`Invalid Binance balance quantity: ${key2}`);
|
|
334389
335359
|
}
|
|
334390
335360
|
async function normalizeBinanceSpotBalanceEvent(exchange, event) {
|
|
334391
335361
|
if (event.e !== "outboundAccountPosition") {
|
|
@@ -334551,12 +335521,14 @@ async function getBinanceMarketId(broker, symbol2) {
|
|
|
334551
335521
|
}
|
|
334552
335522
|
return symbol2.replace("/", "").toUpperCase();
|
|
334553
335523
|
}
|
|
334554
|
-
async function streamBinanceUserData(call, broker, symbol2, subscriptionType, isClosed, archiveContext) {
|
|
334555
|
-
const userDataStream = new BinanceSpotUserDataStream(broker);
|
|
334556
|
-
|
|
334557
|
-
|
|
334558
|
-
|
|
334559
|
-
|
|
335524
|
+
async function streamBinanceUserData(call, broker, symbol2, subscriptionType, isClosed, archiveContext, userDataSource, knownMarketId) {
|
|
335525
|
+
const userDataStream = userDataSource ?? new BinanceSpotUserDataStream(broker);
|
|
335526
|
+
if (!userDataSource) {
|
|
335527
|
+
call.once("close", () => userDataStream.close());
|
|
335528
|
+
call.once("cancelled", () => userDataStream.close());
|
|
335529
|
+
call.once("error", () => userDataStream.close());
|
|
335530
|
+
}
|
|
335531
|
+
const marketId = knownMarketId ?? (subscriptionType === SubscriptionType.ORDERS ? await getBinanceMarketId(broker, symbol2) : null);
|
|
334560
335532
|
try {
|
|
334561
335533
|
for await (const message of userDataStream) {
|
|
334562
335534
|
if (isClosed()) {
|
|
@@ -334648,7 +335620,13 @@ async function runCcxtSubscribeLoop(call, isClosed, symbol2, subscriptionType, w
|
|
|
334648
335620
|
}
|
|
334649
335621
|
}
|
|
334650
335622
|
function createSubscribeHandler(deps) {
|
|
334651
|
-
const {
|
|
335623
|
+
const {
|
|
335624
|
+
brokers,
|
|
335625
|
+
whitelistIps,
|
|
335626
|
+
otelMetrics,
|
|
335627
|
+
brokerArchiver,
|
|
335628
|
+
userDataStreamSupervisor
|
|
335629
|
+
} = deps;
|
|
334652
335630
|
const brokerLifecycle = deps.brokerLifecycle ?? new SubscribeBrokerLifecycle;
|
|
334653
335631
|
return async (call) => {
|
|
334654
335632
|
const subscribeStartTime = Date.now();
|
|
@@ -334783,7 +335761,28 @@ function createSubscribeHandler(deps) {
|
|
|
334783
335761
|
});
|
|
334784
335762
|
return;
|
|
334785
335763
|
}
|
|
334786
|
-
await
|
|
335764
|
+
const marketId = subscriptionType === SubscriptionType.ORDERS ? await getBinanceMarketId(accountBroker, resolvedSymbol) : undefined;
|
|
335765
|
+
let userDataSource;
|
|
335766
|
+
if (selectedBrokerAccount) {
|
|
335767
|
+
if (!userDataStreamSupervisor) {
|
|
335768
|
+
await writeSubscribeError(call, isStreamClosed, {
|
|
335769
|
+
data: JSON.stringify({
|
|
335770
|
+
error: "Configured account user-data supervisor is unavailable"
|
|
335771
|
+
}),
|
|
335772
|
+
timestamp: Date.now(),
|
|
335773
|
+
symbol: resolvedSymbol,
|
|
335774
|
+
type: subscriptionType
|
|
335775
|
+
});
|
|
335776
|
+
return;
|
|
335777
|
+
}
|
|
335778
|
+
userDataSource = userDataStreamSupervisor.subscribe({
|
|
335779
|
+
exchange: normalizedCex,
|
|
335780
|
+
accountSelector: selectedBrokerAccount.label,
|
|
335781
|
+
kind: subscriptionType === SubscriptionType.BALANCE ? "balance" : "orders",
|
|
335782
|
+
marketId
|
|
335783
|
+
});
|
|
335784
|
+
}
|
|
335785
|
+
await streamBinanceUserData(call, accountBroker, resolvedSymbol, subscriptionType, isStreamClosed, streamArchiveContext, userDataSource, marketId);
|
|
334787
335786
|
return;
|
|
334788
335787
|
}
|
|
334789
335788
|
switch (subscriptionType) {
|
|
@@ -335177,7 +336176,7 @@ var CEX_BROKER_PACKAGE_DEFINITION = protoLoader.fromJSON(node_descriptor_default
|
|
|
335177
336176
|
// src/server.ts
|
|
335178
336177
|
var grpcObj = grpc14.loadPackageDefinition(CEX_BROKER_PACKAGE_DEFINITION);
|
|
335179
336178
|
var cexNode = grpcObj.cex_broker;
|
|
335180
|
-
function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker, subscribeBrokerLifecycle) {
|
|
336179
|
+
function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker, subscribeBrokerLifecycle, userDataStreamSupervisor) {
|
|
335181
336180
|
const server = new grpc14.Server;
|
|
335182
336181
|
server.addService(cexNode.cex_service.service, {
|
|
335183
336182
|
ExecuteAction: createExecuteActionHandler({
|
|
@@ -335196,7 +336195,8 @@ function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, ot
|
|
|
335196
336195
|
whitelistIps,
|
|
335197
336196
|
otelMetrics,
|
|
335198
336197
|
brokerArchiver,
|
|
335199
|
-
brokerLifecycle: subscribeBrokerLifecycle
|
|
336198
|
+
brokerLifecycle: subscribeBrokerLifecycle,
|
|
336199
|
+
userDataStreamSupervisor
|
|
335200
336200
|
})
|
|
335201
336201
|
});
|
|
335202
336202
|
return server;
|
|
@@ -335340,13 +336340,14 @@ class CEXBroker {
|
|
|
335340
336340
|
fillArchivePoller;
|
|
335341
336341
|
depositArchivePoller;
|
|
335342
336342
|
accountBalanceArchivePoller;
|
|
336343
|
+
userDataStreamSupervisor;
|
|
335343
336344
|
loadEnvConfig() {
|
|
335344
336345
|
log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
|
|
335345
336346
|
const configMap = {};
|
|
335346
|
-
for (const [
|
|
335347
|
-
if (!
|
|
336347
|
+
for (const [key2, value] of Object.entries(process.env)) {
|
|
336348
|
+
if (!key2.startsWith("CEX_BROKER_"))
|
|
335348
336349
|
continue;
|
|
335349
|
-
let match =
|
|
336350
|
+
let match = key2.match(/^CEX_BROKER_(\w+)_(API_(KEY|SECRET)|ROLE|EMAIL|SUBACCOUNTID|UID)_(\d+)$/);
|
|
335350
336351
|
if (match) {
|
|
335351
336352
|
const broker2 = match[1]?.toLowerCase() ?? "";
|
|
335352
336353
|
const type3 = match[2]?.toLowerCase() ?? "";
|
|
@@ -335375,9 +336376,9 @@ class CEXBroker {
|
|
|
335375
336376
|
}
|
|
335376
336377
|
continue;
|
|
335377
336378
|
}
|
|
335378
|
-
match =
|
|
336379
|
+
match = key2.match(/^CEX_BROKER_(\w+)_(API_(KEY|SECRET)|ROLE|EMAIL|SUBACCOUNTID|UID)$/);
|
|
335379
336380
|
if (!match) {
|
|
335380
|
-
log.warn(`⚠️ Skipping unrecognized env var: ${
|
|
336381
|
+
log.warn(`⚠️ Skipping unrecognized env var: ${key2}`);
|
|
335381
336382
|
continue;
|
|
335382
336383
|
}
|
|
335383
336384
|
const broker = match[1]?.toLowerCase() ?? "";
|
|
@@ -335495,6 +336496,10 @@ class CEXBroker {
|
|
|
335495
336496
|
if (this.server) {
|
|
335496
336497
|
await this.server.forceShutdown();
|
|
335497
336498
|
}
|
|
336499
|
+
if (this.userDataStreamSupervisor) {
|
|
336500
|
+
await this.userDataStreamSupervisor.close();
|
|
336501
|
+
this.userDataStreamSupervisor = undefined;
|
|
336502
|
+
}
|
|
335498
336503
|
if (this.brokerArchiver) {
|
|
335499
336504
|
await this.brokerArchiver.close();
|
|
335500
336505
|
}
|
|
@@ -335506,6 +336511,14 @@ class CEXBroker {
|
|
|
335506
336511
|
}
|
|
335507
336512
|
}
|
|
335508
336513
|
async run() {
|
|
336514
|
+
const marketArchiveState = resolveMarketCaptureArchiveState({
|
|
336515
|
+
archiveEnabled: this.brokerArchiver?.isEnabled() ?? false,
|
|
336516
|
+
marketArchiveEnabled: isMarketArchiveEnabled(),
|
|
336517
|
+
environment: process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT,
|
|
336518
|
+
deploymentId: this.brokerArchiver?.getDeploymentId(),
|
|
336519
|
+
captureBundleId: process.env.CEX_BROKER_CAPTURE_BUNDLE_ID
|
|
336520
|
+
});
|
|
336521
|
+
assertMarketCaptureArchiveStartable(marketArchiveState);
|
|
335509
336522
|
if (this.server) {
|
|
335510
336523
|
await this.server.forceShutdown();
|
|
335511
336524
|
}
|
|
@@ -335529,7 +336542,15 @@ class CEXBroker {
|
|
|
335529
336542
|
if (this.otelMetrics?.isOtelEnabled()) {
|
|
335530
336543
|
await this.otelMetrics.initialize();
|
|
335531
336544
|
}
|
|
335532
|
-
|
|
336545
|
+
if (!this.userDataStreamSupervisor && Object.keys(this.brokers).length > 0) {
|
|
336546
|
+
const publisher = new StreamHealthPublisher(streamHealthPublisherConfigFromEnv());
|
|
336547
|
+
this.userDataStreamSupervisor = new UserDataStreamSupervisor({
|
|
336548
|
+
brokers: this.brokers,
|
|
336549
|
+
publisher
|
|
336550
|
+
});
|
|
336551
|
+
this.userDataStreamSupervisor.start();
|
|
336552
|
+
}
|
|
336553
|
+
this.server = getServer(this.policy, this.brokers, this.whitelistIps, this.useVerity, this.#verityProverUrl, this.otelMetrics, this.brokerArchiver, this.orderActivityTracker, this.withdrawalObservationTracker, undefined, this.userDataStreamSupervisor);
|
|
335533
336554
|
this.server.bindAsync(`0.0.0.0:${this.port}`, grpc15.ServerCredentials.createInsecure(), (err2, port) => {
|
|
335534
336555
|
if (err2) {
|
|
335535
336556
|
log.error(err2);
|