@usherlabs/cex-broker 0.2.38 → 0.2.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/commands/cli.js +1794 -773
- package/dist/handlers/subscribe/handler.d.ts +2 -0
- package/dist/helpers/binance-user-data-stream.d.ts +12 -0
- package/dist/helpers/broker-execution-archive/index.d.ts +1 -1
- package/dist/helpers/broker-execution-archive/rows.d.ts +1 -0
- package/dist/helpers/broker-execution-archive/writer.d.ts +30 -0
- package/dist/helpers/deposit-archive-poller.d.ts +1 -0
- package/dist/helpers/market-data-archive/capture-context.d.ts +24 -3
- package/dist/helpers/market-data-archive/index.d.ts +1 -1
- package/dist/helpers/stream-health-publisher.d.ts +41 -0
- package/dist/helpers/user-data-stream-supervisor.d.ts +27 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1794 -773
- package/dist/index.js.map +18 -16
- package/dist/server.d.ts +2 -1
- package/package.json +10 -2
package/dist/index.js
CHANGED
|
@@ -4,25 +4,43 @@ var __getProtoOf = Object.getPrototypeOf;
|
|
|
4
4
|
var __defProp = Object.defineProperty;
|
|
5
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
6
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
function __accessProp(key) {
|
|
8
|
+
return this[key];
|
|
9
|
+
}
|
|
10
|
+
var __toESMCache_node;
|
|
11
|
+
var __toESMCache_esm;
|
|
7
12
|
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
+
var canCache = mod != null && typeof mod === "object";
|
|
14
|
+
if (canCache) {
|
|
15
|
+
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
+
var cached = cache.get(mod);
|
|
17
|
+
if (cached)
|
|
18
|
+
return cached;
|
|
19
|
+
}
|
|
8
20
|
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
9
21
|
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
10
22
|
for (let key of __getOwnPropNames(mod))
|
|
11
23
|
if (!__hasOwnProp.call(to, key))
|
|
12
24
|
__defProp(to, key, {
|
|
13
|
-
get: (
|
|
25
|
+
get: __accessProp.bind(mod, key),
|
|
14
26
|
enumerable: true
|
|
15
27
|
});
|
|
28
|
+
if (canCache)
|
|
29
|
+
cache.set(mod, to);
|
|
16
30
|
return to;
|
|
17
31
|
};
|
|
18
32
|
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
+
var __returnValue = (v) => v;
|
|
34
|
+
function __exportSetter(name, newValue) {
|
|
35
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
+
}
|
|
19
37
|
var __export = (target, all) => {
|
|
20
38
|
for (var name in all)
|
|
21
39
|
__defProp(target, name, {
|
|
22
40
|
get: all[name],
|
|
23
41
|
enumerable: true,
|
|
24
42
|
configurable: true,
|
|
25
|
-
set: (
|
|
43
|
+
set: __exportSetter.bind(all, name)
|
|
26
44
|
});
|
|
27
45
|
};
|
|
28
46
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
@@ -26363,7 +26381,7 @@ var require_tracestate_impl = __commonJS((exports) => {
|
|
|
26363
26381
|
const value = listMember.slice(i2 + 1, part.length);
|
|
26364
26382
|
if ((0, tracestate_validators_1.validateKey)(key) && (0, tracestate_validators_1.validateValue)(value)) {
|
|
26365
26383
|
agg.set(key, value);
|
|
26366
|
-
}
|
|
26384
|
+
}
|
|
26367
26385
|
}
|
|
26368
26386
|
return agg;
|
|
26369
26387
|
}, new Map);
|
|
@@ -35496,7 +35514,7 @@ var require_TraceState = __commonJS((exports) => {
|
|
|
35496
35514
|
const value = listMember.slice(i2 + 1, part.length);
|
|
35497
35515
|
if ((0, validators_1.validateKey)(key) && (0, validators_1.validateValue)(value)) {
|
|
35498
35516
|
agg.set(key, value);
|
|
35499
|
-
}
|
|
35517
|
+
}
|
|
35500
35518
|
}
|
|
35501
35519
|
return agg;
|
|
35502
35520
|
}, new Map);
|
|
@@ -291220,7 +291238,9 @@ function buildMarketMetadataSnapshotRow(input) {
|
|
|
291220
291238
|
// src/helpers/broker-execution-archive/writer.ts
|
|
291221
291239
|
var import_api_logs2 = __toESM(require_src4(), 1);
|
|
291222
291240
|
import { closeSync, fsyncSync, openSync, writeSync } from "node:fs";
|
|
291223
|
-
import {
|
|
291241
|
+
import {
|
|
291242
|
+
request as httpRequest2
|
|
291243
|
+
} from "node:http";
|
|
291224
291244
|
import { request as httpsRequest2 } from "node:https";
|
|
291225
291245
|
var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
|
|
291226
291246
|
"broker_execution.order_events",
|
|
@@ -291293,6 +291313,7 @@ class BrokerExecutionArchiver {
|
|
|
291293
291313
|
flushIntervalMs;
|
|
291294
291314
|
forwarderTimeoutMs;
|
|
291295
291315
|
queue = [];
|
|
291316
|
+
enqueueTimes = new WeakMap;
|
|
291296
291317
|
stats = {
|
|
291297
291318
|
enqueued: 0,
|
|
291298
291319
|
shed: 0,
|
|
@@ -291301,6 +291322,16 @@ class BrokerExecutionArchiver {
|
|
|
291301
291322
|
};
|
|
291302
291323
|
flushTimer = null;
|
|
291303
291324
|
flushInFlight = null;
|
|
291325
|
+
inFlightBatch = null;
|
|
291326
|
+
inFlightStartedAtMs = null;
|
|
291327
|
+
lastFailureAtMs = null;
|
|
291328
|
+
lastShedAtMs = null;
|
|
291329
|
+
lastSuccessAtMs = null;
|
|
291330
|
+
lastFailureEventSequence = null;
|
|
291331
|
+
lastShedEventSequence = null;
|
|
291332
|
+
lastSuccessEventSequence = null;
|
|
291333
|
+
lastSinkLatencyMs = null;
|
|
291334
|
+
healthEventSequence = 0;
|
|
291304
291335
|
lastShedWarnAtMs = 0;
|
|
291305
291336
|
closing = false;
|
|
291306
291337
|
closed = false;
|
|
@@ -291334,6 +291365,7 @@ class BrokerExecutionArchiver {
|
|
|
291334
291365
|
}
|
|
291335
291366
|
try {
|
|
291336
291367
|
this.flushTimer = setInterval(() => {
|
|
291368
|
+
this.emitArchiveHealthMetrics();
|
|
291337
291369
|
this.flush();
|
|
291338
291370
|
}, this.flushIntervalMs);
|
|
291339
291371
|
this.flushTimer.unref?.();
|
|
@@ -291377,32 +291409,34 @@ class BrokerExecutionArchiver {
|
|
|
291377
291409
|
row: { ...row.row, source: this.source }
|
|
291378
291410
|
};
|
|
291379
291411
|
if (this.queue.length >= this.maxQueueSize) {
|
|
291380
|
-
const
|
|
291381
|
-
if (
|
|
291382
|
-
this.appendLossRecords([
|
|
291412
|
+
const shedEntry = this.queue[0];
|
|
291413
|
+
if (shedEntry) {
|
|
291414
|
+
this.appendLossRecords([shedEntry], "queue_shed");
|
|
291383
291415
|
this.queue.shift();
|
|
291384
291416
|
}
|
|
291385
291417
|
this.stats.shed += 1;
|
|
291418
|
+
this.recordHealthEvent("shed");
|
|
291386
291419
|
this.recordArchiveMetric("cex_archive_rows_shed_total", {
|
|
291387
|
-
table:
|
|
291420
|
+
table: shedEntry?.table ?? "unknown",
|
|
291388
291421
|
source: this.source,
|
|
291389
|
-
feed:
|
|
291422
|
+
feed: shedEntry ? archiveFeed(shedEntry) : "NON_MARKET"
|
|
291390
291423
|
});
|
|
291391
291424
|
this.recordArchiveMetric("cex_archive_queue_saturated_rows_total", {
|
|
291392
|
-
table:
|
|
291425
|
+
table: shedEntry?.table ?? "unknown",
|
|
291393
291426
|
source: this.source,
|
|
291394
|
-
feed:
|
|
291427
|
+
feed: shedEntry ? archiveFeed(shedEntry) : "NON_MARKET"
|
|
291395
291428
|
});
|
|
291396
291429
|
const now3 = Date.now();
|
|
291397
291430
|
if (now3 - this.lastShedWarnAtMs >= SHED_WARN_INTERVAL_MS) {
|
|
291398
291431
|
log.warn("Archive queue full: shedding oldest rows", {
|
|
291399
291432
|
shed_total: this.stats.shed,
|
|
291400
291433
|
queue_max: this.maxQueueSize,
|
|
291401
|
-
table:
|
|
291434
|
+
table: shedEntry?.table ?? "unknown"
|
|
291402
291435
|
});
|
|
291403
291436
|
this.lastShedWarnAtMs = now3;
|
|
291404
291437
|
}
|
|
291405
291438
|
}
|
|
291439
|
+
this.enqueueTimes.set(archiveRow, Date.now());
|
|
291406
291440
|
this.queue.push(archiveRow);
|
|
291407
291441
|
this.stats.enqueued += 1;
|
|
291408
291442
|
this.recordArchiveMetric("cex_archive_rows_enqueued_total", {
|
|
@@ -291410,6 +291444,7 @@ class BrokerExecutionArchiver {
|
|
|
291410
291444
|
source: this.source,
|
|
291411
291445
|
feed: archiveFeed(archiveRow)
|
|
291412
291446
|
});
|
|
291447
|
+
this.emitArchiveHealthMetrics();
|
|
291413
291448
|
if (this.queue.length >= this.batchSize) {
|
|
291414
291449
|
this.flush();
|
|
291415
291450
|
}
|
|
@@ -291424,14 +291459,19 @@ class BrokerExecutionArchiver {
|
|
|
291424
291459
|
if (this.flushInFlight) {
|
|
291425
291460
|
return this.flushInFlight;
|
|
291426
291461
|
}
|
|
291462
|
+
const inFlightState = {};
|
|
291427
291463
|
const inFlight = this.flushBatch().then(() => {
|
|
291428
291464
|
return;
|
|
291429
291465
|
}).finally(() => {
|
|
291466
|
+
if (this.flushInFlight !== inFlightState.promise) {
|
|
291467
|
+
return;
|
|
291468
|
+
}
|
|
291430
291469
|
this.flushInFlight = null;
|
|
291431
291470
|
if (!this.closed && !this.closing && this.enabled && this.queue.length >= this.batchSize) {
|
|
291432
291471
|
queueMicrotask(() => void this.flush());
|
|
291433
291472
|
}
|
|
291434
291473
|
});
|
|
291474
|
+
inFlightState.promise = inFlight;
|
|
291435
291475
|
this.flushInFlight = inFlight;
|
|
291436
291476
|
return inFlight;
|
|
291437
291477
|
}
|
|
@@ -291478,6 +291518,79 @@ class BrokerExecutionArchiver {
|
|
|
291478
291518
|
getQueueDepth() {
|
|
291479
291519
|
return this.queue.length;
|
|
291480
291520
|
}
|
|
291521
|
+
getHealthSnapshot() {
|
|
291522
|
+
const now3 = Date.now();
|
|
291523
|
+
const oldestPendingAtMs = this.oldestPendingEnqueueAtMs();
|
|
291524
|
+
const oldestPendingAgeMs = oldestPendingAtMs === null ? 0 : Math.max(0, now3 - oldestPendingAtMs);
|
|
291525
|
+
const inFlightAgeMs = this.inFlightStartedAtMs === null ? 0 : Math.max(0, now3 - this.inFlightStartedAtMs);
|
|
291526
|
+
const lastUnhealthyEventSequence = Math.max(this.lastFailureEventSequence ?? Number.NEGATIVE_INFINITY, this.lastShedEventSequence ?? Number.NEGATIVE_INFINITY);
|
|
291527
|
+
const recoveredFromEvents = lastUnhealthyEventSequence === Number.NEGATIVE_INFINITY || this.lastSuccessEventSequence !== null && this.lastSuccessEventSequence > lastUnhealthyEventSequence;
|
|
291528
|
+
const healthy = this.enabled && !this.closing && !this.closed && this.queue.length < this.maxQueueSize && oldestPendingAgeMs <= this.forwarderTimeoutMs && inFlightAgeMs <= this.forwarderTimeoutMs && recoveredFromEvents;
|
|
291529
|
+
return {
|
|
291530
|
+
healthy,
|
|
291531
|
+
queue_depth: this.queue.length,
|
|
291532
|
+
oldest_pending_age_ms: oldestPendingAgeMs,
|
|
291533
|
+
shed_total: this.stats.shed,
|
|
291534
|
+
last_failure_at: this.lastFailureAtMs,
|
|
291535
|
+
last_shed_at: this.lastShedAtMs,
|
|
291536
|
+
last_success_at: this.lastSuccessAtMs,
|
|
291537
|
+
sink_latency_ms: this.lastSinkLatencyMs,
|
|
291538
|
+
sink_errors_total: this.stats.forwarderFailures,
|
|
291539
|
+
in_flight: this.inFlightBatch !== null,
|
|
291540
|
+
in_flight_age_ms: inFlightAgeMs
|
|
291541
|
+
};
|
|
291542
|
+
}
|
|
291543
|
+
oldestPendingEnqueueAtMs() {
|
|
291544
|
+
let oldest = null;
|
|
291545
|
+
for (const entry of this.queue) {
|
|
291546
|
+
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
291547
|
+
if (enqueuedAtMs === undefined) {
|
|
291548
|
+
continue;
|
|
291549
|
+
}
|
|
291550
|
+
oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
|
|
291551
|
+
}
|
|
291552
|
+
for (const entry of this.inFlightBatch ?? []) {
|
|
291553
|
+
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
291554
|
+
if (enqueuedAtMs === undefined) {
|
|
291555
|
+
continue;
|
|
291556
|
+
}
|
|
291557
|
+
oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
|
|
291558
|
+
}
|
|
291559
|
+
return oldest;
|
|
291560
|
+
}
|
|
291561
|
+
recordHealthEvent(event) {
|
|
291562
|
+
const eventSequence = ++this.healthEventSequence;
|
|
291563
|
+
const eventTimestampMs = Date.now();
|
|
291564
|
+
switch (event) {
|
|
291565
|
+
case "failure":
|
|
291566
|
+
this.lastFailureAtMs = eventTimestampMs;
|
|
291567
|
+
this.lastFailureEventSequence = eventSequence;
|
|
291568
|
+
return;
|
|
291569
|
+
case "shed":
|
|
291570
|
+
this.lastShedAtMs = eventTimestampMs;
|
|
291571
|
+
this.lastShedEventSequence = eventSequence;
|
|
291572
|
+
return;
|
|
291573
|
+
case "success":
|
|
291574
|
+
this.lastSuccessAtMs = eventTimestampMs;
|
|
291575
|
+
this.lastSuccessEventSequence = eventSequence;
|
|
291576
|
+
return;
|
|
291577
|
+
}
|
|
291578
|
+
}
|
|
291579
|
+
emitArchiveHealthMetrics() {
|
|
291580
|
+
const snapshot = this.getHealthSnapshot();
|
|
291581
|
+
const labels = { source: this.source };
|
|
291582
|
+
this.recordArchiveObservableGauge("cex_archive_health", snapshot.healthy ? 1 : 0, labels);
|
|
291583
|
+
this.recordArchiveGauge("cex_archive_queue_depth", snapshot.queue_depth, labels);
|
|
291584
|
+
this.recordArchiveGauge("cex_archive_oldest_pending_age_ms", snapshot.oldest_pending_age_ms, labels);
|
|
291585
|
+
this.recordArchiveGauge("cex_archive_shed_total", snapshot.shed_total, labels);
|
|
291586
|
+
this.recordArchiveGauge("cex_archive_last_failure_at", snapshot.last_failure_at ?? 0, labels);
|
|
291587
|
+
this.recordArchiveGauge("cex_archive_last_shed_at", snapshot.last_shed_at ?? 0, labels);
|
|
291588
|
+
this.recordArchiveGauge("cex_archive_last_success_at", snapshot.last_success_at ?? 0, labels);
|
|
291589
|
+
this.recordArchiveGauge("cex_archive_sink_latency_ms", snapshot.sink_latency_ms ?? 0, labels);
|
|
291590
|
+
this.recordArchiveGauge("cex_archive_sink_errors_total", snapshot.sink_errors_total, labels);
|
|
291591
|
+
this.recordArchiveGauge("cex_archive_in_flight", snapshot.in_flight ? 1 : 0, labels);
|
|
291592
|
+
this.recordArchiveGauge("cex_archive_in_flight_age_ms", snapshot.in_flight_age_ms, labels);
|
|
291593
|
+
}
|
|
291481
291594
|
closeLossJournal() {
|
|
291482
291595
|
if (this.deadLetterFd === undefined) {
|
|
291483
291596
|
return;
|
|
@@ -291500,8 +291613,9 @@ class BrokerExecutionArchiver {
|
|
|
291500
291613
|
this.appendLossRecords([dropped], "queue_shed");
|
|
291501
291614
|
this.queue.shift();
|
|
291502
291615
|
this.stats.shed += 1;
|
|
291616
|
+
this.recordHealthEvent("shed");
|
|
291503
291617
|
this.recordArchiveMetric("cex_archive_rows_shed_total", {
|
|
291504
|
-
table: dropped
|
|
291618
|
+
table: dropped.table,
|
|
291505
291619
|
source: this.source,
|
|
291506
291620
|
feed: archiveFeed(dropped)
|
|
291507
291621
|
});
|
|
@@ -291560,6 +291674,8 @@ class BrokerExecutionArchiver {
|
|
|
291560
291674
|
if (batch.length === 0) {
|
|
291561
291675
|
return true;
|
|
291562
291676
|
}
|
|
291677
|
+
this.inFlightBatch = batch;
|
|
291678
|
+
this.inFlightStartedAtMs = Date.now();
|
|
291563
291679
|
for (const entry of batch) {
|
|
291564
291680
|
if (isBrokerExecutionArchiveTable(entry.table)) {
|
|
291565
291681
|
this.emitOtelLog(entry);
|
|
@@ -291570,8 +291686,15 @@ class BrokerExecutionArchiver {
|
|
|
291570
291686
|
await this.postToForwarder(batch);
|
|
291571
291687
|
} catch (error) {
|
|
291572
291688
|
this.stats.forwarderFailures += 1;
|
|
291573
|
-
this.
|
|
291574
|
-
this.
|
|
291689
|
+
this.recordHealthEvent("failure");
|
|
291690
|
+
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
291691
|
+
this.queue.unshift(...batch);
|
|
291692
|
+
try {
|
|
291693
|
+
this.enforceQueueBound();
|
|
291694
|
+
} finally {
|
|
291695
|
+
this.clearInFlightBatch(batch);
|
|
291696
|
+
this.emitArchiveHealthMetrics();
|
|
291697
|
+
}
|
|
291575
291698
|
this.recordArchiveMetric("cex_archive_forwarder_failures_total", {
|
|
291576
291699
|
count: batch.length
|
|
291577
291700
|
});
|
|
@@ -291580,14 +291703,26 @@ class BrokerExecutionArchiver {
|
|
|
291580
291703
|
}
|
|
291581
291704
|
}
|
|
291582
291705
|
this.stats.flushed += batch.length;
|
|
291706
|
+
this.recordHealthEvent("success");
|
|
291707
|
+
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
291708
|
+
this.clearInFlightBatch(batch);
|
|
291583
291709
|
this.recordFlushHealth(batch);
|
|
291710
|
+
this.emitArchiveHealthMetrics();
|
|
291584
291711
|
return true;
|
|
291585
291712
|
}
|
|
291713
|
+
clearInFlightBatch(batch) {
|
|
291714
|
+
if (this.inFlightBatch !== batch) {
|
|
291715
|
+
return;
|
|
291716
|
+
}
|
|
291717
|
+
this.inFlightBatch = null;
|
|
291718
|
+
this.inFlightStartedAtMs = null;
|
|
291719
|
+
}
|
|
291586
291720
|
recordFlushHealth(batch) {
|
|
291587
291721
|
const countByTable = new Map;
|
|
291588
291722
|
for (const entry of batch) {
|
|
291589
|
-
const
|
|
291590
|
-
const
|
|
291723
|
+
const row = entry;
|
|
291724
|
+
const key = `${row.table}\x00${archiveFeed(row)}`;
|
|
291725
|
+
const grouped = countByTable.get(key) ?? { row, count: 0 };
|
|
291591
291726
|
grouped.count += 1;
|
|
291592
291727
|
countByTable.set(key, grouped);
|
|
291593
291728
|
}
|
|
@@ -291605,9 +291740,14 @@ class BrokerExecutionArchiver {
|
|
|
291605
291740
|
await this.otelMetrics?.recordCounter(metricName, value, labels);
|
|
291606
291741
|
} catch {}
|
|
291607
291742
|
}
|
|
291608
|
-
async recordArchiveGauge(metricName, value) {
|
|
291743
|
+
async recordArchiveGauge(metricName, value, labels = {}) {
|
|
291744
|
+
try {
|
|
291745
|
+
await this.otelMetrics?.recordGauge(metricName, value, labels);
|
|
291746
|
+
} catch {}
|
|
291747
|
+
}
|
|
291748
|
+
async recordArchiveObservableGauge(metricName, value, labels) {
|
|
291609
291749
|
try {
|
|
291610
|
-
await this.otelMetrics?.
|
|
291750
|
+
await this.otelMetrics?.setObservableGauge(metricName, value, labels);
|
|
291611
291751
|
} catch {}
|
|
291612
291752
|
}
|
|
291613
291753
|
emitOtelLog(entry) {
|
|
@@ -291644,27 +291784,136 @@ class BrokerExecutionArchiver {
|
|
|
291644
291784
|
headers.authorization = `Bearer ${this.forwarderAuthToken}`;
|
|
291645
291785
|
}
|
|
291646
291786
|
return new Promise((resolve, reject) => {
|
|
291647
|
-
|
|
291648
|
-
|
|
291649
|
-
|
|
291650
|
-
|
|
291651
|
-
|
|
291652
|
-
|
|
291653
|
-
|
|
291654
|
-
|
|
291655
|
-
|
|
291656
|
-
|
|
291657
|
-
|
|
291658
|
-
|
|
291787
|
+
let req;
|
|
291788
|
+
let response;
|
|
291789
|
+
let settled = false;
|
|
291790
|
+
let requestClosed = false;
|
|
291791
|
+
let requestCloseFailureScheduled = false;
|
|
291792
|
+
let responseClosed = false;
|
|
291793
|
+
let responseEnded = false;
|
|
291794
|
+
let outerTimer;
|
|
291795
|
+
function cleanupRequestListeners() {
|
|
291796
|
+
req?.removeListener("error", onRequestError);
|
|
291797
|
+
req?.removeListener("timeout", onRequestTimeout);
|
|
291798
|
+
req?.removeListener("close", onRequestClose);
|
|
291799
|
+
}
|
|
291800
|
+
function cleanupResponseListeners() {
|
|
291801
|
+
response?.removeListener("data", onResponseData);
|
|
291802
|
+
response?.removeListener("end", onResponseEnd);
|
|
291803
|
+
response?.removeListener("aborted", onResponseAborted);
|
|
291804
|
+
response?.removeListener("error", onResponseError);
|
|
291805
|
+
response?.removeListener("close", onResponseClose);
|
|
291806
|
+
}
|
|
291807
|
+
function onRequestClose() {
|
|
291808
|
+
requestClosed = true;
|
|
291809
|
+
if (!settled && !response && !requestCloseFailureScheduled) {
|
|
291810
|
+
requestCloseFailureScheduled = true;
|
|
291811
|
+
queueMicrotask(() => {
|
|
291812
|
+
if (!settled && !response) {
|
|
291813
|
+
settle2(new Error("Archive forwarder request closed before response headers"), false, true);
|
|
291814
|
+
}
|
|
291815
|
+
});
|
|
291816
|
+
}
|
|
291817
|
+
if (settled) {
|
|
291818
|
+
cleanupRequestListeners();
|
|
291819
|
+
}
|
|
291820
|
+
}
|
|
291821
|
+
function onRequestError(error) {
|
|
291822
|
+
if (!settled) {
|
|
291823
|
+
settle2(error, true);
|
|
291824
|
+
} else {
|
|
291825
|
+
cleanupRequestListeners();
|
|
291826
|
+
}
|
|
291827
|
+
}
|
|
291828
|
+
function onRequestTimeout() {
|
|
291829
|
+
settle2(new Error("Archive forwarder socket timed out"), true);
|
|
291830
|
+
}
|
|
291831
|
+
function onResponseData() {}
|
|
291832
|
+
function onResponseEnd() {
|
|
291833
|
+
responseEnded = true;
|
|
291834
|
+
if (!response?.complete) {
|
|
291835
|
+
settle2(new Error("Archive forwarder response was incomplete"), true);
|
|
291836
|
+
return;
|
|
291837
|
+
}
|
|
291838
|
+
const status = response.statusCode ?? 0;
|
|
291839
|
+
if (status < 200 || status >= 300) {
|
|
291840
|
+
settle2(new Error(`Archive forwarder returned ${status} ${response.statusMessage ?? ""}`), false);
|
|
291841
|
+
return;
|
|
291842
|
+
}
|
|
291843
|
+
settle2(undefined, false);
|
|
291844
|
+
}
|
|
291845
|
+
function onResponseAborted() {
|
|
291846
|
+
settle2(new Error("Archive forwarder response was aborted"), true);
|
|
291847
|
+
}
|
|
291848
|
+
function onResponseError(error) {
|
|
291849
|
+
if (!settled) {
|
|
291850
|
+
settle2(error, true);
|
|
291851
|
+
}
|
|
291852
|
+
}
|
|
291853
|
+
function onResponseClose() {
|
|
291854
|
+
responseClosed = true;
|
|
291855
|
+
if (!settled && !responseEnded) {
|
|
291856
|
+
settle2(new Error("Archive forwarder response closed before completion"), true);
|
|
291857
|
+
return;
|
|
291858
|
+
}
|
|
291859
|
+
cleanupResponseListeners();
|
|
291860
|
+
}
|
|
291861
|
+
function settle2(error, destroyRequest, retainClosedRequestError = false) {
|
|
291862
|
+
if (settled) {
|
|
291863
|
+
return;
|
|
291864
|
+
}
|
|
291865
|
+
settled = true;
|
|
291866
|
+
if (outerTimer) {
|
|
291867
|
+
clearTimeout(outerTimer);
|
|
291868
|
+
outerTimer = undefined;
|
|
291869
|
+
}
|
|
291870
|
+
response?.removeListener("data", onResponseData);
|
|
291871
|
+
response?.removeListener("end", onResponseEnd);
|
|
291872
|
+
response?.removeListener("aborted", onResponseAborted);
|
|
291873
|
+
if (responseClosed) {
|
|
291874
|
+
cleanupResponseListeners();
|
|
291875
|
+
}
|
|
291876
|
+
if (destroyRequest && req && !requestClosed) {
|
|
291877
|
+
req.destroy(error);
|
|
291878
|
+
} else if (retainClosedRequestError && req && requestClosed) {
|
|
291879
|
+
req.removeListener("timeout", onRequestTimeout);
|
|
291880
|
+
req.removeListener("close", onRequestClose);
|
|
291881
|
+
} else if (!req || requestClosed) {
|
|
291882
|
+
cleanupRequestListeners();
|
|
291883
|
+
}
|
|
291884
|
+
if (error) {
|
|
291885
|
+
reject(error);
|
|
291886
|
+
} else {
|
|
291659
291887
|
resolve();
|
|
291888
|
+
}
|
|
291889
|
+
}
|
|
291890
|
+
outerTimer = setTimeout(() => {
|
|
291891
|
+
settle2(new Error("Archive forwarder request deadline exceeded"), true);
|
|
291892
|
+
}, this.forwarderTimeoutMs);
|
|
291893
|
+
try {
|
|
291894
|
+
req = doRequest(url2, {
|
|
291895
|
+
method: "POST",
|
|
291896
|
+
headers,
|
|
291897
|
+
timeout: this.forwarderTimeoutMs
|
|
291898
|
+
}, (nextResponse) => {
|
|
291899
|
+
response = nextResponse;
|
|
291900
|
+
response.on("data", onResponseData);
|
|
291901
|
+
response.on("end", onResponseEnd);
|
|
291902
|
+
response.on("aborted", onResponseAborted);
|
|
291903
|
+
response.on("error", onResponseError);
|
|
291904
|
+
response.on("close", onResponseClose);
|
|
291905
|
+
if (settled) {
|
|
291906
|
+
response.resume();
|
|
291907
|
+
}
|
|
291660
291908
|
});
|
|
291661
|
-
|
|
291662
|
-
|
|
291663
|
-
|
|
291664
|
-
req.
|
|
291665
|
-
|
|
291666
|
-
|
|
291667
|
-
|
|
291909
|
+
req.on("error", onRequestError);
|
|
291910
|
+
req.on("timeout", onRequestTimeout);
|
|
291911
|
+
req.on("close", onRequestClose);
|
|
291912
|
+
req.write(body);
|
|
291913
|
+
req.end();
|
|
291914
|
+
} catch (error) {
|
|
291915
|
+
settle2(error instanceof Error ? error : new Error(String(error)), true);
|
|
291916
|
+
}
|
|
291668
291917
|
});
|
|
291669
291918
|
}
|
|
291670
291919
|
}
|
|
@@ -292137,9 +292386,18 @@ class AccountBalanceArchivePoller {
|
|
|
292137
292386
|
// src/helpers/deposit-archive-poller.ts
|
|
292138
292387
|
var DEFAULT_CONFIG2 = {
|
|
292139
292388
|
pollIntervalMs: 60000,
|
|
292389
|
+
fetchTimeoutMs: 30000,
|
|
292140
292390
|
lookbackMs: 24 * 60 * 60 * 1000,
|
|
292141
292391
|
depositsLimit: 50
|
|
292142
292392
|
};
|
|
292393
|
+
function withTimeout(promise, timeoutMs, label) {
|
|
292394
|
+
let timer;
|
|
292395
|
+
const expiry = new Promise((_resolve, reject) => {
|
|
292396
|
+
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
292397
|
+
timer.unref?.();
|
|
292398
|
+
});
|
|
292399
|
+
return Promise.race([promise, expiry]).finally(() => clearTimeout(timer));
|
|
292400
|
+
}
|
|
292143
292401
|
var ALL_CURRENCIES_CODE = "*";
|
|
292144
292402
|
function depositTimestamp(record) {
|
|
292145
292403
|
const observedAt = depositField(record, [
|
|
@@ -292252,6 +292510,14 @@ class DepositArchivePoller {
|
|
|
292252
292510
|
return true;
|
|
292253
292511
|
}
|
|
292254
292512
|
async#pollOne(target) {
|
|
292513
|
+
let outcome = "error";
|
|
292514
|
+
try {
|
|
292515
|
+
outcome = await this.#pollTarget(target);
|
|
292516
|
+
} finally {
|
|
292517
|
+
this.params.metrics?.recordCounter("cex_deposit_poller_polls_total", 1, { exchange: target.exchangeId, outcome });
|
|
292518
|
+
}
|
|
292519
|
+
}
|
|
292520
|
+
async#pollTarget(target) {
|
|
292255
292521
|
const exchange = target.account.exchange;
|
|
292256
292522
|
const key = this.#targetKey(target);
|
|
292257
292523
|
if (typeof exchange.fetchDeposits !== "function" || exchange.has?.fetchDeposits === false) {
|
|
@@ -292262,12 +292528,12 @@ class DepositArchivePoller {
|
|
|
292262
292528
|
account: target.account.label
|
|
292263
292529
|
});
|
|
292264
292530
|
}
|
|
292265
|
-
return;
|
|
292531
|
+
return "unsupported";
|
|
292266
292532
|
}
|
|
292267
292533
|
const since = this.#cursors.get(key) ?? Date.now() - this.#config.lookbackMs;
|
|
292268
292534
|
let deposits;
|
|
292269
292535
|
try {
|
|
292270
|
-
deposits = await exchange.fetchDeposits(undefined, since, this.#config.depositsLimit);
|
|
292536
|
+
deposits = await withTimeout(exchange.fetchDeposits(undefined, since, this.#config.depositsLimit), this.#config.fetchTimeoutMs, "fetchDeposits");
|
|
292271
292537
|
} catch (error) {
|
|
292272
292538
|
this.params.metrics?.recordCounter("cex_deposit_poller_errors_total", 1, { exchange: target.exchangeId });
|
|
292273
292539
|
log.warn("Deposit archive poll failed", {
|
|
@@ -292275,10 +292541,10 @@ class DepositArchivePoller {
|
|
|
292275
292541
|
account: target.account.label,
|
|
292276
292542
|
error
|
|
292277
292543
|
});
|
|
292278
|
-
return;
|
|
292544
|
+
return "error";
|
|
292279
292545
|
}
|
|
292280
292546
|
if (!Array.isArray(deposits) || deposits.length === 0) {
|
|
292281
|
-
return;
|
|
292547
|
+
return "ok";
|
|
292282
292548
|
}
|
|
292283
292549
|
let archived = 0;
|
|
292284
292550
|
for (const deposit of deposits) {
|
|
@@ -292326,7 +292592,7 @@ class DepositArchivePoller {
|
|
|
292326
292592
|
network: network === undefined ? undefined : String(network),
|
|
292327
292593
|
externalId: depositTxid,
|
|
292328
292594
|
txid: depositTxid,
|
|
292329
|
-
exchangeTimestamp:
|
|
292595
|
+
exchangeTimestamp: normalizeTimestamp2(creditedAt),
|
|
292330
292596
|
payload: record
|
|
292331
292597
|
}
|
|
292332
292598
|
}));
|
|
@@ -292359,6 +292625,7 @@ class DepositArchivePoller {
|
|
|
292359
292625
|
this.#lastArchivedByTarget.delete(key);
|
|
292360
292626
|
}
|
|
292361
292627
|
}
|
|
292628
|
+
return "ok";
|
|
292362
292629
|
}
|
|
292363
292630
|
#targetKey(target) {
|
|
292364
292631
|
return `${target.exchangeId}|${target.account.label}|${target.code}`;
|
|
@@ -292500,6 +292767,309 @@ class FillArchivePoller {
|
|
|
292500
292767
|
}
|
|
292501
292768
|
}
|
|
292502
292769
|
|
|
292770
|
+
// src/helpers/market-data-archive/capture-contract.ts
|
|
292771
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
292772
|
+
var MARKET_CAPTURE_SCHEMA_VERSION = "1.0.0";
|
|
292773
|
+
var CHECKSUM_ALGORITHM = "sha256-canonical-json-v1";
|
|
292774
|
+
var ARCHIVE_SOURCES = ["broker_read", "broker_write"];
|
|
292775
|
+
var CAPTURE_FEEDS = [
|
|
292776
|
+
"ORDERBOOK",
|
|
292777
|
+
"TICKER",
|
|
292778
|
+
"TRADES",
|
|
292779
|
+
"OHLCV"
|
|
292780
|
+
];
|
|
292781
|
+
var SOURCE_MODES = [
|
|
292782
|
+
"broker_live_stream_v1",
|
|
292783
|
+
"broker_live_sampling_v1",
|
|
292784
|
+
"broker_current_snapshot_v1",
|
|
292785
|
+
"broker_bootstrap_fetch_v1",
|
|
292786
|
+
"external_ccxt_fallback_v1",
|
|
292787
|
+
"external_hummingbot_fallback_v1",
|
|
292788
|
+
"legacy_migration_v1"
|
|
292789
|
+
];
|
|
292790
|
+
var RAW_CAPTURE_SCOPES = [
|
|
292791
|
+
"ccxt_normalized_object",
|
|
292792
|
+
"broker_visible_payload",
|
|
292793
|
+
"exchange_wire_frame"
|
|
292794
|
+
];
|
|
292795
|
+
var CHECKSUM_FIELDS = new Set([
|
|
292796
|
+
"normalized_row_checksum",
|
|
292797
|
+
"raw_checksum",
|
|
292798
|
+
"checksum"
|
|
292799
|
+
]);
|
|
292800
|
+
function canonicalDecimal(value) {
|
|
292801
|
+
if (!Number.isFinite(value)) {
|
|
292802
|
+
throw new Error("Canonical numbers must be finite");
|
|
292803
|
+
}
|
|
292804
|
+
if (Object.is(value, -0)) {
|
|
292805
|
+
return "0";
|
|
292806
|
+
}
|
|
292807
|
+
const rendered = String(value).toLowerCase();
|
|
292808
|
+
if (!rendered.includes("e")) {
|
|
292809
|
+
return rendered;
|
|
292810
|
+
}
|
|
292811
|
+
const [coefficient = "0", exponentText = "0"] = rendered.split("e");
|
|
292812
|
+
const exponent = Number.parseInt(exponentText, 10);
|
|
292813
|
+
const negative = coefficient.startsWith("-");
|
|
292814
|
+
const unsigned = negative ? coefficient.slice(1) : coefficient;
|
|
292815
|
+
const [integer = "0", fraction = ""] = unsigned.split(".");
|
|
292816
|
+
const digits = `${integer}${fraction}`;
|
|
292817
|
+
const decimalIndex = integer.length + exponent;
|
|
292818
|
+
let result;
|
|
292819
|
+
if (decimalIndex <= 0) {
|
|
292820
|
+
result = `0.${"0".repeat(-decimalIndex)}${digits}`;
|
|
292821
|
+
} else if (decimalIndex >= digits.length) {
|
|
292822
|
+
result = `${digits}${"0".repeat(decimalIndex - digits.length)}`;
|
|
292823
|
+
} else {
|
|
292824
|
+
result = `${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
|
|
292825
|
+
}
|
|
292826
|
+
return negative ? `-${result}` : result;
|
|
292827
|
+
}
|
|
292828
|
+
function serializeCanonical(value, stack) {
|
|
292829
|
+
if (value === null)
|
|
292830
|
+
return "null";
|
|
292831
|
+
if (typeof value === "string")
|
|
292832
|
+
return JSON.stringify(value);
|
|
292833
|
+
if (typeof value === "boolean")
|
|
292834
|
+
return value ? "true" : "false";
|
|
292835
|
+
if (typeof value === "number")
|
|
292836
|
+
return canonicalDecimal(value);
|
|
292837
|
+
if (typeof value === "bigint")
|
|
292838
|
+
return value.toString(10);
|
|
292839
|
+
if (value instanceof Date) {
|
|
292840
|
+
if (Number.isNaN(value.getTime())) {
|
|
292841
|
+
throw new Error("Canonical timestamps must be valid");
|
|
292842
|
+
}
|
|
292843
|
+
return value.getTime().toString(10);
|
|
292844
|
+
}
|
|
292845
|
+
if (Array.isArray(value)) {
|
|
292846
|
+
if (stack.has(value))
|
|
292847
|
+
throw new Error("Canonical values must be acyclic");
|
|
292848
|
+
stack.add(value);
|
|
292849
|
+
const result = `[${value.map((entry) => entry === undefined ? "null" : serializeCanonical(entry, stack)).join(",")}]`;
|
|
292850
|
+
stack.delete(value);
|
|
292851
|
+
return result;
|
|
292852
|
+
}
|
|
292853
|
+
if (typeof value === "object") {
|
|
292854
|
+
if (stack.has(value))
|
|
292855
|
+
throw new Error("Canonical values must be acyclic");
|
|
292856
|
+
stack.add(value);
|
|
292857
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([left], [right]) => left.localeCompare(right));
|
|
292858
|
+
const result = `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${serializeCanonical(entry, stack)}`).join(",")}}`;
|
|
292859
|
+
stack.delete(value);
|
|
292860
|
+
return result;
|
|
292861
|
+
}
|
|
292862
|
+
throw new Error(`Unsupported canonical value type: ${typeof value}`);
|
|
292863
|
+
}
|
|
292864
|
+
function canonicalSerialize(value) {
|
|
292865
|
+
return serializeCanonical(value, new Set);
|
|
292866
|
+
}
|
|
292867
|
+
function omitChecksumFields(value) {
|
|
292868
|
+
if (Array.isArray(value))
|
|
292869
|
+
return value.map(omitChecksumFields);
|
|
292870
|
+
if (value && typeof value === "object" && !(value instanceof Date)) {
|
|
292871
|
+
return Object.fromEntries(Object.entries(value).filter(([key]) => !CHECKSUM_FIELDS.has(key)).map(([key, entry]) => [key, omitChecksumFields(entry)]));
|
|
292872
|
+
}
|
|
292873
|
+
return value;
|
|
292874
|
+
}
|
|
292875
|
+
function sha256Canonical(value) {
|
|
292876
|
+
return createHash3("sha256").update(canonicalSerialize(omitChecksumFields(value))).digest("hex");
|
|
292877
|
+
}
|
|
292878
|
+
function normalizeTimestampMs(value, field) {
|
|
292879
|
+
let timestamp;
|
|
292880
|
+
if (value instanceof Date) {
|
|
292881
|
+
timestamp = value.getTime();
|
|
292882
|
+
} else if (typeof value === "number") {
|
|
292883
|
+
timestamp = value;
|
|
292884
|
+
} else if (typeof value === "string" && /^\d+$/.test(value.trim())) {
|
|
292885
|
+
timestamp = Number(value.trim());
|
|
292886
|
+
} else if (typeof value === "string") {
|
|
292887
|
+
timestamp = Date.parse(value);
|
|
292888
|
+
} else {
|
|
292889
|
+
timestamp = Number.NaN;
|
|
292890
|
+
}
|
|
292891
|
+
if (!Number.isSafeInteger(timestamp) || timestamp < 0) {
|
|
292892
|
+
throw new Error(`${field} must be a non-negative millisecond timestamp`);
|
|
292893
|
+
}
|
|
292894
|
+
return timestamp;
|
|
292895
|
+
}
|
|
292896
|
+
function assertCaptureContext(context2) {
|
|
292897
|
+
if (!ARCHIVE_SOURCES.includes(context2.source)) {
|
|
292898
|
+
throw new Error(`Unsupported archive source: ${context2.source}`);
|
|
292899
|
+
}
|
|
292900
|
+
if (!CAPTURE_FEEDS.includes(context2.feed)) {
|
|
292901
|
+
throw new Error(`Unsupported capture feed: ${context2.feed}`);
|
|
292902
|
+
}
|
|
292903
|
+
if (!SOURCE_MODES.includes(context2.sourceMode)) {
|
|
292904
|
+
throw new Error(`Unsupported source mode: ${context2.sourceMode}`);
|
|
292905
|
+
}
|
|
292906
|
+
for (const [field, value] of [
|
|
292907
|
+
["deployment_id", context2.deploymentId],
|
|
292908
|
+
["capture_bundle_id", context2.captureBundleId],
|
|
292909
|
+
["exchange", context2.exchange],
|
|
292910
|
+
["symbol", context2.symbol],
|
|
292911
|
+
["provider", context2.provider]
|
|
292912
|
+
]) {
|
|
292913
|
+
if (!value.trim())
|
|
292914
|
+
throw new Error(`${field} must not be empty`);
|
|
292915
|
+
}
|
|
292916
|
+
}
|
|
292917
|
+
function createRawCapture(context2, input) {
|
|
292918
|
+
assertCaptureContext(context2);
|
|
292919
|
+
if (!RAW_CAPTURE_SCOPES.includes(input.scope)) {
|
|
292920
|
+
throw new Error(`Unsupported raw capture scope: ${input.scope}`);
|
|
292921
|
+
}
|
|
292922
|
+
const eventTimeMs = normalizeTimestampMs(input.eventTimeMs, "event_time_ms");
|
|
292923
|
+
const receivedTimeMs = normalizeTimestampMs(input.receivedTimeMs, "received_time_ms");
|
|
292924
|
+
const redactedPayload = redactStreamPayload(input.payload);
|
|
292925
|
+
const rawChecksum = sha256Canonical(redactedPayload);
|
|
292926
|
+
const rawCaptureId = sha256Canonical({
|
|
292927
|
+
capture_bundle_id: context2.captureBundleId,
|
|
292928
|
+
exchange: context2.exchange.trim().toLowerCase(),
|
|
292929
|
+
feed: context2.feed,
|
|
292930
|
+
raw_capture_scope: input.scope,
|
|
292931
|
+
raw_payload_sha256: rawChecksum,
|
|
292932
|
+
schema_version: context2.schemaVersion,
|
|
292933
|
+
source_mode: context2.sourceMode,
|
|
292934
|
+
source_symbol: context2.symbol.trim(),
|
|
292935
|
+
source_time_ms: eventTimeMs
|
|
292936
|
+
});
|
|
292937
|
+
return {
|
|
292938
|
+
rawCaptureId,
|
|
292939
|
+
rawCaptureScope: input.scope,
|
|
292940
|
+
rawChecksum,
|
|
292941
|
+
redactedPayload,
|
|
292942
|
+
eventTimeMs,
|
|
292943
|
+
receivedTimeMs,
|
|
292944
|
+
checksumAlgorithm: context2.checksumAlgorithm
|
|
292945
|
+
};
|
|
292946
|
+
}
|
|
292947
|
+
function captureCoreFields(context2, rawCapture) {
|
|
292948
|
+
assertCaptureContext(context2);
|
|
292949
|
+
return {
|
|
292950
|
+
source: context2.source,
|
|
292951
|
+
deployment_id: context2.deploymentId,
|
|
292952
|
+
capture_bundle_id: context2.captureBundleId,
|
|
292953
|
+
exchange: context2.exchange.trim().toLowerCase(),
|
|
292954
|
+
symbol: context2.symbol.trim(),
|
|
292955
|
+
trading_pair: context2.symbol.trim().replace("/", "-"),
|
|
292956
|
+
source_symbol: context2.symbol.trim(),
|
|
292957
|
+
asset_type: context2.assetType,
|
|
292958
|
+
feed: context2.feed,
|
|
292959
|
+
provider: context2.provider,
|
|
292960
|
+
source_mode: context2.sourceMode,
|
|
292961
|
+
source_time_ms: rawCapture.eventTimeMs,
|
|
292962
|
+
received_time_ms: rawCapture.receivedTimeMs,
|
|
292963
|
+
raw_capture_id: rawCapture.rawCaptureId,
|
|
292964
|
+
raw_capture_scope: rawCapture.rawCaptureScope,
|
|
292965
|
+
schema_version: context2.schemaVersion,
|
|
292966
|
+
checksum_algorithm: context2.checksumAlgorithm,
|
|
292967
|
+
raw_checksum: rawCapture.rawChecksum,
|
|
292968
|
+
provenance_complete: context2.provenanceComplete ? 1 : 0
|
|
292969
|
+
};
|
|
292970
|
+
}
|
|
292971
|
+
|
|
292972
|
+
// src/helpers/market-data-archive/capture-context.ts
|
|
292973
|
+
function createMarketCaptureContext(input) {
|
|
292974
|
+
const environment = input.environment ?? "development";
|
|
292975
|
+
const deploymentId = input.deploymentId.trim();
|
|
292976
|
+
if (!deploymentId)
|
|
292977
|
+
throw new Error("deployment_id must not be empty");
|
|
292978
|
+
const configuredBundle = input.captureBundleId?.trim();
|
|
292979
|
+
if (environment === "production" && !configuredBundle) {
|
|
292980
|
+
throw new Error("capture_bundle_id is required for production market capture");
|
|
292981
|
+
}
|
|
292982
|
+
const exchange = input.exchange.trim().toLowerCase();
|
|
292983
|
+
const symbol = input.symbol.trim();
|
|
292984
|
+
if (!exchange || !symbol) {
|
|
292985
|
+
throw new Error("exchange and symbol are required for market capture");
|
|
292986
|
+
}
|
|
292987
|
+
return {
|
|
292988
|
+
source: input.source,
|
|
292989
|
+
deploymentId,
|
|
292990
|
+
captureBundleId: configuredBundle ?? `development:${deploymentId}`,
|
|
292991
|
+
exchange,
|
|
292992
|
+
symbol,
|
|
292993
|
+
assetType: input.assetType,
|
|
292994
|
+
feed: input.feed,
|
|
292995
|
+
provider: input.provider?.trim() || `ccxt:${exchange}`,
|
|
292996
|
+
sourceMode: input.sourceMode,
|
|
292997
|
+
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
292998
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
292999
|
+
provenanceComplete: true,
|
|
293000
|
+
timeframe: input.timeframe,
|
|
293001
|
+
accountSelector: input.accountSelector
|
|
293002
|
+
};
|
|
293003
|
+
}
|
|
293004
|
+
function resolveMarketCaptureArchiveState(input) {
|
|
293005
|
+
if (!input.archiveEnabled) {
|
|
293006
|
+
return { enabled: false, reason: "archive_disabled" };
|
|
293007
|
+
}
|
|
293008
|
+
if (!input.marketArchiveEnabled) {
|
|
293009
|
+
return { enabled: false, reason: "market_archive_disabled" };
|
|
293010
|
+
}
|
|
293011
|
+
const environment = input.environment?.trim() || "development";
|
|
293012
|
+
if (environment !== "development" && environment !== "production") {
|
|
293013
|
+
return { enabled: false, reason: "invalid_capture_environment" };
|
|
293014
|
+
}
|
|
293015
|
+
if (environment === "production") {
|
|
293016
|
+
const deploymentId = input.deploymentId?.trim();
|
|
293017
|
+
if (!deploymentId || deploymentId === "unknown") {
|
|
293018
|
+
return { enabled: false, reason: "missing_deployment_id" };
|
|
293019
|
+
}
|
|
293020
|
+
if (!input.captureBundleId?.trim()) {
|
|
293021
|
+
return { enabled: false, reason: "missing_capture_bundle_id" };
|
|
293022
|
+
}
|
|
293023
|
+
}
|
|
293024
|
+
return { enabled: true };
|
|
293025
|
+
}
|
|
293026
|
+
function assertMarketCaptureArchiveStartable(state) {
|
|
293027
|
+
if (state.enabled || state.reason === "archive_disabled" || state.reason === "market_archive_disabled") {
|
|
293028
|
+
return;
|
|
293029
|
+
}
|
|
293030
|
+
throw new Error(`Refusing to start: canonical market-data archival was requested but its capture identity is incomplete (${state.reason}). ` + "Set CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT, CEX_BROKER_DEPLOYMENT_ID and CEX_BROKER_CAPTURE_BUNDLE_ID, " + "or disable archival explicitly via CEX_BROKER_MARKET_ARCHIVE_ENABLED.");
|
|
293031
|
+
}
|
|
293032
|
+
function captureEnvironmentFromEnv(value = process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT) {
|
|
293033
|
+
const environment = value?.trim() || "development";
|
|
293034
|
+
if (environment !== "development" && environment !== "production") {
|
|
293035
|
+
throw new Error("CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT must be development or production");
|
|
293036
|
+
}
|
|
293037
|
+
return environment;
|
|
293038
|
+
}
|
|
293039
|
+
|
|
293040
|
+
// src/helpers/market-data-archive/orderbook-sampler.ts
|
|
293041
|
+
var DEFAULT_ORDERBOOK_INTERVAL_MS = 1000;
|
|
293042
|
+
function getOrderbookIntervalMs() {
|
|
293043
|
+
const raw = process.env.CEX_BROKER_ORDERBOOK_INTERVAL_MS ?? process.env.CEX_BROKER_ORDERBOOK_TOB_INTERVAL_MS;
|
|
293044
|
+
if (!raw) {
|
|
293045
|
+
return DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
293046
|
+
}
|
|
293047
|
+
const parsed = Number.parseInt(raw, 10);
|
|
293048
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
293049
|
+
}
|
|
293050
|
+
function isMarketArchiveEnabled() {
|
|
293051
|
+
return process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED !== "false";
|
|
293052
|
+
}
|
|
293053
|
+
|
|
293054
|
+
class OrderbookSampler {
|
|
293055
|
+
intervalMs;
|
|
293056
|
+
lastEmitMs = null;
|
|
293057
|
+
constructor(intervalMs = getOrderbookIntervalMs()) {
|
|
293058
|
+
this.intervalMs = intervalMs;
|
|
293059
|
+
}
|
|
293060
|
+
shouldEmit(nowMs = Date.now()) {
|
|
293061
|
+
if (this.lastEmitMs !== null && nowMs < this.lastEmitMs) {
|
|
293062
|
+
this.lastEmitMs = nowMs;
|
|
293063
|
+
return true;
|
|
293064
|
+
}
|
|
293065
|
+
if (this.lastEmitMs !== null && nowMs - this.lastEmitMs < this.intervalMs) {
|
|
293066
|
+
return false;
|
|
293067
|
+
}
|
|
293068
|
+
this.lastEmitMs = nowMs;
|
|
293069
|
+
return true;
|
|
293070
|
+
}
|
|
293071
|
+
}
|
|
293072
|
+
|
|
292503
293073
|
// src/helpers/order-activity-tracker.ts
|
|
292504
293074
|
var DEFAULT_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
|
292505
293075
|
|
|
@@ -292883,6 +293453,925 @@ function createOtelLogsFromEnv() {
|
|
|
292883
293453
|
return new OtelLogs(config);
|
|
292884
293454
|
}
|
|
292885
293455
|
|
|
293456
|
+
// src/helpers/stream-health-publisher.ts
|
|
293457
|
+
import { createHash as createHash4, randomUUID } from "node:crypto";
|
|
293458
|
+
import {
|
|
293459
|
+
closeSync as closeSync2,
|
|
293460
|
+
fsyncSync as fsyncSync2,
|
|
293461
|
+
openSync as openSync2,
|
|
293462
|
+
readFileSync,
|
|
293463
|
+
renameSync,
|
|
293464
|
+
statSync,
|
|
293465
|
+
unlinkSync,
|
|
293466
|
+
writeFileSync
|
|
293467
|
+
} from "node:fs";
|
|
293468
|
+
import { request as httpRequest3 } from "node:http";
|
|
293469
|
+
import { request as httpsRequest3 } from "node:https";
|
|
293470
|
+
import { dirname } from "node:path";
|
|
293471
|
+
var SOURCE = "broker_write";
|
|
293472
|
+
var TABLE = "broker_stream_health.snapshots";
|
|
293473
|
+
var PRODUCER_ID = "cex-broker-user-data";
|
|
293474
|
+
var STATE_VERSION = 1;
|
|
293475
|
+
var HEARTBEAT_MS = 30000;
|
|
293476
|
+
var FORWARDER_TIMEOUT_MS = 3000;
|
|
293477
|
+
var IDENTIFIER = /^[a-z0-9][a-z0-9:_-]{0,127}$/;
|
|
293478
|
+
function identifier(value, name) {
|
|
293479
|
+
const normalized = value.trim().toLowerCase();
|
|
293480
|
+
if (!IDENTIFIER.test(normalized)) {
|
|
293481
|
+
throw new Error(`${name} must be a lower-case stream-health identifier`);
|
|
293482
|
+
}
|
|
293483
|
+
return normalized;
|
|
293484
|
+
}
|
|
293485
|
+
function counter(value) {
|
|
293486
|
+
if (!/^(0|[1-9]\d*)$/.test(value)) {
|
|
293487
|
+
throw new Error("Invalid persisted stream-health counter");
|
|
293488
|
+
}
|
|
293489
|
+
return BigInt(value);
|
|
293490
|
+
}
|
|
293491
|
+
function next(value) {
|
|
293492
|
+
return (counter(value) + 1n).toString();
|
|
293493
|
+
}
|
|
293494
|
+
function key(snapshot) {
|
|
293495
|
+
return `exchange:${snapshot.exchange}|account:${snapshot.accountSelector}|stream:${snapshot.streamKind}|scope:${snapshot.accountScope}`;
|
|
293496
|
+
}
|
|
293497
|
+
function registryRevision(snapshots) {
|
|
293498
|
+
const rows = snapshots.map((snapshot) => ({
|
|
293499
|
+
exchange: snapshot.exchange,
|
|
293500
|
+
account_selector: snapshot.accountSelector,
|
|
293501
|
+
account_role: snapshot.accountRole ?? null,
|
|
293502
|
+
stream_kind: snapshot.streamKind,
|
|
293503
|
+
account_scope: snapshot.accountScope,
|
|
293504
|
+
registry_status: snapshot.registryStatus
|
|
293505
|
+
})).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
|
|
293506
|
+
return createHash4("sha256").update(JSON.stringify(rows)).digest("hex");
|
|
293507
|
+
}
|
|
293508
|
+
function validState(value) {
|
|
293509
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
293510
|
+
return false;
|
|
293511
|
+
const state = value;
|
|
293512
|
+
return state.version === STATE_VERSION && state.producerId === PRODUCER_ID && typeof state.producerEpoch === "string" && typeof state.runId === "string" && typeof state.nextBatchSequence === "string" && state.nextStreamSequences !== null && typeof state.nextStreamSequences === "object" && (state.pendingBody === undefined || typeof state.pendingBody === "string");
|
|
293513
|
+
}
|
|
293514
|
+
function forwarderPost(url2, body, authToken, timeoutMs) {
|
|
293515
|
+
const request = url2.protocol === "http:" ? httpRequest3 : httpsRequest3;
|
|
293516
|
+
const headers = {
|
|
293517
|
+
"content-type": "application/json",
|
|
293518
|
+
"content-length": Buffer.byteLength(body)
|
|
293519
|
+
};
|
|
293520
|
+
if (authToken)
|
|
293521
|
+
headers.authorization = `Bearer ${authToken}`;
|
|
293522
|
+
return new Promise((resolve, reject) => {
|
|
293523
|
+
const req = request(url2, { method: "POST", headers, timeout: timeoutMs }, (res) => {
|
|
293524
|
+
res.on("data", () => {});
|
|
293525
|
+
res.on("end", () => {
|
|
293526
|
+
const status = res.statusCode ?? 0;
|
|
293527
|
+
if (status < 200 || status >= 300) {
|
|
293528
|
+
reject(new Error(`Stream health forwarder returned ${status}`));
|
|
293529
|
+
return;
|
|
293530
|
+
}
|
|
293531
|
+
resolve();
|
|
293532
|
+
});
|
|
293533
|
+
});
|
|
293534
|
+
req.on("error", reject);
|
|
293535
|
+
req.on("timeout", () => req.destroy(new Error("Stream health forwarder timed out")));
|
|
293536
|
+
req.write(body);
|
|
293537
|
+
req.end();
|
|
293538
|
+
});
|
|
293539
|
+
}
|
|
293540
|
+
|
|
293541
|
+
class StreamHealthPublisher {
|
|
293542
|
+
#deploymentId;
|
|
293543
|
+
#statePath;
|
|
293544
|
+
#heartbeatMs;
|
|
293545
|
+
#post;
|
|
293546
|
+
#state;
|
|
293547
|
+
#advanceRun;
|
|
293548
|
+
#snapshots = [];
|
|
293549
|
+
#dirty = false;
|
|
293550
|
+
#closed = false;
|
|
293551
|
+
#pumping = null;
|
|
293552
|
+
#heartbeat = null;
|
|
293553
|
+
#retry = null;
|
|
293554
|
+
#retryAttempt = 0;
|
|
293555
|
+
constructor(options) {
|
|
293556
|
+
this.#deploymentId = identifier(options.deploymentId, "deployment_id");
|
|
293557
|
+
this.#statePath = options.statePath.trim();
|
|
293558
|
+
if (!this.#statePath) {
|
|
293559
|
+
throw new Error("CEX_BROKER_STREAM_HEALTH_STATE_PATH is required");
|
|
293560
|
+
}
|
|
293561
|
+
this.#heartbeatMs = options.heartbeatIntervalMs ?? HEARTBEAT_MS;
|
|
293562
|
+
if (!Number.isInteger(this.#heartbeatMs) || this.#heartbeatMs < 1 || this.#heartbeatMs > 60000) {
|
|
293563
|
+
throw new Error("Stream health heartbeat interval must be between 1 and 60000ms");
|
|
293564
|
+
}
|
|
293565
|
+
let url2;
|
|
293566
|
+
try {
|
|
293567
|
+
url2 = new URL(options.forwarderUrl.trim());
|
|
293568
|
+
} catch (error) {
|
|
293569
|
+
throw new Error("CEX_BROKER_ARCHIVE_FORWARDER_URL must be valid", {
|
|
293570
|
+
cause: error
|
|
293571
|
+
});
|
|
293572
|
+
}
|
|
293573
|
+
if (url2.protocol !== "http:" && url2.protocol !== "https:") {
|
|
293574
|
+
throw new Error("CEX_BROKER_ARCHIVE_FORWARDER_URL must use HTTP(S)");
|
|
293575
|
+
}
|
|
293576
|
+
this.#post = options.post ?? ((body) => forwarderPost(url2, body, options.forwarderAuthToken, options.forwarderTimeoutMs ?? FORWARDER_TIMEOUT_MS));
|
|
293577
|
+
const loaded = this.#read();
|
|
293578
|
+
this.#state = loaded ?? {
|
|
293579
|
+
version: STATE_VERSION,
|
|
293580
|
+
producerId: PRODUCER_ID,
|
|
293581
|
+
producerEpoch: "1",
|
|
293582
|
+
runId: randomUUID(),
|
|
293583
|
+
nextBatchSequence: "1",
|
|
293584
|
+
nextStreamSequences: {}
|
|
293585
|
+
};
|
|
293586
|
+
counter(this.#state.producerEpoch);
|
|
293587
|
+
counter(this.#state.nextBatchSequence);
|
|
293588
|
+
for (const value of Object.values(this.#state.nextStreamSequences))
|
|
293589
|
+
counter(value);
|
|
293590
|
+
this.#advanceRun = loaded !== null;
|
|
293591
|
+
if (!loaded)
|
|
293592
|
+
this.#persist();
|
|
293593
|
+
}
|
|
293594
|
+
start() {
|
|
293595
|
+
if (this.#closed || this.#heartbeat)
|
|
293596
|
+
return;
|
|
293597
|
+
this.#heartbeat = setInterval(() => {
|
|
293598
|
+
if (this.#snapshots.length > 0) {
|
|
293599
|
+
this.#dirty = true;
|
|
293600
|
+
this.#schedule();
|
|
293601
|
+
}
|
|
293602
|
+
}, this.#heartbeatMs);
|
|
293603
|
+
this.#heartbeat.unref?.();
|
|
293604
|
+
if (this.#state.pendingBody || this.#dirty)
|
|
293605
|
+
this.#schedule();
|
|
293606
|
+
}
|
|
293607
|
+
publish(snapshots) {
|
|
293608
|
+
if (this.#closed)
|
|
293609
|
+
return;
|
|
293610
|
+
this.#snapshots = snapshots.map((snapshot) => ({ ...snapshot }));
|
|
293611
|
+
this.#dirty = true;
|
|
293612
|
+
this.#schedule();
|
|
293613
|
+
}
|
|
293614
|
+
async close(snapshots, timeoutMs = FORWARDER_TIMEOUT_MS) {
|
|
293615
|
+
if (this.#closed)
|
|
293616
|
+
return;
|
|
293617
|
+
if (this.#heartbeat)
|
|
293618
|
+
clearInterval(this.#heartbeat);
|
|
293619
|
+
this.#heartbeat = null;
|
|
293620
|
+
if (this.#retry)
|
|
293621
|
+
clearTimeout(this.#retry);
|
|
293622
|
+
this.#retry = null;
|
|
293623
|
+
this.#snapshots = snapshots.map((snapshot) => ({ ...snapshot }));
|
|
293624
|
+
this.#dirty = this.#snapshots.length > 0;
|
|
293625
|
+
this.#schedule();
|
|
293626
|
+
await Promise.race([
|
|
293627
|
+
this.#waitForIdle(),
|
|
293628
|
+
new Promise((resolve) => setTimeout(resolve, timeoutMs))
|
|
293629
|
+
]);
|
|
293630
|
+
this.#closed = true;
|
|
293631
|
+
if (this.#retry)
|
|
293632
|
+
clearTimeout(this.#retry);
|
|
293633
|
+
this.#retry = null;
|
|
293634
|
+
}
|
|
293635
|
+
#schedule() {
|
|
293636
|
+
if (this.#closed || this.#pumping)
|
|
293637
|
+
return;
|
|
293638
|
+
this.#pumping = this.#pump().finally(() => {
|
|
293639
|
+
this.#pumping = null;
|
|
293640
|
+
if (!this.#closed && !this.#retry && (this.#state.pendingBody || this.#dirty))
|
|
293641
|
+
this.#schedule();
|
|
293642
|
+
});
|
|
293643
|
+
}
|
|
293644
|
+
async#pump() {
|
|
293645
|
+
if (this.#state.pendingBody && !await this.#deliver())
|
|
293646
|
+
return;
|
|
293647
|
+
if (!this.#dirty || this.#snapshots.length === 0)
|
|
293648
|
+
return;
|
|
293649
|
+
if (this.#advanceRun) {
|
|
293650
|
+
this.#state.producerEpoch = next(this.#state.producerEpoch);
|
|
293651
|
+
this.#state.runId = randomUUID();
|
|
293652
|
+
this.#state.nextBatchSequence = "1";
|
|
293653
|
+
this.#state.nextStreamSequences = {};
|
|
293654
|
+
this.#advanceRun = false;
|
|
293655
|
+
this.#persist();
|
|
293656
|
+
}
|
|
293657
|
+
this.#dirty = false;
|
|
293658
|
+
this.#state.pendingBody = this.#body(this.#snapshots);
|
|
293659
|
+
this.#persist();
|
|
293660
|
+
await this.#deliver();
|
|
293661
|
+
}
|
|
293662
|
+
async#deliver() {
|
|
293663
|
+
const body = this.#state.pendingBody;
|
|
293664
|
+
if (!body)
|
|
293665
|
+
return true;
|
|
293666
|
+
try {
|
|
293667
|
+
await this.#post(body);
|
|
293668
|
+
this.#state.pendingBody = undefined;
|
|
293669
|
+
this.#persist();
|
|
293670
|
+
this.#retryAttempt = 0;
|
|
293671
|
+
return true;
|
|
293672
|
+
} catch {
|
|
293673
|
+
this.#retryLater();
|
|
293674
|
+
return false;
|
|
293675
|
+
}
|
|
293676
|
+
}
|
|
293677
|
+
#body(snapshots) {
|
|
293678
|
+
const ordered3 = [...snapshots].sort((left, right) => key(left).localeCompare(key(right)));
|
|
293679
|
+
if (ordered3.length === 0 || ordered3.length > 1000) {
|
|
293680
|
+
throw new Error("Stream health requires between one and 1000 registry rows");
|
|
293681
|
+
}
|
|
293682
|
+
const batchSequence = this.#state.nextBatchSequence;
|
|
293683
|
+
this.#state.nextBatchSequence = next(batchSequence);
|
|
293684
|
+
const heartbeatAt = new Date().toISOString();
|
|
293685
|
+
const active = ordered3.filter((snapshot) => snapshot.registryStatus === "active").length;
|
|
293686
|
+
const rows = ordered3.map((snapshot) => {
|
|
293687
|
+
const streamKey = key(snapshot);
|
|
293688
|
+
const sequence = this.#state.nextStreamSequences[streamKey] ?? "1";
|
|
293689
|
+
this.#state.nextStreamSequences[streamKey] = next(sequence);
|
|
293690
|
+
return {
|
|
293691
|
+
table: TABLE,
|
|
293692
|
+
row: {
|
|
293693
|
+
producer_id: PRODUCER_ID,
|
|
293694
|
+
producer_epoch: this.#state.producerEpoch,
|
|
293695
|
+
run_id: this.#state.runId,
|
|
293696
|
+
batch_sequence: batchSequence,
|
|
293697
|
+
batch_snapshot_count: String(ordered3.length),
|
|
293698
|
+
batch_active_stream_count: String(active),
|
|
293699
|
+
registry_revision: registryRevision(ordered3),
|
|
293700
|
+
registry_status: snapshot.registryStatus,
|
|
293701
|
+
retired_at: snapshot.retiredAt,
|
|
293702
|
+
exchange: snapshot.exchange,
|
|
293703
|
+
account_selector: snapshot.accountSelector,
|
|
293704
|
+
account_role: snapshot.accountRole ?? null,
|
|
293705
|
+
stream_kind: snapshot.streamKind,
|
|
293706
|
+
account_scope: snapshot.accountScope,
|
|
293707
|
+
sequence,
|
|
293708
|
+
state: snapshot.state,
|
|
293709
|
+
state_changed_at: snapshot.stateChangedAt,
|
|
293710
|
+
last_connected_at: snapshot.lastConnectedAt,
|
|
293711
|
+
last_authenticated_at: snapshot.lastAuthenticatedAt,
|
|
293712
|
+
last_received_at: snapshot.lastReceivedAt,
|
|
293713
|
+
heartbeat_at: heartbeatAt,
|
|
293714
|
+
connect_attempt_count: snapshot.connectAttemptCount,
|
|
293715
|
+
reconnect_count: snapshot.reconnectCount,
|
|
293716
|
+
error_count: snapshot.errorCount,
|
|
293717
|
+
last_failure_kind: snapshot.lastFailureKind,
|
|
293718
|
+
last_failure_reason: snapshot.lastFailureReason,
|
|
293719
|
+
traffic_mode: snapshot.trafficMode,
|
|
293720
|
+
source_watermark: snapshot.sourceWatermark
|
|
293721
|
+
}
|
|
293722
|
+
};
|
|
293723
|
+
});
|
|
293724
|
+
return JSON.stringify({
|
|
293725
|
+
source: SOURCE,
|
|
293726
|
+
deployment_id: this.#deploymentId,
|
|
293727
|
+
rows
|
|
293728
|
+
});
|
|
293729
|
+
}
|
|
293730
|
+
#retryLater() {
|
|
293731
|
+
if (this.#closed || this.#retry)
|
|
293732
|
+
return;
|
|
293733
|
+
const delay = Math.min(1000 * 2 ** this.#retryAttempt, 30000);
|
|
293734
|
+
this.#retryAttempt += 1;
|
|
293735
|
+
this.#retry = setTimeout(() => {
|
|
293736
|
+
this.#retry = null;
|
|
293737
|
+
this.#schedule();
|
|
293738
|
+
}, delay);
|
|
293739
|
+
this.#retry.unref?.();
|
|
293740
|
+
}
|
|
293741
|
+
async#waitForIdle() {
|
|
293742
|
+
while (this.#pumping)
|
|
293743
|
+
await this.#pumping;
|
|
293744
|
+
}
|
|
293745
|
+
#read() {
|
|
293746
|
+
try {
|
|
293747
|
+
const parsed = JSON.parse(readFileSync(this.#statePath, "utf8"));
|
|
293748
|
+
if (!validState(parsed))
|
|
293749
|
+
throw new Error("invalid state shape");
|
|
293750
|
+
return parsed;
|
|
293751
|
+
} catch (error) {
|
|
293752
|
+
if (error.code === "ENOENT")
|
|
293753
|
+
return null;
|
|
293754
|
+
throw new Error("Stream health state cannot be read", { cause: error });
|
|
293755
|
+
}
|
|
293756
|
+
}
|
|
293757
|
+
#persist() {
|
|
293758
|
+
const parent = dirname(this.#statePath);
|
|
293759
|
+
try {
|
|
293760
|
+
if (!statSync(parent).isDirectory())
|
|
293761
|
+
throw new Error("state parent is not a directory");
|
|
293762
|
+
} catch (error) {
|
|
293763
|
+
throw new Error("Stream health state directory is unavailable", {
|
|
293764
|
+
cause: error
|
|
293765
|
+
});
|
|
293766
|
+
}
|
|
293767
|
+
const temporary = `${this.#statePath}.${process.pid}.${randomUUID()}.tmp`;
|
|
293768
|
+
let fd2;
|
|
293769
|
+
try {
|
|
293770
|
+
fd2 = openSync2(temporary, "wx", 384);
|
|
293771
|
+
writeFileSync(fd2, JSON.stringify(this.#state));
|
|
293772
|
+
fsyncSync2(fd2);
|
|
293773
|
+
closeSync2(fd2);
|
|
293774
|
+
fd2 = undefined;
|
|
293775
|
+
renameSync(temporary, this.#statePath);
|
|
293776
|
+
const parentFd = openSync2(parent, "r");
|
|
293777
|
+
try {
|
|
293778
|
+
fsyncSync2(parentFd);
|
|
293779
|
+
} finally {
|
|
293780
|
+
closeSync2(parentFd);
|
|
293781
|
+
}
|
|
293782
|
+
} catch (error) {
|
|
293783
|
+
if (fd2 !== undefined)
|
|
293784
|
+
closeSync2(fd2);
|
|
293785
|
+
try {
|
|
293786
|
+
unlinkSync(temporary);
|
|
293787
|
+
} catch {}
|
|
293788
|
+
throw new Error("Stream health state cannot be persisted", {
|
|
293789
|
+
cause: error
|
|
293790
|
+
});
|
|
293791
|
+
}
|
|
293792
|
+
}
|
|
293793
|
+
}
|
|
293794
|
+
function streamHealthPublisherConfigFromEnv(env = process.env) {
|
|
293795
|
+
if (env.CEX_BROKER_ARCHIVE_ENABLED !== "true") {
|
|
293796
|
+
throw new Error("Configured account user streams require CEX_BROKER_ARCHIVE_ENABLED=true");
|
|
293797
|
+
}
|
|
293798
|
+
const deploymentId = env.CEX_BROKER_DEPLOYMENT_ID?.trim();
|
|
293799
|
+
const forwarderUrl = env.CEX_BROKER_ARCHIVE_FORWARDER_URL?.trim();
|
|
293800
|
+
const statePath = env.CEX_BROKER_STREAM_HEALTH_STATE_PATH?.trim();
|
|
293801
|
+
if (!deploymentId || !forwarderUrl || !statePath) {
|
|
293802
|
+
throw new Error("Configured account user streams require deployment, forwarder, and persistent state configuration");
|
|
293803
|
+
}
|
|
293804
|
+
return {
|
|
293805
|
+
deploymentId,
|
|
293806
|
+
forwarderUrl,
|
|
293807
|
+
statePath,
|
|
293808
|
+
forwarderAuthToken: env.CEX_BROKER_ARCHIVE_FORWARDER_TOKEN?.trim() || undefined
|
|
293809
|
+
};
|
|
293810
|
+
}
|
|
293811
|
+
|
|
293812
|
+
// src/helpers/binance-user-data-stream.ts
|
|
293813
|
+
import { Buffer as Buffer2 } from "node:buffer";
|
|
293814
|
+
import { createHmac } from "node:crypto";
|
|
293815
|
+
var BINANCE_SPOT_WS_API_URL = "wss://ws-api.binance.com:443/ws-api/v3";
|
|
293816
|
+
var DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS = 16;
|
|
293817
|
+
var createWebSocket = (url2) => new wrapper_default(url2);
|
|
293818
|
+
var userDataRequestCounter = 0;
|
|
293819
|
+
function getExchangeString(exchange, key2) {
|
|
293820
|
+
const value = exchange[key2];
|
|
293821
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
293822
|
+
throw new Error(`Binance user-data stream requires exchange.${key2}`);
|
|
293823
|
+
}
|
|
293824
|
+
return value;
|
|
293825
|
+
}
|
|
293826
|
+
function sortedQuery(params) {
|
|
293827
|
+
return Object.entries(params).sort(([left], [right]) => left.localeCompare(right)).map(([key2, value]) => `${encodeURIComponent(key2)}=${encodeURIComponent(String(value))}`).join("&");
|
|
293828
|
+
}
|
|
293829
|
+
function signUserDataStreamParams(exchange, params) {
|
|
293830
|
+
const signParams = exchange.signParams;
|
|
293831
|
+
if (typeof signParams === "function") {
|
|
293832
|
+
return signParams.call(exchange, params);
|
|
293833
|
+
}
|
|
293834
|
+
const secret = getExchangeString(exchange, "secret");
|
|
293835
|
+
return {
|
|
293836
|
+
...params,
|
|
293837
|
+
signature: createHmac("sha256", secret).update(sortedQuery(params)).digest("hex")
|
|
293838
|
+
};
|
|
293839
|
+
}
|
|
293840
|
+
function getBinanceSpotWsApiUrl(exchange) {
|
|
293841
|
+
const urls = exchange.urls;
|
|
293842
|
+
return urls?.api?.ws?.["ws-api"]?.spot ?? BINANCE_SPOT_WS_API_URL;
|
|
293843
|
+
}
|
|
293844
|
+
function getRecord(value) {
|
|
293845
|
+
return typeof value === "object" && value !== null ? value : null;
|
|
293846
|
+
}
|
|
293847
|
+
function getMessage(value) {
|
|
293848
|
+
if (value instanceof Error) {
|
|
293849
|
+
return value.message;
|
|
293850
|
+
}
|
|
293851
|
+
if (typeof value === "string" && value.length > 0) {
|
|
293852
|
+
return value;
|
|
293853
|
+
}
|
|
293854
|
+
const record = getRecord(value);
|
|
293855
|
+
const message = record?.message;
|
|
293856
|
+
return typeof message === "string" && message.length > 0 ? message : null;
|
|
293857
|
+
}
|
|
293858
|
+
function getOptionalExchangeString(exchange, key2) {
|
|
293859
|
+
const value = exchange[key2];
|
|
293860
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
293861
|
+
}
|
|
293862
|
+
function redactDiagnosticMessage(message, secretValues) {
|
|
293863
|
+
let redacted = message;
|
|
293864
|
+
for (const value of secretValues) {
|
|
293865
|
+
if (value.length > 0) {
|
|
293866
|
+
redacted = redacted.split(value).join("[redacted]");
|
|
293867
|
+
}
|
|
293868
|
+
}
|
|
293869
|
+
return redacted.replace(/(\b(?:apiKey|secret|signature)\b\s*=\s*)[^\s&,;)]+/gi, "$1[redacted]").replace(/("(?:apiKey|secret|signature)"\s*:\s*")[^"]*(")/gi, "$1[redacted]$2");
|
|
293870
|
+
}
|
|
293871
|
+
function formatBinanceUserDataWebSocketError(event, secretValues) {
|
|
293872
|
+
const record = getRecord(event);
|
|
293873
|
+
const message = getMessage(record?.error) ?? getMessage(record?.message) ?? getMessage(event);
|
|
293874
|
+
const safeMessage = message === null ? null : redactDiagnosticMessage(message, secretValues);
|
|
293875
|
+
return new Error(safeMessage ? `Binance user-data WebSocket error: ${safeMessage}` : "Binance user-data WebSocket error");
|
|
293876
|
+
}
|
|
293877
|
+
function getCloseReason(value) {
|
|
293878
|
+
if (typeof value === "string") {
|
|
293879
|
+
return value.length > 0 ? value : null;
|
|
293880
|
+
}
|
|
293881
|
+
if (Buffer2.isBuffer(value)) {
|
|
293882
|
+
const reason = value.toString("utf8");
|
|
293883
|
+
return reason.length > 0 ? reason : null;
|
|
293884
|
+
}
|
|
293885
|
+
if (value instanceof Uint8Array) {
|
|
293886
|
+
const reason = Buffer2.from(value).toString("utf8");
|
|
293887
|
+
return reason.length > 0 ? reason : null;
|
|
293888
|
+
}
|
|
293889
|
+
return null;
|
|
293890
|
+
}
|
|
293891
|
+
function formatBinanceUserDataWebSocketClose(codeOrEvent, reasonOrUndefined, secretValues) {
|
|
293892
|
+
const record = getRecord(codeOrEvent);
|
|
293893
|
+
const code = record ? record.code : codeOrEvent;
|
|
293894
|
+
const reason = getCloseReason(record ? record.reason : reasonOrUndefined);
|
|
293895
|
+
const safeReason = reason === null ? null : redactDiagnosticMessage(reason, secretValues);
|
|
293896
|
+
const details = [
|
|
293897
|
+
typeof code === "number" || typeof code === "string" ? `code=${code}` : null,
|
|
293898
|
+
safeReason ? `reason=${safeReason}` : null
|
|
293899
|
+
].filter((detail) => detail !== null);
|
|
293900
|
+
return new Error(details.length > 0 ? `Binance user-data WebSocket closed unexpectedly (${details.join(", ")})` : "Binance user-data WebSocket closed unexpectedly");
|
|
293901
|
+
}
|
|
293902
|
+
function decodeMessageData(data) {
|
|
293903
|
+
if (typeof data === "string") {
|
|
293904
|
+
return data;
|
|
293905
|
+
}
|
|
293906
|
+
if (Buffer2.isBuffer(data)) {
|
|
293907
|
+
return data.toString("utf8");
|
|
293908
|
+
}
|
|
293909
|
+
if (data instanceof ArrayBuffer) {
|
|
293910
|
+
return Buffer2.from(data).toString("utf8");
|
|
293911
|
+
}
|
|
293912
|
+
if (ArrayBuffer.isView(data)) {
|
|
293913
|
+
return Buffer2.from(data.buffer, data.byteOffset, data.byteLength).toString("utf8");
|
|
293914
|
+
}
|
|
293915
|
+
if (Array.isArray(data) && data.every((item) => Buffer2.isBuffer(item))) {
|
|
293916
|
+
return Buffer2.concat(data).toString("utf8");
|
|
293917
|
+
}
|
|
293918
|
+
return data;
|
|
293919
|
+
}
|
|
293920
|
+
|
|
293921
|
+
class BinanceSpotUserDataStream {
|
|
293922
|
+
exchange;
|
|
293923
|
+
ws;
|
|
293924
|
+
secretValues;
|
|
293925
|
+
requestId = `user-data-${Date.now()}-${userDataRequestCounter++}`;
|
|
293926
|
+
maxBufferedEvents;
|
|
293927
|
+
observer;
|
|
293928
|
+
queue = [];
|
|
293929
|
+
waiters = [];
|
|
293930
|
+
closed = false;
|
|
293931
|
+
closeError = null;
|
|
293932
|
+
subscriptionId = null;
|
|
293933
|
+
constructor(exchange, options = {}) {
|
|
293934
|
+
this.exchange = exchange;
|
|
293935
|
+
this.maxBufferedEvents = options.maxBufferedEvents ?? DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS;
|
|
293936
|
+
this.observer = options.observer;
|
|
293937
|
+
this.secretValues = [
|
|
293938
|
+
getOptionalExchangeString(exchange, "apiKey"),
|
|
293939
|
+
getOptionalExchangeString(exchange, "secret")
|
|
293940
|
+
].filter((value) => value !== null);
|
|
293941
|
+
this.ws = createWebSocket(getBinanceSpotWsApiUrl(exchange));
|
|
293942
|
+
this.ws.on("open", () => {
|
|
293943
|
+
this.observer?.onConnected?.();
|
|
293944
|
+
this.subscribe();
|
|
293945
|
+
});
|
|
293946
|
+
this.ws.on("message", (data) => this.handleMessage(data));
|
|
293947
|
+
this.ws.on("error", (error) => this.fail(formatBinanceUserDataWebSocketError(error, this.secretValues), "transport_error"));
|
|
293948
|
+
this.ws.on("close", (code, reason) => this.handleClose(code, reason));
|
|
293949
|
+
}
|
|
293950
|
+
async* [Symbol.asyncIterator]() {
|
|
293951
|
+
while (true) {
|
|
293952
|
+
const event = await this.nextEvent();
|
|
293953
|
+
if (!event) {
|
|
293954
|
+
break;
|
|
293955
|
+
}
|
|
293956
|
+
yield event;
|
|
293957
|
+
}
|
|
293958
|
+
}
|
|
293959
|
+
close() {
|
|
293960
|
+
if (this.closed) {
|
|
293961
|
+
return;
|
|
293962
|
+
}
|
|
293963
|
+
this.closed = true;
|
|
293964
|
+
this.queue.length = 0;
|
|
293965
|
+
try {
|
|
293966
|
+
this.ws.close();
|
|
293967
|
+
} catch {}
|
|
293968
|
+
this.flushWaiters();
|
|
293969
|
+
}
|
|
293970
|
+
handleClose(code, reason) {
|
|
293971
|
+
if (this.closed) {
|
|
293972
|
+
return;
|
|
293973
|
+
}
|
|
293974
|
+
this.fail(formatBinanceUserDataWebSocketClose(code, reason, this.secretValues), "remote_closed");
|
|
293975
|
+
}
|
|
293976
|
+
subscribe() {
|
|
293977
|
+
const apiKey = getExchangeString(this.exchange, "apiKey");
|
|
293978
|
+
const signedParams = signUserDataStreamParams(this.exchange, {
|
|
293979
|
+
apiKey,
|
|
293980
|
+
timestamp: Date.now()
|
|
293981
|
+
});
|
|
293982
|
+
this.ws.send(JSON.stringify({
|
|
293983
|
+
id: this.requestId,
|
|
293984
|
+
method: "userDataStream.subscribe.signature",
|
|
293985
|
+
params: signedParams
|
|
293986
|
+
}));
|
|
293987
|
+
}
|
|
293988
|
+
handleMessage(data) {
|
|
293989
|
+
if (this.closed) {
|
|
293990
|
+
return;
|
|
293991
|
+
}
|
|
293992
|
+
let message;
|
|
293993
|
+
try {
|
|
293994
|
+
const decodedData = decodeMessageData(data);
|
|
293995
|
+
message = typeof decodedData === "string" ? JSON.parse(decodedData) : decodedData;
|
|
293996
|
+
} catch (error) {
|
|
293997
|
+
this.fail(error instanceof Error ? error : new Error("Invalid Binance user-data message"), "protocol_error");
|
|
293998
|
+
return;
|
|
293999
|
+
}
|
|
294000
|
+
if ("id" in message && message.id === this.requestId) {
|
|
294001
|
+
if (message.status !== 200) {
|
|
294002
|
+
this.fail(new Error(message.error?.msg ?? message.error?.message ?? `Binance user-data subscription failed with status ${message.status}`), "auth_failed");
|
|
294003
|
+
return;
|
|
294004
|
+
}
|
|
294005
|
+
this.subscriptionId = message.result?.subscriptionId ?? null;
|
|
294006
|
+
this.observer?.onAuthenticated?.();
|
|
294007
|
+
return;
|
|
294008
|
+
}
|
|
294009
|
+
if ("status" in message && typeof message.status === "number" && message.status !== 200) {
|
|
294010
|
+
const errorMessage = message.error?.msg ?? message.error?.message ?? `Binance user-data request failed with status ${message.status}`;
|
|
294011
|
+
const errorCode2 = message.error?.code;
|
|
294012
|
+
this.fail(new Error(typeof errorCode2 === "number" ? `${errorMessage} (code ${errorCode2})` : errorMessage), "protocol_error");
|
|
294013
|
+
return;
|
|
294014
|
+
}
|
|
294015
|
+
if (!("event" in message) || !message.event) {
|
|
294016
|
+
return;
|
|
294017
|
+
}
|
|
294018
|
+
const subscriptionId = message.subscriptionId ?? this.subscriptionId;
|
|
294019
|
+
if (subscriptionId === null || subscriptionId === undefined) {
|
|
294020
|
+
return;
|
|
294021
|
+
}
|
|
294022
|
+
this.push({ subscriptionId, event: message.event });
|
|
294023
|
+
}
|
|
294024
|
+
push(event) {
|
|
294025
|
+
if (this.closed) {
|
|
294026
|
+
return;
|
|
294027
|
+
}
|
|
294028
|
+
this.observer?.onEvent?.(event);
|
|
294029
|
+
const waiter = this.waiters.shift();
|
|
294030
|
+
if (waiter) {
|
|
294031
|
+
waiter.resolve(event);
|
|
294032
|
+
return;
|
|
294033
|
+
}
|
|
294034
|
+
if (this.queue.length >= this.maxBufferedEvents) {
|
|
294035
|
+
this.fail(new Error(`Binance user-data stream buffered event limit exceeded (${this.maxBufferedEvents}); downstream consumer is not keeping up`), "backpressure");
|
|
294036
|
+
return;
|
|
294037
|
+
}
|
|
294038
|
+
this.queue.push(event);
|
|
294039
|
+
}
|
|
294040
|
+
nextEvent() {
|
|
294041
|
+
const event = this.queue.shift();
|
|
294042
|
+
if (event) {
|
|
294043
|
+
return Promise.resolve(event);
|
|
294044
|
+
}
|
|
294045
|
+
if (this.closeError) {
|
|
294046
|
+
return Promise.reject(this.closeError);
|
|
294047
|
+
}
|
|
294048
|
+
if (this.closed) {
|
|
294049
|
+
return Promise.resolve(null);
|
|
294050
|
+
}
|
|
294051
|
+
return new Promise((resolve, reject) => {
|
|
294052
|
+
this.waiters.push({ resolve, reject });
|
|
294053
|
+
});
|
|
294054
|
+
}
|
|
294055
|
+
fail(error, kind) {
|
|
294056
|
+
if (this.closeError) {
|
|
294057
|
+
return;
|
|
294058
|
+
}
|
|
294059
|
+
this.closeError = error;
|
|
294060
|
+
this.observer?.onFailure?.({ kind, reason: error.message });
|
|
294061
|
+
this.closed = true;
|
|
294062
|
+
this.queue.length = 0;
|
|
294063
|
+
this.flushWaiters();
|
|
294064
|
+
try {
|
|
294065
|
+
this.ws.close();
|
|
294066
|
+
} catch {}
|
|
294067
|
+
}
|
|
294068
|
+
flushWaiters() {
|
|
294069
|
+
const error = this.closeError;
|
|
294070
|
+
for (const waiter of this.waiters.splice(0)) {
|
|
294071
|
+
if (error) {
|
|
294072
|
+
waiter.reject(error);
|
|
294073
|
+
} else {
|
|
294074
|
+
waiter.resolve(null);
|
|
294075
|
+
}
|
|
294076
|
+
}
|
|
294077
|
+
}
|
|
294078
|
+
}
|
|
294079
|
+
function isBinanceBalanceUserDataEvent(event) {
|
|
294080
|
+
return event.e === "outboundAccountPosition" || event.e === "balanceUpdate" || event.e === "externalLockUpdate";
|
|
294081
|
+
}
|
|
294082
|
+
function isBinanceOrderUserDataEvent(event) {
|
|
294083
|
+
return event.e === "executionReport" || event.e === "listStatus";
|
|
294084
|
+
}
|
|
294085
|
+
|
|
294086
|
+
// src/helpers/user-data-stream-supervisor.ts
|
|
294087
|
+
var MAX_SUBSCRIBER_EVENTS = 16;
|
|
294088
|
+
function now3() {
|
|
294089
|
+
return new Date().toISOString();
|
|
294090
|
+
}
|
|
294091
|
+
function retryDelay(attempt) {
|
|
294092
|
+
return Math.min(1000 * 2 ** attempt, 30000);
|
|
294093
|
+
}
|
|
294094
|
+
function safeFailureReason(exchange, reason) {
|
|
294095
|
+
const secrets = [exchange.apiKey, exchange.secret].filter((value) => typeof value === "string" && value.length > 0);
|
|
294096
|
+
return redactSecretLiterals(reason, secrets).replace(/\s+/g, " ").trim().slice(0, 256);
|
|
294097
|
+
}
|
|
294098
|
+
|
|
294099
|
+
class Subscriber {
|
|
294100
|
+
kind;
|
|
294101
|
+
marketId;
|
|
294102
|
+
onClose;
|
|
294103
|
+
#queue = [];
|
|
294104
|
+
#waiters = [];
|
|
294105
|
+
#closed = false;
|
|
294106
|
+
#error = null;
|
|
294107
|
+
constructor(kind, marketId, onClose) {
|
|
294108
|
+
this.kind = kind;
|
|
294109
|
+
this.marketId = marketId;
|
|
294110
|
+
this.onClose = onClose;
|
|
294111
|
+
}
|
|
294112
|
+
push(message) {
|
|
294113
|
+
if (this.#closed || !this.#matches(message.event))
|
|
294114
|
+
return;
|
|
294115
|
+
const waiter = this.#waiters.shift();
|
|
294116
|
+
if (waiter) {
|
|
294117
|
+
waiter.resolve(message);
|
|
294118
|
+
return;
|
|
294119
|
+
}
|
|
294120
|
+
if (this.#queue.length >= MAX_SUBSCRIBER_EVENTS) {
|
|
294121
|
+
this.#fail(new Error("Configured account user-data subscriber fell behind"));
|
|
294122
|
+
return;
|
|
294123
|
+
}
|
|
294124
|
+
this.#queue.push(message);
|
|
294125
|
+
}
|
|
294126
|
+
close() {
|
|
294127
|
+
if (this.#closed)
|
|
294128
|
+
return;
|
|
294129
|
+
this.#closed = true;
|
|
294130
|
+
this.#queue.length = 0;
|
|
294131
|
+
this.onClose();
|
|
294132
|
+
for (const waiter of this.#waiters.splice(0))
|
|
294133
|
+
waiter.resolve(null);
|
|
294134
|
+
}
|
|
294135
|
+
async* [Symbol.asyncIterator]() {
|
|
294136
|
+
while (true) {
|
|
294137
|
+
const event = await this.#next();
|
|
294138
|
+
if (!event)
|
|
294139
|
+
return;
|
|
294140
|
+
yield event;
|
|
294141
|
+
}
|
|
294142
|
+
}
|
|
294143
|
+
#matches(event) {
|
|
294144
|
+
if (this.kind === "balance")
|
|
294145
|
+
return isBinanceBalanceUserDataEvent(event);
|
|
294146
|
+
if (!isBinanceOrderUserDataEvent(event))
|
|
294147
|
+
return false;
|
|
294148
|
+
return !this.marketId || event.s === this.marketId;
|
|
294149
|
+
}
|
|
294150
|
+
#next() {
|
|
294151
|
+
const event = this.#queue.shift();
|
|
294152
|
+
if (event)
|
|
294153
|
+
return Promise.resolve(event);
|
|
294154
|
+
if (this.#error)
|
|
294155
|
+
return Promise.reject(this.#error);
|
|
294156
|
+
if (this.#closed)
|
|
294157
|
+
return Promise.resolve(null);
|
|
294158
|
+
return new Promise((resolve, reject) => this.#waiters.push({ resolve, reject }));
|
|
294159
|
+
}
|
|
294160
|
+
#fail(error) {
|
|
294161
|
+
if (this.#closed)
|
|
294162
|
+
return;
|
|
294163
|
+
this.#closed = true;
|
|
294164
|
+
this.#error = error;
|
|
294165
|
+
this.#queue.length = 0;
|
|
294166
|
+
this.onClose();
|
|
294167
|
+
for (const waiter of this.#waiters.splice(0))
|
|
294168
|
+
waiter.reject(error);
|
|
294169
|
+
}
|
|
294170
|
+
}
|
|
294171
|
+
|
|
294172
|
+
class AccountWorker {
|
|
294173
|
+
exchangeName;
|
|
294174
|
+
account;
|
|
294175
|
+
onChange;
|
|
294176
|
+
#subscribers = new Set;
|
|
294177
|
+
#snapshot;
|
|
294178
|
+
#stopping = false;
|
|
294179
|
+
#stream = null;
|
|
294180
|
+
#retryTimer = null;
|
|
294181
|
+
#retryResolve = null;
|
|
294182
|
+
#run = null;
|
|
294183
|
+
#failureObserved = false;
|
|
294184
|
+
#attempts = 0;
|
|
294185
|
+
constructor(exchangeName, account, onChange) {
|
|
294186
|
+
this.exchangeName = exchangeName;
|
|
294187
|
+
this.account = account;
|
|
294188
|
+
this.onChange = onChange;
|
|
294189
|
+
const timestamp = now3();
|
|
294190
|
+
this.#snapshot = {
|
|
294191
|
+
exchange: exchangeName,
|
|
294192
|
+
accountSelector: account.label,
|
|
294193
|
+
accountRole: account.role,
|
|
294194
|
+
streamKind: "user_data",
|
|
294195
|
+
accountScope: "spot",
|
|
294196
|
+
registryStatus: "active",
|
|
294197
|
+
retiredAt: null,
|
|
294198
|
+
state: "connecting",
|
|
294199
|
+
stateChangedAt: timestamp,
|
|
294200
|
+
lastConnectedAt: null,
|
|
294201
|
+
lastAuthenticatedAt: null,
|
|
294202
|
+
lastReceivedAt: null,
|
|
294203
|
+
connectAttemptCount: "0",
|
|
294204
|
+
reconnectCount: "0",
|
|
294205
|
+
errorCount: "0",
|
|
294206
|
+
lastFailureKind: "none",
|
|
294207
|
+
lastFailureReason: "",
|
|
294208
|
+
trafficMode: "event_driven",
|
|
294209
|
+
sourceWatermark: null
|
|
294210
|
+
};
|
|
294211
|
+
}
|
|
294212
|
+
start() {
|
|
294213
|
+
if (this.exchangeName !== "binance") {
|
|
294214
|
+
this.#fail("unsupported_connector", "Configured exchange has no user-data supervisor");
|
|
294215
|
+
return;
|
|
294216
|
+
}
|
|
294217
|
+
this.#run = this.#connectLoop();
|
|
294218
|
+
}
|
|
294219
|
+
subscribe(options) {
|
|
294220
|
+
if (this.#stopping)
|
|
294221
|
+
throw new Error("Configured account user-data supervisor is stopping");
|
|
294222
|
+
const subscriber = new Subscriber(options.kind, options.marketId, () => {
|
|
294223
|
+
this.#subscribers.delete(subscriber);
|
|
294224
|
+
});
|
|
294225
|
+
this.#subscribers.add(subscriber);
|
|
294226
|
+
return subscriber;
|
|
294227
|
+
}
|
|
294228
|
+
snapshot() {
|
|
294229
|
+
return { ...this.#snapshot };
|
|
294230
|
+
}
|
|
294231
|
+
async stop() {
|
|
294232
|
+
this.#stopping = true;
|
|
294233
|
+
if (this.#retryTimer)
|
|
294234
|
+
clearTimeout(this.#retryTimer);
|
|
294235
|
+
this.#retryTimer = null;
|
|
294236
|
+
this.#retryResolve?.();
|
|
294237
|
+
this.#retryResolve = null;
|
|
294238
|
+
this.#stream?.close();
|
|
294239
|
+
await this.#run;
|
|
294240
|
+
this.#transition("disconnected", "shutdown", "Broker shutdown");
|
|
294241
|
+
for (const subscriber of [...this.#subscribers])
|
|
294242
|
+
subscriber.close();
|
|
294243
|
+
}
|
|
294244
|
+
#transition(state, failureKind, failureReason) {
|
|
294245
|
+
const timestamp = now3();
|
|
294246
|
+
if (this.#snapshot.state !== state) {
|
|
294247
|
+
this.#snapshot.state = state;
|
|
294248
|
+
this.#snapshot.stateChangedAt = timestamp;
|
|
294249
|
+
}
|
|
294250
|
+
if (failureKind) {
|
|
294251
|
+
this.#snapshot.lastFailureKind = failureKind;
|
|
294252
|
+
this.#snapshot.lastFailureReason = failureReason ?? "";
|
|
294253
|
+
}
|
|
294254
|
+
this.onChange();
|
|
294255
|
+
}
|
|
294256
|
+
#connected() {
|
|
294257
|
+
this.#snapshot.lastConnectedAt = now3();
|
|
294258
|
+
this.#transition("connected");
|
|
294259
|
+
}
|
|
294260
|
+
#authenticated() {
|
|
294261
|
+
this.#snapshot.lastAuthenticatedAt = now3();
|
|
294262
|
+
this.onChange();
|
|
294263
|
+
}
|
|
294264
|
+
#received(message) {
|
|
294265
|
+
this.#snapshot.lastReceivedAt = now3();
|
|
294266
|
+
const eventTimestamp = message.event.E;
|
|
294267
|
+
this.#snapshot.sourceWatermark = typeof eventTimestamp === "number" || typeof eventTimestamp === "string" ? String(eventTimestamp).slice(0, 512) : null;
|
|
294268
|
+
for (const subscriber of this.#subscribers)
|
|
294269
|
+
subscriber.push(message);
|
|
294270
|
+
this.onChange();
|
|
294271
|
+
}
|
|
294272
|
+
#fail(kind, reason) {
|
|
294273
|
+
this.#failureObserved = true;
|
|
294274
|
+
this.#snapshot.errorCount = (BigInt(this.#snapshot.errorCount) + 1n).toString();
|
|
294275
|
+
this.#transition("error", kind, safeFailureReason(this.account.exchange, reason));
|
|
294276
|
+
}
|
|
294277
|
+
async#connectLoop() {
|
|
294278
|
+
while (!this.#stopping) {
|
|
294279
|
+
if (this.#attempts > 0) {
|
|
294280
|
+
this.#snapshot.reconnectCount = (BigInt(this.#snapshot.reconnectCount) + 1n).toString();
|
|
294281
|
+
}
|
|
294282
|
+
this.#attempts += 1;
|
|
294283
|
+
this.#snapshot.connectAttemptCount = String(this.#attempts);
|
|
294284
|
+
this.#failureObserved = false;
|
|
294285
|
+
this.#transition("connecting");
|
|
294286
|
+
const stream4 = new BinanceSpotUserDataStream(this.account.exchange, {
|
|
294287
|
+
observer: {
|
|
294288
|
+
onConnected: () => this.#connected(),
|
|
294289
|
+
onAuthenticated: () => this.#authenticated(),
|
|
294290
|
+
onEvent: (message) => this.#received(message),
|
|
294291
|
+
onFailure: (failure) => this.#handleStreamFailure(failure)
|
|
294292
|
+
}
|
|
294293
|
+
});
|
|
294294
|
+
this.#stream = stream4;
|
|
294295
|
+
try {
|
|
294296
|
+
for await (const _event of stream4) {}
|
|
294297
|
+
} catch (error) {
|
|
294298
|
+
if (!this.#stopping && !this.#failureObserved) {
|
|
294299
|
+
this.#fail("transport_error", error instanceof Error ? error.message : "User-data stream failed");
|
|
294300
|
+
}
|
|
294301
|
+
} finally {
|
|
294302
|
+
stream4.close();
|
|
294303
|
+
if (this.#stream === stream4)
|
|
294304
|
+
this.#stream = null;
|
|
294305
|
+
}
|
|
294306
|
+
if (!this.#stopping)
|
|
294307
|
+
await this.#waitForRetry();
|
|
294308
|
+
}
|
|
294309
|
+
}
|
|
294310
|
+
#handleStreamFailure(failure) {
|
|
294311
|
+
this.#fail(failure.kind, failure.reason);
|
|
294312
|
+
}
|
|
294313
|
+
#waitForRetry() {
|
|
294314
|
+
return new Promise((resolve) => {
|
|
294315
|
+
const delay = retryDelay(Math.max(this.#attempts - 1, 0));
|
|
294316
|
+
this.#retryResolve = resolve;
|
|
294317
|
+
this.#retryTimer = setTimeout(() => {
|
|
294318
|
+
this.#retryTimer = null;
|
|
294319
|
+
this.#retryResolve = null;
|
|
294320
|
+
resolve();
|
|
294321
|
+
}, delay);
|
|
294322
|
+
this.#retryTimer.unref?.();
|
|
294323
|
+
});
|
|
294324
|
+
}
|
|
294325
|
+
}
|
|
294326
|
+
|
|
294327
|
+
class UserDataStreamSupervisor {
|
|
294328
|
+
options;
|
|
294329
|
+
#workers = new Map;
|
|
294330
|
+
#started = false;
|
|
294331
|
+
constructor(options) {
|
|
294332
|
+
this.options = options;
|
|
294333
|
+
for (const [exchange, pool] of Object.entries(options.brokers)) {
|
|
294334
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
294335
|
+
const normalizedExchange = exchange.trim().toLowerCase();
|
|
294336
|
+
const worker = new AccountWorker(normalizedExchange, account, () => this.#publish());
|
|
294337
|
+
this.#workers.set(`${normalizedExchange}|${account.label}`, worker);
|
|
294338
|
+
}
|
|
294339
|
+
}
|
|
294340
|
+
if (this.#workers.size === 0) {
|
|
294341
|
+
throw new Error("User-data supervisor requires at least one configured account");
|
|
294342
|
+
}
|
|
294343
|
+
}
|
|
294344
|
+
start() {
|
|
294345
|
+
if (this.#started)
|
|
294346
|
+
return;
|
|
294347
|
+
this.#started = true;
|
|
294348
|
+
this.options.publisher.start();
|
|
294349
|
+
for (const worker of this.#workers.values())
|
|
294350
|
+
worker.start();
|
|
294351
|
+
this.#publish();
|
|
294352
|
+
}
|
|
294353
|
+
subscribe(options) {
|
|
294354
|
+
const exchange = options.exchange.trim().toLowerCase();
|
|
294355
|
+
const worker = this.#workers.get(`${exchange}|${options.accountSelector}`);
|
|
294356
|
+
if (!worker)
|
|
294357
|
+
throw new Error("Configured account user-data stream is unavailable");
|
|
294358
|
+
return worker.subscribe({ kind: options.kind, marketId: options.marketId });
|
|
294359
|
+
}
|
|
294360
|
+
async close() {
|
|
294361
|
+
for (const worker of this.#workers.values())
|
|
294362
|
+
await worker.stop();
|
|
294363
|
+
await this.options.publisher.close(this.#snapshots());
|
|
294364
|
+
}
|
|
294365
|
+
#snapshots() {
|
|
294366
|
+
return [...this.#workers.values()].map((worker) => worker.snapshot());
|
|
294367
|
+
}
|
|
294368
|
+
#publish() {
|
|
294369
|
+
if (!this.#started)
|
|
294370
|
+
return;
|
|
294371
|
+
this.options.publisher.publish(this.#snapshots());
|
|
294372
|
+
}
|
|
294373
|
+
}
|
|
294374
|
+
|
|
292886
294375
|
// src/server.ts
|
|
292887
294376
|
import * as grpc14 from "@grpc/grpc-js";
|
|
292888
294377
|
|
|
@@ -293723,9 +295212,9 @@ function floatSafeRemainder(val, step) {
|
|
|
293723
295212
|
return valInt % stepInt / 10 ** decCount;
|
|
293724
295213
|
}
|
|
293725
295214
|
var EVALUATING = Symbol("evaluating");
|
|
293726
|
-
function defineLazy(object,
|
|
295215
|
+
function defineLazy(object, key2, getter) {
|
|
293727
295216
|
let value = undefined;
|
|
293728
|
-
Object.defineProperty(object,
|
|
295217
|
+
Object.defineProperty(object, key2, {
|
|
293729
295218
|
get() {
|
|
293730
295219
|
if (value === EVALUATING) {
|
|
293731
295220
|
return;
|
|
@@ -293737,7 +295226,7 @@ function defineLazy(object, key, getter) {
|
|
|
293737
295226
|
return value;
|
|
293738
295227
|
},
|
|
293739
295228
|
set(v) {
|
|
293740
|
-
Object.defineProperty(object,
|
|
295229
|
+
Object.defineProperty(object, key2, {
|
|
293741
295230
|
value: v
|
|
293742
295231
|
});
|
|
293743
295232
|
},
|
|
@@ -293769,11 +295258,11 @@ function cloneDef(schema) {
|
|
|
293769
295258
|
function getElementAtPath(obj, path) {
|
|
293770
295259
|
if (!path)
|
|
293771
295260
|
return obj;
|
|
293772
|
-
return path.reduce((acc,
|
|
295261
|
+
return path.reduce((acc, key2) => acc?.[key2], obj);
|
|
293773
295262
|
}
|
|
293774
295263
|
function promiseAllObject(promisesObj) {
|
|
293775
295264
|
const keys2 = Object.keys(promisesObj);
|
|
293776
|
-
const promises = keys2.map((
|
|
295265
|
+
const promises = keys2.map((key2) => promisesObj[key2]);
|
|
293777
295266
|
return Promise.all(promises).then((results) => {
|
|
293778
295267
|
const resolvedObj = {};
|
|
293779
295268
|
for (let i2 = 0;i2 < keys2.length; i2++) {
|
|
@@ -293837,8 +295326,8 @@ function shallowClone(o) {
|
|
|
293837
295326
|
}
|
|
293838
295327
|
function numKeys(data) {
|
|
293839
295328
|
let keyCount = 0;
|
|
293840
|
-
for (const
|
|
293841
|
-
if (Object.prototype.hasOwnProperty.call(data,
|
|
295329
|
+
for (const key2 in data) {
|
|
295330
|
+
if (Object.prototype.hasOwnProperty.call(data, key2)) {
|
|
293842
295331
|
keyCount++;
|
|
293843
295332
|
}
|
|
293844
295333
|
}
|
|
@@ -293981,13 +295470,13 @@ function pick(schema, mask2) {
|
|
|
293981
295470
|
const def = mergeDefs(schema._zod.def, {
|
|
293982
295471
|
get shape() {
|
|
293983
295472
|
const newShape = {};
|
|
293984
|
-
for (const
|
|
293985
|
-
if (!(
|
|
293986
|
-
throw new Error(`Unrecognized key: "${
|
|
295473
|
+
for (const key2 in mask2) {
|
|
295474
|
+
if (!(key2 in currDef.shape)) {
|
|
295475
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
293987
295476
|
}
|
|
293988
|
-
if (!mask2[
|
|
295477
|
+
if (!mask2[key2])
|
|
293989
295478
|
continue;
|
|
293990
|
-
newShape[
|
|
295479
|
+
newShape[key2] = currDef.shape[key2];
|
|
293991
295480
|
}
|
|
293992
295481
|
assignProp(this, "shape", newShape);
|
|
293993
295482
|
return newShape;
|
|
@@ -294006,13 +295495,13 @@ function omit5(schema, mask2) {
|
|
|
294006
295495
|
const def = mergeDefs(schema._zod.def, {
|
|
294007
295496
|
get shape() {
|
|
294008
295497
|
const newShape = { ...schema._zod.def.shape };
|
|
294009
|
-
for (const
|
|
294010
|
-
if (!(
|
|
294011
|
-
throw new Error(`Unrecognized key: "${
|
|
295498
|
+
for (const key2 in mask2) {
|
|
295499
|
+
if (!(key2 in currDef.shape)) {
|
|
295500
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
294012
295501
|
}
|
|
294013
|
-
if (!mask2[
|
|
295502
|
+
if (!mask2[key2])
|
|
294014
295503
|
continue;
|
|
294015
|
-
delete newShape[
|
|
295504
|
+
delete newShape[key2];
|
|
294016
295505
|
}
|
|
294017
295506
|
assignProp(this, "shape", newShape);
|
|
294018
295507
|
return newShape;
|
|
@@ -294029,8 +295518,8 @@ function extend4(schema, shape) {
|
|
|
294029
295518
|
const hasChecks = checks && checks.length > 0;
|
|
294030
295519
|
if (hasChecks) {
|
|
294031
295520
|
const existingShape = schema._zod.def.shape;
|
|
294032
|
-
for (const
|
|
294033
|
-
if (Object.getOwnPropertyDescriptor(existingShape,
|
|
295521
|
+
for (const key2 in shape) {
|
|
295522
|
+
if (Object.getOwnPropertyDescriptor(existingShape, key2) !== undefined) {
|
|
294034
295523
|
throw new Error("Cannot overwrite keys on object schemas containing refinements. Use `.safeExtend()` instead.");
|
|
294035
295524
|
}
|
|
294036
295525
|
}
|
|
@@ -294083,23 +295572,23 @@ function partial(Class, schema, mask2) {
|
|
|
294083
295572
|
const oldShape = schema._zod.def.shape;
|
|
294084
295573
|
const shape = { ...oldShape };
|
|
294085
295574
|
if (mask2) {
|
|
294086
|
-
for (const
|
|
294087
|
-
if (!(
|
|
294088
|
-
throw new Error(`Unrecognized key: "${
|
|
295575
|
+
for (const key2 in mask2) {
|
|
295576
|
+
if (!(key2 in oldShape)) {
|
|
295577
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
294089
295578
|
}
|
|
294090
|
-
if (!mask2[
|
|
295579
|
+
if (!mask2[key2])
|
|
294091
295580
|
continue;
|
|
294092
|
-
shape[
|
|
295581
|
+
shape[key2] = Class ? new Class({
|
|
294093
295582
|
type: "optional",
|
|
294094
|
-
innerType: oldShape[
|
|
294095
|
-
}) : oldShape[
|
|
295583
|
+
innerType: oldShape[key2]
|
|
295584
|
+
}) : oldShape[key2];
|
|
294096
295585
|
}
|
|
294097
295586
|
} else {
|
|
294098
|
-
for (const
|
|
294099
|
-
shape[
|
|
295587
|
+
for (const key2 in oldShape) {
|
|
295588
|
+
shape[key2] = Class ? new Class({
|
|
294100
295589
|
type: "optional",
|
|
294101
|
-
innerType: oldShape[
|
|
294102
|
-
}) : oldShape[
|
|
295590
|
+
innerType: oldShape[key2]
|
|
295591
|
+
}) : oldShape[key2];
|
|
294103
295592
|
}
|
|
294104
295593
|
}
|
|
294105
295594
|
assignProp(this, "shape", shape);
|
|
@@ -294115,22 +295604,22 @@ function required(Class, schema, mask2) {
|
|
|
294115
295604
|
const oldShape = schema._zod.def.shape;
|
|
294116
295605
|
const shape = { ...oldShape };
|
|
294117
295606
|
if (mask2) {
|
|
294118
|
-
for (const
|
|
294119
|
-
if (!(
|
|
294120
|
-
throw new Error(`Unrecognized key: "${
|
|
295607
|
+
for (const key2 in mask2) {
|
|
295608
|
+
if (!(key2 in shape)) {
|
|
295609
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
294121
295610
|
}
|
|
294122
|
-
if (!mask2[
|
|
295611
|
+
if (!mask2[key2])
|
|
294123
295612
|
continue;
|
|
294124
|
-
shape[
|
|
295613
|
+
shape[key2] = new Class({
|
|
294125
295614
|
type: "nonoptional",
|
|
294126
|
-
innerType: oldShape[
|
|
295615
|
+
innerType: oldShape[key2]
|
|
294127
295616
|
});
|
|
294128
295617
|
}
|
|
294129
295618
|
} else {
|
|
294130
|
-
for (const
|
|
294131
|
-
shape[
|
|
295619
|
+
for (const key2 in oldShape) {
|
|
295620
|
+
shape[key2] = new Class({
|
|
294132
295621
|
type: "nonoptional",
|
|
294133
|
-
innerType: oldShape[
|
|
295622
|
+
innerType: oldShape[key2]
|
|
294134
295623
|
});
|
|
294135
295624
|
}
|
|
294136
295625
|
}
|
|
@@ -295880,19 +297369,19 @@ var $ZodArray = /* @__PURE__ */ $constructor("$ZodArray", (inst, def) => {
|
|
|
295880
297369
|
return payload;
|
|
295881
297370
|
};
|
|
295882
297371
|
});
|
|
295883
|
-
function handlePropertyResult(result, final,
|
|
297372
|
+
function handlePropertyResult(result, final, key2, input, isOptionalOut) {
|
|
295884
297373
|
if (result.issues.length) {
|
|
295885
|
-
if (isOptionalOut && !(
|
|
297374
|
+
if (isOptionalOut && !(key2 in input)) {
|
|
295886
297375
|
return;
|
|
295887
297376
|
}
|
|
295888
|
-
final.issues.push(...prefixIssues(
|
|
297377
|
+
final.issues.push(...prefixIssues(key2, result.issues));
|
|
295889
297378
|
}
|
|
295890
297379
|
if (result.value === undefined) {
|
|
295891
|
-
if (
|
|
295892
|
-
final.value[
|
|
297380
|
+
if (key2 in input) {
|
|
297381
|
+
final.value[key2] = undefined;
|
|
295893
297382
|
}
|
|
295894
297383
|
} else {
|
|
295895
|
-
final.value[
|
|
297384
|
+
final.value[key2] = result.value;
|
|
295896
297385
|
}
|
|
295897
297386
|
}
|
|
295898
297387
|
function normalizeDef(def) {
|
|
@@ -295917,18 +297406,18 @@ function handleCatchall(proms, input, payload, ctx, def, inst) {
|
|
|
295917
297406
|
const _catchall = def.catchall._zod;
|
|
295918
297407
|
const t = _catchall.def.type;
|
|
295919
297408
|
const isOptionalOut = _catchall.optout === "optional";
|
|
295920
|
-
for (const
|
|
295921
|
-
if (keySet.has(
|
|
297409
|
+
for (const key2 in input) {
|
|
297410
|
+
if (keySet.has(key2))
|
|
295922
297411
|
continue;
|
|
295923
297412
|
if (t === "never") {
|
|
295924
|
-
unrecognized.push(
|
|
297413
|
+
unrecognized.push(key2);
|
|
295925
297414
|
continue;
|
|
295926
297415
|
}
|
|
295927
|
-
const r = _catchall.run({ value: input[
|
|
297416
|
+
const r = _catchall.run({ value: input[key2], issues: [] }, ctx);
|
|
295928
297417
|
if (r instanceof Promise) {
|
|
295929
|
-
proms.push(r.then((r2) => handlePropertyResult(r2, payload,
|
|
297418
|
+
proms.push(r.then((r2) => handlePropertyResult(r2, payload, key2, input, isOptionalOut)));
|
|
295930
297419
|
} else {
|
|
295931
|
-
handlePropertyResult(r, payload,
|
|
297420
|
+
handlePropertyResult(r, payload, key2, input, isOptionalOut);
|
|
295932
297421
|
}
|
|
295933
297422
|
}
|
|
295934
297423
|
if (unrecognized.length) {
|
|
@@ -295964,12 +297453,12 @@ var $ZodObject = /* @__PURE__ */ $constructor("$ZodObject", (inst, def) => {
|
|
|
295964
297453
|
defineLazy(inst._zod, "propValues", () => {
|
|
295965
297454
|
const shape = def.shape;
|
|
295966
297455
|
const propValues = {};
|
|
295967
|
-
for (const
|
|
295968
|
-
const field = shape[
|
|
297456
|
+
for (const key2 in shape) {
|
|
297457
|
+
const field = shape[key2]._zod;
|
|
295969
297458
|
if (field.values) {
|
|
295970
|
-
propValues[
|
|
297459
|
+
propValues[key2] ?? (propValues[key2] = new Set);
|
|
295971
297460
|
for (const v of field.values)
|
|
295972
|
-
propValues[
|
|
297461
|
+
propValues[key2].add(v);
|
|
295973
297462
|
}
|
|
295974
297463
|
}
|
|
295975
297464
|
return propValues;
|
|
@@ -295992,14 +297481,14 @@ var $ZodObject = /* @__PURE__ */ $constructor("$ZodObject", (inst, def) => {
|
|
|
295992
297481
|
payload.value = {};
|
|
295993
297482
|
const proms = [];
|
|
295994
297483
|
const shape = value.shape;
|
|
295995
|
-
for (const
|
|
295996
|
-
const el = shape[
|
|
297484
|
+
for (const key2 of value.keys) {
|
|
297485
|
+
const el = shape[key2];
|
|
295997
297486
|
const isOptionalOut = el._zod.optout === "optional";
|
|
295998
|
-
const r = el._zod.run({ value: input[
|
|
297487
|
+
const r = el._zod.run({ value: input[key2], issues: [] }, ctx);
|
|
295999
297488
|
if (r instanceof Promise) {
|
|
296000
|
-
proms.push(r.then((r2) => handlePropertyResult(r2, payload,
|
|
297489
|
+
proms.push(r.then((r2) => handlePropertyResult(r2, payload, key2, input, isOptionalOut)));
|
|
296001
297490
|
} else {
|
|
296002
|
-
handlePropertyResult(r, payload,
|
|
297491
|
+
handlePropertyResult(r, payload, key2, input, isOptionalOut);
|
|
296003
297492
|
}
|
|
296004
297493
|
}
|
|
296005
297494
|
if (!catchall) {
|
|
@@ -296015,23 +297504,23 @@ var $ZodObjectJIT = /* @__PURE__ */ $constructor("$ZodObjectJIT", (inst, def) =>
|
|
|
296015
297504
|
const generateFastpass = (shape) => {
|
|
296016
297505
|
const doc = new Doc(["shape", "payload", "ctx"]);
|
|
296017
297506
|
const normalized = _normalized.value;
|
|
296018
|
-
const parseStr = (
|
|
296019
|
-
const k = esc(
|
|
297507
|
+
const parseStr = (key2) => {
|
|
297508
|
+
const k = esc(key2);
|
|
296020
297509
|
return `shape[${k}]._zod.run({ value: input[${k}], issues: [] }, ctx)`;
|
|
296021
297510
|
};
|
|
296022
297511
|
doc.write(`const input = payload.value;`);
|
|
296023
297512
|
const ids = Object.create(null);
|
|
296024
|
-
let
|
|
296025
|
-
for (const
|
|
296026
|
-
ids[
|
|
297513
|
+
let counter2 = 0;
|
|
297514
|
+
for (const key2 of normalized.keys) {
|
|
297515
|
+
ids[key2] = `key_${counter2++}`;
|
|
296027
297516
|
}
|
|
296028
297517
|
doc.write(`const newResult = {};`);
|
|
296029
|
-
for (const
|
|
296030
|
-
const id2 = ids[
|
|
296031
|
-
const k = esc(
|
|
296032
|
-
const schema = shape[
|
|
297518
|
+
for (const key2 of normalized.keys) {
|
|
297519
|
+
const id2 = ids[key2];
|
|
297520
|
+
const k = esc(key2);
|
|
297521
|
+
const schema = shape[key2];
|
|
296033
297522
|
const isOptionalOut = schema?._zod?.optout === "optional";
|
|
296034
|
-
doc.write(`const ${id2} = ${parseStr(
|
|
297523
|
+
doc.write(`const ${id2} = ${parseStr(key2)};`);
|
|
296035
297524
|
if (isOptionalOut) {
|
|
296036
297525
|
doc.write(`
|
|
296037
297526
|
if (${id2}.issues.length) {
|
|
@@ -296317,17 +297806,17 @@ function mergeValues(a, b2) {
|
|
|
296317
297806
|
}
|
|
296318
297807
|
if (isPlainObject2(a) && isPlainObject2(b2)) {
|
|
296319
297808
|
const bKeys = Object.keys(b2);
|
|
296320
|
-
const sharedKeys = Object.keys(a).filter((
|
|
297809
|
+
const sharedKeys = Object.keys(a).filter((key2) => bKeys.indexOf(key2) !== -1);
|
|
296321
297810
|
const newObj = { ...a, ...b2 };
|
|
296322
|
-
for (const
|
|
296323
|
-
const sharedValue = mergeValues(a[
|
|
297811
|
+
for (const key2 of sharedKeys) {
|
|
297812
|
+
const sharedValue = mergeValues(a[key2], b2[key2]);
|
|
296324
297813
|
if (!sharedValue.valid) {
|
|
296325
297814
|
return {
|
|
296326
297815
|
valid: false,
|
|
296327
|
-
mergeErrorPath: [
|
|
297816
|
+
mergeErrorPath: [key2, ...sharedValue.mergeErrorPath]
|
|
296328
297817
|
};
|
|
296329
297818
|
}
|
|
296330
|
-
newObj[
|
|
297819
|
+
newObj[key2] = sharedValue.data;
|
|
296331
297820
|
}
|
|
296332
297821
|
return { valid: true, data: newObj };
|
|
296333
297822
|
}
|
|
@@ -296483,30 +297972,30 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
|
|
|
296483
297972
|
if (values2) {
|
|
296484
297973
|
payload.value = {};
|
|
296485
297974
|
const recordKeys = new Set;
|
|
296486
|
-
for (const
|
|
296487
|
-
if (typeof
|
|
296488
|
-
recordKeys.add(typeof
|
|
296489
|
-
const result = def.valueType._zod.run({ value: input[
|
|
297975
|
+
for (const key2 of values2) {
|
|
297976
|
+
if (typeof key2 === "string" || typeof key2 === "number" || typeof key2 === "symbol") {
|
|
297977
|
+
recordKeys.add(typeof key2 === "number" ? key2.toString() : key2);
|
|
297978
|
+
const result = def.valueType._zod.run({ value: input[key2], issues: [] }, ctx);
|
|
296490
297979
|
if (result instanceof Promise) {
|
|
296491
297980
|
proms.push(result.then((result2) => {
|
|
296492
297981
|
if (result2.issues.length) {
|
|
296493
|
-
payload.issues.push(...prefixIssues(
|
|
297982
|
+
payload.issues.push(...prefixIssues(key2, result2.issues));
|
|
296494
297983
|
}
|
|
296495
|
-
payload.value[
|
|
297984
|
+
payload.value[key2] = result2.value;
|
|
296496
297985
|
}));
|
|
296497
297986
|
} else {
|
|
296498
297987
|
if (result.issues.length) {
|
|
296499
|
-
payload.issues.push(...prefixIssues(
|
|
297988
|
+
payload.issues.push(...prefixIssues(key2, result.issues));
|
|
296500
297989
|
}
|
|
296501
|
-
payload.value[
|
|
297990
|
+
payload.value[key2] = result.value;
|
|
296502
297991
|
}
|
|
296503
297992
|
}
|
|
296504
297993
|
}
|
|
296505
297994
|
let unrecognized;
|
|
296506
|
-
for (const
|
|
296507
|
-
if (!recordKeys.has(
|
|
297995
|
+
for (const key2 in input) {
|
|
297996
|
+
if (!recordKeys.has(key2)) {
|
|
296508
297997
|
unrecognized = unrecognized ?? [];
|
|
296509
|
-
unrecognized.push(
|
|
297998
|
+
unrecognized.push(key2);
|
|
296510
297999
|
}
|
|
296511
298000
|
}
|
|
296512
298001
|
if (unrecognized && unrecognized.length > 0) {
|
|
@@ -296519,16 +298008,16 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
|
|
|
296519
298008
|
}
|
|
296520
298009
|
} else {
|
|
296521
298010
|
payload.value = {};
|
|
296522
|
-
for (const
|
|
296523
|
-
if (
|
|
298011
|
+
for (const key2 of Reflect.ownKeys(input)) {
|
|
298012
|
+
if (key2 === "__proto__")
|
|
296524
298013
|
continue;
|
|
296525
|
-
let keyResult = def.keyType._zod.run({ value:
|
|
298014
|
+
let keyResult = def.keyType._zod.run({ value: key2, issues: [] }, ctx);
|
|
296526
298015
|
if (keyResult instanceof Promise) {
|
|
296527
298016
|
throw new Error("Async schemas not supported in object keys currently");
|
|
296528
298017
|
}
|
|
296529
|
-
const checkNumericKey = typeof
|
|
298018
|
+
const checkNumericKey = typeof key2 === "string" && number3.test(key2) && keyResult.issues.length;
|
|
296530
298019
|
if (checkNumericKey) {
|
|
296531
|
-
const retryResult = def.keyType._zod.run({ value: Number(
|
|
298020
|
+
const retryResult = def.keyType._zod.run({ value: Number(key2), issues: [] }, ctx);
|
|
296532
298021
|
if (retryResult instanceof Promise) {
|
|
296533
298022
|
throw new Error("Async schemas not supported in object keys currently");
|
|
296534
298023
|
}
|
|
@@ -296538,30 +298027,30 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
|
|
|
296538
298027
|
}
|
|
296539
298028
|
if (keyResult.issues.length) {
|
|
296540
298029
|
if (def.mode === "loose") {
|
|
296541
|
-
payload.value[
|
|
298030
|
+
payload.value[key2] = input[key2];
|
|
296542
298031
|
} else {
|
|
296543
298032
|
payload.issues.push({
|
|
296544
298033
|
code: "invalid_key",
|
|
296545
298034
|
origin: "record",
|
|
296546
298035
|
issues: keyResult.issues.map((iss) => finalizeIssue(iss, ctx, config())),
|
|
296547
|
-
input:
|
|
296548
|
-
path: [
|
|
298036
|
+
input: key2,
|
|
298037
|
+
path: [key2],
|
|
296549
298038
|
inst
|
|
296550
298039
|
});
|
|
296551
298040
|
}
|
|
296552
298041
|
continue;
|
|
296553
298042
|
}
|
|
296554
|
-
const result = def.valueType._zod.run({ value: input[
|
|
298043
|
+
const result = def.valueType._zod.run({ value: input[key2], issues: [] }, ctx);
|
|
296555
298044
|
if (result instanceof Promise) {
|
|
296556
298045
|
proms.push(result.then((result2) => {
|
|
296557
298046
|
if (result2.issues.length) {
|
|
296558
|
-
payload.issues.push(...prefixIssues(
|
|
298047
|
+
payload.issues.push(...prefixIssues(key2, result2.issues));
|
|
296559
298048
|
}
|
|
296560
298049
|
payload.value[keyResult.value] = result2.value;
|
|
296561
298050
|
}));
|
|
296562
298051
|
} else {
|
|
296563
298052
|
if (result.issues.length) {
|
|
296564
|
-
payload.issues.push(...prefixIssues(
|
|
298053
|
+
payload.issues.push(...prefixIssues(key2, result.issues));
|
|
296565
298054
|
}
|
|
296566
298055
|
payload.value[keyResult.value] = result.value;
|
|
296567
298056
|
}
|
|
@@ -296588,15 +298077,15 @@ var $ZodMap = /* @__PURE__ */ $constructor("$ZodMap", (inst, def) => {
|
|
|
296588
298077
|
}
|
|
296589
298078
|
const proms = [];
|
|
296590
298079
|
payload.value = new Map;
|
|
296591
|
-
for (const [
|
|
296592
|
-
const keyResult = def.keyType._zod.run({ value:
|
|
298080
|
+
for (const [key2, value] of input) {
|
|
298081
|
+
const keyResult = def.keyType._zod.run({ value: key2, issues: [] }, ctx);
|
|
296593
298082
|
const valueResult = def.valueType._zod.run({ value, issues: [] }, ctx);
|
|
296594
298083
|
if (keyResult instanceof Promise || valueResult instanceof Promise) {
|
|
296595
298084
|
proms.push(Promise.all([keyResult, valueResult]).then(([keyResult2, valueResult2]) => {
|
|
296596
|
-
handleMapResult(keyResult2, valueResult2, payload,
|
|
298085
|
+
handleMapResult(keyResult2, valueResult2, payload, key2, input, inst, ctx);
|
|
296597
298086
|
}));
|
|
296598
298087
|
} else {
|
|
296599
|
-
handleMapResult(keyResult, valueResult, payload,
|
|
298088
|
+
handleMapResult(keyResult, valueResult, payload, key2, input, inst, ctx);
|
|
296600
298089
|
}
|
|
296601
298090
|
}
|
|
296602
298091
|
if (proms.length)
|
|
@@ -296604,10 +298093,10 @@ var $ZodMap = /* @__PURE__ */ $constructor("$ZodMap", (inst, def) => {
|
|
|
296604
298093
|
return payload;
|
|
296605
298094
|
};
|
|
296606
298095
|
});
|
|
296607
|
-
function handleMapResult(keyResult, valueResult, final,
|
|
298096
|
+
function handleMapResult(keyResult, valueResult, final, key2, input, inst, ctx) {
|
|
296608
298097
|
if (keyResult.issues.length) {
|
|
296609
|
-
if (propertyKeyTypes.has(typeof
|
|
296610
|
-
final.issues.push(...prefixIssues(
|
|
298098
|
+
if (propertyKeyTypes.has(typeof key2)) {
|
|
298099
|
+
final.issues.push(...prefixIssues(key2, keyResult.issues));
|
|
296611
298100
|
} else {
|
|
296612
298101
|
final.issues.push({
|
|
296613
298102
|
code: "invalid_key",
|
|
@@ -296619,15 +298108,15 @@ function handleMapResult(keyResult, valueResult, final, key, input, inst, ctx) {
|
|
|
296619
298108
|
}
|
|
296620
298109
|
}
|
|
296621
298110
|
if (valueResult.issues.length) {
|
|
296622
|
-
if (propertyKeyTypes.has(typeof
|
|
296623
|
-
final.issues.push(...prefixIssues(
|
|
298111
|
+
if (propertyKeyTypes.has(typeof key2)) {
|
|
298112
|
+
final.issues.push(...prefixIssues(key2, valueResult.issues));
|
|
296624
298113
|
} else {
|
|
296625
298114
|
final.issues.push({
|
|
296626
298115
|
origin: "map",
|
|
296627
298116
|
code: "invalid_element",
|
|
296628
298117
|
input,
|
|
296629
298118
|
inst,
|
|
296630
|
-
key,
|
|
298119
|
+
key: key2,
|
|
296631
298120
|
issues: valueResult.issues.map((iss) => finalizeIssue(iss, ctx, config()))
|
|
296632
298121
|
});
|
|
296633
298122
|
}
|
|
@@ -296957,12 +298446,12 @@ var $ZodPipe = /* @__PURE__ */ $constructor("$ZodPipe", (inst, def) => {
|
|
|
296957
298446
|
return handlePipeResult(left, def.out, ctx);
|
|
296958
298447
|
};
|
|
296959
298448
|
});
|
|
296960
|
-
function handlePipeResult(left,
|
|
298449
|
+
function handlePipeResult(left, next2, ctx) {
|
|
296961
298450
|
if (left.issues.length) {
|
|
296962
298451
|
left.aborted = true;
|
|
296963
298452
|
return left;
|
|
296964
298453
|
}
|
|
296965
|
-
return
|
|
298454
|
+
return next2._zod.run({ value: left.value, issues: left.issues }, ctx);
|
|
296966
298455
|
}
|
|
296967
298456
|
var $ZodCodec = /* @__PURE__ */ $constructor("$ZodCodec", (inst, def) => {
|
|
296968
298457
|
$ZodType.init(inst, def);
|
|
@@ -303821,8 +305310,8 @@ function extractDefs(ctx, schema) {
|
|
|
303821
305310
|
if (defId)
|
|
303822
305311
|
seen.defId = defId;
|
|
303823
305312
|
const schema2 = seen.schema;
|
|
303824
|
-
for (const
|
|
303825
|
-
delete schema2[
|
|
305313
|
+
for (const key2 in schema2) {
|
|
305314
|
+
delete schema2[key2];
|
|
303826
305315
|
}
|
|
303827
305316
|
schema2.$ref = ref;
|
|
303828
305317
|
};
|
|
@@ -303889,20 +305378,20 @@ function finalize(ctx, schema) {
|
|
|
303889
305378
|
Object.assign(schema2, _cached);
|
|
303890
305379
|
const isParentRef = zodSchema._zod.parent === ref;
|
|
303891
305380
|
if (isParentRef) {
|
|
303892
|
-
for (const
|
|
303893
|
-
if (
|
|
305381
|
+
for (const key2 in schema2) {
|
|
305382
|
+
if (key2 === "$ref" || key2 === "allOf")
|
|
303894
305383
|
continue;
|
|
303895
|
-
if (!(
|
|
303896
|
-
delete schema2[
|
|
305384
|
+
if (!(key2 in _cached)) {
|
|
305385
|
+
delete schema2[key2];
|
|
303897
305386
|
}
|
|
303898
305387
|
}
|
|
303899
305388
|
}
|
|
303900
305389
|
if (refSchema.$ref && refSeen.def) {
|
|
303901
|
-
for (const
|
|
303902
|
-
if (
|
|
305390
|
+
for (const key2 in schema2) {
|
|
305391
|
+
if (key2 === "$ref" || key2 === "allOf")
|
|
303903
305392
|
continue;
|
|
303904
|
-
if (
|
|
303905
|
-
delete schema2[
|
|
305393
|
+
if (key2 in refSeen.def && JSON.stringify(schema2[key2]) === JSON.stringify(refSeen.def[key2])) {
|
|
305394
|
+
delete schema2[key2];
|
|
303906
305395
|
}
|
|
303907
305396
|
}
|
|
303908
305397
|
}
|
|
@@ -303914,11 +305403,11 @@ function finalize(ctx, schema) {
|
|
|
303914
305403
|
if (parentSeen?.schema.$ref) {
|
|
303915
305404
|
schema2.$ref = parentSeen.schema.$ref;
|
|
303916
305405
|
if (parentSeen.def) {
|
|
303917
|
-
for (const
|
|
303918
|
-
if (
|
|
305406
|
+
for (const key2 in schema2) {
|
|
305407
|
+
if (key2 === "$ref" || key2 === "allOf")
|
|
303919
305408
|
continue;
|
|
303920
|
-
if (
|
|
303921
|
-
delete schema2[
|
|
305409
|
+
if (key2 in parentSeen.def && JSON.stringify(schema2[key2]) === JSON.stringify(parentSeen.def[key2])) {
|
|
305410
|
+
delete schema2[key2];
|
|
303922
305411
|
}
|
|
303923
305412
|
}
|
|
303924
305413
|
}
|
|
@@ -303940,7 +305429,7 @@ function finalize(ctx, schema) {
|
|
|
303940
305429
|
result.$schema = "http://json-schema.org/draft-07/schema#";
|
|
303941
305430
|
} else if (ctx.target === "draft-04") {
|
|
303942
305431
|
result.$schema = "http://json-schema.org/draft-04/schema#";
|
|
303943
|
-
} else if (ctx.target === "openapi-3.0") {}
|
|
305432
|
+
} else if (ctx.target === "openapi-3.0") {}
|
|
303944
305433
|
if (ctx.external?.uri) {
|
|
303945
305434
|
const id2 = ctx.external.registry.get(schema)?.id;
|
|
303946
305435
|
if (!id2)
|
|
@@ -304009,8 +305498,8 @@ function isTransforming(_schema, _ctx) {
|
|
|
304009
305498
|
return isTransforming(def.in, ctx) || isTransforming(def.out, ctx);
|
|
304010
305499
|
}
|
|
304011
305500
|
if (def.type === "object") {
|
|
304012
|
-
for (const
|
|
304013
|
-
if (isTransforming(def.shape[
|
|
305501
|
+
for (const key2 in def.shape) {
|
|
305502
|
+
if (isTransforming(def.shape[key2], ctx))
|
|
304014
305503
|
return true;
|
|
304015
305504
|
}
|
|
304016
305505
|
return false;
|
|
@@ -304188,7 +305677,7 @@ var literalProcessor = (schema, ctx, json3, _params) => {
|
|
|
304188
305677
|
if (val === undefined) {
|
|
304189
305678
|
if (ctx.unrepresentable === "throw") {
|
|
304190
305679
|
throw new Error("Literal `undefined` cannot be represented in JSON Schema");
|
|
304191
|
-
}
|
|
305680
|
+
}
|
|
304192
305681
|
} else if (typeof val === "bigint") {
|
|
304193
305682
|
if (ctx.unrepresentable === "throw") {
|
|
304194
305683
|
throw new Error("BigInt literals cannot be represented in JSON Schema");
|
|
@@ -304301,15 +305790,15 @@ var objectProcessor = (schema, ctx, _json, params) => {
|
|
|
304301
305790
|
json3.type = "object";
|
|
304302
305791
|
json3.properties = {};
|
|
304303
305792
|
const shape = def.shape;
|
|
304304
|
-
for (const
|
|
304305
|
-
json3.properties[
|
|
305793
|
+
for (const key2 in shape) {
|
|
305794
|
+
json3.properties[key2] = process2(shape[key2], ctx, {
|
|
304306
305795
|
...params,
|
|
304307
|
-
path: [...params.path, "properties",
|
|
305796
|
+
path: [...params.path, "properties", key2]
|
|
304308
305797
|
});
|
|
304309
305798
|
}
|
|
304310
305799
|
const allKeys = new Set(Object.keys(shape));
|
|
304311
|
-
const requiredKeys = new Set([...allKeys].filter((
|
|
304312
|
-
const v = def.shape[
|
|
305800
|
+
const requiredKeys = new Set([...allKeys].filter((key2) => {
|
|
305801
|
+
const v = def.shape[key2]._zod;
|
|
304313
305802
|
if (ctx.io === "input") {
|
|
304314
305803
|
return v.optin === undefined;
|
|
304315
305804
|
} else {
|
|
@@ -304574,9 +306063,9 @@ function toJSONSchema(input, params) {
|
|
|
304574
306063
|
};
|
|
304575
306064
|
ctx2.external = external;
|
|
304576
306065
|
for (const entry of registry2._idmap.entries()) {
|
|
304577
|
-
const [
|
|
306066
|
+
const [key2, schema] = entry;
|
|
304578
306067
|
extractDefs(ctx2, schema);
|
|
304579
|
-
schemas[
|
|
306068
|
+
schemas[key2] = finalize(ctx2, schema);
|
|
304580
306069
|
}
|
|
304581
306070
|
if (Object.keys(defs).length > 0) {
|
|
304582
306071
|
const defsSegment = ctx2.target === "draft-2020-12" ? "$defs" : "definitions";
|
|
@@ -306133,11 +307622,11 @@ function resolveRef(ref, ctx) {
|
|
|
306133
307622
|
}
|
|
306134
307623
|
const defsKey = ctx.version === "draft-2020-12" ? "$defs" : "definitions";
|
|
306135
307624
|
if (path[0] === defsKey) {
|
|
306136
|
-
const
|
|
306137
|
-
if (!
|
|
307625
|
+
const key2 = path[1];
|
|
307626
|
+
if (!key2 || !ctx.defs[key2]) {
|
|
306138
307627
|
throw new Error(`Reference not found: ${ref}`);
|
|
306139
307628
|
}
|
|
306140
|
-
return ctx.defs[
|
|
307629
|
+
return ctx.defs[key2];
|
|
306141
307630
|
}
|
|
306142
307631
|
throw new Error(`Reference not found: ${ref}`);
|
|
306143
307632
|
}
|
|
@@ -306323,9 +307812,9 @@ function convertBaseSchema(schema, ctx) {
|
|
|
306323
307812
|
const shape = {};
|
|
306324
307813
|
const properties = schema.properties || {};
|
|
306325
307814
|
const requiredSet = new Set(schema.required || []);
|
|
306326
|
-
for (const [
|
|
307815
|
+
for (const [key2, propSchema] of Object.entries(properties)) {
|
|
306327
307816
|
const propZodSchema = convertSchema(propSchema, ctx);
|
|
306328
|
-
shape[
|
|
307817
|
+
shape[key2] = requiredSet.has(key2) ? propZodSchema : propZodSchema.optional();
|
|
306329
307818
|
}
|
|
306330
307819
|
if (schema.propertyNames) {
|
|
306331
307820
|
const keySchema = convertSchema(schema.propertyNames, ctx);
|
|
@@ -306469,20 +307958,20 @@ function convertSchema(schema, ctx) {
|
|
|
306469
307958
|
}
|
|
306470
307959
|
const extraMeta = {};
|
|
306471
307960
|
const coreMetadataKeys = ["$id", "id", "$comment", "$anchor", "$vocabulary", "$dynamicRef", "$dynamicAnchor"];
|
|
306472
|
-
for (const
|
|
306473
|
-
if (
|
|
306474
|
-
extraMeta[
|
|
307961
|
+
for (const key2 of coreMetadataKeys) {
|
|
307962
|
+
if (key2 in schema) {
|
|
307963
|
+
extraMeta[key2] = schema[key2];
|
|
306475
307964
|
}
|
|
306476
307965
|
}
|
|
306477
307966
|
const contentMetadataKeys = ["contentEncoding", "contentMediaType", "contentSchema"];
|
|
306478
|
-
for (const
|
|
306479
|
-
if (
|
|
306480
|
-
extraMeta[
|
|
307967
|
+
for (const key2 of contentMetadataKeys) {
|
|
307968
|
+
if (key2 in schema) {
|
|
307969
|
+
extraMeta[key2] = schema[key2];
|
|
306481
307970
|
}
|
|
306482
307971
|
}
|
|
306483
|
-
for (const
|
|
306484
|
-
if (!RECOGNIZED_KEYS.has(
|
|
306485
|
-
extraMeta[
|
|
307972
|
+
for (const key2 of Object.keys(schema)) {
|
|
307973
|
+
if (!RECOGNIZED_KEYS.has(key2)) {
|
|
307974
|
+
extraMeta[key2] = schema[key2];
|
|
306486
307975
|
}
|
|
306487
307976
|
}
|
|
306488
307977
|
if (Object.keys(extraMeta).length > 0) {
|
|
@@ -306847,7 +308336,7 @@ async function handleDeposit(ctx) {
|
|
|
306847
308336
|
network: depositNetwork?.exchangeNetworkId,
|
|
306848
308337
|
externalId: depositTxid,
|
|
306849
308338
|
txid: depositTxid,
|
|
306850
|
-
exchangeTimestamp:
|
|
308339
|
+
exchangeTimestamp: normalizeTimestamp2(creditedAt),
|
|
306851
308340
|
payload: deposit
|
|
306852
308341
|
}
|
|
306853
308342
|
});
|
|
@@ -307180,208 +308669,6 @@ function buildHistoricalOrderBookUnsupported(payload) {
|
|
|
307180
308669
|
// src/handlers/execute-action/order-book-call.ts
|
|
307181
308670
|
import * as grpc4 from "@grpc/grpc-js";
|
|
307182
308671
|
|
|
307183
|
-
// src/helpers/market-data-archive/capture-contract.ts
|
|
307184
|
-
import { createHash as createHash3 } from "node:crypto";
|
|
307185
|
-
var MARKET_CAPTURE_SCHEMA_VERSION = "1.0.0";
|
|
307186
|
-
var CHECKSUM_ALGORITHM = "sha256-canonical-json-v1";
|
|
307187
|
-
var ARCHIVE_SOURCES = ["broker_read", "broker_write"];
|
|
307188
|
-
var CAPTURE_FEEDS = [
|
|
307189
|
-
"ORDERBOOK",
|
|
307190
|
-
"TICKER",
|
|
307191
|
-
"TRADES",
|
|
307192
|
-
"OHLCV"
|
|
307193
|
-
];
|
|
307194
|
-
var SOURCE_MODES = [
|
|
307195
|
-
"broker_live_stream_v1",
|
|
307196
|
-
"broker_live_sampling_v1",
|
|
307197
|
-
"broker_current_snapshot_v1",
|
|
307198
|
-
"broker_bootstrap_fetch_v1",
|
|
307199
|
-
"external_ccxt_fallback_v1",
|
|
307200
|
-
"external_hummingbot_fallback_v1",
|
|
307201
|
-
"legacy_migration_v1"
|
|
307202
|
-
];
|
|
307203
|
-
var RAW_CAPTURE_SCOPES = [
|
|
307204
|
-
"ccxt_normalized_object",
|
|
307205
|
-
"broker_visible_payload",
|
|
307206
|
-
"exchange_wire_frame"
|
|
307207
|
-
];
|
|
307208
|
-
var CHECKSUM_FIELDS = new Set([
|
|
307209
|
-
"normalized_row_checksum",
|
|
307210
|
-
"raw_checksum",
|
|
307211
|
-
"checksum"
|
|
307212
|
-
]);
|
|
307213
|
-
function canonicalDecimal(value) {
|
|
307214
|
-
if (!Number.isFinite(value)) {
|
|
307215
|
-
throw new Error("Canonical numbers must be finite");
|
|
307216
|
-
}
|
|
307217
|
-
if (Object.is(value, -0)) {
|
|
307218
|
-
return "0";
|
|
307219
|
-
}
|
|
307220
|
-
const rendered = String(value).toLowerCase();
|
|
307221
|
-
if (!rendered.includes("e")) {
|
|
307222
|
-
return rendered;
|
|
307223
|
-
}
|
|
307224
|
-
const [coefficient = "0", exponentText = "0"] = rendered.split("e");
|
|
307225
|
-
const exponent = Number.parseInt(exponentText, 10);
|
|
307226
|
-
const negative = coefficient.startsWith("-");
|
|
307227
|
-
const unsigned = negative ? coefficient.slice(1) : coefficient;
|
|
307228
|
-
const [integer2 = "0", fraction = ""] = unsigned.split(".");
|
|
307229
|
-
const digits = `${integer2}${fraction}`;
|
|
307230
|
-
const decimalIndex = integer2.length + exponent;
|
|
307231
|
-
let result;
|
|
307232
|
-
if (decimalIndex <= 0) {
|
|
307233
|
-
result = `0.${"0".repeat(-decimalIndex)}${digits}`;
|
|
307234
|
-
} else if (decimalIndex >= digits.length) {
|
|
307235
|
-
result = `${digits}${"0".repeat(decimalIndex - digits.length)}`;
|
|
307236
|
-
} else {
|
|
307237
|
-
result = `${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
|
|
307238
|
-
}
|
|
307239
|
-
return negative ? `-${result}` : result;
|
|
307240
|
-
}
|
|
307241
|
-
function serializeCanonical(value, stack) {
|
|
307242
|
-
if (value === null)
|
|
307243
|
-
return "null";
|
|
307244
|
-
if (typeof value === "string")
|
|
307245
|
-
return JSON.stringify(value);
|
|
307246
|
-
if (typeof value === "boolean")
|
|
307247
|
-
return value ? "true" : "false";
|
|
307248
|
-
if (typeof value === "number")
|
|
307249
|
-
return canonicalDecimal(value);
|
|
307250
|
-
if (typeof value === "bigint")
|
|
307251
|
-
return value.toString(10);
|
|
307252
|
-
if (value instanceof Date) {
|
|
307253
|
-
if (Number.isNaN(value.getTime())) {
|
|
307254
|
-
throw new Error("Canonical timestamps must be valid");
|
|
307255
|
-
}
|
|
307256
|
-
return value.getTime().toString(10);
|
|
307257
|
-
}
|
|
307258
|
-
if (Array.isArray(value)) {
|
|
307259
|
-
if (stack.has(value))
|
|
307260
|
-
throw new Error("Canonical values must be acyclic");
|
|
307261
|
-
stack.add(value);
|
|
307262
|
-
const result = `[${value.map((entry) => entry === undefined ? "null" : serializeCanonical(entry, stack)).join(",")}]`;
|
|
307263
|
-
stack.delete(value);
|
|
307264
|
-
return result;
|
|
307265
|
-
}
|
|
307266
|
-
if (typeof value === "object") {
|
|
307267
|
-
if (stack.has(value))
|
|
307268
|
-
throw new Error("Canonical values must be acyclic");
|
|
307269
|
-
stack.add(value);
|
|
307270
|
-
const entries = Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([left], [right]) => left.localeCompare(right));
|
|
307271
|
-
const result = `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${serializeCanonical(entry, stack)}`).join(",")}}`;
|
|
307272
|
-
stack.delete(value);
|
|
307273
|
-
return result;
|
|
307274
|
-
}
|
|
307275
|
-
throw new Error(`Unsupported canonical value type: ${typeof value}`);
|
|
307276
|
-
}
|
|
307277
|
-
function canonicalSerialize(value) {
|
|
307278
|
-
return serializeCanonical(value, new Set);
|
|
307279
|
-
}
|
|
307280
|
-
function omitChecksumFields(value) {
|
|
307281
|
-
if (Array.isArray(value))
|
|
307282
|
-
return value.map(omitChecksumFields);
|
|
307283
|
-
if (value && typeof value === "object" && !(value instanceof Date)) {
|
|
307284
|
-
return Object.fromEntries(Object.entries(value).filter(([key]) => !CHECKSUM_FIELDS.has(key)).map(([key, entry]) => [key, omitChecksumFields(entry)]));
|
|
307285
|
-
}
|
|
307286
|
-
return value;
|
|
307287
|
-
}
|
|
307288
|
-
function sha256Canonical(value) {
|
|
307289
|
-
return createHash3("sha256").update(canonicalSerialize(omitChecksumFields(value))).digest("hex");
|
|
307290
|
-
}
|
|
307291
|
-
function normalizeTimestampMs(value, field) {
|
|
307292
|
-
let timestamp;
|
|
307293
|
-
if (value instanceof Date) {
|
|
307294
|
-
timestamp = value.getTime();
|
|
307295
|
-
} else if (typeof value === "number") {
|
|
307296
|
-
timestamp = value;
|
|
307297
|
-
} else if (typeof value === "string" && /^\d+$/.test(value.trim())) {
|
|
307298
|
-
timestamp = Number(value.trim());
|
|
307299
|
-
} else if (typeof value === "string") {
|
|
307300
|
-
timestamp = Date.parse(value);
|
|
307301
|
-
} else {
|
|
307302
|
-
timestamp = Number.NaN;
|
|
307303
|
-
}
|
|
307304
|
-
if (!Number.isSafeInteger(timestamp) || timestamp < 0) {
|
|
307305
|
-
throw new Error(`${field} must be a non-negative millisecond timestamp`);
|
|
307306
|
-
}
|
|
307307
|
-
return timestamp;
|
|
307308
|
-
}
|
|
307309
|
-
function assertCaptureContext(context2) {
|
|
307310
|
-
if (!ARCHIVE_SOURCES.includes(context2.source)) {
|
|
307311
|
-
throw new Error(`Unsupported archive source: ${context2.source}`);
|
|
307312
|
-
}
|
|
307313
|
-
if (!CAPTURE_FEEDS.includes(context2.feed)) {
|
|
307314
|
-
throw new Error(`Unsupported capture feed: ${context2.feed}`);
|
|
307315
|
-
}
|
|
307316
|
-
if (!SOURCE_MODES.includes(context2.sourceMode)) {
|
|
307317
|
-
throw new Error(`Unsupported source mode: ${context2.sourceMode}`);
|
|
307318
|
-
}
|
|
307319
|
-
for (const [field, value] of [
|
|
307320
|
-
["deployment_id", context2.deploymentId],
|
|
307321
|
-
["capture_bundle_id", context2.captureBundleId],
|
|
307322
|
-
["exchange", context2.exchange],
|
|
307323
|
-
["symbol", context2.symbol],
|
|
307324
|
-
["provider", context2.provider]
|
|
307325
|
-
]) {
|
|
307326
|
-
if (!value.trim())
|
|
307327
|
-
throw new Error(`${field} must not be empty`);
|
|
307328
|
-
}
|
|
307329
|
-
}
|
|
307330
|
-
function createRawCapture(context2, input) {
|
|
307331
|
-
assertCaptureContext(context2);
|
|
307332
|
-
if (!RAW_CAPTURE_SCOPES.includes(input.scope)) {
|
|
307333
|
-
throw new Error(`Unsupported raw capture scope: ${input.scope}`);
|
|
307334
|
-
}
|
|
307335
|
-
const eventTimeMs = normalizeTimestampMs(input.eventTimeMs, "event_time_ms");
|
|
307336
|
-
const receivedTimeMs = normalizeTimestampMs(input.receivedTimeMs, "received_time_ms");
|
|
307337
|
-
const redactedPayload = redactStreamPayload(input.payload);
|
|
307338
|
-
const rawChecksum = sha256Canonical(redactedPayload);
|
|
307339
|
-
const rawCaptureId = sha256Canonical({
|
|
307340
|
-
capture_bundle_id: context2.captureBundleId,
|
|
307341
|
-
exchange: context2.exchange.trim().toLowerCase(),
|
|
307342
|
-
feed: context2.feed,
|
|
307343
|
-
raw_capture_scope: input.scope,
|
|
307344
|
-
raw_payload_sha256: rawChecksum,
|
|
307345
|
-
schema_version: context2.schemaVersion,
|
|
307346
|
-
source_mode: context2.sourceMode,
|
|
307347
|
-
source_symbol: context2.symbol.trim(),
|
|
307348
|
-
source_time_ms: eventTimeMs
|
|
307349
|
-
});
|
|
307350
|
-
return {
|
|
307351
|
-
rawCaptureId,
|
|
307352
|
-
rawCaptureScope: input.scope,
|
|
307353
|
-
rawChecksum,
|
|
307354
|
-
redactedPayload,
|
|
307355
|
-
eventTimeMs,
|
|
307356
|
-
receivedTimeMs,
|
|
307357
|
-
checksumAlgorithm: context2.checksumAlgorithm
|
|
307358
|
-
};
|
|
307359
|
-
}
|
|
307360
|
-
function captureCoreFields(context2, rawCapture) {
|
|
307361
|
-
assertCaptureContext(context2);
|
|
307362
|
-
return {
|
|
307363
|
-
source: context2.source,
|
|
307364
|
-
deployment_id: context2.deploymentId,
|
|
307365
|
-
capture_bundle_id: context2.captureBundleId,
|
|
307366
|
-
exchange: context2.exchange.trim().toLowerCase(),
|
|
307367
|
-
symbol: context2.symbol.trim(),
|
|
307368
|
-
trading_pair: context2.symbol.trim().replace("/", "-"),
|
|
307369
|
-
source_symbol: context2.symbol.trim(),
|
|
307370
|
-
asset_type: context2.assetType,
|
|
307371
|
-
feed: context2.feed,
|
|
307372
|
-
provider: context2.provider,
|
|
307373
|
-
source_mode: context2.sourceMode,
|
|
307374
|
-
source_time_ms: rawCapture.eventTimeMs,
|
|
307375
|
-
received_time_ms: rawCapture.receivedTimeMs,
|
|
307376
|
-
raw_capture_id: rawCapture.rawCaptureId,
|
|
307377
|
-
raw_capture_scope: rawCapture.rawCaptureScope,
|
|
307378
|
-
schema_version: context2.schemaVersion,
|
|
307379
|
-
checksum_algorithm: context2.checksumAlgorithm,
|
|
307380
|
-
raw_checksum: rawCapture.rawChecksum,
|
|
307381
|
-
provenance_complete: context2.provenanceComplete ? 1 : 0
|
|
307382
|
-
};
|
|
307383
|
-
}
|
|
307384
|
-
|
|
307385
308672
|
// src/helpers/market-data-archive/canonical-orderbook.ts
|
|
307386
308673
|
class OrderBookValidationError extends Error {
|
|
307387
308674
|
reason;
|
|
@@ -307549,46 +308836,6 @@ function buildCanonicalOrderBookRows(input) {
|
|
|
307549
308836
|
};
|
|
307550
308837
|
}
|
|
307551
308838
|
|
|
307552
|
-
// src/helpers/market-data-archive/capture-context.ts
|
|
307553
|
-
function createMarketCaptureContext(input) {
|
|
307554
|
-
const environment = input.environment ?? "development";
|
|
307555
|
-
const deploymentId = input.deploymentId.trim();
|
|
307556
|
-
if (!deploymentId)
|
|
307557
|
-
throw new Error("deployment_id must not be empty");
|
|
307558
|
-
const configuredBundle = input.captureBundleId?.trim();
|
|
307559
|
-
if (environment === "production" && !configuredBundle) {
|
|
307560
|
-
throw new Error("capture_bundle_id is required for production market capture");
|
|
307561
|
-
}
|
|
307562
|
-
const exchange = input.exchange.trim().toLowerCase();
|
|
307563
|
-
const symbol2 = input.symbol.trim();
|
|
307564
|
-
if (!exchange || !symbol2) {
|
|
307565
|
-
throw new Error("exchange and symbol are required for market capture");
|
|
307566
|
-
}
|
|
307567
|
-
return {
|
|
307568
|
-
source: input.source,
|
|
307569
|
-
deploymentId,
|
|
307570
|
-
captureBundleId: configuredBundle ?? `development:${deploymentId}`,
|
|
307571
|
-
exchange,
|
|
307572
|
-
symbol: symbol2,
|
|
307573
|
-
assetType: input.assetType,
|
|
307574
|
-
feed: input.feed,
|
|
307575
|
-
provider: input.provider?.trim() || `ccxt:${exchange}`,
|
|
307576
|
-
sourceMode: input.sourceMode,
|
|
307577
|
-
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
307578
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
307579
|
-
provenanceComplete: true,
|
|
307580
|
-
timeframe: input.timeframe,
|
|
307581
|
-
accountSelector: input.accountSelector
|
|
307582
|
-
};
|
|
307583
|
-
}
|
|
307584
|
-
function captureEnvironmentFromEnv(value = process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT) {
|
|
307585
|
-
const environment = value?.trim() || "development";
|
|
307586
|
-
if (environment !== "development" && environment !== "production") {
|
|
307587
|
-
throw new Error("CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT must be development or production");
|
|
307588
|
-
}
|
|
307589
|
-
return environment;
|
|
307590
|
-
}
|
|
307591
|
-
|
|
307592
308839
|
// src/helpers/market-data-archive/ohlcv-bar-tracker.ts
|
|
307593
308840
|
function isFiniteNumber(value) {
|
|
307594
308841
|
return typeof value === "number" && Number.isFinite(value);
|
|
@@ -307723,39 +308970,6 @@ function getOrderbookArchiveDepthLimit() {
|
|
|
307723
308970
|
return Math.min(parsed, MAX_ORDERBOOK_ARCHIVE_DEPTH_LIMIT);
|
|
307724
308971
|
}
|
|
307725
308972
|
|
|
307726
|
-
// src/helpers/market-data-archive/orderbook-sampler.ts
|
|
307727
|
-
var DEFAULT_ORDERBOOK_INTERVAL_MS = 1000;
|
|
307728
|
-
function getOrderbookIntervalMs() {
|
|
307729
|
-
const raw = process.env.CEX_BROKER_ORDERBOOK_INTERVAL_MS ?? process.env.CEX_BROKER_ORDERBOOK_TOB_INTERVAL_MS;
|
|
307730
|
-
if (!raw) {
|
|
307731
|
-
return DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
307732
|
-
}
|
|
307733
|
-
const parsed = Number.parseInt(raw, 10);
|
|
307734
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
307735
|
-
}
|
|
307736
|
-
function isMarketArchiveEnabled() {
|
|
307737
|
-
return process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED !== "false";
|
|
307738
|
-
}
|
|
307739
|
-
|
|
307740
|
-
class OrderbookSampler {
|
|
307741
|
-
intervalMs;
|
|
307742
|
-
lastEmitMs = null;
|
|
307743
|
-
constructor(intervalMs = getOrderbookIntervalMs()) {
|
|
307744
|
-
this.intervalMs = intervalMs;
|
|
307745
|
-
}
|
|
307746
|
-
shouldEmit(nowMs = Date.now()) {
|
|
307747
|
-
if (this.lastEmitMs !== null && nowMs < this.lastEmitMs) {
|
|
307748
|
-
this.lastEmitMs = nowMs;
|
|
307749
|
-
return true;
|
|
307750
|
-
}
|
|
307751
|
-
if (this.lastEmitMs !== null && nowMs - this.lastEmitMs < this.intervalMs) {
|
|
307752
|
-
return false;
|
|
307753
|
-
}
|
|
307754
|
-
this.lastEmitMs = nowMs;
|
|
307755
|
-
return true;
|
|
307756
|
-
}
|
|
307757
|
-
}
|
|
307758
|
-
|
|
307759
308973
|
// src/helpers/market-data-archive/parse-stream.ts
|
|
307760
308974
|
function isFiniteNumber2(value) {
|
|
307761
308975
|
return typeof value === "number" && Number.isFinite(value);
|
|
@@ -307850,10 +309064,10 @@ function parseTicker(value, fallbackMs) {
|
|
|
307850
309064
|
["change", record2.change],
|
|
307851
309065
|
["percentage", record2.percentage]
|
|
307852
309066
|
];
|
|
307853
|
-
for (const [
|
|
309067
|
+
for (const [key2, rawValue] of fields) {
|
|
307854
309068
|
const numeric = toNumber2(rawValue);
|
|
307855
309069
|
if (numeric !== undefined) {
|
|
307856
|
-
parsed[
|
|
309070
|
+
parsed[key2] = numeric;
|
|
307857
309071
|
}
|
|
307858
309072
|
}
|
|
307859
309073
|
return parsed;
|
|
@@ -307870,9 +309084,19 @@ function withNormalizedChecksum(record2) {
|
|
|
307870
309084
|
normalized_row_checksum: sha256Canonical(compact)
|
|
307871
309085
|
};
|
|
307872
309086
|
}
|
|
309087
|
+
function legacyMarketFields(context2, rawCapture) {
|
|
309088
|
+
return {
|
|
309089
|
+
account_selector: context2.accountSelector,
|
|
309090
|
+
broker_observed_timestamp: new Date(rawCapture.receivedTimeMs).toISOString()
|
|
309091
|
+
};
|
|
309092
|
+
}
|
|
309093
|
+
function legacyDecimal8(value) {
|
|
309094
|
+
return value === undefined ? undefined : Number(value.toFixed(8));
|
|
309095
|
+
}
|
|
307873
309096
|
function buildCanonicalCexStreamEventRow(context2, rawCapture) {
|
|
307874
309097
|
const row = withNormalizedChecksum({
|
|
307875
309098
|
...captureCoreFields(context2, rawCapture),
|
|
309099
|
+
...legacyMarketFields(context2, rawCapture),
|
|
307876
309100
|
stream_type: context2.feed,
|
|
307877
309101
|
event_time_ms: rawCapture.eventTimeMs,
|
|
307878
309102
|
payload_encoding: "canonical_json_v1",
|
|
@@ -307886,19 +309110,21 @@ function buildCanonicalTickerEventRow(context2, rawCapture, ticker) {
|
|
|
307886
309110
|
}
|
|
307887
309111
|
const row = withNormalizedChecksum({
|
|
307888
309112
|
...captureCoreFields(context2, rawCapture),
|
|
309113
|
+
...legacyMarketFields(context2, rawCapture),
|
|
307889
309114
|
source_time_ms: ticker.eventTimeMs,
|
|
307890
309115
|
event_time_ms: ticker.eventTimeMs,
|
|
307891
|
-
last: ticker.last,
|
|
307892
|
-
bid: ticker.bid,
|
|
307893
|
-
ask: ticker.ask,
|
|
307894
|
-
high: ticker.high,
|
|
307895
|
-
low: ticker.low,
|
|
307896
|
-
open: ticker.open,
|
|
307897
|
-
close: ticker.close,
|
|
307898
|
-
base_volume: ticker.baseVolume,
|
|
307899
|
-
quote_volume: ticker.quoteVolume,
|
|
307900
|
-
change: ticker.change,
|
|
307901
|
-
percentage: ticker.percentage
|
|
309116
|
+
last: legacyDecimal8(ticker.last),
|
|
309117
|
+
bid: legacyDecimal8(ticker.bid),
|
|
309118
|
+
ask: legacyDecimal8(ticker.ask),
|
|
309119
|
+
high: legacyDecimal8(ticker.high),
|
|
309120
|
+
low: legacyDecimal8(ticker.low),
|
|
309121
|
+
open: legacyDecimal8(ticker.open),
|
|
309122
|
+
close: legacyDecimal8(ticker.close),
|
|
309123
|
+
base_volume: legacyDecimal8(ticker.baseVolume),
|
|
309124
|
+
quote_volume: legacyDecimal8(ticker.quoteVolume),
|
|
309125
|
+
change: legacyDecimal8(ticker.change),
|
|
309126
|
+
percentage: legacyDecimal8(ticker.percentage),
|
|
309127
|
+
payload_json: JSON.stringify(rawCapture.redactedPayload)
|
|
307902
309128
|
});
|
|
307903
309129
|
return { table: "market_data.cex_ticker_events", row };
|
|
307904
309130
|
}
|
|
@@ -307908,13 +309134,14 @@ function buildCanonicalTradeRow(context2, rawCapture, trade) {
|
|
|
307908
309134
|
}
|
|
307909
309135
|
const row = withNormalizedChecksum({
|
|
307910
309136
|
...captureCoreFields(context2, rawCapture),
|
|
309137
|
+
...legacyMarketFields(context2, rawCapture),
|
|
307911
309138
|
source_time_ms: trade.eventTimeMs,
|
|
307912
309139
|
trade_id: trade.tradeId,
|
|
307913
309140
|
event_time_ms: trade.eventTimeMs,
|
|
307914
309141
|
side: trade.side,
|
|
307915
|
-
price: trade.price,
|
|
307916
|
-
amount: trade.amount,
|
|
307917
|
-
cost: trade.cost,
|
|
309142
|
+
price: legacyDecimal8(trade.price),
|
|
309143
|
+
amount: legacyDecimal8(trade.amount),
|
|
309144
|
+
cost: legacyDecimal8(trade.cost),
|
|
307918
309145
|
taker_or_maker: trade.takerOrMaker
|
|
307919
309146
|
});
|
|
307920
309147
|
return { table: "market_data.cex_trades", row };
|
|
@@ -307994,10 +309221,19 @@ function resolveCaptureContext(archiver, input, feed, sourceMode) {
|
|
|
307994
309221
|
environment: captureEnvironmentFromEnv()
|
|
307995
309222
|
});
|
|
307996
309223
|
}
|
|
309224
|
+
function canArchiveMarketData(archiver) {
|
|
309225
|
+
return resolveMarketCaptureArchiveState({
|
|
309226
|
+
archiveEnabled: archiver?.isEnabled() ?? false,
|
|
309227
|
+
marketArchiveEnabled: isMarketArchiveEnabled(),
|
|
309228
|
+
environment: process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT,
|
|
309229
|
+
deploymentId: archiver?.getDeploymentId(),
|
|
309230
|
+
captureBundleId: process.env.CEX_BROKER_CAPTURE_BUNDLE_ID
|
|
309231
|
+
}).enabled;
|
|
309232
|
+
}
|
|
307997
309233
|
function archiveOrderbookInBackground(archiver, otelMetrics, input, options) {
|
|
307998
309234
|
const labels = watchLabels("orderbook", input, archiver, "ORDERBOOK");
|
|
307999
309235
|
recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
|
|
308000
|
-
if (!
|
|
309236
|
+
if (!canArchiveMarketData(archiver)) {
|
|
308001
309237
|
return;
|
|
308002
309238
|
}
|
|
308003
309239
|
if (options?.sampledOut) {
|
|
@@ -308039,7 +309275,7 @@ function archiveOrderbookInBackground(archiver, otelMetrics, input, options) {
|
|
|
308039
309275
|
function archiveOhlcvInBackground(archiver, otelMetrics, tracker, input) {
|
|
308040
309276
|
const labels = watchLabels("ohlcv", input, archiver, "OHLCV");
|
|
308041
309277
|
recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
|
|
308042
|
-
if (!
|
|
309278
|
+
if (!canArchiveMarketData(archiver)) {
|
|
308043
309279
|
return;
|
|
308044
309280
|
}
|
|
308045
309281
|
queueMicrotask(() => {
|
|
@@ -308084,7 +309320,7 @@ function createOhlcvBarTracker() {
|
|
|
308084
309320
|
function archiveMarketRowsInBackground(archiver, otelMetrics, stream4, input, feed, enqueueRows) {
|
|
308085
309321
|
const labels = watchLabels(stream4, input, archiver, feed);
|
|
308086
309322
|
recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
|
|
308087
|
-
if (!
|
|
309323
|
+
if (!canArchiveMarketData(archiver)) {
|
|
308088
309324
|
return;
|
|
308089
309325
|
}
|
|
308090
309326
|
queueMicrotask(() => {
|
|
@@ -309624,282 +310860,16 @@ class SubscribeBrokerLifecycle {
|
|
|
309624
310860
|
// src/handlers/subscribe/handler.ts
|
|
309625
310861
|
import * as grpc13 from "@grpc/grpc-js";
|
|
309626
310862
|
|
|
309627
|
-
// src/helpers/binance-user-data-stream.ts
|
|
309628
|
-
import { Buffer as Buffer2 } from "node:buffer";
|
|
309629
|
-
import { createHmac } from "node:crypto";
|
|
309630
|
-
var BINANCE_SPOT_WS_API_URL = "wss://ws-api.binance.com:443/ws-api/v3";
|
|
309631
|
-
var DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS = 16;
|
|
309632
|
-
var createWebSocket = (url3) => new wrapper_default(url3);
|
|
309633
|
-
var userDataRequestCounter = 0;
|
|
309634
|
-
function getExchangeString(exchange, key) {
|
|
309635
|
-
const value = exchange[key];
|
|
309636
|
-
if (typeof value !== "string" || value.length === 0) {
|
|
309637
|
-
throw new Error(`Binance user-data stream requires exchange.${key}`);
|
|
309638
|
-
}
|
|
309639
|
-
return value;
|
|
309640
|
-
}
|
|
309641
|
-
function sortedQuery(params) {
|
|
309642
|
-
return Object.entries(params).sort(([left], [right]) => left.localeCompare(right)).map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`).join("&");
|
|
309643
|
-
}
|
|
309644
|
-
function signUserDataStreamParams(exchange, params) {
|
|
309645
|
-
const signParams = exchange.signParams;
|
|
309646
|
-
if (typeof signParams === "function") {
|
|
309647
|
-
return signParams.call(exchange, params);
|
|
309648
|
-
}
|
|
309649
|
-
const secret = getExchangeString(exchange, "secret");
|
|
309650
|
-
return {
|
|
309651
|
-
...params,
|
|
309652
|
-
signature: createHmac("sha256", secret).update(sortedQuery(params)).digest("hex")
|
|
309653
|
-
};
|
|
309654
|
-
}
|
|
309655
|
-
function getBinanceSpotWsApiUrl(exchange) {
|
|
309656
|
-
const urls = exchange.urls;
|
|
309657
|
-
return urls?.api?.ws?.["ws-api"]?.spot ?? BINANCE_SPOT_WS_API_URL;
|
|
309658
|
-
}
|
|
309659
|
-
function getRecord(value) {
|
|
309660
|
-
return typeof value === "object" && value !== null ? value : null;
|
|
309661
|
-
}
|
|
309662
|
-
function getMessage(value) {
|
|
309663
|
-
if (value instanceof Error) {
|
|
309664
|
-
return value.message;
|
|
309665
|
-
}
|
|
309666
|
-
if (typeof value === "string" && value.length > 0) {
|
|
309667
|
-
return value;
|
|
309668
|
-
}
|
|
309669
|
-
const record2 = getRecord(value);
|
|
309670
|
-
const message = record2?.message;
|
|
309671
|
-
return typeof message === "string" && message.length > 0 ? message : null;
|
|
309672
|
-
}
|
|
309673
|
-
function getOptionalExchangeString(exchange, key) {
|
|
309674
|
-
const value = exchange[key];
|
|
309675
|
-
return typeof value === "string" && value.length > 0 ? value : null;
|
|
309676
|
-
}
|
|
309677
|
-
function redactDiagnosticMessage(message, secretValues) {
|
|
309678
|
-
let redacted = message;
|
|
309679
|
-
for (const value of secretValues) {
|
|
309680
|
-
if (value.length > 0) {
|
|
309681
|
-
redacted = redacted.split(value).join("[redacted]");
|
|
309682
|
-
}
|
|
309683
|
-
}
|
|
309684
|
-
return redacted.replace(/(\b(?:apiKey|secret|signature)\b\s*=\s*)[^\s&,;)]+/gi, "$1[redacted]").replace(/("(?:apiKey|secret|signature)"\s*:\s*")[^"]*(")/gi, "$1[redacted]$2");
|
|
309685
|
-
}
|
|
309686
|
-
function formatBinanceUserDataWebSocketError(event, secretValues) {
|
|
309687
|
-
const record2 = getRecord(event);
|
|
309688
|
-
const message = getMessage(record2?.error) ?? getMessage(record2?.message) ?? getMessage(event);
|
|
309689
|
-
const safeMessage = message === null ? null : redactDiagnosticMessage(message, secretValues);
|
|
309690
|
-
return new Error(safeMessage ? `Binance user-data WebSocket error: ${safeMessage}` : "Binance user-data WebSocket error");
|
|
309691
|
-
}
|
|
309692
|
-
function getCloseReason(value) {
|
|
309693
|
-
if (typeof value === "string") {
|
|
309694
|
-
return value.length > 0 ? value : null;
|
|
309695
|
-
}
|
|
309696
|
-
if (Buffer2.isBuffer(value)) {
|
|
309697
|
-
const reason = value.toString("utf8");
|
|
309698
|
-
return reason.length > 0 ? reason : null;
|
|
309699
|
-
}
|
|
309700
|
-
if (value instanceof Uint8Array) {
|
|
309701
|
-
const reason = Buffer2.from(value).toString("utf8");
|
|
309702
|
-
return reason.length > 0 ? reason : null;
|
|
309703
|
-
}
|
|
309704
|
-
return null;
|
|
309705
|
-
}
|
|
309706
|
-
function formatBinanceUserDataWebSocketClose(codeOrEvent, reasonOrUndefined, secretValues) {
|
|
309707
|
-
const record2 = getRecord(codeOrEvent);
|
|
309708
|
-
const code = record2 ? record2.code : codeOrEvent;
|
|
309709
|
-
const reason = getCloseReason(record2 ? record2.reason : reasonOrUndefined);
|
|
309710
|
-
const safeReason = reason === null ? null : redactDiagnosticMessage(reason, secretValues);
|
|
309711
|
-
const details = [
|
|
309712
|
-
typeof code === "number" || typeof code === "string" ? `code=${code}` : null,
|
|
309713
|
-
safeReason ? `reason=${safeReason}` : null
|
|
309714
|
-
].filter((detail) => detail !== null);
|
|
309715
|
-
return new Error(details.length > 0 ? `Binance user-data WebSocket closed unexpectedly (${details.join(", ")})` : "Binance user-data WebSocket closed unexpectedly");
|
|
309716
|
-
}
|
|
309717
|
-
function decodeMessageData(data) {
|
|
309718
|
-
if (typeof data === "string") {
|
|
309719
|
-
return data;
|
|
309720
|
-
}
|
|
309721
|
-
if (Buffer2.isBuffer(data)) {
|
|
309722
|
-
return data.toString("utf8");
|
|
309723
|
-
}
|
|
309724
|
-
if (data instanceof ArrayBuffer) {
|
|
309725
|
-
return Buffer2.from(data).toString("utf8");
|
|
309726
|
-
}
|
|
309727
|
-
if (ArrayBuffer.isView(data)) {
|
|
309728
|
-
return Buffer2.from(data.buffer, data.byteOffset, data.byteLength).toString("utf8");
|
|
309729
|
-
}
|
|
309730
|
-
if (Array.isArray(data) && data.every((item) => Buffer2.isBuffer(item))) {
|
|
309731
|
-
return Buffer2.concat(data).toString("utf8");
|
|
309732
|
-
}
|
|
309733
|
-
return data;
|
|
309734
|
-
}
|
|
309735
|
-
|
|
309736
|
-
class BinanceSpotUserDataStream {
|
|
309737
|
-
exchange;
|
|
309738
|
-
ws;
|
|
309739
|
-
secretValues;
|
|
309740
|
-
requestId = `user-data-${Date.now()}-${userDataRequestCounter++}`;
|
|
309741
|
-
maxBufferedEvents;
|
|
309742
|
-
queue = [];
|
|
309743
|
-
waiters = [];
|
|
309744
|
-
closed = false;
|
|
309745
|
-
closeError = null;
|
|
309746
|
-
subscriptionId = null;
|
|
309747
|
-
constructor(exchange, options = {}) {
|
|
309748
|
-
this.exchange = exchange;
|
|
309749
|
-
this.maxBufferedEvents = options.maxBufferedEvents ?? DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS;
|
|
309750
|
-
this.secretValues = [
|
|
309751
|
-
getOptionalExchangeString(exchange, "apiKey"),
|
|
309752
|
-
getOptionalExchangeString(exchange, "secret")
|
|
309753
|
-
].filter((value) => value !== null);
|
|
309754
|
-
this.ws = createWebSocket(getBinanceSpotWsApiUrl(exchange));
|
|
309755
|
-
this.ws.on("open", () => this.subscribe());
|
|
309756
|
-
this.ws.on("message", (data) => this.handleMessage(data));
|
|
309757
|
-
this.ws.on("error", (error48) => this.fail(formatBinanceUserDataWebSocketError(error48, this.secretValues)));
|
|
309758
|
-
this.ws.on("close", (code, reason) => this.handleClose(code, reason));
|
|
309759
|
-
}
|
|
309760
|
-
async* [Symbol.asyncIterator]() {
|
|
309761
|
-
while (true) {
|
|
309762
|
-
const event = await this.nextEvent();
|
|
309763
|
-
if (!event) {
|
|
309764
|
-
break;
|
|
309765
|
-
}
|
|
309766
|
-
yield event;
|
|
309767
|
-
}
|
|
309768
|
-
}
|
|
309769
|
-
close() {
|
|
309770
|
-
if (this.closed) {
|
|
309771
|
-
return;
|
|
309772
|
-
}
|
|
309773
|
-
this.closed = true;
|
|
309774
|
-
this.queue.length = 0;
|
|
309775
|
-
try {
|
|
309776
|
-
this.ws.close();
|
|
309777
|
-
} catch {}
|
|
309778
|
-
this.flushWaiters();
|
|
309779
|
-
}
|
|
309780
|
-
handleClose(code, reason) {
|
|
309781
|
-
if (this.closed) {
|
|
309782
|
-
return;
|
|
309783
|
-
}
|
|
309784
|
-
this.fail(formatBinanceUserDataWebSocketClose(code, reason, this.secretValues));
|
|
309785
|
-
}
|
|
309786
|
-
subscribe() {
|
|
309787
|
-
const apiKey = getExchangeString(this.exchange, "apiKey");
|
|
309788
|
-
const signedParams = signUserDataStreamParams(this.exchange, {
|
|
309789
|
-
apiKey,
|
|
309790
|
-
timestamp: Date.now()
|
|
309791
|
-
});
|
|
309792
|
-
this.ws.send(JSON.stringify({
|
|
309793
|
-
id: this.requestId,
|
|
309794
|
-
method: "userDataStream.subscribe.signature",
|
|
309795
|
-
params: signedParams
|
|
309796
|
-
}));
|
|
309797
|
-
}
|
|
309798
|
-
handleMessage(data) {
|
|
309799
|
-
if (this.closed) {
|
|
309800
|
-
return;
|
|
309801
|
-
}
|
|
309802
|
-
let message;
|
|
309803
|
-
try {
|
|
309804
|
-
const decodedData = decodeMessageData(data);
|
|
309805
|
-
message = typeof decodedData === "string" ? JSON.parse(decodedData) : decodedData;
|
|
309806
|
-
} catch (error48) {
|
|
309807
|
-
this.fail(error48 instanceof Error ? error48 : new Error("Invalid Binance user-data message"));
|
|
309808
|
-
return;
|
|
309809
|
-
}
|
|
309810
|
-
if ("id" in message && message.id === this.requestId) {
|
|
309811
|
-
if (message.status !== 200) {
|
|
309812
|
-
this.fail(new Error(message.error?.msg ?? message.error?.message ?? `Binance user-data subscription failed with status ${message.status}`));
|
|
309813
|
-
return;
|
|
309814
|
-
}
|
|
309815
|
-
this.subscriptionId = message.result?.subscriptionId ?? null;
|
|
309816
|
-
return;
|
|
309817
|
-
}
|
|
309818
|
-
if ("status" in message && typeof message.status === "number" && message.status !== 200) {
|
|
309819
|
-
const errorMessage = message.error?.msg ?? message.error?.message ?? `Binance user-data request failed with status ${message.status}`;
|
|
309820
|
-
const errorCode2 = message.error?.code;
|
|
309821
|
-
this.fail(new Error(typeof errorCode2 === "number" ? `${errorMessage} (code ${errorCode2})` : errorMessage));
|
|
309822
|
-
return;
|
|
309823
|
-
}
|
|
309824
|
-
if (!("event" in message) || !message.event) {
|
|
309825
|
-
return;
|
|
309826
|
-
}
|
|
309827
|
-
const subscriptionId = message.subscriptionId ?? this.subscriptionId;
|
|
309828
|
-
if (subscriptionId === null || subscriptionId === undefined) {
|
|
309829
|
-
return;
|
|
309830
|
-
}
|
|
309831
|
-
this.push({ subscriptionId, event: message.event });
|
|
309832
|
-
}
|
|
309833
|
-
push(event) {
|
|
309834
|
-
if (this.closed) {
|
|
309835
|
-
return;
|
|
309836
|
-
}
|
|
309837
|
-
const waiter = this.waiters.shift();
|
|
309838
|
-
if (waiter) {
|
|
309839
|
-
waiter.resolve(event);
|
|
309840
|
-
return;
|
|
309841
|
-
}
|
|
309842
|
-
if (this.queue.length >= this.maxBufferedEvents) {
|
|
309843
|
-
this.fail(new Error(`Binance user-data stream buffered event limit exceeded (${this.maxBufferedEvents}); downstream consumer is not keeping up`));
|
|
309844
|
-
return;
|
|
309845
|
-
}
|
|
309846
|
-
this.queue.push(event);
|
|
309847
|
-
}
|
|
309848
|
-
nextEvent() {
|
|
309849
|
-
const event = this.queue.shift();
|
|
309850
|
-
if (event) {
|
|
309851
|
-
return Promise.resolve(event);
|
|
309852
|
-
}
|
|
309853
|
-
if (this.closeError) {
|
|
309854
|
-
return Promise.reject(this.closeError);
|
|
309855
|
-
}
|
|
309856
|
-
if (this.closed) {
|
|
309857
|
-
return Promise.resolve(null);
|
|
309858
|
-
}
|
|
309859
|
-
return new Promise((resolve, reject) => {
|
|
309860
|
-
this.waiters.push({ resolve, reject });
|
|
309861
|
-
});
|
|
309862
|
-
}
|
|
309863
|
-
fail(error48) {
|
|
309864
|
-
if (this.closeError) {
|
|
309865
|
-
return;
|
|
309866
|
-
}
|
|
309867
|
-
this.closeError = error48;
|
|
309868
|
-
this.closed = true;
|
|
309869
|
-
this.queue.length = 0;
|
|
309870
|
-
this.flushWaiters();
|
|
309871
|
-
try {
|
|
309872
|
-
this.ws.close();
|
|
309873
|
-
} catch {}
|
|
309874
|
-
}
|
|
309875
|
-
flushWaiters() {
|
|
309876
|
-
const error48 = this.closeError;
|
|
309877
|
-
for (const waiter of this.waiters.splice(0)) {
|
|
309878
|
-
if (error48) {
|
|
309879
|
-
waiter.reject(error48);
|
|
309880
|
-
} else {
|
|
309881
|
-
waiter.resolve(null);
|
|
309882
|
-
}
|
|
309883
|
-
}
|
|
309884
|
-
}
|
|
309885
|
-
}
|
|
309886
|
-
function isBinanceBalanceUserDataEvent(event) {
|
|
309887
|
-
return event.e === "outboundAccountPosition" || event.e === "balanceUpdate" || event.e === "externalLockUpdate";
|
|
309888
|
-
}
|
|
309889
|
-
function isBinanceOrderUserDataEvent(event) {
|
|
309890
|
-
return event.e === "executionReport" || event.e === "listStatus";
|
|
309891
|
-
}
|
|
309892
|
-
|
|
309893
310863
|
// src/helpers/binance-user-data-normalization.ts
|
|
309894
|
-
function requireQuantity(entry,
|
|
309895
|
-
const value = entry[
|
|
310864
|
+
function requireQuantity(entry, key2) {
|
|
310865
|
+
const value = entry[key2];
|
|
309896
310866
|
if (typeof value === "string" && value.trim().length > 0) {
|
|
309897
310867
|
return value;
|
|
309898
310868
|
}
|
|
309899
310869
|
if (typeof value === "number" && Number.isFinite(value)) {
|
|
309900
310870
|
return String(value);
|
|
309901
310871
|
}
|
|
309902
|
-
throw new Error(`Invalid Binance balance quantity: ${
|
|
310872
|
+
throw new Error(`Invalid Binance balance quantity: ${key2}`);
|
|
309903
310873
|
}
|
|
309904
310874
|
async function normalizeBinanceSpotBalanceEvent(exchange, event) {
|
|
309905
310875
|
if (event.e !== "outboundAccountPosition") {
|
|
@@ -310065,12 +311035,14 @@ async function getBinanceMarketId(broker, symbol2) {
|
|
|
310065
311035
|
}
|
|
310066
311036
|
return symbol2.replace("/", "").toUpperCase();
|
|
310067
311037
|
}
|
|
310068
|
-
async function streamBinanceUserData(call, broker, symbol2, subscriptionType, isClosed, archiveContext) {
|
|
310069
|
-
const userDataStream = new BinanceSpotUserDataStream(broker);
|
|
310070
|
-
|
|
310071
|
-
|
|
310072
|
-
|
|
310073
|
-
|
|
311038
|
+
async function streamBinanceUserData(call, broker, symbol2, subscriptionType, isClosed, archiveContext, userDataSource, knownMarketId) {
|
|
311039
|
+
const userDataStream = userDataSource ?? new BinanceSpotUserDataStream(broker);
|
|
311040
|
+
if (!userDataSource) {
|
|
311041
|
+
call.once("close", () => userDataStream.close());
|
|
311042
|
+
call.once("cancelled", () => userDataStream.close());
|
|
311043
|
+
call.once("error", () => userDataStream.close());
|
|
311044
|
+
}
|
|
311045
|
+
const marketId = knownMarketId ?? (subscriptionType === SubscriptionType.ORDERS ? await getBinanceMarketId(broker, symbol2) : null);
|
|
310074
311046
|
try {
|
|
310075
311047
|
for await (const message of userDataStream) {
|
|
310076
311048
|
if (isClosed()) {
|
|
@@ -310162,7 +311134,13 @@ async function runCcxtSubscribeLoop(call, isClosed, symbol2, subscriptionType, w
|
|
|
310162
311134
|
}
|
|
310163
311135
|
}
|
|
310164
311136
|
function createSubscribeHandler(deps) {
|
|
310165
|
-
const {
|
|
311137
|
+
const {
|
|
311138
|
+
brokers,
|
|
311139
|
+
whitelistIps,
|
|
311140
|
+
otelMetrics,
|
|
311141
|
+
brokerArchiver,
|
|
311142
|
+
userDataStreamSupervisor
|
|
311143
|
+
} = deps;
|
|
310166
311144
|
const brokerLifecycle = deps.brokerLifecycle ?? new SubscribeBrokerLifecycle;
|
|
310167
311145
|
return async (call) => {
|
|
310168
311146
|
const subscribeStartTime = Date.now();
|
|
@@ -310297,7 +311275,28 @@ function createSubscribeHandler(deps) {
|
|
|
310297
311275
|
});
|
|
310298
311276
|
return;
|
|
310299
311277
|
}
|
|
310300
|
-
await
|
|
311278
|
+
const marketId = subscriptionType === SubscriptionType.ORDERS ? await getBinanceMarketId(accountBroker, resolvedSymbol) : undefined;
|
|
311279
|
+
let userDataSource;
|
|
311280
|
+
if (selectedBrokerAccount) {
|
|
311281
|
+
if (!userDataStreamSupervisor) {
|
|
311282
|
+
await writeSubscribeError(call, isStreamClosed, {
|
|
311283
|
+
data: JSON.stringify({
|
|
311284
|
+
error: "Configured account user-data supervisor is unavailable"
|
|
311285
|
+
}),
|
|
311286
|
+
timestamp: Date.now(),
|
|
311287
|
+
symbol: resolvedSymbol,
|
|
311288
|
+
type: subscriptionType
|
|
311289
|
+
});
|
|
311290
|
+
return;
|
|
311291
|
+
}
|
|
311292
|
+
userDataSource = userDataStreamSupervisor.subscribe({
|
|
311293
|
+
exchange: normalizedCex,
|
|
311294
|
+
accountSelector: selectedBrokerAccount.label,
|
|
311295
|
+
kind: subscriptionType === SubscriptionType.BALANCE ? "balance" : "orders",
|
|
311296
|
+
marketId
|
|
311297
|
+
});
|
|
311298
|
+
}
|
|
311299
|
+
await streamBinanceUserData(call, accountBroker, resolvedSymbol, subscriptionType, isStreamClosed, streamArchiveContext, userDataSource, marketId);
|
|
310301
311300
|
return;
|
|
310302
311301
|
}
|
|
310303
311302
|
switch (subscriptionType) {
|
|
@@ -310691,7 +311690,7 @@ var CEX_BROKER_PACKAGE_DEFINITION = protoLoader.fromJSON(node_descriptor_default
|
|
|
310691
311690
|
// src/server.ts
|
|
310692
311691
|
var grpcObj = grpc14.loadPackageDefinition(CEX_BROKER_PACKAGE_DEFINITION);
|
|
310693
311692
|
var cexNode = grpcObj.cex_broker;
|
|
310694
|
-
function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker, subscribeBrokerLifecycle) {
|
|
311693
|
+
function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker, subscribeBrokerLifecycle, userDataStreamSupervisor) {
|
|
310695
311694
|
const server = new grpc14.Server;
|
|
310696
311695
|
server.addService(cexNode.cex_service.service, {
|
|
310697
311696
|
ExecuteAction: createExecuteActionHandler({
|
|
@@ -310710,7 +311709,8 @@ function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, ot
|
|
|
310710
311709
|
whitelistIps,
|
|
310711
311710
|
otelMetrics,
|
|
310712
311711
|
brokerArchiver,
|
|
310713
|
-
brokerLifecycle: subscribeBrokerLifecycle
|
|
311712
|
+
brokerLifecycle: subscribeBrokerLifecycle,
|
|
311713
|
+
userDataStreamSupervisor
|
|
310714
311714
|
})
|
|
310715
311715
|
});
|
|
310716
311716
|
return server;
|
|
@@ -310854,13 +311854,14 @@ class CEXBroker {
|
|
|
310854
311854
|
fillArchivePoller;
|
|
310855
311855
|
depositArchivePoller;
|
|
310856
311856
|
accountBalanceArchivePoller;
|
|
311857
|
+
userDataStreamSupervisor;
|
|
310857
311858
|
loadEnvConfig() {
|
|
310858
311859
|
log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
|
|
310859
311860
|
const configMap = {};
|
|
310860
|
-
for (const [
|
|
310861
|
-
if (!
|
|
311861
|
+
for (const [key2, value] of Object.entries(process.env)) {
|
|
311862
|
+
if (!key2.startsWith("CEX_BROKER_"))
|
|
310862
311863
|
continue;
|
|
310863
|
-
let match =
|
|
311864
|
+
let match = key2.match(/^CEX_BROKER_(\w+)_(API_(KEY|SECRET)|ROLE|EMAIL|SUBACCOUNTID|UID)_(\d+)$/);
|
|
310864
311865
|
if (match) {
|
|
310865
311866
|
const broker2 = match[1]?.toLowerCase() ?? "";
|
|
310866
311867
|
const type3 = match[2]?.toLowerCase() ?? "";
|
|
@@ -310889,9 +311890,9 @@ class CEXBroker {
|
|
|
310889
311890
|
}
|
|
310890
311891
|
continue;
|
|
310891
311892
|
}
|
|
310892
|
-
match =
|
|
311893
|
+
match = key2.match(/^CEX_BROKER_(\w+)_(API_(KEY|SECRET)|ROLE|EMAIL|SUBACCOUNTID|UID)$/);
|
|
310893
311894
|
if (!match) {
|
|
310894
|
-
log.warn(`⚠️ Skipping unrecognized env var: ${
|
|
311895
|
+
log.warn(`⚠️ Skipping unrecognized env var: ${key2}`);
|
|
310895
311896
|
continue;
|
|
310896
311897
|
}
|
|
310897
311898
|
const broker = match[1]?.toLowerCase() ?? "";
|
|
@@ -311009,6 +312010,10 @@ class CEXBroker {
|
|
|
311009
312010
|
if (this.server) {
|
|
311010
312011
|
await this.server.forceShutdown();
|
|
311011
312012
|
}
|
|
312013
|
+
if (this.userDataStreamSupervisor) {
|
|
312014
|
+
await this.userDataStreamSupervisor.close();
|
|
312015
|
+
this.userDataStreamSupervisor = undefined;
|
|
312016
|
+
}
|
|
311012
312017
|
if (this.brokerArchiver) {
|
|
311013
312018
|
await this.brokerArchiver.close();
|
|
311014
312019
|
}
|
|
@@ -311020,6 +312025,14 @@ class CEXBroker {
|
|
|
311020
312025
|
}
|
|
311021
312026
|
}
|
|
311022
312027
|
async run() {
|
|
312028
|
+
const marketArchiveState = resolveMarketCaptureArchiveState({
|
|
312029
|
+
archiveEnabled: this.brokerArchiver?.isEnabled() ?? false,
|
|
312030
|
+
marketArchiveEnabled: isMarketArchiveEnabled(),
|
|
312031
|
+
environment: process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT,
|
|
312032
|
+
deploymentId: this.brokerArchiver?.getDeploymentId(),
|
|
312033
|
+
captureBundleId: process.env.CEX_BROKER_CAPTURE_BUNDLE_ID
|
|
312034
|
+
});
|
|
312035
|
+
assertMarketCaptureArchiveStartable(marketArchiveState);
|
|
311023
312036
|
if (this.server) {
|
|
311024
312037
|
await this.server.forceShutdown();
|
|
311025
312038
|
}
|
|
@@ -311043,7 +312056,15 @@ class CEXBroker {
|
|
|
311043
312056
|
if (this.otelMetrics?.isOtelEnabled()) {
|
|
311044
312057
|
await this.otelMetrics.initialize();
|
|
311045
312058
|
}
|
|
311046
|
-
|
|
312059
|
+
if (!this.userDataStreamSupervisor && Object.keys(this.brokers).length > 0) {
|
|
312060
|
+
const publisher = new StreamHealthPublisher(streamHealthPublisherConfigFromEnv());
|
|
312061
|
+
this.userDataStreamSupervisor = new UserDataStreamSupervisor({
|
|
312062
|
+
brokers: this.brokers,
|
|
312063
|
+
publisher
|
|
312064
|
+
});
|
|
312065
|
+
this.userDataStreamSupervisor.start();
|
|
312066
|
+
}
|
|
312067
|
+
this.server = getServer(this.policy, this.brokers, this.whitelistIps, this.useVerity, this.#verityProverUrl, this.otelMetrics, this.brokerArchiver, this.orderActivityTracker, this.withdrawalObservationTracker, undefined, this.userDataStreamSupervisor);
|
|
311047
312068
|
this.server.bindAsync(`0.0.0.0:${this.port}`, grpc15.ServerCredentials.createInsecure(), (err2, port) => {
|
|
311048
312069
|
if (err2) {
|
|
311049
312070
|
log.error(err2);
|
|
@@ -311088,4 +312109,4 @@ export {
|
|
|
311088
312109
|
CEXBroker as default
|
|
311089
312110
|
};
|
|
311090
312111
|
|
|
311091
|
-
//# debugId=
|
|
312112
|
+
//# debugId=14D3F8955E8B1FB864756E2164756E21
|