@usherlabs/cex-broker 0.2.37 → 0.2.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/commands/cli.js +2088 -1259
- package/dist/handlers/subscribe/handler.d.ts +2 -0
- package/dist/helpers/binance-user-data-normalization.d.ts +3 -0
- package/dist/helpers/binance-user-data-stream.d.ts +12 -0
- package/dist/helpers/broker-execution-archive/index.d.ts +1 -1
- package/dist/helpers/broker-execution-archive/rows.d.ts +1 -0
- package/dist/helpers/deposit-archive-poller.d.ts +1 -0
- package/dist/helpers/market-data-archive/capture-context.d.ts +24 -3
- package/dist/helpers/market-data-archive/index.d.ts +1 -1
- package/dist/helpers/stream-health-publisher.d.ts +41 -0
- package/dist/helpers/user-data-stream-supervisor.d.ts +27 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2090 -1261
- package/dist/index.js.map +18 -15
- package/dist/server.d.ts +2 -1
- package/package.json +10 -2
package/dist/index.js
CHANGED
|
@@ -292137,9 +292137,18 @@ class AccountBalanceArchivePoller {
|
|
|
292137
292137
|
// src/helpers/deposit-archive-poller.ts
|
|
292138
292138
|
var DEFAULT_CONFIG2 = {
|
|
292139
292139
|
pollIntervalMs: 60000,
|
|
292140
|
+
fetchTimeoutMs: 30000,
|
|
292140
292141
|
lookbackMs: 24 * 60 * 60 * 1000,
|
|
292141
292142
|
depositsLimit: 50
|
|
292142
292143
|
};
|
|
292144
|
+
function withTimeout(promise, timeoutMs, label) {
|
|
292145
|
+
let timer;
|
|
292146
|
+
const expiry = new Promise((_resolve, reject) => {
|
|
292147
|
+
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
292148
|
+
timer.unref?.();
|
|
292149
|
+
});
|
|
292150
|
+
return Promise.race([promise, expiry]).finally(() => clearTimeout(timer));
|
|
292151
|
+
}
|
|
292143
292152
|
var ALL_CURRENCIES_CODE = "*";
|
|
292144
292153
|
function depositTimestamp(record) {
|
|
292145
292154
|
const observedAt = depositField(record, [
|
|
@@ -292252,6 +292261,14 @@ class DepositArchivePoller {
|
|
|
292252
292261
|
return true;
|
|
292253
292262
|
}
|
|
292254
292263
|
async#pollOne(target) {
|
|
292264
|
+
let outcome = "error";
|
|
292265
|
+
try {
|
|
292266
|
+
outcome = await this.#pollTarget(target);
|
|
292267
|
+
} finally {
|
|
292268
|
+
this.params.metrics?.recordCounter("cex_deposit_poller_polls_total", 1, { exchange: target.exchangeId, outcome });
|
|
292269
|
+
}
|
|
292270
|
+
}
|
|
292271
|
+
async#pollTarget(target) {
|
|
292255
292272
|
const exchange = target.account.exchange;
|
|
292256
292273
|
const key = this.#targetKey(target);
|
|
292257
292274
|
if (typeof exchange.fetchDeposits !== "function" || exchange.has?.fetchDeposits === false) {
|
|
@@ -292262,12 +292279,12 @@ class DepositArchivePoller {
|
|
|
292262
292279
|
account: target.account.label
|
|
292263
292280
|
});
|
|
292264
292281
|
}
|
|
292265
|
-
return;
|
|
292282
|
+
return "unsupported";
|
|
292266
292283
|
}
|
|
292267
292284
|
const since = this.#cursors.get(key) ?? Date.now() - this.#config.lookbackMs;
|
|
292268
292285
|
let deposits;
|
|
292269
292286
|
try {
|
|
292270
|
-
deposits = await exchange.fetchDeposits(undefined, since, this.#config.depositsLimit);
|
|
292287
|
+
deposits = await withTimeout(exchange.fetchDeposits(undefined, since, this.#config.depositsLimit), this.#config.fetchTimeoutMs, "fetchDeposits");
|
|
292271
292288
|
} catch (error) {
|
|
292272
292289
|
this.params.metrics?.recordCounter("cex_deposit_poller_errors_total", 1, { exchange: target.exchangeId });
|
|
292273
292290
|
log.warn("Deposit archive poll failed", {
|
|
@@ -292275,10 +292292,10 @@ class DepositArchivePoller {
|
|
|
292275
292292
|
account: target.account.label,
|
|
292276
292293
|
error
|
|
292277
292294
|
});
|
|
292278
|
-
return;
|
|
292295
|
+
return "error";
|
|
292279
292296
|
}
|
|
292280
292297
|
if (!Array.isArray(deposits) || deposits.length === 0) {
|
|
292281
|
-
return;
|
|
292298
|
+
return "ok";
|
|
292282
292299
|
}
|
|
292283
292300
|
let archived = 0;
|
|
292284
292301
|
for (const deposit of deposits) {
|
|
@@ -292326,7 +292343,7 @@ class DepositArchivePoller {
|
|
|
292326
292343
|
network: network === undefined ? undefined : String(network),
|
|
292327
292344
|
externalId: depositTxid,
|
|
292328
292345
|
txid: depositTxid,
|
|
292329
|
-
exchangeTimestamp:
|
|
292346
|
+
exchangeTimestamp: normalizeTimestamp2(creditedAt),
|
|
292330
292347
|
payload: record
|
|
292331
292348
|
}
|
|
292332
292349
|
}));
|
|
@@ -292359,6 +292376,7 @@ class DepositArchivePoller {
|
|
|
292359
292376
|
this.#lastArchivedByTarget.delete(key);
|
|
292360
292377
|
}
|
|
292361
292378
|
}
|
|
292379
|
+
return "ok";
|
|
292362
292380
|
}
|
|
292363
292381
|
#targetKey(target) {
|
|
292364
292382
|
return `${target.exchangeId}|${target.account.label}|${target.code}`;
|
|
@@ -292500,387 +292518,1609 @@ class FillArchivePoller {
|
|
|
292500
292518
|
}
|
|
292501
292519
|
}
|
|
292502
292520
|
|
|
292503
|
-
// src/helpers/
|
|
292504
|
-
|
|
292505
|
-
|
|
292506
|
-
|
|
292507
|
-
|
|
292508
|
-
|
|
292509
|
-
|
|
292510
|
-
|
|
292521
|
+
// src/helpers/market-data-archive/capture-contract.ts
|
|
292522
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
292523
|
+
var MARKET_CAPTURE_SCHEMA_VERSION = "1.0.0";
|
|
292524
|
+
var CHECKSUM_ALGORITHM = "sha256-canonical-json-v1";
|
|
292525
|
+
var ARCHIVE_SOURCES = ["broker_read", "broker_write"];
|
|
292526
|
+
var CAPTURE_FEEDS = [
|
|
292527
|
+
"ORDERBOOK",
|
|
292528
|
+
"TICKER",
|
|
292529
|
+
"TRADES",
|
|
292530
|
+
"OHLCV"
|
|
292531
|
+
];
|
|
292532
|
+
var SOURCE_MODES = [
|
|
292533
|
+
"broker_live_stream_v1",
|
|
292534
|
+
"broker_live_sampling_v1",
|
|
292535
|
+
"broker_current_snapshot_v1",
|
|
292536
|
+
"broker_bootstrap_fetch_v1",
|
|
292537
|
+
"external_ccxt_fallback_v1",
|
|
292538
|
+
"external_hummingbot_fallback_v1",
|
|
292539
|
+
"legacy_migration_v1"
|
|
292540
|
+
];
|
|
292541
|
+
var RAW_CAPTURE_SCOPES = [
|
|
292542
|
+
"ccxt_normalized_object",
|
|
292543
|
+
"broker_visible_payload",
|
|
292544
|
+
"exchange_wire_frame"
|
|
292545
|
+
];
|
|
292546
|
+
var CHECKSUM_FIELDS = new Set([
|
|
292547
|
+
"normalized_row_checksum",
|
|
292548
|
+
"raw_checksum",
|
|
292549
|
+
"checksum"
|
|
292550
|
+
]);
|
|
292551
|
+
function canonicalDecimal(value) {
|
|
292552
|
+
if (!Number.isFinite(value)) {
|
|
292553
|
+
throw new Error("Canonical numbers must be finite");
|
|
292511
292554
|
}
|
|
292512
|
-
|
|
292513
|
-
|
|
292514
|
-
const trimmedSymbol = symbol.trim();
|
|
292515
|
-
if (!exchange || !accountLabel.trim() || !trimmedSymbol) {
|
|
292516
|
-
return;
|
|
292517
|
-
}
|
|
292518
|
-
const key = `${exchange}|${accountLabel}|${trimmedSymbol}`;
|
|
292519
|
-
this.#entries.set(key, {
|
|
292520
|
-
exchangeId: exchange,
|
|
292521
|
-
accountLabel,
|
|
292522
|
-
symbol: trimmedSymbol,
|
|
292523
|
-
lastActivityAt: now3
|
|
292524
|
-
});
|
|
292555
|
+
if (Object.is(value, -0)) {
|
|
292556
|
+
return "0";
|
|
292525
292557
|
}
|
|
292526
|
-
|
|
292527
|
-
|
|
292528
|
-
|
|
292529
|
-
if (now3 - entry.lastActivityAt > this.#maxAgeMs) {
|
|
292530
|
-
this.#entries.delete(key);
|
|
292531
|
-
continue;
|
|
292532
|
-
}
|
|
292533
|
-
active.push(entry);
|
|
292534
|
-
}
|
|
292535
|
-
return active;
|
|
292558
|
+
const rendered = String(value).toLowerCase();
|
|
292559
|
+
if (!rendered.includes("e")) {
|
|
292560
|
+
return rendered;
|
|
292536
292561
|
}
|
|
292562
|
+
const [coefficient = "0", exponentText = "0"] = rendered.split("e");
|
|
292563
|
+
const exponent = Number.parseInt(exponentText, 10);
|
|
292564
|
+
const negative = coefficient.startsWith("-");
|
|
292565
|
+
const unsigned = negative ? coefficient.slice(1) : coefficient;
|
|
292566
|
+
const [integer = "0", fraction = ""] = unsigned.split(".");
|
|
292567
|
+
const digits = `${integer}${fraction}`;
|
|
292568
|
+
const decimalIndex = integer.length + exponent;
|
|
292569
|
+
let result;
|
|
292570
|
+
if (decimalIndex <= 0) {
|
|
292571
|
+
result = `0.${"0".repeat(-decimalIndex)}${digits}`;
|
|
292572
|
+
} else if (decimalIndex >= digits.length) {
|
|
292573
|
+
result = `${digits}${"0".repeat(decimalIndex - digits.length)}`;
|
|
292574
|
+
} else {
|
|
292575
|
+
result = `${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
|
|
292576
|
+
}
|
|
292577
|
+
return negative ? `-${result}` : result;
|
|
292537
292578
|
}
|
|
292538
|
-
|
|
292539
|
-
|
|
292540
|
-
|
|
292541
|
-
|
|
292542
|
-
|
|
292543
|
-
|
|
292544
|
-
|
|
292545
|
-
|
|
292546
|
-
|
|
292547
|
-
|
|
292548
|
-
|
|
292549
|
-
|
|
292550
|
-
|
|
292551
|
-
|
|
292552
|
-
signal;
|
|
292553
|
-
provider = null;
|
|
292554
|
-
isEnabled = false;
|
|
292555
|
-
serviceName;
|
|
292556
|
-
constructor(config, signal) {
|
|
292557
|
-
this.signal = signal;
|
|
292558
|
-
this.serviceName = config?.serviceName ?? DEFAULT_SERVICE;
|
|
292559
|
-
const endpointResolution = resolveOtlpEndpoint(this.signal, config);
|
|
292560
|
-
if (!endpointResolution) {
|
|
292561
|
-
log.info(`OTel ${signal} disabled: no OTLP endpoint or host provided`);
|
|
292562
|
-
return;
|
|
292563
|
-
}
|
|
292564
|
-
try {
|
|
292565
|
-
this.provider = this.createProvider(endpointResolution.endpoint, this.serviceName, endpointResolution.appendSignalPath);
|
|
292566
|
-
this.onProviderCreated(this.provider);
|
|
292567
|
-
this.isEnabled = true;
|
|
292568
|
-
log.info(`OTel ${signal} enabled: ${endpointResolution.endpoint}`);
|
|
292569
|
-
} catch (error) {
|
|
292570
|
-
log.error(`Failed to initialize OTel ${signal}:`, error);
|
|
292571
|
-
this.isEnabled = false;
|
|
292572
|
-
this.provider = null;
|
|
292579
|
+
function serializeCanonical(value, stack) {
|
|
292580
|
+
if (value === null)
|
|
292581
|
+
return "null";
|
|
292582
|
+
if (typeof value === "string")
|
|
292583
|
+
return JSON.stringify(value);
|
|
292584
|
+
if (typeof value === "boolean")
|
|
292585
|
+
return value ? "true" : "false";
|
|
292586
|
+
if (typeof value === "number")
|
|
292587
|
+
return canonicalDecimal(value);
|
|
292588
|
+
if (typeof value === "bigint")
|
|
292589
|
+
return value.toString(10);
|
|
292590
|
+
if (value instanceof Date) {
|
|
292591
|
+
if (Number.isNaN(value.getTime())) {
|
|
292592
|
+
throw new Error("Canonical timestamps must be valid");
|
|
292573
292593
|
}
|
|
292594
|
+
return value.getTime().toString(10);
|
|
292574
292595
|
}
|
|
292575
|
-
|
|
292576
|
-
|
|
292577
|
-
|
|
292578
|
-
|
|
292579
|
-
|
|
292580
|
-
|
|
292581
|
-
return
|
|
292582
|
-
}
|
|
292583
|
-
isOtelEnabled() {
|
|
292584
|
-
return this.isEnabled && this.provider !== null;
|
|
292596
|
+
if (Array.isArray(value)) {
|
|
292597
|
+
if (stack.has(value))
|
|
292598
|
+
throw new Error("Canonical values must be acyclic");
|
|
292599
|
+
stack.add(value);
|
|
292600
|
+
const result = `[${value.map((entry) => entry === undefined ? "null" : serializeCanonical(entry, stack)).join(",")}]`;
|
|
292601
|
+
stack.delete(value);
|
|
292602
|
+
return result;
|
|
292585
292603
|
}
|
|
292586
|
-
|
|
292587
|
-
if (
|
|
292588
|
-
|
|
292589
|
-
|
|
292590
|
-
|
|
292591
|
-
|
|
292592
|
-
|
|
292593
|
-
|
|
292594
|
-
log.error(`Error shutting down OTel ${this.signal} provider:`, error);
|
|
292595
|
-
}
|
|
292596
|
-
this.provider = null;
|
|
292597
|
-
this.isEnabled = false;
|
|
292598
|
-
this.onProviderClosed();
|
|
292604
|
+
if (typeof value === "object") {
|
|
292605
|
+
if (stack.has(value))
|
|
292606
|
+
throw new Error("Canonical values must be acyclic");
|
|
292607
|
+
stack.add(value);
|
|
292608
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([left], [right]) => left.localeCompare(right));
|
|
292609
|
+
const result = `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${serializeCanonical(entry, stack)}`).join(",")}}`;
|
|
292610
|
+
stack.delete(value);
|
|
292611
|
+
return result;
|
|
292599
292612
|
}
|
|
292613
|
+
throw new Error(`Unsupported canonical value type: ${typeof value}`);
|
|
292600
292614
|
}
|
|
292601
|
-
function
|
|
292602
|
-
|
|
292603
|
-
|
|
292604
|
-
|
|
292605
|
-
|
|
292606
|
-
|
|
292607
|
-
|
|
292615
|
+
function canonicalSerialize(value) {
|
|
292616
|
+
return serializeCanonical(value, new Set);
|
|
292617
|
+
}
|
|
292618
|
+
function omitChecksumFields(value) {
|
|
292619
|
+
if (Array.isArray(value))
|
|
292620
|
+
return value.map(omitChecksumFields);
|
|
292621
|
+
if (value && typeof value === "object" && !(value instanceof Date)) {
|
|
292622
|
+
return Object.fromEntries(Object.entries(value).filter(([key]) => !CHECKSUM_FIELDS.has(key)).map(([key, entry]) => [key, omitChecksumFields(entry)]));
|
|
292608
292623
|
}
|
|
292609
|
-
return
|
|
292624
|
+
return value;
|
|
292610
292625
|
}
|
|
292611
|
-
|
|
292612
|
-
|
|
292613
|
-
|
|
292614
|
-
|
|
292615
|
-
|
|
292616
|
-
|
|
292617
|
-
|
|
292626
|
+
function sha256Canonical(value) {
|
|
292627
|
+
return createHash3("sha256").update(canonicalSerialize(omitChecksumFields(value))).digest("hex");
|
|
292628
|
+
}
|
|
292629
|
+
function normalizeTimestampMs(value, field) {
|
|
292630
|
+
let timestamp;
|
|
292631
|
+
if (value instanceof Date) {
|
|
292632
|
+
timestamp = value.getTime();
|
|
292633
|
+
} else if (typeof value === "number") {
|
|
292634
|
+
timestamp = value;
|
|
292635
|
+
} else if (typeof value === "string" && /^\d+$/.test(value.trim())) {
|
|
292636
|
+
timestamp = Number(value.trim());
|
|
292637
|
+
} else if (typeof value === "string") {
|
|
292638
|
+
timestamp = Date.parse(value);
|
|
292639
|
+
} else {
|
|
292640
|
+
timestamp = Number.NaN;
|
|
292618
292641
|
}
|
|
292619
|
-
|
|
292620
|
-
|
|
292621
|
-
url: appendOtlpPath(endpoint, "metrics", appendSignalPath)
|
|
292622
|
-
});
|
|
292623
|
-
const reader = new import_sdk_metrics.PeriodicExportingMetricReader({
|
|
292624
|
-
exporter,
|
|
292625
|
-
exportIntervalMillis: EXPORT_INTERVAL_MS
|
|
292626
|
-
});
|
|
292627
|
-
const resource = import_resources.resourceFromAttributes({
|
|
292628
|
-
"service.name": serviceName
|
|
292629
|
-
});
|
|
292630
|
-
return new import_sdk_metrics.MeterProvider({
|
|
292631
|
-
resource,
|
|
292632
|
-
readers: [reader]
|
|
292633
|
-
});
|
|
292642
|
+
if (!Number.isSafeInteger(timestamp) || timestamp < 0) {
|
|
292643
|
+
throw new Error(`${field} must be a non-negative millisecond timestamp`);
|
|
292634
292644
|
}
|
|
292635
|
-
|
|
292636
|
-
|
|
292645
|
+
return timestamp;
|
|
292646
|
+
}
|
|
292647
|
+
function assertCaptureContext(context2) {
|
|
292648
|
+
if (!ARCHIVE_SOURCES.includes(context2.source)) {
|
|
292649
|
+
throw new Error(`Unsupported archive source: ${context2.source}`);
|
|
292637
292650
|
}
|
|
292638
|
-
|
|
292639
|
-
|
|
292651
|
+
if (!CAPTURE_FEEDS.includes(context2.feed)) {
|
|
292652
|
+
throw new Error(`Unsupported capture feed: ${context2.feed}`);
|
|
292640
292653
|
}
|
|
292641
|
-
|
|
292642
|
-
|
|
292643
|
-
|
|
292644
|
-
|
|
292654
|
+
if (!SOURCE_MODES.includes(context2.sourceMode)) {
|
|
292655
|
+
throw new Error(`Unsupported source mode: ${context2.sourceMode}`);
|
|
292656
|
+
}
|
|
292657
|
+
for (const [field, value] of [
|
|
292658
|
+
["deployment_id", context2.deploymentId],
|
|
292659
|
+
["capture_bundle_id", context2.captureBundleId],
|
|
292660
|
+
["exchange", context2.exchange],
|
|
292661
|
+
["symbol", context2.symbol],
|
|
292662
|
+
["provider", context2.provider]
|
|
292663
|
+
]) {
|
|
292664
|
+
if (!value.trim())
|
|
292665
|
+
throw new Error(`${field} must not be empty`);
|
|
292666
|
+
}
|
|
292667
|
+
}
|
|
292668
|
+
function createRawCapture(context2, input) {
|
|
292669
|
+
assertCaptureContext(context2);
|
|
292670
|
+
if (!RAW_CAPTURE_SCOPES.includes(input.scope)) {
|
|
292671
|
+
throw new Error(`Unsupported raw capture scope: ${input.scope}`);
|
|
292672
|
+
}
|
|
292673
|
+
const eventTimeMs = normalizeTimestampMs(input.eventTimeMs, "event_time_ms");
|
|
292674
|
+
const receivedTimeMs = normalizeTimestampMs(input.receivedTimeMs, "received_time_ms");
|
|
292675
|
+
const redactedPayload = redactStreamPayload(input.payload);
|
|
292676
|
+
const rawChecksum = sha256Canonical(redactedPayload);
|
|
292677
|
+
const rawCaptureId = sha256Canonical({
|
|
292678
|
+
capture_bundle_id: context2.captureBundleId,
|
|
292679
|
+
exchange: context2.exchange.trim().toLowerCase(),
|
|
292680
|
+
feed: context2.feed,
|
|
292681
|
+
raw_capture_scope: input.scope,
|
|
292682
|
+
raw_payload_sha256: rawChecksum,
|
|
292683
|
+
schema_version: context2.schemaVersion,
|
|
292684
|
+
source_mode: context2.sourceMode,
|
|
292685
|
+
source_symbol: context2.symbol.trim(),
|
|
292686
|
+
source_time_ms: eventTimeMs
|
|
292687
|
+
});
|
|
292688
|
+
return {
|
|
292689
|
+
rawCaptureId,
|
|
292690
|
+
rawCaptureScope: input.scope,
|
|
292691
|
+
rawChecksum,
|
|
292692
|
+
redactedPayload,
|
|
292693
|
+
eventTimeMs,
|
|
292694
|
+
receivedTimeMs,
|
|
292695
|
+
checksumAlgorithm: context2.checksumAlgorithm
|
|
292696
|
+
};
|
|
292697
|
+
}
|
|
292698
|
+
function captureCoreFields(context2, rawCapture) {
|
|
292699
|
+
assertCaptureContext(context2);
|
|
292700
|
+
return {
|
|
292701
|
+
source: context2.source,
|
|
292702
|
+
deployment_id: context2.deploymentId,
|
|
292703
|
+
capture_bundle_id: context2.captureBundleId,
|
|
292704
|
+
exchange: context2.exchange.trim().toLowerCase(),
|
|
292705
|
+
symbol: context2.symbol.trim(),
|
|
292706
|
+
trading_pair: context2.symbol.trim().replace("/", "-"),
|
|
292707
|
+
source_symbol: context2.symbol.trim(),
|
|
292708
|
+
asset_type: context2.assetType,
|
|
292709
|
+
feed: context2.feed,
|
|
292710
|
+
provider: context2.provider,
|
|
292711
|
+
source_mode: context2.sourceMode,
|
|
292712
|
+
source_time_ms: rawCapture.eventTimeMs,
|
|
292713
|
+
received_time_ms: rawCapture.receivedTimeMs,
|
|
292714
|
+
raw_capture_id: rawCapture.rawCaptureId,
|
|
292715
|
+
raw_capture_scope: rawCapture.rawCaptureScope,
|
|
292716
|
+
schema_version: context2.schemaVersion,
|
|
292717
|
+
checksum_algorithm: context2.checksumAlgorithm,
|
|
292718
|
+
raw_checksum: rawCapture.rawChecksum,
|
|
292719
|
+
provenance_complete: context2.provenanceComplete ? 1 : 0
|
|
292720
|
+
};
|
|
292721
|
+
}
|
|
292722
|
+
|
|
292723
|
+
// src/helpers/market-data-archive/capture-context.ts
|
|
292724
|
+
function createMarketCaptureContext(input) {
|
|
292725
|
+
const environment = input.environment ?? "development";
|
|
292726
|
+
const deploymentId = input.deploymentId.trim();
|
|
292727
|
+
if (!deploymentId)
|
|
292728
|
+
throw new Error("deployment_id must not be empty");
|
|
292729
|
+
const configuredBundle = input.captureBundleId?.trim();
|
|
292730
|
+
if (environment === "production" && !configuredBundle) {
|
|
292731
|
+
throw new Error("capture_bundle_id is required for production market capture");
|
|
292732
|
+
}
|
|
292733
|
+
const exchange = input.exchange.trim().toLowerCase();
|
|
292734
|
+
const symbol = input.symbol.trim();
|
|
292735
|
+
if (!exchange || !symbol) {
|
|
292736
|
+
throw new Error("exchange and symbol are required for market capture");
|
|
292737
|
+
}
|
|
292738
|
+
return {
|
|
292739
|
+
source: input.source,
|
|
292740
|
+
deploymentId,
|
|
292741
|
+
captureBundleId: configuredBundle ?? `development:${deploymentId}`,
|
|
292742
|
+
exchange,
|
|
292743
|
+
symbol,
|
|
292744
|
+
assetType: input.assetType,
|
|
292745
|
+
feed: input.feed,
|
|
292746
|
+
provider: input.provider?.trim() || `ccxt:${exchange}`,
|
|
292747
|
+
sourceMode: input.sourceMode,
|
|
292748
|
+
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
292749
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
292750
|
+
provenanceComplete: true,
|
|
292751
|
+
timeframe: input.timeframe,
|
|
292752
|
+
accountSelector: input.accountSelector
|
|
292753
|
+
};
|
|
292754
|
+
}
|
|
292755
|
+
function resolveMarketCaptureArchiveState(input) {
|
|
292756
|
+
if (!input.archiveEnabled) {
|
|
292757
|
+
return { enabled: false, reason: "archive_disabled" };
|
|
292758
|
+
}
|
|
292759
|
+
if (!input.marketArchiveEnabled) {
|
|
292760
|
+
return { enabled: false, reason: "market_archive_disabled" };
|
|
292761
|
+
}
|
|
292762
|
+
const environment = input.environment?.trim() || "development";
|
|
292763
|
+
if (environment !== "development" && environment !== "production") {
|
|
292764
|
+
return { enabled: false, reason: "invalid_capture_environment" };
|
|
292765
|
+
}
|
|
292766
|
+
if (environment === "production") {
|
|
292767
|
+
const deploymentId = input.deploymentId?.trim();
|
|
292768
|
+
if (!deploymentId || deploymentId === "unknown") {
|
|
292769
|
+
return { enabled: false, reason: "missing_deployment_id" };
|
|
292770
|
+
}
|
|
292771
|
+
if (!input.captureBundleId?.trim()) {
|
|
292772
|
+
return { enabled: false, reason: "missing_capture_bundle_id" };
|
|
292773
|
+
}
|
|
292774
|
+
}
|
|
292775
|
+
return { enabled: true };
|
|
292776
|
+
}
|
|
292777
|
+
function assertMarketCaptureArchiveStartable(state) {
|
|
292778
|
+
if (state.enabled || state.reason === "archive_disabled" || state.reason === "market_archive_disabled") {
|
|
292779
|
+
return;
|
|
292780
|
+
}
|
|
292781
|
+
throw new Error(`Refusing to start: canonical market-data archival was requested but its capture identity is incomplete (${state.reason}). ` + "Set CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT, CEX_BROKER_DEPLOYMENT_ID and CEX_BROKER_CAPTURE_BUNDLE_ID, " + "or disable archival explicitly via CEX_BROKER_MARKET_ARCHIVE_ENABLED.");
|
|
292782
|
+
}
|
|
292783
|
+
function captureEnvironmentFromEnv(value = process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT) {
|
|
292784
|
+
const environment = value?.trim() || "development";
|
|
292785
|
+
if (environment !== "development" && environment !== "production") {
|
|
292786
|
+
throw new Error("CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT must be development or production");
|
|
292787
|
+
}
|
|
292788
|
+
return environment;
|
|
292789
|
+
}
|
|
292790
|
+
|
|
292791
|
+
// src/helpers/market-data-archive/orderbook-sampler.ts
|
|
292792
|
+
var DEFAULT_ORDERBOOK_INTERVAL_MS = 1000;
|
|
292793
|
+
function getOrderbookIntervalMs() {
|
|
292794
|
+
const raw = process.env.CEX_BROKER_ORDERBOOK_INTERVAL_MS ?? process.env.CEX_BROKER_ORDERBOOK_TOB_INTERVAL_MS;
|
|
292795
|
+
if (!raw) {
|
|
292796
|
+
return DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
292797
|
+
}
|
|
292798
|
+
const parsed = Number.parseInt(raw, 10);
|
|
292799
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
292800
|
+
}
|
|
292801
|
+
function isMarketArchiveEnabled() {
|
|
292802
|
+
return process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED !== "false";
|
|
292803
|
+
}
|
|
292804
|
+
|
|
292805
|
+
class OrderbookSampler {
|
|
292806
|
+
intervalMs;
|
|
292807
|
+
lastEmitMs = null;
|
|
292808
|
+
constructor(intervalMs = getOrderbookIntervalMs()) {
|
|
292809
|
+
this.intervalMs = intervalMs;
|
|
292810
|
+
}
|
|
292811
|
+
shouldEmit(nowMs = Date.now()) {
|
|
292812
|
+
if (this.lastEmitMs !== null && nowMs < this.lastEmitMs) {
|
|
292813
|
+
this.lastEmitMs = nowMs;
|
|
292814
|
+
return true;
|
|
292815
|
+
}
|
|
292816
|
+
if (this.lastEmitMs !== null && nowMs - this.lastEmitMs < this.intervalMs) {
|
|
292817
|
+
return false;
|
|
292818
|
+
}
|
|
292819
|
+
this.lastEmitMs = nowMs;
|
|
292820
|
+
return true;
|
|
292821
|
+
}
|
|
292822
|
+
}
|
|
292823
|
+
|
|
292824
|
+
// src/helpers/order-activity-tracker.ts
|
|
292825
|
+
var DEFAULT_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
|
292826
|
+
|
|
292827
|
+
class OrderActivityTracker {
|
|
292828
|
+
#entries = new Map;
|
|
292829
|
+
#maxAgeMs;
|
|
292830
|
+
constructor(options) {
|
|
292831
|
+
this.#maxAgeMs = options?.maxAgeMs ?? DEFAULT_MAX_AGE_MS;
|
|
292832
|
+
}
|
|
292833
|
+
record(exchangeId, accountLabel, symbol, now3 = Date.now()) {
|
|
292834
|
+
const exchange = exchangeId.trim().toLowerCase();
|
|
292835
|
+
const trimmedSymbol = symbol.trim();
|
|
292836
|
+
if (!exchange || !accountLabel.trim() || !trimmedSymbol) {
|
|
292837
|
+
return;
|
|
292838
|
+
}
|
|
292839
|
+
const key = `${exchange}|${accountLabel}|${trimmedSymbol}`;
|
|
292840
|
+
this.#entries.set(key, {
|
|
292841
|
+
exchangeId: exchange,
|
|
292842
|
+
accountLabel,
|
|
292843
|
+
symbol: trimmedSymbol,
|
|
292844
|
+
lastActivityAt: now3
|
|
292845
|
+
});
|
|
292846
|
+
}
|
|
292847
|
+
list(now3 = Date.now()) {
|
|
292848
|
+
const active = [];
|
|
292849
|
+
for (const [key, entry] of this.#entries) {
|
|
292850
|
+
if (now3 - entry.lastActivityAt > this.#maxAgeMs) {
|
|
292851
|
+
this.#entries.delete(key);
|
|
292852
|
+
continue;
|
|
292853
|
+
}
|
|
292854
|
+
active.push(entry);
|
|
292855
|
+
}
|
|
292856
|
+
return active;
|
|
292857
|
+
}
|
|
292858
|
+
}
|
|
292859
|
+
|
|
292860
|
+
// src/helpers/otel.ts
|
|
292861
|
+
var import_api2 = __toESM(require_src2(), 1);
|
|
292862
|
+
var import_api_logs3 = __toESM(require_src4(), 1);
|
|
292863
|
+
var import_exporter_logs_otlp_http = __toESM(require_src11(), 1);
|
|
292864
|
+
var import_exporter_metrics_otlp_http = __toESM(require_src12(), 1);
|
|
292865
|
+
var import_resources = __toESM(require_src8(), 1);
|
|
292866
|
+
var import_sdk_logs = __toESM(require_src13(), 1);
|
|
292867
|
+
var import_sdk_metrics = __toESM(require_src9(), 1);
|
|
292868
|
+
var DEFAULT_SERVICE = "cex-broker";
|
|
292869
|
+
var DEFAULT_OTLP_PORT = 4318;
|
|
292870
|
+
var EXPORT_INTERVAL_MS = 5000;
|
|
292871
|
+
|
|
292872
|
+
class BaseOtelSignal {
|
|
292873
|
+
signal;
|
|
292874
|
+
provider = null;
|
|
292875
|
+
isEnabled = false;
|
|
292876
|
+
serviceName;
|
|
292877
|
+
constructor(config, signal) {
|
|
292878
|
+
this.signal = signal;
|
|
292879
|
+
this.serviceName = config?.serviceName ?? DEFAULT_SERVICE;
|
|
292880
|
+
const endpointResolution = resolveOtlpEndpoint(this.signal, config);
|
|
292881
|
+
if (!endpointResolution) {
|
|
292882
|
+
log.info(`OTel ${signal} disabled: no OTLP endpoint or host provided`);
|
|
292883
|
+
return;
|
|
292884
|
+
}
|
|
292885
|
+
try {
|
|
292886
|
+
this.provider = this.createProvider(endpointResolution.endpoint, this.serviceName, endpointResolution.appendSignalPath);
|
|
292887
|
+
this.onProviderCreated(this.provider);
|
|
292888
|
+
this.isEnabled = true;
|
|
292889
|
+
log.info(`OTel ${signal} enabled: ${endpointResolution.endpoint}`);
|
|
292890
|
+
} catch (error) {
|
|
292891
|
+
log.error(`Failed to initialize OTel ${signal}:`, error);
|
|
292892
|
+
this.isEnabled = false;
|
|
292893
|
+
this.provider = null;
|
|
292894
|
+
}
|
|
292895
|
+
}
|
|
292896
|
+
onProviderCreated(_provider) {}
|
|
292897
|
+
onProviderClosed() {}
|
|
292898
|
+
getProvider() {
|
|
292899
|
+
return this.provider;
|
|
292900
|
+
}
|
|
292901
|
+
getServiceName() {
|
|
292902
|
+
return this.serviceName;
|
|
292903
|
+
}
|
|
292904
|
+
isOtelEnabled() {
|
|
292905
|
+
return this.isEnabled && this.provider !== null;
|
|
292906
|
+
}
|
|
292907
|
+
async close() {
|
|
292908
|
+
if (!this.provider) {
|
|
292909
|
+
return;
|
|
292910
|
+
}
|
|
292911
|
+
try {
|
|
292912
|
+
await this.shutdownProvider(this.provider);
|
|
292913
|
+
log.info(`OTel ${this.signal} provider shut down`);
|
|
292914
|
+
} catch (error) {
|
|
292915
|
+
log.error(`Error shutting down OTel ${this.signal} provider:`, error);
|
|
292916
|
+
}
|
|
292917
|
+
this.provider = null;
|
|
292918
|
+
this.isEnabled = false;
|
|
292919
|
+
this.onProviderClosed();
|
|
292920
|
+
}
|
|
292921
|
+
}
|
|
292922
|
+
function toAttributes(labels, service) {
|
|
292923
|
+
const attrs = { ...labels, service };
|
|
292924
|
+
for (const key of Object.keys(attrs)) {
|
|
292925
|
+
const v = attrs[key];
|
|
292926
|
+
if (typeof v !== "string" && typeof v !== "number") {
|
|
292927
|
+
attrs[key] = String(v);
|
|
292928
|
+
}
|
|
292929
|
+
}
|
|
292930
|
+
return attrs;
|
|
292931
|
+
}
|
|
292932
|
+
|
|
292933
|
+
class OtelMetrics extends BaseOtelSignal {
|
|
292934
|
+
counters = new Map;
|
|
292935
|
+
histograms = new Map;
|
|
292936
|
+
observableGauges = new Map;
|
|
292937
|
+
constructor(config) {
|
|
292938
|
+
super(config, "metrics");
|
|
292939
|
+
}
|
|
292940
|
+
createProvider(endpoint, serviceName, appendSignalPath) {
|
|
292941
|
+
const exporter = new import_exporter_metrics_otlp_http.OTLPMetricExporter({
|
|
292942
|
+
url: appendOtlpPath(endpoint, "metrics", appendSignalPath)
|
|
292943
|
+
});
|
|
292944
|
+
const reader = new import_sdk_metrics.PeriodicExportingMetricReader({
|
|
292945
|
+
exporter,
|
|
292946
|
+
exportIntervalMillis: EXPORT_INTERVAL_MS
|
|
292947
|
+
});
|
|
292948
|
+
const resource = import_resources.resourceFromAttributes({
|
|
292949
|
+
"service.name": serviceName
|
|
292950
|
+
});
|
|
292951
|
+
return new import_sdk_metrics.MeterProvider({
|
|
292952
|
+
resource,
|
|
292953
|
+
readers: [reader]
|
|
292954
|
+
});
|
|
292955
|
+
}
|
|
292956
|
+
onProviderCreated(provider) {
|
|
292957
|
+
import_api2.metrics.setGlobalMeterProvider(provider);
|
|
292958
|
+
}
|
|
292959
|
+
shutdownProvider(provider) {
|
|
292960
|
+
return provider.shutdown();
|
|
292961
|
+
}
|
|
292962
|
+
async initialize() {
|
|
292963
|
+
if (this.isOtelEnabled()) {
|
|
292964
|
+
log.info("OTel metrics initialized (storage is handled by the collector)");
|
|
292965
|
+
}
|
|
292966
|
+
}
|
|
292967
|
+
async insertMetric(metric) {
|
|
292968
|
+
if (!this.isOtelEnabled())
|
|
292969
|
+
return;
|
|
292970
|
+
const labels = metric.labels ? JSON.parse(metric.labels) : {};
|
|
292971
|
+
try {
|
|
292972
|
+
if (metric.metric_type === "counter") {
|
|
292973
|
+
await this.recordCounter(metric.metric_name, metric.value, labels, metric.service);
|
|
292974
|
+
} else if (metric.metric_type === "gauge") {
|
|
292975
|
+
await this.recordGauge(metric.metric_name, metric.value, labels, metric.service);
|
|
292976
|
+
} else {
|
|
292977
|
+
await this.recordHistogram(metric.metric_name, metric.value, labels, metric.service);
|
|
292978
|
+
}
|
|
292979
|
+
} catch {}
|
|
292980
|
+
}
|
|
292981
|
+
async insertMetrics(metricsList) {
|
|
292982
|
+
if (!this.isOtelEnabled() || metricsList.length === 0)
|
|
292983
|
+
return;
|
|
292984
|
+
for (const m of metricsList) {
|
|
292985
|
+
await this.insertMetric(m);
|
|
292986
|
+
}
|
|
292987
|
+
}
|
|
292988
|
+
async recordCounter(metricName, value, labels, service = this.getServiceName()) {
|
|
292989
|
+
const provider = this.getProvider();
|
|
292990
|
+
if (!this.isOtelEnabled() || !provider)
|
|
292991
|
+
return;
|
|
292992
|
+
try {
|
|
292993
|
+
let counter = this.counters.get(metricName);
|
|
292994
|
+
if (!counter) {
|
|
292995
|
+
const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
|
|
292996
|
+
counter = meter.createCounter(metricName, { description: metricName });
|
|
292997
|
+
this.counters.set(metricName, counter);
|
|
292998
|
+
}
|
|
292999
|
+
counter.add(value, toAttributes(labels, service));
|
|
293000
|
+
} catch (error) {
|
|
293001
|
+
log.error("Failed to record counter:", error);
|
|
293002
|
+
}
|
|
293003
|
+
}
|
|
293004
|
+
async recordGauge(metricName, value, labels, service = this.getServiceName()) {
|
|
293005
|
+
const provider = this.getProvider();
|
|
293006
|
+
if (!this.isOtelEnabled() || !provider)
|
|
293007
|
+
return;
|
|
293008
|
+
try {
|
|
293009
|
+
let hist = this.histograms.get(`gauge_${metricName}`);
|
|
293010
|
+
if (!hist) {
|
|
293011
|
+
const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
|
|
293012
|
+
hist = meter.createHistogram(`${metricName}_gauge`, {
|
|
293013
|
+
description: metricName
|
|
293014
|
+
});
|
|
293015
|
+
this.histograms.set(`gauge_${metricName}`, hist);
|
|
293016
|
+
}
|
|
293017
|
+
hist.record(value, toAttributes(labels, service));
|
|
293018
|
+
} catch (error) {
|
|
293019
|
+
log.error("Failed to record gauge:", error);
|
|
293020
|
+
}
|
|
293021
|
+
}
|
|
293022
|
+
async setObservableGauge(metricName, value, labels, service = this.getServiceName()) {
|
|
293023
|
+
const provider = this.getProvider();
|
|
293024
|
+
if (!this.isOtelEnabled() || !provider)
|
|
293025
|
+
return;
|
|
293026
|
+
try {
|
|
293027
|
+
let state = this.observableGauges.get(metricName);
|
|
293028
|
+
if (!state) {
|
|
293029
|
+
const observations = new Map;
|
|
293030
|
+
const instrument = provider.getMeter("cex-broker-metrics", "1.0.0").createObservableGauge(metricName, { description: metricName });
|
|
293031
|
+
instrument.addCallback((result) => {
|
|
293032
|
+
for (const observation of observations.values()) {
|
|
293033
|
+
result.observe(observation.value, observation.attributes);
|
|
293034
|
+
}
|
|
293035
|
+
});
|
|
293036
|
+
state = { instrument, observations };
|
|
293037
|
+
this.observableGauges.set(metricName, state);
|
|
293038
|
+
}
|
|
293039
|
+
const attributes = toAttributes(labels, service);
|
|
293040
|
+
state.observations.set(stableAttributeKey(attributes), {
|
|
293041
|
+
value,
|
|
293042
|
+
attributes
|
|
293043
|
+
});
|
|
293044
|
+
} catch (error) {
|
|
293045
|
+
log.error("Failed to set observable gauge:", error);
|
|
293046
|
+
}
|
|
293047
|
+
}
|
|
293048
|
+
async recordHistogram(metricName, value, labels, service = this.getServiceName()) {
|
|
293049
|
+
const provider = this.getProvider();
|
|
293050
|
+
if (!this.isOtelEnabled() || !provider)
|
|
293051
|
+
return;
|
|
293052
|
+
try {
|
|
293053
|
+
let hist = this.histograms.get(metricName);
|
|
293054
|
+
if (!hist) {
|
|
293055
|
+
const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
|
|
293056
|
+
hist = meter.createHistogram(metricName, { description: metricName });
|
|
293057
|
+
this.histograms.set(metricName, hist);
|
|
293058
|
+
}
|
|
293059
|
+
hist.record(value, toAttributes(labels, service));
|
|
293060
|
+
} catch (error) {
|
|
293061
|
+
log.error("Failed to record histogram:", error);
|
|
293062
|
+
}
|
|
293063
|
+
}
|
|
293064
|
+
}
|
|
293065
|
+
|
|
293066
|
+
class OtelLogs extends BaseOtelSignal {
|
|
293067
|
+
logger = null;
|
|
293068
|
+
constructor(config) {
|
|
293069
|
+
super(config, "logs");
|
|
293070
|
+
}
|
|
293071
|
+
createProvider(endpoint, serviceName, appendSignalPath) {
|
|
293072
|
+
const exporter = new import_exporter_logs_otlp_http.OTLPLogExporter({
|
|
293073
|
+
url: appendOtlpPath(endpoint, "logs", appendSignalPath)
|
|
293074
|
+
});
|
|
293075
|
+
const processor = new import_sdk_logs.BatchLogRecordProcessor(exporter);
|
|
293076
|
+
const resource = import_resources.resourceFromAttributes({
|
|
293077
|
+
"service.name": serviceName
|
|
293078
|
+
});
|
|
293079
|
+
return new import_sdk_logs.LoggerProvider({
|
|
293080
|
+
resource,
|
|
293081
|
+
processors: [processor]
|
|
293082
|
+
});
|
|
293083
|
+
}
|
|
293084
|
+
onProviderCreated(provider) {
|
|
293085
|
+
import_api_logs3.logs.setGlobalLoggerProvider(provider);
|
|
293086
|
+
this.logger = provider.getLogger("cex-broker-logs", "1.0.0");
|
|
293087
|
+
}
|
|
293088
|
+
shutdownProvider(provider) {
|
|
293089
|
+
return provider.forceFlush().then(() => provider.shutdown());
|
|
293090
|
+
}
|
|
293091
|
+
onProviderClosed() {
|
|
293092
|
+
this.logger = null;
|
|
293093
|
+
}
|
|
293094
|
+
emit(record) {
|
|
293095
|
+
if (!this.isOtelEnabled() || !this.logger) {
|
|
293096
|
+
return;
|
|
293097
|
+
}
|
|
293098
|
+
this.logger.emit(record);
|
|
293099
|
+
}
|
|
293100
|
+
}
|
|
293101
|
+
function resolveOtlpEndpoint(signal, config) {
|
|
293102
|
+
if (config?.otlpEndpoint) {
|
|
293103
|
+
return {
|
|
293104
|
+
endpoint: normalizeOtlpEndpoint(config.otlpEndpoint),
|
|
293105
|
+
appendSignalPath: true
|
|
293106
|
+
};
|
|
293107
|
+
}
|
|
293108
|
+
const signalEndpoint = signal === "metrics" ? process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT : process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
|
|
293109
|
+
if (signalEndpoint) {
|
|
293110
|
+
return {
|
|
293111
|
+
endpoint: signalEndpoint,
|
|
293112
|
+
appendSignalPath: false
|
|
293113
|
+
};
|
|
293114
|
+
}
|
|
293115
|
+
if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) {
|
|
293116
|
+
return {
|
|
293117
|
+
endpoint: normalizeOtlpEndpoint(process.env.OTEL_EXPORTER_OTLP_ENDPOINT),
|
|
293118
|
+
appendSignalPath: true
|
|
293119
|
+
};
|
|
293120
|
+
}
|
|
293121
|
+
if (config?.host) {
|
|
293122
|
+
const protocol = config.protocol || "http";
|
|
293123
|
+
const port = config.port ?? DEFAULT_OTLP_PORT;
|
|
293124
|
+
return {
|
|
293125
|
+
endpoint: `${protocol}://${config.host}:${port}`,
|
|
293126
|
+
appendSignalPath: true
|
|
293127
|
+
};
|
|
293128
|
+
}
|
|
293129
|
+
return null;
|
|
293130
|
+
}
|
|
293131
|
+
function appendOtlpPath(endpoint, signal, appendSignalPath) {
|
|
293132
|
+
if (!appendSignalPath) {
|
|
293133
|
+
return endpoint;
|
|
293134
|
+
}
|
|
293135
|
+
const baseEndpoint = normalizeOtlpEndpoint(endpoint);
|
|
293136
|
+
return `${baseEndpoint}/v1/${signal}`;
|
|
293137
|
+
}
|
|
293138
|
+
function normalizeOtlpEndpoint(endpoint) {
|
|
293139
|
+
return endpoint.replace(/\/v1\/(metrics|logs)\/?$/, "").replace(/\/+$/, "");
|
|
293140
|
+
}
|
|
293141
|
+
function getOtelHostFromEnv() {
|
|
293142
|
+
return process.env.CEX_BROKER_OTEL_HOST ?? process.env.CEX_BROKER_CLICKHOUSE_HOST;
|
|
293143
|
+
}
|
|
293144
|
+
function getOtelPortFromEnv() {
|
|
293145
|
+
const port = process.env.CEX_BROKER_OTEL_PORT ?? process.env.CEX_BROKER_CLICKHOUSE_PORT;
|
|
293146
|
+
return port ? Number.parseInt(port, 10) : undefined;
|
|
293147
|
+
}
|
|
293148
|
+
function getOtelProtocolFromEnv() {
|
|
293149
|
+
const protocol = process.env.CEX_BROKER_OTEL_PROTOCOL ?? process.env.CEX_BROKER_CLICKHOUSE_PROTOCOL;
|
|
293150
|
+
return protocol || "http";
|
|
293151
|
+
}
|
|
293152
|
+
function createOtelMetricsFromEnv(options = {}) {
|
|
293153
|
+
const otlpEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
|
|
293154
|
+
const serviceName = process.env.OTEL_SERVICE_NAME || options.defaultServiceName || DEFAULT_SERVICE;
|
|
293155
|
+
if (otlpEndpoint) {
|
|
293156
|
+
return new OtelMetrics({
|
|
293157
|
+
otlpEndpoint,
|
|
293158
|
+
serviceName
|
|
293159
|
+
});
|
|
293160
|
+
}
|
|
293161
|
+
if (options.allowLegacyBrokerConfig === false) {
|
|
293162
|
+
return new OtelMetrics({ serviceName });
|
|
293163
|
+
}
|
|
293164
|
+
const host = getOtelHostFromEnv();
|
|
293165
|
+
if (!host)
|
|
293166
|
+
return new OtelMetrics({ serviceName });
|
|
293167
|
+
const port = getOtelPortFromEnv();
|
|
293168
|
+
const config = {
|
|
293169
|
+
host,
|
|
293170
|
+
port: port ?? DEFAULT_OTLP_PORT,
|
|
293171
|
+
protocol: getOtelProtocolFromEnv(),
|
|
293172
|
+
serviceName
|
|
293173
|
+
};
|
|
293174
|
+
return new OtelMetrics(config);
|
|
293175
|
+
}
|
|
293176
|
+
function stableAttributeKey(attributes) {
|
|
293177
|
+
return JSON.stringify(Object.entries(attributes).sort(([left], [right]) => left.localeCompare(right)));
|
|
293178
|
+
}
|
|
293179
|
+
function createOtelLogsFromEnv() {
|
|
293180
|
+
const logsEndpoint = process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
|
|
293181
|
+
const genericEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
|
|
293182
|
+
const host = getOtelHostFromEnv();
|
|
293183
|
+
if (logsEndpoint) {
|
|
293184
|
+
return new OtelLogs({
|
|
293185
|
+
serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
|
|
293186
|
+
});
|
|
293187
|
+
}
|
|
293188
|
+
if (genericEndpoint) {
|
|
293189
|
+
return new OtelLogs({
|
|
293190
|
+
otlpEndpoint: genericEndpoint,
|
|
293191
|
+
serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
|
|
293192
|
+
});
|
|
292645
293193
|
}
|
|
292646
|
-
|
|
292647
|
-
|
|
293194
|
+
if (!host) {
|
|
293195
|
+
return new OtelLogs;
|
|
293196
|
+
}
|
|
293197
|
+
const port = getOtelPortFromEnv();
|
|
293198
|
+
const config = {
|
|
293199
|
+
host,
|
|
293200
|
+
port: port ?? DEFAULT_OTLP_PORT,
|
|
293201
|
+
protocol: getOtelProtocolFromEnv(),
|
|
293202
|
+
serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
|
|
293203
|
+
};
|
|
293204
|
+
return new OtelLogs(config);
|
|
293205
|
+
}
|
|
293206
|
+
|
|
293207
|
+
// src/helpers/stream-health-publisher.ts
|
|
293208
|
+
import { createHash as createHash4, randomUUID } from "node:crypto";
|
|
293209
|
+
import {
|
|
293210
|
+
closeSync as closeSync2,
|
|
293211
|
+
fsyncSync as fsyncSync2,
|
|
293212
|
+
openSync as openSync2,
|
|
293213
|
+
readFileSync,
|
|
293214
|
+
renameSync,
|
|
293215
|
+
statSync,
|
|
293216
|
+
unlinkSync,
|
|
293217
|
+
writeFileSync
|
|
293218
|
+
} from "node:fs";
|
|
293219
|
+
import { request as httpRequest3 } from "node:http";
|
|
293220
|
+
import { request as httpsRequest3 } from "node:https";
|
|
293221
|
+
import { dirname } from "node:path";
|
|
293222
|
+
var SOURCE = "broker_write";
|
|
293223
|
+
var TABLE = "broker_stream_health.snapshots";
|
|
293224
|
+
var PRODUCER_ID = "cex-broker-user-data";
|
|
293225
|
+
var STATE_VERSION = 1;
|
|
293226
|
+
var HEARTBEAT_MS = 30000;
|
|
293227
|
+
var FORWARDER_TIMEOUT_MS = 3000;
|
|
293228
|
+
var IDENTIFIER = /^[a-z0-9][a-z0-9:_-]{0,127}$/;
|
|
293229
|
+
function identifier(value, name) {
|
|
293230
|
+
const normalized = value.trim().toLowerCase();
|
|
293231
|
+
if (!IDENTIFIER.test(normalized)) {
|
|
293232
|
+
throw new Error(`${name} must be a lower-case stream-health identifier`);
|
|
293233
|
+
}
|
|
293234
|
+
return normalized;
|
|
293235
|
+
}
|
|
293236
|
+
function counter(value) {
|
|
293237
|
+
if (!/^(0|[1-9]\d*)$/.test(value)) {
|
|
293238
|
+
throw new Error("Invalid persisted stream-health counter");
|
|
293239
|
+
}
|
|
293240
|
+
return BigInt(value);
|
|
293241
|
+
}
|
|
293242
|
+
function next(value) {
|
|
293243
|
+
return (counter(value) + 1n).toString();
|
|
293244
|
+
}
|
|
293245
|
+
function key(snapshot) {
|
|
293246
|
+
return `exchange:${snapshot.exchange}|account:${snapshot.accountSelector}|stream:${snapshot.streamKind}|scope:${snapshot.accountScope}`;
|
|
293247
|
+
}
|
|
293248
|
+
function registryRevision(snapshots) {
|
|
293249
|
+
const rows = snapshots.map((snapshot) => ({
|
|
293250
|
+
exchange: snapshot.exchange,
|
|
293251
|
+
account_selector: snapshot.accountSelector,
|
|
293252
|
+
account_role: snapshot.accountRole ?? null,
|
|
293253
|
+
stream_kind: snapshot.streamKind,
|
|
293254
|
+
account_scope: snapshot.accountScope,
|
|
293255
|
+
registry_status: snapshot.registryStatus
|
|
293256
|
+
})).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
|
|
293257
|
+
return createHash4("sha256").update(JSON.stringify(rows)).digest("hex");
|
|
293258
|
+
}
|
|
293259
|
+
function validState(value) {
|
|
293260
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
293261
|
+
return false;
|
|
293262
|
+
const state = value;
|
|
293263
|
+
return state.version === STATE_VERSION && state.producerId === PRODUCER_ID && typeof state.producerEpoch === "string" && typeof state.runId === "string" && typeof state.nextBatchSequence === "string" && state.nextStreamSequences !== null && typeof state.nextStreamSequences === "object" && (state.pendingBody === undefined || typeof state.pendingBody === "string");
|
|
293264
|
+
}
|
|
293265
|
+
function forwarderPost(url2, body, authToken, timeoutMs) {
|
|
293266
|
+
const request = url2.protocol === "http:" ? httpRequest3 : httpsRequest3;
|
|
293267
|
+
const headers = {
|
|
293268
|
+
"content-type": "application/json",
|
|
293269
|
+
"content-length": Buffer.byteLength(body)
|
|
293270
|
+
};
|
|
293271
|
+
if (authToken)
|
|
293272
|
+
headers.authorization = `Bearer ${authToken}`;
|
|
293273
|
+
return new Promise((resolve, reject) => {
|
|
293274
|
+
const req = request(url2, { method: "POST", headers, timeout: timeoutMs }, (res) => {
|
|
293275
|
+
res.on("data", () => {});
|
|
293276
|
+
res.on("end", () => {
|
|
293277
|
+
const status = res.statusCode ?? 0;
|
|
293278
|
+
if (status < 200 || status >= 300) {
|
|
293279
|
+
reject(new Error(`Stream health forwarder returned ${status}`));
|
|
293280
|
+
return;
|
|
293281
|
+
}
|
|
293282
|
+
resolve();
|
|
293283
|
+
});
|
|
293284
|
+
});
|
|
293285
|
+
req.on("error", reject);
|
|
293286
|
+
req.on("timeout", () => req.destroy(new Error("Stream health forwarder timed out")));
|
|
293287
|
+
req.write(body);
|
|
293288
|
+
req.end();
|
|
293289
|
+
});
|
|
293290
|
+
}
|
|
293291
|
+
|
|
293292
|
+
class StreamHealthPublisher {
|
|
293293
|
+
#deploymentId;
|
|
293294
|
+
#statePath;
|
|
293295
|
+
#heartbeatMs;
|
|
293296
|
+
#post;
|
|
293297
|
+
#state;
|
|
293298
|
+
#advanceRun;
|
|
293299
|
+
#snapshots = [];
|
|
293300
|
+
#dirty = false;
|
|
293301
|
+
#closed = false;
|
|
293302
|
+
#pumping = null;
|
|
293303
|
+
#heartbeat = null;
|
|
293304
|
+
#retry = null;
|
|
293305
|
+
#retryAttempt = 0;
|
|
293306
|
+
constructor(options) {
|
|
293307
|
+
this.#deploymentId = identifier(options.deploymentId, "deployment_id");
|
|
293308
|
+
this.#statePath = options.statePath.trim();
|
|
293309
|
+
if (!this.#statePath) {
|
|
293310
|
+
throw new Error("CEX_BROKER_STREAM_HEALTH_STATE_PATH is required");
|
|
293311
|
+
}
|
|
293312
|
+
this.#heartbeatMs = options.heartbeatIntervalMs ?? HEARTBEAT_MS;
|
|
293313
|
+
if (!Number.isInteger(this.#heartbeatMs) || this.#heartbeatMs < 1 || this.#heartbeatMs > 60000) {
|
|
293314
|
+
throw new Error("Stream health heartbeat interval must be between 1 and 60000ms");
|
|
293315
|
+
}
|
|
293316
|
+
let url2;
|
|
293317
|
+
try {
|
|
293318
|
+
url2 = new URL(options.forwarderUrl.trim());
|
|
293319
|
+
} catch (error) {
|
|
293320
|
+
throw new Error("CEX_BROKER_ARCHIVE_FORWARDER_URL must be valid", {
|
|
293321
|
+
cause: error
|
|
293322
|
+
});
|
|
293323
|
+
}
|
|
293324
|
+
if (url2.protocol !== "http:" && url2.protocol !== "https:") {
|
|
293325
|
+
throw new Error("CEX_BROKER_ARCHIVE_FORWARDER_URL must use HTTP(S)");
|
|
293326
|
+
}
|
|
293327
|
+
this.#post = options.post ?? ((body) => forwarderPost(url2, body, options.forwarderAuthToken, options.forwarderTimeoutMs ?? FORWARDER_TIMEOUT_MS));
|
|
293328
|
+
const loaded = this.#read();
|
|
293329
|
+
this.#state = loaded ?? {
|
|
293330
|
+
version: STATE_VERSION,
|
|
293331
|
+
producerId: PRODUCER_ID,
|
|
293332
|
+
producerEpoch: "1",
|
|
293333
|
+
runId: randomUUID(),
|
|
293334
|
+
nextBatchSequence: "1",
|
|
293335
|
+
nextStreamSequences: {}
|
|
293336
|
+
};
|
|
293337
|
+
counter(this.#state.producerEpoch);
|
|
293338
|
+
counter(this.#state.nextBatchSequence);
|
|
293339
|
+
for (const value of Object.values(this.#state.nextStreamSequences))
|
|
293340
|
+
counter(value);
|
|
293341
|
+
this.#advanceRun = loaded !== null;
|
|
293342
|
+
if (!loaded)
|
|
293343
|
+
this.#persist();
|
|
293344
|
+
}
|
|
293345
|
+
start() {
|
|
293346
|
+
if (this.#closed || this.#heartbeat)
|
|
292648
293347
|
return;
|
|
292649
|
-
|
|
293348
|
+
this.#heartbeat = setInterval(() => {
|
|
293349
|
+
if (this.#snapshots.length > 0) {
|
|
293350
|
+
this.#dirty = true;
|
|
293351
|
+
this.#schedule();
|
|
293352
|
+
}
|
|
293353
|
+
}, this.#heartbeatMs);
|
|
293354
|
+
this.#heartbeat.unref?.();
|
|
293355
|
+
if (this.#state.pendingBody || this.#dirty)
|
|
293356
|
+
this.#schedule();
|
|
293357
|
+
}
|
|
293358
|
+
publish(snapshots) {
|
|
293359
|
+
if (this.#closed)
|
|
293360
|
+
return;
|
|
293361
|
+
this.#snapshots = snapshots.map((snapshot) => ({ ...snapshot }));
|
|
293362
|
+
this.#dirty = true;
|
|
293363
|
+
this.#schedule();
|
|
293364
|
+
}
|
|
293365
|
+
async close(snapshots, timeoutMs = FORWARDER_TIMEOUT_MS) {
|
|
293366
|
+
if (this.#closed)
|
|
293367
|
+
return;
|
|
293368
|
+
if (this.#heartbeat)
|
|
293369
|
+
clearInterval(this.#heartbeat);
|
|
293370
|
+
this.#heartbeat = null;
|
|
293371
|
+
if (this.#retry)
|
|
293372
|
+
clearTimeout(this.#retry);
|
|
293373
|
+
this.#retry = null;
|
|
293374
|
+
this.#snapshots = snapshots.map((snapshot) => ({ ...snapshot }));
|
|
293375
|
+
this.#dirty = this.#snapshots.length > 0;
|
|
293376
|
+
this.#schedule();
|
|
293377
|
+
await Promise.race([
|
|
293378
|
+
this.#waitForIdle(),
|
|
293379
|
+
new Promise((resolve) => setTimeout(resolve, timeoutMs))
|
|
293380
|
+
]);
|
|
293381
|
+
this.#closed = true;
|
|
293382
|
+
if (this.#retry)
|
|
293383
|
+
clearTimeout(this.#retry);
|
|
293384
|
+
this.#retry = null;
|
|
293385
|
+
}
|
|
293386
|
+
#schedule() {
|
|
293387
|
+
if (this.#closed || this.#pumping)
|
|
293388
|
+
return;
|
|
293389
|
+
this.#pumping = this.#pump().finally(() => {
|
|
293390
|
+
this.#pumping = null;
|
|
293391
|
+
if (!this.#closed && !this.#retry && (this.#state.pendingBody || this.#dirty))
|
|
293392
|
+
this.#schedule();
|
|
293393
|
+
});
|
|
293394
|
+
}
|
|
293395
|
+
async#pump() {
|
|
293396
|
+
if (this.#state.pendingBody && !await this.#deliver())
|
|
293397
|
+
return;
|
|
293398
|
+
if (!this.#dirty || this.#snapshots.length === 0)
|
|
293399
|
+
return;
|
|
293400
|
+
if (this.#advanceRun) {
|
|
293401
|
+
this.#state.producerEpoch = next(this.#state.producerEpoch);
|
|
293402
|
+
this.#state.runId = randomUUID();
|
|
293403
|
+
this.#state.nextBatchSequence = "1";
|
|
293404
|
+
this.#state.nextStreamSequences = {};
|
|
293405
|
+
this.#advanceRun = false;
|
|
293406
|
+
this.#persist();
|
|
293407
|
+
}
|
|
293408
|
+
this.#dirty = false;
|
|
293409
|
+
this.#state.pendingBody = this.#body(this.#snapshots);
|
|
293410
|
+
this.#persist();
|
|
293411
|
+
await this.#deliver();
|
|
293412
|
+
}
|
|
293413
|
+
async#deliver() {
|
|
293414
|
+
const body = this.#state.pendingBody;
|
|
293415
|
+
if (!body)
|
|
293416
|
+
return true;
|
|
292650
293417
|
try {
|
|
292651
|
-
|
|
292652
|
-
|
|
292653
|
-
|
|
292654
|
-
|
|
292655
|
-
|
|
292656
|
-
|
|
293418
|
+
await this.#post(body);
|
|
293419
|
+
this.#state.pendingBody = undefined;
|
|
293420
|
+
this.#persist();
|
|
293421
|
+
this.#retryAttempt = 0;
|
|
293422
|
+
return true;
|
|
293423
|
+
} catch {
|
|
293424
|
+
this.#retryLater();
|
|
293425
|
+
return false;
|
|
293426
|
+
}
|
|
293427
|
+
}
|
|
293428
|
+
#body(snapshots) {
|
|
293429
|
+
const ordered3 = [...snapshots].sort((left, right) => key(left).localeCompare(key(right)));
|
|
293430
|
+
if (ordered3.length === 0 || ordered3.length > 1000) {
|
|
293431
|
+
throw new Error("Stream health requires between one and 1000 registry rows");
|
|
293432
|
+
}
|
|
293433
|
+
const batchSequence = this.#state.nextBatchSequence;
|
|
293434
|
+
this.#state.nextBatchSequence = next(batchSequence);
|
|
293435
|
+
const heartbeatAt = new Date().toISOString();
|
|
293436
|
+
const active = ordered3.filter((snapshot) => snapshot.registryStatus === "active").length;
|
|
293437
|
+
const rows = ordered3.map((snapshot) => {
|
|
293438
|
+
const streamKey = key(snapshot);
|
|
293439
|
+
const sequence = this.#state.nextStreamSequences[streamKey] ?? "1";
|
|
293440
|
+
this.#state.nextStreamSequences[streamKey] = next(sequence);
|
|
293441
|
+
return {
|
|
293442
|
+
table: TABLE,
|
|
293443
|
+
row: {
|
|
293444
|
+
producer_id: PRODUCER_ID,
|
|
293445
|
+
producer_epoch: this.#state.producerEpoch,
|
|
293446
|
+
run_id: this.#state.runId,
|
|
293447
|
+
batch_sequence: batchSequence,
|
|
293448
|
+
batch_snapshot_count: String(ordered3.length),
|
|
293449
|
+
batch_active_stream_count: String(active),
|
|
293450
|
+
registry_revision: registryRevision(ordered3),
|
|
293451
|
+
registry_status: snapshot.registryStatus,
|
|
293452
|
+
retired_at: snapshot.retiredAt,
|
|
293453
|
+
exchange: snapshot.exchange,
|
|
293454
|
+
account_selector: snapshot.accountSelector,
|
|
293455
|
+
account_role: snapshot.accountRole ?? null,
|
|
293456
|
+
stream_kind: snapshot.streamKind,
|
|
293457
|
+
account_scope: snapshot.accountScope,
|
|
293458
|
+
sequence,
|
|
293459
|
+
state: snapshot.state,
|
|
293460
|
+
state_changed_at: snapshot.stateChangedAt,
|
|
293461
|
+
last_connected_at: snapshot.lastConnectedAt,
|
|
293462
|
+
last_authenticated_at: snapshot.lastAuthenticatedAt,
|
|
293463
|
+
last_received_at: snapshot.lastReceivedAt,
|
|
293464
|
+
heartbeat_at: heartbeatAt,
|
|
293465
|
+
connect_attempt_count: snapshot.connectAttemptCount,
|
|
293466
|
+
reconnect_count: snapshot.reconnectCount,
|
|
293467
|
+
error_count: snapshot.errorCount,
|
|
293468
|
+
last_failure_kind: snapshot.lastFailureKind,
|
|
293469
|
+
last_failure_reason: snapshot.lastFailureReason,
|
|
293470
|
+
traffic_mode: snapshot.trafficMode,
|
|
293471
|
+
source_watermark: snapshot.sourceWatermark
|
|
293472
|
+
}
|
|
293473
|
+
};
|
|
293474
|
+
});
|
|
293475
|
+
return JSON.stringify({
|
|
293476
|
+
source: SOURCE,
|
|
293477
|
+
deployment_id: this.#deploymentId,
|
|
293478
|
+
rows
|
|
293479
|
+
});
|
|
293480
|
+
}
|
|
293481
|
+
#retryLater() {
|
|
293482
|
+
if (this.#closed || this.#retry)
|
|
293483
|
+
return;
|
|
293484
|
+
const delay = Math.min(1000 * 2 ** this.#retryAttempt, 30000);
|
|
293485
|
+
this.#retryAttempt += 1;
|
|
293486
|
+
this.#retry = setTimeout(() => {
|
|
293487
|
+
this.#retry = null;
|
|
293488
|
+
this.#schedule();
|
|
293489
|
+
}, delay);
|
|
293490
|
+
this.#retry.unref?.();
|
|
293491
|
+
}
|
|
293492
|
+
async#waitForIdle() {
|
|
293493
|
+
while (this.#pumping)
|
|
293494
|
+
await this.#pumping;
|
|
293495
|
+
}
|
|
293496
|
+
#read() {
|
|
293497
|
+
try {
|
|
293498
|
+
const parsed = JSON.parse(readFileSync(this.#statePath, "utf8"));
|
|
293499
|
+
if (!validState(parsed))
|
|
293500
|
+
throw new Error("invalid state shape");
|
|
293501
|
+
return parsed;
|
|
293502
|
+
} catch (error) {
|
|
293503
|
+
if (error.code === "ENOENT")
|
|
293504
|
+
return null;
|
|
293505
|
+
throw new Error("Stream health state cannot be read", { cause: error });
|
|
293506
|
+
}
|
|
293507
|
+
}
|
|
293508
|
+
#persist() {
|
|
293509
|
+
const parent = dirname(this.#statePath);
|
|
293510
|
+
try {
|
|
293511
|
+
if (!statSync(parent).isDirectory())
|
|
293512
|
+
throw new Error("state parent is not a directory");
|
|
293513
|
+
} catch (error) {
|
|
293514
|
+
throw new Error("Stream health state directory is unavailable", {
|
|
293515
|
+
cause: error
|
|
293516
|
+
});
|
|
293517
|
+
}
|
|
293518
|
+
const temporary = `${this.#statePath}.${process.pid}.${randomUUID()}.tmp`;
|
|
293519
|
+
let fd2;
|
|
293520
|
+
try {
|
|
293521
|
+
fd2 = openSync2(temporary, "wx", 384);
|
|
293522
|
+
writeFileSync(fd2, JSON.stringify(this.#state));
|
|
293523
|
+
fsyncSync2(fd2);
|
|
293524
|
+
closeSync2(fd2);
|
|
293525
|
+
fd2 = undefined;
|
|
293526
|
+
renameSync(temporary, this.#statePath);
|
|
293527
|
+
const parentFd = openSync2(parent, "r");
|
|
293528
|
+
try {
|
|
293529
|
+
fsyncSync2(parentFd);
|
|
293530
|
+
} finally {
|
|
293531
|
+
closeSync2(parentFd);
|
|
293532
|
+
}
|
|
293533
|
+
} catch (error) {
|
|
293534
|
+
if (fd2 !== undefined)
|
|
293535
|
+
closeSync2(fd2);
|
|
293536
|
+
try {
|
|
293537
|
+
unlinkSync(temporary);
|
|
293538
|
+
} catch {}
|
|
293539
|
+
throw new Error("Stream health state cannot be persisted", {
|
|
293540
|
+
cause: error
|
|
293541
|
+
});
|
|
293542
|
+
}
|
|
293543
|
+
}
|
|
293544
|
+
}
|
|
293545
|
+
function streamHealthPublisherConfigFromEnv(env = process.env) {
|
|
293546
|
+
if (env.CEX_BROKER_ARCHIVE_ENABLED !== "true") {
|
|
293547
|
+
throw new Error("Configured account user streams require CEX_BROKER_ARCHIVE_ENABLED=true");
|
|
293548
|
+
}
|
|
293549
|
+
const deploymentId = env.CEX_BROKER_DEPLOYMENT_ID?.trim();
|
|
293550
|
+
const forwarderUrl = env.CEX_BROKER_ARCHIVE_FORWARDER_URL?.trim();
|
|
293551
|
+
const statePath = env.CEX_BROKER_STREAM_HEALTH_STATE_PATH?.trim();
|
|
293552
|
+
if (!deploymentId || !forwarderUrl || !statePath) {
|
|
293553
|
+
throw new Error("Configured account user streams require deployment, forwarder, and persistent state configuration");
|
|
293554
|
+
}
|
|
293555
|
+
return {
|
|
293556
|
+
deploymentId,
|
|
293557
|
+
forwarderUrl,
|
|
293558
|
+
statePath,
|
|
293559
|
+
forwarderAuthToken: env.CEX_BROKER_ARCHIVE_FORWARDER_TOKEN?.trim() || undefined
|
|
293560
|
+
};
|
|
293561
|
+
}
|
|
293562
|
+
|
|
293563
|
+
// src/helpers/binance-user-data-stream.ts
|
|
293564
|
+
import { Buffer as Buffer2 } from "node:buffer";
|
|
293565
|
+
import { createHmac } from "node:crypto";
|
|
293566
|
+
var BINANCE_SPOT_WS_API_URL = "wss://ws-api.binance.com:443/ws-api/v3";
|
|
293567
|
+
var DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS = 16;
|
|
293568
|
+
var createWebSocket = (url2) => new wrapper_default(url2);
|
|
293569
|
+
var userDataRequestCounter = 0;
|
|
293570
|
+
function getExchangeString(exchange, key2) {
|
|
293571
|
+
const value = exchange[key2];
|
|
293572
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
293573
|
+
throw new Error(`Binance user-data stream requires exchange.${key2}`);
|
|
293574
|
+
}
|
|
293575
|
+
return value;
|
|
293576
|
+
}
|
|
293577
|
+
function sortedQuery(params) {
|
|
293578
|
+
return Object.entries(params).sort(([left], [right]) => left.localeCompare(right)).map(([key2, value]) => `${encodeURIComponent(key2)}=${encodeURIComponent(String(value))}`).join("&");
|
|
293579
|
+
}
|
|
293580
|
+
function signUserDataStreamParams(exchange, params) {
|
|
293581
|
+
const signParams = exchange.signParams;
|
|
293582
|
+
if (typeof signParams === "function") {
|
|
293583
|
+
return signParams.call(exchange, params);
|
|
293584
|
+
}
|
|
293585
|
+
const secret = getExchangeString(exchange, "secret");
|
|
293586
|
+
return {
|
|
293587
|
+
...params,
|
|
293588
|
+
signature: createHmac("sha256", secret).update(sortedQuery(params)).digest("hex")
|
|
293589
|
+
};
|
|
293590
|
+
}
|
|
293591
|
+
function getBinanceSpotWsApiUrl(exchange) {
|
|
293592
|
+
const urls = exchange.urls;
|
|
293593
|
+
return urls?.api?.ws?.["ws-api"]?.spot ?? BINANCE_SPOT_WS_API_URL;
|
|
293594
|
+
}
|
|
293595
|
+
function getRecord(value) {
|
|
293596
|
+
return typeof value === "object" && value !== null ? value : null;
|
|
293597
|
+
}
|
|
293598
|
+
function getMessage(value) {
|
|
293599
|
+
if (value instanceof Error) {
|
|
293600
|
+
return value.message;
|
|
293601
|
+
}
|
|
293602
|
+
if (typeof value === "string" && value.length > 0) {
|
|
293603
|
+
return value;
|
|
293604
|
+
}
|
|
293605
|
+
const record = getRecord(value);
|
|
293606
|
+
const message = record?.message;
|
|
293607
|
+
return typeof message === "string" && message.length > 0 ? message : null;
|
|
293608
|
+
}
|
|
293609
|
+
function getOptionalExchangeString(exchange, key2) {
|
|
293610
|
+
const value = exchange[key2];
|
|
293611
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
293612
|
+
}
|
|
293613
|
+
function redactDiagnosticMessage(message, secretValues) {
|
|
293614
|
+
let redacted = message;
|
|
293615
|
+
for (const value of secretValues) {
|
|
293616
|
+
if (value.length > 0) {
|
|
293617
|
+
redacted = redacted.split(value).join("[redacted]");
|
|
293618
|
+
}
|
|
293619
|
+
}
|
|
293620
|
+
return redacted.replace(/(\b(?:apiKey|secret|signature)\b\s*=\s*)[^\s&,;)]+/gi, "$1[redacted]").replace(/("(?:apiKey|secret|signature)"\s*:\s*")[^"]*(")/gi, "$1[redacted]$2");
|
|
293621
|
+
}
|
|
293622
|
+
function formatBinanceUserDataWebSocketError(event, secretValues) {
|
|
293623
|
+
const record = getRecord(event);
|
|
293624
|
+
const message = getMessage(record?.error) ?? getMessage(record?.message) ?? getMessage(event);
|
|
293625
|
+
const safeMessage = message === null ? null : redactDiagnosticMessage(message, secretValues);
|
|
293626
|
+
return new Error(safeMessage ? `Binance user-data WebSocket error: ${safeMessage}` : "Binance user-data WebSocket error");
|
|
293627
|
+
}
|
|
293628
|
+
function getCloseReason(value) {
|
|
293629
|
+
if (typeof value === "string") {
|
|
293630
|
+
return value.length > 0 ? value : null;
|
|
293631
|
+
}
|
|
293632
|
+
if (Buffer2.isBuffer(value)) {
|
|
293633
|
+
const reason = value.toString("utf8");
|
|
293634
|
+
return reason.length > 0 ? reason : null;
|
|
293635
|
+
}
|
|
293636
|
+
if (value instanceof Uint8Array) {
|
|
293637
|
+
const reason = Buffer2.from(value).toString("utf8");
|
|
293638
|
+
return reason.length > 0 ? reason : null;
|
|
293639
|
+
}
|
|
293640
|
+
return null;
|
|
293641
|
+
}
|
|
293642
|
+
function formatBinanceUserDataWebSocketClose(codeOrEvent, reasonOrUndefined, secretValues) {
|
|
293643
|
+
const record = getRecord(codeOrEvent);
|
|
293644
|
+
const code = record ? record.code : codeOrEvent;
|
|
293645
|
+
const reason = getCloseReason(record ? record.reason : reasonOrUndefined);
|
|
293646
|
+
const safeReason = reason === null ? null : redactDiagnosticMessage(reason, secretValues);
|
|
293647
|
+
const details = [
|
|
293648
|
+
typeof code === "number" || typeof code === "string" ? `code=${code}` : null,
|
|
293649
|
+
safeReason ? `reason=${safeReason}` : null
|
|
293650
|
+
].filter((detail) => detail !== null);
|
|
293651
|
+
return new Error(details.length > 0 ? `Binance user-data WebSocket closed unexpectedly (${details.join(", ")})` : "Binance user-data WebSocket closed unexpectedly");
|
|
293652
|
+
}
|
|
293653
|
+
function decodeMessageData(data) {
|
|
293654
|
+
if (typeof data === "string") {
|
|
293655
|
+
return data;
|
|
293656
|
+
}
|
|
293657
|
+
if (Buffer2.isBuffer(data)) {
|
|
293658
|
+
return data.toString("utf8");
|
|
293659
|
+
}
|
|
293660
|
+
if (data instanceof ArrayBuffer) {
|
|
293661
|
+
return Buffer2.from(data).toString("utf8");
|
|
293662
|
+
}
|
|
293663
|
+
if (ArrayBuffer.isView(data)) {
|
|
293664
|
+
return Buffer2.from(data.buffer, data.byteOffset, data.byteLength).toString("utf8");
|
|
293665
|
+
}
|
|
293666
|
+
if (Array.isArray(data) && data.every((item) => Buffer2.isBuffer(item))) {
|
|
293667
|
+
return Buffer2.concat(data).toString("utf8");
|
|
293668
|
+
}
|
|
293669
|
+
return data;
|
|
293670
|
+
}
|
|
293671
|
+
|
|
293672
|
+
class BinanceSpotUserDataStream {
|
|
293673
|
+
exchange;
|
|
293674
|
+
ws;
|
|
293675
|
+
secretValues;
|
|
293676
|
+
requestId = `user-data-${Date.now()}-${userDataRequestCounter++}`;
|
|
293677
|
+
maxBufferedEvents;
|
|
293678
|
+
observer;
|
|
293679
|
+
queue = [];
|
|
293680
|
+
waiters = [];
|
|
293681
|
+
closed = false;
|
|
293682
|
+
closeError = null;
|
|
293683
|
+
subscriptionId = null;
|
|
293684
|
+
constructor(exchange, options = {}) {
|
|
293685
|
+
this.exchange = exchange;
|
|
293686
|
+
this.maxBufferedEvents = options.maxBufferedEvents ?? DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS;
|
|
293687
|
+
this.observer = options.observer;
|
|
293688
|
+
this.secretValues = [
|
|
293689
|
+
getOptionalExchangeString(exchange, "apiKey"),
|
|
293690
|
+
getOptionalExchangeString(exchange, "secret")
|
|
293691
|
+
].filter((value) => value !== null);
|
|
293692
|
+
this.ws = createWebSocket(getBinanceSpotWsApiUrl(exchange));
|
|
293693
|
+
this.ws.on("open", () => {
|
|
293694
|
+
this.observer?.onConnected?.();
|
|
293695
|
+
this.subscribe();
|
|
293696
|
+
});
|
|
293697
|
+
this.ws.on("message", (data) => this.handleMessage(data));
|
|
293698
|
+
this.ws.on("error", (error) => this.fail(formatBinanceUserDataWebSocketError(error, this.secretValues), "transport_error"));
|
|
293699
|
+
this.ws.on("close", (code, reason) => this.handleClose(code, reason));
|
|
293700
|
+
}
|
|
293701
|
+
async* [Symbol.asyncIterator]() {
|
|
293702
|
+
while (true) {
|
|
293703
|
+
const event = await this.nextEvent();
|
|
293704
|
+
if (!event) {
|
|
293705
|
+
break;
|
|
292657
293706
|
}
|
|
293707
|
+
yield event;
|
|
293708
|
+
}
|
|
293709
|
+
}
|
|
293710
|
+
close() {
|
|
293711
|
+
if (this.closed) {
|
|
293712
|
+
return;
|
|
293713
|
+
}
|
|
293714
|
+
this.closed = true;
|
|
293715
|
+
this.queue.length = 0;
|
|
293716
|
+
try {
|
|
293717
|
+
this.ws.close();
|
|
292658
293718
|
} catch {}
|
|
293719
|
+
this.flushWaiters();
|
|
293720
|
+
}
|
|
293721
|
+
handleClose(code, reason) {
|
|
293722
|
+
if (this.closed) {
|
|
293723
|
+
return;
|
|
293724
|
+
}
|
|
293725
|
+
this.fail(formatBinanceUserDataWebSocketClose(code, reason, this.secretValues), "remote_closed");
|
|
293726
|
+
}
|
|
293727
|
+
subscribe() {
|
|
293728
|
+
const apiKey = getExchangeString(this.exchange, "apiKey");
|
|
293729
|
+
const signedParams = signUserDataStreamParams(this.exchange, {
|
|
293730
|
+
apiKey,
|
|
293731
|
+
timestamp: Date.now()
|
|
293732
|
+
});
|
|
293733
|
+
this.ws.send(JSON.stringify({
|
|
293734
|
+
id: this.requestId,
|
|
293735
|
+
method: "userDataStream.subscribe.signature",
|
|
293736
|
+
params: signedParams
|
|
293737
|
+
}));
|
|
292659
293738
|
}
|
|
292660
|
-
|
|
292661
|
-
if (
|
|
293739
|
+
handleMessage(data) {
|
|
293740
|
+
if (this.closed) {
|
|
292662
293741
|
return;
|
|
292663
|
-
for (const m of metricsList) {
|
|
292664
|
-
await this.insertMetric(m);
|
|
292665
293742
|
}
|
|
292666
|
-
|
|
292667
|
-
async recordCounter(metricName, value, labels, service = this.getServiceName()) {
|
|
292668
|
-
const provider = this.getProvider();
|
|
292669
|
-
if (!this.isOtelEnabled() || !provider)
|
|
292670
|
-
return;
|
|
293743
|
+
let message;
|
|
292671
293744
|
try {
|
|
292672
|
-
|
|
292673
|
-
|
|
292674
|
-
const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
|
|
292675
|
-
counter = meter.createCounter(metricName, { description: metricName });
|
|
292676
|
-
this.counters.set(metricName, counter);
|
|
292677
|
-
}
|
|
292678
|
-
counter.add(value, toAttributes(labels, service));
|
|
293745
|
+
const decodedData = decodeMessageData(data);
|
|
293746
|
+
message = typeof decodedData === "string" ? JSON.parse(decodedData) : decodedData;
|
|
292679
293747
|
} catch (error) {
|
|
292680
|
-
|
|
292681
|
-
}
|
|
292682
|
-
}
|
|
292683
|
-
async recordGauge(metricName, value, labels, service = this.getServiceName()) {
|
|
292684
|
-
const provider = this.getProvider();
|
|
292685
|
-
if (!this.isOtelEnabled() || !provider)
|
|
293748
|
+
this.fail(error instanceof Error ? error : new Error("Invalid Binance user-data message"), "protocol_error");
|
|
292686
293749
|
return;
|
|
292687
|
-
|
|
292688
|
-
|
|
292689
|
-
if (
|
|
292690
|
-
|
|
292691
|
-
|
|
292692
|
-
description: metricName
|
|
292693
|
-
});
|
|
292694
|
-
this.histograms.set(`gauge_${metricName}`, hist);
|
|
293750
|
+
}
|
|
293751
|
+
if ("id" in message && message.id === this.requestId) {
|
|
293752
|
+
if (message.status !== 200) {
|
|
293753
|
+
this.fail(new Error(message.error?.msg ?? message.error?.message ?? `Binance user-data subscription failed with status ${message.status}`), "auth_failed");
|
|
293754
|
+
return;
|
|
292695
293755
|
}
|
|
292696
|
-
|
|
292697
|
-
|
|
292698
|
-
|
|
293756
|
+
this.subscriptionId = message.result?.subscriptionId ?? null;
|
|
293757
|
+
this.observer?.onAuthenticated?.();
|
|
293758
|
+
return;
|
|
292699
293759
|
}
|
|
293760
|
+
if ("status" in message && typeof message.status === "number" && message.status !== 200) {
|
|
293761
|
+
const errorMessage = message.error?.msg ?? message.error?.message ?? `Binance user-data request failed with status ${message.status}`;
|
|
293762
|
+
const errorCode2 = message.error?.code;
|
|
293763
|
+
this.fail(new Error(typeof errorCode2 === "number" ? `${errorMessage} (code ${errorCode2})` : errorMessage), "protocol_error");
|
|
293764
|
+
return;
|
|
293765
|
+
}
|
|
293766
|
+
if (!("event" in message) || !message.event) {
|
|
293767
|
+
return;
|
|
293768
|
+
}
|
|
293769
|
+
const subscriptionId = message.subscriptionId ?? this.subscriptionId;
|
|
293770
|
+
if (subscriptionId === null || subscriptionId === undefined) {
|
|
293771
|
+
return;
|
|
293772
|
+
}
|
|
293773
|
+
this.push({ subscriptionId, event: message.event });
|
|
292700
293774
|
}
|
|
292701
|
-
|
|
292702
|
-
|
|
292703
|
-
|
|
293775
|
+
push(event) {
|
|
293776
|
+
if (this.closed) {
|
|
293777
|
+
return;
|
|
293778
|
+
}
|
|
293779
|
+
this.observer?.onEvent?.(event);
|
|
293780
|
+
const waiter = this.waiters.shift();
|
|
293781
|
+
if (waiter) {
|
|
293782
|
+
waiter.resolve(event);
|
|
293783
|
+
return;
|
|
293784
|
+
}
|
|
293785
|
+
if (this.queue.length >= this.maxBufferedEvents) {
|
|
293786
|
+
this.fail(new Error(`Binance user-data stream buffered event limit exceeded (${this.maxBufferedEvents}); downstream consumer is not keeping up`), "backpressure");
|
|
292704
293787
|
return;
|
|
292705
|
-
try {
|
|
292706
|
-
let state = this.observableGauges.get(metricName);
|
|
292707
|
-
if (!state) {
|
|
292708
|
-
const observations = new Map;
|
|
292709
|
-
const instrument = provider.getMeter("cex-broker-metrics", "1.0.0").createObservableGauge(metricName, { description: metricName });
|
|
292710
|
-
instrument.addCallback((result) => {
|
|
292711
|
-
for (const observation of observations.values()) {
|
|
292712
|
-
result.observe(observation.value, observation.attributes);
|
|
292713
|
-
}
|
|
292714
|
-
});
|
|
292715
|
-
state = { instrument, observations };
|
|
292716
|
-
this.observableGauges.set(metricName, state);
|
|
292717
|
-
}
|
|
292718
|
-
const attributes = toAttributes(labels, service);
|
|
292719
|
-
state.observations.set(stableAttributeKey(attributes), {
|
|
292720
|
-
value,
|
|
292721
|
-
attributes
|
|
292722
|
-
});
|
|
292723
|
-
} catch (error) {
|
|
292724
|
-
log.error("Failed to set observable gauge:", error);
|
|
292725
293788
|
}
|
|
293789
|
+
this.queue.push(event);
|
|
292726
293790
|
}
|
|
292727
|
-
|
|
292728
|
-
const
|
|
292729
|
-
if (
|
|
293791
|
+
nextEvent() {
|
|
293792
|
+
const event = this.queue.shift();
|
|
293793
|
+
if (event) {
|
|
293794
|
+
return Promise.resolve(event);
|
|
293795
|
+
}
|
|
293796
|
+
if (this.closeError) {
|
|
293797
|
+
return Promise.reject(this.closeError);
|
|
293798
|
+
}
|
|
293799
|
+
if (this.closed) {
|
|
293800
|
+
return Promise.resolve(null);
|
|
293801
|
+
}
|
|
293802
|
+
return new Promise((resolve, reject) => {
|
|
293803
|
+
this.waiters.push({ resolve, reject });
|
|
293804
|
+
});
|
|
293805
|
+
}
|
|
293806
|
+
fail(error, kind) {
|
|
293807
|
+
if (this.closeError) {
|
|
292730
293808
|
return;
|
|
293809
|
+
}
|
|
293810
|
+
this.closeError = error;
|
|
293811
|
+
this.observer?.onFailure?.({ kind, reason: error.message });
|
|
293812
|
+
this.closed = true;
|
|
293813
|
+
this.queue.length = 0;
|
|
293814
|
+
this.flushWaiters();
|
|
292731
293815
|
try {
|
|
292732
|
-
|
|
292733
|
-
|
|
292734
|
-
|
|
292735
|
-
|
|
292736
|
-
|
|
293816
|
+
this.ws.close();
|
|
293817
|
+
} catch {}
|
|
293818
|
+
}
|
|
293819
|
+
flushWaiters() {
|
|
293820
|
+
const error = this.closeError;
|
|
293821
|
+
for (const waiter of this.waiters.splice(0)) {
|
|
293822
|
+
if (error) {
|
|
293823
|
+
waiter.reject(error);
|
|
293824
|
+
} else {
|
|
293825
|
+
waiter.resolve(null);
|
|
292737
293826
|
}
|
|
292738
|
-
hist.record(value, toAttributes(labels, service));
|
|
292739
|
-
} catch (error) {
|
|
292740
|
-
log.error("Failed to record histogram:", error);
|
|
292741
293827
|
}
|
|
292742
293828
|
}
|
|
292743
293829
|
}
|
|
293830
|
+
function isBinanceBalanceUserDataEvent(event) {
|
|
293831
|
+
return event.e === "outboundAccountPosition" || event.e === "balanceUpdate" || event.e === "externalLockUpdate";
|
|
293832
|
+
}
|
|
293833
|
+
function isBinanceOrderUserDataEvent(event) {
|
|
293834
|
+
return event.e === "executionReport" || event.e === "listStatus";
|
|
293835
|
+
}
|
|
292744
293836
|
|
|
292745
|
-
|
|
292746
|
-
|
|
292747
|
-
|
|
292748
|
-
|
|
292749
|
-
|
|
292750
|
-
|
|
292751
|
-
|
|
292752
|
-
|
|
292753
|
-
|
|
292754
|
-
|
|
292755
|
-
|
|
292756
|
-
|
|
292757
|
-
|
|
292758
|
-
|
|
292759
|
-
|
|
292760
|
-
|
|
292761
|
-
|
|
292762
|
-
|
|
292763
|
-
|
|
292764
|
-
|
|
292765
|
-
|
|
292766
|
-
|
|
292767
|
-
|
|
292768
|
-
|
|
292769
|
-
|
|
292770
|
-
onProviderClosed() {
|
|
292771
|
-
this.logger = null;
|
|
293837
|
+
// src/helpers/user-data-stream-supervisor.ts
|
|
293838
|
+
var MAX_SUBSCRIBER_EVENTS = 16;
|
|
293839
|
+
function now3() {
|
|
293840
|
+
return new Date().toISOString();
|
|
293841
|
+
}
|
|
293842
|
+
function retryDelay(attempt) {
|
|
293843
|
+
return Math.min(1000 * 2 ** attempt, 30000);
|
|
293844
|
+
}
|
|
293845
|
+
function safeFailureReason(exchange, reason) {
|
|
293846
|
+
const secrets = [exchange.apiKey, exchange.secret].filter((value) => typeof value === "string" && value.length > 0);
|
|
293847
|
+
return redactSecretLiterals(reason, secrets).replace(/\s+/g, " ").trim().slice(0, 256);
|
|
293848
|
+
}
|
|
293849
|
+
|
|
293850
|
+
class Subscriber {
|
|
293851
|
+
kind;
|
|
293852
|
+
marketId;
|
|
293853
|
+
onClose;
|
|
293854
|
+
#queue = [];
|
|
293855
|
+
#waiters = [];
|
|
293856
|
+
#closed = false;
|
|
293857
|
+
#error = null;
|
|
293858
|
+
constructor(kind, marketId, onClose) {
|
|
293859
|
+
this.kind = kind;
|
|
293860
|
+
this.marketId = marketId;
|
|
293861
|
+
this.onClose = onClose;
|
|
292772
293862
|
}
|
|
292773
|
-
|
|
292774
|
-
if (
|
|
293863
|
+
push(message) {
|
|
293864
|
+
if (this.#closed || !this.#matches(message.event))
|
|
293865
|
+
return;
|
|
293866
|
+
const waiter = this.#waiters.shift();
|
|
293867
|
+
if (waiter) {
|
|
293868
|
+
waiter.resolve(message);
|
|
292775
293869
|
return;
|
|
292776
293870
|
}
|
|
292777
|
-
this.
|
|
293871
|
+
if (this.#queue.length >= MAX_SUBSCRIBER_EVENTS) {
|
|
293872
|
+
this.#fail(new Error("Configured account user-data subscriber fell behind"));
|
|
293873
|
+
return;
|
|
293874
|
+
}
|
|
293875
|
+
this.#queue.push(message);
|
|
292778
293876
|
}
|
|
292779
|
-
|
|
292780
|
-
|
|
292781
|
-
|
|
292782
|
-
|
|
292783
|
-
|
|
292784
|
-
|
|
292785
|
-
|
|
293877
|
+
close() {
|
|
293878
|
+
if (this.#closed)
|
|
293879
|
+
return;
|
|
293880
|
+
this.#closed = true;
|
|
293881
|
+
this.#queue.length = 0;
|
|
293882
|
+
this.onClose();
|
|
293883
|
+
for (const waiter of this.#waiters.splice(0))
|
|
293884
|
+
waiter.resolve(null);
|
|
292786
293885
|
}
|
|
292787
|
-
|
|
292788
|
-
|
|
292789
|
-
|
|
292790
|
-
|
|
292791
|
-
|
|
292792
|
-
|
|
293886
|
+
async* [Symbol.asyncIterator]() {
|
|
293887
|
+
while (true) {
|
|
293888
|
+
const event = await this.#next();
|
|
293889
|
+
if (!event)
|
|
293890
|
+
return;
|
|
293891
|
+
yield event;
|
|
293892
|
+
}
|
|
292793
293893
|
}
|
|
292794
|
-
|
|
292795
|
-
|
|
292796
|
-
|
|
292797
|
-
|
|
292798
|
-
|
|
293894
|
+
#matches(event) {
|
|
293895
|
+
if (this.kind === "balance")
|
|
293896
|
+
return isBinanceBalanceUserDataEvent(event);
|
|
293897
|
+
if (!isBinanceOrderUserDataEvent(event))
|
|
293898
|
+
return false;
|
|
293899
|
+
return !this.marketId || event.s === this.marketId;
|
|
292799
293900
|
}
|
|
292800
|
-
|
|
292801
|
-
const
|
|
292802
|
-
|
|
292803
|
-
|
|
292804
|
-
|
|
292805
|
-
|
|
293901
|
+
#next() {
|
|
293902
|
+
const event = this.#queue.shift();
|
|
293903
|
+
if (event)
|
|
293904
|
+
return Promise.resolve(event);
|
|
293905
|
+
if (this.#error)
|
|
293906
|
+
return Promise.reject(this.#error);
|
|
293907
|
+
if (this.#closed)
|
|
293908
|
+
return Promise.resolve(null);
|
|
293909
|
+
return new Promise((resolve, reject) => this.#waiters.push({ resolve, reject }));
|
|
293910
|
+
}
|
|
293911
|
+
#fail(error) {
|
|
293912
|
+
if (this.#closed)
|
|
293913
|
+
return;
|
|
293914
|
+
this.#closed = true;
|
|
293915
|
+
this.#error = error;
|
|
293916
|
+
this.#queue.length = 0;
|
|
293917
|
+
this.onClose();
|
|
293918
|
+
for (const waiter of this.#waiters.splice(0))
|
|
293919
|
+
waiter.reject(error);
|
|
293920
|
+
}
|
|
293921
|
+
}
|
|
293922
|
+
|
|
293923
|
+
class AccountWorker {
|
|
293924
|
+
exchangeName;
|
|
293925
|
+
account;
|
|
293926
|
+
onChange;
|
|
293927
|
+
#subscribers = new Set;
|
|
293928
|
+
#snapshot;
|
|
293929
|
+
#stopping = false;
|
|
293930
|
+
#stream = null;
|
|
293931
|
+
#retryTimer = null;
|
|
293932
|
+
#retryResolve = null;
|
|
293933
|
+
#run = null;
|
|
293934
|
+
#failureObserved = false;
|
|
293935
|
+
#attempts = 0;
|
|
293936
|
+
constructor(exchangeName, account, onChange) {
|
|
293937
|
+
this.exchangeName = exchangeName;
|
|
293938
|
+
this.account = account;
|
|
293939
|
+
this.onChange = onChange;
|
|
293940
|
+
const timestamp = now3();
|
|
293941
|
+
this.#snapshot = {
|
|
293942
|
+
exchange: exchangeName,
|
|
293943
|
+
accountSelector: account.label,
|
|
293944
|
+
accountRole: account.role,
|
|
293945
|
+
streamKind: "user_data",
|
|
293946
|
+
accountScope: "spot",
|
|
293947
|
+
registryStatus: "active",
|
|
293948
|
+
retiredAt: null,
|
|
293949
|
+
state: "connecting",
|
|
293950
|
+
stateChangedAt: timestamp,
|
|
293951
|
+
lastConnectedAt: null,
|
|
293952
|
+
lastAuthenticatedAt: null,
|
|
293953
|
+
lastReceivedAt: null,
|
|
293954
|
+
connectAttemptCount: "0",
|
|
293955
|
+
reconnectCount: "0",
|
|
293956
|
+
errorCount: "0",
|
|
293957
|
+
lastFailureKind: "none",
|
|
293958
|
+
lastFailureReason: "",
|
|
293959
|
+
trafficMode: "event_driven",
|
|
293960
|
+
sourceWatermark: null
|
|
292806
293961
|
};
|
|
292807
293962
|
}
|
|
292808
|
-
|
|
292809
|
-
|
|
292810
|
-
|
|
292811
|
-
|
|
292812
|
-
|
|
293963
|
+
start() {
|
|
293964
|
+
if (this.exchangeName !== "binance") {
|
|
293965
|
+
this.#fail("unsupported_connector", "Configured exchange has no user-data supervisor");
|
|
293966
|
+
return;
|
|
293967
|
+
}
|
|
293968
|
+
this.#run = this.#connectLoop();
|
|
292813
293969
|
}
|
|
292814
|
-
|
|
292815
|
-
|
|
292816
|
-
|
|
292817
|
-
|
|
292818
|
-
|
|
292819
|
-
}
|
|
292820
|
-
function getOtelHostFromEnv() {
|
|
292821
|
-
return process.env.CEX_BROKER_OTEL_HOST ?? process.env.CEX_BROKER_CLICKHOUSE_HOST;
|
|
292822
|
-
}
|
|
292823
|
-
function getOtelPortFromEnv() {
|
|
292824
|
-
const port = process.env.CEX_BROKER_OTEL_PORT ?? process.env.CEX_BROKER_CLICKHOUSE_PORT;
|
|
292825
|
-
return port ? Number.parseInt(port, 10) : undefined;
|
|
292826
|
-
}
|
|
292827
|
-
function getOtelProtocolFromEnv() {
|
|
292828
|
-
const protocol = process.env.CEX_BROKER_OTEL_PROTOCOL ?? process.env.CEX_BROKER_CLICKHOUSE_PROTOCOL;
|
|
292829
|
-
return protocol || "http";
|
|
292830
|
-
}
|
|
292831
|
-
function createOtelMetricsFromEnv(options = {}) {
|
|
292832
|
-
const otlpEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
|
|
292833
|
-
const serviceName = process.env.OTEL_SERVICE_NAME || options.defaultServiceName || DEFAULT_SERVICE;
|
|
292834
|
-
if (otlpEndpoint) {
|
|
292835
|
-
return new OtelMetrics({
|
|
292836
|
-
otlpEndpoint,
|
|
292837
|
-
serviceName
|
|
293970
|
+
subscribe(options) {
|
|
293971
|
+
if (this.#stopping)
|
|
293972
|
+
throw new Error("Configured account user-data supervisor is stopping");
|
|
293973
|
+
const subscriber = new Subscriber(options.kind, options.marketId, () => {
|
|
293974
|
+
this.#subscribers.delete(subscriber);
|
|
292838
293975
|
});
|
|
293976
|
+
this.#subscribers.add(subscriber);
|
|
293977
|
+
return subscriber;
|
|
293978
|
+
}
|
|
293979
|
+
snapshot() {
|
|
293980
|
+
return { ...this.#snapshot };
|
|
293981
|
+
}
|
|
293982
|
+
async stop() {
|
|
293983
|
+
this.#stopping = true;
|
|
293984
|
+
if (this.#retryTimer)
|
|
293985
|
+
clearTimeout(this.#retryTimer);
|
|
293986
|
+
this.#retryTimer = null;
|
|
293987
|
+
this.#retryResolve?.();
|
|
293988
|
+
this.#retryResolve = null;
|
|
293989
|
+
this.#stream?.close();
|
|
293990
|
+
await this.#run;
|
|
293991
|
+
this.#transition("disconnected", "shutdown", "Broker shutdown");
|
|
293992
|
+
for (const subscriber of [...this.#subscribers])
|
|
293993
|
+
subscriber.close();
|
|
293994
|
+
}
|
|
293995
|
+
#transition(state, failureKind, failureReason) {
|
|
293996
|
+
const timestamp = now3();
|
|
293997
|
+
if (this.#snapshot.state !== state) {
|
|
293998
|
+
this.#snapshot.state = state;
|
|
293999
|
+
this.#snapshot.stateChangedAt = timestamp;
|
|
294000
|
+
}
|
|
294001
|
+
if (failureKind) {
|
|
294002
|
+
this.#snapshot.lastFailureKind = failureKind;
|
|
294003
|
+
this.#snapshot.lastFailureReason = failureReason ?? "";
|
|
294004
|
+
}
|
|
294005
|
+
this.onChange();
|
|
294006
|
+
}
|
|
294007
|
+
#connected() {
|
|
294008
|
+
this.#snapshot.lastConnectedAt = now3();
|
|
294009
|
+
this.#transition("connected");
|
|
294010
|
+
}
|
|
294011
|
+
#authenticated() {
|
|
294012
|
+
this.#snapshot.lastAuthenticatedAt = now3();
|
|
294013
|
+
this.onChange();
|
|
294014
|
+
}
|
|
294015
|
+
#received(message) {
|
|
294016
|
+
this.#snapshot.lastReceivedAt = now3();
|
|
294017
|
+
const eventTimestamp = message.event.E;
|
|
294018
|
+
this.#snapshot.sourceWatermark = typeof eventTimestamp === "number" || typeof eventTimestamp === "string" ? String(eventTimestamp).slice(0, 512) : null;
|
|
294019
|
+
for (const subscriber of this.#subscribers)
|
|
294020
|
+
subscriber.push(message);
|
|
294021
|
+
this.onChange();
|
|
294022
|
+
}
|
|
294023
|
+
#fail(kind, reason) {
|
|
294024
|
+
this.#failureObserved = true;
|
|
294025
|
+
this.#snapshot.errorCount = (BigInt(this.#snapshot.errorCount) + 1n).toString();
|
|
294026
|
+
this.#transition("error", kind, safeFailureReason(this.account.exchange, reason));
|
|
294027
|
+
}
|
|
294028
|
+
async#connectLoop() {
|
|
294029
|
+
while (!this.#stopping) {
|
|
294030
|
+
if (this.#attempts > 0) {
|
|
294031
|
+
this.#snapshot.reconnectCount = (BigInt(this.#snapshot.reconnectCount) + 1n).toString();
|
|
294032
|
+
}
|
|
294033
|
+
this.#attempts += 1;
|
|
294034
|
+
this.#snapshot.connectAttemptCount = String(this.#attempts);
|
|
294035
|
+
this.#failureObserved = false;
|
|
294036
|
+
this.#transition("connecting");
|
|
294037
|
+
const stream4 = new BinanceSpotUserDataStream(this.account.exchange, {
|
|
294038
|
+
observer: {
|
|
294039
|
+
onConnected: () => this.#connected(),
|
|
294040
|
+
onAuthenticated: () => this.#authenticated(),
|
|
294041
|
+
onEvent: (message) => this.#received(message),
|
|
294042
|
+
onFailure: (failure) => this.#handleStreamFailure(failure)
|
|
294043
|
+
}
|
|
294044
|
+
});
|
|
294045
|
+
this.#stream = stream4;
|
|
294046
|
+
try {
|
|
294047
|
+
for await (const _event of stream4) {}
|
|
294048
|
+
} catch (error) {
|
|
294049
|
+
if (!this.#stopping && !this.#failureObserved) {
|
|
294050
|
+
this.#fail("transport_error", error instanceof Error ? error.message : "User-data stream failed");
|
|
294051
|
+
}
|
|
294052
|
+
} finally {
|
|
294053
|
+
stream4.close();
|
|
294054
|
+
if (this.#stream === stream4)
|
|
294055
|
+
this.#stream = null;
|
|
294056
|
+
}
|
|
294057
|
+
if (!this.#stopping)
|
|
294058
|
+
await this.#waitForRetry();
|
|
294059
|
+
}
|
|
292839
294060
|
}
|
|
292840
|
-
|
|
292841
|
-
|
|
294061
|
+
#handleStreamFailure(failure) {
|
|
294062
|
+
this.#fail(failure.kind, failure.reason);
|
|
292842
294063
|
}
|
|
292843
|
-
|
|
292844
|
-
|
|
292845
|
-
|
|
292846
|
-
|
|
292847
|
-
|
|
292848
|
-
|
|
292849
|
-
|
|
292850
|
-
|
|
292851
|
-
|
|
292852
|
-
|
|
292853
|
-
return new OtelMetrics(config);
|
|
292854
|
-
}
|
|
292855
|
-
function stableAttributeKey(attributes) {
|
|
292856
|
-
return JSON.stringify(Object.entries(attributes).sort(([left], [right]) => left.localeCompare(right)));
|
|
292857
|
-
}
|
|
292858
|
-
function createOtelLogsFromEnv() {
|
|
292859
|
-
const logsEndpoint = process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
|
|
292860
|
-
const genericEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
|
|
292861
|
-
const host = getOtelHostFromEnv();
|
|
292862
|
-
if (logsEndpoint) {
|
|
292863
|
-
return new OtelLogs({
|
|
292864
|
-
serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
|
|
294064
|
+
#waitForRetry() {
|
|
294065
|
+
return new Promise((resolve) => {
|
|
294066
|
+
const delay = retryDelay(Math.max(this.#attempts - 1, 0));
|
|
294067
|
+
this.#retryResolve = resolve;
|
|
294068
|
+
this.#retryTimer = setTimeout(() => {
|
|
294069
|
+
this.#retryTimer = null;
|
|
294070
|
+
this.#retryResolve = null;
|
|
294071
|
+
resolve();
|
|
294072
|
+
}, delay);
|
|
294073
|
+
this.#retryTimer.unref?.();
|
|
292865
294074
|
});
|
|
292866
294075
|
}
|
|
292867
|
-
|
|
292868
|
-
|
|
292869
|
-
|
|
292870
|
-
|
|
292871
|
-
|
|
294076
|
+
}
|
|
294077
|
+
|
|
294078
|
+
class UserDataStreamSupervisor {
|
|
294079
|
+
options;
|
|
294080
|
+
#workers = new Map;
|
|
294081
|
+
#started = false;
|
|
294082
|
+
constructor(options) {
|
|
294083
|
+
this.options = options;
|
|
294084
|
+
for (const [exchange, pool] of Object.entries(options.brokers)) {
|
|
294085
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
294086
|
+
const normalizedExchange = exchange.trim().toLowerCase();
|
|
294087
|
+
const worker = new AccountWorker(normalizedExchange, account, () => this.#publish());
|
|
294088
|
+
this.#workers.set(`${normalizedExchange}|${account.label}`, worker);
|
|
294089
|
+
}
|
|
294090
|
+
}
|
|
294091
|
+
if (this.#workers.size === 0) {
|
|
294092
|
+
throw new Error("User-data supervisor requires at least one configured account");
|
|
294093
|
+
}
|
|
292872
294094
|
}
|
|
292873
|
-
|
|
292874
|
-
|
|
294095
|
+
start() {
|
|
294096
|
+
if (this.#started)
|
|
294097
|
+
return;
|
|
294098
|
+
this.#started = true;
|
|
294099
|
+
this.options.publisher.start();
|
|
294100
|
+
for (const worker of this.#workers.values())
|
|
294101
|
+
worker.start();
|
|
294102
|
+
this.#publish();
|
|
294103
|
+
}
|
|
294104
|
+
subscribe(options) {
|
|
294105
|
+
const exchange = options.exchange.trim().toLowerCase();
|
|
294106
|
+
const worker = this.#workers.get(`${exchange}|${options.accountSelector}`);
|
|
294107
|
+
if (!worker)
|
|
294108
|
+
throw new Error("Configured account user-data stream is unavailable");
|
|
294109
|
+
return worker.subscribe({ kind: options.kind, marketId: options.marketId });
|
|
294110
|
+
}
|
|
294111
|
+
async close() {
|
|
294112
|
+
for (const worker of this.#workers.values())
|
|
294113
|
+
await worker.stop();
|
|
294114
|
+
await this.options.publisher.close(this.#snapshots());
|
|
294115
|
+
}
|
|
294116
|
+
#snapshots() {
|
|
294117
|
+
return [...this.#workers.values()].map((worker) => worker.snapshot());
|
|
294118
|
+
}
|
|
294119
|
+
#publish() {
|
|
294120
|
+
if (!this.#started)
|
|
294121
|
+
return;
|
|
294122
|
+
this.options.publisher.publish(this.#snapshots());
|
|
292875
294123
|
}
|
|
292876
|
-
const port = getOtelPortFromEnv();
|
|
292877
|
-
const config = {
|
|
292878
|
-
host,
|
|
292879
|
-
port: port ?? DEFAULT_OTLP_PORT,
|
|
292880
|
-
protocol: getOtelProtocolFromEnv(),
|
|
292881
|
-
serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
|
|
292882
|
-
};
|
|
292883
|
-
return new OtelLogs(config);
|
|
292884
294124
|
}
|
|
292885
294125
|
|
|
292886
294126
|
// src/server.ts
|
|
@@ -293723,9 +294963,9 @@ function floatSafeRemainder(val, step) {
|
|
|
293723
294963
|
return valInt % stepInt / 10 ** decCount;
|
|
293724
294964
|
}
|
|
293725
294965
|
var EVALUATING = Symbol("evaluating");
|
|
293726
|
-
function defineLazy(object,
|
|
294966
|
+
function defineLazy(object, key2, getter) {
|
|
293727
294967
|
let value = undefined;
|
|
293728
|
-
Object.defineProperty(object,
|
|
294968
|
+
Object.defineProperty(object, key2, {
|
|
293729
294969
|
get() {
|
|
293730
294970
|
if (value === EVALUATING) {
|
|
293731
294971
|
return;
|
|
@@ -293737,7 +294977,7 @@ function defineLazy(object, key, getter) {
|
|
|
293737
294977
|
return value;
|
|
293738
294978
|
},
|
|
293739
294979
|
set(v) {
|
|
293740
|
-
Object.defineProperty(object,
|
|
294980
|
+
Object.defineProperty(object, key2, {
|
|
293741
294981
|
value: v
|
|
293742
294982
|
});
|
|
293743
294983
|
},
|
|
@@ -293769,11 +295009,11 @@ function cloneDef(schema) {
|
|
|
293769
295009
|
function getElementAtPath(obj, path) {
|
|
293770
295010
|
if (!path)
|
|
293771
295011
|
return obj;
|
|
293772
|
-
return path.reduce((acc,
|
|
295012
|
+
return path.reduce((acc, key2) => acc?.[key2], obj);
|
|
293773
295013
|
}
|
|
293774
295014
|
function promiseAllObject(promisesObj) {
|
|
293775
295015
|
const keys2 = Object.keys(promisesObj);
|
|
293776
|
-
const promises = keys2.map((
|
|
295016
|
+
const promises = keys2.map((key2) => promisesObj[key2]);
|
|
293777
295017
|
return Promise.all(promises).then((results) => {
|
|
293778
295018
|
const resolvedObj = {};
|
|
293779
295019
|
for (let i2 = 0;i2 < keys2.length; i2++) {
|
|
@@ -293837,8 +295077,8 @@ function shallowClone(o) {
|
|
|
293837
295077
|
}
|
|
293838
295078
|
function numKeys(data) {
|
|
293839
295079
|
let keyCount = 0;
|
|
293840
|
-
for (const
|
|
293841
|
-
if (Object.prototype.hasOwnProperty.call(data,
|
|
295080
|
+
for (const key2 in data) {
|
|
295081
|
+
if (Object.prototype.hasOwnProperty.call(data, key2)) {
|
|
293842
295082
|
keyCount++;
|
|
293843
295083
|
}
|
|
293844
295084
|
}
|
|
@@ -293981,13 +295221,13 @@ function pick(schema, mask2) {
|
|
|
293981
295221
|
const def = mergeDefs(schema._zod.def, {
|
|
293982
295222
|
get shape() {
|
|
293983
295223
|
const newShape = {};
|
|
293984
|
-
for (const
|
|
293985
|
-
if (!(
|
|
293986
|
-
throw new Error(`Unrecognized key: "${
|
|
295224
|
+
for (const key2 in mask2) {
|
|
295225
|
+
if (!(key2 in currDef.shape)) {
|
|
295226
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
293987
295227
|
}
|
|
293988
|
-
if (!mask2[
|
|
295228
|
+
if (!mask2[key2])
|
|
293989
295229
|
continue;
|
|
293990
|
-
newShape[
|
|
295230
|
+
newShape[key2] = currDef.shape[key2];
|
|
293991
295231
|
}
|
|
293992
295232
|
assignProp(this, "shape", newShape);
|
|
293993
295233
|
return newShape;
|
|
@@ -294006,13 +295246,13 @@ function omit5(schema, mask2) {
|
|
|
294006
295246
|
const def = mergeDefs(schema._zod.def, {
|
|
294007
295247
|
get shape() {
|
|
294008
295248
|
const newShape = { ...schema._zod.def.shape };
|
|
294009
|
-
for (const
|
|
294010
|
-
if (!(
|
|
294011
|
-
throw new Error(`Unrecognized key: "${
|
|
295249
|
+
for (const key2 in mask2) {
|
|
295250
|
+
if (!(key2 in currDef.shape)) {
|
|
295251
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
294012
295252
|
}
|
|
294013
|
-
if (!mask2[
|
|
295253
|
+
if (!mask2[key2])
|
|
294014
295254
|
continue;
|
|
294015
|
-
delete newShape[
|
|
295255
|
+
delete newShape[key2];
|
|
294016
295256
|
}
|
|
294017
295257
|
assignProp(this, "shape", newShape);
|
|
294018
295258
|
return newShape;
|
|
@@ -294029,8 +295269,8 @@ function extend4(schema, shape) {
|
|
|
294029
295269
|
const hasChecks = checks && checks.length > 0;
|
|
294030
295270
|
if (hasChecks) {
|
|
294031
295271
|
const existingShape = schema._zod.def.shape;
|
|
294032
|
-
for (const
|
|
294033
|
-
if (Object.getOwnPropertyDescriptor(existingShape,
|
|
295272
|
+
for (const key2 in shape) {
|
|
295273
|
+
if (Object.getOwnPropertyDescriptor(existingShape, key2) !== undefined) {
|
|
294034
295274
|
throw new Error("Cannot overwrite keys on object schemas containing refinements. Use `.safeExtend()` instead.");
|
|
294035
295275
|
}
|
|
294036
295276
|
}
|
|
@@ -294083,23 +295323,23 @@ function partial(Class, schema, mask2) {
|
|
|
294083
295323
|
const oldShape = schema._zod.def.shape;
|
|
294084
295324
|
const shape = { ...oldShape };
|
|
294085
295325
|
if (mask2) {
|
|
294086
|
-
for (const
|
|
294087
|
-
if (!(
|
|
294088
|
-
throw new Error(`Unrecognized key: "${
|
|
295326
|
+
for (const key2 in mask2) {
|
|
295327
|
+
if (!(key2 in oldShape)) {
|
|
295328
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
294089
295329
|
}
|
|
294090
|
-
if (!mask2[
|
|
295330
|
+
if (!mask2[key2])
|
|
294091
295331
|
continue;
|
|
294092
|
-
shape[
|
|
295332
|
+
shape[key2] = Class ? new Class({
|
|
294093
295333
|
type: "optional",
|
|
294094
|
-
innerType: oldShape[
|
|
294095
|
-
}) : oldShape[
|
|
295334
|
+
innerType: oldShape[key2]
|
|
295335
|
+
}) : oldShape[key2];
|
|
294096
295336
|
}
|
|
294097
295337
|
} else {
|
|
294098
|
-
for (const
|
|
294099
|
-
shape[
|
|
295338
|
+
for (const key2 in oldShape) {
|
|
295339
|
+
shape[key2] = Class ? new Class({
|
|
294100
295340
|
type: "optional",
|
|
294101
|
-
innerType: oldShape[
|
|
294102
|
-
}) : oldShape[
|
|
295341
|
+
innerType: oldShape[key2]
|
|
295342
|
+
}) : oldShape[key2];
|
|
294103
295343
|
}
|
|
294104
295344
|
}
|
|
294105
295345
|
assignProp(this, "shape", shape);
|
|
@@ -294115,22 +295355,22 @@ function required(Class, schema, mask2) {
|
|
|
294115
295355
|
const oldShape = schema._zod.def.shape;
|
|
294116
295356
|
const shape = { ...oldShape };
|
|
294117
295357
|
if (mask2) {
|
|
294118
|
-
for (const
|
|
294119
|
-
if (!(
|
|
294120
|
-
throw new Error(`Unrecognized key: "${
|
|
295358
|
+
for (const key2 in mask2) {
|
|
295359
|
+
if (!(key2 in shape)) {
|
|
295360
|
+
throw new Error(`Unrecognized key: "${key2}"`);
|
|
294121
295361
|
}
|
|
294122
|
-
if (!mask2[
|
|
295362
|
+
if (!mask2[key2])
|
|
294123
295363
|
continue;
|
|
294124
|
-
shape[
|
|
295364
|
+
shape[key2] = new Class({
|
|
294125
295365
|
type: "nonoptional",
|
|
294126
|
-
innerType: oldShape[
|
|
295366
|
+
innerType: oldShape[key2]
|
|
294127
295367
|
});
|
|
294128
295368
|
}
|
|
294129
295369
|
} else {
|
|
294130
|
-
for (const
|
|
294131
|
-
shape[
|
|
295370
|
+
for (const key2 in oldShape) {
|
|
295371
|
+
shape[key2] = new Class({
|
|
294132
295372
|
type: "nonoptional",
|
|
294133
|
-
innerType: oldShape[
|
|
295373
|
+
innerType: oldShape[key2]
|
|
294134
295374
|
});
|
|
294135
295375
|
}
|
|
294136
295376
|
}
|
|
@@ -295880,19 +297120,19 @@ var $ZodArray = /* @__PURE__ */ $constructor("$ZodArray", (inst, def) => {
|
|
|
295880
297120
|
return payload;
|
|
295881
297121
|
};
|
|
295882
297122
|
});
|
|
295883
|
-
function handlePropertyResult(result, final,
|
|
297123
|
+
function handlePropertyResult(result, final, key2, input, isOptionalOut) {
|
|
295884
297124
|
if (result.issues.length) {
|
|
295885
|
-
if (isOptionalOut && !(
|
|
297125
|
+
if (isOptionalOut && !(key2 in input)) {
|
|
295886
297126
|
return;
|
|
295887
297127
|
}
|
|
295888
|
-
final.issues.push(...prefixIssues(
|
|
297128
|
+
final.issues.push(...prefixIssues(key2, result.issues));
|
|
295889
297129
|
}
|
|
295890
297130
|
if (result.value === undefined) {
|
|
295891
|
-
if (
|
|
295892
|
-
final.value[
|
|
297131
|
+
if (key2 in input) {
|
|
297132
|
+
final.value[key2] = undefined;
|
|
295893
297133
|
}
|
|
295894
297134
|
} else {
|
|
295895
|
-
final.value[
|
|
297135
|
+
final.value[key2] = result.value;
|
|
295896
297136
|
}
|
|
295897
297137
|
}
|
|
295898
297138
|
function normalizeDef(def) {
|
|
@@ -295917,18 +297157,18 @@ function handleCatchall(proms, input, payload, ctx, def, inst) {
|
|
|
295917
297157
|
const _catchall = def.catchall._zod;
|
|
295918
297158
|
const t = _catchall.def.type;
|
|
295919
297159
|
const isOptionalOut = _catchall.optout === "optional";
|
|
295920
|
-
for (const
|
|
295921
|
-
if (keySet.has(
|
|
297160
|
+
for (const key2 in input) {
|
|
297161
|
+
if (keySet.has(key2))
|
|
295922
297162
|
continue;
|
|
295923
297163
|
if (t === "never") {
|
|
295924
|
-
unrecognized.push(
|
|
297164
|
+
unrecognized.push(key2);
|
|
295925
297165
|
continue;
|
|
295926
297166
|
}
|
|
295927
|
-
const r = _catchall.run({ value: input[
|
|
297167
|
+
const r = _catchall.run({ value: input[key2], issues: [] }, ctx);
|
|
295928
297168
|
if (r instanceof Promise) {
|
|
295929
|
-
proms.push(r.then((r2) => handlePropertyResult(r2, payload,
|
|
297169
|
+
proms.push(r.then((r2) => handlePropertyResult(r2, payload, key2, input, isOptionalOut)));
|
|
295930
297170
|
} else {
|
|
295931
|
-
handlePropertyResult(r, payload,
|
|
297171
|
+
handlePropertyResult(r, payload, key2, input, isOptionalOut);
|
|
295932
297172
|
}
|
|
295933
297173
|
}
|
|
295934
297174
|
if (unrecognized.length) {
|
|
@@ -295964,12 +297204,12 @@ var $ZodObject = /* @__PURE__ */ $constructor("$ZodObject", (inst, def) => {
|
|
|
295964
297204
|
defineLazy(inst._zod, "propValues", () => {
|
|
295965
297205
|
const shape = def.shape;
|
|
295966
297206
|
const propValues = {};
|
|
295967
|
-
for (const
|
|
295968
|
-
const field = shape[
|
|
297207
|
+
for (const key2 in shape) {
|
|
297208
|
+
const field = shape[key2]._zod;
|
|
295969
297209
|
if (field.values) {
|
|
295970
|
-
propValues[
|
|
297210
|
+
propValues[key2] ?? (propValues[key2] = new Set);
|
|
295971
297211
|
for (const v of field.values)
|
|
295972
|
-
propValues[
|
|
297212
|
+
propValues[key2].add(v);
|
|
295973
297213
|
}
|
|
295974
297214
|
}
|
|
295975
297215
|
return propValues;
|
|
@@ -295992,14 +297232,14 @@ var $ZodObject = /* @__PURE__ */ $constructor("$ZodObject", (inst, def) => {
|
|
|
295992
297232
|
payload.value = {};
|
|
295993
297233
|
const proms = [];
|
|
295994
297234
|
const shape = value.shape;
|
|
295995
|
-
for (const
|
|
295996
|
-
const el = shape[
|
|
297235
|
+
for (const key2 of value.keys) {
|
|
297236
|
+
const el = shape[key2];
|
|
295997
297237
|
const isOptionalOut = el._zod.optout === "optional";
|
|
295998
|
-
const r = el._zod.run({ value: input[
|
|
297238
|
+
const r = el._zod.run({ value: input[key2], issues: [] }, ctx);
|
|
295999
297239
|
if (r instanceof Promise) {
|
|
296000
|
-
proms.push(r.then((r2) => handlePropertyResult(r2, payload,
|
|
297240
|
+
proms.push(r.then((r2) => handlePropertyResult(r2, payload, key2, input, isOptionalOut)));
|
|
296001
297241
|
} else {
|
|
296002
|
-
handlePropertyResult(r, payload,
|
|
297242
|
+
handlePropertyResult(r, payload, key2, input, isOptionalOut);
|
|
296003
297243
|
}
|
|
296004
297244
|
}
|
|
296005
297245
|
if (!catchall) {
|
|
@@ -296015,23 +297255,23 @@ var $ZodObjectJIT = /* @__PURE__ */ $constructor("$ZodObjectJIT", (inst, def) =>
|
|
|
296015
297255
|
const generateFastpass = (shape) => {
|
|
296016
297256
|
const doc = new Doc(["shape", "payload", "ctx"]);
|
|
296017
297257
|
const normalized = _normalized.value;
|
|
296018
|
-
const parseStr = (
|
|
296019
|
-
const k = esc(
|
|
297258
|
+
const parseStr = (key2) => {
|
|
297259
|
+
const k = esc(key2);
|
|
296020
297260
|
return `shape[${k}]._zod.run({ value: input[${k}], issues: [] }, ctx)`;
|
|
296021
297261
|
};
|
|
296022
297262
|
doc.write(`const input = payload.value;`);
|
|
296023
297263
|
const ids = Object.create(null);
|
|
296024
|
-
let
|
|
296025
|
-
for (const
|
|
296026
|
-
ids[
|
|
297264
|
+
let counter2 = 0;
|
|
297265
|
+
for (const key2 of normalized.keys) {
|
|
297266
|
+
ids[key2] = `key_${counter2++}`;
|
|
296027
297267
|
}
|
|
296028
297268
|
doc.write(`const newResult = {};`);
|
|
296029
|
-
for (const
|
|
296030
|
-
const id2 = ids[
|
|
296031
|
-
const k = esc(
|
|
296032
|
-
const schema = shape[
|
|
297269
|
+
for (const key2 of normalized.keys) {
|
|
297270
|
+
const id2 = ids[key2];
|
|
297271
|
+
const k = esc(key2);
|
|
297272
|
+
const schema = shape[key2];
|
|
296033
297273
|
const isOptionalOut = schema?._zod?.optout === "optional";
|
|
296034
|
-
doc.write(`const ${id2} = ${parseStr(
|
|
297274
|
+
doc.write(`const ${id2} = ${parseStr(key2)};`);
|
|
296035
297275
|
if (isOptionalOut) {
|
|
296036
297276
|
doc.write(`
|
|
296037
297277
|
if (${id2}.issues.length) {
|
|
@@ -296317,17 +297557,17 @@ function mergeValues(a, b2) {
|
|
|
296317
297557
|
}
|
|
296318
297558
|
if (isPlainObject2(a) && isPlainObject2(b2)) {
|
|
296319
297559
|
const bKeys = Object.keys(b2);
|
|
296320
|
-
const sharedKeys = Object.keys(a).filter((
|
|
297560
|
+
const sharedKeys = Object.keys(a).filter((key2) => bKeys.indexOf(key2) !== -1);
|
|
296321
297561
|
const newObj = { ...a, ...b2 };
|
|
296322
|
-
for (const
|
|
296323
|
-
const sharedValue = mergeValues(a[
|
|
297562
|
+
for (const key2 of sharedKeys) {
|
|
297563
|
+
const sharedValue = mergeValues(a[key2], b2[key2]);
|
|
296324
297564
|
if (!sharedValue.valid) {
|
|
296325
297565
|
return {
|
|
296326
297566
|
valid: false,
|
|
296327
|
-
mergeErrorPath: [
|
|
297567
|
+
mergeErrorPath: [key2, ...sharedValue.mergeErrorPath]
|
|
296328
297568
|
};
|
|
296329
297569
|
}
|
|
296330
|
-
newObj[
|
|
297570
|
+
newObj[key2] = sharedValue.data;
|
|
296331
297571
|
}
|
|
296332
297572
|
return { valid: true, data: newObj };
|
|
296333
297573
|
}
|
|
@@ -296483,30 +297723,30 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
|
|
|
296483
297723
|
if (values2) {
|
|
296484
297724
|
payload.value = {};
|
|
296485
297725
|
const recordKeys = new Set;
|
|
296486
|
-
for (const
|
|
296487
|
-
if (typeof
|
|
296488
|
-
recordKeys.add(typeof
|
|
296489
|
-
const result = def.valueType._zod.run({ value: input[
|
|
297726
|
+
for (const key2 of values2) {
|
|
297727
|
+
if (typeof key2 === "string" || typeof key2 === "number" || typeof key2 === "symbol") {
|
|
297728
|
+
recordKeys.add(typeof key2 === "number" ? key2.toString() : key2);
|
|
297729
|
+
const result = def.valueType._zod.run({ value: input[key2], issues: [] }, ctx);
|
|
296490
297730
|
if (result instanceof Promise) {
|
|
296491
297731
|
proms.push(result.then((result2) => {
|
|
296492
297732
|
if (result2.issues.length) {
|
|
296493
|
-
payload.issues.push(...prefixIssues(
|
|
297733
|
+
payload.issues.push(...prefixIssues(key2, result2.issues));
|
|
296494
297734
|
}
|
|
296495
|
-
payload.value[
|
|
297735
|
+
payload.value[key2] = result2.value;
|
|
296496
297736
|
}));
|
|
296497
297737
|
} else {
|
|
296498
297738
|
if (result.issues.length) {
|
|
296499
|
-
payload.issues.push(...prefixIssues(
|
|
297739
|
+
payload.issues.push(...prefixIssues(key2, result.issues));
|
|
296500
297740
|
}
|
|
296501
|
-
payload.value[
|
|
297741
|
+
payload.value[key2] = result.value;
|
|
296502
297742
|
}
|
|
296503
297743
|
}
|
|
296504
297744
|
}
|
|
296505
297745
|
let unrecognized;
|
|
296506
|
-
for (const
|
|
296507
|
-
if (!recordKeys.has(
|
|
297746
|
+
for (const key2 in input) {
|
|
297747
|
+
if (!recordKeys.has(key2)) {
|
|
296508
297748
|
unrecognized = unrecognized ?? [];
|
|
296509
|
-
unrecognized.push(
|
|
297749
|
+
unrecognized.push(key2);
|
|
296510
297750
|
}
|
|
296511
297751
|
}
|
|
296512
297752
|
if (unrecognized && unrecognized.length > 0) {
|
|
@@ -296519,16 +297759,16 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
|
|
|
296519
297759
|
}
|
|
296520
297760
|
} else {
|
|
296521
297761
|
payload.value = {};
|
|
296522
|
-
for (const
|
|
296523
|
-
if (
|
|
297762
|
+
for (const key2 of Reflect.ownKeys(input)) {
|
|
297763
|
+
if (key2 === "__proto__")
|
|
296524
297764
|
continue;
|
|
296525
|
-
let keyResult = def.keyType._zod.run({ value:
|
|
297765
|
+
let keyResult = def.keyType._zod.run({ value: key2, issues: [] }, ctx);
|
|
296526
297766
|
if (keyResult instanceof Promise) {
|
|
296527
297767
|
throw new Error("Async schemas not supported in object keys currently");
|
|
296528
297768
|
}
|
|
296529
|
-
const checkNumericKey = typeof
|
|
297769
|
+
const checkNumericKey = typeof key2 === "string" && number3.test(key2) && keyResult.issues.length;
|
|
296530
297770
|
if (checkNumericKey) {
|
|
296531
|
-
const retryResult = def.keyType._zod.run({ value: Number(
|
|
297771
|
+
const retryResult = def.keyType._zod.run({ value: Number(key2), issues: [] }, ctx);
|
|
296532
297772
|
if (retryResult instanceof Promise) {
|
|
296533
297773
|
throw new Error("Async schemas not supported in object keys currently");
|
|
296534
297774
|
}
|
|
@@ -296538,30 +297778,30 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
|
|
|
296538
297778
|
}
|
|
296539
297779
|
if (keyResult.issues.length) {
|
|
296540
297780
|
if (def.mode === "loose") {
|
|
296541
|
-
payload.value[
|
|
297781
|
+
payload.value[key2] = input[key2];
|
|
296542
297782
|
} else {
|
|
296543
297783
|
payload.issues.push({
|
|
296544
297784
|
code: "invalid_key",
|
|
296545
297785
|
origin: "record",
|
|
296546
297786
|
issues: keyResult.issues.map((iss) => finalizeIssue(iss, ctx, config())),
|
|
296547
|
-
input:
|
|
296548
|
-
path: [
|
|
297787
|
+
input: key2,
|
|
297788
|
+
path: [key2],
|
|
296549
297789
|
inst
|
|
296550
297790
|
});
|
|
296551
297791
|
}
|
|
296552
297792
|
continue;
|
|
296553
297793
|
}
|
|
296554
|
-
const result = def.valueType._zod.run({ value: input[
|
|
297794
|
+
const result = def.valueType._zod.run({ value: input[key2], issues: [] }, ctx);
|
|
296555
297795
|
if (result instanceof Promise) {
|
|
296556
297796
|
proms.push(result.then((result2) => {
|
|
296557
297797
|
if (result2.issues.length) {
|
|
296558
|
-
payload.issues.push(...prefixIssues(
|
|
297798
|
+
payload.issues.push(...prefixIssues(key2, result2.issues));
|
|
296559
297799
|
}
|
|
296560
297800
|
payload.value[keyResult.value] = result2.value;
|
|
296561
297801
|
}));
|
|
296562
297802
|
} else {
|
|
296563
297803
|
if (result.issues.length) {
|
|
296564
|
-
payload.issues.push(...prefixIssues(
|
|
297804
|
+
payload.issues.push(...prefixIssues(key2, result.issues));
|
|
296565
297805
|
}
|
|
296566
297806
|
payload.value[keyResult.value] = result.value;
|
|
296567
297807
|
}
|
|
@@ -296588,15 +297828,15 @@ var $ZodMap = /* @__PURE__ */ $constructor("$ZodMap", (inst, def) => {
|
|
|
296588
297828
|
}
|
|
296589
297829
|
const proms = [];
|
|
296590
297830
|
payload.value = new Map;
|
|
296591
|
-
for (const [
|
|
296592
|
-
const keyResult = def.keyType._zod.run({ value:
|
|
297831
|
+
for (const [key2, value] of input) {
|
|
297832
|
+
const keyResult = def.keyType._zod.run({ value: key2, issues: [] }, ctx);
|
|
296593
297833
|
const valueResult = def.valueType._zod.run({ value, issues: [] }, ctx);
|
|
296594
297834
|
if (keyResult instanceof Promise || valueResult instanceof Promise) {
|
|
296595
297835
|
proms.push(Promise.all([keyResult, valueResult]).then(([keyResult2, valueResult2]) => {
|
|
296596
|
-
handleMapResult(keyResult2, valueResult2, payload,
|
|
297836
|
+
handleMapResult(keyResult2, valueResult2, payload, key2, input, inst, ctx);
|
|
296597
297837
|
}));
|
|
296598
297838
|
} else {
|
|
296599
|
-
handleMapResult(keyResult, valueResult, payload,
|
|
297839
|
+
handleMapResult(keyResult, valueResult, payload, key2, input, inst, ctx);
|
|
296600
297840
|
}
|
|
296601
297841
|
}
|
|
296602
297842
|
if (proms.length)
|
|
@@ -296604,10 +297844,10 @@ var $ZodMap = /* @__PURE__ */ $constructor("$ZodMap", (inst, def) => {
|
|
|
296604
297844
|
return payload;
|
|
296605
297845
|
};
|
|
296606
297846
|
});
|
|
296607
|
-
function handleMapResult(keyResult, valueResult, final,
|
|
297847
|
+
function handleMapResult(keyResult, valueResult, final, key2, input, inst, ctx) {
|
|
296608
297848
|
if (keyResult.issues.length) {
|
|
296609
|
-
if (propertyKeyTypes.has(typeof
|
|
296610
|
-
final.issues.push(...prefixIssues(
|
|
297849
|
+
if (propertyKeyTypes.has(typeof key2)) {
|
|
297850
|
+
final.issues.push(...prefixIssues(key2, keyResult.issues));
|
|
296611
297851
|
} else {
|
|
296612
297852
|
final.issues.push({
|
|
296613
297853
|
code: "invalid_key",
|
|
@@ -296619,15 +297859,15 @@ function handleMapResult(keyResult, valueResult, final, key, input, inst, ctx) {
|
|
|
296619
297859
|
}
|
|
296620
297860
|
}
|
|
296621
297861
|
if (valueResult.issues.length) {
|
|
296622
|
-
if (propertyKeyTypes.has(typeof
|
|
296623
|
-
final.issues.push(...prefixIssues(
|
|
297862
|
+
if (propertyKeyTypes.has(typeof key2)) {
|
|
297863
|
+
final.issues.push(...prefixIssues(key2, valueResult.issues));
|
|
296624
297864
|
} else {
|
|
296625
297865
|
final.issues.push({
|
|
296626
297866
|
origin: "map",
|
|
296627
297867
|
code: "invalid_element",
|
|
296628
297868
|
input,
|
|
296629
297869
|
inst,
|
|
296630
|
-
key,
|
|
297870
|
+
key: key2,
|
|
296631
297871
|
issues: valueResult.issues.map((iss) => finalizeIssue(iss, ctx, config()))
|
|
296632
297872
|
});
|
|
296633
297873
|
}
|
|
@@ -296957,12 +298197,12 @@ var $ZodPipe = /* @__PURE__ */ $constructor("$ZodPipe", (inst, def) => {
|
|
|
296957
298197
|
return handlePipeResult(left, def.out, ctx);
|
|
296958
298198
|
};
|
|
296959
298199
|
});
|
|
296960
|
-
function handlePipeResult(left,
|
|
298200
|
+
function handlePipeResult(left, next2, ctx) {
|
|
296961
298201
|
if (left.issues.length) {
|
|
296962
298202
|
left.aborted = true;
|
|
296963
298203
|
return left;
|
|
296964
298204
|
}
|
|
296965
|
-
return
|
|
298205
|
+
return next2._zod.run({ value: left.value, issues: left.issues }, ctx);
|
|
296966
298206
|
}
|
|
296967
298207
|
var $ZodCodec = /* @__PURE__ */ $constructor("$ZodCodec", (inst, def) => {
|
|
296968
298208
|
$ZodType.init(inst, def);
|
|
@@ -303821,8 +305061,8 @@ function extractDefs(ctx, schema) {
|
|
|
303821
305061
|
if (defId)
|
|
303822
305062
|
seen.defId = defId;
|
|
303823
305063
|
const schema2 = seen.schema;
|
|
303824
|
-
for (const
|
|
303825
|
-
delete schema2[
|
|
305064
|
+
for (const key2 in schema2) {
|
|
305065
|
+
delete schema2[key2];
|
|
303826
305066
|
}
|
|
303827
305067
|
schema2.$ref = ref;
|
|
303828
305068
|
};
|
|
@@ -303889,20 +305129,20 @@ function finalize(ctx, schema) {
|
|
|
303889
305129
|
Object.assign(schema2, _cached);
|
|
303890
305130
|
const isParentRef = zodSchema._zod.parent === ref;
|
|
303891
305131
|
if (isParentRef) {
|
|
303892
|
-
for (const
|
|
303893
|
-
if (
|
|
305132
|
+
for (const key2 in schema2) {
|
|
305133
|
+
if (key2 === "$ref" || key2 === "allOf")
|
|
303894
305134
|
continue;
|
|
303895
|
-
if (!(
|
|
303896
|
-
delete schema2[
|
|
305135
|
+
if (!(key2 in _cached)) {
|
|
305136
|
+
delete schema2[key2];
|
|
303897
305137
|
}
|
|
303898
305138
|
}
|
|
303899
305139
|
}
|
|
303900
305140
|
if (refSchema.$ref && refSeen.def) {
|
|
303901
|
-
for (const
|
|
303902
|
-
if (
|
|
305141
|
+
for (const key2 in schema2) {
|
|
305142
|
+
if (key2 === "$ref" || key2 === "allOf")
|
|
303903
305143
|
continue;
|
|
303904
|
-
if (
|
|
303905
|
-
delete schema2[
|
|
305144
|
+
if (key2 in refSeen.def && JSON.stringify(schema2[key2]) === JSON.stringify(refSeen.def[key2])) {
|
|
305145
|
+
delete schema2[key2];
|
|
303906
305146
|
}
|
|
303907
305147
|
}
|
|
303908
305148
|
}
|
|
@@ -303914,11 +305154,11 @@ function finalize(ctx, schema) {
|
|
|
303914
305154
|
if (parentSeen?.schema.$ref) {
|
|
303915
305155
|
schema2.$ref = parentSeen.schema.$ref;
|
|
303916
305156
|
if (parentSeen.def) {
|
|
303917
|
-
for (const
|
|
303918
|
-
if (
|
|
305157
|
+
for (const key2 in schema2) {
|
|
305158
|
+
if (key2 === "$ref" || key2 === "allOf")
|
|
303919
305159
|
continue;
|
|
303920
|
-
if (
|
|
303921
|
-
delete schema2[
|
|
305160
|
+
if (key2 in parentSeen.def && JSON.stringify(schema2[key2]) === JSON.stringify(parentSeen.def[key2])) {
|
|
305161
|
+
delete schema2[key2];
|
|
303922
305162
|
}
|
|
303923
305163
|
}
|
|
303924
305164
|
}
|
|
@@ -304009,8 +305249,8 @@ function isTransforming(_schema, _ctx) {
|
|
|
304009
305249
|
return isTransforming(def.in, ctx) || isTransforming(def.out, ctx);
|
|
304010
305250
|
}
|
|
304011
305251
|
if (def.type === "object") {
|
|
304012
|
-
for (const
|
|
304013
|
-
if (isTransforming(def.shape[
|
|
305252
|
+
for (const key2 in def.shape) {
|
|
305253
|
+
if (isTransforming(def.shape[key2], ctx))
|
|
304014
305254
|
return true;
|
|
304015
305255
|
}
|
|
304016
305256
|
return false;
|
|
@@ -304301,15 +305541,15 @@ var objectProcessor = (schema, ctx, _json, params) => {
|
|
|
304301
305541
|
json3.type = "object";
|
|
304302
305542
|
json3.properties = {};
|
|
304303
305543
|
const shape = def.shape;
|
|
304304
|
-
for (const
|
|
304305
|
-
json3.properties[
|
|
305544
|
+
for (const key2 in shape) {
|
|
305545
|
+
json3.properties[key2] = process2(shape[key2], ctx, {
|
|
304306
305546
|
...params,
|
|
304307
|
-
path: [...params.path, "properties",
|
|
305547
|
+
path: [...params.path, "properties", key2]
|
|
304308
305548
|
});
|
|
304309
305549
|
}
|
|
304310
305550
|
const allKeys = new Set(Object.keys(shape));
|
|
304311
|
-
const requiredKeys = new Set([...allKeys].filter((
|
|
304312
|
-
const v = def.shape[
|
|
305551
|
+
const requiredKeys = new Set([...allKeys].filter((key2) => {
|
|
305552
|
+
const v = def.shape[key2]._zod;
|
|
304313
305553
|
if (ctx.io === "input") {
|
|
304314
305554
|
return v.optin === undefined;
|
|
304315
305555
|
} else {
|
|
@@ -304574,9 +305814,9 @@ function toJSONSchema(input, params) {
|
|
|
304574
305814
|
};
|
|
304575
305815
|
ctx2.external = external;
|
|
304576
305816
|
for (const entry of registry2._idmap.entries()) {
|
|
304577
|
-
const [
|
|
305817
|
+
const [key2, schema] = entry;
|
|
304578
305818
|
extractDefs(ctx2, schema);
|
|
304579
|
-
schemas[
|
|
305819
|
+
schemas[key2] = finalize(ctx2, schema);
|
|
304580
305820
|
}
|
|
304581
305821
|
if (Object.keys(defs).length > 0) {
|
|
304582
305822
|
const defsSegment = ctx2.target === "draft-2020-12" ? "$defs" : "definitions";
|
|
@@ -306133,11 +307373,11 @@ function resolveRef(ref, ctx) {
|
|
|
306133
307373
|
}
|
|
306134
307374
|
const defsKey = ctx.version === "draft-2020-12" ? "$defs" : "definitions";
|
|
306135
307375
|
if (path[0] === defsKey) {
|
|
306136
|
-
const
|
|
306137
|
-
if (!
|
|
307376
|
+
const key2 = path[1];
|
|
307377
|
+
if (!key2 || !ctx.defs[key2]) {
|
|
306138
307378
|
throw new Error(`Reference not found: ${ref}`);
|
|
306139
307379
|
}
|
|
306140
|
-
return ctx.defs[
|
|
307380
|
+
return ctx.defs[key2];
|
|
306141
307381
|
}
|
|
306142
307382
|
throw new Error(`Reference not found: ${ref}`);
|
|
306143
307383
|
}
|
|
@@ -306323,9 +307563,9 @@ function convertBaseSchema(schema, ctx) {
|
|
|
306323
307563
|
const shape = {};
|
|
306324
307564
|
const properties = schema.properties || {};
|
|
306325
307565
|
const requiredSet = new Set(schema.required || []);
|
|
306326
|
-
for (const [
|
|
307566
|
+
for (const [key2, propSchema] of Object.entries(properties)) {
|
|
306327
307567
|
const propZodSchema = convertSchema(propSchema, ctx);
|
|
306328
|
-
shape[
|
|
307568
|
+
shape[key2] = requiredSet.has(key2) ? propZodSchema : propZodSchema.optional();
|
|
306329
307569
|
}
|
|
306330
307570
|
if (schema.propertyNames) {
|
|
306331
307571
|
const keySchema = convertSchema(schema.propertyNames, ctx);
|
|
@@ -306469,20 +307709,20 @@ function convertSchema(schema, ctx) {
|
|
|
306469
307709
|
}
|
|
306470
307710
|
const extraMeta = {};
|
|
306471
307711
|
const coreMetadataKeys = ["$id", "id", "$comment", "$anchor", "$vocabulary", "$dynamicRef", "$dynamicAnchor"];
|
|
306472
|
-
for (const
|
|
306473
|
-
if (
|
|
306474
|
-
extraMeta[
|
|
307712
|
+
for (const key2 of coreMetadataKeys) {
|
|
307713
|
+
if (key2 in schema) {
|
|
307714
|
+
extraMeta[key2] = schema[key2];
|
|
306475
307715
|
}
|
|
306476
307716
|
}
|
|
306477
307717
|
const contentMetadataKeys = ["contentEncoding", "contentMediaType", "contentSchema"];
|
|
306478
|
-
for (const
|
|
306479
|
-
if (
|
|
306480
|
-
extraMeta[
|
|
307718
|
+
for (const key2 of contentMetadataKeys) {
|
|
307719
|
+
if (key2 in schema) {
|
|
307720
|
+
extraMeta[key2] = schema[key2];
|
|
306481
307721
|
}
|
|
306482
307722
|
}
|
|
306483
|
-
for (const
|
|
306484
|
-
if (!RECOGNIZED_KEYS.has(
|
|
306485
|
-
extraMeta[
|
|
307723
|
+
for (const key2 of Object.keys(schema)) {
|
|
307724
|
+
if (!RECOGNIZED_KEYS.has(key2)) {
|
|
307725
|
+
extraMeta[key2] = schema[key2];
|
|
306486
307726
|
}
|
|
306487
307727
|
}
|
|
306488
307728
|
if (Object.keys(extraMeta).length > 0) {
|
|
@@ -306847,7 +308087,7 @@ async function handleDeposit(ctx) {
|
|
|
306847
308087
|
network: depositNetwork?.exchangeNetworkId,
|
|
306848
308088
|
externalId: depositTxid,
|
|
306849
308089
|
txid: depositTxid,
|
|
306850
|
-
exchangeTimestamp:
|
|
308090
|
+
exchangeTimestamp: normalizeTimestamp2(creditedAt),
|
|
306851
308091
|
payload: deposit
|
|
306852
308092
|
}
|
|
306853
308093
|
});
|
|
@@ -307068,320 +308308,118 @@ function parseOrderBookCallPayload(payload, request) {
|
|
|
307068
308308
|
}
|
|
307069
308309
|
if (parsedStart.timestamp >= parsedEnd.timestamp) {
|
|
307070
308310
|
return {
|
|
307071
|
-
kind: "error",
|
|
307072
|
-
message: "ValidationError: start must be before end"
|
|
307073
|
-
};
|
|
307074
|
-
}
|
|
307075
|
-
const parsedCadence = parseCadence(payloadValue(payload, "cadence"));
|
|
307076
|
-
if (!parsedCadence.ok) {
|
|
307077
|
-
return { kind: "error", message: parsedCadence.message };
|
|
307078
|
-
}
|
|
307079
|
-
return {
|
|
307080
|
-
kind: "order_book",
|
|
307081
|
-
payload: {
|
|
307082
|
-
...parsed,
|
|
307083
|
-
start: parsedStart.value,
|
|
307084
|
-
end: parsedEnd.value,
|
|
307085
|
-
cadence: parsedCadence.value
|
|
307086
|
-
}
|
|
307087
|
-
};
|
|
307088
|
-
}
|
|
307089
|
-
function parseOptionalDepthLimit(value) {
|
|
307090
|
-
const parsed = parsePositiveInteger(nonEmptyString(value), "depthLimit");
|
|
307091
|
-
return parsed.ok ? parsed.value : undefined;
|
|
307092
|
-
}
|
|
307093
|
-
function scalarByAlias(payload, aliases) {
|
|
307094
|
-
for (const alias of aliases) {
|
|
307095
|
-
const value = payload[alias];
|
|
307096
|
-
if (isScalar(value)) {
|
|
307097
|
-
return value;
|
|
307098
|
-
}
|
|
307099
|
-
}
|
|
307100
|
-
return;
|
|
307101
|
-
}
|
|
307102
|
-
function normalizeSide(payload, side, depthLimit) {
|
|
307103
|
-
const rawLevels = payload[side];
|
|
307104
|
-
if (!Array.isArray(rawLevels)) {
|
|
307105
|
-
throw new Error(`Malformed order book: ${side} must be an array`);
|
|
307106
|
-
}
|
|
307107
|
-
return rawLevels.slice(0, depthLimit).map((level, index2) => {
|
|
307108
|
-
if (!Array.isArray(level) || level.length < 2) {
|
|
307109
|
-
throw new Error(`Malformed order book: ${side}[${index2}] must be [price, amount]`);
|
|
307110
|
-
}
|
|
307111
|
-
const price = Number(level[0]);
|
|
307112
|
-
const amount = Number(level[1]);
|
|
307113
|
-
if (!Number.isFinite(price) || !Number.isFinite(amount)) {
|
|
307114
|
-
throw new Error(`Malformed order book: ${side}[${index2}] must be numeric`);
|
|
307115
|
-
}
|
|
307116
|
-
return [price, amount];
|
|
307117
|
-
});
|
|
307118
|
-
}
|
|
307119
|
-
function normalizeOrderBookSnapshot(orderBook, options) {
|
|
307120
|
-
if (!isRecord(orderBook)) {
|
|
307121
|
-
throw new Error("Malformed order book: expected object");
|
|
307122
|
-
}
|
|
307123
|
-
const receivedTimestamp = options.receivedTimestamp ?? Date.now();
|
|
307124
|
-
const timestamp = scalarByAlias(orderBook, ["timestamp"]) ?? receivedTimestamp;
|
|
307125
|
-
const sequence = scalarByAlias(orderBook, [
|
|
307126
|
-
"sequence",
|
|
307127
|
-
"updateId",
|
|
307128
|
-
"lastUpdateId",
|
|
307129
|
-
"nonce"
|
|
307130
|
-
]);
|
|
307131
|
-
const normalized = {
|
|
307132
|
-
bids: normalizeSide(orderBook, "bids", options.depthLimit),
|
|
307133
|
-
asks: normalizeSide(orderBook, "asks", options.depthLimit),
|
|
307134
|
-
timestamp,
|
|
307135
|
-
receivedTimestamp,
|
|
307136
|
-
exchange: options.exchange,
|
|
307137
|
-
symbol: options.symbol,
|
|
307138
|
-
depthLimit: options.depthLimit
|
|
307139
|
-
};
|
|
307140
|
-
if (sequence !== undefined) {
|
|
307141
|
-
normalized.sequence = sequence;
|
|
307142
|
-
}
|
|
307143
|
-
return normalized;
|
|
307144
|
-
}
|
|
307145
|
-
function supportsBrokerMethod(broker, method) {
|
|
307146
|
-
const fn = broker[method];
|
|
307147
|
-
const hasValue = broker.has?.[method];
|
|
307148
|
-
return typeof fn === "function" && hasValue !== false;
|
|
307149
|
-
}
|
|
307150
|
-
function buildOrderBookCapability(broker, payload) {
|
|
307151
|
-
return {
|
|
307152
|
-
exchange: payload.exchange,
|
|
307153
|
-
symbol: payload.symbol,
|
|
307154
|
-
provider: "ccxt_order_book",
|
|
307155
|
-
maxDepth: payload.depthLimit,
|
|
307156
|
-
timestampPrecision: "milliseconds",
|
|
307157
|
-
constructionMode: payload.constructionMode,
|
|
307158
|
-
supportsCurrentSnapshot: supportsBrokerMethod(broker, "fetchOrderBook"),
|
|
307159
|
-
supportsLiveStream: supportsBrokerMethod(broker, "watchOrderBook"),
|
|
307160
|
-
supportsHistoricalSnapshots: false,
|
|
307161
|
-
supportsSampledTopN: false,
|
|
307162
|
-
supportsExactL2Reconstruction: false
|
|
307163
|
-
};
|
|
307164
|
-
}
|
|
307165
|
-
function buildHistoricalOrderBookUnsupported(payload) {
|
|
307166
|
-
return {
|
|
307167
|
-
exchange: payload.exchange,
|
|
307168
|
-
symbol: payload.symbol,
|
|
307169
|
-
provider: "ccxt_order_book",
|
|
307170
|
-
constructionMode: payload.constructionMode,
|
|
307171
|
-
depthLimit: payload.depthLimit,
|
|
307172
|
-
start: payload.start,
|
|
307173
|
-
end: payload.end,
|
|
307174
|
-
cadence: payload.cadence,
|
|
307175
|
-
unsupported: true,
|
|
307176
|
-
unsupportedReason: HISTORICAL_ORDER_BOOK_PROVIDER_UNSUPPORTED
|
|
307177
|
-
};
|
|
307178
|
-
}
|
|
307179
|
-
|
|
307180
|
-
// src/handlers/execute-action/order-book-call.ts
|
|
307181
|
-
import * as grpc4 from "@grpc/grpc-js";
|
|
307182
|
-
|
|
307183
|
-
// src/helpers/market-data-archive/capture-contract.ts
|
|
307184
|
-
import { createHash as createHash3 } from "node:crypto";
|
|
307185
|
-
var MARKET_CAPTURE_SCHEMA_VERSION = "1.0.0";
|
|
307186
|
-
var CHECKSUM_ALGORITHM = "sha256-canonical-json-v1";
|
|
307187
|
-
var ARCHIVE_SOURCES = ["broker_read", "broker_write"];
|
|
307188
|
-
var CAPTURE_FEEDS = [
|
|
307189
|
-
"ORDERBOOK",
|
|
307190
|
-
"TICKER",
|
|
307191
|
-
"TRADES",
|
|
307192
|
-
"OHLCV"
|
|
307193
|
-
];
|
|
307194
|
-
var SOURCE_MODES = [
|
|
307195
|
-
"broker_live_stream_v1",
|
|
307196
|
-
"broker_live_sampling_v1",
|
|
307197
|
-
"broker_current_snapshot_v1",
|
|
307198
|
-
"broker_bootstrap_fetch_v1",
|
|
307199
|
-
"external_ccxt_fallback_v1",
|
|
307200
|
-
"external_hummingbot_fallback_v1",
|
|
307201
|
-
"legacy_migration_v1"
|
|
307202
|
-
];
|
|
307203
|
-
var RAW_CAPTURE_SCOPES = [
|
|
307204
|
-
"ccxt_normalized_object",
|
|
307205
|
-
"broker_visible_payload",
|
|
307206
|
-
"exchange_wire_frame"
|
|
307207
|
-
];
|
|
307208
|
-
var CHECKSUM_FIELDS = new Set([
|
|
307209
|
-
"normalized_row_checksum",
|
|
307210
|
-
"raw_checksum",
|
|
307211
|
-
"checksum"
|
|
307212
|
-
]);
|
|
307213
|
-
function canonicalDecimal(value) {
|
|
307214
|
-
if (!Number.isFinite(value)) {
|
|
307215
|
-
throw new Error("Canonical numbers must be finite");
|
|
307216
|
-
}
|
|
307217
|
-
if (Object.is(value, -0)) {
|
|
307218
|
-
return "0";
|
|
307219
|
-
}
|
|
307220
|
-
const rendered = String(value).toLowerCase();
|
|
307221
|
-
if (!rendered.includes("e")) {
|
|
307222
|
-
return rendered;
|
|
307223
|
-
}
|
|
307224
|
-
const [coefficient = "0", exponentText = "0"] = rendered.split("e");
|
|
307225
|
-
const exponent = Number.parseInt(exponentText, 10);
|
|
307226
|
-
const negative = coefficient.startsWith("-");
|
|
307227
|
-
const unsigned = negative ? coefficient.slice(1) : coefficient;
|
|
307228
|
-
const [integer2 = "0", fraction = ""] = unsigned.split(".");
|
|
307229
|
-
const digits = `${integer2}${fraction}`;
|
|
307230
|
-
const decimalIndex = integer2.length + exponent;
|
|
307231
|
-
let result;
|
|
307232
|
-
if (decimalIndex <= 0) {
|
|
307233
|
-
result = `0.${"0".repeat(-decimalIndex)}${digits}`;
|
|
307234
|
-
} else if (decimalIndex >= digits.length) {
|
|
307235
|
-
result = `${digits}${"0".repeat(decimalIndex - digits.length)}`;
|
|
307236
|
-
} else {
|
|
307237
|
-
result = `${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
|
|
307238
|
-
}
|
|
307239
|
-
return negative ? `-${result}` : result;
|
|
307240
|
-
}
|
|
307241
|
-
function serializeCanonical(value, stack) {
|
|
307242
|
-
if (value === null)
|
|
307243
|
-
return "null";
|
|
307244
|
-
if (typeof value === "string")
|
|
307245
|
-
return JSON.stringify(value);
|
|
307246
|
-
if (typeof value === "boolean")
|
|
307247
|
-
return value ? "true" : "false";
|
|
307248
|
-
if (typeof value === "number")
|
|
307249
|
-
return canonicalDecimal(value);
|
|
307250
|
-
if (typeof value === "bigint")
|
|
307251
|
-
return value.toString(10);
|
|
307252
|
-
if (value instanceof Date) {
|
|
307253
|
-
if (Number.isNaN(value.getTime())) {
|
|
307254
|
-
throw new Error("Canonical timestamps must be valid");
|
|
307255
|
-
}
|
|
307256
|
-
return value.getTime().toString(10);
|
|
307257
|
-
}
|
|
307258
|
-
if (Array.isArray(value)) {
|
|
307259
|
-
if (stack.has(value))
|
|
307260
|
-
throw new Error("Canonical values must be acyclic");
|
|
307261
|
-
stack.add(value);
|
|
307262
|
-
const result = `[${value.map((entry) => entry === undefined ? "null" : serializeCanonical(entry, stack)).join(",")}]`;
|
|
307263
|
-
stack.delete(value);
|
|
307264
|
-
return result;
|
|
307265
|
-
}
|
|
307266
|
-
if (typeof value === "object") {
|
|
307267
|
-
if (stack.has(value))
|
|
307268
|
-
throw new Error("Canonical values must be acyclic");
|
|
307269
|
-
stack.add(value);
|
|
307270
|
-
const entries = Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([left], [right]) => left.localeCompare(right));
|
|
307271
|
-
const result = `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${serializeCanonical(entry, stack)}`).join(",")}}`;
|
|
307272
|
-
stack.delete(value);
|
|
307273
|
-
return result;
|
|
308311
|
+
kind: "error",
|
|
308312
|
+
message: "ValidationError: start must be before end"
|
|
308313
|
+
};
|
|
307274
308314
|
}
|
|
307275
|
-
|
|
307276
|
-
|
|
307277
|
-
|
|
307278
|
-
return serializeCanonical(value, new Set);
|
|
307279
|
-
}
|
|
307280
|
-
function omitChecksumFields(value) {
|
|
307281
|
-
if (Array.isArray(value))
|
|
307282
|
-
return value.map(omitChecksumFields);
|
|
307283
|
-
if (value && typeof value === "object" && !(value instanceof Date)) {
|
|
307284
|
-
return Object.fromEntries(Object.entries(value).filter(([key]) => !CHECKSUM_FIELDS.has(key)).map(([key, entry]) => [key, omitChecksumFields(entry)]));
|
|
308315
|
+
const parsedCadence = parseCadence(payloadValue(payload, "cadence"));
|
|
308316
|
+
if (!parsedCadence.ok) {
|
|
308317
|
+
return { kind: "error", message: parsedCadence.message };
|
|
307285
308318
|
}
|
|
307286
|
-
return
|
|
308319
|
+
return {
|
|
308320
|
+
kind: "order_book",
|
|
308321
|
+
payload: {
|
|
308322
|
+
...parsed,
|
|
308323
|
+
start: parsedStart.value,
|
|
308324
|
+
end: parsedEnd.value,
|
|
308325
|
+
cadence: parsedCadence.value
|
|
308326
|
+
}
|
|
308327
|
+
};
|
|
307287
308328
|
}
|
|
307288
|
-
function
|
|
307289
|
-
|
|
308329
|
+
function parseOptionalDepthLimit(value) {
|
|
308330
|
+
const parsed = parsePositiveInteger(nonEmptyString(value), "depthLimit");
|
|
308331
|
+
return parsed.ok ? parsed.value : undefined;
|
|
307290
308332
|
}
|
|
307291
|
-
function
|
|
307292
|
-
|
|
307293
|
-
|
|
307294
|
-
|
|
307295
|
-
|
|
307296
|
-
|
|
307297
|
-
} else if (typeof value === "string" && /^\d+$/.test(value.trim())) {
|
|
307298
|
-
timestamp = Number(value.trim());
|
|
307299
|
-
} else if (typeof value === "string") {
|
|
307300
|
-
timestamp = Date.parse(value);
|
|
307301
|
-
} else {
|
|
307302
|
-
timestamp = Number.NaN;
|
|
307303
|
-
}
|
|
307304
|
-
if (!Number.isSafeInteger(timestamp) || timestamp < 0) {
|
|
307305
|
-
throw new Error(`${field} must be a non-negative millisecond timestamp`);
|
|
308333
|
+
function scalarByAlias(payload, aliases) {
|
|
308334
|
+
for (const alias of aliases) {
|
|
308335
|
+
const value = payload[alias];
|
|
308336
|
+
if (isScalar(value)) {
|
|
308337
|
+
return value;
|
|
308338
|
+
}
|
|
307306
308339
|
}
|
|
307307
|
-
return
|
|
308340
|
+
return;
|
|
307308
308341
|
}
|
|
307309
|
-
function
|
|
307310
|
-
|
|
307311
|
-
|
|
307312
|
-
|
|
307313
|
-
if (!CAPTURE_FEEDS.includes(context2.feed)) {
|
|
307314
|
-
throw new Error(`Unsupported capture feed: ${context2.feed}`);
|
|
308342
|
+
function normalizeSide(payload, side, depthLimit) {
|
|
308343
|
+
const rawLevels = payload[side];
|
|
308344
|
+
if (!Array.isArray(rawLevels)) {
|
|
308345
|
+
throw new Error(`Malformed order book: ${side} must be an array`);
|
|
307315
308346
|
}
|
|
307316
|
-
|
|
307317
|
-
|
|
308347
|
+
return rawLevels.slice(0, depthLimit).map((level, index2) => {
|
|
308348
|
+
if (!Array.isArray(level) || level.length < 2) {
|
|
308349
|
+
throw new Error(`Malformed order book: ${side}[${index2}] must be [price, amount]`);
|
|
308350
|
+
}
|
|
308351
|
+
const price = Number(level[0]);
|
|
308352
|
+
const amount = Number(level[1]);
|
|
308353
|
+
if (!Number.isFinite(price) || !Number.isFinite(amount)) {
|
|
308354
|
+
throw new Error(`Malformed order book: ${side}[${index2}] must be numeric`);
|
|
308355
|
+
}
|
|
308356
|
+
return [price, amount];
|
|
308357
|
+
});
|
|
308358
|
+
}
|
|
308359
|
+
function normalizeOrderBookSnapshot(orderBook, options) {
|
|
308360
|
+
if (!isRecord(orderBook)) {
|
|
308361
|
+
throw new Error("Malformed order book: expected object");
|
|
307318
308362
|
}
|
|
307319
|
-
|
|
307320
|
-
|
|
307321
|
-
|
|
307322
|
-
|
|
307323
|
-
|
|
307324
|
-
|
|
307325
|
-
|
|
307326
|
-
|
|
307327
|
-
|
|
308363
|
+
const receivedTimestamp = options.receivedTimestamp ?? Date.now();
|
|
308364
|
+
const timestamp = scalarByAlias(orderBook, ["timestamp"]) ?? receivedTimestamp;
|
|
308365
|
+
const sequence = scalarByAlias(orderBook, [
|
|
308366
|
+
"sequence",
|
|
308367
|
+
"updateId",
|
|
308368
|
+
"lastUpdateId",
|
|
308369
|
+
"nonce"
|
|
308370
|
+
]);
|
|
308371
|
+
const normalized = {
|
|
308372
|
+
bids: normalizeSide(orderBook, "bids", options.depthLimit),
|
|
308373
|
+
asks: normalizeSide(orderBook, "asks", options.depthLimit),
|
|
308374
|
+
timestamp,
|
|
308375
|
+
receivedTimestamp,
|
|
308376
|
+
exchange: options.exchange,
|
|
308377
|
+
symbol: options.symbol,
|
|
308378
|
+
depthLimit: options.depthLimit
|
|
308379
|
+
};
|
|
308380
|
+
if (sequence !== undefined) {
|
|
308381
|
+
normalized.sequence = sequence;
|
|
307328
308382
|
}
|
|
308383
|
+
return normalized;
|
|
307329
308384
|
}
|
|
307330
|
-
function
|
|
307331
|
-
|
|
307332
|
-
|
|
307333
|
-
|
|
307334
|
-
|
|
307335
|
-
|
|
307336
|
-
const receivedTimeMs = normalizeTimestampMs(input.receivedTimeMs, "received_time_ms");
|
|
307337
|
-
const redactedPayload = redactStreamPayload(input.payload);
|
|
307338
|
-
const rawChecksum = sha256Canonical(redactedPayload);
|
|
307339
|
-
const rawCaptureId = sha256Canonical({
|
|
307340
|
-
capture_bundle_id: context2.captureBundleId,
|
|
307341
|
-
exchange: context2.exchange.trim().toLowerCase(),
|
|
307342
|
-
feed: context2.feed,
|
|
307343
|
-
raw_capture_scope: input.scope,
|
|
307344
|
-
raw_payload_sha256: rawChecksum,
|
|
307345
|
-
schema_version: context2.schemaVersion,
|
|
307346
|
-
source_mode: context2.sourceMode,
|
|
307347
|
-
source_symbol: context2.symbol.trim(),
|
|
307348
|
-
source_time_ms: eventTimeMs
|
|
307349
|
-
});
|
|
308385
|
+
function supportsBrokerMethod(broker, method) {
|
|
308386
|
+
const fn = broker[method];
|
|
308387
|
+
const hasValue = broker.has?.[method];
|
|
308388
|
+
return typeof fn === "function" && hasValue !== false;
|
|
308389
|
+
}
|
|
308390
|
+
function buildOrderBookCapability(broker, payload) {
|
|
307350
308391
|
return {
|
|
307351
|
-
|
|
307352
|
-
|
|
307353
|
-
|
|
307354
|
-
|
|
307355
|
-
|
|
307356
|
-
|
|
307357
|
-
|
|
308392
|
+
exchange: payload.exchange,
|
|
308393
|
+
symbol: payload.symbol,
|
|
308394
|
+
provider: "ccxt_order_book",
|
|
308395
|
+
maxDepth: payload.depthLimit,
|
|
308396
|
+
timestampPrecision: "milliseconds",
|
|
308397
|
+
constructionMode: payload.constructionMode,
|
|
308398
|
+
supportsCurrentSnapshot: supportsBrokerMethod(broker, "fetchOrderBook"),
|
|
308399
|
+
supportsLiveStream: supportsBrokerMethod(broker, "watchOrderBook"),
|
|
308400
|
+
supportsHistoricalSnapshots: false,
|
|
308401
|
+
supportsSampledTopN: false,
|
|
308402
|
+
supportsExactL2Reconstruction: false
|
|
307358
308403
|
};
|
|
307359
308404
|
}
|
|
307360
|
-
function
|
|
307361
|
-
assertCaptureContext(context2);
|
|
308405
|
+
function buildHistoricalOrderBookUnsupported(payload) {
|
|
307362
308406
|
return {
|
|
307363
|
-
|
|
307364
|
-
|
|
307365
|
-
|
|
307366
|
-
|
|
307367
|
-
|
|
307368
|
-
|
|
307369
|
-
|
|
307370
|
-
|
|
307371
|
-
|
|
307372
|
-
|
|
307373
|
-
source_mode: context2.sourceMode,
|
|
307374
|
-
source_time_ms: rawCapture.eventTimeMs,
|
|
307375
|
-
received_time_ms: rawCapture.receivedTimeMs,
|
|
307376
|
-
raw_capture_id: rawCapture.rawCaptureId,
|
|
307377
|
-
raw_capture_scope: rawCapture.rawCaptureScope,
|
|
307378
|
-
schema_version: context2.schemaVersion,
|
|
307379
|
-
checksum_algorithm: context2.checksumAlgorithm,
|
|
307380
|
-
raw_checksum: rawCapture.rawChecksum,
|
|
307381
|
-
provenance_complete: context2.provenanceComplete ? 1 : 0
|
|
308407
|
+
exchange: payload.exchange,
|
|
308408
|
+
symbol: payload.symbol,
|
|
308409
|
+
provider: "ccxt_order_book",
|
|
308410
|
+
constructionMode: payload.constructionMode,
|
|
308411
|
+
depthLimit: payload.depthLimit,
|
|
308412
|
+
start: payload.start,
|
|
308413
|
+
end: payload.end,
|
|
308414
|
+
cadence: payload.cadence,
|
|
308415
|
+
unsupported: true,
|
|
308416
|
+
unsupportedReason: HISTORICAL_ORDER_BOOK_PROVIDER_UNSUPPORTED
|
|
307382
308417
|
};
|
|
307383
308418
|
}
|
|
307384
308419
|
|
|
308420
|
+
// src/handlers/execute-action/order-book-call.ts
|
|
308421
|
+
import * as grpc4 from "@grpc/grpc-js";
|
|
308422
|
+
|
|
307385
308423
|
// src/helpers/market-data-archive/canonical-orderbook.ts
|
|
307386
308424
|
class OrderBookValidationError extends Error {
|
|
307387
308425
|
reason;
|
|
@@ -307549,46 +308587,6 @@ function buildCanonicalOrderBookRows(input) {
|
|
|
307549
308587
|
};
|
|
307550
308588
|
}
|
|
307551
308589
|
|
|
307552
|
-
// src/helpers/market-data-archive/capture-context.ts
|
|
307553
|
-
function createMarketCaptureContext(input) {
|
|
307554
|
-
const environment = input.environment ?? "development";
|
|
307555
|
-
const deploymentId = input.deploymentId.trim();
|
|
307556
|
-
if (!deploymentId)
|
|
307557
|
-
throw new Error("deployment_id must not be empty");
|
|
307558
|
-
const configuredBundle = input.captureBundleId?.trim();
|
|
307559
|
-
if (environment === "production" && !configuredBundle) {
|
|
307560
|
-
throw new Error("capture_bundle_id is required for production market capture");
|
|
307561
|
-
}
|
|
307562
|
-
const exchange = input.exchange.trim().toLowerCase();
|
|
307563
|
-
const symbol2 = input.symbol.trim();
|
|
307564
|
-
if (!exchange || !symbol2) {
|
|
307565
|
-
throw new Error("exchange and symbol are required for market capture");
|
|
307566
|
-
}
|
|
307567
|
-
return {
|
|
307568
|
-
source: input.source,
|
|
307569
|
-
deploymentId,
|
|
307570
|
-
captureBundleId: configuredBundle ?? `development:${deploymentId}`,
|
|
307571
|
-
exchange,
|
|
307572
|
-
symbol: symbol2,
|
|
307573
|
-
assetType: input.assetType,
|
|
307574
|
-
feed: input.feed,
|
|
307575
|
-
provider: input.provider?.trim() || `ccxt:${exchange}`,
|
|
307576
|
-
sourceMode: input.sourceMode,
|
|
307577
|
-
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
307578
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
307579
|
-
provenanceComplete: true,
|
|
307580
|
-
timeframe: input.timeframe,
|
|
307581
|
-
accountSelector: input.accountSelector
|
|
307582
|
-
};
|
|
307583
|
-
}
|
|
307584
|
-
function captureEnvironmentFromEnv(value = process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT) {
|
|
307585
|
-
const environment = value?.trim() || "development";
|
|
307586
|
-
if (environment !== "development" && environment !== "production") {
|
|
307587
|
-
throw new Error("CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT must be development or production");
|
|
307588
|
-
}
|
|
307589
|
-
return environment;
|
|
307590
|
-
}
|
|
307591
|
-
|
|
307592
308590
|
// src/helpers/market-data-archive/ohlcv-bar-tracker.ts
|
|
307593
308591
|
function isFiniteNumber(value) {
|
|
307594
308592
|
return typeof value === "number" && Number.isFinite(value);
|
|
@@ -307713,47 +308711,14 @@ var DEFAULT_ORDERBOOK_ARCHIVE_DEPTH_LIMIT = 25;
|
|
|
307713
308711
|
var MAX_ORDERBOOK_ARCHIVE_DEPTH_LIMIT = 500;
|
|
307714
308712
|
function getOrderbookArchiveDepthLimit() {
|
|
307715
308713
|
const raw = process.env.CEX_BROKER_ORDERBOOK_ARCHIVE_DEPTH_LIMIT;
|
|
307716
|
-
if (!raw) {
|
|
307717
|
-
return DEFAULT_ORDERBOOK_ARCHIVE_DEPTH_LIMIT;
|
|
307718
|
-
}
|
|
307719
|
-
const parsed = Number.parseInt(raw, 10);
|
|
307720
|
-
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
307721
|
-
return DEFAULT_ORDERBOOK_ARCHIVE_DEPTH_LIMIT;
|
|
307722
|
-
}
|
|
307723
|
-
return Math.min(parsed, MAX_ORDERBOOK_ARCHIVE_DEPTH_LIMIT);
|
|
307724
|
-
}
|
|
307725
|
-
|
|
307726
|
-
// src/helpers/market-data-archive/orderbook-sampler.ts
|
|
307727
|
-
var DEFAULT_ORDERBOOK_INTERVAL_MS = 1000;
|
|
307728
|
-
function getOrderbookIntervalMs() {
|
|
307729
|
-
const raw = process.env.CEX_BROKER_ORDERBOOK_INTERVAL_MS ?? process.env.CEX_BROKER_ORDERBOOK_TOB_INTERVAL_MS;
|
|
307730
|
-
if (!raw) {
|
|
307731
|
-
return DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
307732
|
-
}
|
|
307733
|
-
const parsed = Number.parseInt(raw, 10);
|
|
307734
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_ORDERBOOK_INTERVAL_MS;
|
|
307735
|
-
}
|
|
307736
|
-
function isMarketArchiveEnabled() {
|
|
307737
|
-
return process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED !== "false";
|
|
307738
|
-
}
|
|
307739
|
-
|
|
307740
|
-
class OrderbookSampler {
|
|
307741
|
-
intervalMs;
|
|
307742
|
-
lastEmitMs = null;
|
|
307743
|
-
constructor(intervalMs = getOrderbookIntervalMs()) {
|
|
307744
|
-
this.intervalMs = intervalMs;
|
|
308714
|
+
if (!raw) {
|
|
308715
|
+
return DEFAULT_ORDERBOOK_ARCHIVE_DEPTH_LIMIT;
|
|
307745
308716
|
}
|
|
307746
|
-
|
|
307747
|
-
|
|
307748
|
-
|
|
307749
|
-
return true;
|
|
307750
|
-
}
|
|
307751
|
-
if (this.lastEmitMs !== null && nowMs - this.lastEmitMs < this.intervalMs) {
|
|
307752
|
-
return false;
|
|
307753
|
-
}
|
|
307754
|
-
this.lastEmitMs = nowMs;
|
|
307755
|
-
return true;
|
|
308717
|
+
const parsed = Number.parseInt(raw, 10);
|
|
308718
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
308719
|
+
return DEFAULT_ORDERBOOK_ARCHIVE_DEPTH_LIMIT;
|
|
307756
308720
|
}
|
|
308721
|
+
return Math.min(parsed, MAX_ORDERBOOK_ARCHIVE_DEPTH_LIMIT);
|
|
307757
308722
|
}
|
|
307758
308723
|
|
|
307759
308724
|
// src/helpers/market-data-archive/parse-stream.ts
|
|
@@ -307850,10 +308815,10 @@ function parseTicker(value, fallbackMs) {
|
|
|
307850
308815
|
["change", record2.change],
|
|
307851
308816
|
["percentage", record2.percentage]
|
|
307852
308817
|
];
|
|
307853
|
-
for (const [
|
|
308818
|
+
for (const [key2, rawValue] of fields) {
|
|
307854
308819
|
const numeric = toNumber2(rawValue);
|
|
307855
308820
|
if (numeric !== undefined) {
|
|
307856
|
-
parsed[
|
|
308821
|
+
parsed[key2] = numeric;
|
|
307857
308822
|
}
|
|
307858
308823
|
}
|
|
307859
308824
|
return parsed;
|
|
@@ -307870,9 +308835,19 @@ function withNormalizedChecksum(record2) {
|
|
|
307870
308835
|
normalized_row_checksum: sha256Canonical(compact)
|
|
307871
308836
|
};
|
|
307872
308837
|
}
|
|
308838
|
+
function legacyMarketFields(context2, rawCapture) {
|
|
308839
|
+
return {
|
|
308840
|
+
account_selector: context2.accountSelector,
|
|
308841
|
+
broker_observed_timestamp: new Date(rawCapture.receivedTimeMs).toISOString()
|
|
308842
|
+
};
|
|
308843
|
+
}
|
|
308844
|
+
function legacyDecimal8(value) {
|
|
308845
|
+
return value === undefined ? undefined : Number(value.toFixed(8));
|
|
308846
|
+
}
|
|
307873
308847
|
function buildCanonicalCexStreamEventRow(context2, rawCapture) {
|
|
307874
308848
|
const row = withNormalizedChecksum({
|
|
307875
308849
|
...captureCoreFields(context2, rawCapture),
|
|
308850
|
+
...legacyMarketFields(context2, rawCapture),
|
|
307876
308851
|
stream_type: context2.feed,
|
|
307877
308852
|
event_time_ms: rawCapture.eventTimeMs,
|
|
307878
308853
|
payload_encoding: "canonical_json_v1",
|
|
@@ -307886,19 +308861,21 @@ function buildCanonicalTickerEventRow(context2, rawCapture, ticker) {
|
|
|
307886
308861
|
}
|
|
307887
308862
|
const row = withNormalizedChecksum({
|
|
307888
308863
|
...captureCoreFields(context2, rawCapture),
|
|
308864
|
+
...legacyMarketFields(context2, rawCapture),
|
|
307889
308865
|
source_time_ms: ticker.eventTimeMs,
|
|
307890
308866
|
event_time_ms: ticker.eventTimeMs,
|
|
307891
|
-
last: ticker.last,
|
|
307892
|
-
bid: ticker.bid,
|
|
307893
|
-
ask: ticker.ask,
|
|
307894
|
-
high: ticker.high,
|
|
307895
|
-
low: ticker.low,
|
|
307896
|
-
open: ticker.open,
|
|
307897
|
-
close: ticker.close,
|
|
307898
|
-
base_volume: ticker.baseVolume,
|
|
307899
|
-
quote_volume: ticker.quoteVolume,
|
|
307900
|
-
change: ticker.change,
|
|
307901
|
-
percentage: ticker.percentage
|
|
308867
|
+
last: legacyDecimal8(ticker.last),
|
|
308868
|
+
bid: legacyDecimal8(ticker.bid),
|
|
308869
|
+
ask: legacyDecimal8(ticker.ask),
|
|
308870
|
+
high: legacyDecimal8(ticker.high),
|
|
308871
|
+
low: legacyDecimal8(ticker.low),
|
|
308872
|
+
open: legacyDecimal8(ticker.open),
|
|
308873
|
+
close: legacyDecimal8(ticker.close),
|
|
308874
|
+
base_volume: legacyDecimal8(ticker.baseVolume),
|
|
308875
|
+
quote_volume: legacyDecimal8(ticker.quoteVolume),
|
|
308876
|
+
change: legacyDecimal8(ticker.change),
|
|
308877
|
+
percentage: legacyDecimal8(ticker.percentage),
|
|
308878
|
+
payload_json: JSON.stringify(rawCapture.redactedPayload)
|
|
307902
308879
|
});
|
|
307903
308880
|
return { table: "market_data.cex_ticker_events", row };
|
|
307904
308881
|
}
|
|
@@ -307908,13 +308885,14 @@ function buildCanonicalTradeRow(context2, rawCapture, trade) {
|
|
|
307908
308885
|
}
|
|
307909
308886
|
const row = withNormalizedChecksum({
|
|
307910
308887
|
...captureCoreFields(context2, rawCapture),
|
|
308888
|
+
...legacyMarketFields(context2, rawCapture),
|
|
307911
308889
|
source_time_ms: trade.eventTimeMs,
|
|
307912
308890
|
trade_id: trade.tradeId,
|
|
307913
308891
|
event_time_ms: trade.eventTimeMs,
|
|
307914
308892
|
side: trade.side,
|
|
307915
|
-
price: trade.price,
|
|
307916
|
-
amount: trade.amount,
|
|
307917
|
-
cost: trade.cost,
|
|
308893
|
+
price: legacyDecimal8(trade.price),
|
|
308894
|
+
amount: legacyDecimal8(trade.amount),
|
|
308895
|
+
cost: legacyDecimal8(trade.cost),
|
|
307918
308896
|
taker_or_maker: trade.takerOrMaker
|
|
307919
308897
|
});
|
|
307920
308898
|
return { table: "market_data.cex_trades", row };
|
|
@@ -307994,10 +308972,19 @@ function resolveCaptureContext(archiver, input, feed, sourceMode) {
|
|
|
307994
308972
|
environment: captureEnvironmentFromEnv()
|
|
307995
308973
|
});
|
|
307996
308974
|
}
|
|
308975
|
+
function canArchiveMarketData(archiver) {
|
|
308976
|
+
return resolveMarketCaptureArchiveState({
|
|
308977
|
+
archiveEnabled: archiver?.isEnabled() ?? false,
|
|
308978
|
+
marketArchiveEnabled: isMarketArchiveEnabled(),
|
|
308979
|
+
environment: process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT,
|
|
308980
|
+
deploymentId: archiver?.getDeploymentId(),
|
|
308981
|
+
captureBundleId: process.env.CEX_BROKER_CAPTURE_BUNDLE_ID
|
|
308982
|
+
}).enabled;
|
|
308983
|
+
}
|
|
307997
308984
|
function archiveOrderbookInBackground(archiver, otelMetrics, input, options) {
|
|
307998
308985
|
const labels = watchLabels("orderbook", input, archiver, "ORDERBOOK");
|
|
307999
308986
|
recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
|
|
308000
|
-
if (!
|
|
308987
|
+
if (!canArchiveMarketData(archiver)) {
|
|
308001
308988
|
return;
|
|
308002
308989
|
}
|
|
308003
308990
|
if (options?.sampledOut) {
|
|
@@ -308039,7 +309026,7 @@ function archiveOrderbookInBackground(archiver, otelMetrics, input, options) {
|
|
|
308039
309026
|
function archiveOhlcvInBackground(archiver, otelMetrics, tracker, input) {
|
|
308040
309027
|
const labels = watchLabels("ohlcv", input, archiver, "OHLCV");
|
|
308041
309028
|
recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
|
|
308042
|
-
if (!
|
|
309029
|
+
if (!canArchiveMarketData(archiver)) {
|
|
308043
309030
|
return;
|
|
308044
309031
|
}
|
|
308045
309032
|
queueMicrotask(() => {
|
|
@@ -308084,7 +309071,7 @@ function createOhlcvBarTracker() {
|
|
|
308084
309071
|
function archiveMarketRowsInBackground(archiver, otelMetrics, stream4, input, feed, enqueueRows) {
|
|
308085
309072
|
const labels = watchLabels(stream4, input, archiver, feed);
|
|
308086
309073
|
recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
|
|
308087
|
-
if (!
|
|
309074
|
+
if (!canArchiveMarketData(archiver)) {
|
|
308088
309075
|
return;
|
|
308089
309076
|
}
|
|
308090
309077
|
queueMicrotask(() => {
|
|
@@ -309511,383 +310498,173 @@ function createExecuteActionHandler(deps) {
|
|
|
309511
310498
|
}, null);
|
|
309512
310499
|
}
|
|
309513
310500
|
const normalizedCex = cex3.trim().toLowerCase();
|
|
309514
|
-
const metadata = call.metadata;
|
|
309515
|
-
const selectedBrokerAccount = selectBrokerAccountForCex(normalizedCex, brokers, metadata);
|
|
309516
|
-
const broker = selectedBrokerAccount?.exchange ?? createBroker(normalizedCex, call.metadata) ?? (isPublicMarketDataAction(action, call.request.payload) ? createPublicBroker(normalizedCex) : null);
|
|
309517
|
-
if (!broker) {
|
|
309518
|
-
return wrappedCallback({
|
|
309519
|
-
code: grpc12.status.UNAUTHENTICATED,
|
|
309520
|
-
message: `This Exchange is not registered and No API metadata was found`
|
|
309521
|
-
}, null);
|
|
309522
|
-
}
|
|
309523
|
-
const verity = { proof: "" };
|
|
309524
|
-
const applyVerityToBroker = (targetBroker) => {
|
|
309525
|
-
if (!useVerity)
|
|
309526
|
-
return;
|
|
309527
|
-
const override = buildHttpClientOverrideFromMetadata(metadata, verityProverUrl, (proof, notaryPubKey) => {
|
|
309528
|
-
verity.proof = proof;
|
|
309529
|
-
log.debug(`Verity proof:`, { proof, notaryPubKey });
|
|
309530
|
-
});
|
|
309531
|
-
targetBroker.setHttpClientOverride(override, verityHttpClientOverridePredicate);
|
|
309532
|
-
};
|
|
309533
|
-
const preludeCtx = {
|
|
309534
|
-
call,
|
|
309535
|
-
wrappedCallback,
|
|
309536
|
-
action,
|
|
309537
|
-
policy,
|
|
309538
|
-
brokers,
|
|
309539
|
-
metadata,
|
|
309540
|
-
normalizedCex,
|
|
309541
|
-
cex: cex3,
|
|
309542
|
-
symbol: symbol2,
|
|
309543
|
-
selectedBrokerAccount,
|
|
309544
|
-
broker,
|
|
309545
|
-
verity,
|
|
309546
|
-
applyVerityToBroker,
|
|
309547
|
-
useVerity,
|
|
309548
|
-
verityProverUrl,
|
|
309549
|
-
otelMetrics,
|
|
309550
|
-
brokerArchiver,
|
|
309551
|
-
orderActivityTracker,
|
|
309552
|
-
withdrawalObservationTracker
|
|
309553
|
-
};
|
|
309554
|
-
if (action === Action.Call) {
|
|
309555
|
-
const handled = await handleOrderBookCall(preludeCtx);
|
|
309556
|
-
if (handled)
|
|
309557
|
-
return;
|
|
309558
|
-
}
|
|
309559
|
-
applyVerityToBroker(broker);
|
|
309560
|
-
const ctx = { ...preludeCtx, broker };
|
|
309561
|
-
await dispatchExecuteAction(ctx);
|
|
309562
|
-
} catch (error48) {
|
|
309563
|
-
safeLogError("ExecuteAction unhandled error", error48);
|
|
309564
|
-
return wrappedCallback({
|
|
309565
|
-
code: grpc12.status.INTERNAL,
|
|
309566
|
-
message: "ExecuteAction failed unexpectedly"
|
|
309567
|
-
}, null);
|
|
309568
|
-
}
|
|
309569
|
-
};
|
|
309570
|
-
}
|
|
309571
|
-
// src/handlers/subscribe/broker-lifecycle.ts
|
|
309572
|
-
class SubscribeBrokerLifecycle {
|
|
309573
|
-
#brokers = new Map;
|
|
309574
|
-
#closing = new Map;
|
|
309575
|
-
#shuttingDown = false;
|
|
309576
|
-
register(broker, context2) {
|
|
309577
|
-
this.#brokers.set(broker, context2);
|
|
309578
|
-
if (this.#shuttingDown) {
|
|
309579
|
-
this.close(broker);
|
|
309580
|
-
}
|
|
309581
|
-
}
|
|
309582
|
-
close(broker) {
|
|
309583
|
-
const existing = this.#closing.get(broker);
|
|
309584
|
-
if (existing) {
|
|
309585
|
-
return existing;
|
|
309586
|
-
}
|
|
309587
|
-
const context2 = this.#brokers.get(broker) ?? {
|
|
309588
|
-
cex: "unknown",
|
|
309589
|
-
symbol: "unknown"
|
|
309590
|
-
};
|
|
309591
|
-
this.#brokers.delete(broker);
|
|
309592
|
-
const closing = (async () => {
|
|
309593
|
-
try {
|
|
309594
|
-
await broker.close();
|
|
309595
|
-
log.debug("Request-scoped Subscribe broker closed", context2);
|
|
309596
|
-
return "closed";
|
|
309597
|
-
} catch (error48) {
|
|
309598
|
-
log.warn("Failed to close request-scoped Subscribe broker", {
|
|
309599
|
-
...context2,
|
|
309600
|
-
error: error48
|
|
309601
|
-
});
|
|
309602
|
-
return "failed";
|
|
309603
|
-
} finally {
|
|
309604
|
-
this.#closing.delete(broker);
|
|
309605
|
-
}
|
|
309606
|
-
})();
|
|
309607
|
-
this.#closing.set(broker, closing);
|
|
309608
|
-
return closing;
|
|
309609
|
-
}
|
|
309610
|
-
async closeAll() {
|
|
309611
|
-
this.#shuttingDown = true;
|
|
309612
|
-
let failed = 0;
|
|
309613
|
-
while (this.#brokers.size > 0 || this.#closing.size > 0) {
|
|
309614
|
-
const inFlight = [...this.#closing.values()];
|
|
309615
|
-
const fresh = [...this.#brokers.keys()].map((broker) => this.close(broker));
|
|
309616
|
-
const outcomes = await Promise.all([...fresh, ...inFlight]);
|
|
309617
|
-
failed += outcomes.filter((outcome) => outcome === "failed").length;
|
|
309618
|
-
}
|
|
309619
|
-
if (failed > 0) {
|
|
309620
|
-
throw new Error(`${failed} request-scoped Subscribe broker(s) failed to close`);
|
|
309621
|
-
}
|
|
309622
|
-
}
|
|
309623
|
-
}
|
|
309624
|
-
// src/handlers/subscribe/handler.ts
|
|
309625
|
-
import * as grpc13 from "@grpc/grpc-js";
|
|
309626
|
-
|
|
309627
|
-
// src/helpers/binance-user-data-stream.ts
|
|
309628
|
-
import { Buffer as Buffer2 } from "node:buffer";
|
|
309629
|
-
import { createHmac } from "node:crypto";
|
|
309630
|
-
var BINANCE_SPOT_WS_API_URL = "wss://ws-api.binance.com:443/ws-api/v3";
|
|
309631
|
-
var DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS = 16;
|
|
309632
|
-
var createWebSocket = (url3) => new wrapper_default(url3);
|
|
309633
|
-
var userDataRequestCounter = 0;
|
|
309634
|
-
function getExchangeString(exchange, key) {
|
|
309635
|
-
const value = exchange[key];
|
|
309636
|
-
if (typeof value !== "string" || value.length === 0) {
|
|
309637
|
-
throw new Error(`Binance user-data stream requires exchange.${key}`);
|
|
309638
|
-
}
|
|
309639
|
-
return value;
|
|
309640
|
-
}
|
|
309641
|
-
function sortedQuery(params) {
|
|
309642
|
-
return Object.entries(params).sort(([left], [right]) => left.localeCompare(right)).map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`).join("&");
|
|
309643
|
-
}
|
|
309644
|
-
function signUserDataStreamParams(exchange, params) {
|
|
309645
|
-
const signParams = exchange.signParams;
|
|
309646
|
-
if (typeof signParams === "function") {
|
|
309647
|
-
return signParams.call(exchange, params);
|
|
309648
|
-
}
|
|
309649
|
-
const secret = getExchangeString(exchange, "secret");
|
|
309650
|
-
return {
|
|
309651
|
-
...params,
|
|
309652
|
-
signature: createHmac("sha256", secret).update(sortedQuery(params)).digest("hex")
|
|
309653
|
-
};
|
|
309654
|
-
}
|
|
309655
|
-
function getBinanceSpotWsApiUrl(exchange) {
|
|
309656
|
-
const urls = exchange.urls;
|
|
309657
|
-
return urls?.api?.ws?.["ws-api"]?.spot ?? BINANCE_SPOT_WS_API_URL;
|
|
309658
|
-
}
|
|
309659
|
-
function getRecord(value) {
|
|
309660
|
-
return typeof value === "object" && value !== null ? value : null;
|
|
309661
|
-
}
|
|
309662
|
-
function getMessage(value) {
|
|
309663
|
-
if (value instanceof Error) {
|
|
309664
|
-
return value.message;
|
|
309665
|
-
}
|
|
309666
|
-
if (typeof value === "string" && value.length > 0) {
|
|
309667
|
-
return value;
|
|
309668
|
-
}
|
|
309669
|
-
const record2 = getRecord(value);
|
|
309670
|
-
const message = record2?.message;
|
|
309671
|
-
return typeof message === "string" && message.length > 0 ? message : null;
|
|
309672
|
-
}
|
|
309673
|
-
function getOptionalExchangeString(exchange, key) {
|
|
309674
|
-
const value = exchange[key];
|
|
309675
|
-
return typeof value === "string" && value.length > 0 ? value : null;
|
|
309676
|
-
}
|
|
309677
|
-
function redactDiagnosticMessage(message, secretValues) {
|
|
309678
|
-
let redacted = message;
|
|
309679
|
-
for (const value of secretValues) {
|
|
309680
|
-
if (value.length > 0) {
|
|
309681
|
-
redacted = redacted.split(value).join("[redacted]");
|
|
309682
|
-
}
|
|
309683
|
-
}
|
|
309684
|
-
return redacted.replace(/(\b(?:apiKey|secret|signature)\b\s*=\s*)[^\s&,;)]+/gi, "$1[redacted]").replace(/("(?:apiKey|secret|signature)"\s*:\s*")[^"]*(")/gi, "$1[redacted]$2");
|
|
309685
|
-
}
|
|
309686
|
-
function formatBinanceUserDataWebSocketError(event, secretValues) {
|
|
309687
|
-
const record2 = getRecord(event);
|
|
309688
|
-
const message = getMessage(record2?.error) ?? getMessage(record2?.message) ?? getMessage(event);
|
|
309689
|
-
const safeMessage = message === null ? null : redactDiagnosticMessage(message, secretValues);
|
|
309690
|
-
return new Error(safeMessage ? `Binance user-data WebSocket error: ${safeMessage}` : "Binance user-data WebSocket error");
|
|
309691
|
-
}
|
|
309692
|
-
function getCloseReason(value) {
|
|
309693
|
-
if (typeof value === "string") {
|
|
309694
|
-
return value.length > 0 ? value : null;
|
|
309695
|
-
}
|
|
309696
|
-
if (Buffer2.isBuffer(value)) {
|
|
309697
|
-
const reason = value.toString("utf8");
|
|
309698
|
-
return reason.length > 0 ? reason : null;
|
|
309699
|
-
}
|
|
309700
|
-
if (value instanceof Uint8Array) {
|
|
309701
|
-
const reason = Buffer2.from(value).toString("utf8");
|
|
309702
|
-
return reason.length > 0 ? reason : null;
|
|
309703
|
-
}
|
|
309704
|
-
return null;
|
|
309705
|
-
}
|
|
309706
|
-
function formatBinanceUserDataWebSocketClose(codeOrEvent, reasonOrUndefined, secretValues) {
|
|
309707
|
-
const record2 = getRecord(codeOrEvent);
|
|
309708
|
-
const code = record2 ? record2.code : codeOrEvent;
|
|
309709
|
-
const reason = getCloseReason(record2 ? record2.reason : reasonOrUndefined);
|
|
309710
|
-
const safeReason = reason === null ? null : redactDiagnosticMessage(reason, secretValues);
|
|
309711
|
-
const details = [
|
|
309712
|
-
typeof code === "number" || typeof code === "string" ? `code=${code}` : null,
|
|
309713
|
-
safeReason ? `reason=${safeReason}` : null
|
|
309714
|
-
].filter((detail) => detail !== null);
|
|
309715
|
-
return new Error(details.length > 0 ? `Binance user-data WebSocket closed unexpectedly (${details.join(", ")})` : "Binance user-data WebSocket closed unexpectedly");
|
|
309716
|
-
}
|
|
309717
|
-
function decodeMessageData(data) {
|
|
309718
|
-
if (typeof data === "string") {
|
|
309719
|
-
return data;
|
|
309720
|
-
}
|
|
309721
|
-
if (Buffer2.isBuffer(data)) {
|
|
309722
|
-
return data.toString("utf8");
|
|
309723
|
-
}
|
|
309724
|
-
if (data instanceof ArrayBuffer) {
|
|
309725
|
-
return Buffer2.from(data).toString("utf8");
|
|
309726
|
-
}
|
|
309727
|
-
if (ArrayBuffer.isView(data)) {
|
|
309728
|
-
return Buffer2.from(data.buffer, data.byteOffset, data.byteLength).toString("utf8");
|
|
309729
|
-
}
|
|
309730
|
-
if (Array.isArray(data) && data.every((item) => Buffer2.isBuffer(item))) {
|
|
309731
|
-
return Buffer2.concat(data).toString("utf8");
|
|
309732
|
-
}
|
|
309733
|
-
return data;
|
|
309734
|
-
}
|
|
309735
|
-
|
|
309736
|
-
class BinanceSpotUserDataStream {
|
|
309737
|
-
exchange;
|
|
309738
|
-
ws;
|
|
309739
|
-
secretValues;
|
|
309740
|
-
requestId = `user-data-${Date.now()}-${userDataRequestCounter++}`;
|
|
309741
|
-
maxBufferedEvents;
|
|
309742
|
-
queue = [];
|
|
309743
|
-
waiters = [];
|
|
309744
|
-
closed = false;
|
|
309745
|
-
closeError = null;
|
|
309746
|
-
subscriptionId = null;
|
|
309747
|
-
constructor(exchange, options = {}) {
|
|
309748
|
-
this.exchange = exchange;
|
|
309749
|
-
this.maxBufferedEvents = options.maxBufferedEvents ?? DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS;
|
|
309750
|
-
this.secretValues = [
|
|
309751
|
-
getOptionalExchangeString(exchange, "apiKey"),
|
|
309752
|
-
getOptionalExchangeString(exchange, "secret")
|
|
309753
|
-
].filter((value) => value !== null);
|
|
309754
|
-
this.ws = createWebSocket(getBinanceSpotWsApiUrl(exchange));
|
|
309755
|
-
this.ws.on("open", () => this.subscribe());
|
|
309756
|
-
this.ws.on("message", (data) => this.handleMessage(data));
|
|
309757
|
-
this.ws.on("error", (error48) => this.fail(formatBinanceUserDataWebSocketError(error48, this.secretValues)));
|
|
309758
|
-
this.ws.on("close", (code, reason) => this.handleClose(code, reason));
|
|
309759
|
-
}
|
|
309760
|
-
async* [Symbol.asyncIterator]() {
|
|
309761
|
-
while (true) {
|
|
309762
|
-
const event = await this.nextEvent();
|
|
309763
|
-
if (!event) {
|
|
309764
|
-
break;
|
|
310501
|
+
const metadata = call.metadata;
|
|
310502
|
+
const selectedBrokerAccount = selectBrokerAccountForCex(normalizedCex, brokers, metadata);
|
|
310503
|
+
const broker = selectedBrokerAccount?.exchange ?? createBroker(normalizedCex, call.metadata) ?? (isPublicMarketDataAction(action, call.request.payload) ? createPublicBroker(normalizedCex) : null);
|
|
310504
|
+
if (!broker) {
|
|
310505
|
+
return wrappedCallback({
|
|
310506
|
+
code: grpc12.status.UNAUTHENTICATED,
|
|
310507
|
+
message: `This Exchange is not registered and No API metadata was found`
|
|
310508
|
+
}, null);
|
|
309765
310509
|
}
|
|
309766
|
-
|
|
309767
|
-
|
|
309768
|
-
|
|
309769
|
-
|
|
309770
|
-
|
|
309771
|
-
|
|
310510
|
+
const verity = { proof: "" };
|
|
310511
|
+
const applyVerityToBroker = (targetBroker) => {
|
|
310512
|
+
if (!useVerity)
|
|
310513
|
+
return;
|
|
310514
|
+
const override = buildHttpClientOverrideFromMetadata(metadata, verityProverUrl, (proof, notaryPubKey) => {
|
|
310515
|
+
verity.proof = proof;
|
|
310516
|
+
log.debug(`Verity proof:`, { proof, notaryPubKey });
|
|
310517
|
+
});
|
|
310518
|
+
targetBroker.setHttpClientOverride(override, verityHttpClientOverridePredicate);
|
|
310519
|
+
};
|
|
310520
|
+
const preludeCtx = {
|
|
310521
|
+
call,
|
|
310522
|
+
wrappedCallback,
|
|
310523
|
+
action,
|
|
310524
|
+
policy,
|
|
310525
|
+
brokers,
|
|
310526
|
+
metadata,
|
|
310527
|
+
normalizedCex,
|
|
310528
|
+
cex: cex3,
|
|
310529
|
+
symbol: symbol2,
|
|
310530
|
+
selectedBrokerAccount,
|
|
310531
|
+
broker,
|
|
310532
|
+
verity,
|
|
310533
|
+
applyVerityToBroker,
|
|
310534
|
+
useVerity,
|
|
310535
|
+
verityProverUrl,
|
|
310536
|
+
otelMetrics,
|
|
310537
|
+
brokerArchiver,
|
|
310538
|
+
orderActivityTracker,
|
|
310539
|
+
withdrawalObservationTracker
|
|
310540
|
+
};
|
|
310541
|
+
if (action === Action.Call) {
|
|
310542
|
+
const handled = await handleOrderBookCall(preludeCtx);
|
|
310543
|
+
if (handled)
|
|
310544
|
+
return;
|
|
310545
|
+
}
|
|
310546
|
+
applyVerityToBroker(broker);
|
|
310547
|
+
const ctx = { ...preludeCtx, broker };
|
|
310548
|
+
await dispatchExecuteAction(ctx);
|
|
310549
|
+
} catch (error48) {
|
|
310550
|
+
safeLogError("ExecuteAction unhandled error", error48);
|
|
310551
|
+
return wrappedCallback({
|
|
310552
|
+
code: grpc12.status.INTERNAL,
|
|
310553
|
+
message: "ExecuteAction failed unexpectedly"
|
|
310554
|
+
}, null);
|
|
309772
310555
|
}
|
|
309773
|
-
|
|
309774
|
-
|
|
309775
|
-
|
|
309776
|
-
|
|
309777
|
-
|
|
309778
|
-
|
|
309779
|
-
|
|
309780
|
-
|
|
309781
|
-
|
|
309782
|
-
|
|
310556
|
+
};
|
|
310557
|
+
}
|
|
310558
|
+
// src/handlers/subscribe/broker-lifecycle.ts
|
|
310559
|
+
class SubscribeBrokerLifecycle {
|
|
310560
|
+
#brokers = new Map;
|
|
310561
|
+
#closing = new Map;
|
|
310562
|
+
#shuttingDown = false;
|
|
310563
|
+
register(broker, context2) {
|
|
310564
|
+
this.#brokers.set(broker, context2);
|
|
310565
|
+
if (this.#shuttingDown) {
|
|
310566
|
+
this.close(broker);
|
|
309783
310567
|
}
|
|
309784
|
-
this.fail(formatBinanceUserDataWebSocketClose(code, reason, this.secretValues));
|
|
309785
310568
|
}
|
|
309786
|
-
|
|
309787
|
-
const
|
|
309788
|
-
|
|
309789
|
-
|
|
309790
|
-
timestamp: Date.now()
|
|
309791
|
-
});
|
|
309792
|
-
this.ws.send(JSON.stringify({
|
|
309793
|
-
id: this.requestId,
|
|
309794
|
-
method: "userDataStream.subscribe.signature",
|
|
309795
|
-
params: signedParams
|
|
309796
|
-
}));
|
|
309797
|
-
}
|
|
309798
|
-
handleMessage(data) {
|
|
309799
|
-
if (this.closed) {
|
|
309800
|
-
return;
|
|
309801
|
-
}
|
|
309802
|
-
let message;
|
|
309803
|
-
try {
|
|
309804
|
-
const decodedData = decodeMessageData(data);
|
|
309805
|
-
message = typeof decodedData === "string" ? JSON.parse(decodedData) : decodedData;
|
|
309806
|
-
} catch (error48) {
|
|
309807
|
-
this.fail(error48 instanceof Error ? error48 : new Error("Invalid Binance user-data message"));
|
|
309808
|
-
return;
|
|
310569
|
+
close(broker) {
|
|
310570
|
+
const existing = this.#closing.get(broker);
|
|
310571
|
+
if (existing) {
|
|
310572
|
+
return existing;
|
|
309809
310573
|
}
|
|
309810
|
-
|
|
309811
|
-
|
|
309812
|
-
|
|
309813
|
-
|
|
310574
|
+
const context2 = this.#brokers.get(broker) ?? {
|
|
310575
|
+
cex: "unknown",
|
|
310576
|
+
symbol: "unknown"
|
|
310577
|
+
};
|
|
310578
|
+
this.#brokers.delete(broker);
|
|
310579
|
+
const closing = (async () => {
|
|
310580
|
+
try {
|
|
310581
|
+
await broker.close();
|
|
310582
|
+
log.debug("Request-scoped Subscribe broker closed", context2);
|
|
310583
|
+
return "closed";
|
|
310584
|
+
} catch (error48) {
|
|
310585
|
+
log.warn("Failed to close request-scoped Subscribe broker", {
|
|
310586
|
+
...context2,
|
|
310587
|
+
error: error48
|
|
310588
|
+
});
|
|
310589
|
+
return "failed";
|
|
310590
|
+
} finally {
|
|
310591
|
+
this.#closing.delete(broker);
|
|
309814
310592
|
}
|
|
309815
|
-
|
|
309816
|
-
|
|
309817
|
-
|
|
309818
|
-
if ("status" in message && typeof message.status === "number" && message.status !== 200) {
|
|
309819
|
-
const errorMessage = message.error?.msg ?? message.error?.message ?? `Binance user-data request failed with status ${message.status}`;
|
|
309820
|
-
const errorCode2 = message.error?.code;
|
|
309821
|
-
this.fail(new Error(typeof errorCode2 === "number" ? `${errorMessage} (code ${errorCode2})` : errorMessage));
|
|
309822
|
-
return;
|
|
309823
|
-
}
|
|
309824
|
-
if (!("event" in message) || !message.event) {
|
|
309825
|
-
return;
|
|
309826
|
-
}
|
|
309827
|
-
const subscriptionId = message.subscriptionId ?? this.subscriptionId;
|
|
309828
|
-
if (subscriptionId === null || subscriptionId === undefined) {
|
|
309829
|
-
return;
|
|
309830
|
-
}
|
|
309831
|
-
this.push({ subscriptionId, event: message.event });
|
|
310593
|
+
})();
|
|
310594
|
+
this.#closing.set(broker, closing);
|
|
310595
|
+
return closing;
|
|
309832
310596
|
}
|
|
309833
|
-
|
|
309834
|
-
|
|
309835
|
-
|
|
309836
|
-
|
|
309837
|
-
|
|
309838
|
-
|
|
309839
|
-
|
|
309840
|
-
|
|
310597
|
+
async closeAll() {
|
|
310598
|
+
this.#shuttingDown = true;
|
|
310599
|
+
let failed = 0;
|
|
310600
|
+
while (this.#brokers.size > 0 || this.#closing.size > 0) {
|
|
310601
|
+
const inFlight = [...this.#closing.values()];
|
|
310602
|
+
const fresh = [...this.#brokers.keys()].map((broker) => this.close(broker));
|
|
310603
|
+
const outcomes = await Promise.all([...fresh, ...inFlight]);
|
|
310604
|
+
failed += outcomes.filter((outcome) => outcome === "failed").length;
|
|
309841
310605
|
}
|
|
309842
|
-
if (
|
|
309843
|
-
|
|
309844
|
-
return;
|
|
310606
|
+
if (failed > 0) {
|
|
310607
|
+
throw new Error(`${failed} request-scoped Subscribe broker(s) failed to close`);
|
|
309845
310608
|
}
|
|
309846
|
-
this.queue.push(event);
|
|
309847
310609
|
}
|
|
309848
|
-
|
|
309849
|
-
|
|
309850
|
-
|
|
309851
|
-
|
|
309852
|
-
|
|
309853
|
-
|
|
309854
|
-
|
|
309855
|
-
|
|
309856
|
-
|
|
309857
|
-
return Promise.resolve(null);
|
|
309858
|
-
}
|
|
309859
|
-
return new Promise((resolve, reject) => {
|
|
309860
|
-
this.waiters.push({ resolve, reject });
|
|
309861
|
-
});
|
|
310610
|
+
}
|
|
310611
|
+
// src/handlers/subscribe/handler.ts
|
|
310612
|
+
import * as grpc13 from "@grpc/grpc-js";
|
|
310613
|
+
|
|
310614
|
+
// src/helpers/binance-user-data-normalization.ts
|
|
310615
|
+
function requireQuantity(entry, key2) {
|
|
310616
|
+
const value = entry[key2];
|
|
310617
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
310618
|
+
return value;
|
|
309862
310619
|
}
|
|
309863
|
-
|
|
309864
|
-
|
|
309865
|
-
return;
|
|
309866
|
-
}
|
|
309867
|
-
this.closeError = error48;
|
|
309868
|
-
this.closed = true;
|
|
309869
|
-
this.queue.length = 0;
|
|
309870
|
-
this.flushWaiters();
|
|
309871
|
-
try {
|
|
309872
|
-
this.ws.close();
|
|
309873
|
-
} catch {}
|
|
310620
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
310621
|
+
return String(value);
|
|
309874
310622
|
}
|
|
309875
|
-
|
|
309876
|
-
|
|
309877
|
-
|
|
309878
|
-
|
|
309879
|
-
|
|
309880
|
-
|
|
309881
|
-
|
|
309882
|
-
|
|
310623
|
+
throw new Error(`Invalid Binance balance quantity: ${key2}`);
|
|
310624
|
+
}
|
|
310625
|
+
async function normalizeBinanceSpotBalanceEvent(exchange, event) {
|
|
310626
|
+
if (event.e !== "outboundAccountPosition") {
|
|
310627
|
+
return exchange.fetchBalance({ type: "spot" });
|
|
310628
|
+
}
|
|
310629
|
+
if (!Array.isArray(event.B)) {
|
|
310630
|
+
throw new Error("Invalid Binance outboundAccountPosition balances");
|
|
310631
|
+
}
|
|
310632
|
+
const timestamp = typeof event.E === "number" && Number.isFinite(event.E) ? event.E : undefined;
|
|
310633
|
+
const balance = {
|
|
310634
|
+
info: event,
|
|
310635
|
+
...timestamp !== undefined && {
|
|
310636
|
+
timestamp,
|
|
310637
|
+
datetime: new Date(timestamp).toISOString()
|
|
310638
|
+
}
|
|
310639
|
+
};
|
|
310640
|
+
for (const rawEntry of event.B) {
|
|
310641
|
+
const entry = asRecord(rawEntry);
|
|
310642
|
+
const asset = entry?.a;
|
|
310643
|
+
if (!entry || typeof asset !== "string" || asset.length === 0) {
|
|
310644
|
+
throw new Error("Invalid Binance outboundAccountPosition asset");
|
|
309883
310645
|
}
|
|
310646
|
+
balance[asset] = {
|
|
310647
|
+
free: requireQuantity(entry, "f"),
|
|
310648
|
+
used: requireQuantity(entry, "l")
|
|
310649
|
+
};
|
|
309884
310650
|
}
|
|
310651
|
+
return exchange.safeBalance(balance);
|
|
309885
310652
|
}
|
|
309886
|
-
function
|
|
309887
|
-
|
|
310653
|
+
function getTradeId(value) {
|
|
310654
|
+
if (typeof value === "number" && Number.isFinite(value) || typeof value === "string" && value.length > 0) {
|
|
310655
|
+
const tradeId = String(value);
|
|
310656
|
+
return tradeId === "-1" ? undefined : tradeId;
|
|
310657
|
+
}
|
|
310658
|
+
return;
|
|
309888
310659
|
}
|
|
309889
|
-
function
|
|
309890
|
-
|
|
310660
|
+
function normalizeBinanceExecutionReport(exchange, event) {
|
|
310661
|
+
const parsed = asRecord(exchange.parseWsOrder(event));
|
|
310662
|
+
if (!parsed) {
|
|
310663
|
+
throw new Error("Binance executionReport did not parse as an order");
|
|
310664
|
+
}
|
|
310665
|
+
const { fee: _fee, fees: _fees, ...order } = parsed;
|
|
310666
|
+
const tradeId = getTradeId(event.t);
|
|
310667
|
+
return tradeId === undefined ? order : { ...order, tradeId };
|
|
309891
310668
|
}
|
|
309892
310669
|
// src/helpers/market-data-archive/ohlcv-bootstrap.ts
|
|
309893
310670
|
var DEFAULT_OHLCV_BOOTSTRAP_LIMIT = 100;
|
|
@@ -310009,12 +310786,14 @@ async function getBinanceMarketId(broker, symbol2) {
|
|
|
310009
310786
|
}
|
|
310010
310787
|
return symbol2.replace("/", "").toUpperCase();
|
|
310011
310788
|
}
|
|
310012
|
-
async function streamBinanceUserData(call, broker, symbol2, subscriptionType, isClosed, archiveContext) {
|
|
310013
|
-
const userDataStream = new BinanceSpotUserDataStream(broker);
|
|
310014
|
-
|
|
310015
|
-
|
|
310016
|
-
|
|
310017
|
-
|
|
310789
|
+
async function streamBinanceUserData(call, broker, symbol2, subscriptionType, isClosed, archiveContext, userDataSource, knownMarketId) {
|
|
310790
|
+
const userDataStream = userDataSource ?? new BinanceSpotUserDataStream(broker);
|
|
310791
|
+
if (!userDataSource) {
|
|
310792
|
+
call.once("close", () => userDataStream.close());
|
|
310793
|
+
call.once("cancelled", () => userDataStream.close());
|
|
310794
|
+
call.once("error", () => userDataStream.close());
|
|
310795
|
+
}
|
|
310796
|
+
const marketId = knownMarketId ?? (subscriptionType === SubscriptionType.ORDERS ? await getBinanceMarketId(broker, symbol2) : null);
|
|
310018
310797
|
try {
|
|
310019
310798
|
for await (const message of userDataStream) {
|
|
310020
310799
|
if (isClosed()) {
|
|
@@ -310035,17 +310814,6 @@ async function streamBinanceUserData(call, broker, symbol2, subscriptionType, is
|
|
|
310035
310814
|
}
|
|
310036
310815
|
}
|
|
310037
310816
|
const receivedTimestamp = Date.now();
|
|
310038
|
-
if (!await writeSubscribeFrame(call, isClosed, {
|
|
310039
|
-
data: JSON.stringify({
|
|
310040
|
-
subscriptionId: message.subscriptionId,
|
|
310041
|
-
event
|
|
310042
|
-
}),
|
|
310043
|
-
timestamp: receivedTimestamp,
|
|
310044
|
-
symbol: symbol2,
|
|
310045
|
-
type: subscriptionType
|
|
310046
|
-
})) {
|
|
310047
|
-
break;
|
|
310048
|
-
}
|
|
310049
310817
|
const archiveSubscriptionType = subscriptionType === SubscriptionType.BALANCE ? "BALANCE" : "ORDERS";
|
|
310050
310818
|
archiveSubscribeStreamInBackground(archiveContext?.archiver, {
|
|
310051
310819
|
exchange: archiveContext?.exchange ?? "binance",
|
|
@@ -310066,6 +310834,18 @@ async function streamBinanceUserData(call, broker, symbol2, subscriptionType, is
|
|
|
310066
310834
|
receivedTimestamp
|
|
310067
310835
|
});
|
|
310068
310836
|
}
|
|
310837
|
+
if (subscriptionType === SubscriptionType.ORDERS && event.e === "listStatus") {
|
|
310838
|
+
continue;
|
|
310839
|
+
}
|
|
310840
|
+
const data = subscriptionType === SubscriptionType.BALANCE ? await normalizeBinanceSpotBalanceEvent(broker, event) : normalizeBinanceExecutionReport(broker, event);
|
|
310841
|
+
if (!await writeSubscribeFrame(call, isClosed, {
|
|
310842
|
+
data: JSON.stringify(data),
|
|
310843
|
+
timestamp: receivedTimestamp,
|
|
310844
|
+
symbol: symbol2,
|
|
310845
|
+
type: subscriptionType
|
|
310846
|
+
})) {
|
|
310847
|
+
break;
|
|
310848
|
+
}
|
|
310069
310849
|
}
|
|
310070
310850
|
} finally {
|
|
310071
310851
|
userDataStream.close();
|
|
@@ -310105,7 +310885,13 @@ async function runCcxtSubscribeLoop(call, isClosed, symbol2, subscriptionType, w
|
|
|
310105
310885
|
}
|
|
310106
310886
|
}
|
|
310107
310887
|
function createSubscribeHandler(deps) {
|
|
310108
|
-
const {
|
|
310888
|
+
const {
|
|
310889
|
+
brokers,
|
|
310890
|
+
whitelistIps,
|
|
310891
|
+
otelMetrics,
|
|
310892
|
+
brokerArchiver,
|
|
310893
|
+
userDataStreamSupervisor
|
|
310894
|
+
} = deps;
|
|
310109
310895
|
const brokerLifecycle = deps.brokerLifecycle ?? new SubscribeBrokerLifecycle;
|
|
310110
310896
|
return async (call) => {
|
|
310111
310897
|
const subscribeStartTime = Date.now();
|
|
@@ -310240,7 +311026,28 @@ function createSubscribeHandler(deps) {
|
|
|
310240
311026
|
});
|
|
310241
311027
|
return;
|
|
310242
311028
|
}
|
|
310243
|
-
await
|
|
311029
|
+
const marketId = subscriptionType === SubscriptionType.ORDERS ? await getBinanceMarketId(accountBroker, resolvedSymbol) : undefined;
|
|
311030
|
+
let userDataSource;
|
|
311031
|
+
if (selectedBrokerAccount) {
|
|
311032
|
+
if (!userDataStreamSupervisor) {
|
|
311033
|
+
await writeSubscribeError(call, isStreamClosed, {
|
|
311034
|
+
data: JSON.stringify({
|
|
311035
|
+
error: "Configured account user-data supervisor is unavailable"
|
|
311036
|
+
}),
|
|
311037
|
+
timestamp: Date.now(),
|
|
311038
|
+
symbol: resolvedSymbol,
|
|
311039
|
+
type: subscriptionType
|
|
311040
|
+
});
|
|
311041
|
+
return;
|
|
311042
|
+
}
|
|
311043
|
+
userDataSource = userDataStreamSupervisor.subscribe({
|
|
311044
|
+
exchange: normalizedCex,
|
|
311045
|
+
accountSelector: selectedBrokerAccount.label,
|
|
311046
|
+
kind: subscriptionType === SubscriptionType.BALANCE ? "balance" : "orders",
|
|
311047
|
+
marketId
|
|
311048
|
+
});
|
|
311049
|
+
}
|
|
311050
|
+
await streamBinanceUserData(call, accountBroker, resolvedSymbol, subscriptionType, isStreamClosed, streamArchiveContext, userDataSource, marketId);
|
|
310244
311051
|
return;
|
|
310245
311052
|
}
|
|
310246
311053
|
switch (subscriptionType) {
|
|
@@ -310634,7 +311441,7 @@ var CEX_BROKER_PACKAGE_DEFINITION = protoLoader.fromJSON(node_descriptor_default
|
|
|
310634
311441
|
// src/server.ts
|
|
310635
311442
|
var grpcObj = grpc14.loadPackageDefinition(CEX_BROKER_PACKAGE_DEFINITION);
|
|
310636
311443
|
var cexNode = grpcObj.cex_broker;
|
|
310637
|
-
function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker, subscribeBrokerLifecycle) {
|
|
311444
|
+
function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker, subscribeBrokerLifecycle, userDataStreamSupervisor) {
|
|
310638
311445
|
const server = new grpc14.Server;
|
|
310639
311446
|
server.addService(cexNode.cex_service.service, {
|
|
310640
311447
|
ExecuteAction: createExecuteActionHandler({
|
|
@@ -310653,7 +311460,8 @@ function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, ot
|
|
|
310653
311460
|
whitelistIps,
|
|
310654
311461
|
otelMetrics,
|
|
310655
311462
|
brokerArchiver,
|
|
310656
|
-
brokerLifecycle: subscribeBrokerLifecycle
|
|
311463
|
+
brokerLifecycle: subscribeBrokerLifecycle,
|
|
311464
|
+
userDataStreamSupervisor
|
|
310657
311465
|
})
|
|
310658
311466
|
});
|
|
310659
311467
|
return server;
|
|
@@ -310797,13 +311605,14 @@ class CEXBroker {
|
|
|
310797
311605
|
fillArchivePoller;
|
|
310798
311606
|
depositArchivePoller;
|
|
310799
311607
|
accountBalanceArchivePoller;
|
|
311608
|
+
userDataStreamSupervisor;
|
|
310800
311609
|
loadEnvConfig() {
|
|
310801
311610
|
log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
|
|
310802
311611
|
const configMap = {};
|
|
310803
|
-
for (const [
|
|
310804
|
-
if (!
|
|
311612
|
+
for (const [key2, value] of Object.entries(process.env)) {
|
|
311613
|
+
if (!key2.startsWith("CEX_BROKER_"))
|
|
310805
311614
|
continue;
|
|
310806
|
-
let match =
|
|
311615
|
+
let match = key2.match(/^CEX_BROKER_(\w+)_(API_(KEY|SECRET)|ROLE|EMAIL|SUBACCOUNTID|UID)_(\d+)$/);
|
|
310807
311616
|
if (match) {
|
|
310808
311617
|
const broker2 = match[1]?.toLowerCase() ?? "";
|
|
310809
311618
|
const type3 = match[2]?.toLowerCase() ?? "";
|
|
@@ -310832,9 +311641,9 @@ class CEXBroker {
|
|
|
310832
311641
|
}
|
|
310833
311642
|
continue;
|
|
310834
311643
|
}
|
|
310835
|
-
match =
|
|
311644
|
+
match = key2.match(/^CEX_BROKER_(\w+)_(API_(KEY|SECRET)|ROLE|EMAIL|SUBACCOUNTID|UID)$/);
|
|
310836
311645
|
if (!match) {
|
|
310837
|
-
log.warn(`⚠️ Skipping unrecognized env var: ${
|
|
311646
|
+
log.warn(`⚠️ Skipping unrecognized env var: ${key2}`);
|
|
310838
311647
|
continue;
|
|
310839
311648
|
}
|
|
310840
311649
|
const broker = match[1]?.toLowerCase() ?? "";
|
|
@@ -310952,6 +311761,10 @@ class CEXBroker {
|
|
|
310952
311761
|
if (this.server) {
|
|
310953
311762
|
await this.server.forceShutdown();
|
|
310954
311763
|
}
|
|
311764
|
+
if (this.userDataStreamSupervisor) {
|
|
311765
|
+
await this.userDataStreamSupervisor.close();
|
|
311766
|
+
this.userDataStreamSupervisor = undefined;
|
|
311767
|
+
}
|
|
310955
311768
|
if (this.brokerArchiver) {
|
|
310956
311769
|
await this.brokerArchiver.close();
|
|
310957
311770
|
}
|
|
@@ -310963,6 +311776,14 @@ class CEXBroker {
|
|
|
310963
311776
|
}
|
|
310964
311777
|
}
|
|
310965
311778
|
async run() {
|
|
311779
|
+
const marketArchiveState = resolveMarketCaptureArchiveState({
|
|
311780
|
+
archiveEnabled: this.brokerArchiver?.isEnabled() ?? false,
|
|
311781
|
+
marketArchiveEnabled: isMarketArchiveEnabled(),
|
|
311782
|
+
environment: process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT,
|
|
311783
|
+
deploymentId: this.brokerArchiver?.getDeploymentId(),
|
|
311784
|
+
captureBundleId: process.env.CEX_BROKER_CAPTURE_BUNDLE_ID
|
|
311785
|
+
});
|
|
311786
|
+
assertMarketCaptureArchiveStartable(marketArchiveState);
|
|
310966
311787
|
if (this.server) {
|
|
310967
311788
|
await this.server.forceShutdown();
|
|
310968
311789
|
}
|
|
@@ -310986,7 +311807,15 @@ class CEXBroker {
|
|
|
310986
311807
|
if (this.otelMetrics?.isOtelEnabled()) {
|
|
310987
311808
|
await this.otelMetrics.initialize();
|
|
310988
311809
|
}
|
|
310989
|
-
|
|
311810
|
+
if (!this.userDataStreamSupervisor && Object.keys(this.brokers).length > 0) {
|
|
311811
|
+
const publisher = new StreamHealthPublisher(streamHealthPublisherConfigFromEnv());
|
|
311812
|
+
this.userDataStreamSupervisor = new UserDataStreamSupervisor({
|
|
311813
|
+
brokers: this.brokers,
|
|
311814
|
+
publisher
|
|
311815
|
+
});
|
|
311816
|
+
this.userDataStreamSupervisor.start();
|
|
311817
|
+
}
|
|
311818
|
+
this.server = getServer(this.policy, this.brokers, this.whitelistIps, this.useVerity, this.#verityProverUrl, this.otelMetrics, this.brokerArchiver, this.orderActivityTracker, this.withdrawalObservationTracker, undefined, this.userDataStreamSupervisor);
|
|
310990
311819
|
this.server.bindAsync(`0.0.0.0:${this.port}`, grpc15.ServerCredentials.createInsecure(), (err2, port) => {
|
|
310991
311820
|
if (err2) {
|
|
310992
311821
|
log.error(err2);
|
|
@@ -311031,4 +311860,4 @@ export {
|
|
|
311031
311860
|
CEXBroker as default
|
|
311032
311861
|
};
|
|
311033
311862
|
|
|
311034
|
-
//# debugId=
|
|
311863
|
+
//# debugId=66D8B9B2EE90476F64756E2164756E21
|