@usherlabs/cex-broker 0.2.23 → 0.2.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cli.js +484 -8
- package/dist/handlers/execute-action/context.d.ts +2 -1
- package/dist/handlers/execute-action/handler.d.ts +2 -1
- package/dist/helpers/account-balance-archive-poller.d.ts +19 -0
- package/dist/helpers/broker-execution-archive/capture.d.ts +6 -0
- package/dist/helpers/broker-execution-archive/index.d.ts +5 -4
- package/dist/helpers/broker-execution-archive/rows.d.ts +18 -0
- package/dist/helpers/broker-execution-archive/types.d.ts +4 -2
- package/dist/helpers/broker-execution-archive/withdrawal-observation-tracker.d.ts +23 -0
- package/dist/helpers/broker-execution-archive/writer.d.ts +3 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +485 -9
- package/dist/index.js.map +15 -13
- package/dist/server.d.ts +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -274461,6 +274461,9 @@ function compactUndefined(record) {
|
|
|
274461
274461
|
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
274462
274462
|
}
|
|
274463
274463
|
|
|
274464
|
+
// src/helpers/broker-execution-archive/rows.ts
|
|
274465
|
+
import { createHash as createHash2 } from "crypto";
|
|
274466
|
+
|
|
274464
274467
|
// src/helpers/broker-execution-archive/redact.ts
|
|
274465
274468
|
import { createHash } from "crypto";
|
|
274466
274469
|
var SECRET_KEY_PATTERN = /\b(api[_-]?key|api[_-]?secret|secret|signature|passphrase|password|token|credential)\b/i;
|
|
@@ -274533,6 +274536,8 @@ function hashMarketMetadata(payload) {
|
|
|
274533
274536
|
var BROKER_WRITE_SOURCE = "broker_write";
|
|
274534
274537
|
var ARCHIVE_SCHEMA_VERSION = "1";
|
|
274535
274538
|
var FILL_EVENT_KIND = "trade_history_fill";
|
|
274539
|
+
var ACCOUNT_BALANCE_SCOPE = "spot";
|
|
274540
|
+
var ACCOUNT_BALANCE_PRECISION_BASIS = "ccxt_normalized_number";
|
|
274536
274541
|
|
|
274537
274542
|
// src/helpers/broker-execution-archive/rows.ts
|
|
274538
274543
|
function firstString2(...values2) {
|
|
@@ -274566,6 +274571,151 @@ function normalizeTimestamp2(value) {
|
|
|
274566
274571
|
const date = new Date(ms);
|
|
274567
274572
|
return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
|
|
274568
274573
|
}
|
|
274574
|
+
function decimalString(value) {
|
|
274575
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
274576
|
+
return;
|
|
274577
|
+
}
|
|
274578
|
+
if (Object.is(value, -0)) {
|
|
274579
|
+
return "0";
|
|
274580
|
+
}
|
|
274581
|
+
const rendered = String(value).toLowerCase();
|
|
274582
|
+
if (!rendered.includes("e")) {
|
|
274583
|
+
return rendered;
|
|
274584
|
+
}
|
|
274585
|
+
const [coefficient = "", rawExponent = "0"] = rendered.split("e");
|
|
274586
|
+
const exponent = Number.parseInt(rawExponent, 10);
|
|
274587
|
+
const negative = coefficient.startsWith("-");
|
|
274588
|
+
const unsigned = negative ? coefficient.slice(1) : coefficient;
|
|
274589
|
+
const [integer = "0", fraction = ""] = unsigned.split(".");
|
|
274590
|
+
const digits = `${integer}${fraction}`;
|
|
274591
|
+
const decimalIndex = integer.length + exponent;
|
|
274592
|
+
let expanded;
|
|
274593
|
+
if (decimalIndex <= 0) {
|
|
274594
|
+
expanded = `0.${"0".repeat(-decimalIndex)}${digits}`;
|
|
274595
|
+
} else if (decimalIndex >= digits.length) {
|
|
274596
|
+
expanded = `${digits}${"0".repeat(decimalIndex - digits.length)}`;
|
|
274597
|
+
} else {
|
|
274598
|
+
expanded = `${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
|
|
274599
|
+
}
|
|
274600
|
+
return negative ? `-${expanded}` : expanded;
|
|
274601
|
+
}
|
|
274602
|
+
function normalizedBalanceMap(value) {
|
|
274603
|
+
const record = asRecord(value);
|
|
274604
|
+
if (!record) {
|
|
274605
|
+
return {};
|
|
274606
|
+
}
|
|
274607
|
+
const entries = [];
|
|
274608
|
+
for (const [rawAsset, quantity] of Object.entries(record)) {
|
|
274609
|
+
const asset = rawAsset.trim();
|
|
274610
|
+
const normalized = decimalString(quantity);
|
|
274611
|
+
if (asset && normalized !== undefined) {
|
|
274612
|
+
entries.push([asset, normalized]);
|
|
274613
|
+
}
|
|
274614
|
+
}
|
|
274615
|
+
return Object.fromEntries(entries.sort(([left], [right]) => left.localeCompare(right)));
|
|
274616
|
+
}
|
|
274617
|
+
function sortedBalanceMap(map) {
|
|
274618
|
+
return Object.fromEntries(Object.entries(map).sort(([left], [right]) => left.localeCompare(right)));
|
|
274619
|
+
}
|
|
274620
|
+
var BALANCE_METADATA_KEYS = new Set([
|
|
274621
|
+
"info",
|
|
274622
|
+
"timestamp",
|
|
274623
|
+
"datetime",
|
|
274624
|
+
"free",
|
|
274625
|
+
"used",
|
|
274626
|
+
"total",
|
|
274627
|
+
"debt"
|
|
274628
|
+
]);
|
|
274629
|
+
function normalizeCcxtBalanceForArchive(balance) {
|
|
274630
|
+
const record = asRecord(balance) ?? {};
|
|
274631
|
+
const aggregateRecords = {
|
|
274632
|
+
free: asRecord(record.free),
|
|
274633
|
+
used: asRecord(record.used),
|
|
274634
|
+
total: asRecord(record.total)
|
|
274635
|
+
};
|
|
274636
|
+
const maps = {
|
|
274637
|
+
free: normalizedBalanceMap(record.free),
|
|
274638
|
+
used: normalizedBalanceMap(record.used),
|
|
274639
|
+
total: normalizedBalanceMap(record.total)
|
|
274640
|
+
};
|
|
274641
|
+
const assetEntryAssets = new Set;
|
|
274642
|
+
for (const [rawAsset, value] of Object.entries(record)) {
|
|
274643
|
+
if (BALANCE_METADATA_KEYS.has(rawAsset)) {
|
|
274644
|
+
continue;
|
|
274645
|
+
}
|
|
274646
|
+
const asset = rawAsset.trim();
|
|
274647
|
+
const assetEntry = asRecord(value);
|
|
274648
|
+
if (!asset || !assetEntry || !(("free" in assetEntry) || ("used" in assetEntry) || ("total" in assetEntry))) {
|
|
274649
|
+
continue;
|
|
274650
|
+
}
|
|
274651
|
+
assetEntryAssets.add(asset);
|
|
274652
|
+
for (const field of ["free", "used", "total"]) {
|
|
274653
|
+
const normalized = decimalString(assetEntry[field]);
|
|
274654
|
+
if (normalized !== undefined && maps[field][asset] === undefined) {
|
|
274655
|
+
maps[field][asset] = normalized;
|
|
274656
|
+
}
|
|
274657
|
+
}
|
|
274658
|
+
}
|
|
274659
|
+
const reportedAssets = new Set(assetEntryAssets);
|
|
274660
|
+
for (const aggregateRecord of Object.values(aggregateRecords)) {
|
|
274661
|
+
for (const rawAsset of Object.keys(aggregateRecord ?? {})) {
|
|
274662
|
+
const asset = rawAsset.trim();
|
|
274663
|
+
if (asset) {
|
|
274664
|
+
reportedAssets.add(asset);
|
|
274665
|
+
}
|
|
274666
|
+
}
|
|
274667
|
+
}
|
|
274668
|
+
for (const map of Object.values(maps)) {
|
|
274669
|
+
for (const asset of Object.keys(map)) {
|
|
274670
|
+
reportedAssets.add(asset);
|
|
274671
|
+
}
|
|
274672
|
+
}
|
|
274673
|
+
return {
|
|
274674
|
+
exchangeTimestamp: normalizeTimestamp2(record.timestamp ?? record.datetime),
|
|
274675
|
+
reportedAssets: [...reportedAssets].sort(),
|
|
274676
|
+
assetEntryAssets: [...assetEntryAssets].sort(),
|
|
274677
|
+
freeBalances: sortedBalanceMap(maps.free),
|
|
274678
|
+
usedBalances: sortedBalanceMap(maps.used),
|
|
274679
|
+
totalBalances: sortedBalanceMap(maps.total),
|
|
274680
|
+
freeMapPresent: aggregateRecords.free !== undefined,
|
|
274681
|
+
usedMapPresent: aggregateRecords.used !== undefined,
|
|
274682
|
+
totalMapPresent: aggregateRecords.total !== undefined
|
|
274683
|
+
};
|
|
274684
|
+
}
|
|
274685
|
+
function buildAccountBalanceSnapshotRow(input) {
|
|
274686
|
+
const { tags, balance } = input;
|
|
274687
|
+
const observationId = createHash2("sha256").update(JSON.stringify({
|
|
274688
|
+
deployment_id: tags.deployment_id,
|
|
274689
|
+
exchange: tags.exchange,
|
|
274690
|
+
account_selector: tags.account_selector,
|
|
274691
|
+
balance_scope: ACCOUNT_BALANCE_SCOPE,
|
|
274692
|
+
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
274693
|
+
balance
|
|
274694
|
+
})).digest("hex");
|
|
274695
|
+
return {
|
|
274696
|
+
table: "broker_account.balance_snapshots",
|
|
274697
|
+
row: compactUndefined2({
|
|
274698
|
+
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
274699
|
+
exchange_timestamp: balance.exchangeTimestamp,
|
|
274700
|
+
source: tags.source,
|
|
274701
|
+
deployment_id: tags.deployment_id,
|
|
274702
|
+
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
274703
|
+
exchange: tags.exchange,
|
|
274704
|
+
account_selector: tags.account_selector,
|
|
274705
|
+
balance_scope: ACCOUNT_BALANCE_SCOPE,
|
|
274706
|
+
observation_id: observationId,
|
|
274707
|
+
reported_assets: balance.reportedAssets,
|
|
274708
|
+
asset_entry_assets: balance.assetEntryAssets,
|
|
274709
|
+
free_balances: balance.freeBalances,
|
|
274710
|
+
used_balances: balance.usedBalances,
|
|
274711
|
+
total_balances: balance.totalBalances,
|
|
274712
|
+
aggregate_free_map_present: Number(balance.freeMapPresent),
|
|
274713
|
+
aggregate_used_map_present: Number(balance.usedMapPresent),
|
|
274714
|
+
aggregate_total_map_present: Number(balance.totalMapPresent),
|
|
274715
|
+
precision_basis: ACCOUNT_BALANCE_PRECISION_BASIS
|
|
274716
|
+
})
|
|
274717
|
+
};
|
|
274718
|
+
}
|
|
274569
274719
|
function compactUndefined2(record) {
|
|
274570
274720
|
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
274571
274721
|
}
|
|
@@ -274810,6 +274960,52 @@ function archiveTransferEventInBackground(archiver, input) {
|
|
|
274810
274960
|
}
|
|
274811
274961
|
});
|
|
274812
274962
|
}
|
|
274963
|
+
function archiveWithdrawalObservationsInBackground(archiver, tracker, input) {
|
|
274964
|
+
try {
|
|
274965
|
+
if (!archiver?.isEnabled() || !Array.isArray(input.transactions)) {
|
|
274966
|
+
return;
|
|
274967
|
+
}
|
|
274968
|
+
const brokerObservedTimestamp = new Date().toISOString();
|
|
274969
|
+
for (const [resultIndex, transaction] of input.transactions.entries()) {
|
|
274970
|
+
const normalized = normalizeCcxtTransactionForArchive(transaction);
|
|
274971
|
+
const assetSymbol = normalized.assetSymbol;
|
|
274972
|
+
if (!tracker.shouldArchive({
|
|
274973
|
+
exchange: input.exchange,
|
|
274974
|
+
accountSelector: input.accountSelector,
|
|
274975
|
+
assetSymbol,
|
|
274976
|
+
transaction,
|
|
274977
|
+
normalized
|
|
274978
|
+
})) {
|
|
274979
|
+
continue;
|
|
274980
|
+
}
|
|
274981
|
+
archiveTransferEventInBackground(archiver, {
|
|
274982
|
+
exchange: input.exchange,
|
|
274983
|
+
accountSelector: input.accountSelector,
|
|
274984
|
+
assetSymbol,
|
|
274985
|
+
brokerObservedTimestamp,
|
|
274986
|
+
transfer: {
|
|
274987
|
+
eventKind: "withdrawal",
|
|
274988
|
+
lifecycleAction: "observe_withdrawal",
|
|
274989
|
+
status: normalized.status,
|
|
274990
|
+
amount: normalized.amount,
|
|
274991
|
+
address: normalized.address,
|
|
274992
|
+
network: normalized.network,
|
|
274993
|
+
externalId: normalized.externalId,
|
|
274994
|
+
txid: normalized.txid,
|
|
274995
|
+
resultIndex,
|
|
274996
|
+
feeAmount: normalized.feeAmount,
|
|
274997
|
+
feeCurrency: normalized.feeCurrency,
|
|
274998
|
+
exchangeTimestamp: normalized.exchangeTimestamp,
|
|
274999
|
+
payload: transaction
|
|
275000
|
+
}
|
|
275001
|
+
});
|
|
275002
|
+
}
|
|
275003
|
+
} catch (archiveError) {
|
|
275004
|
+
log.warn("Failed to archive withdrawal observations", {
|
|
275005
|
+
error: archiveError
|
|
275006
|
+
});
|
|
275007
|
+
}
|
|
275008
|
+
}
|
|
274813
275009
|
async function captureMarketMetadataSnapshot(archiver, broker, input) {
|
|
274814
275010
|
if (!archiver?.canPersistMarketMetadataSnapshot()) {
|
|
274815
275011
|
return;
|
|
@@ -274855,6 +275051,106 @@ async function captureMarketMetadataSnapshot(archiver, broker, input) {
|
|
|
274855
275051
|
return;
|
|
274856
275052
|
}
|
|
274857
275053
|
}
|
|
275054
|
+
// src/helpers/broker-execution-archive/withdrawal-observation-tracker.ts
|
|
275055
|
+
var DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES = 1e4;
|
|
275056
|
+
var VENUE_LIFECYCLE_EVIDENCE_FIELDS = [
|
|
275057
|
+
"updated",
|
|
275058
|
+
"updatedAt",
|
|
275059
|
+
"updated_at",
|
|
275060
|
+
"updateTime",
|
|
275061
|
+
"update_time",
|
|
275062
|
+
"updateTimestamp",
|
|
275063
|
+
"lastUpdateTimestamp",
|
|
275064
|
+
"completed",
|
|
275065
|
+
"completedAt",
|
|
275066
|
+
"completed_at",
|
|
275067
|
+
"completeTime",
|
|
275068
|
+
"complete_time",
|
|
275069
|
+
"completionTime",
|
|
275070
|
+
"successTime"
|
|
275071
|
+
];
|
|
275072
|
+
function fingerprintEvidenceValue(value) {
|
|
275073
|
+
if (value === undefined || value === null) {
|
|
275074
|
+
return value;
|
|
275075
|
+
}
|
|
275076
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
275077
|
+
return value;
|
|
275078
|
+
}
|
|
275079
|
+
if (typeof value === "bigint") {
|
|
275080
|
+
return value.toString();
|
|
275081
|
+
}
|
|
275082
|
+
try {
|
|
275083
|
+
return JSON.stringify(value);
|
|
275084
|
+
} catch {
|
|
275085
|
+
return String(value);
|
|
275086
|
+
}
|
|
275087
|
+
}
|
|
275088
|
+
function venueLifecycleEvidence(transaction) {
|
|
275089
|
+
const record = asRecord(transaction);
|
|
275090
|
+
const info = asRecord(record?.info);
|
|
275091
|
+
const evidence = [];
|
|
275092
|
+
for (const [scope, source] of [
|
|
275093
|
+
["transaction", record],
|
|
275094
|
+
["info", info]
|
|
275095
|
+
]) {
|
|
275096
|
+
for (const field of VENUE_LIFECYCLE_EVIDENCE_FIELDS) {
|
|
275097
|
+
const value = source?.[field];
|
|
275098
|
+
if (value !== undefined && value !== null) {
|
|
275099
|
+
evidence.push([scope, field, fingerprintEvidenceValue(value)]);
|
|
275100
|
+
}
|
|
275101
|
+
}
|
|
275102
|
+
}
|
|
275103
|
+
return evidence;
|
|
275104
|
+
}
|
|
275105
|
+
function observationFingerprint(observation) {
|
|
275106
|
+
const { normalized, transaction } = observation;
|
|
275107
|
+
return JSON.stringify({
|
|
275108
|
+
status: normalized.status ?? null,
|
|
275109
|
+
txid: normalized.txid ?? null,
|
|
275110
|
+
amount: normalized.amount ?? null,
|
|
275111
|
+
feeAmount: normalized.feeAmount ?? null,
|
|
275112
|
+
feeCurrency: normalized.feeCurrency ?? null,
|
|
275113
|
+
address: normalized.address ?? null,
|
|
275114
|
+
network: normalized.network ?? null,
|
|
275115
|
+
exchangeTimestamp: normalized.exchangeTimestamp ?? null,
|
|
275116
|
+
venueLifecycleEvidence: venueLifecycleEvidence(transaction)
|
|
275117
|
+
});
|
|
275118
|
+
}
|
|
275119
|
+
|
|
275120
|
+
class WithdrawalObservationTracker {
|
|
275121
|
+
#fingerprints = new Map;
|
|
275122
|
+
#maxEntries;
|
|
275123
|
+
#missingIdSequence = 0n;
|
|
275124
|
+
constructor(options) {
|
|
275125
|
+
const maxEntries = options?.maxEntries ?? DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES;
|
|
275126
|
+
this.#maxEntries = Number.isFinite(maxEntries) ? Math.max(1, Math.floor(maxEntries)) : DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES;
|
|
275127
|
+
}
|
|
275128
|
+
shouldArchive(observation) {
|
|
275129
|
+
const externalId = observation.normalized.externalId?.trim();
|
|
275130
|
+
const identity = JSON.stringify([
|
|
275131
|
+
observation.exchange.trim().toLowerCase() || "unknown",
|
|
275132
|
+
observation.accountSelector?.trim() || "unknown",
|
|
275133
|
+
observation.assetSymbol?.trim().toUpperCase() || "unknown",
|
|
275134
|
+
externalId || `missing:${++this.#missingIdSequence}`
|
|
275135
|
+
]);
|
|
275136
|
+
const fingerprint = observationFingerprint(observation);
|
|
275137
|
+
if (this.#fingerprints.get(identity) === fingerprint) {
|
|
275138
|
+
return false;
|
|
275139
|
+
}
|
|
275140
|
+
this.#fingerprints.delete(identity);
|
|
275141
|
+
this.#fingerprints.set(identity, fingerprint);
|
|
275142
|
+
while (this.#fingerprints.size > this.#maxEntries) {
|
|
275143
|
+
const oldest = this.#fingerprints.keys().next().value;
|
|
275144
|
+
if (oldest === undefined)
|
|
275145
|
+
break;
|
|
275146
|
+
this.#fingerprints.delete(oldest);
|
|
275147
|
+
}
|
|
275148
|
+
return true;
|
|
275149
|
+
}
|
|
275150
|
+
getSize() {
|
|
275151
|
+
return this.#fingerprints.size;
|
|
275152
|
+
}
|
|
275153
|
+
}
|
|
274858
275154
|
// src/helpers/broker-execution-archive/writer.ts
|
|
274859
275155
|
import { request as httpRequest2 } from "http";
|
|
274860
275156
|
import { request as httpsRequest2 } from "https";
|
|
@@ -275000,6 +275296,15 @@ function isMarketArchiveTable(table) {
|
|
|
275000
275296
|
}
|
|
275001
275297
|
|
|
275002
275298
|
// src/helpers/broker-execution-archive/writer.ts
|
|
275299
|
+
var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
|
|
275300
|
+
"broker_execution.order_events",
|
|
275301
|
+
"broker_execution.market_metadata_snapshots",
|
|
275302
|
+
"broker_execution.transfer_events",
|
|
275303
|
+
"broker_execution.fill_events"
|
|
275304
|
+
]);
|
|
275305
|
+
function isBrokerExecutionArchiveTable(table) {
|
|
275306
|
+
return BROKER_EXECUTION_ARCHIVE_TABLES.has(table);
|
|
275307
|
+
}
|
|
275003
275308
|
var DEFAULT_MAX_QUEUE_SIZE = 1e4;
|
|
275004
275309
|
var DEFAULT_BATCH_SIZE = 10;
|
|
275005
275310
|
var DEFAULT_FLUSH_INTERVAL_MS = 1000;
|
|
@@ -275084,6 +275389,9 @@ class BrokerExecutionArchiver {
|
|
|
275084
275389
|
canPersistMarketMetadataSnapshot() {
|
|
275085
275390
|
return this.isEnabled() && (Boolean(this.otelLogs?.isOtelEnabled()) || Boolean(this.forwarderUrl));
|
|
275086
275391
|
}
|
|
275392
|
+
canPersistAccountBalanceSnapshots() {
|
|
275393
|
+
return this.isEnabled() && Boolean(this.forwarderUrl);
|
|
275394
|
+
}
|
|
275087
275395
|
enqueue(row) {
|
|
275088
275396
|
if (!this.enabled || this.closed || !this.otelLogs && !this.forwarderUrl) {
|
|
275089
275397
|
return;
|
|
@@ -275095,6 +275403,9 @@ class BrokerExecutionArchiver {
|
|
|
275095
275403
|
}
|
|
275096
275404
|
return;
|
|
275097
275405
|
}
|
|
275406
|
+
if (row.table === "broker_account.balance_snapshots" && !this.forwarderUrl) {
|
|
275407
|
+
return;
|
|
275408
|
+
}
|
|
275098
275409
|
if (this.queue.length >= this.maxQueueSize) {
|
|
275099
275410
|
this.queue.shift();
|
|
275100
275411
|
this.stats.shed += 1;
|
|
@@ -275173,7 +275484,7 @@ class BrokerExecutionArchiver {
|
|
|
275173
275484
|
return true;
|
|
275174
275485
|
}
|
|
275175
275486
|
for (const entry of batch) {
|
|
275176
|
-
if (
|
|
275487
|
+
if (isBrokerExecutionArchiveTable(entry.table)) {
|
|
275177
275488
|
this.emitOtelLog(entry);
|
|
275178
275489
|
}
|
|
275179
275490
|
}
|
|
@@ -275309,8 +275620,138 @@ function parsePositiveInt(value, fallback) {
|
|
|
275309
275620
|
const parsed = Number.parseInt(value, 10);
|
|
275310
275621
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
275311
275622
|
}
|
|
275312
|
-
// src/helpers/
|
|
275623
|
+
// src/helpers/account-balance-archive-poller.ts
|
|
275313
275624
|
var DEFAULT_CONFIG = {
|
|
275625
|
+
pollIntervalMs: 60000
|
|
275626
|
+
};
|
|
275627
|
+
var metricLabels = (target) => ({
|
|
275628
|
+
exchange: target.exchangeId,
|
|
275629
|
+
account_selector: target.account.label,
|
|
275630
|
+
balance_scope: ACCOUNT_BALANCE_SCOPE
|
|
275631
|
+
});
|
|
275632
|
+
|
|
275633
|
+
class AccountBalanceArchivePoller {
|
|
275634
|
+
params;
|
|
275635
|
+
#timer = null;
|
|
275636
|
+
#stopped = false;
|
|
275637
|
+
#running = null;
|
|
275638
|
+
#lastSuccessMs = new Map;
|
|
275639
|
+
#config;
|
|
275640
|
+
constructor(params) {
|
|
275641
|
+
this.params = params;
|
|
275642
|
+
this.#config = { ...DEFAULT_CONFIG, ...params.config };
|
|
275643
|
+
}
|
|
275644
|
+
start() {
|
|
275645
|
+
if (this.#timer || this.#stopped || !this.params.archiver.canPersistAccountBalanceSnapshots()) {
|
|
275646
|
+
return;
|
|
275647
|
+
}
|
|
275648
|
+
log.info("\uD83D\uDCB0 Account balance archive poller started", {
|
|
275649
|
+
balanceScope: ACCOUNT_BALANCE_SCOPE
|
|
275650
|
+
});
|
|
275651
|
+
this.#schedule(0);
|
|
275652
|
+
}
|
|
275653
|
+
async stop() {
|
|
275654
|
+
this.#stopped = true;
|
|
275655
|
+
if (this.#timer) {
|
|
275656
|
+
clearTimeout(this.#timer);
|
|
275657
|
+
this.#timer = null;
|
|
275658
|
+
}
|
|
275659
|
+
await this.#running;
|
|
275660
|
+
}
|
|
275661
|
+
async pollAllOnce() {
|
|
275662
|
+
if (this.#stopped || this.#running || !this.params.archiver.canPersistAccountBalanceSnapshots()) {
|
|
275663
|
+
return false;
|
|
275664
|
+
}
|
|
275665
|
+
this.#running = this.#pollAllSequentially();
|
|
275666
|
+
try {
|
|
275667
|
+
return await this.#running;
|
|
275668
|
+
} finally {
|
|
275669
|
+
this.#running = null;
|
|
275670
|
+
}
|
|
275671
|
+
}
|
|
275672
|
+
#targets() {
|
|
275673
|
+
const targets = [];
|
|
275674
|
+
for (const [exchangeId, pool] of Object.entries(this.params.brokers)) {
|
|
275675
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
275676
|
+
targets.push({ exchangeId, account });
|
|
275677
|
+
}
|
|
275678
|
+
}
|
|
275679
|
+
return targets;
|
|
275680
|
+
}
|
|
275681
|
+
async#pollAllSequentially() {
|
|
275682
|
+
for (const target of this.#targets()) {
|
|
275683
|
+
if (this.#stopped) {
|
|
275684
|
+
break;
|
|
275685
|
+
}
|
|
275686
|
+
await this.#pollOne(target);
|
|
275687
|
+
}
|
|
275688
|
+
return true;
|
|
275689
|
+
}
|
|
275690
|
+
async#pollOne(target) {
|
|
275691
|
+
const labels = metricLabels(target);
|
|
275692
|
+
this.params.metrics?.recordCounter("cex_account_balance_poll_attempts_total", 1, labels);
|
|
275693
|
+
try {
|
|
275694
|
+
const exchange = target.account.exchange;
|
|
275695
|
+
const balance = await exchange.fetchBalance({
|
|
275696
|
+
type: ACCOUNT_BALANCE_SCOPE
|
|
275697
|
+
});
|
|
275698
|
+
const observedAt = new Date;
|
|
275699
|
+
const normalized = normalizeCcxtBalanceForArchive(balance);
|
|
275700
|
+
this.params.archiver.enqueue(buildAccountBalanceSnapshotRow({
|
|
275701
|
+
tags: buildCommonArchiveTags({
|
|
275702
|
+
deploymentId: this.params.archiver.getDeploymentId(),
|
|
275703
|
+
accountSelector: target.account.label,
|
|
275704
|
+
exchange: target.exchangeId,
|
|
275705
|
+
brokerObservedTimestamp: observedAt.toISOString()
|
|
275706
|
+
}),
|
|
275707
|
+
balance: normalized
|
|
275708
|
+
}));
|
|
275709
|
+
const successMs = Date.now();
|
|
275710
|
+
this.#lastSuccessMs.set(this.#targetKey(target), successMs);
|
|
275711
|
+
this.params.metrics?.recordCounter("cex_account_balance_poll_successes_total", 1, labels);
|
|
275712
|
+
this.params.metrics?.recordGauge("cex_account_balance_poll_last_success_timestamp_seconds", Math.floor(successMs / 1000), labels);
|
|
275713
|
+
this.#recordFreshness(labels, successMs, successMs);
|
|
275714
|
+
} catch (error) {
|
|
275715
|
+
this.params.metrics?.recordCounter("cex_account_balance_poll_failures_total", 1, labels);
|
|
275716
|
+
const now3 = Date.now();
|
|
275717
|
+
const lastSuccess = this.#lastSuccessMs.get(this.#targetKey(target));
|
|
275718
|
+
if (lastSuccess !== undefined) {
|
|
275719
|
+
this.#recordFreshness(labels, lastSuccess, now3);
|
|
275720
|
+
}
|
|
275721
|
+
log.warn("Account balance archive poll failed", {
|
|
275722
|
+
exchange: target.exchangeId,
|
|
275723
|
+
account: target.account.label,
|
|
275724
|
+
balanceScope: ACCOUNT_BALANCE_SCOPE,
|
|
275725
|
+
errorType: error instanceof Error ? error.name : "unknown"
|
|
275726
|
+
});
|
|
275727
|
+
}
|
|
275728
|
+
}
|
|
275729
|
+
#recordFreshness(labels, lastSuccessMs, nowMs) {
|
|
275730
|
+
this.params.metrics?.recordGauge("cex_account_balance_poll_freshness_seconds", Math.max(0, (nowMs - lastSuccessMs) / 1000), labels);
|
|
275731
|
+
}
|
|
275732
|
+
#targetKey(target) {
|
|
275733
|
+
return `${target.exchangeId}|${target.account.label}|${ACCOUNT_BALANCE_SCOPE}`;
|
|
275734
|
+
}
|
|
275735
|
+
#schedule(delayMs) {
|
|
275736
|
+
this.#timer = setTimeout(() => void this.#tick(), delayMs);
|
|
275737
|
+
this.#timer.unref?.();
|
|
275738
|
+
}
|
|
275739
|
+
async#tick() {
|
|
275740
|
+
this.#timer = null;
|
|
275741
|
+
try {
|
|
275742
|
+
await this.pollAllOnce();
|
|
275743
|
+
} catch (error) {
|
|
275744
|
+
log.error("Account balance archive poller tick failed", error);
|
|
275745
|
+
} finally {
|
|
275746
|
+
if (!this.#stopped) {
|
|
275747
|
+
this.#schedule(this.#config.pollIntervalMs);
|
|
275748
|
+
}
|
|
275749
|
+
}
|
|
275750
|
+
}
|
|
275751
|
+
}
|
|
275752
|
+
|
|
275753
|
+
// src/helpers/fill-archive-poller.ts
|
|
275754
|
+
var DEFAULT_CONFIG2 = {
|
|
275314
275755
|
pollIntervalMs: 60000,
|
|
275315
275756
|
lookbackMs: 24 * 60 * 60 * 1000,
|
|
275316
275757
|
tradesLimit: 500
|
|
@@ -275335,7 +275776,7 @@ class FillArchivePoller {
|
|
|
275335
275776
|
#config;
|
|
275336
275777
|
constructor(params) {
|
|
275337
275778
|
this.params = params;
|
|
275338
|
-
this.#config = { ...
|
|
275779
|
+
this.#config = { ...DEFAULT_CONFIG2, ...params.config };
|
|
275339
275780
|
}
|
|
275340
275781
|
start() {
|
|
275341
275782
|
if (this.#timer || this.#stopped) {
|
|
@@ -294640,6 +295081,13 @@ async function handleTreasuryCall(ctx) {
|
|
|
294640
295081
|
}, null);
|
|
294641
295082
|
}
|
|
294642
295083
|
const result = await fn.apply(broker, argsArray);
|
|
295084
|
+
if (callValue.functionName === "fetchWithdrawals") {
|
|
295085
|
+
archiveWithdrawalObservationsInBackground(ctx.brokerArchiver, ctx.withdrawalObservationTracker, {
|
|
295086
|
+
exchange: ctx.normalizedCex,
|
|
295087
|
+
accountSelector: ctx.selectedBrokerAccount?.label,
|
|
295088
|
+
transactions: result
|
|
295089
|
+
});
|
|
295090
|
+
}
|
|
294643
295091
|
ctx.wrappedCallback(null, {
|
|
294644
295092
|
proof: ctx.verity.proof,
|
|
294645
295093
|
result: JSON.stringify(result)
|
|
@@ -294818,6 +295266,7 @@ function createExecuteActionHandler(deps) {
|
|
|
294818
295266
|
brokerArchiver,
|
|
294819
295267
|
orderActivityTracker
|
|
294820
295268
|
} = deps;
|
|
295269
|
+
const withdrawalObservationTracker = deps.withdrawalObservationTracker ?? new WithdrawalObservationTracker;
|
|
294821
295270
|
return async (call, callback) => {
|
|
294822
295271
|
const startTime = Date.now();
|
|
294823
295272
|
const { action: rawAction, cex: cex3, symbol: symbol2 } = call.request;
|
|
@@ -294896,7 +295345,8 @@ function createExecuteActionHandler(deps) {
|
|
|
294896
295345
|
verityProverUrl,
|
|
294897
295346
|
otelMetrics,
|
|
294898
295347
|
brokerArchiver,
|
|
294899
|
-
orderActivityTracker
|
|
295348
|
+
orderActivityTracker,
|
|
295349
|
+
withdrawalObservationTracker
|
|
294900
295350
|
};
|
|
294901
295351
|
if (action === Action.Call) {
|
|
294902
295352
|
const handled = await handleOrderBookCall(preludeCtx);
|
|
@@ -294932,6 +295382,7 @@ import WebSocket2 from "ws";
|
|
|
294932
295382
|
var BINANCE_SPOT_WS_API_URL = "wss://ws-api.binance.com:443/ws-api/v3";
|
|
294933
295383
|
var DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS = 16;
|
|
294934
295384
|
var createWebSocket = (url3) => new WebSocket2(url3);
|
|
295385
|
+
var userDataRequestCounter = 0;
|
|
294935
295386
|
function getExchangeString(exchange, key) {
|
|
294936
295387
|
const value = exchange[key];
|
|
294937
295388
|
if (typeof value !== "string" || value.length === 0) {
|
|
@@ -295038,7 +295489,7 @@ class BinanceSpotUserDataStream {
|
|
|
295038
295489
|
exchange;
|
|
295039
295490
|
ws;
|
|
295040
295491
|
secretValues;
|
|
295041
|
-
requestId = `user-data-${Date.now()}-${
|
|
295492
|
+
requestId = `user-data-${Date.now()}-${userDataRequestCounter++}`;
|
|
295042
295493
|
maxBufferedEvents;
|
|
295043
295494
|
queue = [];
|
|
295044
295495
|
waiters = [];
|
|
@@ -295116,6 +295567,12 @@ class BinanceSpotUserDataStream {
|
|
|
295116
295567
|
this.subscriptionId = message.result?.subscriptionId ?? null;
|
|
295117
295568
|
return;
|
|
295118
295569
|
}
|
|
295570
|
+
if ("status" in message && typeof message.status === "number" && message.status !== 200) {
|
|
295571
|
+
const errorMessage = message.error?.msg ?? message.error?.message ?? `Binance user-data request failed with status ${message.status}`;
|
|
295572
|
+
const errorCode = message.error?.code;
|
|
295573
|
+
this.fail(new Error(typeof errorCode === "number" ? `${errorMessage} (code ${errorCode})` : errorMessage));
|
|
295574
|
+
return;
|
|
295575
|
+
}
|
|
295119
295576
|
if (!("event" in message) || !message.event) {
|
|
295120
295577
|
return;
|
|
295121
295578
|
}
|
|
@@ -296486,7 +296943,7 @@ var CEX_BROKER_PACKAGE_DEFINITION = protoLoader.fromJSON(node_descriptor_default
|
|
|
296486
296943
|
// src/server.ts
|
|
296487
296944
|
var grpcObj = grpc14.loadPackageDefinition(CEX_BROKER_PACKAGE_DEFINITION);
|
|
296488
296945
|
var cexNode = grpcObj.cex_broker;
|
|
296489
|
-
function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker) {
|
|
296946
|
+
function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker) {
|
|
296490
296947
|
const server = new grpc14.Server;
|
|
296491
296948
|
server.addService(cexNode.cex_service.service, {
|
|
296492
296949
|
ExecuteAction: createExecuteActionHandler({
|
|
@@ -296497,7 +296954,8 @@ function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, ot
|
|
|
296497
296954
|
verityProverUrl,
|
|
296498
296955
|
otelMetrics,
|
|
296499
296956
|
brokerArchiver,
|
|
296500
|
-
orderActivityTracker
|
|
296957
|
+
orderActivityTracker,
|
|
296958
|
+
withdrawalObservationTracker
|
|
296501
296959
|
}),
|
|
296502
296960
|
Subscribe: createSubscribeHandler({
|
|
296503
296961
|
brokers,
|
|
@@ -296643,7 +297101,9 @@ class CEXBroker {
|
|
|
296643
297101
|
brokerArchiver;
|
|
296644
297102
|
depositReconciler;
|
|
296645
297103
|
orderActivityTracker = new OrderActivityTracker;
|
|
297104
|
+
withdrawalObservationTracker = new WithdrawalObservationTracker;
|
|
296646
297105
|
fillArchivePoller;
|
|
297106
|
+
accountBalanceArchivePoller;
|
|
296647
297107
|
loadEnvConfig() {
|
|
296648
297108
|
log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
|
|
296649
297109
|
const configMap = {};
|
|
@@ -296788,6 +297248,10 @@ class CEXBroker {
|
|
|
296788
297248
|
this.fillArchivePoller.stop();
|
|
296789
297249
|
this.fillArchivePoller = undefined;
|
|
296790
297250
|
}
|
|
297251
|
+
if (this.accountBalanceArchivePoller) {
|
|
297252
|
+
await this.accountBalanceArchivePoller.stop();
|
|
297253
|
+
this.accountBalanceArchivePoller = undefined;
|
|
297254
|
+
}
|
|
296791
297255
|
if (this.server) {
|
|
296792
297256
|
await this.server.forceShutdown();
|
|
296793
297257
|
}
|
|
@@ -296813,11 +297277,15 @@ class CEXBroker {
|
|
|
296813
297277
|
this.fillArchivePoller.stop();
|
|
296814
297278
|
this.fillArchivePoller = undefined;
|
|
296815
297279
|
}
|
|
297280
|
+
if (this.accountBalanceArchivePoller) {
|
|
297281
|
+
await this.accountBalanceArchivePoller.stop();
|
|
297282
|
+
this.accountBalanceArchivePoller = undefined;
|
|
297283
|
+
}
|
|
296816
297284
|
log.info(`Running CEXBroker at ${new Date().toISOString()}`);
|
|
296817
297285
|
if (this.otelMetrics?.isOtelEnabled()) {
|
|
296818
297286
|
await this.otelMetrics.initialize();
|
|
296819
297287
|
}
|
|
296820
|
-
this.server = getServer(this.policy, this.brokers, this.whitelistIps, this.useVerity, this.#verityProverUrl, this.otelMetrics, this.brokerArchiver, this.orderActivityTracker);
|
|
297288
|
+
this.server = getServer(this.policy, this.brokers, this.whitelistIps, this.useVerity, this.#verityProverUrl, this.otelMetrics, this.brokerArchiver, this.orderActivityTracker, this.withdrawalObservationTracker);
|
|
296821
297289
|
this.server.bindAsync(`0.0.0.0:${this.port}`, grpc15.ServerCredentials.createInsecure(), (err2, port) => {
|
|
296822
297290
|
if (err2) {
|
|
296823
297291
|
log.error(err2);
|
|
@@ -296841,6 +297309,14 @@ class CEXBroker {
|
|
|
296841
297309
|
});
|
|
296842
297310
|
this.fillArchivePoller.start();
|
|
296843
297311
|
}
|
|
297312
|
+
if (this.brokerArchiver?.canPersistAccountBalanceSnapshots()) {
|
|
297313
|
+
this.accountBalanceArchivePoller = new AccountBalanceArchivePoller({
|
|
297314
|
+
brokers: this.brokers,
|
|
297315
|
+
archiver: this.brokerArchiver,
|
|
297316
|
+
metrics: this.otelMetrics
|
|
297317
|
+
});
|
|
297318
|
+
this.accountBalanceArchivePoller.start();
|
|
297319
|
+
}
|
|
296844
297320
|
return this;
|
|
296845
297321
|
}
|
|
296846
297322
|
}
|
|
@@ -296848,4 +297324,4 @@ export {
|
|
|
296848
297324
|
CEXBroker as default
|
|
296849
297325
|
};
|
|
296850
297326
|
|
|
296851
|
-
//# debugId=
|
|
297327
|
+
//# debugId=186957481DC5AB2064756E2164756E21
|