@usherlabs/cex-broker 0.2.41 → 0.2.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cli.js +666 -319
- package/dist/helpers/balance-update-archive-consumer.d.ts +16 -0
- package/dist/helpers/broker-execution-archive/index.d.ts +2 -2
- package/dist/helpers/broker-execution-archive/rows.d.ts +25 -0
- package/dist/helpers/broker-execution-archive/types.d.ts +5 -3
- package/dist/helpers/broker-execution-archive/writer.d.ts +1 -1
- package/dist/helpers/user-asset-archive-poller.d.ts +19 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +667 -320
- package/dist/index.js.map +11 -9
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -290525,6 +290525,20 @@ function validateDeposit(policy, exchange, network, ticker) {
|
|
|
290525
290525
|
return { valid: true };
|
|
290526
290526
|
}
|
|
290527
290527
|
|
|
290528
|
+
// src/helpers/broker-execution-archive/rows.ts
|
|
290529
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
290530
|
+
|
|
290531
|
+
// src/helpers/shared/guards.ts
|
|
290532
|
+
function isRecord(value) {
|
|
290533
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
290534
|
+
}
|
|
290535
|
+
function asRecord(value) {
|
|
290536
|
+
return isRecord(value) ? value : undefined;
|
|
290537
|
+
}
|
|
290538
|
+
|
|
290539
|
+
// src/helpers/broker-execution-archive/redact.ts
|
|
290540
|
+
import { createHash } from "node:crypto";
|
|
290541
|
+
|
|
290528
290542
|
// src/helpers/shared/errors.ts
|
|
290529
290543
|
var MAX_ERROR_DETAIL_LENGTH = 512;
|
|
290530
290544
|
var REDACTED_ERROR_MESSAGE = "redacted_error";
|
|
@@ -290584,226 +290598,7 @@ function safeLogRedactedError(context2, error) {
|
|
|
290584
290598
|
}
|
|
290585
290599
|
}
|
|
290586
290600
|
|
|
290587
|
-
// src/helpers/shared/guards.ts
|
|
290588
|
-
function isRecord(value) {
|
|
290589
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
290590
|
-
}
|
|
290591
|
-
function asRecord(value) {
|
|
290592
|
-
return isRecord(value) ? value : undefined;
|
|
290593
|
-
}
|
|
290594
|
-
|
|
290595
|
-
// src/helpers/order-telemetry.ts
|
|
290596
|
-
var NUMERIC_METRICS = [
|
|
290597
|
-
["requestedQuantity", "cex_market_action_requested_quantity"],
|
|
290598
|
-
["requestedNotional", "cex_market_action_requested_notional"],
|
|
290599
|
-
["executedBaseQuantity", "cex_market_action_executed_base_quantity"],
|
|
290600
|
-
["executedQuoteQuantity", "cex_market_action_executed_quote_quantity"],
|
|
290601
|
-
["averageExecutionPrice", "cex_market_action_average_execution_price"],
|
|
290602
|
-
["filledAmount", "cex_market_action_filled_amount"],
|
|
290603
|
-
["remainingAmount", "cex_market_action_remaining_amount"],
|
|
290604
|
-
["feeAmount", "cex_market_action_fee_amount"],
|
|
290605
|
-
["feeRate", "cex_market_action_fee_rate"]
|
|
290606
|
-
];
|
|
290607
|
-
async function emitOrderExecutionTelemetry(otelMetrics, context2, order, error) {
|
|
290608
|
-
try {
|
|
290609
|
-
const telemetry = buildOrderExecutionTelemetry(context2, order, error);
|
|
290610
|
-
log.info("CEX market action execution telemetry", telemetry);
|
|
290611
|
-
const labels = {
|
|
290612
|
-
action: telemetry.action,
|
|
290613
|
-
cex: telemetry.cex,
|
|
290614
|
-
account: telemetry.accountLabel,
|
|
290615
|
-
symbol: telemetry.symbol,
|
|
290616
|
-
side: telemetry.side,
|
|
290617
|
-
order_type: telemetry.orderType,
|
|
290618
|
-
status: telemetry.status
|
|
290619
|
-
};
|
|
290620
|
-
await otelMetrics?.recordCounter("cex_market_action_executions_total", 1, {
|
|
290621
|
-
...labels,
|
|
290622
|
-
result: error ? "error" : "ok"
|
|
290623
|
-
});
|
|
290624
|
-
for (const [key, metricName] of NUMERIC_METRICS) {
|
|
290625
|
-
const value = telemetry[key];
|
|
290626
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
290627
|
-
await otelMetrics?.recordHistogram(metricName, value, labels);
|
|
290628
|
-
}
|
|
290629
|
-
}
|
|
290630
|
-
return telemetry;
|
|
290631
|
-
} catch (telemetryError) {
|
|
290632
|
-
try {
|
|
290633
|
-
log.error("Failed to emit CEX order telemetry", {
|
|
290634
|
-
error: telemetryError
|
|
290635
|
-
});
|
|
290636
|
-
} catch {}
|
|
290637
|
-
return;
|
|
290638
|
-
}
|
|
290639
|
-
}
|
|
290640
|
-
function buildOrderExecutionTelemetry(context2, order, error) {
|
|
290641
|
-
const record = asRecord(order);
|
|
290642
|
-
const info = asRecord(record?.info);
|
|
290643
|
-
const fees = getFees(record, info);
|
|
290644
|
-
const fee = summarizeFees(fees);
|
|
290645
|
-
const executedBaseQuantity = firstNumber(record?.filled, info?.executedQty, info?.cumExecQty) ?? computeFilledFromAmount(record);
|
|
290646
|
-
const executedQuoteQuantity = firstNumber(record?.cost, info?.cummulativeQuoteQty, info?.cumQuote, info?.cumExecValue);
|
|
290647
|
-
const averageExecutionPrice = firstNumber(record?.average, info?.avgPrice) ?? computeAveragePrice(executedBaseQuantity, executedQuoteQuantity);
|
|
290648
|
-
const exchangeTimestamp = normalizeTimestamp(firstValue(record?.timestamp, info?.time, info?.transactTime, record?.datetime));
|
|
290649
|
-
const status = firstString(record?.status, info?.status) ?? (error ? "failed" : "unknown");
|
|
290650
|
-
const errorRecord = error instanceof Error ? error : undefined;
|
|
290651
|
-
return compactUndefined({
|
|
290652
|
-
event: "cex_market_action_execution",
|
|
290653
|
-
action: context2.action,
|
|
290654
|
-
cex: context2.cex.trim().toLowerCase() || "unknown",
|
|
290655
|
-
accountLabel: context2.accountLabel ?? "unknown",
|
|
290656
|
-
symbol: firstString(record?.symbol, info?.symbol, context2.symbol) ?? "unknown",
|
|
290657
|
-
side: firstString(record?.side, info?.side, context2.side) ?? "unknown",
|
|
290658
|
-
orderType: firstString(record?.type, info?.type, context2.orderType) ?? "unknown",
|
|
290659
|
-
orderId: firstString(record?.id, info?.orderId, info?.orderID),
|
|
290660
|
-
orderAuthor: context2.orderAuthor,
|
|
290661
|
-
clientOrderId: firstString(context2.clientOrderId, record?.clientOrderId, record?.clientOrderID, record?.clientOid, info?.clientOrderId, info?.clientOrderID, info?.clientOid),
|
|
290662
|
-
idempotencyId: context2.idempotencyId,
|
|
290663
|
-
makerActionId: context2.makerActionId,
|
|
290664
|
-
status: status.toLowerCase(),
|
|
290665
|
-
requestedQuantity: context2.requestedQuantity ?? firstNumber(record?.amount),
|
|
290666
|
-
requestedNotional: context2.requestedNotional,
|
|
290667
|
-
executedBaseQuantity,
|
|
290668
|
-
executedQuoteQuantity,
|
|
290669
|
-
averageExecutionPrice,
|
|
290670
|
-
filledAmount: firstNumber(record?.filled, info?.executedQty),
|
|
290671
|
-
remainingAmount: firstNumber(record?.remaining, info?.remainingQty),
|
|
290672
|
-
feeAmount: fee.amount,
|
|
290673
|
-
feeCurrency: fee.currency,
|
|
290674
|
-
feeRate: fee.rate,
|
|
290675
|
-
exchangeTimestamp,
|
|
290676
|
-
brokerObservedTimestamp: context2.brokerObservedTimestamp ?? new Date().toISOString(),
|
|
290677
|
-
errorType: errorRecord?.name,
|
|
290678
|
-
errorMessage: errorRecord ? REDACTED_ERROR_MESSAGE : undefined
|
|
290679
|
-
});
|
|
290680
|
-
}
|
|
290681
|
-
function emitOrderExecutionTelemetryInBackground(otelMetrics, context2, order, error) {
|
|
290682
|
-
emitOrderExecutionTelemetry(otelMetrics, context2, order, error).catch((telemetryError) => {
|
|
290683
|
-
try {
|
|
290684
|
-
log.warn("Telemetry emit failed", { error: telemetryError });
|
|
290685
|
-
} catch {
|
|
290686
|
-
console.warn("Telemetry emit failed", telemetryError);
|
|
290687
|
-
}
|
|
290688
|
-
});
|
|
290689
|
-
}
|
|
290690
|
-
function extractOrderTelemetryIds(params) {
|
|
290691
|
-
const record = params ?? {};
|
|
290692
|
-
return {
|
|
290693
|
-
clientOrderId: firstString(record.clientOrderId, record.clientOrderID, record.newClientOrderId, record.clientOid),
|
|
290694
|
-
idempotencyId: firstString(record.idempotencyId, record.idempotencyID, record.idempotencyKey, record.requestId, record.requestID),
|
|
290695
|
-
makerActionId: firstString(record.makerActionId, record.maker_action_id, record.actionId, record.action_id)
|
|
290696
|
-
};
|
|
290697
|
-
}
|
|
290698
|
-
function firstValue(...values2) {
|
|
290699
|
-
return values2.find((value) => value !== undefined && value !== null);
|
|
290700
|
-
}
|
|
290701
|
-
function firstString(...values2) {
|
|
290702
|
-
for (const value of values2) {
|
|
290703
|
-
if (typeof value === "string" && value.trim()) {
|
|
290704
|
-
return value.trim();
|
|
290705
|
-
}
|
|
290706
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
290707
|
-
return String(value);
|
|
290708
|
-
}
|
|
290709
|
-
}
|
|
290710
|
-
return;
|
|
290711
|
-
}
|
|
290712
|
-
function firstNumber(...values2) {
|
|
290713
|
-
for (const value of values2) {
|
|
290714
|
-
const numberValue = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
|
290715
|
-
if (Number.isFinite(numberValue)) {
|
|
290716
|
-
return numberValue;
|
|
290717
|
-
}
|
|
290718
|
-
}
|
|
290719
|
-
return;
|
|
290720
|
-
}
|
|
290721
|
-
function computeFilledFromAmount(record) {
|
|
290722
|
-
const amount = firstNumber(record?.amount);
|
|
290723
|
-
const remaining = firstNumber(record?.remaining);
|
|
290724
|
-
if (amount === undefined || remaining === undefined) {
|
|
290725
|
-
return;
|
|
290726
|
-
}
|
|
290727
|
-
return amount - remaining;
|
|
290728
|
-
}
|
|
290729
|
-
function computeAveragePrice(executedBaseQuantity, executedQuoteQuantity) {
|
|
290730
|
-
if (executedBaseQuantity === undefined || executedQuoteQuantity === undefined || executedBaseQuantity === 0) {
|
|
290731
|
-
return;
|
|
290732
|
-
}
|
|
290733
|
-
return executedQuoteQuantity / executedBaseQuantity;
|
|
290734
|
-
}
|
|
290735
|
-
function normalizeTimestamp(value) {
|
|
290736
|
-
if (typeof value === "string" && value.trim()) {
|
|
290737
|
-
return value.trim();
|
|
290738
|
-
}
|
|
290739
|
-
const timestamp = firstNumber(value);
|
|
290740
|
-
if (timestamp === undefined) {
|
|
290741
|
-
return;
|
|
290742
|
-
}
|
|
290743
|
-
const date = new Date(timestamp);
|
|
290744
|
-
return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
|
|
290745
|
-
}
|
|
290746
|
-
function getFees(record, info) {
|
|
290747
|
-
const fees = [];
|
|
290748
|
-
if (record?.fee)
|
|
290749
|
-
fees.push(record.fee);
|
|
290750
|
-
if (Array.isArray(record?.fees))
|
|
290751
|
-
fees.push(...record.fees);
|
|
290752
|
-
if (Array.isArray(record?.trades)) {
|
|
290753
|
-
for (const trade of record.trades) {
|
|
290754
|
-
const tradeRecord = asRecord(trade);
|
|
290755
|
-
if (tradeRecord?.fee)
|
|
290756
|
-
fees.push(tradeRecord.fee);
|
|
290757
|
-
if (Array.isArray(tradeRecord?.fees))
|
|
290758
|
-
fees.push(...tradeRecord.fees);
|
|
290759
|
-
}
|
|
290760
|
-
}
|
|
290761
|
-
if (Array.isArray(info?.fills)) {
|
|
290762
|
-
for (const fill of info.fills) {
|
|
290763
|
-
const fillRecord = asRecord(fill);
|
|
290764
|
-
const commission = firstNumber(fillRecord?.commission);
|
|
290765
|
-
if (commission !== undefined) {
|
|
290766
|
-
fees.push({
|
|
290767
|
-
cost: commission,
|
|
290768
|
-
currency: firstString(fillRecord?.commissionAsset)
|
|
290769
|
-
});
|
|
290770
|
-
}
|
|
290771
|
-
}
|
|
290772
|
-
}
|
|
290773
|
-
return fees;
|
|
290774
|
-
}
|
|
290775
|
-
function summarizeFees(fees) {
|
|
290776
|
-
let amount = 0;
|
|
290777
|
-
let amountFound = false;
|
|
290778
|
-
let currency;
|
|
290779
|
-
let rate;
|
|
290780
|
-
for (const rawFee of fees) {
|
|
290781
|
-
const fee = asRecord(rawFee);
|
|
290782
|
-
if (!fee)
|
|
290783
|
-
continue;
|
|
290784
|
-
const cost = firstNumber(fee.cost, fee.amount);
|
|
290785
|
-
if (cost !== undefined) {
|
|
290786
|
-
amount += cost;
|
|
290787
|
-
amountFound = true;
|
|
290788
|
-
}
|
|
290789
|
-
currency ??= firstString(fee.currency);
|
|
290790
|
-
rate ??= firstNumber(fee.rate);
|
|
290791
|
-
}
|
|
290792
|
-
return {
|
|
290793
|
-
amount: amountFound ? amount : undefined,
|
|
290794
|
-
currency,
|
|
290795
|
-
rate
|
|
290796
|
-
};
|
|
290797
|
-
}
|
|
290798
|
-
function compactUndefined(record) {
|
|
290799
|
-
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
290800
|
-
}
|
|
290801
|
-
|
|
290802
|
-
// src/helpers/broker-execution-archive/rows.ts
|
|
290803
|
-
import { createHash as createHash2 } from "node:crypto";
|
|
290804
|
-
|
|
290805
290601
|
// src/helpers/broker-execution-archive/redact.ts
|
|
290806
|
-
import { createHash } from "node:crypto";
|
|
290807
290602
|
var SECRET_KEY_PATTERN = /\b(api[_-]?key|api[_-]?secret|secret|signature|passphrase|password|token|credential)\b/i;
|
|
290808
290603
|
var SECRET_VALUE_PATTERN = /(\b(?:apiKey|api_key|apiSecret|api_secret|secret|signature|passphrase|password|token)\b\s*[=:]\s*)[^\s&,;)}\]]+/gi;
|
|
290809
290604
|
var SECRET_JSON_PATTERN = /("(?:apiKey|api_key|apiSecret|api_secret|secret|signature|passphrase|password|token)"\s*:\s*")[^"]*(")/gi;
|
|
@@ -290876,9 +290671,11 @@ var ARCHIVE_SCHEMA_VERSION = "1";
|
|
|
290876
290671
|
var FILL_EVENT_KIND = "trade_history_fill";
|
|
290877
290672
|
var ACCOUNT_BALANCE_SCOPE = "spot";
|
|
290878
290673
|
var ACCOUNT_BALANCE_PRECISION_BASIS = "ccxt_normalized_number";
|
|
290674
|
+
var USER_ASSET_BALANCE_SCOPE = "user_asset";
|
|
290675
|
+
var USER_ASSET_PRECISION_BASIS = "venue_raw_string";
|
|
290879
290676
|
|
|
290880
290677
|
// src/helpers/broker-execution-archive/rows.ts
|
|
290881
|
-
function
|
|
290678
|
+
function firstString(...values2) {
|
|
290882
290679
|
for (const value of values2) {
|
|
290883
290680
|
if (typeof value === "string" && value.trim()) {
|
|
290884
290681
|
return value.trim();
|
|
@@ -290889,7 +290686,7 @@ function firstString2(...values2) {
|
|
|
290889
290686
|
}
|
|
290890
290687
|
return;
|
|
290891
290688
|
}
|
|
290892
|
-
function
|
|
290689
|
+
function firstNumber(...values2) {
|
|
290893
290690
|
for (const value of values2) {
|
|
290894
290691
|
const numeric = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
|
290895
290692
|
if (Number.isFinite(numeric)) {
|
|
@@ -290898,11 +290695,11 @@ function firstNumber2(...values2) {
|
|
|
290898
290695
|
}
|
|
290899
290696
|
return;
|
|
290900
290697
|
}
|
|
290901
|
-
function
|
|
290698
|
+
function normalizeTimestamp(value) {
|
|
290902
290699
|
if (typeof value === "string" && value.trim()) {
|
|
290903
290700
|
return value.trim();
|
|
290904
290701
|
}
|
|
290905
|
-
const ms =
|
|
290702
|
+
const ms = firstNumber(value);
|
|
290906
290703
|
if (ms === undefined) {
|
|
290907
290704
|
return;
|
|
290908
290705
|
}
|
|
@@ -291009,7 +290806,7 @@ function normalizeCcxtBalanceForArchive(balance) {
|
|
|
291009
290806
|
}
|
|
291010
290807
|
}
|
|
291011
290808
|
return {
|
|
291012
|
-
exchangeTimestamp:
|
|
290809
|
+
exchangeTimestamp: normalizeTimestamp(record.timestamp ?? record.datetime),
|
|
291013
290810
|
reportedAssets: [...reportedAssets].sort(),
|
|
291014
290811
|
assetEntryAssets: [...assetEntryAssets].sort(),
|
|
291015
290812
|
freeBalances: sortedBalanceMap(maps.free),
|
|
@@ -291032,7 +290829,7 @@ function buildAccountBalanceSnapshotRow(input) {
|
|
|
291032
290829
|
})).digest("hex");
|
|
291033
290830
|
return {
|
|
291034
290831
|
table: "broker_account.balance_snapshots",
|
|
291035
|
-
row:
|
|
290832
|
+
row: compactUndefined({
|
|
291036
290833
|
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
291037
290834
|
exchange_timestamp: balance.exchangeTimestamp,
|
|
291038
290835
|
source: tags.source,
|
|
@@ -291054,7 +290851,101 @@ function buildAccountBalanceSnapshotRow(input) {
|
|
|
291054
290851
|
})
|
|
291055
290852
|
};
|
|
291056
290853
|
}
|
|
291057
|
-
|
|
290854
|
+
var USER_ASSET_BUCKETS = {
|
|
290855
|
+
freeBalances: "free",
|
|
290856
|
+
lockedBalances: "locked",
|
|
290857
|
+
freezeBalances: "freeze",
|
|
290858
|
+
withdrawingBalances: "withdrawing"
|
|
290859
|
+
};
|
|
290860
|
+
function userAssetQuantity(value) {
|
|
290861
|
+
if (typeof value === "number") {
|
|
290862
|
+
return decimalString(value);
|
|
290863
|
+
}
|
|
290864
|
+
if (typeof value !== "string") {
|
|
290865
|
+
return;
|
|
290866
|
+
}
|
|
290867
|
+
const trimmed = value.trim();
|
|
290868
|
+
if (!trimmed || !Number.isFinite(Number(trimmed))) {
|
|
290869
|
+
return;
|
|
290870
|
+
}
|
|
290871
|
+
return trimmed;
|
|
290872
|
+
}
|
|
290873
|
+
function normalizeBinanceUserAssetsForArchive(response) {
|
|
290874
|
+
if (!Array.isArray(response)) {
|
|
290875
|
+
throw new Error("binance_user_asset_malformed_response: getUserAsset did not return an array");
|
|
290876
|
+
}
|
|
290877
|
+
const reportedAssets = [];
|
|
290878
|
+
const incompleteAssets = [];
|
|
290879
|
+
const maps = {
|
|
290880
|
+
freeBalances: {},
|
|
290881
|
+
lockedBalances: {},
|
|
290882
|
+
freezeBalances: {},
|
|
290883
|
+
withdrawingBalances: {}
|
|
290884
|
+
};
|
|
290885
|
+
for (const entry of response) {
|
|
290886
|
+
const record = asRecord(entry);
|
|
290887
|
+
const asset = typeof record?.asset === "string" ? record.asset.trim() : undefined;
|
|
290888
|
+
if (!record || !asset) {
|
|
290889
|
+
throw new Error("binance_user_asset_malformed_response: getUserAsset entry has no asset key");
|
|
290890
|
+
}
|
|
290891
|
+
if (reportedAssets.includes(asset)) {
|
|
290892
|
+
throw new Error(`binance_user_asset_malformed_response: getUserAsset returned duplicate entries for ${asset}`);
|
|
290893
|
+
}
|
|
290894
|
+
reportedAssets.push(asset);
|
|
290895
|
+
let complete = true;
|
|
290896
|
+
for (const [field, venueKey] of Object.entries(USER_ASSET_BUCKETS)) {
|
|
290897
|
+
const quantity = userAssetQuantity(record[venueKey]);
|
|
290898
|
+
if (quantity === undefined) {
|
|
290899
|
+
complete = false;
|
|
290900
|
+
continue;
|
|
290901
|
+
}
|
|
290902
|
+
maps[field][asset] = quantity;
|
|
290903
|
+
}
|
|
290904
|
+
if (!complete) {
|
|
290905
|
+
incompleteAssets.push(asset);
|
|
290906
|
+
}
|
|
290907
|
+
}
|
|
290908
|
+
return {
|
|
290909
|
+
reportedAssets: reportedAssets.sort(),
|
|
290910
|
+
incompleteAssets: incompleteAssets.sort(),
|
|
290911
|
+
freeBalances: sortedBalanceMap(maps.freeBalances),
|
|
290912
|
+
lockedBalances: sortedBalanceMap(maps.lockedBalances),
|
|
290913
|
+
freezeBalances: sortedBalanceMap(maps.freezeBalances),
|
|
290914
|
+
withdrawingBalances: sortedBalanceMap(maps.withdrawingBalances)
|
|
290915
|
+
};
|
|
290916
|
+
}
|
|
290917
|
+
function buildUserAssetSnapshotRow(input) {
|
|
290918
|
+
const { tags, userAssets } = input;
|
|
290919
|
+
const observationId = createHash2("sha256").update(JSON.stringify({
|
|
290920
|
+
deployment_id: tags.deployment_id,
|
|
290921
|
+
exchange: tags.exchange,
|
|
290922
|
+
account_selector: tags.account_selector,
|
|
290923
|
+
balance_scope: USER_ASSET_BALANCE_SCOPE,
|
|
290924
|
+
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
290925
|
+
userAssets
|
|
290926
|
+
})).digest("hex");
|
|
290927
|
+
return {
|
|
290928
|
+
table: "broker_account.user_asset_snapshots",
|
|
290929
|
+
row: {
|
|
290930
|
+
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
290931
|
+
source: tags.source,
|
|
290932
|
+
deployment_id: tags.deployment_id,
|
|
290933
|
+
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
290934
|
+
exchange: tags.exchange,
|
|
290935
|
+
account_selector: tags.account_selector,
|
|
290936
|
+
balance_scope: USER_ASSET_BALANCE_SCOPE,
|
|
290937
|
+
observation_id: observationId,
|
|
290938
|
+
reported_assets: userAssets.reportedAssets,
|
|
290939
|
+
incomplete_assets: userAssets.incompleteAssets,
|
|
290940
|
+
free_balances: userAssets.freeBalances,
|
|
290941
|
+
locked_balances: userAssets.lockedBalances,
|
|
290942
|
+
freeze_balances: userAssets.freezeBalances,
|
|
290943
|
+
withdrawing_balances: userAssets.withdrawingBalances,
|
|
290944
|
+
precision_basis: USER_ASSET_PRECISION_BASIS
|
|
290945
|
+
}
|
|
290946
|
+
};
|
|
290947
|
+
}
|
|
290948
|
+
function compactUndefined(record) {
|
|
291058
290949
|
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
291059
290950
|
}
|
|
291060
290951
|
function buildCommonArchiveTags(input) {
|
|
@@ -291071,7 +290962,7 @@ function buildOrderEventArchiveRow(input) {
|
|
|
291071
290962
|
const { tags, telemetry, action } = input;
|
|
291072
290963
|
return {
|
|
291073
290964
|
table: "broker_execution.order_events",
|
|
291074
|
-
row:
|
|
290965
|
+
row: compactUndefined({
|
|
291075
290966
|
...tags,
|
|
291076
290967
|
event_kind: input.eventKind ?? "execute_action",
|
|
291077
290968
|
action,
|
|
@@ -291108,35 +290999,35 @@ function buildSubscribeStreamArchiveRow(input) {
|
|
|
291108
290999
|
const info = asRecord(record?.info);
|
|
291109
291000
|
return {
|
|
291110
291001
|
table: "broker_execution.order_events",
|
|
291111
|
-
row:
|
|
291002
|
+
row: compactUndefined({
|
|
291112
291003
|
...input.tags,
|
|
291113
291004
|
event_kind: "subscribe_stream",
|
|
291114
291005
|
subscription_type: input.subscriptionType,
|
|
291115
|
-
order_id:
|
|
291116
|
-
client_order_id:
|
|
291117
|
-
status:
|
|
291006
|
+
order_id: firstString(record?.id, record?.orderId, record?.i, info?.orderId, info?.i),
|
|
291007
|
+
client_order_id: firstString(record?.clientOrderId, record?.clientOrderID, record?.c, info?.clientOrderId, info?.c),
|
|
291008
|
+
status: firstString(record?.status, record?.X, info?.status, info?.X)?.toLowerCase(),
|
|
291118
291009
|
payload_json: JSON.stringify(redactedPayload)
|
|
291119
291010
|
})
|
|
291120
291011
|
};
|
|
291121
291012
|
}
|
|
291122
291013
|
function quantityString(...values2) {
|
|
291123
|
-
return
|
|
291014
|
+
return firstString(...values2);
|
|
291124
291015
|
}
|
|
291125
291016
|
function extractBinanceInternalTransferId(response) {
|
|
291126
291017
|
const record = asRecord(response);
|
|
291127
|
-
return
|
|
291018
|
+
return firstString(record?.txnId, record?.tranId);
|
|
291128
291019
|
}
|
|
291129
291020
|
function buildTransferEventArchiveRow(input) {
|
|
291130
291021
|
const { tags, transfer } = input;
|
|
291131
291022
|
return {
|
|
291132
291023
|
table: "broker_execution.transfer_events",
|
|
291133
|
-
row:
|
|
291024
|
+
row: compactUndefined({
|
|
291134
291025
|
...tags,
|
|
291135
291026
|
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
291136
291027
|
event_kind: transfer.eventKind,
|
|
291137
291028
|
lifecycle_action: transfer.lifecycleAction,
|
|
291138
291029
|
status: transfer.status ?? "",
|
|
291139
|
-
asset_symbol: tags.symbol,
|
|
291030
|
+
asset_symbol: transfer.assetSymbol ?? tags.symbol,
|
|
291140
291031
|
amount: transfer.amount,
|
|
291141
291032
|
address: transfer.address,
|
|
291142
291033
|
network: transfer.network,
|
|
@@ -291156,18 +291047,18 @@ function normalizeCcxtTransactionForArchive(transaction) {
|
|
|
291156
291047
|
const record = asRecord(transaction);
|
|
291157
291048
|
const info = asRecord(record?.info);
|
|
291158
291049
|
const fee = asRecord(record?.fee);
|
|
291159
|
-
return
|
|
291160
|
-
externalId:
|
|
291161
|
-
clientWithdrawalId:
|
|
291162
|
-
txid:
|
|
291163
|
-
address:
|
|
291164
|
-
network:
|
|
291050
|
+
return compactUndefined({
|
|
291051
|
+
externalId: firstString(record?.id, info?.id, record?.txid, info?.txId),
|
|
291052
|
+
clientWithdrawalId: firstString(info?.withdrawOrderId),
|
|
291053
|
+
txid: firstString(record?.txid, info?.txId, info?.txid, info?.tx_hash),
|
|
291054
|
+
address: firstString(record?.address, record?.addressTo, info?.address),
|
|
291055
|
+
network: firstString(record?.network, info?.network),
|
|
291165
291056
|
amount: quantityString(info?.amount, record?.amount),
|
|
291166
|
-
assetSymbol:
|
|
291167
|
-
status:
|
|
291057
|
+
assetSymbol: firstString(record?.currency, info?.coin, info?.asset),
|
|
291058
|
+
status: firstString(record?.status, info?.status)?.toLowerCase(),
|
|
291168
291059
|
feeAmount: quantityString(fee?.cost, record?.feeCost),
|
|
291169
|
-
feeCurrency:
|
|
291170
|
-
exchangeTimestamp:
|
|
291060
|
+
feeCurrency: firstString(fee?.currency, record?.feeCurrency),
|
|
291061
|
+
exchangeTimestamp: normalizeTimestamp(firstValueForTransfer(record?.timestamp, record?.datetime, info?.applyTime))
|
|
291171
291062
|
});
|
|
291172
291063
|
}
|
|
291173
291064
|
function firstValueForTransfer(...values2) {
|
|
@@ -291177,7 +291068,7 @@ function buildFillEventArchiveRow(input) {
|
|
|
291177
291068
|
const { tags, fill } = input;
|
|
291178
291069
|
return {
|
|
291179
291070
|
table: "broker_execution.fill_events",
|
|
291180
|
-
row:
|
|
291071
|
+
row: compactUndefined({
|
|
291181
291072
|
...tags,
|
|
291182
291073
|
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
291183
291074
|
event_kind: FILL_EVENT_KIND,
|
|
@@ -291198,45 +291089,242 @@ function buildFillEventArchiveRow(input) {
|
|
|
291198
291089
|
})
|
|
291199
291090
|
};
|
|
291200
291091
|
}
|
|
291201
|
-
function normalizeCcxtTradeForArchive(trade) {
|
|
291202
|
-
const record = asRecord(trade);
|
|
291203
|
-
const info = asRecord(record?.info);
|
|
291204
|
-
const fee = asRecord(record?.fee);
|
|
291205
|
-
return {
|
|
291206
|
-
orderId:
|
|
291207
|
-
clientOrderId:
|
|
291208
|
-
fillId:
|
|
291209
|
-
side:
|
|
291210
|
-
orderType:
|
|
291211
|
-
price: quantityString(info?.price, record?.price),
|
|
291212
|
-
baseQuantity: quantityString(info?.qty, record?.amount),
|
|
291213
|
-
quoteQuantity: quantityString(info?.quoteQty, record?.cost),
|
|
291214
|
-
feeAmount: quantityString(fee?.cost, info?.commission),
|
|
291215
|
-
feeCurrency:
|
|
291216
|
-
feeRate: quantityString(fee?.rate),
|
|
291217
|
-
exchangeTimestamp:
|
|
291218
|
-
payload: trade
|
|
291219
|
-
};
|
|
291092
|
+
function normalizeCcxtTradeForArchive(trade) {
|
|
291093
|
+
const record = asRecord(trade);
|
|
291094
|
+
const info = asRecord(record?.info);
|
|
291095
|
+
const fee = asRecord(record?.fee);
|
|
291096
|
+
return {
|
|
291097
|
+
orderId: firstString(record?.order, info?.orderId, info?.orderID),
|
|
291098
|
+
clientOrderId: firstString(record?.clientOrderId, info?.clientOrderId, info?.origClientOrderId),
|
|
291099
|
+
fillId: firstString(record?.id, info?.id, info?.tradeId),
|
|
291100
|
+
side: firstString(record?.side, info?.side)?.toLowerCase(),
|
|
291101
|
+
orderType: firstString(record?.type, info?.type)?.toLowerCase(),
|
|
291102
|
+
price: quantityString(info?.price, record?.price),
|
|
291103
|
+
baseQuantity: quantityString(info?.qty, record?.amount),
|
|
291104
|
+
quoteQuantity: quantityString(info?.quoteQty, record?.cost),
|
|
291105
|
+
feeAmount: quantityString(fee?.cost, info?.commission),
|
|
291106
|
+
feeCurrency: firstString(fee?.currency, info?.commissionAsset),
|
|
291107
|
+
feeRate: quantityString(fee?.rate),
|
|
291108
|
+
exchangeTimestamp: normalizeTimestamp(firstValueForTransfer(record?.timestamp, record?.datetime, info?.time)),
|
|
291109
|
+
payload: trade
|
|
291110
|
+
};
|
|
291111
|
+
}
|
|
291112
|
+
function buildMarketMetadataSnapshotRow(input) {
|
|
291113
|
+
const redactedSnapshot = redactStreamPayload(input.marketSnapshot);
|
|
291114
|
+
const metadataHash = hashMarketMetadata(redactedSnapshot);
|
|
291115
|
+
return {
|
|
291116
|
+
table: "broker_execution.market_metadata_snapshots",
|
|
291117
|
+
row: compactUndefined({
|
|
291118
|
+
...input.tags,
|
|
291119
|
+
client_order_id: input.clientOrderId,
|
|
291120
|
+
order_id: input.orderId,
|
|
291121
|
+
maker_action_id: input.makerActionId,
|
|
291122
|
+
idempotency_id: input.idempotencyId,
|
|
291123
|
+
market_metadata_hash: metadataHash,
|
|
291124
|
+
snapshot_json: JSON.stringify(redactedSnapshot)
|
|
291125
|
+
})
|
|
291126
|
+
};
|
|
291127
|
+
}
|
|
291128
|
+
|
|
291129
|
+
// src/helpers/order-telemetry.ts
|
|
291130
|
+
var NUMERIC_METRICS = [
|
|
291131
|
+
["requestedQuantity", "cex_market_action_requested_quantity"],
|
|
291132
|
+
["requestedNotional", "cex_market_action_requested_notional"],
|
|
291133
|
+
["executedBaseQuantity", "cex_market_action_executed_base_quantity"],
|
|
291134
|
+
["executedQuoteQuantity", "cex_market_action_executed_quote_quantity"],
|
|
291135
|
+
["averageExecutionPrice", "cex_market_action_average_execution_price"],
|
|
291136
|
+
["filledAmount", "cex_market_action_filled_amount"],
|
|
291137
|
+
["remainingAmount", "cex_market_action_remaining_amount"],
|
|
291138
|
+
["feeAmount", "cex_market_action_fee_amount"],
|
|
291139
|
+
["feeRate", "cex_market_action_fee_rate"]
|
|
291140
|
+
];
|
|
291141
|
+
async function emitOrderExecutionTelemetry(otelMetrics, context2, order, error) {
|
|
291142
|
+
try {
|
|
291143
|
+
const telemetry = buildOrderExecutionTelemetry(context2, order, error);
|
|
291144
|
+
log.info("CEX market action execution telemetry", telemetry);
|
|
291145
|
+
const labels = {
|
|
291146
|
+
action: telemetry.action,
|
|
291147
|
+
cex: telemetry.cex,
|
|
291148
|
+
account: telemetry.accountLabel,
|
|
291149
|
+
symbol: telemetry.symbol,
|
|
291150
|
+
side: telemetry.side,
|
|
291151
|
+
order_type: telemetry.orderType,
|
|
291152
|
+
status: telemetry.status
|
|
291153
|
+
};
|
|
291154
|
+
await otelMetrics?.recordCounter("cex_market_action_executions_total", 1, {
|
|
291155
|
+
...labels,
|
|
291156
|
+
result: error ? "error" : "ok"
|
|
291157
|
+
});
|
|
291158
|
+
for (const [key, metricName] of NUMERIC_METRICS) {
|
|
291159
|
+
const value = telemetry[key];
|
|
291160
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
291161
|
+
await otelMetrics?.recordHistogram(metricName, value, labels);
|
|
291162
|
+
}
|
|
291163
|
+
}
|
|
291164
|
+
return telemetry;
|
|
291165
|
+
} catch (telemetryError) {
|
|
291166
|
+
try {
|
|
291167
|
+
log.error("Failed to emit CEX order telemetry", {
|
|
291168
|
+
error: telemetryError
|
|
291169
|
+
});
|
|
291170
|
+
} catch {}
|
|
291171
|
+
return;
|
|
291172
|
+
}
|
|
291173
|
+
}
|
|
291174
|
+
function buildOrderExecutionTelemetry(context2, order, error) {
|
|
291175
|
+
const record = asRecord(order);
|
|
291176
|
+
const info = asRecord(record?.info);
|
|
291177
|
+
const fees = getFees(record, info);
|
|
291178
|
+
const fee = summarizeFees(fees);
|
|
291179
|
+
const executedBaseQuantity = firstNumber2(record?.filled, info?.executedQty, info?.cumExecQty) ?? computeFilledFromAmount(record);
|
|
291180
|
+
const executedQuoteQuantity = firstNumber2(record?.cost, info?.cummulativeQuoteQty, info?.cumQuote, info?.cumExecValue);
|
|
291181
|
+
const averageExecutionPrice = firstNumber2(record?.average, info?.avgPrice) ?? computeAveragePrice(executedBaseQuantity, executedQuoteQuantity);
|
|
291182
|
+
const exchangeTimestamp = normalizeTimestamp(firstValue(record?.timestamp, info?.time, info?.transactTime, record?.datetime));
|
|
291183
|
+
const status = firstString2(record?.status, info?.status) ?? (error ? "failed" : "unknown");
|
|
291184
|
+
const errorRecord = error instanceof Error ? error : undefined;
|
|
291185
|
+
return compactUndefined2({
|
|
291186
|
+
event: "cex_market_action_execution",
|
|
291187
|
+
action: context2.action,
|
|
291188
|
+
cex: context2.cex.trim().toLowerCase() || "unknown",
|
|
291189
|
+
accountLabel: context2.accountLabel ?? "unknown",
|
|
291190
|
+
symbol: firstString2(record?.symbol, info?.symbol, context2.symbol) ?? "unknown",
|
|
291191
|
+
side: firstString2(record?.side, info?.side, context2.side) ?? "unknown",
|
|
291192
|
+
orderType: firstString2(record?.type, info?.type, context2.orderType) ?? "unknown",
|
|
291193
|
+
orderId: firstString2(record?.id, info?.orderId, info?.orderID),
|
|
291194
|
+
orderAuthor: context2.orderAuthor,
|
|
291195
|
+
clientOrderId: firstString2(context2.clientOrderId, record?.clientOrderId, record?.clientOrderID, record?.clientOid, info?.clientOrderId, info?.clientOrderID, info?.clientOid),
|
|
291196
|
+
idempotencyId: context2.idempotencyId,
|
|
291197
|
+
makerActionId: context2.makerActionId,
|
|
291198
|
+
status: status.toLowerCase(),
|
|
291199
|
+
requestedQuantity: context2.requestedQuantity ?? firstNumber2(record?.amount),
|
|
291200
|
+
requestedNotional: context2.requestedNotional,
|
|
291201
|
+
executedBaseQuantity,
|
|
291202
|
+
executedQuoteQuantity,
|
|
291203
|
+
averageExecutionPrice,
|
|
291204
|
+
filledAmount: firstNumber2(record?.filled, info?.executedQty),
|
|
291205
|
+
remainingAmount: firstNumber2(record?.remaining, info?.remainingQty),
|
|
291206
|
+
feeAmount: fee.amount,
|
|
291207
|
+
feeCurrency: fee.currency,
|
|
291208
|
+
feeRate: fee.rate,
|
|
291209
|
+
exchangeTimestamp,
|
|
291210
|
+
brokerObservedTimestamp: context2.brokerObservedTimestamp ?? new Date().toISOString(),
|
|
291211
|
+
errorType: errorRecord?.name,
|
|
291212
|
+
errorMessage: errorRecord ? REDACTED_ERROR_MESSAGE : undefined
|
|
291213
|
+
});
|
|
291214
|
+
}
|
|
291215
|
+
function emitOrderExecutionTelemetryInBackground(otelMetrics, context2, order, error) {
|
|
291216
|
+
emitOrderExecutionTelemetry(otelMetrics, context2, order, error).catch((telemetryError) => {
|
|
291217
|
+
try {
|
|
291218
|
+
log.warn("Telemetry emit failed", { error: telemetryError });
|
|
291219
|
+
} catch {
|
|
291220
|
+
console.warn("Telemetry emit failed", telemetryError);
|
|
291221
|
+
}
|
|
291222
|
+
});
|
|
291223
|
+
}
|
|
291224
|
+
function extractOrderTelemetryIds(params) {
|
|
291225
|
+
const record = params ?? {};
|
|
291226
|
+
return {
|
|
291227
|
+
clientOrderId: firstString2(record.clientOrderId, record.clientOrderID, record.newClientOrderId, record.clientOid),
|
|
291228
|
+
idempotencyId: firstString2(record.idempotencyId, record.idempotencyID, record.idempotencyKey, record.requestId, record.requestID),
|
|
291229
|
+
makerActionId: firstString2(record.makerActionId, record.maker_action_id, record.actionId, record.action_id)
|
|
291230
|
+
};
|
|
291231
|
+
}
|
|
291232
|
+
function firstValue(...values2) {
|
|
291233
|
+
return values2.find((value) => value !== undefined && value !== null);
|
|
291234
|
+
}
|
|
291235
|
+
function firstString2(...values2) {
|
|
291236
|
+
for (const value of values2) {
|
|
291237
|
+
if (typeof value === "string" && value.trim()) {
|
|
291238
|
+
return value.trim();
|
|
291239
|
+
}
|
|
291240
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
291241
|
+
return String(value);
|
|
291242
|
+
}
|
|
291243
|
+
}
|
|
291244
|
+
return;
|
|
291245
|
+
}
|
|
291246
|
+
function firstNumber2(...values2) {
|
|
291247
|
+
for (const value of values2) {
|
|
291248
|
+
const numberValue = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
|
291249
|
+
if (Number.isFinite(numberValue)) {
|
|
291250
|
+
return numberValue;
|
|
291251
|
+
}
|
|
291252
|
+
}
|
|
291253
|
+
return;
|
|
291254
|
+
}
|
|
291255
|
+
function computeFilledFromAmount(record) {
|
|
291256
|
+
const amount = firstNumber2(record?.amount);
|
|
291257
|
+
const remaining = firstNumber2(record?.remaining);
|
|
291258
|
+
if (amount === undefined || remaining === undefined) {
|
|
291259
|
+
return;
|
|
291260
|
+
}
|
|
291261
|
+
return amount - remaining;
|
|
291262
|
+
}
|
|
291263
|
+
function computeAveragePrice(executedBaseQuantity, executedQuoteQuantity) {
|
|
291264
|
+
if (executedBaseQuantity === undefined || executedQuoteQuantity === undefined || executedBaseQuantity === 0) {
|
|
291265
|
+
return;
|
|
291266
|
+
}
|
|
291267
|
+
return executedQuoteQuantity / executedBaseQuantity;
|
|
291268
|
+
}
|
|
291269
|
+
function getFees(record, info) {
|
|
291270
|
+
const fees = [];
|
|
291271
|
+
if (record?.fee)
|
|
291272
|
+
fees.push(record.fee);
|
|
291273
|
+
if (Array.isArray(record?.fees))
|
|
291274
|
+
fees.push(...record.fees);
|
|
291275
|
+
if (Array.isArray(record?.trades)) {
|
|
291276
|
+
for (const trade of record.trades) {
|
|
291277
|
+
const tradeRecord = asRecord(trade);
|
|
291278
|
+
if (tradeRecord?.fee)
|
|
291279
|
+
fees.push(tradeRecord.fee);
|
|
291280
|
+
if (Array.isArray(tradeRecord?.fees))
|
|
291281
|
+
fees.push(...tradeRecord.fees);
|
|
291282
|
+
}
|
|
291283
|
+
}
|
|
291284
|
+
if (Array.isArray(info?.fills)) {
|
|
291285
|
+
for (const fill of info.fills) {
|
|
291286
|
+
const fillRecord = asRecord(fill);
|
|
291287
|
+
const commission = firstNumber2(fillRecord?.commission);
|
|
291288
|
+
if (commission !== undefined) {
|
|
291289
|
+
fees.push({
|
|
291290
|
+
cost: commission,
|
|
291291
|
+
currency: firstString2(fillRecord?.commissionAsset)
|
|
291292
|
+
});
|
|
291293
|
+
}
|
|
291294
|
+
}
|
|
291295
|
+
}
|
|
291296
|
+
return fees;
|
|
291220
291297
|
}
|
|
291221
|
-
function
|
|
291222
|
-
|
|
291223
|
-
|
|
291298
|
+
function summarizeFees(fees) {
|
|
291299
|
+
let amount = 0;
|
|
291300
|
+
let amountFound = false;
|
|
291301
|
+
let currency;
|
|
291302
|
+
let rate;
|
|
291303
|
+
for (const rawFee of fees) {
|
|
291304
|
+
const fee = asRecord(rawFee);
|
|
291305
|
+
if (!fee)
|
|
291306
|
+
continue;
|
|
291307
|
+
const cost = firstNumber2(fee.cost, fee.amount);
|
|
291308
|
+
if (cost !== undefined) {
|
|
291309
|
+
amount += cost;
|
|
291310
|
+
amountFound = true;
|
|
291311
|
+
}
|
|
291312
|
+
currency ??= firstString2(fee.currency);
|
|
291313
|
+
rate ??= firstNumber2(fee.rate);
|
|
291314
|
+
}
|
|
291224
291315
|
return {
|
|
291225
|
-
|
|
291226
|
-
|
|
291227
|
-
|
|
291228
|
-
client_order_id: input.clientOrderId,
|
|
291229
|
-
order_id: input.orderId,
|
|
291230
|
-
maker_action_id: input.makerActionId,
|
|
291231
|
-
idempotency_id: input.idempotencyId,
|
|
291232
|
-
market_metadata_hash: metadataHash,
|
|
291233
|
-
snapshot_json: JSON.stringify(redactedSnapshot)
|
|
291234
|
-
})
|
|
291316
|
+
amount: amountFound ? amount : undefined,
|
|
291317
|
+
currency,
|
|
291318
|
+
rate
|
|
291235
291319
|
};
|
|
291236
291320
|
}
|
|
291321
|
+
function compactUndefined2(record) {
|
|
291322
|
+
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
291323
|
+
}
|
|
291237
291324
|
|
|
291238
291325
|
// src/helpers/broker-execution-archive/writer.ts
|
|
291239
291326
|
var import_api_logs2 = __toESM(require_src4(), 1);
|
|
291327
|
+
import { randomUUID } from "node:crypto";
|
|
291240
291328
|
import { closeSync, fsyncSync, openSync, writeSync } from "node:fs";
|
|
291241
291329
|
import {
|
|
291242
291330
|
request as httpRequest2
|
|
@@ -291268,6 +291356,7 @@ var DEFAULT_BATCH_SIZE = 10;
|
|
|
291268
291356
|
var DEFAULT_FLUSH_INTERVAL_MS = 1000;
|
|
291269
291357
|
var DEFAULT_FORWARDER_TIMEOUT_MS = 3000;
|
|
291270
291358
|
var SHED_WARN_INTERVAL_MS = 60000;
|
|
291359
|
+
var MAX_PINNED_BATCH_ATTEMPTS = 10;
|
|
291271
291360
|
var MARKET_FEEDS = new Set(["ORDERBOOK", "TICKER", "TRADES", "OHLCV"]);
|
|
291272
291361
|
function archiveFeed(row) {
|
|
291273
291362
|
const declared = row.row.feed ?? row.row.stream_type;
|
|
@@ -291320,6 +291409,7 @@ class BrokerExecutionArchiver {
|
|
|
291320
291409
|
flushed: 0,
|
|
291321
291410
|
forwarderFailures: 0
|
|
291322
291411
|
};
|
|
291412
|
+
pendingRetry = null;
|
|
291323
291413
|
flushTimer = null;
|
|
291324
291414
|
flushInFlight = null;
|
|
291325
291415
|
inFlightBatch = null;
|
|
@@ -291453,7 +291543,7 @@ class BrokerExecutionArchiver {
|
|
|
291453
291543
|
queueMicrotask(() => this.enqueue(row));
|
|
291454
291544
|
}
|
|
291455
291545
|
async flush() {
|
|
291456
|
-
if (!this.enabled || this.closed || this.queue.length === 0) {
|
|
291546
|
+
if (!this.enabled || this.closed || this.queue.length === 0 && this.pendingRetry === null) {
|
|
291457
291547
|
return;
|
|
291458
291548
|
}
|
|
291459
291549
|
if (this.flushInFlight) {
|
|
@@ -291483,16 +291573,18 @@ class BrokerExecutionArchiver {
|
|
|
291483
291573
|
}
|
|
291484
291574
|
let closeError;
|
|
291485
291575
|
try {
|
|
291486
|
-
while (this.queue.length > 0 || this.flushInFlight) {
|
|
291576
|
+
while (this.pendingRetry !== null || this.queue.length > 0 || this.flushInFlight) {
|
|
291487
291577
|
if (this.flushInFlight) {
|
|
291488
291578
|
await this.flushInFlight;
|
|
291489
291579
|
continue;
|
|
291490
291580
|
}
|
|
291491
|
-
|
|
291492
|
-
|
|
291493
|
-
|
|
291494
|
-
|
|
291581
|
+
if (!await this.flushBatch()) {
|
|
291582
|
+
const undelivered = [
|
|
291583
|
+
...this.pendingRetry?.rows ?? [],
|
|
291584
|
+
...this.queue
|
|
291585
|
+
];
|
|
291495
291586
|
this.appendLossRecords(undelivered, "shutdown_forwarder_failure");
|
|
291587
|
+
this.pendingRetry = null;
|
|
291496
291588
|
this.queue.length = 0;
|
|
291497
291589
|
break;
|
|
291498
291590
|
}
|
|
@@ -291516,7 +291608,7 @@ class BrokerExecutionArchiver {
|
|
|
291516
291608
|
return { ...this.stats };
|
|
291517
291609
|
}
|
|
291518
291610
|
getQueueDepth() {
|
|
291519
|
-
return this.queue.length;
|
|
291611
|
+
return this.queue.length + (this.pendingRetry?.rows.length ?? 0);
|
|
291520
291612
|
}
|
|
291521
291613
|
getHealthSnapshot() {
|
|
291522
291614
|
const now3 = Date.now();
|
|
@@ -291528,7 +291620,7 @@ class BrokerExecutionArchiver {
|
|
|
291528
291620
|
const healthy = this.enabled && !this.closing && !this.closed && this.queue.length < this.maxQueueSize && oldestPendingAgeMs <= this.forwarderTimeoutMs && inFlightAgeMs <= this.forwarderTimeoutMs && recoveredFromEvents;
|
|
291529
291621
|
return {
|
|
291530
291622
|
healthy,
|
|
291531
|
-
queue_depth: this.
|
|
291623
|
+
queue_depth: this.getQueueDepth(),
|
|
291532
291624
|
oldest_pending_age_ms: oldestPendingAgeMs,
|
|
291533
291625
|
shed_total: this.stats.shed,
|
|
291534
291626
|
last_failure_at: this.lastFailureAtMs,
|
|
@@ -291542,6 +291634,13 @@ class BrokerExecutionArchiver {
|
|
|
291542
291634
|
}
|
|
291543
291635
|
oldestPendingEnqueueAtMs() {
|
|
291544
291636
|
let oldest = null;
|
|
291637
|
+
for (const entry of this.pendingRetry?.rows ?? []) {
|
|
291638
|
+
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
291639
|
+
if (enqueuedAtMs === undefined) {
|
|
291640
|
+
continue;
|
|
291641
|
+
}
|
|
291642
|
+
oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
|
|
291643
|
+
}
|
|
291545
291644
|
for (const entry of this.queue) {
|
|
291546
291645
|
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
291547
291646
|
if (enqueuedAtMs === undefined) {
|
|
@@ -291604,28 +291703,6 @@ class BrokerExecutionArchiver {
|
|
|
291604
291703
|
this.deadLetterFd = undefined;
|
|
291605
291704
|
}
|
|
291606
291705
|
}
|
|
291607
|
-
enforceQueueBound() {
|
|
291608
|
-
while (this.queue.length > this.maxQueueSize) {
|
|
291609
|
-
const dropped = this.queue[0];
|
|
291610
|
-
if (!dropped) {
|
|
291611
|
-
return;
|
|
291612
|
-
}
|
|
291613
|
-
this.appendLossRecords([dropped], "queue_shed");
|
|
291614
|
-
this.queue.shift();
|
|
291615
|
-
this.stats.shed += 1;
|
|
291616
|
-
this.recordHealthEvent("shed");
|
|
291617
|
-
this.recordArchiveMetric("cex_archive_rows_shed_total", {
|
|
291618
|
-
table: dropped.table,
|
|
291619
|
-
source: this.source,
|
|
291620
|
-
feed: archiveFeed(dropped)
|
|
291621
|
-
});
|
|
291622
|
-
this.recordArchiveMetric("cex_archive_queue_saturated_rows_total", {
|
|
291623
|
-
table: dropped.table,
|
|
291624
|
-
source: this.source,
|
|
291625
|
-
feed: archiveFeed(dropped)
|
|
291626
|
-
});
|
|
291627
|
-
}
|
|
291628
|
-
}
|
|
291629
291706
|
appendLossRecords(rows, reason) {
|
|
291630
291707
|
if (rows.length === 0) {
|
|
291631
291708
|
return;
|
|
@@ -291670,10 +291747,12 @@ class BrokerExecutionArchiver {
|
|
|
291670
291747
|
}
|
|
291671
291748
|
}
|
|
291672
291749
|
async flushBatch() {
|
|
291673
|
-
const
|
|
291750
|
+
const pinned = this.pendingRetry;
|
|
291751
|
+
const batch = pinned ? pinned.rows : this.queue.splice(0, this.batchSize);
|
|
291674
291752
|
if (batch.length === 0) {
|
|
291675
291753
|
return true;
|
|
291676
291754
|
}
|
|
291755
|
+
const batchId = pinned?.batchId ?? randomUUID();
|
|
291677
291756
|
this.inFlightBatch = batch;
|
|
291678
291757
|
this.inFlightStartedAtMs = Date.now();
|
|
291679
291758
|
for (const entry of batch) {
|
|
@@ -291683,14 +291762,23 @@ class BrokerExecutionArchiver {
|
|
|
291683
291762
|
}
|
|
291684
291763
|
if (this.forwarderUrl) {
|
|
291685
291764
|
try {
|
|
291686
|
-
await this.postToForwarder(batch);
|
|
291765
|
+
await this.postToForwarder(batch, batchId);
|
|
291687
291766
|
} catch (error) {
|
|
291688
291767
|
this.stats.forwarderFailures += 1;
|
|
291689
291768
|
this.recordHealthEvent("failure");
|
|
291690
291769
|
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
291691
|
-
|
|
291770
|
+
const attempts = (pinned?.attempts ?? 0) + 1;
|
|
291692
291771
|
try {
|
|
291693
|
-
|
|
291772
|
+
if (attempts >= MAX_PINNED_BATCH_ATTEMPTS) {
|
|
291773
|
+
this.pendingRetry = null;
|
|
291774
|
+
this.appendLossRecords(batch, "retry_exhausted");
|
|
291775
|
+
log.warn("Broker execution archive gave up on a batch", {
|
|
291776
|
+
attempts,
|
|
291777
|
+
rows: batch.length
|
|
291778
|
+
});
|
|
291779
|
+
} else {
|
|
291780
|
+
this.pendingRetry = { batchId, rows: batch, attempts };
|
|
291781
|
+
}
|
|
291694
291782
|
} finally {
|
|
291695
291783
|
this.clearInFlightBatch(batch);
|
|
291696
291784
|
this.emitArchiveHealthMetrics();
|
|
@@ -291702,6 +291790,7 @@ class BrokerExecutionArchiver {
|
|
|
291702
291790
|
return false;
|
|
291703
291791
|
}
|
|
291704
291792
|
}
|
|
291793
|
+
this.pendingRetry = null;
|
|
291705
291794
|
this.stats.flushed += batch.length;
|
|
291706
291795
|
this.recordHealthEvent("success");
|
|
291707
291796
|
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
@@ -291765,13 +291854,14 @@ class BrokerExecutionArchiver {
|
|
|
291765
291854
|
log.warn("Broker execution archive OTLP emit failed", { error });
|
|
291766
291855
|
}
|
|
291767
291856
|
}
|
|
291768
|
-
postToForwarder(batch) {
|
|
291857
|
+
postToForwarder(batch, batchId) {
|
|
291769
291858
|
if (!this.forwarderUrl || batch.length === 0) {
|
|
291770
291859
|
return Promise.resolve();
|
|
291771
291860
|
}
|
|
291772
291861
|
const body = JSON.stringify({
|
|
291773
291862
|
source: this.source,
|
|
291774
291863
|
deployment_id: this.deploymentId,
|
|
291864
|
+
batch_id: batchId,
|
|
291775
291865
|
rows: batch
|
|
291776
291866
|
});
|
|
291777
291867
|
const url2 = new URL(this.forwarderUrl);
|
|
@@ -292383,6 +292473,94 @@ class AccountBalanceArchivePoller {
|
|
|
292383
292473
|
}
|
|
292384
292474
|
}
|
|
292385
292475
|
|
|
292476
|
+
// src/helpers/balance-update-archive-consumer.ts
|
|
292477
|
+
class BalanceUpdateArchiveConsumer {
|
|
292478
|
+
params;
|
|
292479
|
+
#started = false;
|
|
292480
|
+
#stopping = false;
|
|
292481
|
+
#subscriptions = new Set;
|
|
292482
|
+
#runs = [];
|
|
292483
|
+
constructor(params) {
|
|
292484
|
+
this.params = params;
|
|
292485
|
+
}
|
|
292486
|
+
start() {
|
|
292487
|
+
if (this.#started || this.#stopping)
|
|
292488
|
+
return;
|
|
292489
|
+
this.#started = true;
|
|
292490
|
+
const targets = this.#targets();
|
|
292491
|
+
log.info("\uD83D\uDCB8 Balance-update archive consumer started", {
|
|
292492
|
+
accounts: targets.length
|
|
292493
|
+
});
|
|
292494
|
+
for (const target of targets) {
|
|
292495
|
+
this.#runs.push(this.#consume(target));
|
|
292496
|
+
}
|
|
292497
|
+
}
|
|
292498
|
+
async stop() {
|
|
292499
|
+
this.#stopping = true;
|
|
292500
|
+
for (const subscription of [...this.#subscriptions]) {
|
|
292501
|
+
subscription.close();
|
|
292502
|
+
}
|
|
292503
|
+
await Promise.all(this.#runs);
|
|
292504
|
+
}
|
|
292505
|
+
#targets() {
|
|
292506
|
+
const targets = [];
|
|
292507
|
+
for (const [exchange, pool] of Object.entries(this.params.brokers)) {
|
|
292508
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
292509
|
+
targets.push({ exchange, accountSelector: account.label });
|
|
292510
|
+
}
|
|
292511
|
+
}
|
|
292512
|
+
return targets;
|
|
292513
|
+
}
|
|
292514
|
+
async#consume(target) {
|
|
292515
|
+
while (!this.#stopping) {
|
|
292516
|
+
let subscription;
|
|
292517
|
+
try {
|
|
292518
|
+
subscription = this.params.userDataStreamSupervisor.subscribe({
|
|
292519
|
+
exchange: target.exchange,
|
|
292520
|
+
accountSelector: target.accountSelector,
|
|
292521
|
+
kind: "balance"
|
|
292522
|
+
});
|
|
292523
|
+
this.#subscriptions.add(subscription);
|
|
292524
|
+
for await (const message of subscription) {
|
|
292525
|
+
if (this.#stopping)
|
|
292526
|
+
break;
|
|
292527
|
+
const event = message.event;
|
|
292528
|
+
if (event.e !== "balanceUpdate")
|
|
292529
|
+
continue;
|
|
292530
|
+
this.params.archiver.enqueue(buildTransferEventArchiveRow({
|
|
292531
|
+
tags: buildCommonArchiveTags({
|
|
292532
|
+
deploymentId: this.params.archiver.getDeploymentId(),
|
|
292533
|
+
accountSelector: target.accountSelector,
|
|
292534
|
+
exchange: target.exchange
|
|
292535
|
+
}),
|
|
292536
|
+
transfer: {
|
|
292537
|
+
eventKind: "balance_delta",
|
|
292538
|
+
lifecycleAction: "observe_balance_update",
|
|
292539
|
+
amount: typeof event.d === "string" ? event.d : undefined,
|
|
292540
|
+
assetSymbol: typeof event.a === "string" ? event.a : undefined,
|
|
292541
|
+
exchangeTimestamp: normalizeTimestamp(event.T),
|
|
292542
|
+
payload: event
|
|
292543
|
+
}
|
|
292544
|
+
}));
|
|
292545
|
+
}
|
|
292546
|
+
} catch (error) {
|
|
292547
|
+
if (!this.#stopping) {
|
|
292548
|
+
log.warn("Balance-update archive subscription failed", {
|
|
292549
|
+
exchange: target.exchange,
|
|
292550
|
+
accountSelector: target.accountSelector,
|
|
292551
|
+
errorType: error instanceof Error ? error.name : typeof error
|
|
292552
|
+
});
|
|
292553
|
+
}
|
|
292554
|
+
} finally {
|
|
292555
|
+
if (subscription) {
|
|
292556
|
+
this.#subscriptions.delete(subscription);
|
|
292557
|
+
subscription.close();
|
|
292558
|
+
}
|
|
292559
|
+
}
|
|
292560
|
+
}
|
|
292561
|
+
}
|
|
292562
|
+
}
|
|
292563
|
+
|
|
292386
292564
|
// src/helpers/deposit-archive-poller.ts
|
|
292387
292565
|
var DEFAULT_CONFIG2 = {
|
|
292388
292566
|
pollIntervalMs: 60000,
|
|
@@ -292592,7 +292770,7 @@ class DepositArchivePoller {
|
|
|
292592
292770
|
network: network === undefined ? undefined : String(network),
|
|
292593
292771
|
externalId: depositTxid,
|
|
292594
292772
|
txid: depositTxid,
|
|
292595
|
-
exchangeTimestamp:
|
|
292773
|
+
exchangeTimestamp: normalizeTimestamp(creditedAt),
|
|
292596
292774
|
payload: record
|
|
292597
292775
|
}
|
|
292598
292776
|
}));
|
|
@@ -293454,7 +293632,7 @@ function createOtelLogsFromEnv() {
|
|
|
293454
293632
|
}
|
|
293455
293633
|
|
|
293456
293634
|
// src/helpers/stream-health-publisher.ts
|
|
293457
|
-
import { createHash as createHash4, randomUUID } from "node:crypto";
|
|
293635
|
+
import { createHash as createHash4, randomUUID as randomUUID2 } from "node:crypto";
|
|
293458
293636
|
import {
|
|
293459
293637
|
closeSync as closeSync2,
|
|
293460
293638
|
fsyncSync as fsyncSync2,
|
|
@@ -293579,7 +293757,7 @@ class StreamHealthPublisher {
|
|
|
293579
293757
|
version: STATE_VERSION,
|
|
293580
293758
|
producerId: PRODUCER_ID,
|
|
293581
293759
|
producerEpoch: "1",
|
|
293582
|
-
runId:
|
|
293760
|
+
runId: randomUUID2(),
|
|
293583
293761
|
nextBatchSequence: "1",
|
|
293584
293762
|
nextStreamSequences: {}
|
|
293585
293763
|
};
|
|
@@ -293648,7 +293826,7 @@ class StreamHealthPublisher {
|
|
|
293648
293826
|
return;
|
|
293649
293827
|
if (this.#advanceRun) {
|
|
293650
293828
|
this.#state.producerEpoch = next(this.#state.producerEpoch);
|
|
293651
|
-
this.#state.runId =
|
|
293829
|
+
this.#state.runId = randomUUID2();
|
|
293652
293830
|
this.#state.nextBatchSequence = "1";
|
|
293653
293831
|
this.#state.nextStreamSequences = {};
|
|
293654
293832
|
this.#advanceRun = false;
|
|
@@ -293764,7 +293942,7 @@ class StreamHealthPublisher {
|
|
|
293764
293942
|
cause: error
|
|
293765
293943
|
});
|
|
293766
293944
|
}
|
|
293767
|
-
const temporary = `${this.#statePath}.${process.pid}.${
|
|
293945
|
+
const temporary = `${this.#statePath}.${process.pid}.${randomUUID2()}.tmp`;
|
|
293768
293946
|
let fd2;
|
|
293769
293947
|
try {
|
|
293770
293948
|
fd2 = openSync2(temporary, "wx", 384);
|
|
@@ -293809,6 +293987,143 @@ function streamHealthPublisherConfigFromEnv(env = process.env) {
|
|
|
293809
293987
|
};
|
|
293810
293988
|
}
|
|
293811
293989
|
|
|
293990
|
+
// src/helpers/user-asset-archive-poller.ts
|
|
293991
|
+
var DEFAULT_CONFIG4 = {
|
|
293992
|
+
pollIntervalMs: 60000
|
|
293993
|
+
};
|
|
293994
|
+
var USER_ASSET_EXCHANGE_ID = "binance";
|
|
293995
|
+
var metricLabels2 = (target) => ({
|
|
293996
|
+
exchange: target.exchangeId,
|
|
293997
|
+
account_selector: target.account.label,
|
|
293998
|
+
balance_scope: USER_ASSET_BALANCE_SCOPE
|
|
293999
|
+
});
|
|
294000
|
+
|
|
294001
|
+
class UserAssetArchivePoller {
|
|
294002
|
+
params;
|
|
294003
|
+
#timer = null;
|
|
294004
|
+
#stopped = false;
|
|
294005
|
+
#running = null;
|
|
294006
|
+
#lastSuccessMs = new Map;
|
|
294007
|
+
#config;
|
|
294008
|
+
constructor(params) {
|
|
294009
|
+
this.params = params;
|
|
294010
|
+
this.#config = { ...DEFAULT_CONFIG4, ...params.config };
|
|
294011
|
+
}
|
|
294012
|
+
start() {
|
|
294013
|
+
if (this.#timer || this.#stopped || !this.params.archiver.canPersistAccountBalanceSnapshots() || this.#targets().length === 0) {
|
|
294014
|
+
return;
|
|
294015
|
+
}
|
|
294016
|
+
log.info("\uD83E\uDDCA User asset archive poller started", {
|
|
294017
|
+
balanceScope: USER_ASSET_BALANCE_SCOPE
|
|
294018
|
+
});
|
|
294019
|
+
this.#schedule(0);
|
|
294020
|
+
}
|
|
294021
|
+
async stop() {
|
|
294022
|
+
this.#stopped = true;
|
|
294023
|
+
if (this.#timer) {
|
|
294024
|
+
clearTimeout(this.#timer);
|
|
294025
|
+
this.#timer = null;
|
|
294026
|
+
}
|
|
294027
|
+
await this.#running;
|
|
294028
|
+
}
|
|
294029
|
+
async pollAllOnce() {
|
|
294030
|
+
if (this.#stopped || this.#running || !this.params.archiver.canPersistAccountBalanceSnapshots()) {
|
|
294031
|
+
return false;
|
|
294032
|
+
}
|
|
294033
|
+
this.#running = this.#pollAllSequentially();
|
|
294034
|
+
try {
|
|
294035
|
+
return await this.#running;
|
|
294036
|
+
} finally {
|
|
294037
|
+
this.#running = null;
|
|
294038
|
+
}
|
|
294039
|
+
}
|
|
294040
|
+
#targets() {
|
|
294041
|
+
const targets = [];
|
|
294042
|
+
for (const [exchangeId, pool] of Object.entries(this.params.brokers)) {
|
|
294043
|
+
if (exchangeId.trim().toLowerCase() !== USER_ASSET_EXCHANGE_ID) {
|
|
294044
|
+
continue;
|
|
294045
|
+
}
|
|
294046
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
294047
|
+
targets.push({ exchangeId, account });
|
|
294048
|
+
}
|
|
294049
|
+
}
|
|
294050
|
+
return targets;
|
|
294051
|
+
}
|
|
294052
|
+
async#pollAllSequentially() {
|
|
294053
|
+
for (const target of this.#targets()) {
|
|
294054
|
+
if (this.#stopped) {
|
|
294055
|
+
break;
|
|
294056
|
+
}
|
|
294057
|
+
await this.#pollOne(target);
|
|
294058
|
+
}
|
|
294059
|
+
return true;
|
|
294060
|
+
}
|
|
294061
|
+
async#pollOne(target) {
|
|
294062
|
+
const labels = metricLabels2(target);
|
|
294063
|
+
this.params.metrics?.recordCounter("cex_user_asset_poll_attempts_total", 1, labels);
|
|
294064
|
+
try {
|
|
294065
|
+
const exchange = target.account.exchange;
|
|
294066
|
+
if (typeof exchange.sapiV3PostAssetGetUserAsset !== "function") {
|
|
294067
|
+
throw new Error("binance_user_asset_unavailable: getUserAsset is not defined on this exchange instance");
|
|
294068
|
+
}
|
|
294069
|
+
const response = await exchange.sapiV3PostAssetGetUserAsset({});
|
|
294070
|
+
const observedAt = new Date;
|
|
294071
|
+
const normalized = normalizeBinanceUserAssetsForArchive(response);
|
|
294072
|
+
this.params.archiver.enqueue(buildUserAssetSnapshotRow({
|
|
294073
|
+
tags: buildCommonArchiveTags({
|
|
294074
|
+
deploymentId: this.params.archiver.getDeploymentId(),
|
|
294075
|
+
accountSelector: target.account.label,
|
|
294076
|
+
exchange: target.exchangeId,
|
|
294077
|
+
brokerObservedTimestamp: observedAt.toISOString()
|
|
294078
|
+
}),
|
|
294079
|
+
userAssets: normalized
|
|
294080
|
+
}));
|
|
294081
|
+
const successMs = Date.now();
|
|
294082
|
+
this.#lastSuccessMs.set(this.#targetKey(target), successMs);
|
|
294083
|
+
this.params.metrics?.recordCounter("cex_user_asset_poll_successes_total", 1, labels);
|
|
294084
|
+
this.params.metrics?.recordGauge("cex_user_asset_poll_last_success_timestamp_seconds", Math.floor(successMs / 1000), labels);
|
|
294085
|
+
this.#recordFreshness(labels, successMs, successMs);
|
|
294086
|
+
} catch (error) {
|
|
294087
|
+
rethrowArchiveDurabilityError(error);
|
|
294088
|
+
this.params.metrics?.recordCounter("cex_user_asset_poll_failures_total", 1, labels);
|
|
294089
|
+
const now3 = Date.now();
|
|
294090
|
+
const lastSuccess = this.#lastSuccessMs.get(this.#targetKey(target));
|
|
294091
|
+
if (lastSuccess !== undefined) {
|
|
294092
|
+
this.#recordFreshness(labels, lastSuccess, now3);
|
|
294093
|
+
}
|
|
294094
|
+
log.warn("User asset archive poll failed", {
|
|
294095
|
+
exchange: target.exchangeId,
|
|
294096
|
+
account: target.account.label,
|
|
294097
|
+
balanceScope: USER_ASSET_BALANCE_SCOPE,
|
|
294098
|
+
errorType: error instanceof Error ? error.name : "unknown"
|
|
294099
|
+
});
|
|
294100
|
+
}
|
|
294101
|
+
}
|
|
294102
|
+
#recordFreshness(labels, lastSuccessMs, nowMs) {
|
|
294103
|
+
this.params.metrics?.recordGauge("cex_user_asset_poll_freshness_seconds", Math.max(0, (nowMs - lastSuccessMs) / 1000), labels);
|
|
294104
|
+
}
|
|
294105
|
+
#targetKey(target) {
|
|
294106
|
+
return `${target.exchangeId}|${target.account.label}|${USER_ASSET_BALANCE_SCOPE}`;
|
|
294107
|
+
}
|
|
294108
|
+
#schedule(delayMs) {
|
|
294109
|
+
this.#timer = setTimeout(() => void this.#tick(), delayMs);
|
|
294110
|
+
this.#timer.unref?.();
|
|
294111
|
+
}
|
|
294112
|
+
async#tick() {
|
|
294113
|
+
this.#timer = null;
|
|
294114
|
+
try {
|
|
294115
|
+
await this.pollAllOnce();
|
|
294116
|
+
} catch (error) {
|
|
294117
|
+
rethrowArchiveDurabilityError(error);
|
|
294118
|
+
log.error("User asset archive poller tick failed", error);
|
|
294119
|
+
} finally {
|
|
294120
|
+
if (!this.#stopped) {
|
|
294121
|
+
this.#schedule(this.#config.pollIntervalMs);
|
|
294122
|
+
}
|
|
294123
|
+
}
|
|
294124
|
+
}
|
|
294125
|
+
}
|
|
294126
|
+
|
|
293812
294127
|
// src/helpers/binance-user-data-stream.ts
|
|
293813
294128
|
import { Buffer as Buffer2 } from "node:buffer";
|
|
293814
294129
|
import { createHmac } from "node:crypto";
|
|
@@ -308336,7 +308651,7 @@ async function handleDeposit(ctx) {
|
|
|
308336
308651
|
network: depositNetwork?.exchangeNetworkId,
|
|
308337
308652
|
externalId: depositTxid,
|
|
308338
308653
|
txid: depositTxid,
|
|
308339
|
-
exchangeTimestamp:
|
|
308654
|
+
exchangeTimestamp: normalizeTimestamp(creditedAt),
|
|
308340
308655
|
payload: deposit
|
|
308341
308656
|
}
|
|
308342
308657
|
});
|
|
@@ -311854,6 +312169,8 @@ class CEXBroker {
|
|
|
311854
312169
|
fillArchivePoller;
|
|
311855
312170
|
depositArchivePoller;
|
|
311856
312171
|
accountBalanceArchivePoller;
|
|
312172
|
+
userAssetArchivePoller;
|
|
312173
|
+
balanceUpdateArchiveConsumer;
|
|
311857
312174
|
userDataStreamSupervisor;
|
|
311858
312175
|
loadEnvConfig() {
|
|
311859
312176
|
log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
|
|
@@ -312007,6 +312324,14 @@ class CEXBroker {
|
|
|
312007
312324
|
await this.accountBalanceArchivePoller.stop();
|
|
312008
312325
|
this.accountBalanceArchivePoller = undefined;
|
|
312009
312326
|
}
|
|
312327
|
+
if (this.userAssetArchivePoller) {
|
|
312328
|
+
await this.userAssetArchivePoller.stop();
|
|
312329
|
+
this.userAssetArchivePoller = undefined;
|
|
312330
|
+
}
|
|
312331
|
+
if (this.balanceUpdateArchiveConsumer) {
|
|
312332
|
+
await this.balanceUpdateArchiveConsumer.stop();
|
|
312333
|
+
this.balanceUpdateArchiveConsumer = undefined;
|
|
312334
|
+
}
|
|
312010
312335
|
if (this.server) {
|
|
312011
312336
|
await this.server.forceShutdown();
|
|
312012
312337
|
}
|
|
@@ -312052,6 +312377,14 @@ class CEXBroker {
|
|
|
312052
312377
|
await this.accountBalanceArchivePoller.stop();
|
|
312053
312378
|
this.accountBalanceArchivePoller = undefined;
|
|
312054
312379
|
}
|
|
312380
|
+
if (this.userAssetArchivePoller) {
|
|
312381
|
+
await this.userAssetArchivePoller.stop();
|
|
312382
|
+
this.userAssetArchivePoller = undefined;
|
|
312383
|
+
}
|
|
312384
|
+
if (this.balanceUpdateArchiveConsumer) {
|
|
312385
|
+
await this.balanceUpdateArchiveConsumer.stop();
|
|
312386
|
+
this.balanceUpdateArchiveConsumer = undefined;
|
|
312387
|
+
}
|
|
312055
312388
|
log.info(`Running CEXBroker at ${new Date().toISOString()}`);
|
|
312056
312389
|
if (this.otelMetrics?.isOtelEnabled()) {
|
|
312057
312390
|
await this.otelMetrics.initialize();
|
|
@@ -312093,6 +312426,14 @@ class CEXBroker {
|
|
|
312093
312426
|
metrics: this.otelMetrics
|
|
312094
312427
|
});
|
|
312095
312428
|
this.depositArchivePoller.start();
|
|
312429
|
+
if (this.userDataStreamSupervisor) {
|
|
312430
|
+
this.balanceUpdateArchiveConsumer = new BalanceUpdateArchiveConsumer({
|
|
312431
|
+
brokers: this.brokers,
|
|
312432
|
+
archiver: this.brokerArchiver,
|
|
312433
|
+
userDataStreamSupervisor: this.userDataStreamSupervisor
|
|
312434
|
+
});
|
|
312435
|
+
this.balanceUpdateArchiveConsumer.start();
|
|
312436
|
+
}
|
|
312096
312437
|
}
|
|
312097
312438
|
if (this.brokerArchiver?.canPersistAccountBalanceSnapshots()) {
|
|
312098
312439
|
this.accountBalanceArchivePoller = new AccountBalanceArchivePoller({
|
|
@@ -312101,6 +312442,12 @@ class CEXBroker {
|
|
|
312101
312442
|
metrics: this.otelMetrics
|
|
312102
312443
|
});
|
|
312103
312444
|
this.accountBalanceArchivePoller.start();
|
|
312445
|
+
this.userAssetArchivePoller = new UserAssetArchivePoller({
|
|
312446
|
+
brokers: this.brokers,
|
|
312447
|
+
archiver: this.brokerArchiver,
|
|
312448
|
+
metrics: this.otelMetrics
|
|
312449
|
+
});
|
|
312450
|
+
this.userAssetArchivePoller.start();
|
|
312104
312451
|
}
|
|
312105
312452
|
return this;
|
|
312106
312453
|
}
|
|
@@ -312109,4 +312456,4 @@ export {
|
|
|
312109
312456
|
CEXBroker as default
|
|
312110
312457
|
};
|
|
312111
312458
|
|
|
312112
|
-
//# debugId=
|
|
312459
|
+
//# debugId=2BB56A1EBD617FCC64756E2164756E21
|