@usherlabs/cex-broker 0.2.41 → 0.2.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cli.js +666 -319
- package/dist/helpers/balance-update-archive-consumer.d.ts +16 -0
- package/dist/helpers/broker-execution-archive/index.d.ts +2 -2
- package/dist/helpers/broker-execution-archive/rows.d.ts +25 -0
- package/dist/helpers/broker-execution-archive/types.d.ts +5 -3
- package/dist/helpers/broker-execution-archive/writer.d.ts +1 -1
- package/dist/helpers/user-asset-archive-poller.d.ts +19 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +667 -320
- package/dist/index.js.map +11 -9
- package/package.json +1 -1
package/dist/commands/cli.js
CHANGED
|
@@ -315011,6 +315011,20 @@ function validateDeposit(policy, exchange, network, ticker) {
|
|
|
315011
315011
|
return { valid: true };
|
|
315012
315012
|
}
|
|
315013
315013
|
|
|
315014
|
+
// src/helpers/broker-execution-archive/rows.ts
|
|
315015
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
315016
|
+
|
|
315017
|
+
// src/helpers/shared/guards.ts
|
|
315018
|
+
function isRecord(value) {
|
|
315019
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
315020
|
+
}
|
|
315021
|
+
function asRecord(value) {
|
|
315022
|
+
return isRecord(value) ? value : undefined;
|
|
315023
|
+
}
|
|
315024
|
+
|
|
315025
|
+
// src/helpers/broker-execution-archive/redact.ts
|
|
315026
|
+
import { createHash } from "node:crypto";
|
|
315027
|
+
|
|
315014
315028
|
// src/helpers/shared/errors.ts
|
|
315015
315029
|
var MAX_ERROR_DETAIL_LENGTH = 512;
|
|
315016
315030
|
var REDACTED_ERROR_MESSAGE = "redacted_error";
|
|
@@ -315070,226 +315084,7 @@ function safeLogRedactedError(context2, error) {
|
|
|
315070
315084
|
}
|
|
315071
315085
|
}
|
|
315072
315086
|
|
|
315073
|
-
// src/helpers/shared/guards.ts
|
|
315074
|
-
function isRecord(value) {
|
|
315075
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
315076
|
-
}
|
|
315077
|
-
function asRecord(value) {
|
|
315078
|
-
return isRecord(value) ? value : undefined;
|
|
315079
|
-
}
|
|
315080
|
-
|
|
315081
|
-
// src/helpers/order-telemetry.ts
|
|
315082
|
-
var NUMERIC_METRICS = [
|
|
315083
|
-
["requestedQuantity", "cex_market_action_requested_quantity"],
|
|
315084
|
-
["requestedNotional", "cex_market_action_requested_notional"],
|
|
315085
|
-
["executedBaseQuantity", "cex_market_action_executed_base_quantity"],
|
|
315086
|
-
["executedQuoteQuantity", "cex_market_action_executed_quote_quantity"],
|
|
315087
|
-
["averageExecutionPrice", "cex_market_action_average_execution_price"],
|
|
315088
|
-
["filledAmount", "cex_market_action_filled_amount"],
|
|
315089
|
-
["remainingAmount", "cex_market_action_remaining_amount"],
|
|
315090
|
-
["feeAmount", "cex_market_action_fee_amount"],
|
|
315091
|
-
["feeRate", "cex_market_action_fee_rate"]
|
|
315092
|
-
];
|
|
315093
|
-
async function emitOrderExecutionTelemetry(otelMetrics, context2, order, error) {
|
|
315094
|
-
try {
|
|
315095
|
-
const telemetry = buildOrderExecutionTelemetry(context2, order, error);
|
|
315096
|
-
log.info("CEX market action execution telemetry", telemetry);
|
|
315097
|
-
const labels = {
|
|
315098
|
-
action: telemetry.action,
|
|
315099
|
-
cex: telemetry.cex,
|
|
315100
|
-
account: telemetry.accountLabel,
|
|
315101
|
-
symbol: telemetry.symbol,
|
|
315102
|
-
side: telemetry.side,
|
|
315103
|
-
order_type: telemetry.orderType,
|
|
315104
|
-
status: telemetry.status
|
|
315105
|
-
};
|
|
315106
|
-
await otelMetrics?.recordCounter("cex_market_action_executions_total", 1, {
|
|
315107
|
-
...labels,
|
|
315108
|
-
result: error ? "error" : "ok"
|
|
315109
|
-
});
|
|
315110
|
-
for (const [key, metricName] of NUMERIC_METRICS) {
|
|
315111
|
-
const value = telemetry[key];
|
|
315112
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
315113
|
-
await otelMetrics?.recordHistogram(metricName, value, labels);
|
|
315114
|
-
}
|
|
315115
|
-
}
|
|
315116
|
-
return telemetry;
|
|
315117
|
-
} catch (telemetryError) {
|
|
315118
|
-
try {
|
|
315119
|
-
log.error("Failed to emit CEX order telemetry", {
|
|
315120
|
-
error: telemetryError
|
|
315121
|
-
});
|
|
315122
|
-
} catch {}
|
|
315123
|
-
return;
|
|
315124
|
-
}
|
|
315125
|
-
}
|
|
315126
|
-
function buildOrderExecutionTelemetry(context2, order, error) {
|
|
315127
|
-
const record = asRecord(order);
|
|
315128
|
-
const info = asRecord(record?.info);
|
|
315129
|
-
const fees = getFees(record, info);
|
|
315130
|
-
const fee = summarizeFees(fees);
|
|
315131
|
-
const executedBaseQuantity = firstNumber(record?.filled, info?.executedQty, info?.cumExecQty) ?? computeFilledFromAmount(record);
|
|
315132
|
-
const executedQuoteQuantity = firstNumber(record?.cost, info?.cummulativeQuoteQty, info?.cumQuote, info?.cumExecValue);
|
|
315133
|
-
const averageExecutionPrice = firstNumber(record?.average, info?.avgPrice) ?? computeAveragePrice(executedBaseQuantity, executedQuoteQuantity);
|
|
315134
|
-
const exchangeTimestamp = normalizeTimestamp(firstValue(record?.timestamp, info?.time, info?.transactTime, record?.datetime));
|
|
315135
|
-
const status = firstString(record?.status, info?.status) ?? (error ? "failed" : "unknown");
|
|
315136
|
-
const errorRecord = error instanceof Error ? error : undefined;
|
|
315137
|
-
return compactUndefined({
|
|
315138
|
-
event: "cex_market_action_execution",
|
|
315139
|
-
action: context2.action,
|
|
315140
|
-
cex: context2.cex.trim().toLowerCase() || "unknown",
|
|
315141
|
-
accountLabel: context2.accountLabel ?? "unknown",
|
|
315142
|
-
symbol: firstString(record?.symbol, info?.symbol, context2.symbol) ?? "unknown",
|
|
315143
|
-
side: firstString(record?.side, info?.side, context2.side) ?? "unknown",
|
|
315144
|
-
orderType: firstString(record?.type, info?.type, context2.orderType) ?? "unknown",
|
|
315145
|
-
orderId: firstString(record?.id, info?.orderId, info?.orderID),
|
|
315146
|
-
orderAuthor: context2.orderAuthor,
|
|
315147
|
-
clientOrderId: firstString(context2.clientOrderId, record?.clientOrderId, record?.clientOrderID, record?.clientOid, info?.clientOrderId, info?.clientOrderID, info?.clientOid),
|
|
315148
|
-
idempotencyId: context2.idempotencyId,
|
|
315149
|
-
makerActionId: context2.makerActionId,
|
|
315150
|
-
status: status.toLowerCase(),
|
|
315151
|
-
requestedQuantity: context2.requestedQuantity ?? firstNumber(record?.amount),
|
|
315152
|
-
requestedNotional: context2.requestedNotional,
|
|
315153
|
-
executedBaseQuantity,
|
|
315154
|
-
executedQuoteQuantity,
|
|
315155
|
-
averageExecutionPrice,
|
|
315156
|
-
filledAmount: firstNumber(record?.filled, info?.executedQty),
|
|
315157
|
-
remainingAmount: firstNumber(record?.remaining, info?.remainingQty),
|
|
315158
|
-
feeAmount: fee.amount,
|
|
315159
|
-
feeCurrency: fee.currency,
|
|
315160
|
-
feeRate: fee.rate,
|
|
315161
|
-
exchangeTimestamp,
|
|
315162
|
-
brokerObservedTimestamp: context2.brokerObservedTimestamp ?? new Date().toISOString(),
|
|
315163
|
-
errorType: errorRecord?.name,
|
|
315164
|
-
errorMessage: errorRecord ? REDACTED_ERROR_MESSAGE : undefined
|
|
315165
|
-
});
|
|
315166
|
-
}
|
|
315167
|
-
function emitOrderExecutionTelemetryInBackground(otelMetrics, context2, order, error) {
|
|
315168
|
-
emitOrderExecutionTelemetry(otelMetrics, context2, order, error).catch((telemetryError) => {
|
|
315169
|
-
try {
|
|
315170
|
-
log.warn("Telemetry emit failed", { error: telemetryError });
|
|
315171
|
-
} catch {
|
|
315172
|
-
console.warn("Telemetry emit failed", telemetryError);
|
|
315173
|
-
}
|
|
315174
|
-
});
|
|
315175
|
-
}
|
|
315176
|
-
function extractOrderTelemetryIds(params) {
|
|
315177
|
-
const record = params ?? {};
|
|
315178
|
-
return {
|
|
315179
|
-
clientOrderId: firstString(record.clientOrderId, record.clientOrderID, record.newClientOrderId, record.clientOid),
|
|
315180
|
-
idempotencyId: firstString(record.idempotencyId, record.idempotencyID, record.idempotencyKey, record.requestId, record.requestID),
|
|
315181
|
-
makerActionId: firstString(record.makerActionId, record.maker_action_id, record.actionId, record.action_id)
|
|
315182
|
-
};
|
|
315183
|
-
}
|
|
315184
|
-
function firstValue(...values2) {
|
|
315185
|
-
return values2.find((value) => value !== undefined && value !== null);
|
|
315186
|
-
}
|
|
315187
|
-
function firstString(...values2) {
|
|
315188
|
-
for (const value of values2) {
|
|
315189
|
-
if (typeof value === "string" && value.trim()) {
|
|
315190
|
-
return value.trim();
|
|
315191
|
-
}
|
|
315192
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
315193
|
-
return String(value);
|
|
315194
|
-
}
|
|
315195
|
-
}
|
|
315196
|
-
return;
|
|
315197
|
-
}
|
|
315198
|
-
function firstNumber(...values2) {
|
|
315199
|
-
for (const value of values2) {
|
|
315200
|
-
const numberValue = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
|
315201
|
-
if (Number.isFinite(numberValue)) {
|
|
315202
|
-
return numberValue;
|
|
315203
|
-
}
|
|
315204
|
-
}
|
|
315205
|
-
return;
|
|
315206
|
-
}
|
|
315207
|
-
function computeFilledFromAmount(record) {
|
|
315208
|
-
const amount = firstNumber(record?.amount);
|
|
315209
|
-
const remaining = firstNumber(record?.remaining);
|
|
315210
|
-
if (amount === undefined || remaining === undefined) {
|
|
315211
|
-
return;
|
|
315212
|
-
}
|
|
315213
|
-
return amount - remaining;
|
|
315214
|
-
}
|
|
315215
|
-
function computeAveragePrice(executedBaseQuantity, executedQuoteQuantity) {
|
|
315216
|
-
if (executedBaseQuantity === undefined || executedQuoteQuantity === undefined || executedBaseQuantity === 0) {
|
|
315217
|
-
return;
|
|
315218
|
-
}
|
|
315219
|
-
return executedQuoteQuantity / executedBaseQuantity;
|
|
315220
|
-
}
|
|
315221
|
-
function normalizeTimestamp(value) {
|
|
315222
|
-
if (typeof value === "string" && value.trim()) {
|
|
315223
|
-
return value.trim();
|
|
315224
|
-
}
|
|
315225
|
-
const timestamp = firstNumber(value);
|
|
315226
|
-
if (timestamp === undefined) {
|
|
315227
|
-
return;
|
|
315228
|
-
}
|
|
315229
|
-
const date = new Date(timestamp);
|
|
315230
|
-
return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
|
|
315231
|
-
}
|
|
315232
|
-
function getFees(record, info) {
|
|
315233
|
-
const fees = [];
|
|
315234
|
-
if (record?.fee)
|
|
315235
|
-
fees.push(record.fee);
|
|
315236
|
-
if (Array.isArray(record?.fees))
|
|
315237
|
-
fees.push(...record.fees);
|
|
315238
|
-
if (Array.isArray(record?.trades)) {
|
|
315239
|
-
for (const trade of record.trades) {
|
|
315240
|
-
const tradeRecord = asRecord(trade);
|
|
315241
|
-
if (tradeRecord?.fee)
|
|
315242
|
-
fees.push(tradeRecord.fee);
|
|
315243
|
-
if (Array.isArray(tradeRecord?.fees))
|
|
315244
|
-
fees.push(...tradeRecord.fees);
|
|
315245
|
-
}
|
|
315246
|
-
}
|
|
315247
|
-
if (Array.isArray(info?.fills)) {
|
|
315248
|
-
for (const fill of info.fills) {
|
|
315249
|
-
const fillRecord = asRecord(fill);
|
|
315250
|
-
const commission = firstNumber(fillRecord?.commission);
|
|
315251
|
-
if (commission !== undefined) {
|
|
315252
|
-
fees.push({
|
|
315253
|
-
cost: commission,
|
|
315254
|
-
currency: firstString(fillRecord?.commissionAsset)
|
|
315255
|
-
});
|
|
315256
|
-
}
|
|
315257
|
-
}
|
|
315258
|
-
}
|
|
315259
|
-
return fees;
|
|
315260
|
-
}
|
|
315261
|
-
function summarizeFees(fees) {
|
|
315262
|
-
let amount = 0;
|
|
315263
|
-
let amountFound = false;
|
|
315264
|
-
let currency;
|
|
315265
|
-
let rate;
|
|
315266
|
-
for (const rawFee of fees) {
|
|
315267
|
-
const fee = asRecord(rawFee);
|
|
315268
|
-
if (!fee)
|
|
315269
|
-
continue;
|
|
315270
|
-
const cost = firstNumber(fee.cost, fee.amount);
|
|
315271
|
-
if (cost !== undefined) {
|
|
315272
|
-
amount += cost;
|
|
315273
|
-
amountFound = true;
|
|
315274
|
-
}
|
|
315275
|
-
currency ??= firstString(fee.currency);
|
|
315276
|
-
rate ??= firstNumber(fee.rate);
|
|
315277
|
-
}
|
|
315278
|
-
return {
|
|
315279
|
-
amount: amountFound ? amount : undefined,
|
|
315280
|
-
currency,
|
|
315281
|
-
rate
|
|
315282
|
-
};
|
|
315283
|
-
}
|
|
315284
|
-
function compactUndefined(record) {
|
|
315285
|
-
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
315286
|
-
}
|
|
315287
|
-
|
|
315288
|
-
// src/helpers/broker-execution-archive/rows.ts
|
|
315289
|
-
import { createHash as createHash2 } from "node:crypto";
|
|
315290
|
-
|
|
315291
315087
|
// src/helpers/broker-execution-archive/redact.ts
|
|
315292
|
-
import { createHash } from "node:crypto";
|
|
315293
315088
|
var SECRET_KEY_PATTERN = /\b(api[_-]?key|api[_-]?secret|secret|signature|passphrase|password|token|credential)\b/i;
|
|
315294
315089
|
var SECRET_VALUE_PATTERN = /(\b(?:apiKey|api_key|apiSecret|api_secret|secret|signature|passphrase|password|token)\b\s*[=:]\s*)[^\s&,;)}\]]+/gi;
|
|
315295
315090
|
var SECRET_JSON_PATTERN = /("(?:apiKey|api_key|apiSecret|api_secret|secret|signature|passphrase|password|token)"\s*:\s*")[^"]*(")/gi;
|
|
@@ -315362,9 +315157,11 @@ var ARCHIVE_SCHEMA_VERSION = "1";
|
|
|
315362
315157
|
var FILL_EVENT_KIND = "trade_history_fill";
|
|
315363
315158
|
var ACCOUNT_BALANCE_SCOPE = "spot";
|
|
315364
315159
|
var ACCOUNT_BALANCE_PRECISION_BASIS = "ccxt_normalized_number";
|
|
315160
|
+
var USER_ASSET_BALANCE_SCOPE = "user_asset";
|
|
315161
|
+
var USER_ASSET_PRECISION_BASIS = "venue_raw_string";
|
|
315365
315162
|
|
|
315366
315163
|
// src/helpers/broker-execution-archive/rows.ts
|
|
315367
|
-
function
|
|
315164
|
+
function firstString(...values2) {
|
|
315368
315165
|
for (const value of values2) {
|
|
315369
315166
|
if (typeof value === "string" && value.trim()) {
|
|
315370
315167
|
return value.trim();
|
|
@@ -315375,7 +315172,7 @@ function firstString2(...values2) {
|
|
|
315375
315172
|
}
|
|
315376
315173
|
return;
|
|
315377
315174
|
}
|
|
315378
|
-
function
|
|
315175
|
+
function firstNumber(...values2) {
|
|
315379
315176
|
for (const value of values2) {
|
|
315380
315177
|
const numeric = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
|
315381
315178
|
if (Number.isFinite(numeric)) {
|
|
@@ -315384,11 +315181,11 @@ function firstNumber2(...values2) {
|
|
|
315384
315181
|
}
|
|
315385
315182
|
return;
|
|
315386
315183
|
}
|
|
315387
|
-
function
|
|
315184
|
+
function normalizeTimestamp(value) {
|
|
315388
315185
|
if (typeof value === "string" && value.trim()) {
|
|
315389
315186
|
return value.trim();
|
|
315390
315187
|
}
|
|
315391
|
-
const ms =
|
|
315188
|
+
const ms = firstNumber(value);
|
|
315392
315189
|
if (ms === undefined) {
|
|
315393
315190
|
return;
|
|
315394
315191
|
}
|
|
@@ -315495,7 +315292,7 @@ function normalizeCcxtBalanceForArchive(balance) {
|
|
|
315495
315292
|
}
|
|
315496
315293
|
}
|
|
315497
315294
|
return {
|
|
315498
|
-
exchangeTimestamp:
|
|
315295
|
+
exchangeTimestamp: normalizeTimestamp(record.timestamp ?? record.datetime),
|
|
315499
315296
|
reportedAssets: [...reportedAssets].sort(),
|
|
315500
315297
|
assetEntryAssets: [...assetEntryAssets].sort(),
|
|
315501
315298
|
freeBalances: sortedBalanceMap(maps.free),
|
|
@@ -315518,7 +315315,7 @@ function buildAccountBalanceSnapshotRow(input) {
|
|
|
315518
315315
|
})).digest("hex");
|
|
315519
315316
|
return {
|
|
315520
315317
|
table: "broker_account.balance_snapshots",
|
|
315521
|
-
row:
|
|
315318
|
+
row: compactUndefined({
|
|
315522
315319
|
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
315523
315320
|
exchange_timestamp: balance.exchangeTimestamp,
|
|
315524
315321
|
source: tags.source,
|
|
@@ -315540,7 +315337,101 @@ function buildAccountBalanceSnapshotRow(input) {
|
|
|
315540
315337
|
})
|
|
315541
315338
|
};
|
|
315542
315339
|
}
|
|
315543
|
-
|
|
315340
|
+
var USER_ASSET_BUCKETS = {
|
|
315341
|
+
freeBalances: "free",
|
|
315342
|
+
lockedBalances: "locked",
|
|
315343
|
+
freezeBalances: "freeze",
|
|
315344
|
+
withdrawingBalances: "withdrawing"
|
|
315345
|
+
};
|
|
315346
|
+
function userAssetQuantity(value) {
|
|
315347
|
+
if (typeof value === "number") {
|
|
315348
|
+
return decimalString(value);
|
|
315349
|
+
}
|
|
315350
|
+
if (typeof value !== "string") {
|
|
315351
|
+
return;
|
|
315352
|
+
}
|
|
315353
|
+
const trimmed = value.trim();
|
|
315354
|
+
if (!trimmed || !Number.isFinite(Number(trimmed))) {
|
|
315355
|
+
return;
|
|
315356
|
+
}
|
|
315357
|
+
return trimmed;
|
|
315358
|
+
}
|
|
315359
|
+
function normalizeBinanceUserAssetsForArchive(response) {
|
|
315360
|
+
if (!Array.isArray(response)) {
|
|
315361
|
+
throw new Error("binance_user_asset_malformed_response: getUserAsset did not return an array");
|
|
315362
|
+
}
|
|
315363
|
+
const reportedAssets = [];
|
|
315364
|
+
const incompleteAssets = [];
|
|
315365
|
+
const maps = {
|
|
315366
|
+
freeBalances: {},
|
|
315367
|
+
lockedBalances: {},
|
|
315368
|
+
freezeBalances: {},
|
|
315369
|
+
withdrawingBalances: {}
|
|
315370
|
+
};
|
|
315371
|
+
for (const entry of response) {
|
|
315372
|
+
const record = asRecord(entry);
|
|
315373
|
+
const asset = typeof record?.asset === "string" ? record.asset.trim() : undefined;
|
|
315374
|
+
if (!record || !asset) {
|
|
315375
|
+
throw new Error("binance_user_asset_malformed_response: getUserAsset entry has no asset key");
|
|
315376
|
+
}
|
|
315377
|
+
if (reportedAssets.includes(asset)) {
|
|
315378
|
+
throw new Error(`binance_user_asset_malformed_response: getUserAsset returned duplicate entries for ${asset}`);
|
|
315379
|
+
}
|
|
315380
|
+
reportedAssets.push(asset);
|
|
315381
|
+
let complete = true;
|
|
315382
|
+
for (const [field, venueKey] of Object.entries(USER_ASSET_BUCKETS)) {
|
|
315383
|
+
const quantity = userAssetQuantity(record[venueKey]);
|
|
315384
|
+
if (quantity === undefined) {
|
|
315385
|
+
complete = false;
|
|
315386
|
+
continue;
|
|
315387
|
+
}
|
|
315388
|
+
maps[field][asset] = quantity;
|
|
315389
|
+
}
|
|
315390
|
+
if (!complete) {
|
|
315391
|
+
incompleteAssets.push(asset);
|
|
315392
|
+
}
|
|
315393
|
+
}
|
|
315394
|
+
return {
|
|
315395
|
+
reportedAssets: reportedAssets.sort(),
|
|
315396
|
+
incompleteAssets: incompleteAssets.sort(),
|
|
315397
|
+
freeBalances: sortedBalanceMap(maps.freeBalances),
|
|
315398
|
+
lockedBalances: sortedBalanceMap(maps.lockedBalances),
|
|
315399
|
+
freezeBalances: sortedBalanceMap(maps.freezeBalances),
|
|
315400
|
+
withdrawingBalances: sortedBalanceMap(maps.withdrawingBalances)
|
|
315401
|
+
};
|
|
315402
|
+
}
|
|
315403
|
+
function buildUserAssetSnapshotRow(input) {
|
|
315404
|
+
const { tags, userAssets } = input;
|
|
315405
|
+
const observationId = createHash2("sha256").update(JSON.stringify({
|
|
315406
|
+
deployment_id: tags.deployment_id,
|
|
315407
|
+
exchange: tags.exchange,
|
|
315408
|
+
account_selector: tags.account_selector,
|
|
315409
|
+
balance_scope: USER_ASSET_BALANCE_SCOPE,
|
|
315410
|
+
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
315411
|
+
userAssets
|
|
315412
|
+
})).digest("hex");
|
|
315413
|
+
return {
|
|
315414
|
+
table: "broker_account.user_asset_snapshots",
|
|
315415
|
+
row: {
|
|
315416
|
+
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
315417
|
+
source: tags.source,
|
|
315418
|
+
deployment_id: tags.deployment_id,
|
|
315419
|
+
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
315420
|
+
exchange: tags.exchange,
|
|
315421
|
+
account_selector: tags.account_selector,
|
|
315422
|
+
balance_scope: USER_ASSET_BALANCE_SCOPE,
|
|
315423
|
+
observation_id: observationId,
|
|
315424
|
+
reported_assets: userAssets.reportedAssets,
|
|
315425
|
+
incomplete_assets: userAssets.incompleteAssets,
|
|
315426
|
+
free_balances: userAssets.freeBalances,
|
|
315427
|
+
locked_balances: userAssets.lockedBalances,
|
|
315428
|
+
freeze_balances: userAssets.freezeBalances,
|
|
315429
|
+
withdrawing_balances: userAssets.withdrawingBalances,
|
|
315430
|
+
precision_basis: USER_ASSET_PRECISION_BASIS
|
|
315431
|
+
}
|
|
315432
|
+
};
|
|
315433
|
+
}
|
|
315434
|
+
function compactUndefined(record) {
|
|
315544
315435
|
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
315545
315436
|
}
|
|
315546
315437
|
function buildCommonArchiveTags(input) {
|
|
@@ -315557,7 +315448,7 @@ function buildOrderEventArchiveRow(input) {
|
|
|
315557
315448
|
const { tags, telemetry, action } = input;
|
|
315558
315449
|
return {
|
|
315559
315450
|
table: "broker_execution.order_events",
|
|
315560
|
-
row:
|
|
315451
|
+
row: compactUndefined({
|
|
315561
315452
|
...tags,
|
|
315562
315453
|
event_kind: input.eventKind ?? "execute_action",
|
|
315563
315454
|
action,
|
|
@@ -315594,35 +315485,35 @@ function buildSubscribeStreamArchiveRow(input) {
|
|
|
315594
315485
|
const info = asRecord(record?.info);
|
|
315595
315486
|
return {
|
|
315596
315487
|
table: "broker_execution.order_events",
|
|
315597
|
-
row:
|
|
315488
|
+
row: compactUndefined({
|
|
315598
315489
|
...input.tags,
|
|
315599
315490
|
event_kind: "subscribe_stream",
|
|
315600
315491
|
subscription_type: input.subscriptionType,
|
|
315601
|
-
order_id:
|
|
315602
|
-
client_order_id:
|
|
315603
|
-
status:
|
|
315492
|
+
order_id: firstString(record?.id, record?.orderId, record?.i, info?.orderId, info?.i),
|
|
315493
|
+
client_order_id: firstString(record?.clientOrderId, record?.clientOrderID, record?.c, info?.clientOrderId, info?.c),
|
|
315494
|
+
status: firstString(record?.status, record?.X, info?.status, info?.X)?.toLowerCase(),
|
|
315604
315495
|
payload_json: JSON.stringify(redactedPayload)
|
|
315605
315496
|
})
|
|
315606
315497
|
};
|
|
315607
315498
|
}
|
|
315608
315499
|
function quantityString(...values2) {
|
|
315609
|
-
return
|
|
315500
|
+
return firstString(...values2);
|
|
315610
315501
|
}
|
|
315611
315502
|
function extractBinanceInternalTransferId(response) {
|
|
315612
315503
|
const record = asRecord(response);
|
|
315613
|
-
return
|
|
315504
|
+
return firstString(record?.txnId, record?.tranId);
|
|
315614
315505
|
}
|
|
315615
315506
|
function buildTransferEventArchiveRow(input) {
|
|
315616
315507
|
const { tags, transfer } = input;
|
|
315617
315508
|
return {
|
|
315618
315509
|
table: "broker_execution.transfer_events",
|
|
315619
|
-
row:
|
|
315510
|
+
row: compactUndefined({
|
|
315620
315511
|
...tags,
|
|
315621
315512
|
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
315622
315513
|
event_kind: transfer.eventKind,
|
|
315623
315514
|
lifecycle_action: transfer.lifecycleAction,
|
|
315624
315515
|
status: transfer.status ?? "",
|
|
315625
|
-
asset_symbol: tags.symbol,
|
|
315516
|
+
asset_symbol: transfer.assetSymbol ?? tags.symbol,
|
|
315626
315517
|
amount: transfer.amount,
|
|
315627
315518
|
address: transfer.address,
|
|
315628
315519
|
network: transfer.network,
|
|
@@ -315642,18 +315533,18 @@ function normalizeCcxtTransactionForArchive(transaction) {
|
|
|
315642
315533
|
const record = asRecord(transaction);
|
|
315643
315534
|
const info = asRecord(record?.info);
|
|
315644
315535
|
const fee = asRecord(record?.fee);
|
|
315645
|
-
return
|
|
315646
|
-
externalId:
|
|
315647
|
-
clientWithdrawalId:
|
|
315648
|
-
txid:
|
|
315649
|
-
address:
|
|
315650
|
-
network:
|
|
315536
|
+
return compactUndefined({
|
|
315537
|
+
externalId: firstString(record?.id, info?.id, record?.txid, info?.txId),
|
|
315538
|
+
clientWithdrawalId: firstString(info?.withdrawOrderId),
|
|
315539
|
+
txid: firstString(record?.txid, info?.txId, info?.txid, info?.tx_hash),
|
|
315540
|
+
address: firstString(record?.address, record?.addressTo, info?.address),
|
|
315541
|
+
network: firstString(record?.network, info?.network),
|
|
315651
315542
|
amount: quantityString(info?.amount, record?.amount),
|
|
315652
|
-
assetSymbol:
|
|
315653
|
-
status:
|
|
315543
|
+
assetSymbol: firstString(record?.currency, info?.coin, info?.asset),
|
|
315544
|
+
status: firstString(record?.status, info?.status)?.toLowerCase(),
|
|
315654
315545
|
feeAmount: quantityString(fee?.cost, record?.feeCost),
|
|
315655
|
-
feeCurrency:
|
|
315656
|
-
exchangeTimestamp:
|
|
315546
|
+
feeCurrency: firstString(fee?.currency, record?.feeCurrency),
|
|
315547
|
+
exchangeTimestamp: normalizeTimestamp(firstValueForTransfer(record?.timestamp, record?.datetime, info?.applyTime))
|
|
315657
315548
|
});
|
|
315658
315549
|
}
|
|
315659
315550
|
function firstValueForTransfer(...values2) {
|
|
@@ -315663,7 +315554,7 @@ function buildFillEventArchiveRow(input) {
|
|
|
315663
315554
|
const { tags, fill } = input;
|
|
315664
315555
|
return {
|
|
315665
315556
|
table: "broker_execution.fill_events",
|
|
315666
|
-
row:
|
|
315557
|
+
row: compactUndefined({
|
|
315667
315558
|
...tags,
|
|
315668
315559
|
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
315669
315560
|
event_kind: FILL_EVENT_KIND,
|
|
@@ -315684,45 +315575,242 @@ function buildFillEventArchiveRow(input) {
|
|
|
315684
315575
|
})
|
|
315685
315576
|
};
|
|
315686
315577
|
}
|
|
315687
|
-
function normalizeCcxtTradeForArchive(trade) {
|
|
315688
|
-
const record = asRecord(trade);
|
|
315689
|
-
const info = asRecord(record?.info);
|
|
315690
|
-
const fee = asRecord(record?.fee);
|
|
315691
|
-
return {
|
|
315692
|
-
orderId:
|
|
315693
|
-
clientOrderId:
|
|
315694
|
-
fillId:
|
|
315695
|
-
side:
|
|
315696
|
-
orderType:
|
|
315697
|
-
price: quantityString(info?.price, record?.price),
|
|
315698
|
-
baseQuantity: quantityString(info?.qty, record?.amount),
|
|
315699
|
-
quoteQuantity: quantityString(info?.quoteQty, record?.cost),
|
|
315700
|
-
feeAmount: quantityString(fee?.cost, info?.commission),
|
|
315701
|
-
feeCurrency:
|
|
315702
|
-
feeRate: quantityString(fee?.rate),
|
|
315703
|
-
exchangeTimestamp:
|
|
315704
|
-
payload: trade
|
|
315705
|
-
};
|
|
315578
|
+
function normalizeCcxtTradeForArchive(trade) {
|
|
315579
|
+
const record = asRecord(trade);
|
|
315580
|
+
const info = asRecord(record?.info);
|
|
315581
|
+
const fee = asRecord(record?.fee);
|
|
315582
|
+
return {
|
|
315583
|
+
orderId: firstString(record?.order, info?.orderId, info?.orderID),
|
|
315584
|
+
clientOrderId: firstString(record?.clientOrderId, info?.clientOrderId, info?.origClientOrderId),
|
|
315585
|
+
fillId: firstString(record?.id, info?.id, info?.tradeId),
|
|
315586
|
+
side: firstString(record?.side, info?.side)?.toLowerCase(),
|
|
315587
|
+
orderType: firstString(record?.type, info?.type)?.toLowerCase(),
|
|
315588
|
+
price: quantityString(info?.price, record?.price),
|
|
315589
|
+
baseQuantity: quantityString(info?.qty, record?.amount),
|
|
315590
|
+
quoteQuantity: quantityString(info?.quoteQty, record?.cost),
|
|
315591
|
+
feeAmount: quantityString(fee?.cost, info?.commission),
|
|
315592
|
+
feeCurrency: firstString(fee?.currency, info?.commissionAsset),
|
|
315593
|
+
feeRate: quantityString(fee?.rate),
|
|
315594
|
+
exchangeTimestamp: normalizeTimestamp(firstValueForTransfer(record?.timestamp, record?.datetime, info?.time)),
|
|
315595
|
+
payload: trade
|
|
315596
|
+
};
|
|
315597
|
+
}
|
|
315598
|
+
function buildMarketMetadataSnapshotRow(input) {
|
|
315599
|
+
const redactedSnapshot = redactStreamPayload(input.marketSnapshot);
|
|
315600
|
+
const metadataHash = hashMarketMetadata(redactedSnapshot);
|
|
315601
|
+
return {
|
|
315602
|
+
table: "broker_execution.market_metadata_snapshots",
|
|
315603
|
+
row: compactUndefined({
|
|
315604
|
+
...input.tags,
|
|
315605
|
+
client_order_id: input.clientOrderId,
|
|
315606
|
+
order_id: input.orderId,
|
|
315607
|
+
maker_action_id: input.makerActionId,
|
|
315608
|
+
idempotency_id: input.idempotencyId,
|
|
315609
|
+
market_metadata_hash: metadataHash,
|
|
315610
|
+
snapshot_json: JSON.stringify(redactedSnapshot)
|
|
315611
|
+
})
|
|
315612
|
+
};
|
|
315613
|
+
}
|
|
315614
|
+
|
|
315615
|
+
// src/helpers/order-telemetry.ts
|
|
315616
|
+
var NUMERIC_METRICS = [
|
|
315617
|
+
["requestedQuantity", "cex_market_action_requested_quantity"],
|
|
315618
|
+
["requestedNotional", "cex_market_action_requested_notional"],
|
|
315619
|
+
["executedBaseQuantity", "cex_market_action_executed_base_quantity"],
|
|
315620
|
+
["executedQuoteQuantity", "cex_market_action_executed_quote_quantity"],
|
|
315621
|
+
["averageExecutionPrice", "cex_market_action_average_execution_price"],
|
|
315622
|
+
["filledAmount", "cex_market_action_filled_amount"],
|
|
315623
|
+
["remainingAmount", "cex_market_action_remaining_amount"],
|
|
315624
|
+
["feeAmount", "cex_market_action_fee_amount"],
|
|
315625
|
+
["feeRate", "cex_market_action_fee_rate"]
|
|
315626
|
+
];
|
|
315627
|
+
async function emitOrderExecutionTelemetry(otelMetrics, context2, order, error) {
|
|
315628
|
+
try {
|
|
315629
|
+
const telemetry = buildOrderExecutionTelemetry(context2, order, error);
|
|
315630
|
+
log.info("CEX market action execution telemetry", telemetry);
|
|
315631
|
+
const labels = {
|
|
315632
|
+
action: telemetry.action,
|
|
315633
|
+
cex: telemetry.cex,
|
|
315634
|
+
account: telemetry.accountLabel,
|
|
315635
|
+
symbol: telemetry.symbol,
|
|
315636
|
+
side: telemetry.side,
|
|
315637
|
+
order_type: telemetry.orderType,
|
|
315638
|
+
status: telemetry.status
|
|
315639
|
+
};
|
|
315640
|
+
await otelMetrics?.recordCounter("cex_market_action_executions_total", 1, {
|
|
315641
|
+
...labels,
|
|
315642
|
+
result: error ? "error" : "ok"
|
|
315643
|
+
});
|
|
315644
|
+
for (const [key, metricName] of NUMERIC_METRICS) {
|
|
315645
|
+
const value = telemetry[key];
|
|
315646
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
315647
|
+
await otelMetrics?.recordHistogram(metricName, value, labels);
|
|
315648
|
+
}
|
|
315649
|
+
}
|
|
315650
|
+
return telemetry;
|
|
315651
|
+
} catch (telemetryError) {
|
|
315652
|
+
try {
|
|
315653
|
+
log.error("Failed to emit CEX order telemetry", {
|
|
315654
|
+
error: telemetryError
|
|
315655
|
+
});
|
|
315656
|
+
} catch {}
|
|
315657
|
+
return;
|
|
315658
|
+
}
|
|
315659
|
+
}
|
|
315660
|
+
function buildOrderExecutionTelemetry(context2, order, error) {
|
|
315661
|
+
const record = asRecord(order);
|
|
315662
|
+
const info = asRecord(record?.info);
|
|
315663
|
+
const fees = getFees(record, info);
|
|
315664
|
+
const fee = summarizeFees(fees);
|
|
315665
|
+
const executedBaseQuantity = firstNumber2(record?.filled, info?.executedQty, info?.cumExecQty) ?? computeFilledFromAmount(record);
|
|
315666
|
+
const executedQuoteQuantity = firstNumber2(record?.cost, info?.cummulativeQuoteQty, info?.cumQuote, info?.cumExecValue);
|
|
315667
|
+
const averageExecutionPrice = firstNumber2(record?.average, info?.avgPrice) ?? computeAveragePrice(executedBaseQuantity, executedQuoteQuantity);
|
|
315668
|
+
const exchangeTimestamp = normalizeTimestamp(firstValue(record?.timestamp, info?.time, info?.transactTime, record?.datetime));
|
|
315669
|
+
const status = firstString2(record?.status, info?.status) ?? (error ? "failed" : "unknown");
|
|
315670
|
+
const errorRecord = error instanceof Error ? error : undefined;
|
|
315671
|
+
return compactUndefined2({
|
|
315672
|
+
event: "cex_market_action_execution",
|
|
315673
|
+
action: context2.action,
|
|
315674
|
+
cex: context2.cex.trim().toLowerCase() || "unknown",
|
|
315675
|
+
accountLabel: context2.accountLabel ?? "unknown",
|
|
315676
|
+
symbol: firstString2(record?.symbol, info?.symbol, context2.symbol) ?? "unknown",
|
|
315677
|
+
side: firstString2(record?.side, info?.side, context2.side) ?? "unknown",
|
|
315678
|
+
orderType: firstString2(record?.type, info?.type, context2.orderType) ?? "unknown",
|
|
315679
|
+
orderId: firstString2(record?.id, info?.orderId, info?.orderID),
|
|
315680
|
+
orderAuthor: context2.orderAuthor,
|
|
315681
|
+
clientOrderId: firstString2(context2.clientOrderId, record?.clientOrderId, record?.clientOrderID, record?.clientOid, info?.clientOrderId, info?.clientOrderID, info?.clientOid),
|
|
315682
|
+
idempotencyId: context2.idempotencyId,
|
|
315683
|
+
makerActionId: context2.makerActionId,
|
|
315684
|
+
status: status.toLowerCase(),
|
|
315685
|
+
requestedQuantity: context2.requestedQuantity ?? firstNumber2(record?.amount),
|
|
315686
|
+
requestedNotional: context2.requestedNotional,
|
|
315687
|
+
executedBaseQuantity,
|
|
315688
|
+
executedQuoteQuantity,
|
|
315689
|
+
averageExecutionPrice,
|
|
315690
|
+
filledAmount: firstNumber2(record?.filled, info?.executedQty),
|
|
315691
|
+
remainingAmount: firstNumber2(record?.remaining, info?.remainingQty),
|
|
315692
|
+
feeAmount: fee.amount,
|
|
315693
|
+
feeCurrency: fee.currency,
|
|
315694
|
+
feeRate: fee.rate,
|
|
315695
|
+
exchangeTimestamp,
|
|
315696
|
+
brokerObservedTimestamp: context2.brokerObservedTimestamp ?? new Date().toISOString(),
|
|
315697
|
+
errorType: errorRecord?.name,
|
|
315698
|
+
errorMessage: errorRecord ? REDACTED_ERROR_MESSAGE : undefined
|
|
315699
|
+
});
|
|
315700
|
+
}
|
|
315701
|
+
function emitOrderExecutionTelemetryInBackground(otelMetrics, context2, order, error) {
|
|
315702
|
+
emitOrderExecutionTelemetry(otelMetrics, context2, order, error).catch((telemetryError) => {
|
|
315703
|
+
try {
|
|
315704
|
+
log.warn("Telemetry emit failed", { error: telemetryError });
|
|
315705
|
+
} catch {
|
|
315706
|
+
console.warn("Telemetry emit failed", telemetryError);
|
|
315707
|
+
}
|
|
315708
|
+
});
|
|
315709
|
+
}
|
|
315710
|
+
function extractOrderTelemetryIds(params) {
|
|
315711
|
+
const record = params ?? {};
|
|
315712
|
+
return {
|
|
315713
|
+
clientOrderId: firstString2(record.clientOrderId, record.clientOrderID, record.newClientOrderId, record.clientOid),
|
|
315714
|
+
idempotencyId: firstString2(record.idempotencyId, record.idempotencyID, record.idempotencyKey, record.requestId, record.requestID),
|
|
315715
|
+
makerActionId: firstString2(record.makerActionId, record.maker_action_id, record.actionId, record.action_id)
|
|
315716
|
+
};
|
|
315717
|
+
}
|
|
315718
|
+
function firstValue(...values2) {
|
|
315719
|
+
return values2.find((value) => value !== undefined && value !== null);
|
|
315720
|
+
}
|
|
315721
|
+
function firstString2(...values2) {
|
|
315722
|
+
for (const value of values2) {
|
|
315723
|
+
if (typeof value === "string" && value.trim()) {
|
|
315724
|
+
return value.trim();
|
|
315725
|
+
}
|
|
315726
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
315727
|
+
return String(value);
|
|
315728
|
+
}
|
|
315729
|
+
}
|
|
315730
|
+
return;
|
|
315731
|
+
}
|
|
315732
|
+
function firstNumber2(...values2) {
|
|
315733
|
+
for (const value of values2) {
|
|
315734
|
+
const numberValue = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
|
315735
|
+
if (Number.isFinite(numberValue)) {
|
|
315736
|
+
return numberValue;
|
|
315737
|
+
}
|
|
315738
|
+
}
|
|
315739
|
+
return;
|
|
315740
|
+
}
|
|
315741
|
+
function computeFilledFromAmount(record) {
|
|
315742
|
+
const amount = firstNumber2(record?.amount);
|
|
315743
|
+
const remaining = firstNumber2(record?.remaining);
|
|
315744
|
+
if (amount === undefined || remaining === undefined) {
|
|
315745
|
+
return;
|
|
315746
|
+
}
|
|
315747
|
+
return amount - remaining;
|
|
315748
|
+
}
|
|
315749
|
+
function computeAveragePrice(executedBaseQuantity, executedQuoteQuantity) {
|
|
315750
|
+
if (executedBaseQuantity === undefined || executedQuoteQuantity === undefined || executedBaseQuantity === 0) {
|
|
315751
|
+
return;
|
|
315752
|
+
}
|
|
315753
|
+
return executedQuoteQuantity / executedBaseQuantity;
|
|
315754
|
+
}
|
|
315755
|
+
function getFees(record, info) {
|
|
315756
|
+
const fees = [];
|
|
315757
|
+
if (record?.fee)
|
|
315758
|
+
fees.push(record.fee);
|
|
315759
|
+
if (Array.isArray(record?.fees))
|
|
315760
|
+
fees.push(...record.fees);
|
|
315761
|
+
if (Array.isArray(record?.trades)) {
|
|
315762
|
+
for (const trade of record.trades) {
|
|
315763
|
+
const tradeRecord = asRecord(trade);
|
|
315764
|
+
if (tradeRecord?.fee)
|
|
315765
|
+
fees.push(tradeRecord.fee);
|
|
315766
|
+
if (Array.isArray(tradeRecord?.fees))
|
|
315767
|
+
fees.push(...tradeRecord.fees);
|
|
315768
|
+
}
|
|
315769
|
+
}
|
|
315770
|
+
if (Array.isArray(info?.fills)) {
|
|
315771
|
+
for (const fill of info.fills) {
|
|
315772
|
+
const fillRecord = asRecord(fill);
|
|
315773
|
+
const commission = firstNumber2(fillRecord?.commission);
|
|
315774
|
+
if (commission !== undefined) {
|
|
315775
|
+
fees.push({
|
|
315776
|
+
cost: commission,
|
|
315777
|
+
currency: firstString2(fillRecord?.commissionAsset)
|
|
315778
|
+
});
|
|
315779
|
+
}
|
|
315780
|
+
}
|
|
315781
|
+
}
|
|
315782
|
+
return fees;
|
|
315706
315783
|
}
|
|
315707
|
-
function
|
|
315708
|
-
|
|
315709
|
-
|
|
315784
|
+
function summarizeFees(fees) {
|
|
315785
|
+
let amount = 0;
|
|
315786
|
+
let amountFound = false;
|
|
315787
|
+
let currency;
|
|
315788
|
+
let rate;
|
|
315789
|
+
for (const rawFee of fees) {
|
|
315790
|
+
const fee = asRecord(rawFee);
|
|
315791
|
+
if (!fee)
|
|
315792
|
+
continue;
|
|
315793
|
+
const cost = firstNumber2(fee.cost, fee.amount);
|
|
315794
|
+
if (cost !== undefined) {
|
|
315795
|
+
amount += cost;
|
|
315796
|
+
amountFound = true;
|
|
315797
|
+
}
|
|
315798
|
+
currency ??= firstString2(fee.currency);
|
|
315799
|
+
rate ??= firstNumber2(fee.rate);
|
|
315800
|
+
}
|
|
315710
315801
|
return {
|
|
315711
|
-
|
|
315712
|
-
|
|
315713
|
-
|
|
315714
|
-
client_order_id: input.clientOrderId,
|
|
315715
|
-
order_id: input.orderId,
|
|
315716
|
-
maker_action_id: input.makerActionId,
|
|
315717
|
-
idempotency_id: input.idempotencyId,
|
|
315718
|
-
market_metadata_hash: metadataHash,
|
|
315719
|
-
snapshot_json: JSON.stringify(redactedSnapshot)
|
|
315720
|
-
})
|
|
315802
|
+
amount: amountFound ? amount : undefined,
|
|
315803
|
+
currency,
|
|
315804
|
+
rate
|
|
315721
315805
|
};
|
|
315722
315806
|
}
|
|
315807
|
+
function compactUndefined2(record) {
|
|
315808
|
+
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
315809
|
+
}
|
|
315723
315810
|
|
|
315724
315811
|
// src/helpers/broker-execution-archive/writer.ts
|
|
315725
315812
|
var import_api_logs2 = __toESM(require_src7(), 1);
|
|
315813
|
+
import { randomUUID } from "node:crypto";
|
|
315726
315814
|
import { closeSync, fsyncSync, openSync, writeSync } from "node:fs";
|
|
315727
315815
|
import {
|
|
315728
315816
|
request as httpRequest2
|
|
@@ -315754,6 +315842,7 @@ var DEFAULT_BATCH_SIZE = 10;
|
|
|
315754
315842
|
var DEFAULT_FLUSH_INTERVAL_MS = 1000;
|
|
315755
315843
|
var DEFAULT_FORWARDER_TIMEOUT_MS = 3000;
|
|
315756
315844
|
var SHED_WARN_INTERVAL_MS = 60000;
|
|
315845
|
+
var MAX_PINNED_BATCH_ATTEMPTS = 10;
|
|
315757
315846
|
var MARKET_FEEDS = new Set(["ORDERBOOK", "TICKER", "TRADES", "OHLCV"]);
|
|
315758
315847
|
function archiveFeed(row) {
|
|
315759
315848
|
const declared = row.row.feed ?? row.row.stream_type;
|
|
@@ -315806,6 +315895,7 @@ class BrokerExecutionArchiver {
|
|
|
315806
315895
|
flushed: 0,
|
|
315807
315896
|
forwarderFailures: 0
|
|
315808
315897
|
};
|
|
315898
|
+
pendingRetry = null;
|
|
315809
315899
|
flushTimer = null;
|
|
315810
315900
|
flushInFlight = null;
|
|
315811
315901
|
inFlightBatch = null;
|
|
@@ -315939,7 +316029,7 @@ class BrokerExecutionArchiver {
|
|
|
315939
316029
|
queueMicrotask(() => this.enqueue(row));
|
|
315940
316030
|
}
|
|
315941
316031
|
async flush() {
|
|
315942
|
-
if (!this.enabled || this.closed || this.queue.length === 0) {
|
|
316032
|
+
if (!this.enabled || this.closed || this.queue.length === 0 && this.pendingRetry === null) {
|
|
315943
316033
|
return;
|
|
315944
316034
|
}
|
|
315945
316035
|
if (this.flushInFlight) {
|
|
@@ -315969,16 +316059,18 @@ class BrokerExecutionArchiver {
|
|
|
315969
316059
|
}
|
|
315970
316060
|
let closeError;
|
|
315971
316061
|
try {
|
|
315972
|
-
while (this.queue.length > 0 || this.flushInFlight) {
|
|
316062
|
+
while (this.pendingRetry !== null || this.queue.length > 0 || this.flushInFlight) {
|
|
315973
316063
|
if (this.flushInFlight) {
|
|
315974
316064
|
await this.flushInFlight;
|
|
315975
316065
|
continue;
|
|
315976
316066
|
}
|
|
315977
|
-
|
|
315978
|
-
|
|
315979
|
-
|
|
315980
|
-
|
|
316067
|
+
if (!await this.flushBatch()) {
|
|
316068
|
+
const undelivered = [
|
|
316069
|
+
...this.pendingRetry?.rows ?? [],
|
|
316070
|
+
...this.queue
|
|
316071
|
+
];
|
|
315981
316072
|
this.appendLossRecords(undelivered, "shutdown_forwarder_failure");
|
|
316073
|
+
this.pendingRetry = null;
|
|
315982
316074
|
this.queue.length = 0;
|
|
315983
316075
|
break;
|
|
315984
316076
|
}
|
|
@@ -316002,7 +316094,7 @@ class BrokerExecutionArchiver {
|
|
|
316002
316094
|
return { ...this.stats };
|
|
316003
316095
|
}
|
|
316004
316096
|
getQueueDepth() {
|
|
316005
|
-
return this.queue.length;
|
|
316097
|
+
return this.queue.length + (this.pendingRetry?.rows.length ?? 0);
|
|
316006
316098
|
}
|
|
316007
316099
|
getHealthSnapshot() {
|
|
316008
316100
|
const now3 = Date.now();
|
|
@@ -316014,7 +316106,7 @@ class BrokerExecutionArchiver {
|
|
|
316014
316106
|
const healthy = this.enabled && !this.closing && !this.closed && this.queue.length < this.maxQueueSize && oldestPendingAgeMs <= this.forwarderTimeoutMs && inFlightAgeMs <= this.forwarderTimeoutMs && recoveredFromEvents;
|
|
316015
316107
|
return {
|
|
316016
316108
|
healthy,
|
|
316017
|
-
queue_depth: this.
|
|
316109
|
+
queue_depth: this.getQueueDepth(),
|
|
316018
316110
|
oldest_pending_age_ms: oldestPendingAgeMs,
|
|
316019
316111
|
shed_total: this.stats.shed,
|
|
316020
316112
|
last_failure_at: this.lastFailureAtMs,
|
|
@@ -316028,6 +316120,13 @@ class BrokerExecutionArchiver {
|
|
|
316028
316120
|
}
|
|
316029
316121
|
oldestPendingEnqueueAtMs() {
|
|
316030
316122
|
let oldest = null;
|
|
316123
|
+
for (const entry of this.pendingRetry?.rows ?? []) {
|
|
316124
|
+
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
316125
|
+
if (enqueuedAtMs === undefined) {
|
|
316126
|
+
continue;
|
|
316127
|
+
}
|
|
316128
|
+
oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
|
|
316129
|
+
}
|
|
316031
316130
|
for (const entry of this.queue) {
|
|
316032
316131
|
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
316033
316132
|
if (enqueuedAtMs === undefined) {
|
|
@@ -316090,28 +316189,6 @@ class BrokerExecutionArchiver {
|
|
|
316090
316189
|
this.deadLetterFd = undefined;
|
|
316091
316190
|
}
|
|
316092
316191
|
}
|
|
316093
|
-
enforceQueueBound() {
|
|
316094
|
-
while (this.queue.length > this.maxQueueSize) {
|
|
316095
|
-
const dropped = this.queue[0];
|
|
316096
|
-
if (!dropped) {
|
|
316097
|
-
return;
|
|
316098
|
-
}
|
|
316099
|
-
this.appendLossRecords([dropped], "queue_shed");
|
|
316100
|
-
this.queue.shift();
|
|
316101
|
-
this.stats.shed += 1;
|
|
316102
|
-
this.recordHealthEvent("shed");
|
|
316103
|
-
this.recordArchiveMetric("cex_archive_rows_shed_total", {
|
|
316104
|
-
table: dropped.table,
|
|
316105
|
-
source: this.source,
|
|
316106
|
-
feed: archiveFeed(dropped)
|
|
316107
|
-
});
|
|
316108
|
-
this.recordArchiveMetric("cex_archive_queue_saturated_rows_total", {
|
|
316109
|
-
table: dropped.table,
|
|
316110
|
-
source: this.source,
|
|
316111
|
-
feed: archiveFeed(dropped)
|
|
316112
|
-
});
|
|
316113
|
-
}
|
|
316114
|
-
}
|
|
316115
316192
|
appendLossRecords(rows, reason) {
|
|
316116
316193
|
if (rows.length === 0) {
|
|
316117
316194
|
return;
|
|
@@ -316156,10 +316233,12 @@ class BrokerExecutionArchiver {
|
|
|
316156
316233
|
}
|
|
316157
316234
|
}
|
|
316158
316235
|
async flushBatch() {
|
|
316159
|
-
const
|
|
316236
|
+
const pinned = this.pendingRetry;
|
|
316237
|
+
const batch = pinned ? pinned.rows : this.queue.splice(0, this.batchSize);
|
|
316160
316238
|
if (batch.length === 0) {
|
|
316161
316239
|
return true;
|
|
316162
316240
|
}
|
|
316241
|
+
const batchId = pinned?.batchId ?? randomUUID();
|
|
316163
316242
|
this.inFlightBatch = batch;
|
|
316164
316243
|
this.inFlightStartedAtMs = Date.now();
|
|
316165
316244
|
for (const entry of batch) {
|
|
@@ -316169,14 +316248,23 @@ class BrokerExecutionArchiver {
|
|
|
316169
316248
|
}
|
|
316170
316249
|
if (this.forwarderUrl) {
|
|
316171
316250
|
try {
|
|
316172
|
-
await this.postToForwarder(batch);
|
|
316251
|
+
await this.postToForwarder(batch, batchId);
|
|
316173
316252
|
} catch (error) {
|
|
316174
316253
|
this.stats.forwarderFailures += 1;
|
|
316175
316254
|
this.recordHealthEvent("failure");
|
|
316176
316255
|
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
316177
|
-
|
|
316256
|
+
const attempts = (pinned?.attempts ?? 0) + 1;
|
|
316178
316257
|
try {
|
|
316179
|
-
|
|
316258
|
+
if (attempts >= MAX_PINNED_BATCH_ATTEMPTS) {
|
|
316259
|
+
this.pendingRetry = null;
|
|
316260
|
+
this.appendLossRecords(batch, "retry_exhausted");
|
|
316261
|
+
log.warn("Broker execution archive gave up on a batch", {
|
|
316262
|
+
attempts,
|
|
316263
|
+
rows: batch.length
|
|
316264
|
+
});
|
|
316265
|
+
} else {
|
|
316266
|
+
this.pendingRetry = { batchId, rows: batch, attempts };
|
|
316267
|
+
}
|
|
316180
316268
|
} finally {
|
|
316181
316269
|
this.clearInFlightBatch(batch);
|
|
316182
316270
|
this.emitArchiveHealthMetrics();
|
|
@@ -316188,6 +316276,7 @@ class BrokerExecutionArchiver {
|
|
|
316188
316276
|
return false;
|
|
316189
316277
|
}
|
|
316190
316278
|
}
|
|
316279
|
+
this.pendingRetry = null;
|
|
316191
316280
|
this.stats.flushed += batch.length;
|
|
316192
316281
|
this.recordHealthEvent("success");
|
|
316193
316282
|
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
@@ -316251,13 +316340,14 @@ class BrokerExecutionArchiver {
|
|
|
316251
316340
|
log.warn("Broker execution archive OTLP emit failed", { error });
|
|
316252
316341
|
}
|
|
316253
316342
|
}
|
|
316254
|
-
postToForwarder(batch) {
|
|
316343
|
+
postToForwarder(batch, batchId) {
|
|
316255
316344
|
if (!this.forwarderUrl || batch.length === 0) {
|
|
316256
316345
|
return Promise.resolve();
|
|
316257
316346
|
}
|
|
316258
316347
|
const body = JSON.stringify({
|
|
316259
316348
|
source: this.source,
|
|
316260
316349
|
deployment_id: this.deploymentId,
|
|
316350
|
+
batch_id: batchId,
|
|
316261
316351
|
rows: batch
|
|
316262
316352
|
});
|
|
316263
316353
|
const url2 = new URL(this.forwarderUrl);
|
|
@@ -316869,6 +316959,94 @@ class AccountBalanceArchivePoller {
|
|
|
316869
316959
|
}
|
|
316870
316960
|
}
|
|
316871
316961
|
|
|
316962
|
+
// src/helpers/balance-update-archive-consumer.ts
|
|
316963
|
+
class BalanceUpdateArchiveConsumer {
|
|
316964
|
+
params;
|
|
316965
|
+
#started = false;
|
|
316966
|
+
#stopping = false;
|
|
316967
|
+
#subscriptions = new Set;
|
|
316968
|
+
#runs = [];
|
|
316969
|
+
constructor(params) {
|
|
316970
|
+
this.params = params;
|
|
316971
|
+
}
|
|
316972
|
+
start() {
|
|
316973
|
+
if (this.#started || this.#stopping)
|
|
316974
|
+
return;
|
|
316975
|
+
this.#started = true;
|
|
316976
|
+
const targets = this.#targets();
|
|
316977
|
+
log.info("\uD83D\uDCB8 Balance-update archive consumer started", {
|
|
316978
|
+
accounts: targets.length
|
|
316979
|
+
});
|
|
316980
|
+
for (const target of targets) {
|
|
316981
|
+
this.#runs.push(this.#consume(target));
|
|
316982
|
+
}
|
|
316983
|
+
}
|
|
316984
|
+
async stop() {
|
|
316985
|
+
this.#stopping = true;
|
|
316986
|
+
for (const subscription of [...this.#subscriptions]) {
|
|
316987
|
+
subscription.close();
|
|
316988
|
+
}
|
|
316989
|
+
await Promise.all(this.#runs);
|
|
316990
|
+
}
|
|
316991
|
+
#targets() {
|
|
316992
|
+
const targets = [];
|
|
316993
|
+
for (const [exchange, pool] of Object.entries(this.params.brokers)) {
|
|
316994
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
316995
|
+
targets.push({ exchange, accountSelector: account.label });
|
|
316996
|
+
}
|
|
316997
|
+
}
|
|
316998
|
+
return targets;
|
|
316999
|
+
}
|
|
317000
|
+
async#consume(target) {
|
|
317001
|
+
while (!this.#stopping) {
|
|
317002
|
+
let subscription;
|
|
317003
|
+
try {
|
|
317004
|
+
subscription = this.params.userDataStreamSupervisor.subscribe({
|
|
317005
|
+
exchange: target.exchange,
|
|
317006
|
+
accountSelector: target.accountSelector,
|
|
317007
|
+
kind: "balance"
|
|
317008
|
+
});
|
|
317009
|
+
this.#subscriptions.add(subscription);
|
|
317010
|
+
for await (const message of subscription) {
|
|
317011
|
+
if (this.#stopping)
|
|
317012
|
+
break;
|
|
317013
|
+
const event = message.event;
|
|
317014
|
+
if (event.e !== "balanceUpdate")
|
|
317015
|
+
continue;
|
|
317016
|
+
this.params.archiver.enqueue(buildTransferEventArchiveRow({
|
|
317017
|
+
tags: buildCommonArchiveTags({
|
|
317018
|
+
deploymentId: this.params.archiver.getDeploymentId(),
|
|
317019
|
+
accountSelector: target.accountSelector,
|
|
317020
|
+
exchange: target.exchange
|
|
317021
|
+
}),
|
|
317022
|
+
transfer: {
|
|
317023
|
+
eventKind: "balance_delta",
|
|
317024
|
+
lifecycleAction: "observe_balance_update",
|
|
317025
|
+
amount: typeof event.d === "string" ? event.d : undefined,
|
|
317026
|
+
assetSymbol: typeof event.a === "string" ? event.a : undefined,
|
|
317027
|
+
exchangeTimestamp: normalizeTimestamp(event.T),
|
|
317028
|
+
payload: event
|
|
317029
|
+
}
|
|
317030
|
+
}));
|
|
317031
|
+
}
|
|
317032
|
+
} catch (error) {
|
|
317033
|
+
if (!this.#stopping) {
|
|
317034
|
+
log.warn("Balance-update archive subscription failed", {
|
|
317035
|
+
exchange: target.exchange,
|
|
317036
|
+
accountSelector: target.accountSelector,
|
|
317037
|
+
errorType: error instanceof Error ? error.name : typeof error
|
|
317038
|
+
});
|
|
317039
|
+
}
|
|
317040
|
+
} finally {
|
|
317041
|
+
if (subscription) {
|
|
317042
|
+
this.#subscriptions.delete(subscription);
|
|
317043
|
+
subscription.close();
|
|
317044
|
+
}
|
|
317045
|
+
}
|
|
317046
|
+
}
|
|
317047
|
+
}
|
|
317048
|
+
}
|
|
317049
|
+
|
|
316872
317050
|
// src/helpers/deposit-archive-poller.ts
|
|
316873
317051
|
var DEFAULT_CONFIG2 = {
|
|
316874
317052
|
pollIntervalMs: 60000,
|
|
@@ -317078,7 +317256,7 @@ class DepositArchivePoller {
|
|
|
317078
317256
|
network: network === undefined ? undefined : String(network),
|
|
317079
317257
|
externalId: depositTxid,
|
|
317080
317258
|
txid: depositTxid,
|
|
317081
|
-
exchangeTimestamp:
|
|
317259
|
+
exchangeTimestamp: normalizeTimestamp(creditedAt),
|
|
317082
317260
|
payload: record
|
|
317083
317261
|
}
|
|
317084
317262
|
}));
|
|
@@ -317940,7 +318118,7 @@ function createOtelLogsFromEnv() {
|
|
|
317940
318118
|
}
|
|
317941
318119
|
|
|
317942
318120
|
// src/helpers/stream-health-publisher.ts
|
|
317943
|
-
import { createHash as createHash4, randomUUID } from "node:crypto";
|
|
318121
|
+
import { createHash as createHash4, randomUUID as randomUUID2 } from "node:crypto";
|
|
317944
318122
|
import {
|
|
317945
318123
|
closeSync as closeSync2,
|
|
317946
318124
|
fsyncSync as fsyncSync2,
|
|
@@ -318065,7 +318243,7 @@ class StreamHealthPublisher {
|
|
|
318065
318243
|
version: STATE_VERSION,
|
|
318066
318244
|
producerId: PRODUCER_ID,
|
|
318067
318245
|
producerEpoch: "1",
|
|
318068
|
-
runId:
|
|
318246
|
+
runId: randomUUID2(),
|
|
318069
318247
|
nextBatchSequence: "1",
|
|
318070
318248
|
nextStreamSequences: {}
|
|
318071
318249
|
};
|
|
@@ -318134,7 +318312,7 @@ class StreamHealthPublisher {
|
|
|
318134
318312
|
return;
|
|
318135
318313
|
if (this.#advanceRun) {
|
|
318136
318314
|
this.#state.producerEpoch = next(this.#state.producerEpoch);
|
|
318137
|
-
this.#state.runId =
|
|
318315
|
+
this.#state.runId = randomUUID2();
|
|
318138
318316
|
this.#state.nextBatchSequence = "1";
|
|
318139
318317
|
this.#state.nextStreamSequences = {};
|
|
318140
318318
|
this.#advanceRun = false;
|
|
@@ -318250,7 +318428,7 @@ class StreamHealthPublisher {
|
|
|
318250
318428
|
cause: error
|
|
318251
318429
|
});
|
|
318252
318430
|
}
|
|
318253
|
-
const temporary = `${this.#statePath}.${process.pid}.${
|
|
318431
|
+
const temporary = `${this.#statePath}.${process.pid}.${randomUUID2()}.tmp`;
|
|
318254
318432
|
let fd2;
|
|
318255
318433
|
try {
|
|
318256
318434
|
fd2 = openSync2(temporary, "wx", 384);
|
|
@@ -318295,6 +318473,143 @@ function streamHealthPublisherConfigFromEnv(env = process.env) {
|
|
|
318295
318473
|
};
|
|
318296
318474
|
}
|
|
318297
318475
|
|
|
318476
|
+
// src/helpers/user-asset-archive-poller.ts
|
|
318477
|
+
var DEFAULT_CONFIG4 = {
|
|
318478
|
+
pollIntervalMs: 60000
|
|
318479
|
+
};
|
|
318480
|
+
var USER_ASSET_EXCHANGE_ID = "binance";
|
|
318481
|
+
var metricLabels2 = (target) => ({
|
|
318482
|
+
exchange: target.exchangeId,
|
|
318483
|
+
account_selector: target.account.label,
|
|
318484
|
+
balance_scope: USER_ASSET_BALANCE_SCOPE
|
|
318485
|
+
});
|
|
318486
|
+
|
|
318487
|
+
class UserAssetArchivePoller {
|
|
318488
|
+
params;
|
|
318489
|
+
#timer = null;
|
|
318490
|
+
#stopped = false;
|
|
318491
|
+
#running = null;
|
|
318492
|
+
#lastSuccessMs = new Map;
|
|
318493
|
+
#config;
|
|
318494
|
+
constructor(params) {
|
|
318495
|
+
this.params = params;
|
|
318496
|
+
this.#config = { ...DEFAULT_CONFIG4, ...params.config };
|
|
318497
|
+
}
|
|
318498
|
+
start() {
|
|
318499
|
+
if (this.#timer || this.#stopped || !this.params.archiver.canPersistAccountBalanceSnapshots() || this.#targets().length === 0) {
|
|
318500
|
+
return;
|
|
318501
|
+
}
|
|
318502
|
+
log.info("\uD83E\uDDCA User asset archive poller started", {
|
|
318503
|
+
balanceScope: USER_ASSET_BALANCE_SCOPE
|
|
318504
|
+
});
|
|
318505
|
+
this.#schedule(0);
|
|
318506
|
+
}
|
|
318507
|
+
async stop() {
|
|
318508
|
+
this.#stopped = true;
|
|
318509
|
+
if (this.#timer) {
|
|
318510
|
+
clearTimeout(this.#timer);
|
|
318511
|
+
this.#timer = null;
|
|
318512
|
+
}
|
|
318513
|
+
await this.#running;
|
|
318514
|
+
}
|
|
318515
|
+
async pollAllOnce() {
|
|
318516
|
+
if (this.#stopped || this.#running || !this.params.archiver.canPersistAccountBalanceSnapshots()) {
|
|
318517
|
+
return false;
|
|
318518
|
+
}
|
|
318519
|
+
this.#running = this.#pollAllSequentially();
|
|
318520
|
+
try {
|
|
318521
|
+
return await this.#running;
|
|
318522
|
+
} finally {
|
|
318523
|
+
this.#running = null;
|
|
318524
|
+
}
|
|
318525
|
+
}
|
|
318526
|
+
#targets() {
|
|
318527
|
+
const targets = [];
|
|
318528
|
+
for (const [exchangeId, pool] of Object.entries(this.params.brokers)) {
|
|
318529
|
+
if (exchangeId.trim().toLowerCase() !== USER_ASSET_EXCHANGE_ID) {
|
|
318530
|
+
continue;
|
|
318531
|
+
}
|
|
318532
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
318533
|
+
targets.push({ exchangeId, account });
|
|
318534
|
+
}
|
|
318535
|
+
}
|
|
318536
|
+
return targets;
|
|
318537
|
+
}
|
|
318538
|
+
async#pollAllSequentially() {
|
|
318539
|
+
for (const target of this.#targets()) {
|
|
318540
|
+
if (this.#stopped) {
|
|
318541
|
+
break;
|
|
318542
|
+
}
|
|
318543
|
+
await this.#pollOne(target);
|
|
318544
|
+
}
|
|
318545
|
+
return true;
|
|
318546
|
+
}
|
|
318547
|
+
async#pollOne(target) {
|
|
318548
|
+
const labels = metricLabels2(target);
|
|
318549
|
+
this.params.metrics?.recordCounter("cex_user_asset_poll_attempts_total", 1, labels);
|
|
318550
|
+
try {
|
|
318551
|
+
const exchange = target.account.exchange;
|
|
318552
|
+
if (typeof exchange.sapiV3PostAssetGetUserAsset !== "function") {
|
|
318553
|
+
throw new Error("binance_user_asset_unavailable: getUserAsset is not defined on this exchange instance");
|
|
318554
|
+
}
|
|
318555
|
+
const response = await exchange.sapiV3PostAssetGetUserAsset({});
|
|
318556
|
+
const observedAt = new Date;
|
|
318557
|
+
const normalized = normalizeBinanceUserAssetsForArchive(response);
|
|
318558
|
+
this.params.archiver.enqueue(buildUserAssetSnapshotRow({
|
|
318559
|
+
tags: buildCommonArchiveTags({
|
|
318560
|
+
deploymentId: this.params.archiver.getDeploymentId(),
|
|
318561
|
+
accountSelector: target.account.label,
|
|
318562
|
+
exchange: target.exchangeId,
|
|
318563
|
+
brokerObservedTimestamp: observedAt.toISOString()
|
|
318564
|
+
}),
|
|
318565
|
+
userAssets: normalized
|
|
318566
|
+
}));
|
|
318567
|
+
const successMs = Date.now();
|
|
318568
|
+
this.#lastSuccessMs.set(this.#targetKey(target), successMs);
|
|
318569
|
+
this.params.metrics?.recordCounter("cex_user_asset_poll_successes_total", 1, labels);
|
|
318570
|
+
this.params.metrics?.recordGauge("cex_user_asset_poll_last_success_timestamp_seconds", Math.floor(successMs / 1000), labels);
|
|
318571
|
+
this.#recordFreshness(labels, successMs, successMs);
|
|
318572
|
+
} catch (error) {
|
|
318573
|
+
rethrowArchiveDurabilityError(error);
|
|
318574
|
+
this.params.metrics?.recordCounter("cex_user_asset_poll_failures_total", 1, labels);
|
|
318575
|
+
const now3 = Date.now();
|
|
318576
|
+
const lastSuccess = this.#lastSuccessMs.get(this.#targetKey(target));
|
|
318577
|
+
if (lastSuccess !== undefined) {
|
|
318578
|
+
this.#recordFreshness(labels, lastSuccess, now3);
|
|
318579
|
+
}
|
|
318580
|
+
log.warn("User asset archive poll failed", {
|
|
318581
|
+
exchange: target.exchangeId,
|
|
318582
|
+
account: target.account.label,
|
|
318583
|
+
balanceScope: USER_ASSET_BALANCE_SCOPE,
|
|
318584
|
+
errorType: error instanceof Error ? error.name : "unknown"
|
|
318585
|
+
});
|
|
318586
|
+
}
|
|
318587
|
+
}
|
|
318588
|
+
#recordFreshness(labels, lastSuccessMs, nowMs) {
|
|
318589
|
+
this.params.metrics?.recordGauge("cex_user_asset_poll_freshness_seconds", Math.max(0, (nowMs - lastSuccessMs) / 1000), labels);
|
|
318590
|
+
}
|
|
318591
|
+
#targetKey(target) {
|
|
318592
|
+
return `${target.exchangeId}|${target.account.label}|${USER_ASSET_BALANCE_SCOPE}`;
|
|
318593
|
+
}
|
|
318594
|
+
#schedule(delayMs) {
|
|
318595
|
+
this.#timer = setTimeout(() => void this.#tick(), delayMs);
|
|
318596
|
+
this.#timer.unref?.();
|
|
318597
|
+
}
|
|
318598
|
+
async#tick() {
|
|
318599
|
+
this.#timer = null;
|
|
318600
|
+
try {
|
|
318601
|
+
await this.pollAllOnce();
|
|
318602
|
+
} catch (error) {
|
|
318603
|
+
rethrowArchiveDurabilityError(error);
|
|
318604
|
+
log.error("User asset archive poller tick failed", error);
|
|
318605
|
+
} finally {
|
|
318606
|
+
if (!this.#stopped) {
|
|
318607
|
+
this.#schedule(this.#config.pollIntervalMs);
|
|
318608
|
+
}
|
|
318609
|
+
}
|
|
318610
|
+
}
|
|
318611
|
+
}
|
|
318612
|
+
|
|
318298
318613
|
// src/helpers/binance-user-data-stream.ts
|
|
318299
318614
|
import { Buffer as Buffer2 } from "node:buffer";
|
|
318300
318615
|
import { createHmac } from "node:crypto";
|
|
@@ -332822,7 +333137,7 @@ async function handleDeposit(ctx) {
|
|
|
332822
333137
|
network: depositNetwork?.exchangeNetworkId,
|
|
332823
333138
|
externalId: depositTxid,
|
|
332824
333139
|
txid: depositTxid,
|
|
332825
|
-
exchangeTimestamp:
|
|
333140
|
+
exchangeTimestamp: normalizeTimestamp(creditedAt),
|
|
332826
333141
|
payload: deposit
|
|
332827
333142
|
}
|
|
332828
333143
|
});
|
|
@@ -336340,6 +336655,8 @@ class CEXBroker {
|
|
|
336340
336655
|
fillArchivePoller;
|
|
336341
336656
|
depositArchivePoller;
|
|
336342
336657
|
accountBalanceArchivePoller;
|
|
336658
|
+
userAssetArchivePoller;
|
|
336659
|
+
balanceUpdateArchiveConsumer;
|
|
336343
336660
|
userDataStreamSupervisor;
|
|
336344
336661
|
loadEnvConfig() {
|
|
336345
336662
|
log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
|
|
@@ -336493,6 +336810,14 @@ class CEXBroker {
|
|
|
336493
336810
|
await this.accountBalanceArchivePoller.stop();
|
|
336494
336811
|
this.accountBalanceArchivePoller = undefined;
|
|
336495
336812
|
}
|
|
336813
|
+
if (this.userAssetArchivePoller) {
|
|
336814
|
+
await this.userAssetArchivePoller.stop();
|
|
336815
|
+
this.userAssetArchivePoller = undefined;
|
|
336816
|
+
}
|
|
336817
|
+
if (this.balanceUpdateArchiveConsumer) {
|
|
336818
|
+
await this.balanceUpdateArchiveConsumer.stop();
|
|
336819
|
+
this.balanceUpdateArchiveConsumer = undefined;
|
|
336820
|
+
}
|
|
336496
336821
|
if (this.server) {
|
|
336497
336822
|
await this.server.forceShutdown();
|
|
336498
336823
|
}
|
|
@@ -336538,6 +336863,14 @@ class CEXBroker {
|
|
|
336538
336863
|
await this.accountBalanceArchivePoller.stop();
|
|
336539
336864
|
this.accountBalanceArchivePoller = undefined;
|
|
336540
336865
|
}
|
|
336866
|
+
if (this.userAssetArchivePoller) {
|
|
336867
|
+
await this.userAssetArchivePoller.stop();
|
|
336868
|
+
this.userAssetArchivePoller = undefined;
|
|
336869
|
+
}
|
|
336870
|
+
if (this.balanceUpdateArchiveConsumer) {
|
|
336871
|
+
await this.balanceUpdateArchiveConsumer.stop();
|
|
336872
|
+
this.balanceUpdateArchiveConsumer = undefined;
|
|
336873
|
+
}
|
|
336541
336874
|
log.info(`Running CEXBroker at ${new Date().toISOString()}`);
|
|
336542
336875
|
if (this.otelMetrics?.isOtelEnabled()) {
|
|
336543
336876
|
await this.otelMetrics.initialize();
|
|
@@ -336579,6 +336912,14 @@ class CEXBroker {
|
|
|
336579
336912
|
metrics: this.otelMetrics
|
|
336580
336913
|
});
|
|
336581
336914
|
this.depositArchivePoller.start();
|
|
336915
|
+
if (this.userDataStreamSupervisor) {
|
|
336916
|
+
this.balanceUpdateArchiveConsumer = new BalanceUpdateArchiveConsumer({
|
|
336917
|
+
brokers: this.brokers,
|
|
336918
|
+
archiver: this.brokerArchiver,
|
|
336919
|
+
userDataStreamSupervisor: this.userDataStreamSupervisor
|
|
336920
|
+
});
|
|
336921
|
+
this.balanceUpdateArchiveConsumer.start();
|
|
336922
|
+
}
|
|
336582
336923
|
}
|
|
336583
336924
|
if (this.brokerArchiver?.canPersistAccountBalanceSnapshots()) {
|
|
336584
336925
|
this.accountBalanceArchivePoller = new AccountBalanceArchivePoller({
|
|
@@ -336587,6 +336928,12 @@ class CEXBroker {
|
|
|
336587
336928
|
metrics: this.otelMetrics
|
|
336588
336929
|
});
|
|
336589
336930
|
this.accountBalanceArchivePoller.start();
|
|
336931
|
+
this.userAssetArchivePoller = new UserAssetArchivePoller({
|
|
336932
|
+
brokers: this.brokers,
|
|
336933
|
+
archiver: this.brokerArchiver,
|
|
336934
|
+
metrics: this.otelMetrics
|
|
336935
|
+
});
|
|
336936
|
+
this.userAssetArchivePoller.start();
|
|
336590
336937
|
}
|
|
336591
336938
|
return this;
|
|
336592
336939
|
}
|