@usherlabs/cex-broker 0.2.40 → 0.2.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cli.js +928 -332
- package/dist/helpers/balance-update-archive-consumer.d.ts +16 -0
- package/dist/helpers/broker-execution-archive/index.d.ts +2 -2
- package/dist/helpers/broker-execution-archive/rows.d.ts +25 -0
- package/dist/helpers/broker-execution-archive/types.d.ts +5 -3
- package/dist/helpers/broker-execution-archive/writer.d.ts +31 -1
- package/dist/helpers/user-asset-archive-poller.d.ts +19 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +928 -332
- package/dist/index.js.map +11 -9
- package/package.json +1 -1
package/dist/commands/cli.js
CHANGED
|
@@ -6,25 +6,43 @@ var __getProtoOf = Object.getPrototypeOf;
|
|
|
6
6
|
var __defProp = Object.defineProperty;
|
|
7
7
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
8
8
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
function __accessProp(key) {
|
|
10
|
+
return this[key];
|
|
11
|
+
}
|
|
12
|
+
var __toESMCache_node;
|
|
13
|
+
var __toESMCache_esm;
|
|
9
14
|
var __toESM = (mod, isNodeMode, target) => {
|
|
15
|
+
var canCache = mod != null && typeof mod === "object";
|
|
16
|
+
if (canCache) {
|
|
17
|
+
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
18
|
+
var cached = cache.get(mod);
|
|
19
|
+
if (cached)
|
|
20
|
+
return cached;
|
|
21
|
+
}
|
|
10
22
|
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
11
23
|
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
12
24
|
for (let key of __getOwnPropNames(mod))
|
|
13
25
|
if (!__hasOwnProp.call(to, key))
|
|
14
26
|
__defProp(to, key, {
|
|
15
|
-
get: (
|
|
27
|
+
get: __accessProp.bind(mod, key),
|
|
16
28
|
enumerable: true
|
|
17
29
|
});
|
|
30
|
+
if (canCache)
|
|
31
|
+
cache.set(mod, to);
|
|
18
32
|
return to;
|
|
19
33
|
};
|
|
20
34
|
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
35
|
+
var __returnValue = (v) => v;
|
|
36
|
+
function __exportSetter(name, newValue) {
|
|
37
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
38
|
+
}
|
|
21
39
|
var __export = (target, all) => {
|
|
22
40
|
for (var name in all)
|
|
23
41
|
__defProp(target, name, {
|
|
24
42
|
get: all[name],
|
|
25
43
|
enumerable: true,
|
|
26
44
|
configurable: true,
|
|
27
|
-
set: (
|
|
45
|
+
set: __exportSetter.bind(all, name)
|
|
28
46
|
});
|
|
29
47
|
};
|
|
30
48
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
@@ -997,7 +1015,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
997
1015
|
this._exitCallback = (err) => {
|
|
998
1016
|
if (err.code !== "commander.executeSubCommandAsync") {
|
|
999
1017
|
throw err;
|
|
1000
|
-
}
|
|
1018
|
+
}
|
|
1001
1019
|
};
|
|
1002
1020
|
}
|
|
1003
1021
|
return this;
|
|
@@ -50030,7 +50048,7 @@ var require_tracestate_impl = __commonJS((exports) => {
|
|
|
50030
50048
|
const value = listMember.slice(i2 + 1, part.length);
|
|
50031
50049
|
if ((0, tracestate_validators_1.validateKey)(key) && (0, tracestate_validators_1.validateValue)(value)) {
|
|
50032
50050
|
agg.set(key, value);
|
|
50033
|
-
}
|
|
50051
|
+
}
|
|
50034
50052
|
}
|
|
50035
50053
|
return agg;
|
|
50036
50054
|
}, new Map);
|
|
@@ -59163,7 +59181,7 @@ var require_TraceState = __commonJS((exports) => {
|
|
|
59163
59181
|
const value = listMember.slice(i2 + 1, part.length);
|
|
59164
59182
|
if ((0, validators_1.validateKey)(key) && (0, validators_1.validateValue)(value)) {
|
|
59165
59183
|
agg.set(key, value);
|
|
59166
|
-
}
|
|
59184
|
+
}
|
|
59167
59185
|
}
|
|
59168
59186
|
return agg;
|
|
59169
59187
|
}, new Map);
|
|
@@ -314993,6 +315011,20 @@ function validateDeposit(policy, exchange, network, ticker) {
|
|
|
314993
315011
|
return { valid: true };
|
|
314994
315012
|
}
|
|
314995
315013
|
|
|
315014
|
+
// src/helpers/broker-execution-archive/rows.ts
|
|
315015
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
315016
|
+
|
|
315017
|
+
// src/helpers/shared/guards.ts
|
|
315018
|
+
function isRecord(value) {
|
|
315019
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
315020
|
+
}
|
|
315021
|
+
function asRecord(value) {
|
|
315022
|
+
return isRecord(value) ? value : undefined;
|
|
315023
|
+
}
|
|
315024
|
+
|
|
315025
|
+
// src/helpers/broker-execution-archive/redact.ts
|
|
315026
|
+
import { createHash } from "node:crypto";
|
|
315027
|
+
|
|
314996
315028
|
// src/helpers/shared/errors.ts
|
|
314997
315029
|
var MAX_ERROR_DETAIL_LENGTH = 512;
|
|
314998
315030
|
var REDACTED_ERROR_MESSAGE = "redacted_error";
|
|
@@ -315052,226 +315084,7 @@ function safeLogRedactedError(context2, error) {
|
|
|
315052
315084
|
}
|
|
315053
315085
|
}
|
|
315054
315086
|
|
|
315055
|
-
// src/helpers/shared/guards.ts
|
|
315056
|
-
function isRecord(value) {
|
|
315057
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
315058
|
-
}
|
|
315059
|
-
function asRecord(value) {
|
|
315060
|
-
return isRecord(value) ? value : undefined;
|
|
315061
|
-
}
|
|
315062
|
-
|
|
315063
|
-
// src/helpers/order-telemetry.ts
|
|
315064
|
-
var NUMERIC_METRICS = [
|
|
315065
|
-
["requestedQuantity", "cex_market_action_requested_quantity"],
|
|
315066
|
-
["requestedNotional", "cex_market_action_requested_notional"],
|
|
315067
|
-
["executedBaseQuantity", "cex_market_action_executed_base_quantity"],
|
|
315068
|
-
["executedQuoteQuantity", "cex_market_action_executed_quote_quantity"],
|
|
315069
|
-
["averageExecutionPrice", "cex_market_action_average_execution_price"],
|
|
315070
|
-
["filledAmount", "cex_market_action_filled_amount"],
|
|
315071
|
-
["remainingAmount", "cex_market_action_remaining_amount"],
|
|
315072
|
-
["feeAmount", "cex_market_action_fee_amount"],
|
|
315073
|
-
["feeRate", "cex_market_action_fee_rate"]
|
|
315074
|
-
];
|
|
315075
|
-
async function emitOrderExecutionTelemetry(otelMetrics, context2, order, error) {
|
|
315076
|
-
try {
|
|
315077
|
-
const telemetry = buildOrderExecutionTelemetry(context2, order, error);
|
|
315078
|
-
log.info("CEX market action execution telemetry", telemetry);
|
|
315079
|
-
const labels = {
|
|
315080
|
-
action: telemetry.action,
|
|
315081
|
-
cex: telemetry.cex,
|
|
315082
|
-
account: telemetry.accountLabel,
|
|
315083
|
-
symbol: telemetry.symbol,
|
|
315084
|
-
side: telemetry.side,
|
|
315085
|
-
order_type: telemetry.orderType,
|
|
315086
|
-
status: telemetry.status
|
|
315087
|
-
};
|
|
315088
|
-
await otelMetrics?.recordCounter("cex_market_action_executions_total", 1, {
|
|
315089
|
-
...labels,
|
|
315090
|
-
result: error ? "error" : "ok"
|
|
315091
|
-
});
|
|
315092
|
-
for (const [key, metricName] of NUMERIC_METRICS) {
|
|
315093
|
-
const value = telemetry[key];
|
|
315094
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
315095
|
-
await otelMetrics?.recordHistogram(metricName, value, labels);
|
|
315096
|
-
}
|
|
315097
|
-
}
|
|
315098
|
-
return telemetry;
|
|
315099
|
-
} catch (telemetryError) {
|
|
315100
|
-
try {
|
|
315101
|
-
log.error("Failed to emit CEX order telemetry", {
|
|
315102
|
-
error: telemetryError
|
|
315103
|
-
});
|
|
315104
|
-
} catch {}
|
|
315105
|
-
return;
|
|
315106
|
-
}
|
|
315107
|
-
}
|
|
315108
|
-
function buildOrderExecutionTelemetry(context2, order, error) {
|
|
315109
|
-
const record = asRecord(order);
|
|
315110
|
-
const info = asRecord(record?.info);
|
|
315111
|
-
const fees = getFees(record, info);
|
|
315112
|
-
const fee = summarizeFees(fees);
|
|
315113
|
-
const executedBaseQuantity = firstNumber(record?.filled, info?.executedQty, info?.cumExecQty) ?? computeFilledFromAmount(record);
|
|
315114
|
-
const executedQuoteQuantity = firstNumber(record?.cost, info?.cummulativeQuoteQty, info?.cumQuote, info?.cumExecValue);
|
|
315115
|
-
const averageExecutionPrice = firstNumber(record?.average, info?.avgPrice) ?? computeAveragePrice(executedBaseQuantity, executedQuoteQuantity);
|
|
315116
|
-
const exchangeTimestamp = normalizeTimestamp(firstValue(record?.timestamp, info?.time, info?.transactTime, record?.datetime));
|
|
315117
|
-
const status = firstString(record?.status, info?.status) ?? (error ? "failed" : "unknown");
|
|
315118
|
-
const errorRecord = error instanceof Error ? error : undefined;
|
|
315119
|
-
return compactUndefined({
|
|
315120
|
-
event: "cex_market_action_execution",
|
|
315121
|
-
action: context2.action,
|
|
315122
|
-
cex: context2.cex.trim().toLowerCase() || "unknown",
|
|
315123
|
-
accountLabel: context2.accountLabel ?? "unknown",
|
|
315124
|
-
symbol: firstString(record?.symbol, info?.symbol, context2.symbol) ?? "unknown",
|
|
315125
|
-
side: firstString(record?.side, info?.side, context2.side) ?? "unknown",
|
|
315126
|
-
orderType: firstString(record?.type, info?.type, context2.orderType) ?? "unknown",
|
|
315127
|
-
orderId: firstString(record?.id, info?.orderId, info?.orderID),
|
|
315128
|
-
orderAuthor: context2.orderAuthor,
|
|
315129
|
-
clientOrderId: firstString(context2.clientOrderId, record?.clientOrderId, record?.clientOrderID, record?.clientOid, info?.clientOrderId, info?.clientOrderID, info?.clientOid),
|
|
315130
|
-
idempotencyId: context2.idempotencyId,
|
|
315131
|
-
makerActionId: context2.makerActionId,
|
|
315132
|
-
status: status.toLowerCase(),
|
|
315133
|
-
requestedQuantity: context2.requestedQuantity ?? firstNumber(record?.amount),
|
|
315134
|
-
requestedNotional: context2.requestedNotional,
|
|
315135
|
-
executedBaseQuantity,
|
|
315136
|
-
executedQuoteQuantity,
|
|
315137
|
-
averageExecutionPrice,
|
|
315138
|
-
filledAmount: firstNumber(record?.filled, info?.executedQty),
|
|
315139
|
-
remainingAmount: firstNumber(record?.remaining, info?.remainingQty),
|
|
315140
|
-
feeAmount: fee.amount,
|
|
315141
|
-
feeCurrency: fee.currency,
|
|
315142
|
-
feeRate: fee.rate,
|
|
315143
|
-
exchangeTimestamp,
|
|
315144
|
-
brokerObservedTimestamp: context2.brokerObservedTimestamp ?? new Date().toISOString(),
|
|
315145
|
-
errorType: errorRecord?.name,
|
|
315146
|
-
errorMessage: errorRecord ? REDACTED_ERROR_MESSAGE : undefined
|
|
315147
|
-
});
|
|
315148
|
-
}
|
|
315149
|
-
function emitOrderExecutionTelemetryInBackground(otelMetrics, context2, order, error) {
|
|
315150
|
-
emitOrderExecutionTelemetry(otelMetrics, context2, order, error).catch((telemetryError) => {
|
|
315151
|
-
try {
|
|
315152
|
-
log.warn("Telemetry emit failed", { error: telemetryError });
|
|
315153
|
-
} catch {
|
|
315154
|
-
console.warn("Telemetry emit failed", telemetryError);
|
|
315155
|
-
}
|
|
315156
|
-
});
|
|
315157
|
-
}
|
|
315158
|
-
function extractOrderTelemetryIds(params) {
|
|
315159
|
-
const record = params ?? {};
|
|
315160
|
-
return {
|
|
315161
|
-
clientOrderId: firstString(record.clientOrderId, record.clientOrderID, record.newClientOrderId, record.clientOid),
|
|
315162
|
-
idempotencyId: firstString(record.idempotencyId, record.idempotencyID, record.idempotencyKey, record.requestId, record.requestID),
|
|
315163
|
-
makerActionId: firstString(record.makerActionId, record.maker_action_id, record.actionId, record.action_id)
|
|
315164
|
-
};
|
|
315165
|
-
}
|
|
315166
|
-
function firstValue(...values2) {
|
|
315167
|
-
return values2.find((value) => value !== undefined && value !== null);
|
|
315168
|
-
}
|
|
315169
|
-
function firstString(...values2) {
|
|
315170
|
-
for (const value of values2) {
|
|
315171
|
-
if (typeof value === "string" && value.trim()) {
|
|
315172
|
-
return value.trim();
|
|
315173
|
-
}
|
|
315174
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
315175
|
-
return String(value);
|
|
315176
|
-
}
|
|
315177
|
-
}
|
|
315178
|
-
return;
|
|
315179
|
-
}
|
|
315180
|
-
function firstNumber(...values2) {
|
|
315181
|
-
for (const value of values2) {
|
|
315182
|
-
const numberValue = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
|
315183
|
-
if (Number.isFinite(numberValue)) {
|
|
315184
|
-
return numberValue;
|
|
315185
|
-
}
|
|
315186
|
-
}
|
|
315187
|
-
return;
|
|
315188
|
-
}
|
|
315189
|
-
function computeFilledFromAmount(record) {
|
|
315190
|
-
const amount = firstNumber(record?.amount);
|
|
315191
|
-
const remaining = firstNumber(record?.remaining);
|
|
315192
|
-
if (amount === undefined || remaining === undefined) {
|
|
315193
|
-
return;
|
|
315194
|
-
}
|
|
315195
|
-
return amount - remaining;
|
|
315196
|
-
}
|
|
315197
|
-
function computeAveragePrice(executedBaseQuantity, executedQuoteQuantity) {
|
|
315198
|
-
if (executedBaseQuantity === undefined || executedQuoteQuantity === undefined || executedBaseQuantity === 0) {
|
|
315199
|
-
return;
|
|
315200
|
-
}
|
|
315201
|
-
return executedQuoteQuantity / executedBaseQuantity;
|
|
315202
|
-
}
|
|
315203
|
-
function normalizeTimestamp(value) {
|
|
315204
|
-
if (typeof value === "string" && value.trim()) {
|
|
315205
|
-
return value.trim();
|
|
315206
|
-
}
|
|
315207
|
-
const timestamp = firstNumber(value);
|
|
315208
|
-
if (timestamp === undefined) {
|
|
315209
|
-
return;
|
|
315210
|
-
}
|
|
315211
|
-
const date = new Date(timestamp);
|
|
315212
|
-
return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
|
|
315213
|
-
}
|
|
315214
|
-
function getFees(record, info) {
|
|
315215
|
-
const fees = [];
|
|
315216
|
-
if (record?.fee)
|
|
315217
|
-
fees.push(record.fee);
|
|
315218
|
-
if (Array.isArray(record?.fees))
|
|
315219
|
-
fees.push(...record.fees);
|
|
315220
|
-
if (Array.isArray(record?.trades)) {
|
|
315221
|
-
for (const trade of record.trades) {
|
|
315222
|
-
const tradeRecord = asRecord(trade);
|
|
315223
|
-
if (tradeRecord?.fee)
|
|
315224
|
-
fees.push(tradeRecord.fee);
|
|
315225
|
-
if (Array.isArray(tradeRecord?.fees))
|
|
315226
|
-
fees.push(...tradeRecord.fees);
|
|
315227
|
-
}
|
|
315228
|
-
}
|
|
315229
|
-
if (Array.isArray(info?.fills)) {
|
|
315230
|
-
for (const fill of info.fills) {
|
|
315231
|
-
const fillRecord = asRecord(fill);
|
|
315232
|
-
const commission = firstNumber(fillRecord?.commission);
|
|
315233
|
-
if (commission !== undefined) {
|
|
315234
|
-
fees.push({
|
|
315235
|
-
cost: commission,
|
|
315236
|
-
currency: firstString(fillRecord?.commissionAsset)
|
|
315237
|
-
});
|
|
315238
|
-
}
|
|
315239
|
-
}
|
|
315240
|
-
}
|
|
315241
|
-
return fees;
|
|
315242
|
-
}
|
|
315243
|
-
function summarizeFees(fees) {
|
|
315244
|
-
let amount = 0;
|
|
315245
|
-
let amountFound = false;
|
|
315246
|
-
let currency;
|
|
315247
|
-
let rate;
|
|
315248
|
-
for (const rawFee of fees) {
|
|
315249
|
-
const fee = asRecord(rawFee);
|
|
315250
|
-
if (!fee)
|
|
315251
|
-
continue;
|
|
315252
|
-
const cost = firstNumber(fee.cost, fee.amount);
|
|
315253
|
-
if (cost !== undefined) {
|
|
315254
|
-
amount += cost;
|
|
315255
|
-
amountFound = true;
|
|
315256
|
-
}
|
|
315257
|
-
currency ??= firstString(fee.currency);
|
|
315258
|
-
rate ??= firstNumber(fee.rate);
|
|
315259
|
-
}
|
|
315260
|
-
return {
|
|
315261
|
-
amount: amountFound ? amount : undefined,
|
|
315262
|
-
currency,
|
|
315263
|
-
rate
|
|
315264
|
-
};
|
|
315265
|
-
}
|
|
315266
|
-
function compactUndefined(record) {
|
|
315267
|
-
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
315268
|
-
}
|
|
315269
|
-
|
|
315270
|
-
// src/helpers/broker-execution-archive/rows.ts
|
|
315271
|
-
import { createHash as createHash2 } from "node:crypto";
|
|
315272
|
-
|
|
315273
315087
|
// src/helpers/broker-execution-archive/redact.ts
|
|
315274
|
-
import { createHash } from "node:crypto";
|
|
315275
315088
|
var SECRET_KEY_PATTERN = /\b(api[_-]?key|api[_-]?secret|secret|signature|passphrase|password|token|credential)\b/i;
|
|
315276
315089
|
var SECRET_VALUE_PATTERN = /(\b(?:apiKey|api_key|apiSecret|api_secret|secret|signature|passphrase|password|token)\b\s*[=:]\s*)[^\s&,;)}\]]+/gi;
|
|
315277
315090
|
var SECRET_JSON_PATTERN = /("(?:apiKey|api_key|apiSecret|api_secret|secret|signature|passphrase|password|token)"\s*:\s*")[^"]*(")/gi;
|
|
@@ -315344,9 +315157,11 @@ var ARCHIVE_SCHEMA_VERSION = "1";
|
|
|
315344
315157
|
var FILL_EVENT_KIND = "trade_history_fill";
|
|
315345
315158
|
var ACCOUNT_BALANCE_SCOPE = "spot";
|
|
315346
315159
|
var ACCOUNT_BALANCE_PRECISION_BASIS = "ccxt_normalized_number";
|
|
315160
|
+
var USER_ASSET_BALANCE_SCOPE = "user_asset";
|
|
315161
|
+
var USER_ASSET_PRECISION_BASIS = "venue_raw_string";
|
|
315347
315162
|
|
|
315348
315163
|
// src/helpers/broker-execution-archive/rows.ts
|
|
315349
|
-
function
|
|
315164
|
+
function firstString(...values2) {
|
|
315350
315165
|
for (const value of values2) {
|
|
315351
315166
|
if (typeof value === "string" && value.trim()) {
|
|
315352
315167
|
return value.trim();
|
|
@@ -315357,7 +315172,7 @@ function firstString2(...values2) {
|
|
|
315357
315172
|
}
|
|
315358
315173
|
return;
|
|
315359
315174
|
}
|
|
315360
|
-
function
|
|
315175
|
+
function firstNumber(...values2) {
|
|
315361
315176
|
for (const value of values2) {
|
|
315362
315177
|
const numeric = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
|
315363
315178
|
if (Number.isFinite(numeric)) {
|
|
@@ -315366,11 +315181,11 @@ function firstNumber2(...values2) {
|
|
|
315366
315181
|
}
|
|
315367
315182
|
return;
|
|
315368
315183
|
}
|
|
315369
|
-
function
|
|
315184
|
+
function normalizeTimestamp(value) {
|
|
315370
315185
|
if (typeof value === "string" && value.trim()) {
|
|
315371
315186
|
return value.trim();
|
|
315372
315187
|
}
|
|
315373
|
-
const ms =
|
|
315188
|
+
const ms = firstNumber(value);
|
|
315374
315189
|
if (ms === undefined) {
|
|
315375
315190
|
return;
|
|
315376
315191
|
}
|
|
@@ -315477,7 +315292,7 @@ function normalizeCcxtBalanceForArchive(balance) {
|
|
|
315477
315292
|
}
|
|
315478
315293
|
}
|
|
315479
315294
|
return {
|
|
315480
|
-
exchangeTimestamp:
|
|
315295
|
+
exchangeTimestamp: normalizeTimestamp(record.timestamp ?? record.datetime),
|
|
315481
315296
|
reportedAssets: [...reportedAssets].sort(),
|
|
315482
315297
|
assetEntryAssets: [...assetEntryAssets].sort(),
|
|
315483
315298
|
freeBalances: sortedBalanceMap(maps.free),
|
|
@@ -315500,7 +315315,7 @@ function buildAccountBalanceSnapshotRow(input) {
|
|
|
315500
315315
|
})).digest("hex");
|
|
315501
315316
|
return {
|
|
315502
315317
|
table: "broker_account.balance_snapshots",
|
|
315503
|
-
row:
|
|
315318
|
+
row: compactUndefined({
|
|
315504
315319
|
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
315505
315320
|
exchange_timestamp: balance.exchangeTimestamp,
|
|
315506
315321
|
source: tags.source,
|
|
@@ -315522,7 +315337,101 @@ function buildAccountBalanceSnapshotRow(input) {
|
|
|
315522
315337
|
})
|
|
315523
315338
|
};
|
|
315524
315339
|
}
|
|
315525
|
-
|
|
315340
|
+
var USER_ASSET_BUCKETS = {
|
|
315341
|
+
freeBalances: "free",
|
|
315342
|
+
lockedBalances: "locked",
|
|
315343
|
+
freezeBalances: "freeze",
|
|
315344
|
+
withdrawingBalances: "withdrawing"
|
|
315345
|
+
};
|
|
315346
|
+
function userAssetQuantity(value) {
|
|
315347
|
+
if (typeof value === "number") {
|
|
315348
|
+
return decimalString(value);
|
|
315349
|
+
}
|
|
315350
|
+
if (typeof value !== "string") {
|
|
315351
|
+
return;
|
|
315352
|
+
}
|
|
315353
|
+
const trimmed = value.trim();
|
|
315354
|
+
if (!trimmed || !Number.isFinite(Number(trimmed))) {
|
|
315355
|
+
return;
|
|
315356
|
+
}
|
|
315357
|
+
return trimmed;
|
|
315358
|
+
}
|
|
315359
|
+
function normalizeBinanceUserAssetsForArchive(response) {
|
|
315360
|
+
if (!Array.isArray(response)) {
|
|
315361
|
+
throw new Error("binance_user_asset_malformed_response: getUserAsset did not return an array");
|
|
315362
|
+
}
|
|
315363
|
+
const reportedAssets = [];
|
|
315364
|
+
const incompleteAssets = [];
|
|
315365
|
+
const maps = {
|
|
315366
|
+
freeBalances: {},
|
|
315367
|
+
lockedBalances: {},
|
|
315368
|
+
freezeBalances: {},
|
|
315369
|
+
withdrawingBalances: {}
|
|
315370
|
+
};
|
|
315371
|
+
for (const entry of response) {
|
|
315372
|
+
const record = asRecord(entry);
|
|
315373
|
+
const asset = typeof record?.asset === "string" ? record.asset.trim() : undefined;
|
|
315374
|
+
if (!record || !asset) {
|
|
315375
|
+
throw new Error("binance_user_asset_malformed_response: getUserAsset entry has no asset key");
|
|
315376
|
+
}
|
|
315377
|
+
if (reportedAssets.includes(asset)) {
|
|
315378
|
+
throw new Error(`binance_user_asset_malformed_response: getUserAsset returned duplicate entries for ${asset}`);
|
|
315379
|
+
}
|
|
315380
|
+
reportedAssets.push(asset);
|
|
315381
|
+
let complete = true;
|
|
315382
|
+
for (const [field, venueKey] of Object.entries(USER_ASSET_BUCKETS)) {
|
|
315383
|
+
const quantity = userAssetQuantity(record[venueKey]);
|
|
315384
|
+
if (quantity === undefined) {
|
|
315385
|
+
complete = false;
|
|
315386
|
+
continue;
|
|
315387
|
+
}
|
|
315388
|
+
maps[field][asset] = quantity;
|
|
315389
|
+
}
|
|
315390
|
+
if (!complete) {
|
|
315391
|
+
incompleteAssets.push(asset);
|
|
315392
|
+
}
|
|
315393
|
+
}
|
|
315394
|
+
return {
|
|
315395
|
+
reportedAssets: reportedAssets.sort(),
|
|
315396
|
+
incompleteAssets: incompleteAssets.sort(),
|
|
315397
|
+
freeBalances: sortedBalanceMap(maps.freeBalances),
|
|
315398
|
+
lockedBalances: sortedBalanceMap(maps.lockedBalances),
|
|
315399
|
+
freezeBalances: sortedBalanceMap(maps.freezeBalances),
|
|
315400
|
+
withdrawingBalances: sortedBalanceMap(maps.withdrawingBalances)
|
|
315401
|
+
};
|
|
315402
|
+
}
|
|
315403
|
+
function buildUserAssetSnapshotRow(input) {
|
|
315404
|
+
const { tags, userAssets } = input;
|
|
315405
|
+
const observationId = createHash2("sha256").update(JSON.stringify({
|
|
315406
|
+
deployment_id: tags.deployment_id,
|
|
315407
|
+
exchange: tags.exchange,
|
|
315408
|
+
account_selector: tags.account_selector,
|
|
315409
|
+
balance_scope: USER_ASSET_BALANCE_SCOPE,
|
|
315410
|
+
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
315411
|
+
userAssets
|
|
315412
|
+
})).digest("hex");
|
|
315413
|
+
return {
|
|
315414
|
+
table: "broker_account.user_asset_snapshots",
|
|
315415
|
+
row: {
|
|
315416
|
+
broker_observed_timestamp: tags.broker_observed_timestamp,
|
|
315417
|
+
source: tags.source,
|
|
315418
|
+
deployment_id: tags.deployment_id,
|
|
315419
|
+
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
315420
|
+
exchange: tags.exchange,
|
|
315421
|
+
account_selector: tags.account_selector,
|
|
315422
|
+
balance_scope: USER_ASSET_BALANCE_SCOPE,
|
|
315423
|
+
observation_id: observationId,
|
|
315424
|
+
reported_assets: userAssets.reportedAssets,
|
|
315425
|
+
incomplete_assets: userAssets.incompleteAssets,
|
|
315426
|
+
free_balances: userAssets.freeBalances,
|
|
315427
|
+
locked_balances: userAssets.lockedBalances,
|
|
315428
|
+
freeze_balances: userAssets.freezeBalances,
|
|
315429
|
+
withdrawing_balances: userAssets.withdrawingBalances,
|
|
315430
|
+
precision_basis: USER_ASSET_PRECISION_BASIS
|
|
315431
|
+
}
|
|
315432
|
+
};
|
|
315433
|
+
}
|
|
315434
|
+
function compactUndefined(record) {
|
|
315526
315435
|
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
315527
315436
|
}
|
|
315528
315437
|
function buildCommonArchiveTags(input) {
|
|
@@ -315539,7 +315448,7 @@ function buildOrderEventArchiveRow(input) {
|
|
|
315539
315448
|
const { tags, telemetry, action } = input;
|
|
315540
315449
|
return {
|
|
315541
315450
|
table: "broker_execution.order_events",
|
|
315542
|
-
row:
|
|
315451
|
+
row: compactUndefined({
|
|
315543
315452
|
...tags,
|
|
315544
315453
|
event_kind: input.eventKind ?? "execute_action",
|
|
315545
315454
|
action,
|
|
@@ -315576,35 +315485,35 @@ function buildSubscribeStreamArchiveRow(input) {
|
|
|
315576
315485
|
const info = asRecord(record?.info);
|
|
315577
315486
|
return {
|
|
315578
315487
|
table: "broker_execution.order_events",
|
|
315579
|
-
row:
|
|
315488
|
+
row: compactUndefined({
|
|
315580
315489
|
...input.tags,
|
|
315581
315490
|
event_kind: "subscribe_stream",
|
|
315582
315491
|
subscription_type: input.subscriptionType,
|
|
315583
|
-
order_id:
|
|
315584
|
-
client_order_id:
|
|
315585
|
-
status:
|
|
315492
|
+
order_id: firstString(record?.id, record?.orderId, record?.i, info?.orderId, info?.i),
|
|
315493
|
+
client_order_id: firstString(record?.clientOrderId, record?.clientOrderID, record?.c, info?.clientOrderId, info?.c),
|
|
315494
|
+
status: firstString(record?.status, record?.X, info?.status, info?.X)?.toLowerCase(),
|
|
315586
315495
|
payload_json: JSON.stringify(redactedPayload)
|
|
315587
315496
|
})
|
|
315588
315497
|
};
|
|
315589
315498
|
}
|
|
315590
315499
|
function quantityString(...values2) {
|
|
315591
|
-
return
|
|
315500
|
+
return firstString(...values2);
|
|
315592
315501
|
}
|
|
315593
315502
|
function extractBinanceInternalTransferId(response) {
|
|
315594
315503
|
const record = asRecord(response);
|
|
315595
|
-
return
|
|
315504
|
+
return firstString(record?.txnId, record?.tranId);
|
|
315596
315505
|
}
|
|
315597
315506
|
function buildTransferEventArchiveRow(input) {
|
|
315598
315507
|
const { tags, transfer } = input;
|
|
315599
315508
|
return {
|
|
315600
315509
|
table: "broker_execution.transfer_events",
|
|
315601
|
-
row:
|
|
315510
|
+
row: compactUndefined({
|
|
315602
315511
|
...tags,
|
|
315603
315512
|
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
315604
315513
|
event_kind: transfer.eventKind,
|
|
315605
315514
|
lifecycle_action: transfer.lifecycleAction,
|
|
315606
315515
|
status: transfer.status ?? "",
|
|
315607
|
-
asset_symbol: tags.symbol,
|
|
315516
|
+
asset_symbol: transfer.assetSymbol ?? tags.symbol,
|
|
315608
315517
|
amount: transfer.amount,
|
|
315609
315518
|
address: transfer.address,
|
|
315610
315519
|
network: transfer.network,
|
|
@@ -315624,18 +315533,18 @@ function normalizeCcxtTransactionForArchive(transaction) {
|
|
|
315624
315533
|
const record = asRecord(transaction);
|
|
315625
315534
|
const info = asRecord(record?.info);
|
|
315626
315535
|
const fee = asRecord(record?.fee);
|
|
315627
|
-
return
|
|
315628
|
-
externalId:
|
|
315629
|
-
clientWithdrawalId:
|
|
315630
|
-
txid:
|
|
315631
|
-
address:
|
|
315632
|
-
network:
|
|
315536
|
+
return compactUndefined({
|
|
315537
|
+
externalId: firstString(record?.id, info?.id, record?.txid, info?.txId),
|
|
315538
|
+
clientWithdrawalId: firstString(info?.withdrawOrderId),
|
|
315539
|
+
txid: firstString(record?.txid, info?.txId, info?.txid, info?.tx_hash),
|
|
315540
|
+
address: firstString(record?.address, record?.addressTo, info?.address),
|
|
315541
|
+
network: firstString(record?.network, info?.network),
|
|
315633
315542
|
amount: quantityString(info?.amount, record?.amount),
|
|
315634
|
-
assetSymbol:
|
|
315635
|
-
status:
|
|
315543
|
+
assetSymbol: firstString(record?.currency, info?.coin, info?.asset),
|
|
315544
|
+
status: firstString(record?.status, info?.status)?.toLowerCase(),
|
|
315636
315545
|
feeAmount: quantityString(fee?.cost, record?.feeCost),
|
|
315637
|
-
feeCurrency:
|
|
315638
|
-
exchangeTimestamp:
|
|
315546
|
+
feeCurrency: firstString(fee?.currency, record?.feeCurrency),
|
|
315547
|
+
exchangeTimestamp: normalizeTimestamp(firstValueForTransfer(record?.timestamp, record?.datetime, info?.applyTime))
|
|
315639
315548
|
});
|
|
315640
315549
|
}
|
|
315641
315550
|
function firstValueForTransfer(...values2) {
|
|
@@ -315645,7 +315554,7 @@ function buildFillEventArchiveRow(input) {
|
|
|
315645
315554
|
const { tags, fill } = input;
|
|
315646
315555
|
return {
|
|
315647
315556
|
table: "broker_execution.fill_events",
|
|
315648
|
-
row:
|
|
315557
|
+
row: compactUndefined({
|
|
315649
315558
|
...tags,
|
|
315650
315559
|
schema_version: ARCHIVE_SCHEMA_VERSION,
|
|
315651
315560
|
event_kind: FILL_EVENT_KIND,
|
|
@@ -315671,18 +315580,18 @@ function normalizeCcxtTradeForArchive(trade) {
|
|
|
315671
315580
|
const info = asRecord(record?.info);
|
|
315672
315581
|
const fee = asRecord(record?.fee);
|
|
315673
315582
|
return {
|
|
315674
|
-
orderId:
|
|
315675
|
-
clientOrderId:
|
|
315676
|
-
fillId:
|
|
315677
|
-
side:
|
|
315678
|
-
orderType:
|
|
315583
|
+
orderId: firstString(record?.order, info?.orderId, info?.orderID),
|
|
315584
|
+
clientOrderId: firstString(record?.clientOrderId, info?.clientOrderId, info?.origClientOrderId),
|
|
315585
|
+
fillId: firstString(record?.id, info?.id, info?.tradeId),
|
|
315586
|
+
side: firstString(record?.side, info?.side)?.toLowerCase(),
|
|
315587
|
+
orderType: firstString(record?.type, info?.type)?.toLowerCase(),
|
|
315679
315588
|
price: quantityString(info?.price, record?.price),
|
|
315680
315589
|
baseQuantity: quantityString(info?.qty, record?.amount),
|
|
315681
315590
|
quoteQuantity: quantityString(info?.quoteQty, record?.cost),
|
|
315682
315591
|
feeAmount: quantityString(fee?.cost, info?.commission),
|
|
315683
|
-
feeCurrency:
|
|
315592
|
+
feeCurrency: firstString(fee?.currency, info?.commissionAsset),
|
|
315684
315593
|
feeRate: quantityString(fee?.rate),
|
|
315685
|
-
exchangeTimestamp:
|
|
315594
|
+
exchangeTimestamp: normalizeTimestamp(firstValueForTransfer(record?.timestamp, record?.datetime, info?.time)),
|
|
315686
315595
|
payload: trade
|
|
315687
315596
|
};
|
|
315688
315597
|
}
|
|
@@ -315691,7 +315600,7 @@ function buildMarketMetadataSnapshotRow(input) {
|
|
|
315691
315600
|
const metadataHash = hashMarketMetadata(redactedSnapshot);
|
|
315692
315601
|
return {
|
|
315693
315602
|
table: "broker_execution.market_metadata_snapshots",
|
|
315694
|
-
row:
|
|
315603
|
+
row: compactUndefined({
|
|
315695
315604
|
...input.tags,
|
|
315696
315605
|
client_order_id: input.clientOrderId,
|
|
315697
315606
|
order_id: input.orderId,
|
|
@@ -315703,10 +315612,209 @@ function buildMarketMetadataSnapshotRow(input) {
|
|
|
315703
315612
|
};
|
|
315704
315613
|
}
|
|
315705
315614
|
|
|
315615
|
+
// src/helpers/order-telemetry.ts
|
|
315616
|
+
var NUMERIC_METRICS = [
|
|
315617
|
+
["requestedQuantity", "cex_market_action_requested_quantity"],
|
|
315618
|
+
["requestedNotional", "cex_market_action_requested_notional"],
|
|
315619
|
+
["executedBaseQuantity", "cex_market_action_executed_base_quantity"],
|
|
315620
|
+
["executedQuoteQuantity", "cex_market_action_executed_quote_quantity"],
|
|
315621
|
+
["averageExecutionPrice", "cex_market_action_average_execution_price"],
|
|
315622
|
+
["filledAmount", "cex_market_action_filled_amount"],
|
|
315623
|
+
["remainingAmount", "cex_market_action_remaining_amount"],
|
|
315624
|
+
["feeAmount", "cex_market_action_fee_amount"],
|
|
315625
|
+
["feeRate", "cex_market_action_fee_rate"]
|
|
315626
|
+
];
|
|
315627
|
+
async function emitOrderExecutionTelemetry(otelMetrics, context2, order, error) {
|
|
315628
|
+
try {
|
|
315629
|
+
const telemetry = buildOrderExecutionTelemetry(context2, order, error);
|
|
315630
|
+
log.info("CEX market action execution telemetry", telemetry);
|
|
315631
|
+
const labels = {
|
|
315632
|
+
action: telemetry.action,
|
|
315633
|
+
cex: telemetry.cex,
|
|
315634
|
+
account: telemetry.accountLabel,
|
|
315635
|
+
symbol: telemetry.symbol,
|
|
315636
|
+
side: telemetry.side,
|
|
315637
|
+
order_type: telemetry.orderType,
|
|
315638
|
+
status: telemetry.status
|
|
315639
|
+
};
|
|
315640
|
+
await otelMetrics?.recordCounter("cex_market_action_executions_total", 1, {
|
|
315641
|
+
...labels,
|
|
315642
|
+
result: error ? "error" : "ok"
|
|
315643
|
+
});
|
|
315644
|
+
for (const [key, metricName] of NUMERIC_METRICS) {
|
|
315645
|
+
const value = telemetry[key];
|
|
315646
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
315647
|
+
await otelMetrics?.recordHistogram(metricName, value, labels);
|
|
315648
|
+
}
|
|
315649
|
+
}
|
|
315650
|
+
return telemetry;
|
|
315651
|
+
} catch (telemetryError) {
|
|
315652
|
+
try {
|
|
315653
|
+
log.error("Failed to emit CEX order telemetry", {
|
|
315654
|
+
error: telemetryError
|
|
315655
|
+
});
|
|
315656
|
+
} catch {}
|
|
315657
|
+
return;
|
|
315658
|
+
}
|
|
315659
|
+
}
|
|
315660
|
+
function buildOrderExecutionTelemetry(context2, order, error) {
|
|
315661
|
+
const record = asRecord(order);
|
|
315662
|
+
const info = asRecord(record?.info);
|
|
315663
|
+
const fees = getFees(record, info);
|
|
315664
|
+
const fee = summarizeFees(fees);
|
|
315665
|
+
const executedBaseQuantity = firstNumber2(record?.filled, info?.executedQty, info?.cumExecQty) ?? computeFilledFromAmount(record);
|
|
315666
|
+
const executedQuoteQuantity = firstNumber2(record?.cost, info?.cummulativeQuoteQty, info?.cumQuote, info?.cumExecValue);
|
|
315667
|
+
const averageExecutionPrice = firstNumber2(record?.average, info?.avgPrice) ?? computeAveragePrice(executedBaseQuantity, executedQuoteQuantity);
|
|
315668
|
+
const exchangeTimestamp = normalizeTimestamp(firstValue(record?.timestamp, info?.time, info?.transactTime, record?.datetime));
|
|
315669
|
+
const status = firstString2(record?.status, info?.status) ?? (error ? "failed" : "unknown");
|
|
315670
|
+
const errorRecord = error instanceof Error ? error : undefined;
|
|
315671
|
+
return compactUndefined2({
|
|
315672
|
+
event: "cex_market_action_execution",
|
|
315673
|
+
action: context2.action,
|
|
315674
|
+
cex: context2.cex.trim().toLowerCase() || "unknown",
|
|
315675
|
+
accountLabel: context2.accountLabel ?? "unknown",
|
|
315676
|
+
symbol: firstString2(record?.symbol, info?.symbol, context2.symbol) ?? "unknown",
|
|
315677
|
+
side: firstString2(record?.side, info?.side, context2.side) ?? "unknown",
|
|
315678
|
+
orderType: firstString2(record?.type, info?.type, context2.orderType) ?? "unknown",
|
|
315679
|
+
orderId: firstString2(record?.id, info?.orderId, info?.orderID),
|
|
315680
|
+
orderAuthor: context2.orderAuthor,
|
|
315681
|
+
clientOrderId: firstString2(context2.clientOrderId, record?.clientOrderId, record?.clientOrderID, record?.clientOid, info?.clientOrderId, info?.clientOrderID, info?.clientOid),
|
|
315682
|
+
idempotencyId: context2.idempotencyId,
|
|
315683
|
+
makerActionId: context2.makerActionId,
|
|
315684
|
+
status: status.toLowerCase(),
|
|
315685
|
+
requestedQuantity: context2.requestedQuantity ?? firstNumber2(record?.amount),
|
|
315686
|
+
requestedNotional: context2.requestedNotional,
|
|
315687
|
+
executedBaseQuantity,
|
|
315688
|
+
executedQuoteQuantity,
|
|
315689
|
+
averageExecutionPrice,
|
|
315690
|
+
filledAmount: firstNumber2(record?.filled, info?.executedQty),
|
|
315691
|
+
remainingAmount: firstNumber2(record?.remaining, info?.remainingQty),
|
|
315692
|
+
feeAmount: fee.amount,
|
|
315693
|
+
feeCurrency: fee.currency,
|
|
315694
|
+
feeRate: fee.rate,
|
|
315695
|
+
exchangeTimestamp,
|
|
315696
|
+
brokerObservedTimestamp: context2.brokerObservedTimestamp ?? new Date().toISOString(),
|
|
315697
|
+
errorType: errorRecord?.name,
|
|
315698
|
+
errorMessage: errorRecord ? REDACTED_ERROR_MESSAGE : undefined
|
|
315699
|
+
});
|
|
315700
|
+
}
|
|
315701
|
+
function emitOrderExecutionTelemetryInBackground(otelMetrics, context2, order, error) {
|
|
315702
|
+
emitOrderExecutionTelemetry(otelMetrics, context2, order, error).catch((telemetryError) => {
|
|
315703
|
+
try {
|
|
315704
|
+
log.warn("Telemetry emit failed", { error: telemetryError });
|
|
315705
|
+
} catch {
|
|
315706
|
+
console.warn("Telemetry emit failed", telemetryError);
|
|
315707
|
+
}
|
|
315708
|
+
});
|
|
315709
|
+
}
|
|
315710
|
+
function extractOrderTelemetryIds(params) {
|
|
315711
|
+
const record = params ?? {};
|
|
315712
|
+
return {
|
|
315713
|
+
clientOrderId: firstString2(record.clientOrderId, record.clientOrderID, record.newClientOrderId, record.clientOid),
|
|
315714
|
+
idempotencyId: firstString2(record.idempotencyId, record.idempotencyID, record.idempotencyKey, record.requestId, record.requestID),
|
|
315715
|
+
makerActionId: firstString2(record.makerActionId, record.maker_action_id, record.actionId, record.action_id)
|
|
315716
|
+
};
|
|
315717
|
+
}
|
|
315718
|
+
function firstValue(...values2) {
|
|
315719
|
+
return values2.find((value) => value !== undefined && value !== null);
|
|
315720
|
+
}
|
|
315721
|
+
function firstString2(...values2) {
|
|
315722
|
+
for (const value of values2) {
|
|
315723
|
+
if (typeof value === "string" && value.trim()) {
|
|
315724
|
+
return value.trim();
|
|
315725
|
+
}
|
|
315726
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
315727
|
+
return String(value);
|
|
315728
|
+
}
|
|
315729
|
+
}
|
|
315730
|
+
return;
|
|
315731
|
+
}
|
|
315732
|
+
function firstNumber2(...values2) {
|
|
315733
|
+
for (const value of values2) {
|
|
315734
|
+
const numberValue = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
|
315735
|
+
if (Number.isFinite(numberValue)) {
|
|
315736
|
+
return numberValue;
|
|
315737
|
+
}
|
|
315738
|
+
}
|
|
315739
|
+
return;
|
|
315740
|
+
}
|
|
315741
|
+
function computeFilledFromAmount(record) {
|
|
315742
|
+
const amount = firstNumber2(record?.amount);
|
|
315743
|
+
const remaining = firstNumber2(record?.remaining);
|
|
315744
|
+
if (amount === undefined || remaining === undefined) {
|
|
315745
|
+
return;
|
|
315746
|
+
}
|
|
315747
|
+
return amount - remaining;
|
|
315748
|
+
}
|
|
315749
|
+
function computeAveragePrice(executedBaseQuantity, executedQuoteQuantity) {
|
|
315750
|
+
if (executedBaseQuantity === undefined || executedQuoteQuantity === undefined || executedBaseQuantity === 0) {
|
|
315751
|
+
return;
|
|
315752
|
+
}
|
|
315753
|
+
return executedQuoteQuantity / executedBaseQuantity;
|
|
315754
|
+
}
|
|
315755
|
+
function getFees(record, info) {
|
|
315756
|
+
const fees = [];
|
|
315757
|
+
if (record?.fee)
|
|
315758
|
+
fees.push(record.fee);
|
|
315759
|
+
if (Array.isArray(record?.fees))
|
|
315760
|
+
fees.push(...record.fees);
|
|
315761
|
+
if (Array.isArray(record?.trades)) {
|
|
315762
|
+
for (const trade of record.trades) {
|
|
315763
|
+
const tradeRecord = asRecord(trade);
|
|
315764
|
+
if (tradeRecord?.fee)
|
|
315765
|
+
fees.push(tradeRecord.fee);
|
|
315766
|
+
if (Array.isArray(tradeRecord?.fees))
|
|
315767
|
+
fees.push(...tradeRecord.fees);
|
|
315768
|
+
}
|
|
315769
|
+
}
|
|
315770
|
+
if (Array.isArray(info?.fills)) {
|
|
315771
|
+
for (const fill of info.fills) {
|
|
315772
|
+
const fillRecord = asRecord(fill);
|
|
315773
|
+
const commission = firstNumber2(fillRecord?.commission);
|
|
315774
|
+
if (commission !== undefined) {
|
|
315775
|
+
fees.push({
|
|
315776
|
+
cost: commission,
|
|
315777
|
+
currency: firstString2(fillRecord?.commissionAsset)
|
|
315778
|
+
});
|
|
315779
|
+
}
|
|
315780
|
+
}
|
|
315781
|
+
}
|
|
315782
|
+
return fees;
|
|
315783
|
+
}
|
|
315784
|
+
function summarizeFees(fees) {
|
|
315785
|
+
let amount = 0;
|
|
315786
|
+
let amountFound = false;
|
|
315787
|
+
let currency;
|
|
315788
|
+
let rate;
|
|
315789
|
+
for (const rawFee of fees) {
|
|
315790
|
+
const fee = asRecord(rawFee);
|
|
315791
|
+
if (!fee)
|
|
315792
|
+
continue;
|
|
315793
|
+
const cost = firstNumber2(fee.cost, fee.amount);
|
|
315794
|
+
if (cost !== undefined) {
|
|
315795
|
+
amount += cost;
|
|
315796
|
+
amountFound = true;
|
|
315797
|
+
}
|
|
315798
|
+
currency ??= firstString2(fee.currency);
|
|
315799
|
+
rate ??= firstNumber2(fee.rate);
|
|
315800
|
+
}
|
|
315801
|
+
return {
|
|
315802
|
+
amount: amountFound ? amount : undefined,
|
|
315803
|
+
currency,
|
|
315804
|
+
rate
|
|
315805
|
+
};
|
|
315806
|
+
}
|
|
315807
|
+
function compactUndefined2(record) {
|
|
315808
|
+
return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
|
|
315809
|
+
}
|
|
315810
|
+
|
|
315706
315811
|
// src/helpers/broker-execution-archive/writer.ts
|
|
315707
315812
|
var import_api_logs2 = __toESM(require_src7(), 1);
|
|
315813
|
+
import { randomUUID } from "node:crypto";
|
|
315708
315814
|
import { closeSync, fsyncSync, openSync, writeSync } from "node:fs";
|
|
315709
|
-
import {
|
|
315815
|
+
import {
|
|
315816
|
+
request as httpRequest2
|
|
315817
|
+
} from "node:http";
|
|
315710
315818
|
import { request as httpsRequest2 } from "node:https";
|
|
315711
315819
|
var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
|
|
315712
315820
|
"broker_execution.order_events",
|
|
@@ -315734,6 +315842,7 @@ var DEFAULT_BATCH_SIZE = 10;
|
|
|
315734
315842
|
var DEFAULT_FLUSH_INTERVAL_MS = 1000;
|
|
315735
315843
|
var DEFAULT_FORWARDER_TIMEOUT_MS = 3000;
|
|
315736
315844
|
var SHED_WARN_INTERVAL_MS = 60000;
|
|
315845
|
+
var MAX_PINNED_BATCH_ATTEMPTS = 10;
|
|
315737
315846
|
var MARKET_FEEDS = new Set(["ORDERBOOK", "TICKER", "TRADES", "OHLCV"]);
|
|
315738
315847
|
function archiveFeed(row) {
|
|
315739
315848
|
const declared = row.row.feed ?? row.row.stream_type;
|
|
@@ -315779,14 +315888,26 @@ class BrokerExecutionArchiver {
|
|
|
315779
315888
|
flushIntervalMs;
|
|
315780
315889
|
forwarderTimeoutMs;
|
|
315781
315890
|
queue = [];
|
|
315891
|
+
enqueueTimes = new WeakMap;
|
|
315782
315892
|
stats = {
|
|
315783
315893
|
enqueued: 0,
|
|
315784
315894
|
shed: 0,
|
|
315785
315895
|
flushed: 0,
|
|
315786
315896
|
forwarderFailures: 0
|
|
315787
315897
|
};
|
|
315898
|
+
pendingRetry = null;
|
|
315788
315899
|
flushTimer = null;
|
|
315789
315900
|
flushInFlight = null;
|
|
315901
|
+
inFlightBatch = null;
|
|
315902
|
+
inFlightStartedAtMs = null;
|
|
315903
|
+
lastFailureAtMs = null;
|
|
315904
|
+
lastShedAtMs = null;
|
|
315905
|
+
lastSuccessAtMs = null;
|
|
315906
|
+
lastFailureEventSequence = null;
|
|
315907
|
+
lastShedEventSequence = null;
|
|
315908
|
+
lastSuccessEventSequence = null;
|
|
315909
|
+
lastSinkLatencyMs = null;
|
|
315910
|
+
healthEventSequence = 0;
|
|
315790
315911
|
lastShedWarnAtMs = 0;
|
|
315791
315912
|
closing = false;
|
|
315792
315913
|
closed = false;
|
|
@@ -315820,6 +315941,7 @@ class BrokerExecutionArchiver {
|
|
|
315820
315941
|
}
|
|
315821
315942
|
try {
|
|
315822
315943
|
this.flushTimer = setInterval(() => {
|
|
315944
|
+
this.emitArchiveHealthMetrics();
|
|
315823
315945
|
this.flush();
|
|
315824
315946
|
}, this.flushIntervalMs);
|
|
315825
315947
|
this.flushTimer.unref?.();
|
|
@@ -315863,32 +315985,34 @@ class BrokerExecutionArchiver {
|
|
|
315863
315985
|
row: { ...row.row, source: this.source }
|
|
315864
315986
|
};
|
|
315865
315987
|
if (this.queue.length >= this.maxQueueSize) {
|
|
315866
|
-
const
|
|
315867
|
-
if (
|
|
315868
|
-
this.appendLossRecords([
|
|
315988
|
+
const shedEntry = this.queue[0];
|
|
315989
|
+
if (shedEntry) {
|
|
315990
|
+
this.appendLossRecords([shedEntry], "queue_shed");
|
|
315869
315991
|
this.queue.shift();
|
|
315870
315992
|
}
|
|
315871
315993
|
this.stats.shed += 1;
|
|
315994
|
+
this.recordHealthEvent("shed");
|
|
315872
315995
|
this.recordArchiveMetric("cex_archive_rows_shed_total", {
|
|
315873
|
-
table:
|
|
315996
|
+
table: shedEntry?.table ?? "unknown",
|
|
315874
315997
|
source: this.source,
|
|
315875
|
-
feed:
|
|
315998
|
+
feed: shedEntry ? archiveFeed(shedEntry) : "NON_MARKET"
|
|
315876
315999
|
});
|
|
315877
316000
|
this.recordArchiveMetric("cex_archive_queue_saturated_rows_total", {
|
|
315878
|
-
table:
|
|
316001
|
+
table: shedEntry?.table ?? "unknown",
|
|
315879
316002
|
source: this.source,
|
|
315880
|
-
feed:
|
|
316003
|
+
feed: shedEntry ? archiveFeed(shedEntry) : "NON_MARKET"
|
|
315881
316004
|
});
|
|
315882
316005
|
const now3 = Date.now();
|
|
315883
316006
|
if (now3 - this.lastShedWarnAtMs >= SHED_WARN_INTERVAL_MS) {
|
|
315884
316007
|
log.warn("Archive queue full: shedding oldest rows", {
|
|
315885
316008
|
shed_total: this.stats.shed,
|
|
315886
316009
|
queue_max: this.maxQueueSize,
|
|
315887
|
-
table:
|
|
316010
|
+
table: shedEntry?.table ?? "unknown"
|
|
315888
316011
|
});
|
|
315889
316012
|
this.lastShedWarnAtMs = now3;
|
|
315890
316013
|
}
|
|
315891
316014
|
}
|
|
316015
|
+
this.enqueueTimes.set(archiveRow, Date.now());
|
|
315892
316016
|
this.queue.push(archiveRow);
|
|
315893
316017
|
this.stats.enqueued += 1;
|
|
315894
316018
|
this.recordArchiveMetric("cex_archive_rows_enqueued_total", {
|
|
@@ -315896,6 +316020,7 @@ class BrokerExecutionArchiver {
|
|
|
315896
316020
|
source: this.source,
|
|
315897
316021
|
feed: archiveFeed(archiveRow)
|
|
315898
316022
|
});
|
|
316023
|
+
this.emitArchiveHealthMetrics();
|
|
315899
316024
|
if (this.queue.length >= this.batchSize) {
|
|
315900
316025
|
this.flush();
|
|
315901
316026
|
}
|
|
@@ -315904,20 +316029,25 @@ class BrokerExecutionArchiver {
|
|
|
315904
316029
|
queueMicrotask(() => this.enqueue(row));
|
|
315905
316030
|
}
|
|
315906
316031
|
async flush() {
|
|
315907
|
-
if (!this.enabled || this.closed || this.queue.length === 0) {
|
|
316032
|
+
if (!this.enabled || this.closed || this.queue.length === 0 && this.pendingRetry === null) {
|
|
315908
316033
|
return;
|
|
315909
316034
|
}
|
|
315910
316035
|
if (this.flushInFlight) {
|
|
315911
316036
|
return this.flushInFlight;
|
|
315912
316037
|
}
|
|
316038
|
+
const inFlightState = {};
|
|
315913
316039
|
const inFlight = this.flushBatch().then(() => {
|
|
315914
316040
|
return;
|
|
315915
316041
|
}).finally(() => {
|
|
316042
|
+
if (this.flushInFlight !== inFlightState.promise) {
|
|
316043
|
+
return;
|
|
316044
|
+
}
|
|
315916
316045
|
this.flushInFlight = null;
|
|
315917
316046
|
if (!this.closed && !this.closing && this.enabled && this.queue.length >= this.batchSize) {
|
|
315918
316047
|
queueMicrotask(() => void this.flush());
|
|
315919
316048
|
}
|
|
315920
316049
|
});
|
|
316050
|
+
inFlightState.promise = inFlight;
|
|
315921
316051
|
this.flushInFlight = inFlight;
|
|
315922
316052
|
return inFlight;
|
|
315923
316053
|
}
|
|
@@ -315929,16 +316059,18 @@ class BrokerExecutionArchiver {
|
|
|
315929
316059
|
}
|
|
315930
316060
|
let closeError;
|
|
315931
316061
|
try {
|
|
315932
|
-
while (this.queue.length > 0 || this.flushInFlight) {
|
|
316062
|
+
while (this.pendingRetry !== null || this.queue.length > 0 || this.flushInFlight) {
|
|
315933
316063
|
if (this.flushInFlight) {
|
|
315934
316064
|
await this.flushInFlight;
|
|
315935
316065
|
continue;
|
|
315936
316066
|
}
|
|
315937
|
-
|
|
315938
|
-
|
|
315939
|
-
|
|
315940
|
-
|
|
316067
|
+
if (!await this.flushBatch()) {
|
|
316068
|
+
const undelivered = [
|
|
316069
|
+
...this.pendingRetry?.rows ?? [],
|
|
316070
|
+
...this.queue
|
|
316071
|
+
];
|
|
315941
316072
|
this.appendLossRecords(undelivered, "shutdown_forwarder_failure");
|
|
316073
|
+
this.pendingRetry = null;
|
|
315942
316074
|
this.queue.length = 0;
|
|
315943
316075
|
break;
|
|
315944
316076
|
}
|
|
@@ -315962,7 +316094,87 @@ class BrokerExecutionArchiver {
|
|
|
315962
316094
|
return { ...this.stats };
|
|
315963
316095
|
}
|
|
315964
316096
|
getQueueDepth() {
|
|
315965
|
-
return this.queue.length;
|
|
316097
|
+
return this.queue.length + (this.pendingRetry?.rows.length ?? 0);
|
|
316098
|
+
}
|
|
316099
|
+
getHealthSnapshot() {
|
|
316100
|
+
const now3 = Date.now();
|
|
316101
|
+
const oldestPendingAtMs = this.oldestPendingEnqueueAtMs();
|
|
316102
|
+
const oldestPendingAgeMs = oldestPendingAtMs === null ? 0 : Math.max(0, now3 - oldestPendingAtMs);
|
|
316103
|
+
const inFlightAgeMs = this.inFlightStartedAtMs === null ? 0 : Math.max(0, now3 - this.inFlightStartedAtMs);
|
|
316104
|
+
const lastUnhealthyEventSequence = Math.max(this.lastFailureEventSequence ?? Number.NEGATIVE_INFINITY, this.lastShedEventSequence ?? Number.NEGATIVE_INFINITY);
|
|
316105
|
+
const recoveredFromEvents = lastUnhealthyEventSequence === Number.NEGATIVE_INFINITY || this.lastSuccessEventSequence !== null && this.lastSuccessEventSequence > lastUnhealthyEventSequence;
|
|
316106
|
+
const healthy = this.enabled && !this.closing && !this.closed && this.queue.length < this.maxQueueSize && oldestPendingAgeMs <= this.forwarderTimeoutMs && inFlightAgeMs <= this.forwarderTimeoutMs && recoveredFromEvents;
|
|
316107
|
+
return {
|
|
316108
|
+
healthy,
|
|
316109
|
+
queue_depth: this.getQueueDepth(),
|
|
316110
|
+
oldest_pending_age_ms: oldestPendingAgeMs,
|
|
316111
|
+
shed_total: this.stats.shed,
|
|
316112
|
+
last_failure_at: this.lastFailureAtMs,
|
|
316113
|
+
last_shed_at: this.lastShedAtMs,
|
|
316114
|
+
last_success_at: this.lastSuccessAtMs,
|
|
316115
|
+
sink_latency_ms: this.lastSinkLatencyMs,
|
|
316116
|
+
sink_errors_total: this.stats.forwarderFailures,
|
|
316117
|
+
in_flight: this.inFlightBatch !== null,
|
|
316118
|
+
in_flight_age_ms: inFlightAgeMs
|
|
316119
|
+
};
|
|
316120
|
+
}
|
|
316121
|
+
oldestPendingEnqueueAtMs() {
|
|
316122
|
+
let oldest = null;
|
|
316123
|
+
for (const entry of this.pendingRetry?.rows ?? []) {
|
|
316124
|
+
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
316125
|
+
if (enqueuedAtMs === undefined) {
|
|
316126
|
+
continue;
|
|
316127
|
+
}
|
|
316128
|
+
oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
|
|
316129
|
+
}
|
|
316130
|
+
for (const entry of this.queue) {
|
|
316131
|
+
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
316132
|
+
if (enqueuedAtMs === undefined) {
|
|
316133
|
+
continue;
|
|
316134
|
+
}
|
|
316135
|
+
oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
|
|
316136
|
+
}
|
|
316137
|
+
for (const entry of this.inFlightBatch ?? []) {
|
|
316138
|
+
const enqueuedAtMs = this.enqueueTimes.get(entry);
|
|
316139
|
+
if (enqueuedAtMs === undefined) {
|
|
316140
|
+
continue;
|
|
316141
|
+
}
|
|
316142
|
+
oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
|
|
316143
|
+
}
|
|
316144
|
+
return oldest;
|
|
316145
|
+
}
|
|
316146
|
+
recordHealthEvent(event) {
|
|
316147
|
+
const eventSequence = ++this.healthEventSequence;
|
|
316148
|
+
const eventTimestampMs = Date.now();
|
|
316149
|
+
switch (event) {
|
|
316150
|
+
case "failure":
|
|
316151
|
+
this.lastFailureAtMs = eventTimestampMs;
|
|
316152
|
+
this.lastFailureEventSequence = eventSequence;
|
|
316153
|
+
return;
|
|
316154
|
+
case "shed":
|
|
316155
|
+
this.lastShedAtMs = eventTimestampMs;
|
|
316156
|
+
this.lastShedEventSequence = eventSequence;
|
|
316157
|
+
return;
|
|
316158
|
+
case "success":
|
|
316159
|
+
this.lastSuccessAtMs = eventTimestampMs;
|
|
316160
|
+
this.lastSuccessEventSequence = eventSequence;
|
|
316161
|
+
return;
|
|
316162
|
+
}
|
|
316163
|
+
}
|
|
316164
|
+
emitArchiveHealthMetrics() {
|
|
316165
|
+
const snapshot = this.getHealthSnapshot();
|
|
316166
|
+
const labels = { source: this.source };
|
|
316167
|
+
this.recordArchiveObservableGauge("cex_archive_health", snapshot.healthy ? 1 : 0, labels);
|
|
316168
|
+
this.recordArchiveGauge("cex_archive_queue_depth", snapshot.queue_depth, labels);
|
|
316169
|
+
this.recordArchiveGauge("cex_archive_oldest_pending_age_ms", snapshot.oldest_pending_age_ms, labels);
|
|
316170
|
+
this.recordArchiveGauge("cex_archive_shed_total", snapshot.shed_total, labels);
|
|
316171
|
+
this.recordArchiveGauge("cex_archive_last_failure_at", snapshot.last_failure_at ?? 0, labels);
|
|
316172
|
+
this.recordArchiveGauge("cex_archive_last_shed_at", snapshot.last_shed_at ?? 0, labels);
|
|
316173
|
+
this.recordArchiveGauge("cex_archive_last_success_at", snapshot.last_success_at ?? 0, labels);
|
|
316174
|
+
this.recordArchiveGauge("cex_archive_sink_latency_ms", snapshot.sink_latency_ms ?? 0, labels);
|
|
316175
|
+
this.recordArchiveGauge("cex_archive_sink_errors_total", snapshot.sink_errors_total, labels);
|
|
316176
|
+
this.recordArchiveGauge("cex_archive_in_flight", snapshot.in_flight ? 1 : 0, labels);
|
|
316177
|
+
this.recordArchiveGauge("cex_archive_in_flight_age_ms", snapshot.in_flight_age_ms, labels);
|
|
315966
316178
|
}
|
|
315967
316179
|
closeLossJournal() {
|
|
315968
316180
|
if (this.deadLetterFd === undefined) {
|
|
@@ -315977,27 +316189,6 @@ class BrokerExecutionArchiver {
|
|
|
315977
316189
|
this.deadLetterFd = undefined;
|
|
315978
316190
|
}
|
|
315979
316191
|
}
|
|
315980
|
-
enforceQueueBound() {
|
|
315981
|
-
while (this.queue.length > this.maxQueueSize) {
|
|
315982
|
-
const dropped = this.queue[0];
|
|
315983
|
-
if (!dropped) {
|
|
315984
|
-
return;
|
|
315985
|
-
}
|
|
315986
|
-
this.appendLossRecords([dropped], "queue_shed");
|
|
315987
|
-
this.queue.shift();
|
|
315988
|
-
this.stats.shed += 1;
|
|
315989
|
-
this.recordArchiveMetric("cex_archive_rows_shed_total", {
|
|
315990
|
-
table: dropped?.table ?? "unknown",
|
|
315991
|
-
source: this.source,
|
|
315992
|
-
feed: archiveFeed(dropped)
|
|
315993
|
-
});
|
|
315994
|
-
this.recordArchiveMetric("cex_archive_queue_saturated_rows_total", {
|
|
315995
|
-
table: dropped.table,
|
|
315996
|
-
source: this.source,
|
|
315997
|
-
feed: archiveFeed(dropped)
|
|
315998
|
-
});
|
|
315999
|
-
}
|
|
316000
|
-
}
|
|
316001
316192
|
appendLossRecords(rows, reason) {
|
|
316002
316193
|
if (rows.length === 0) {
|
|
316003
316194
|
return;
|
|
@@ -316042,10 +316233,14 @@ class BrokerExecutionArchiver {
|
|
|
316042
316233
|
}
|
|
316043
316234
|
}
|
|
316044
316235
|
async flushBatch() {
|
|
316045
|
-
const
|
|
316236
|
+
const pinned = this.pendingRetry;
|
|
316237
|
+
const batch = pinned ? pinned.rows : this.queue.splice(0, this.batchSize);
|
|
316046
316238
|
if (batch.length === 0) {
|
|
316047
316239
|
return true;
|
|
316048
316240
|
}
|
|
316241
|
+
const batchId = pinned?.batchId ?? randomUUID();
|
|
316242
|
+
this.inFlightBatch = batch;
|
|
316243
|
+
this.inFlightStartedAtMs = Date.now();
|
|
316049
316244
|
for (const entry of batch) {
|
|
316050
316245
|
if (isBrokerExecutionArchiveTable(entry.table)) {
|
|
316051
316246
|
this.emitOtelLog(entry);
|
|
@@ -316053,11 +316248,27 @@ class BrokerExecutionArchiver {
|
|
|
316053
316248
|
}
|
|
316054
316249
|
if (this.forwarderUrl) {
|
|
316055
316250
|
try {
|
|
316056
|
-
await this.postToForwarder(batch);
|
|
316251
|
+
await this.postToForwarder(batch, batchId);
|
|
316057
316252
|
} catch (error) {
|
|
316058
316253
|
this.stats.forwarderFailures += 1;
|
|
316059
|
-
this.
|
|
316060
|
-
this.
|
|
316254
|
+
this.recordHealthEvent("failure");
|
|
316255
|
+
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
316256
|
+
const attempts = (pinned?.attempts ?? 0) + 1;
|
|
316257
|
+
try {
|
|
316258
|
+
if (attempts >= MAX_PINNED_BATCH_ATTEMPTS) {
|
|
316259
|
+
this.pendingRetry = null;
|
|
316260
|
+
this.appendLossRecords(batch, "retry_exhausted");
|
|
316261
|
+
log.warn("Broker execution archive gave up on a batch", {
|
|
316262
|
+
attempts,
|
|
316263
|
+
rows: batch.length
|
|
316264
|
+
});
|
|
316265
|
+
} else {
|
|
316266
|
+
this.pendingRetry = { batchId, rows: batch, attempts };
|
|
316267
|
+
}
|
|
316268
|
+
} finally {
|
|
316269
|
+
this.clearInFlightBatch(batch);
|
|
316270
|
+
this.emitArchiveHealthMetrics();
|
|
316271
|
+
}
|
|
316061
316272
|
this.recordArchiveMetric("cex_archive_forwarder_failures_total", {
|
|
316062
316273
|
count: batch.length
|
|
316063
316274
|
});
|
|
@@ -316065,15 +316276,28 @@ class BrokerExecutionArchiver {
|
|
|
316065
316276
|
return false;
|
|
316066
316277
|
}
|
|
316067
316278
|
}
|
|
316279
|
+
this.pendingRetry = null;
|
|
316068
316280
|
this.stats.flushed += batch.length;
|
|
316281
|
+
this.recordHealthEvent("success");
|
|
316282
|
+
this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
|
|
316283
|
+
this.clearInFlightBatch(batch);
|
|
316069
316284
|
this.recordFlushHealth(batch);
|
|
316285
|
+
this.emitArchiveHealthMetrics();
|
|
316070
316286
|
return true;
|
|
316071
316287
|
}
|
|
316288
|
+
clearInFlightBatch(batch) {
|
|
316289
|
+
if (this.inFlightBatch !== batch) {
|
|
316290
|
+
return;
|
|
316291
|
+
}
|
|
316292
|
+
this.inFlightBatch = null;
|
|
316293
|
+
this.inFlightStartedAtMs = null;
|
|
316294
|
+
}
|
|
316072
316295
|
recordFlushHealth(batch) {
|
|
316073
316296
|
const countByTable = new Map;
|
|
316074
316297
|
for (const entry of batch) {
|
|
316075
|
-
const
|
|
316076
|
-
const
|
|
316298
|
+
const row = entry;
|
|
316299
|
+
const key = `${row.table}\x00${archiveFeed(row)}`;
|
|
316300
|
+
const grouped = countByTable.get(key) ?? { row, count: 0 };
|
|
316077
316301
|
grouped.count += 1;
|
|
316078
316302
|
countByTable.set(key, grouped);
|
|
316079
316303
|
}
|
|
@@ -316091,9 +316315,14 @@ class BrokerExecutionArchiver {
|
|
|
316091
316315
|
await this.otelMetrics?.recordCounter(metricName, value, labels);
|
|
316092
316316
|
} catch {}
|
|
316093
316317
|
}
|
|
316094
|
-
async recordArchiveGauge(metricName, value) {
|
|
316318
|
+
async recordArchiveGauge(metricName, value, labels = {}) {
|
|
316095
316319
|
try {
|
|
316096
|
-
await this.otelMetrics?.recordGauge(metricName, value,
|
|
316320
|
+
await this.otelMetrics?.recordGauge(metricName, value, labels);
|
|
316321
|
+
} catch {}
|
|
316322
|
+
}
|
|
316323
|
+
async recordArchiveObservableGauge(metricName, value, labels) {
|
|
316324
|
+
try {
|
|
316325
|
+
await this.otelMetrics?.setObservableGauge(metricName, value, labels);
|
|
316097
316326
|
} catch {}
|
|
316098
316327
|
}
|
|
316099
316328
|
emitOtelLog(entry) {
|
|
@@ -316111,13 +316340,14 @@ class BrokerExecutionArchiver {
|
|
|
316111
316340
|
log.warn("Broker execution archive OTLP emit failed", { error });
|
|
316112
316341
|
}
|
|
316113
316342
|
}
|
|
316114
|
-
postToForwarder(batch) {
|
|
316343
|
+
postToForwarder(batch, batchId) {
|
|
316115
316344
|
if (!this.forwarderUrl || batch.length === 0) {
|
|
316116
316345
|
return Promise.resolve();
|
|
316117
316346
|
}
|
|
316118
316347
|
const body = JSON.stringify({
|
|
316119
316348
|
source: this.source,
|
|
316120
316349
|
deployment_id: this.deploymentId,
|
|
316350
|
+
batch_id: batchId,
|
|
316121
316351
|
rows: batch
|
|
316122
316352
|
});
|
|
316123
316353
|
const url2 = new URL(this.forwarderUrl);
|
|
@@ -316130,27 +316360,136 @@ class BrokerExecutionArchiver {
|
|
|
316130
316360
|
headers.authorization = `Bearer ${this.forwarderAuthToken}`;
|
|
316131
316361
|
}
|
|
316132
316362
|
return new Promise((resolve, reject) => {
|
|
316133
|
-
|
|
316134
|
-
|
|
316135
|
-
|
|
316136
|
-
|
|
316137
|
-
|
|
316138
|
-
|
|
316139
|
-
|
|
316140
|
-
|
|
316141
|
-
|
|
316142
|
-
|
|
316143
|
-
|
|
316144
|
-
|
|
316363
|
+
let req;
|
|
316364
|
+
let response;
|
|
316365
|
+
let settled = false;
|
|
316366
|
+
let requestClosed = false;
|
|
316367
|
+
let requestCloseFailureScheduled = false;
|
|
316368
|
+
let responseClosed = false;
|
|
316369
|
+
let responseEnded = false;
|
|
316370
|
+
let outerTimer;
|
|
316371
|
+
function cleanupRequestListeners() {
|
|
316372
|
+
req?.removeListener("error", onRequestError);
|
|
316373
|
+
req?.removeListener("timeout", onRequestTimeout);
|
|
316374
|
+
req?.removeListener("close", onRequestClose);
|
|
316375
|
+
}
|
|
316376
|
+
function cleanupResponseListeners() {
|
|
316377
|
+
response?.removeListener("data", onResponseData);
|
|
316378
|
+
response?.removeListener("end", onResponseEnd);
|
|
316379
|
+
response?.removeListener("aborted", onResponseAborted);
|
|
316380
|
+
response?.removeListener("error", onResponseError);
|
|
316381
|
+
response?.removeListener("close", onResponseClose);
|
|
316382
|
+
}
|
|
316383
|
+
function onRequestClose() {
|
|
316384
|
+
requestClosed = true;
|
|
316385
|
+
if (!settled && !response && !requestCloseFailureScheduled) {
|
|
316386
|
+
requestCloseFailureScheduled = true;
|
|
316387
|
+
queueMicrotask(() => {
|
|
316388
|
+
if (!settled && !response) {
|
|
316389
|
+
settle2(new Error("Archive forwarder request closed before response headers"), false, true);
|
|
316390
|
+
}
|
|
316391
|
+
});
|
|
316392
|
+
}
|
|
316393
|
+
if (settled) {
|
|
316394
|
+
cleanupRequestListeners();
|
|
316395
|
+
}
|
|
316396
|
+
}
|
|
316397
|
+
function onRequestError(error) {
|
|
316398
|
+
if (!settled) {
|
|
316399
|
+
settle2(error, true);
|
|
316400
|
+
} else {
|
|
316401
|
+
cleanupRequestListeners();
|
|
316402
|
+
}
|
|
316403
|
+
}
|
|
316404
|
+
function onRequestTimeout() {
|
|
316405
|
+
settle2(new Error("Archive forwarder socket timed out"), true);
|
|
316406
|
+
}
|
|
316407
|
+
function onResponseData() {}
|
|
316408
|
+
function onResponseEnd() {
|
|
316409
|
+
responseEnded = true;
|
|
316410
|
+
if (!response?.complete) {
|
|
316411
|
+
settle2(new Error("Archive forwarder response was incomplete"), true);
|
|
316412
|
+
return;
|
|
316413
|
+
}
|
|
316414
|
+
const status = response.statusCode ?? 0;
|
|
316415
|
+
if (status < 200 || status >= 300) {
|
|
316416
|
+
settle2(new Error(`Archive forwarder returned ${status} ${response.statusMessage ?? ""}`), false);
|
|
316417
|
+
return;
|
|
316418
|
+
}
|
|
316419
|
+
settle2(undefined, false);
|
|
316420
|
+
}
|
|
316421
|
+
function onResponseAborted() {
|
|
316422
|
+
settle2(new Error("Archive forwarder response was aborted"), true);
|
|
316423
|
+
}
|
|
316424
|
+
function onResponseError(error) {
|
|
316425
|
+
if (!settled) {
|
|
316426
|
+
settle2(error, true);
|
|
316427
|
+
}
|
|
316428
|
+
}
|
|
316429
|
+
function onResponseClose() {
|
|
316430
|
+
responseClosed = true;
|
|
316431
|
+
if (!settled && !responseEnded) {
|
|
316432
|
+
settle2(new Error("Archive forwarder response closed before completion"), true);
|
|
316433
|
+
return;
|
|
316434
|
+
}
|
|
316435
|
+
cleanupResponseListeners();
|
|
316436
|
+
}
|
|
316437
|
+
function settle2(error, destroyRequest, retainClosedRequestError = false) {
|
|
316438
|
+
if (settled) {
|
|
316439
|
+
return;
|
|
316440
|
+
}
|
|
316441
|
+
settled = true;
|
|
316442
|
+
if (outerTimer) {
|
|
316443
|
+
clearTimeout(outerTimer);
|
|
316444
|
+
outerTimer = undefined;
|
|
316445
|
+
}
|
|
316446
|
+
response?.removeListener("data", onResponseData);
|
|
316447
|
+
response?.removeListener("end", onResponseEnd);
|
|
316448
|
+
response?.removeListener("aborted", onResponseAborted);
|
|
316449
|
+
if (responseClosed) {
|
|
316450
|
+
cleanupResponseListeners();
|
|
316451
|
+
}
|
|
316452
|
+
if (destroyRequest && req && !requestClosed) {
|
|
316453
|
+
req.destroy(error);
|
|
316454
|
+
} else if (retainClosedRequestError && req && requestClosed) {
|
|
316455
|
+
req.removeListener("timeout", onRequestTimeout);
|
|
316456
|
+
req.removeListener("close", onRequestClose);
|
|
316457
|
+
} else if (!req || requestClosed) {
|
|
316458
|
+
cleanupRequestListeners();
|
|
316459
|
+
}
|
|
316460
|
+
if (error) {
|
|
316461
|
+
reject(error);
|
|
316462
|
+
} else {
|
|
316145
316463
|
resolve();
|
|
316464
|
+
}
|
|
316465
|
+
}
|
|
316466
|
+
outerTimer = setTimeout(() => {
|
|
316467
|
+
settle2(new Error("Archive forwarder request deadline exceeded"), true);
|
|
316468
|
+
}, this.forwarderTimeoutMs);
|
|
316469
|
+
try {
|
|
316470
|
+
req = doRequest(url2, {
|
|
316471
|
+
method: "POST",
|
|
316472
|
+
headers,
|
|
316473
|
+
timeout: this.forwarderTimeoutMs
|
|
316474
|
+
}, (nextResponse) => {
|
|
316475
|
+
response = nextResponse;
|
|
316476
|
+
response.on("data", onResponseData);
|
|
316477
|
+
response.on("end", onResponseEnd);
|
|
316478
|
+
response.on("aborted", onResponseAborted);
|
|
316479
|
+
response.on("error", onResponseError);
|
|
316480
|
+
response.on("close", onResponseClose);
|
|
316481
|
+
if (settled) {
|
|
316482
|
+
response.resume();
|
|
316483
|
+
}
|
|
316146
316484
|
});
|
|
316147
|
-
|
|
316148
|
-
|
|
316149
|
-
|
|
316150
|
-
req.
|
|
316151
|
-
|
|
316152
|
-
|
|
316153
|
-
|
|
316485
|
+
req.on("error", onRequestError);
|
|
316486
|
+
req.on("timeout", onRequestTimeout);
|
|
316487
|
+
req.on("close", onRequestClose);
|
|
316488
|
+
req.write(body);
|
|
316489
|
+
req.end();
|
|
316490
|
+
} catch (error) {
|
|
316491
|
+
settle2(error instanceof Error ? error : new Error(String(error)), true);
|
|
316492
|
+
}
|
|
316154
316493
|
});
|
|
316155
316494
|
}
|
|
316156
316495
|
}
|
|
@@ -316620,6 +316959,94 @@ class AccountBalanceArchivePoller {
|
|
|
316620
316959
|
}
|
|
316621
316960
|
}
|
|
316622
316961
|
|
|
316962
|
+
// src/helpers/balance-update-archive-consumer.ts
|
|
316963
|
+
class BalanceUpdateArchiveConsumer {
|
|
316964
|
+
params;
|
|
316965
|
+
#started = false;
|
|
316966
|
+
#stopping = false;
|
|
316967
|
+
#subscriptions = new Set;
|
|
316968
|
+
#runs = [];
|
|
316969
|
+
constructor(params) {
|
|
316970
|
+
this.params = params;
|
|
316971
|
+
}
|
|
316972
|
+
start() {
|
|
316973
|
+
if (this.#started || this.#stopping)
|
|
316974
|
+
return;
|
|
316975
|
+
this.#started = true;
|
|
316976
|
+
const targets = this.#targets();
|
|
316977
|
+
log.info("\uD83D\uDCB8 Balance-update archive consumer started", {
|
|
316978
|
+
accounts: targets.length
|
|
316979
|
+
});
|
|
316980
|
+
for (const target of targets) {
|
|
316981
|
+
this.#runs.push(this.#consume(target));
|
|
316982
|
+
}
|
|
316983
|
+
}
|
|
316984
|
+
async stop() {
|
|
316985
|
+
this.#stopping = true;
|
|
316986
|
+
for (const subscription of [...this.#subscriptions]) {
|
|
316987
|
+
subscription.close();
|
|
316988
|
+
}
|
|
316989
|
+
await Promise.all(this.#runs);
|
|
316990
|
+
}
|
|
316991
|
+
#targets() {
|
|
316992
|
+
const targets = [];
|
|
316993
|
+
for (const [exchange, pool] of Object.entries(this.params.brokers)) {
|
|
316994
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
316995
|
+
targets.push({ exchange, accountSelector: account.label });
|
|
316996
|
+
}
|
|
316997
|
+
}
|
|
316998
|
+
return targets;
|
|
316999
|
+
}
|
|
317000
|
+
async#consume(target) {
|
|
317001
|
+
while (!this.#stopping) {
|
|
317002
|
+
let subscription;
|
|
317003
|
+
try {
|
|
317004
|
+
subscription = this.params.userDataStreamSupervisor.subscribe({
|
|
317005
|
+
exchange: target.exchange,
|
|
317006
|
+
accountSelector: target.accountSelector,
|
|
317007
|
+
kind: "balance"
|
|
317008
|
+
});
|
|
317009
|
+
this.#subscriptions.add(subscription);
|
|
317010
|
+
for await (const message of subscription) {
|
|
317011
|
+
if (this.#stopping)
|
|
317012
|
+
break;
|
|
317013
|
+
const event = message.event;
|
|
317014
|
+
if (event.e !== "balanceUpdate")
|
|
317015
|
+
continue;
|
|
317016
|
+
this.params.archiver.enqueue(buildTransferEventArchiveRow({
|
|
317017
|
+
tags: buildCommonArchiveTags({
|
|
317018
|
+
deploymentId: this.params.archiver.getDeploymentId(),
|
|
317019
|
+
accountSelector: target.accountSelector,
|
|
317020
|
+
exchange: target.exchange
|
|
317021
|
+
}),
|
|
317022
|
+
transfer: {
|
|
317023
|
+
eventKind: "balance_delta",
|
|
317024
|
+
lifecycleAction: "observe_balance_update",
|
|
317025
|
+
amount: typeof event.d === "string" ? event.d : undefined,
|
|
317026
|
+
assetSymbol: typeof event.a === "string" ? event.a : undefined,
|
|
317027
|
+
exchangeTimestamp: normalizeTimestamp(event.T),
|
|
317028
|
+
payload: event
|
|
317029
|
+
}
|
|
317030
|
+
}));
|
|
317031
|
+
}
|
|
317032
|
+
} catch (error) {
|
|
317033
|
+
if (!this.#stopping) {
|
|
317034
|
+
log.warn("Balance-update archive subscription failed", {
|
|
317035
|
+
exchange: target.exchange,
|
|
317036
|
+
accountSelector: target.accountSelector,
|
|
317037
|
+
errorType: error instanceof Error ? error.name : typeof error
|
|
317038
|
+
});
|
|
317039
|
+
}
|
|
317040
|
+
} finally {
|
|
317041
|
+
if (subscription) {
|
|
317042
|
+
this.#subscriptions.delete(subscription);
|
|
317043
|
+
subscription.close();
|
|
317044
|
+
}
|
|
317045
|
+
}
|
|
317046
|
+
}
|
|
317047
|
+
}
|
|
317048
|
+
}
|
|
317049
|
+
|
|
316623
317050
|
// src/helpers/deposit-archive-poller.ts
|
|
316624
317051
|
var DEFAULT_CONFIG2 = {
|
|
316625
317052
|
pollIntervalMs: 60000,
|
|
@@ -316829,7 +317256,7 @@ class DepositArchivePoller {
|
|
|
316829
317256
|
network: network === undefined ? undefined : String(network),
|
|
316830
317257
|
externalId: depositTxid,
|
|
316831
317258
|
txid: depositTxid,
|
|
316832
|
-
exchangeTimestamp:
|
|
317259
|
+
exchangeTimestamp: normalizeTimestamp(creditedAt),
|
|
316833
317260
|
payload: record
|
|
316834
317261
|
}
|
|
316835
317262
|
}));
|
|
@@ -317691,7 +318118,7 @@ function createOtelLogsFromEnv() {
|
|
|
317691
318118
|
}
|
|
317692
318119
|
|
|
317693
318120
|
// src/helpers/stream-health-publisher.ts
|
|
317694
|
-
import { createHash as createHash4, randomUUID } from "node:crypto";
|
|
318121
|
+
import { createHash as createHash4, randomUUID as randomUUID2 } from "node:crypto";
|
|
317695
318122
|
import {
|
|
317696
318123
|
closeSync as closeSync2,
|
|
317697
318124
|
fsyncSync as fsyncSync2,
|
|
@@ -317816,7 +318243,7 @@ class StreamHealthPublisher {
|
|
|
317816
318243
|
version: STATE_VERSION,
|
|
317817
318244
|
producerId: PRODUCER_ID,
|
|
317818
318245
|
producerEpoch: "1",
|
|
317819
|
-
runId:
|
|
318246
|
+
runId: randomUUID2(),
|
|
317820
318247
|
nextBatchSequence: "1",
|
|
317821
318248
|
nextStreamSequences: {}
|
|
317822
318249
|
};
|
|
@@ -317885,7 +318312,7 @@ class StreamHealthPublisher {
|
|
|
317885
318312
|
return;
|
|
317886
318313
|
if (this.#advanceRun) {
|
|
317887
318314
|
this.#state.producerEpoch = next(this.#state.producerEpoch);
|
|
317888
|
-
this.#state.runId =
|
|
318315
|
+
this.#state.runId = randomUUID2();
|
|
317889
318316
|
this.#state.nextBatchSequence = "1";
|
|
317890
318317
|
this.#state.nextStreamSequences = {};
|
|
317891
318318
|
this.#advanceRun = false;
|
|
@@ -318001,7 +318428,7 @@ class StreamHealthPublisher {
|
|
|
318001
318428
|
cause: error
|
|
318002
318429
|
});
|
|
318003
318430
|
}
|
|
318004
|
-
const temporary = `${this.#statePath}.${process.pid}.${
|
|
318431
|
+
const temporary = `${this.#statePath}.${process.pid}.${randomUUID2()}.tmp`;
|
|
318005
318432
|
let fd2;
|
|
318006
318433
|
try {
|
|
318007
318434
|
fd2 = openSync2(temporary, "wx", 384);
|
|
@@ -318046,6 +318473,143 @@ function streamHealthPublisherConfigFromEnv(env = process.env) {
|
|
|
318046
318473
|
};
|
|
318047
318474
|
}
|
|
318048
318475
|
|
|
318476
|
+
// src/helpers/user-asset-archive-poller.ts
|
|
318477
|
+
var DEFAULT_CONFIG4 = {
|
|
318478
|
+
pollIntervalMs: 60000
|
|
318479
|
+
};
|
|
318480
|
+
var USER_ASSET_EXCHANGE_ID = "binance";
|
|
318481
|
+
var metricLabels2 = (target) => ({
|
|
318482
|
+
exchange: target.exchangeId,
|
|
318483
|
+
account_selector: target.account.label,
|
|
318484
|
+
balance_scope: USER_ASSET_BALANCE_SCOPE
|
|
318485
|
+
});
|
|
318486
|
+
|
|
318487
|
+
class UserAssetArchivePoller {
|
|
318488
|
+
params;
|
|
318489
|
+
#timer = null;
|
|
318490
|
+
#stopped = false;
|
|
318491
|
+
#running = null;
|
|
318492
|
+
#lastSuccessMs = new Map;
|
|
318493
|
+
#config;
|
|
318494
|
+
constructor(params) {
|
|
318495
|
+
this.params = params;
|
|
318496
|
+
this.#config = { ...DEFAULT_CONFIG4, ...params.config };
|
|
318497
|
+
}
|
|
318498
|
+
start() {
|
|
318499
|
+
if (this.#timer || this.#stopped || !this.params.archiver.canPersistAccountBalanceSnapshots() || this.#targets().length === 0) {
|
|
318500
|
+
return;
|
|
318501
|
+
}
|
|
318502
|
+
log.info("\uD83E\uDDCA User asset archive poller started", {
|
|
318503
|
+
balanceScope: USER_ASSET_BALANCE_SCOPE
|
|
318504
|
+
});
|
|
318505
|
+
this.#schedule(0);
|
|
318506
|
+
}
|
|
318507
|
+
async stop() {
|
|
318508
|
+
this.#stopped = true;
|
|
318509
|
+
if (this.#timer) {
|
|
318510
|
+
clearTimeout(this.#timer);
|
|
318511
|
+
this.#timer = null;
|
|
318512
|
+
}
|
|
318513
|
+
await this.#running;
|
|
318514
|
+
}
|
|
318515
|
+
async pollAllOnce() {
|
|
318516
|
+
if (this.#stopped || this.#running || !this.params.archiver.canPersistAccountBalanceSnapshots()) {
|
|
318517
|
+
return false;
|
|
318518
|
+
}
|
|
318519
|
+
this.#running = this.#pollAllSequentially();
|
|
318520
|
+
try {
|
|
318521
|
+
return await this.#running;
|
|
318522
|
+
} finally {
|
|
318523
|
+
this.#running = null;
|
|
318524
|
+
}
|
|
318525
|
+
}
|
|
318526
|
+
#targets() {
|
|
318527
|
+
const targets = [];
|
|
318528
|
+
for (const [exchangeId, pool] of Object.entries(this.params.brokers)) {
|
|
318529
|
+
if (exchangeId.trim().toLowerCase() !== USER_ASSET_EXCHANGE_ID) {
|
|
318530
|
+
continue;
|
|
318531
|
+
}
|
|
318532
|
+
for (const account of [pool.primary, ...pool.secondaryBrokers]) {
|
|
318533
|
+
targets.push({ exchangeId, account });
|
|
318534
|
+
}
|
|
318535
|
+
}
|
|
318536
|
+
return targets;
|
|
318537
|
+
}
|
|
318538
|
+
async#pollAllSequentially() {
|
|
318539
|
+
for (const target of this.#targets()) {
|
|
318540
|
+
if (this.#stopped) {
|
|
318541
|
+
break;
|
|
318542
|
+
}
|
|
318543
|
+
await this.#pollOne(target);
|
|
318544
|
+
}
|
|
318545
|
+
return true;
|
|
318546
|
+
}
|
|
318547
|
+
async#pollOne(target) {
|
|
318548
|
+
const labels = metricLabels2(target);
|
|
318549
|
+
this.params.metrics?.recordCounter("cex_user_asset_poll_attempts_total", 1, labels);
|
|
318550
|
+
try {
|
|
318551
|
+
const exchange = target.account.exchange;
|
|
318552
|
+
if (typeof exchange.sapiV3PostAssetGetUserAsset !== "function") {
|
|
318553
|
+
throw new Error("binance_user_asset_unavailable: getUserAsset is not defined on this exchange instance");
|
|
318554
|
+
}
|
|
318555
|
+
const response = await exchange.sapiV3PostAssetGetUserAsset({});
|
|
318556
|
+
const observedAt = new Date;
|
|
318557
|
+
const normalized = normalizeBinanceUserAssetsForArchive(response);
|
|
318558
|
+
this.params.archiver.enqueue(buildUserAssetSnapshotRow({
|
|
318559
|
+
tags: buildCommonArchiveTags({
|
|
318560
|
+
deploymentId: this.params.archiver.getDeploymentId(),
|
|
318561
|
+
accountSelector: target.account.label,
|
|
318562
|
+
exchange: target.exchangeId,
|
|
318563
|
+
brokerObservedTimestamp: observedAt.toISOString()
|
|
318564
|
+
}),
|
|
318565
|
+
userAssets: normalized
|
|
318566
|
+
}));
|
|
318567
|
+
const successMs = Date.now();
|
|
318568
|
+
this.#lastSuccessMs.set(this.#targetKey(target), successMs);
|
|
318569
|
+
this.params.metrics?.recordCounter("cex_user_asset_poll_successes_total", 1, labels);
|
|
318570
|
+
this.params.metrics?.recordGauge("cex_user_asset_poll_last_success_timestamp_seconds", Math.floor(successMs / 1000), labels);
|
|
318571
|
+
this.#recordFreshness(labels, successMs, successMs);
|
|
318572
|
+
} catch (error) {
|
|
318573
|
+
rethrowArchiveDurabilityError(error);
|
|
318574
|
+
this.params.metrics?.recordCounter("cex_user_asset_poll_failures_total", 1, labels);
|
|
318575
|
+
const now3 = Date.now();
|
|
318576
|
+
const lastSuccess = this.#lastSuccessMs.get(this.#targetKey(target));
|
|
318577
|
+
if (lastSuccess !== undefined) {
|
|
318578
|
+
this.#recordFreshness(labels, lastSuccess, now3);
|
|
318579
|
+
}
|
|
318580
|
+
log.warn("User asset archive poll failed", {
|
|
318581
|
+
exchange: target.exchangeId,
|
|
318582
|
+
account: target.account.label,
|
|
318583
|
+
balanceScope: USER_ASSET_BALANCE_SCOPE,
|
|
318584
|
+
errorType: error instanceof Error ? error.name : "unknown"
|
|
318585
|
+
});
|
|
318586
|
+
}
|
|
318587
|
+
}
|
|
318588
|
+
#recordFreshness(labels, lastSuccessMs, nowMs) {
|
|
318589
|
+
this.params.metrics?.recordGauge("cex_user_asset_poll_freshness_seconds", Math.max(0, (nowMs - lastSuccessMs) / 1000), labels);
|
|
318590
|
+
}
|
|
318591
|
+
#targetKey(target) {
|
|
318592
|
+
return `${target.exchangeId}|${target.account.label}|${USER_ASSET_BALANCE_SCOPE}`;
|
|
318593
|
+
}
|
|
318594
|
+
#schedule(delayMs) {
|
|
318595
|
+
this.#timer = setTimeout(() => void this.#tick(), delayMs);
|
|
318596
|
+
this.#timer.unref?.();
|
|
318597
|
+
}
|
|
318598
|
+
async#tick() {
|
|
318599
|
+
this.#timer = null;
|
|
318600
|
+
try {
|
|
318601
|
+
await this.pollAllOnce();
|
|
318602
|
+
} catch (error) {
|
|
318603
|
+
rethrowArchiveDurabilityError(error);
|
|
318604
|
+
log.error("User asset archive poller tick failed", error);
|
|
318605
|
+
} finally {
|
|
318606
|
+
if (!this.#stopped) {
|
|
318607
|
+
this.#schedule(this.#config.pollIntervalMs);
|
|
318608
|
+
}
|
|
318609
|
+
}
|
|
318610
|
+
}
|
|
318611
|
+
}
|
|
318612
|
+
|
|
318049
318613
|
// src/helpers/binance-user-data-stream.ts
|
|
318050
318614
|
import { Buffer as Buffer2 } from "node:buffer";
|
|
318051
318615
|
import { createHmac } from "node:crypto";
|
|
@@ -329666,7 +330230,7 @@ function finalize(ctx, schema) {
|
|
|
329666
330230
|
result.$schema = "http://json-schema.org/draft-07/schema#";
|
|
329667
330231
|
} else if (ctx.target === "draft-04") {
|
|
329668
330232
|
result.$schema = "http://json-schema.org/draft-04/schema#";
|
|
329669
|
-
} else if (ctx.target === "openapi-3.0") {}
|
|
330233
|
+
} else if (ctx.target === "openapi-3.0") {}
|
|
329670
330234
|
if (ctx.external?.uri) {
|
|
329671
330235
|
const id2 = ctx.external.registry.get(schema)?.id;
|
|
329672
330236
|
if (!id2)
|
|
@@ -329914,7 +330478,7 @@ var literalProcessor = (schema, ctx, json3, _params) => {
|
|
|
329914
330478
|
if (val === undefined) {
|
|
329915
330479
|
if (ctx.unrepresentable === "throw") {
|
|
329916
330480
|
throw new Error("Literal `undefined` cannot be represented in JSON Schema");
|
|
329917
|
-
}
|
|
330481
|
+
}
|
|
329918
330482
|
} else if (typeof val === "bigint") {
|
|
329919
330483
|
if (ctx.unrepresentable === "throw") {
|
|
329920
330484
|
throw new Error("BigInt literals cannot be represented in JSON Schema");
|
|
@@ -332573,7 +333137,7 @@ async function handleDeposit(ctx) {
|
|
|
332573
333137
|
network: depositNetwork?.exchangeNetworkId,
|
|
332574
333138
|
externalId: depositTxid,
|
|
332575
333139
|
txid: depositTxid,
|
|
332576
|
-
exchangeTimestamp:
|
|
333140
|
+
exchangeTimestamp: normalizeTimestamp(creditedAt),
|
|
332577
333141
|
payload: deposit
|
|
332578
333142
|
}
|
|
332579
333143
|
});
|
|
@@ -336091,6 +336655,8 @@ class CEXBroker {
|
|
|
336091
336655
|
fillArchivePoller;
|
|
336092
336656
|
depositArchivePoller;
|
|
336093
336657
|
accountBalanceArchivePoller;
|
|
336658
|
+
userAssetArchivePoller;
|
|
336659
|
+
balanceUpdateArchiveConsumer;
|
|
336094
336660
|
userDataStreamSupervisor;
|
|
336095
336661
|
loadEnvConfig() {
|
|
336096
336662
|
log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
|
|
@@ -336244,6 +336810,14 @@ class CEXBroker {
|
|
|
336244
336810
|
await this.accountBalanceArchivePoller.stop();
|
|
336245
336811
|
this.accountBalanceArchivePoller = undefined;
|
|
336246
336812
|
}
|
|
336813
|
+
if (this.userAssetArchivePoller) {
|
|
336814
|
+
await this.userAssetArchivePoller.stop();
|
|
336815
|
+
this.userAssetArchivePoller = undefined;
|
|
336816
|
+
}
|
|
336817
|
+
if (this.balanceUpdateArchiveConsumer) {
|
|
336818
|
+
await this.balanceUpdateArchiveConsumer.stop();
|
|
336819
|
+
this.balanceUpdateArchiveConsumer = undefined;
|
|
336820
|
+
}
|
|
336247
336821
|
if (this.server) {
|
|
336248
336822
|
await this.server.forceShutdown();
|
|
336249
336823
|
}
|
|
@@ -336289,6 +336863,14 @@ class CEXBroker {
|
|
|
336289
336863
|
await this.accountBalanceArchivePoller.stop();
|
|
336290
336864
|
this.accountBalanceArchivePoller = undefined;
|
|
336291
336865
|
}
|
|
336866
|
+
if (this.userAssetArchivePoller) {
|
|
336867
|
+
await this.userAssetArchivePoller.stop();
|
|
336868
|
+
this.userAssetArchivePoller = undefined;
|
|
336869
|
+
}
|
|
336870
|
+
if (this.balanceUpdateArchiveConsumer) {
|
|
336871
|
+
await this.balanceUpdateArchiveConsumer.stop();
|
|
336872
|
+
this.balanceUpdateArchiveConsumer = undefined;
|
|
336873
|
+
}
|
|
336292
336874
|
log.info(`Running CEXBroker at ${new Date().toISOString()}`);
|
|
336293
336875
|
if (this.otelMetrics?.isOtelEnabled()) {
|
|
336294
336876
|
await this.otelMetrics.initialize();
|
|
@@ -336330,6 +336912,14 @@ class CEXBroker {
|
|
|
336330
336912
|
metrics: this.otelMetrics
|
|
336331
336913
|
});
|
|
336332
336914
|
this.depositArchivePoller.start();
|
|
336915
|
+
if (this.userDataStreamSupervisor) {
|
|
336916
|
+
this.balanceUpdateArchiveConsumer = new BalanceUpdateArchiveConsumer({
|
|
336917
|
+
brokers: this.brokers,
|
|
336918
|
+
archiver: this.brokerArchiver,
|
|
336919
|
+
userDataStreamSupervisor: this.userDataStreamSupervisor
|
|
336920
|
+
});
|
|
336921
|
+
this.balanceUpdateArchiveConsumer.start();
|
|
336922
|
+
}
|
|
336333
336923
|
}
|
|
336334
336924
|
if (this.brokerArchiver?.canPersistAccountBalanceSnapshots()) {
|
|
336335
336925
|
this.accountBalanceArchivePoller = new AccountBalanceArchivePoller({
|
|
@@ -336338,6 +336928,12 @@ class CEXBroker {
|
|
|
336338
336928
|
metrics: this.otelMetrics
|
|
336339
336929
|
});
|
|
336340
336930
|
this.accountBalanceArchivePoller.start();
|
|
336931
|
+
this.userAssetArchivePoller = new UserAssetArchivePoller({
|
|
336932
|
+
brokers: this.brokers,
|
|
336933
|
+
archiver: this.brokerArchiver,
|
|
336934
|
+
metrics: this.otelMetrics
|
|
336935
|
+
});
|
|
336936
|
+
this.userAssetArchivePoller.start();
|
|
336341
336937
|
}
|
|
336342
336938
|
return this;
|
|
336343
336939
|
}
|