@usherlabs/cex-broker 0.2.18 → 0.2.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -313774,9 +313774,6 @@ function createBrokerPool(cfg) {
313774
313774
  }
313775
313775
  return pool;
313776
313776
  }
313777
- function selectBroker(brokers, metadata) {
313778
- return selectBrokerAccount(brokers, metadata)?.exchange ?? null;
313779
- }
313780
313777
  function getCurrentBrokerSelector(metadata) {
313781
313778
  const use_secondary_key = metadata.get("use-secondary-key");
313782
313779
  if (!use_secondary_key || use_secondary_key.length === 0) {
@@ -314996,262 +314993,1034 @@ function validateDeposit(policy, exchange, network, ticker) {
314996
314993
  return { valid: true };
314997
314994
  }
314998
314995
 
314999
- // src/helpers/otel.ts
315000
- var import_api2 = __toESM(require_src5(), 1);
315001
- var import_api_logs2 = __toESM(require_src7(), 1);
315002
- var import_exporter_logs_otlp_http = __toESM(require_src14(), 1);
315003
- var import_exporter_metrics_otlp_http = __toESM(require_src15(), 1);
315004
- var import_resources = __toESM(require_src11(), 1);
315005
- var import_sdk_logs = __toESM(require_src16(), 1);
315006
- var import_sdk_metrics = __toESM(require_src12(), 1);
315007
- var DEFAULT_SERVICE = "cex-broker";
315008
- var DEFAULT_OTLP_PORT = 4318;
315009
- var EXPORT_INTERVAL_MS = 5000;
314996
+ // src/helpers/shared/guards.ts
314997
+ function isRecord(value) {
314998
+ return value !== null && typeof value === "object" && !Array.isArray(value);
314999
+ }
315000
+ function asRecord(value) {
315001
+ return isRecord(value) ? value : undefined;
315002
+ }
315010
315003
 
315011
- class BaseOtelSignal {
315012
- signal;
315013
- provider = null;
315014
- isEnabled = false;
315015
- serviceName;
315016
- constructor(config, signal) {
315017
- this.signal = signal;
315018
- this.serviceName = config?.serviceName ?? DEFAULT_SERVICE;
315019
- const endpointResolution = resolveOtlpEndpoint(this.signal, config);
315020
- if (!endpointResolution) {
315021
- log.info(`OTel ${signal} disabled: no OTLP endpoint or host provided`);
315022
- return;
315004
+ // src/helpers/order-telemetry.ts
315005
+ var NUMERIC_METRICS = [
315006
+ ["requestedQuantity", "cex_market_action_requested_quantity"],
315007
+ ["requestedNotional", "cex_market_action_requested_notional"],
315008
+ ["executedBaseQuantity", "cex_market_action_executed_base_quantity"],
315009
+ ["executedQuoteQuantity", "cex_market_action_executed_quote_quantity"],
315010
+ ["averageExecutionPrice", "cex_market_action_average_execution_price"],
315011
+ ["filledAmount", "cex_market_action_filled_amount"],
315012
+ ["remainingAmount", "cex_market_action_remaining_amount"],
315013
+ ["feeAmount", "cex_market_action_fee_amount"],
315014
+ ["feeRate", "cex_market_action_fee_rate"]
315015
+ ];
315016
+ var REDACTED_ERROR_MESSAGE = "redacted_error";
315017
+ async function emitOrderExecutionTelemetry(otelMetrics, context2, order, error) {
315018
+ try {
315019
+ const telemetry = buildOrderExecutionTelemetry(context2, order, error);
315020
+ log.info("CEX market action execution telemetry", telemetry);
315021
+ const labels = {
315022
+ action: telemetry.action,
315023
+ cex: telemetry.cex,
315024
+ account: telemetry.accountLabel,
315025
+ symbol: telemetry.symbol,
315026
+ side: telemetry.side,
315027
+ order_type: telemetry.orderType,
315028
+ status: telemetry.status
315029
+ };
315030
+ await otelMetrics?.recordCounter("cex_market_action_executions_total", 1, {
315031
+ ...labels,
315032
+ result: error ? "error" : "ok"
315033
+ });
315034
+ for (const [key, metricName] of NUMERIC_METRICS) {
315035
+ const value = telemetry[key];
315036
+ if (typeof value === "number" && Number.isFinite(value)) {
315037
+ await otelMetrics?.recordHistogram(metricName, value, labels);
315038
+ }
315023
315039
  }
315040
+ return telemetry;
315041
+ } catch (telemetryError) {
315024
315042
  try {
315025
- this.provider = this.createProvider(endpointResolution.endpoint, this.serviceName, endpointResolution.appendSignalPath);
315026
- this.onProviderCreated(this.provider);
315027
- this.isEnabled = true;
315028
- log.info(`OTel ${signal} enabled: ${endpointResolution.endpoint}`);
315029
- } catch (error) {
315030
- log.error(`Failed to initialize OTel ${signal}:`, error);
315031
- this.isEnabled = false;
315032
- this.provider = null;
315043
+ log.error("Failed to emit CEX order telemetry", {
315044
+ error: telemetryError
315045
+ });
315046
+ } catch {}
315047
+ return;
315048
+ }
315049
+ }
315050
+ function buildOrderExecutionTelemetry(context2, order, error) {
315051
+ const record = asRecord(order);
315052
+ const info = asRecord(record?.info);
315053
+ const fees = getFees(record, info);
315054
+ const fee = summarizeFees(fees);
315055
+ const executedBaseQuantity = firstNumber(record?.filled, info?.executedQty, info?.cumExecQty) ?? computeFilledFromAmount(record);
315056
+ const executedQuoteQuantity = firstNumber(record?.cost, info?.cummulativeQuoteQty, info?.cumQuote, info?.cumExecValue);
315057
+ const averageExecutionPrice = firstNumber(record?.average, info?.avgPrice) ?? computeAveragePrice(executedBaseQuantity, executedQuoteQuantity);
315058
+ const exchangeTimestamp = normalizeTimestamp(firstValue(record?.timestamp, info?.time, info?.transactTime, record?.datetime));
315059
+ const status = firstString(record?.status, info?.status) ?? (error ? "failed" : "unknown");
315060
+ const errorRecord = error instanceof Error ? error : undefined;
315061
+ return compactUndefined({
315062
+ event: "cex_market_action_execution",
315063
+ action: context2.action,
315064
+ cex: context2.cex.trim().toLowerCase() || "unknown",
315065
+ accountLabel: context2.accountLabel ?? "unknown",
315066
+ symbol: firstString(record?.symbol, info?.symbol, context2.symbol) ?? "unknown",
315067
+ side: firstString(record?.side, info?.side, context2.side) ?? "unknown",
315068
+ orderType: firstString(record?.type, info?.type, context2.orderType) ?? "unknown",
315069
+ orderId: firstString(record?.id, info?.orderId, info?.orderID),
315070
+ clientOrderId: firstString(context2.clientOrderId, record?.clientOrderId, record?.clientOrderID, record?.clientOid, info?.clientOrderId, info?.clientOrderID, info?.clientOid),
315071
+ idempotencyId: context2.idempotencyId,
315072
+ makerActionId: context2.makerActionId,
315073
+ status: status.toLowerCase(),
315074
+ requestedQuantity: context2.requestedQuantity ?? firstNumber(record?.amount),
315075
+ requestedNotional: context2.requestedNotional,
315076
+ executedBaseQuantity,
315077
+ executedQuoteQuantity,
315078
+ averageExecutionPrice,
315079
+ filledAmount: firstNumber(record?.filled, info?.executedQty),
315080
+ remainingAmount: firstNumber(record?.remaining, info?.remainingQty),
315081
+ feeAmount: fee.amount,
315082
+ feeCurrency: fee.currency,
315083
+ feeRate: fee.rate,
315084
+ exchangeTimestamp,
315085
+ brokerObservedTimestamp: context2.brokerObservedTimestamp ?? new Date().toISOString(),
315086
+ errorType: errorRecord?.name,
315087
+ errorMessage: errorRecord ? REDACTED_ERROR_MESSAGE : undefined
315088
+ });
315089
+ }
315090
+ function emitOrderExecutionTelemetryInBackground(otelMetrics, context2, order, error) {
315091
+ emitOrderExecutionTelemetry(otelMetrics, context2, order, error).catch((telemetryError) => {
315092
+ try {
315093
+ log.warn("Telemetry emit failed", { error: telemetryError });
315094
+ } catch {
315095
+ console.warn("Telemetry emit failed", telemetryError);
315096
+ }
315097
+ });
315098
+ }
315099
+ function extractOrderTelemetryIds(params) {
315100
+ const record = params ?? {};
315101
+ return {
315102
+ clientOrderId: firstString(record.clientOrderId, record.clientOrderID, record.newClientOrderId, record.clientOid),
315103
+ idempotencyId: firstString(record.idempotencyId, record.idempotencyID, record.idempotencyKey, record.requestId, record.requestID),
315104
+ makerActionId: firstString(record.makerActionId, record.maker_action_id, record.actionId, record.action_id)
315105
+ };
315106
+ }
315107
+ function firstValue(...values2) {
315108
+ return values2.find((value) => value !== undefined && value !== null);
315109
+ }
315110
+ function firstString(...values2) {
315111
+ for (const value of values2) {
315112
+ if (typeof value === "string" && value.trim()) {
315113
+ return value.trim();
315114
+ }
315115
+ if (typeof value === "number" && Number.isFinite(value)) {
315116
+ return String(value);
315033
315117
  }
315034
315118
  }
315035
- onProviderCreated(_provider) {}
315036
- onProviderClosed() {}
315037
- getProvider() {
315038
- return this.provider;
315119
+ return;
315120
+ }
315121
+ function firstNumber(...values2) {
315122
+ for (const value of values2) {
315123
+ const numberValue = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
315124
+ if (Number.isFinite(numberValue)) {
315125
+ return numberValue;
315126
+ }
315039
315127
  }
315040
- getServiceName() {
315041
- return this.serviceName;
315128
+ return;
315129
+ }
315130
+ function computeFilledFromAmount(record) {
315131
+ const amount = firstNumber(record?.amount);
315132
+ const remaining = firstNumber(record?.remaining);
315133
+ if (amount === undefined || remaining === undefined) {
315134
+ return;
315042
315135
  }
315043
- isOtelEnabled() {
315044
- return this.isEnabled && this.provider !== null;
315136
+ return amount - remaining;
315137
+ }
315138
+ function computeAveragePrice(executedBaseQuantity, executedQuoteQuantity) {
315139
+ if (executedBaseQuantity === undefined || executedQuoteQuantity === undefined || executedBaseQuantity === 0) {
315140
+ return;
315045
315141
  }
315046
- async close() {
315047
- if (!this.provider) {
315048
- return;
315142
+ return executedQuoteQuantity / executedBaseQuantity;
315143
+ }
315144
+ function normalizeTimestamp(value) {
315145
+ if (typeof value === "string" && value.trim()) {
315146
+ return value.trim();
315147
+ }
315148
+ const timestamp = firstNumber(value);
315149
+ if (timestamp === undefined) {
315150
+ return;
315151
+ }
315152
+ const date = new Date(timestamp);
315153
+ return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
315154
+ }
315155
+ function getFees(record, info) {
315156
+ const fees = [];
315157
+ if (record?.fee)
315158
+ fees.push(record.fee);
315159
+ if (Array.isArray(record?.fees))
315160
+ fees.push(...record.fees);
315161
+ if (Array.isArray(record?.trades)) {
315162
+ for (const trade of record.trades) {
315163
+ const tradeRecord = asRecord(trade);
315164
+ if (tradeRecord?.fee)
315165
+ fees.push(tradeRecord.fee);
315166
+ if (Array.isArray(tradeRecord?.fees))
315167
+ fees.push(...tradeRecord.fees);
315049
315168
  }
315050
- try {
315051
- await this.shutdownProvider(this.provider);
315052
- log.info(`OTel ${this.signal} provider shut down`);
315053
- } catch (error) {
315054
- log.error(`Error shutting down OTel ${this.signal} provider:`, error);
315169
+ }
315170
+ if (Array.isArray(info?.fills)) {
315171
+ for (const fill of info.fills) {
315172
+ const fillRecord = asRecord(fill);
315173
+ const commission = firstNumber(fillRecord?.commission);
315174
+ if (commission !== undefined) {
315175
+ fees.push({
315176
+ cost: commission,
315177
+ currency: firstString(fillRecord?.commissionAsset)
315178
+ });
315179
+ }
315055
315180
  }
315056
- this.provider = null;
315057
- this.isEnabled = false;
315058
- this.onProviderClosed();
315059
315181
  }
315182
+ return fees;
315060
315183
  }
315061
- function toAttributes(labels, service) {
315062
- const attrs = { ...labels, service };
315063
- for (const key of Object.keys(attrs)) {
315064
- const v = attrs[key];
315065
- if (typeof v !== "string" && typeof v !== "number") {
315066
- attrs[key] = String(v);
315184
+ function summarizeFees(fees) {
315185
+ let amount = 0;
315186
+ let amountFound = false;
315187
+ let currency;
315188
+ let rate;
315189
+ for (const rawFee of fees) {
315190
+ const fee = asRecord(rawFee);
315191
+ if (!fee)
315192
+ continue;
315193
+ const cost = firstNumber(fee.cost, fee.amount);
315194
+ if (cost !== undefined) {
315195
+ amount += cost;
315196
+ amountFound = true;
315067
315197
  }
315198
+ currency ??= firstString(fee.currency);
315199
+ rate ??= firstNumber(fee.rate);
315068
315200
  }
315069
- return attrs;
315201
+ return {
315202
+ amount: amountFound ? amount : undefined,
315203
+ currency,
315204
+ rate
315205
+ };
315206
+ }
315207
+ function compactUndefined(record) {
315208
+ return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
315070
315209
  }
315071
315210
 
315072
- class OtelMetrics extends BaseOtelSignal {
315073
- counters = new Map;
315074
- histograms = new Map;
315075
- constructor(config) {
315076
- super(config, "metrics");
315211
+ // src/helpers/broker-execution-archive/redact.ts
315212
+ import { createHash } from "node:crypto";
315213
+ var SECRET_KEY_PATTERN = /\b(api[_-]?key|api[_-]?secret|secret|signature|passphrase|password|token|credential)\b/i;
315214
+ var SECRET_VALUE_PATTERN = /(\b(?:apiKey|api_key|apiSecret|api_secret|secret|signature|passphrase|password|token)\b\s*[=:]\s*)[^\s&,;)}\]]+/gi;
315215
+ var SECRET_JSON_PATTERN = /("(?:apiKey|api_key|apiSecret|api_secret|secret|signature|passphrase|password|token)"\s*:\s*")[^"]*(")/gi;
315216
+ function redactSecretLiterals(value, secretLiterals = []) {
315217
+ let redacted = value;
315218
+ for (const literal of secretLiterals) {
315219
+ if (literal.length > 0) {
315220
+ redacted = redacted.split(literal).join("[redacted]");
315221
+ }
315077
315222
  }
315078
- createProvider(endpoint, serviceName, appendSignalPath) {
315079
- const exporter = new import_exporter_metrics_otlp_http.OTLPMetricExporter({
315080
- url: appendOtlpPath(endpoint, "metrics", appendSignalPath)
315081
- });
315082
- const reader = new import_sdk_metrics.PeriodicExportingMetricReader({
315083
- exporter,
315084
- exportIntervalMillis: EXPORT_INTERVAL_MS
315085
- });
315086
- const resource = import_resources.resourceFromAttributes({
315087
- "service.name": serviceName
315088
- });
315089
- return new import_sdk_metrics.MeterProvider({
315090
- resource,
315091
- readers: [reader]
315092
- });
315223
+ return redacted.replace(SECRET_VALUE_PATTERN, "$1[redacted]").replace(SECRET_JSON_PATTERN, "$1[redacted]$2");
315224
+ }
315225
+ function redactUnknownValue(value, secretLiterals) {
315226
+ if (value === null || value === undefined) {
315227
+ return value;
315093
315228
  }
315094
- onProviderCreated(provider) {
315095
- import_api2.metrics.setGlobalMeterProvider(provider);
315229
+ if (typeof value === "string") {
315230
+ return redactSecretLiterals(value, secretLiterals);
315096
315231
  }
315097
- shutdownProvider(provider) {
315098
- return provider.shutdown();
315232
+ if (typeof value === "number" || typeof value === "boolean") {
315233
+ return value;
315099
315234
  }
315100
- async initialize() {
315101
- if (this.isOtelEnabled()) {
315102
- log.info("OTel metrics initialized (storage is handled by the collector)");
315235
+ if (Array.isArray(value)) {
315236
+ return value.map((entry) => redactUnknownValue(entry, secretLiterals));
315237
+ }
315238
+ if (typeof value === "object") {
315239
+ const record = value;
315240
+ const redacted = {};
315241
+ for (const [key, entry] of Object.entries(record)) {
315242
+ if (SECRET_KEY_PATTERN.test(key)) {
315243
+ redacted[key] = "[redacted]";
315244
+ continue;
315245
+ }
315246
+ redacted[key] = redactUnknownValue(entry, secretLiterals);
315103
315247
  }
315248
+ return redacted;
315104
315249
  }
315105
- async insertMetric(metric) {
315106
- if (!this.isOtelEnabled())
315107
- return;
315108
- const labels = metric.labels ? JSON.parse(metric.labels) : {};
315109
- try {
315110
- if (metric.metric_type === "counter") {
315111
- await this.recordCounter(metric.metric_name, metric.value, labels, metric.service);
315112
- } else if (metric.metric_type === "gauge") {
315113
- await this.recordGauge(metric.metric_name, metric.value, labels, metric.service);
315114
- } else {
315115
- await this.recordHistogram(metric.metric_name, metric.value, labels, metric.service);
315116
- }
315117
- } catch {}
315250
+ return String(value);
315251
+ }
315252
+ function redactStreamPayload(payload, secretLiterals = []) {
315253
+ if (Array.isArray(payload)) {
315254
+ return {
315255
+ items: payload.map((entry) => redactUnknownValue(entry, secretLiterals))
315256
+ };
315118
315257
  }
315119
- async insertMetrics(metricsList) {
315120
- if (!this.isOtelEnabled() || metricsList.length === 0)
315258
+ const record = asRecord(payload);
315259
+ if (!record) {
315260
+ return {};
315261
+ }
315262
+ return redactUnknownValue(record, secretLiterals);
315263
+ }
315264
+ function hashMarketMetadata(payload) {
315265
+ if (payload === undefined || payload === null) {
315266
+ return;
315267
+ }
315268
+ try {
315269
+ const normalized = JSON.stringify(payload);
315270
+ if (!normalized || normalized === "{}") {
315121
315271
  return;
315122
- for (const m of metricsList) {
315123
- await this.insertMetric(m);
315124
315272
  }
315273
+ return createHash("sha256").update(normalized).digest("hex");
315274
+ } catch {
315275
+ return;
315125
315276
  }
315126
- async recordCounter(metricName, value, labels, service = this.getServiceName()) {
315127
- const provider = this.getProvider();
315128
- if (!this.isOtelEnabled() || !provider)
315129
- return;
315277
+ }
315278
+
315279
+ // src/helpers/broker-execution-archive/types.ts
315280
+ var BROKER_WRITE_SOURCE = "broker_write";
315281
+
315282
+ // src/helpers/broker-execution-archive/rows.ts
315283
+ function firstString2(...values2) {
315284
+ for (const value of values2) {
315285
+ if (typeof value === "string" && value.trim()) {
315286
+ return value.trim();
315287
+ }
315288
+ if (typeof value === "number" && Number.isFinite(value)) {
315289
+ return String(value);
315290
+ }
315291
+ }
315292
+ return;
315293
+ }
315294
+ function compactUndefined2(record) {
315295
+ return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
315296
+ }
315297
+ function buildCommonArchiveTags(input) {
315298
+ return {
315299
+ source: BROKER_WRITE_SOURCE,
315300
+ deployment_id: input.deploymentId,
315301
+ account_selector: input.accountSelector ?? "unknown",
315302
+ exchange: input.exchange.trim().toLowerCase() || "unknown",
315303
+ symbol: input.symbol?.trim() || "unknown",
315304
+ broker_observed_timestamp: input.brokerObservedTimestamp ?? new Date().toISOString()
315305
+ };
315306
+ }
315307
+ function buildOrderEventArchiveRow(input) {
315308
+ const { tags, telemetry, action } = input;
315309
+ return {
315310
+ table: "broker_execution.order_events",
315311
+ row: compactUndefined2({
315312
+ ...tags,
315313
+ event_kind: input.eventKind ?? "execute_action",
315314
+ action,
315315
+ subscription_type: input.subscriptionType,
315316
+ order_id: telemetry.orderId,
315317
+ client_order_id: telemetry.clientOrderId,
315318
+ idempotency_id: telemetry.idempotencyId,
315319
+ maker_action_id: telemetry.makerActionId,
315320
+ market_metadata_hash: input.marketMetadataHash,
315321
+ status: telemetry.status,
315322
+ side: telemetry.side,
315323
+ order_type: telemetry.orderType,
315324
+ requested_quantity: telemetry.requestedQuantity,
315325
+ requested_notional: telemetry.requestedNotional,
315326
+ executed_base_quantity: telemetry.executedBaseQuantity,
315327
+ executed_quote_quantity: telemetry.executedQuoteQuantity,
315328
+ average_execution_price: telemetry.averageExecutionPrice,
315329
+ filled_amount: telemetry.filledAmount,
315330
+ remaining_amount: telemetry.remainingAmount,
315331
+ fee_amount: telemetry.feeAmount,
315332
+ fee_currency: telemetry.feeCurrency,
315333
+ fee_rate: telemetry.feeRate,
315334
+ exchange_timestamp: telemetry.exchangeTimestamp,
315335
+ error_type: telemetry.errorType,
315336
+ error_message: telemetry.errorMessage,
315337
+ payload_json: JSON.stringify(telemetry)
315338
+ })
315339
+ };
315340
+ }
315341
+ function buildSubscribeStreamArchiveRow(input) {
315342
+ const redactedPayload = redactStreamPayload(input.streamPayload, input.secretLiterals);
315343
+ const record = asRecord(input.streamPayload);
315344
+ const info = asRecord(record?.info);
315345
+ return {
315346
+ table: "broker_execution.order_events",
315347
+ row: compactUndefined2({
315348
+ ...input.tags,
315349
+ event_kind: "subscribe_stream",
315350
+ subscription_type: input.subscriptionType,
315351
+ order_id: firstString2(record?.id, record?.orderId, record?.i, info?.orderId, info?.i),
315352
+ client_order_id: firstString2(record?.clientOrderId, record?.clientOrderID, record?.c, info?.clientOrderId, info?.c),
315353
+ status: firstString2(record?.status, record?.X, info?.status, info?.X)?.toLowerCase(),
315354
+ payload_json: JSON.stringify(redactedPayload)
315355
+ })
315356
+ };
315357
+ }
315358
+ function buildMarketMetadataSnapshotRow(input) {
315359
+ const redactedSnapshot = redactStreamPayload(input.marketSnapshot);
315360
+ const metadataHash = hashMarketMetadata(redactedSnapshot);
315361
+ return {
315362
+ table: "broker_execution.market_metadata_snapshots",
315363
+ row: compactUndefined2({
315364
+ ...input.tags,
315365
+ client_order_id: input.clientOrderId,
315366
+ order_id: input.orderId,
315367
+ maker_action_id: input.makerActionId,
315368
+ idempotency_id: input.idempotencyId,
315369
+ market_metadata_hash: metadataHash,
315370
+ snapshot_json: JSON.stringify(redactedSnapshot)
315371
+ })
315372
+ };
315373
+ }
315374
+
315375
+ // src/helpers/broker-execution-archive/capture.ts
315376
+ function archiveOrderExecutionInBackground(archiver, context2, order, error, options) {
315377
+ if (!archiver?.isEnabled()) {
315378
+ return;
315379
+ }
315380
+ queueMicrotask(() => {
315130
315381
  try {
315131
- let counter = this.counters.get(metricName);
315132
- if (!counter) {
315133
- const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
315134
- counter = meter.createCounter(metricName, { description: metricName });
315135
- this.counters.set(metricName, counter);
315136
- }
315137
- counter.add(value, toAttributes(labels, service));
315138
- } catch (error) {
315139
- log.error("Failed to record counter:", error);
315382
+ const telemetry = buildOrderExecutionTelemetry(context2, order, error);
315383
+ const tags = buildCommonArchiveTags({
315384
+ deploymentId: archiver.getDeploymentId(),
315385
+ accountSelector: context2.accountLabel,
315386
+ exchange: context2.cex,
315387
+ symbol: telemetry.symbol,
315388
+ brokerObservedTimestamp: telemetry.brokerObservedTimestamp
315389
+ });
315390
+ archiver.enqueue(buildOrderEventArchiveRow({
315391
+ tags,
315392
+ action: context2.action,
315393
+ telemetry,
315394
+ marketMetadataHash: options?.marketMetadataHash
315395
+ }));
315396
+ } catch (archiveError) {
315397
+ log.warn("Failed to archive order execution", { error: archiveError });
315140
315398
  }
315399
+ });
315400
+ }
315401
+ function archiveSubscribeStreamInBackground(archiver, input) {
315402
+ if (!archiver?.isEnabled()) {
315403
+ return;
315141
315404
  }
315142
- async recordGauge(metricName, value, labels, service = this.getServiceName()) {
315143
- const provider = this.getProvider();
315144
- if (!this.isOtelEnabled() || !provider)
315145
- return;
315405
+ queueMicrotask(() => {
315146
315406
  try {
315147
- let hist = this.histograms.get(`gauge_${metricName}`);
315148
- if (!hist) {
315149
- const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
315150
- hist = meter.createHistogram(`${metricName}_gauge`, {
315151
- description: metricName
315152
- });
315153
- this.histograms.set(`gauge_${metricName}`, hist);
315154
- }
315155
- hist.record(value, toAttributes(labels, service));
315156
- } catch (error) {
315157
- log.error("Failed to record gauge:", error);
315407
+ const tags = buildCommonArchiveTags({
315408
+ deploymentId: archiver.getDeploymentId(),
315409
+ accountSelector: input.accountSelector,
315410
+ exchange: input.exchange,
315411
+ symbol: input.symbol
315412
+ });
315413
+ archiver.enqueue(buildSubscribeStreamArchiveRow({
315414
+ tags,
315415
+ subscriptionType: input.subscriptionType,
315416
+ streamPayload: input.streamPayload,
315417
+ secretLiterals: input.secretLiterals
315418
+ }));
315419
+ } catch (archiveError) {
315420
+ log.warn("Failed to archive subscribe stream event", {
315421
+ error: archiveError
315422
+ });
315158
315423
  }
315424
+ });
315425
+ }
315426
+ async function captureMarketMetadataSnapshot(archiver, broker, input) {
315427
+ if (!archiver?.canPersistMarketMetadataSnapshot()) {
315428
+ return;
315159
315429
  }
315160
- async recordHistogram(metricName, value, labels, service = this.getServiceName()) {
315161
- const provider = this.getProvider();
315162
- if (!this.isOtelEnabled() || !provider)
315430
+ try {
315431
+ const fetchOrderBook = broker.fetchOrderBook;
315432
+ if (typeof fetchOrderBook !== "function") {
315163
315433
  return;
315164
- try {
315165
- let hist = this.histograms.get(metricName);
315166
- if (!hist) {
315167
- const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
315168
- hist = meter.createHistogram(metricName, { description: metricName });
315169
- this.histograms.set(metricName, hist);
315170
- }
315171
- hist.record(value, toAttributes(labels, service));
315172
- } catch (error) {
315173
- log.error("Failed to record histogram:", error);
315174
315434
  }
315435
+ const orderBook = await fetchOrderBook.call(broker, input.symbol, 5);
315436
+ const record = asRecord(orderBook);
315437
+ const snapshot = {
315438
+ action: input.action,
315439
+ symbol: input.symbol,
315440
+ bids: record?.bids,
315441
+ asks: record?.asks,
315442
+ timestamp: record?.timestamp ?? Date.now(),
315443
+ datetime: record?.datetime,
315444
+ nonce: record?.nonce
315445
+ };
315446
+ const tags = buildCommonArchiveTags({
315447
+ deploymentId: archiver.getDeploymentId(),
315448
+ accountSelector: input.accountSelector,
315449
+ exchange: input.exchange,
315450
+ symbol: input.symbol,
315451
+ brokerObservedTimestamp: input.brokerObservedTimestamp
315452
+ });
315453
+ const row = buildMarketMetadataSnapshotRow({
315454
+ tags,
315455
+ clientOrderId: input.clientOrderId,
315456
+ orderId: input.orderId,
315457
+ makerActionId: input.makerActionId,
315458
+ idempotencyId: input.idempotencyId,
315459
+ marketSnapshot: snapshot
315460
+ });
315461
+ archiver.enqueue(row);
315462
+ const hash4 = row.row.market_metadata_hash;
315463
+ return typeof hash4 === "string" ? hash4 : undefined;
315464
+ } catch (archiveError) {
315465
+ log.warn("Failed to capture market metadata snapshot", {
315466
+ error: archiveError
315467
+ });
315468
+ return;
315175
315469
  }
315176
315470
  }
315471
+ // src/helpers/broker-execution-archive/writer.ts
315472
+ var import_api_logs2 = __toESM(require_src7(), 1);
315177
315473
 
315178
- class OtelLogs extends BaseOtelSignal {
315179
- logger = null;
315180
- constructor(config) {
315181
- super(config, "logs");
315474
+ // src/helpers/market-data-archive/types.ts
315475
+ var MARKET_ARCHIVE_TABLES = new Set([
315476
+ "market_data.orderbook_snapshots",
315477
+ "market_data.candles",
315478
+ "market_data.cex_stream_events",
315479
+ "market_data.cex_ticker_events",
315480
+ "market_data.cex_trades"
315481
+ ]);
315482
+ function isMarketArchiveTable(table) {
315483
+ return MARKET_ARCHIVE_TABLES.has(table);
315484
+ }
315485
+
315486
+ // src/helpers/broker-execution-archive/writer.ts
315487
+ var DEFAULT_MAX_QUEUE_SIZE = 1e4;
315488
+ var DEFAULT_BATCH_SIZE = 10;
315489
+ var DEFAULT_FLUSH_INTERVAL_MS = 1000;
315490
+ var DEFAULT_FORWARDER_TIMEOUT_MS = 3000;
315491
+ var DEFAULT_ARCHIVE_FORWARDER_PATH = "/archive";
315492
+ var DEFAULT_ARCHIVE_FORWARDER_PORT = 8090;
315493
+ function isArchiveOtelLogsEnabled() {
315494
+ return process.env.CEX_BROKER_ARCHIVE_OTEL_LOGS_ENABLED === "true";
315495
+ }
315496
+ function resolveArchiveForwarderUrlFromEnv() {
315497
+ const explicit = process.env.CEX_BROKER_ARCHIVE_FORWARDER_URL?.trim();
315498
+ if (explicit) {
315499
+ return explicit;
315500
+ }
315501
+ const host = process.env.CEX_BROKER_ARCHIVE_FORWARDER_HOST?.trim() || process.env.CEX_BROKER_CLICKHOUSE_HOST?.trim();
315502
+ if (!host) {
315503
+ return;
315182
315504
  }
315183
- createProvider(endpoint, serviceName, appendSignalPath) {
315184
- const exporter = new import_exporter_logs_otlp_http.OTLPLogExporter({
315185
- url: appendOtlpPath(endpoint, "logs", appendSignalPath)
315186
- });
315187
- const processor = new import_sdk_logs.BatchLogRecordProcessor(exporter);
315188
- const resource = import_resources.resourceFromAttributes({
315189
- "service.name": serviceName
315505
+ const protocol = process.env.CEX_BROKER_CLICKHOUSE_PROTOCOL || "http";
315506
+ const port = parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_FORWARDER_PORT, DEFAULT_ARCHIVE_FORWARDER_PORT);
315507
+ const path = process.env.CEX_BROKER_ARCHIVE_FORWARDER_PATH?.trim() || DEFAULT_ARCHIVE_FORWARDER_PATH;
315508
+ const normalizedPath = path.startsWith("/") ? path : `/${path}`;
315509
+ return `${protocol}://${host}:${port}${normalizedPath}`;
315510
+ }
315511
+
315512
+ class BrokerExecutionArchiver {
315513
+ deploymentId;
315514
+ otelLogs;
315515
+ otelMetrics;
315516
+ forwarderUrl;
315517
+ maxQueueSize;
315518
+ batchSize;
315519
+ flushIntervalMs;
315520
+ forwarderTimeoutMs;
315521
+ queue = [];
315522
+ stats = {
315523
+ enqueued: 0,
315524
+ shed: 0,
315525
+ flushed: 0,
315526
+ forwarderFailures: 0
315527
+ };
315528
+ flushTimer = null;
315529
+ flushInFlight = null;
315530
+ closed = false;
315531
+ loggedMissingMarketForwarder = false;
315532
+ enabled;
315533
+ forwarderAuthToken;
315534
+ constructor(options) {
315535
+ this.deploymentId = options.deploymentId?.trim() || process.env.CEX_BROKER_DEPLOYMENT_ID?.trim() || "unknown";
315536
+ this.otelLogs = options.otelLogs;
315537
+ this.otelMetrics = options.otelMetrics;
315538
+ this.forwarderUrl = options.forwarderUrl?.trim();
315539
+ this.maxQueueSize = options.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE;
315540
+ this.batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
315541
+ this.flushIntervalMs = options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS;
315542
+ this.forwarderTimeoutMs = options.forwarderTimeoutMs ?? DEFAULT_FORWARDER_TIMEOUT_MS;
315543
+ this.enabled = options.enabled;
315544
+ this.forwarderAuthToken = process.env.CEX_BROKER_ARCHIVE_FORWARDER_TOKEN?.trim() || undefined;
315545
+ if (this.enabled) {
315546
+ this.flushTimer = setInterval(() => {
315547
+ this.flush().catch((error) => {
315548
+ log.warn("Broker execution archive flush failed", { error });
315549
+ });
315550
+ }, this.flushIntervalMs);
315551
+ this.flushTimer.unref?.();
315552
+ }
315553
+ }
315554
+ static disabled() {
315555
+ return new BrokerExecutionArchiver({ enabled: false });
315556
+ }
315557
+ static create(options) {
315558
+ const hasSink = Boolean(options.otelLogs?.isOtelEnabled()) || Boolean(options.forwarderUrl);
315559
+ const enabled = process.env.CEX_BROKER_ARCHIVE_ENABLED !== "false" && hasSink;
315560
+ return new BrokerExecutionArchiver({ ...options, enabled });
315561
+ }
315562
+ getDeploymentId() {
315563
+ return this.deploymentId;
315564
+ }
315565
+ isEnabled() {
315566
+ return this.enabled && !this.closed && (Boolean(this.otelLogs) || Boolean(this.forwarderUrl));
315567
+ }
315568
+ canPersistMarketMetadataSnapshot() {
315569
+ return this.isEnabled() && (Boolean(this.otelLogs?.isOtelEnabled()) || Boolean(this.forwarderUrl));
315570
+ }
315571
+ enqueue(row) {
315572
+ if (!this.enabled || this.closed || !this.otelLogs && !this.forwarderUrl) {
315573
+ return;
315574
+ }
315575
+ if (isMarketArchiveTable(row.table) && !this.forwarderUrl) {
315576
+ if (!this.loggedMissingMarketForwarder) {
315577
+ this.loggedMissingMarketForwarder = true;
315578
+ log.warn("Market data archive row dropped: configure CEX_BROKER_ARCHIVE_FORWARDER_URL or CEX_BROKER_CLICKHOUSE_HOST", { table: row.table });
315579
+ }
315580
+ return;
315581
+ }
315582
+ if (this.queue.length >= this.maxQueueSize) {
315583
+ this.queue.shift();
315584
+ this.stats.shed += 1;
315585
+ this.recordArchiveMetric("cex_archive_rows_shed_total", {
315586
+ table: row.table
315587
+ });
315588
+ }
315589
+ this.queue.push(row);
315590
+ this.stats.enqueued += 1;
315591
+ this.recordArchiveMetric("cex_archive_rows_enqueued_total", {
315592
+ table: row.table
315190
315593
  });
315191
- return new import_sdk_logs.LoggerProvider({
315192
- resource,
315193
- processors: [processor]
315594
+ if (this.queue.length >= this.batchSize) {
315595
+ this.flush().catch((error) => {
315596
+ log.warn("Broker execution archive flush failed", { error });
315597
+ });
315598
+ }
315599
+ }
315600
+ enqueueInBackground(row) {
315601
+ queueMicrotask(() => this.enqueue(row));
315602
+ }
315603
+ async flush() {
315604
+ if (!this.enabled || this.closed || this.queue.length === 0) {
315605
+ return;
315606
+ }
315607
+ if (this.flushInFlight) {
315608
+ return this.flushInFlight;
315609
+ }
315610
+ const inFlight = this.flushBatch().then(() => {
315611
+ return;
315612
+ }).finally(() => {
315613
+ this.flushInFlight = null;
315194
315614
  });
315615
+ this.flushInFlight = inFlight;
315616
+ return inFlight;
315195
315617
  }
315196
- onProviderCreated(provider) {
315197
- import_api_logs2.logs.setGlobalLoggerProvider(provider);
315198
- this.logger = provider.getLogger("cex-broker-logs", "1.0.0");
315618
+ async close() {
315619
+ if (this.flushTimer) {
315620
+ clearInterval(this.flushTimer);
315621
+ this.flushTimer = null;
315622
+ }
315623
+ while (this.queue.length > 0 || this.flushInFlight) {
315624
+ if (this.flushInFlight) {
315625
+ await this.flushInFlight;
315626
+ continue;
315627
+ }
315628
+ const depthBefore = this.queue.length;
315629
+ const flushed = await this.flushBatch();
315630
+ if (!flushed && this.queue.length >= depthBefore && depthBefore > 0) {
315631
+ const dropped = this.queue.length;
315632
+ this.queue.length = 0;
315633
+ log.warn(`Archive shutdown dropped ${dropped} undelivered row(s) after forwarder failure`);
315634
+ break;
315635
+ }
315636
+ }
315637
+ this.closed = true;
315199
315638
  }
315200
- shutdownProvider(provider) {
315201
- return provider.forceFlush().then(() => provider.shutdown());
315639
+ getStats() {
315640
+ return { ...this.stats };
315202
315641
  }
315203
- onProviderClosed() {
315204
- this.logger = null;
315642
+ getQueueDepth() {
315643
+ return this.queue.length;
315205
315644
  }
315206
- emit(record) {
315207
- if (!this.isOtelEnabled() || !this.logger) {
315208
- return;
315645
+ enforceQueueBound() {
315646
+ while (this.queue.length > this.maxQueueSize) {
315647
+ const dropped = this.queue.shift();
315648
+ this.stats.shed += 1;
315649
+ this.recordArchiveMetric("cex_archive_rows_shed_total", {
315650
+ table: dropped?.table ?? "unknown"
315651
+ });
315209
315652
  }
315210
- this.logger.emit(record);
315211
315653
  }
315212
- }
315213
- function resolveOtlpEndpoint(signal, config) {
315214
- if (config?.otlpEndpoint) {
315215
- return {
315216
- endpoint: normalizeOtlpEndpoint(config.otlpEndpoint),
315217
- appendSignalPath: true
315218
- };
315654
+ async flushBatch() {
315655
+ const batch = this.queue.splice(0, this.batchSize);
315656
+ if (batch.length === 0) {
315657
+ return true;
315658
+ }
315659
+ for (const entry of batch) {
315660
+ if (!isMarketArchiveTable(entry.table)) {
315661
+ this.emitOtelLog(entry);
315662
+ }
315663
+ }
315664
+ if (this.forwarderUrl) {
315665
+ try {
315666
+ await this.postToForwarder(batch);
315667
+ } catch (error) {
315668
+ this.stats.forwarderFailures += 1;
315669
+ this.queue.push(...batch);
315670
+ this.enforceQueueBound();
315671
+ this.recordArchiveMetric("cex_archive_forwarder_failures_total", {
315672
+ count: batch.length
315673
+ });
315674
+ log.warn("Broker execution archive forwarder failed", { error });
315675
+ return false;
315676
+ }
315677
+ }
315678
+ this.stats.flushed += batch.length;
315679
+ return true;
315219
315680
  }
315220
- const signalEndpoint = signal === "metrics" ? process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT : process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
315221
- if (signalEndpoint) {
315222
- return {
315223
- endpoint: signalEndpoint,
315224
- appendSignalPath: false
315225
- };
315681
+ async recordArchiveMetric(metricName, labels) {
315682
+ try {
315683
+ await this.otelMetrics?.recordCounter(metricName, 1, labels);
315684
+ } catch {}
315226
315685
  }
315227
- if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) {
315228
- return {
315229
- endpoint: normalizeOtlpEndpoint(process.env.OTEL_EXPORTER_OTLP_ENDPOINT),
315230
- appendSignalPath: true
315231
- };
315686
+ emitOtelLog(entry) {
315687
+ if (!this.otelLogs?.isOtelEnabled()) {
315688
+ return;
315689
+ }
315690
+ try {
315691
+ this.otelLogs.emit({
315692
+ body: entry.table,
315693
+ severityNumber: import_api_logs2.SeverityNumber.INFO,
315694
+ severityText: "INFO",
315695
+ attributes: flattenArchiveAttributes(entry)
315696
+ });
315697
+ } catch (error) {
315698
+ log.warn("Broker execution archive OTLP emit failed", { error });
315699
+ }
315232
315700
  }
315233
- if (config?.host) {
315234
- const protocol = config.protocol || "http";
315235
- const port = config.port ?? DEFAULT_OTLP_PORT;
315236
- return {
315237
- endpoint: `${protocol}://${config.host}:${port}`,
315238
- appendSignalPath: true
315701
+ async postToForwarder(batch) {
315702
+ if (!this.forwarderUrl || batch.length === 0) {
315703
+ return;
315704
+ }
315705
+ const controller = new AbortController;
315706
+ const timeout2 = setTimeout(() => controller.abort(), this.forwarderTimeoutMs);
315707
+ const headers = {
315708
+ "content-type": "application/json"
315239
315709
  };
315710
+ if (this.forwarderAuthToken) {
315711
+ headers.authorization = `Bearer ${this.forwarderAuthToken}`;
315712
+ }
315713
+ try {
315714
+ const response = await fetch(this.forwarderUrl, {
315715
+ method: "POST",
315716
+ headers,
315717
+ body: JSON.stringify({
315718
+ source: "broker_write",
315719
+ deployment_id: this.deploymentId,
315720
+ rows: batch
315721
+ }),
315722
+ signal: controller.signal
315723
+ });
315724
+ if (!response.ok) {
315725
+ throw new Error(`Archive forwarder returned ${response.status} ${response.statusText}`);
315726
+ }
315727
+ } finally {
315728
+ clearTimeout(timeout2);
315729
+ }
315240
315730
  }
315241
- return null;
315242
315731
  }
315243
- function appendOtlpPath(endpoint, signal, appendSignalPath) {
315244
- if (!appendSignalPath) {
315245
- return endpoint;
315732
+ function flattenArchiveAttributes(entry) {
315733
+ const attributes = {
315734
+ ch_table: entry.table
315735
+ };
315736
+ for (const [key, value] of Object.entries(entry.row)) {
315737
+ if (value === undefined || value === null) {
315738
+ continue;
315739
+ }
315740
+ if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
315741
+ attributes[key] = value;
315742
+ } else {
315743
+ attributes[key] = JSON.stringify(value);
315744
+ }
315246
315745
  }
315247
- const baseEndpoint = normalizeOtlpEndpoint(endpoint);
315248
- return `${baseEndpoint}/v1/${signal}`;
315746
+ return attributes;
315249
315747
  }
315250
- function normalizeOtlpEndpoint(endpoint) {
315251
- return endpoint.replace(/\/v1\/(metrics|logs)\/?$/, "").replace(/\/+$/, "");
315748
+ function createBrokerExecutionArchiverFromEnv(otelLogs, otelMetrics) {
315749
+ const forwarderUrl = resolveArchiveForwarderUrlFromEnv();
315750
+ const archiveOtelLogs = isArchiveOtelLogsEnabled() ? otelLogs : undefined;
315751
+ return BrokerExecutionArchiver.create({
315752
+ otelLogs: archiveOtelLogs,
315753
+ otelMetrics,
315754
+ forwarderUrl,
315755
+ deploymentId: process.env.CEX_BROKER_DEPLOYMENT_ID,
315756
+ maxQueueSize: parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_QUEUE_MAX, DEFAULT_MAX_QUEUE_SIZE),
315757
+ batchSize: parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_BATCH_SIZE, DEFAULT_BATCH_SIZE),
315758
+ flushIntervalMs: parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_FLUSH_INTERVAL_MS, DEFAULT_FLUSH_INTERVAL_MS)
315759
+ });
315252
315760
  }
315253
- function getOtelHostFromEnv() {
315254
- return process.env.CEX_BROKER_OTEL_HOST ?? process.env.CEX_BROKER_CLICKHOUSE_HOST;
315761
+ function parsePositiveInt(value, fallback) {
315762
+ if (!value) {
315763
+ return fallback;
315764
+ }
315765
+ const parsed = Number.parseInt(value, 10);
315766
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
315767
+ }
315768
+ // src/helpers/otel.ts
315769
+ var import_api2 = __toESM(require_src5(), 1);
315770
+ var import_api_logs3 = __toESM(require_src7(), 1);
315771
+ var import_exporter_logs_otlp_http = __toESM(require_src14(), 1);
315772
+ var import_exporter_metrics_otlp_http = __toESM(require_src15(), 1);
315773
+ var import_resources = __toESM(require_src11(), 1);
315774
+ var import_sdk_logs = __toESM(require_src16(), 1);
315775
+ var import_sdk_metrics = __toESM(require_src12(), 1);
315776
+ var DEFAULT_SERVICE = "cex-broker";
315777
+ var DEFAULT_OTLP_PORT = 4318;
315778
+ var EXPORT_INTERVAL_MS = 5000;
315779
+
315780
+ class BaseOtelSignal {
315781
+ signal;
315782
+ provider = null;
315783
+ isEnabled = false;
315784
+ serviceName;
315785
+ constructor(config, signal) {
315786
+ this.signal = signal;
315787
+ this.serviceName = config?.serviceName ?? DEFAULT_SERVICE;
315788
+ const endpointResolution = resolveOtlpEndpoint(this.signal, config);
315789
+ if (!endpointResolution) {
315790
+ log.info(`OTel ${signal} disabled: no OTLP endpoint or host provided`);
315791
+ return;
315792
+ }
315793
+ try {
315794
+ this.provider = this.createProvider(endpointResolution.endpoint, this.serviceName, endpointResolution.appendSignalPath);
315795
+ this.onProviderCreated(this.provider);
315796
+ this.isEnabled = true;
315797
+ log.info(`OTel ${signal} enabled: ${endpointResolution.endpoint}`);
315798
+ } catch (error) {
315799
+ log.error(`Failed to initialize OTel ${signal}:`, error);
315800
+ this.isEnabled = false;
315801
+ this.provider = null;
315802
+ }
315803
+ }
315804
+ onProviderCreated(_provider) {}
315805
+ onProviderClosed() {}
315806
+ getProvider() {
315807
+ return this.provider;
315808
+ }
315809
+ getServiceName() {
315810
+ return this.serviceName;
315811
+ }
315812
+ isOtelEnabled() {
315813
+ return this.isEnabled && this.provider !== null;
315814
+ }
315815
+ async close() {
315816
+ if (!this.provider) {
315817
+ return;
315818
+ }
315819
+ try {
315820
+ await this.shutdownProvider(this.provider);
315821
+ log.info(`OTel ${this.signal} provider shut down`);
315822
+ } catch (error) {
315823
+ log.error(`Error shutting down OTel ${this.signal} provider:`, error);
315824
+ }
315825
+ this.provider = null;
315826
+ this.isEnabled = false;
315827
+ this.onProviderClosed();
315828
+ }
315829
+ }
315830
+ function toAttributes(labels, service) {
315831
+ const attrs = { ...labels, service };
315832
+ for (const key of Object.keys(attrs)) {
315833
+ const v = attrs[key];
315834
+ if (typeof v !== "string" && typeof v !== "number") {
315835
+ attrs[key] = String(v);
315836
+ }
315837
+ }
315838
+ return attrs;
315839
+ }
315840
+
315841
+ class OtelMetrics extends BaseOtelSignal {
315842
+ counters = new Map;
315843
+ histograms = new Map;
315844
+ constructor(config) {
315845
+ super(config, "metrics");
315846
+ }
315847
+ createProvider(endpoint, serviceName, appendSignalPath) {
315848
+ const exporter = new import_exporter_metrics_otlp_http.OTLPMetricExporter({
315849
+ url: appendOtlpPath(endpoint, "metrics", appendSignalPath)
315850
+ });
315851
+ const reader = new import_sdk_metrics.PeriodicExportingMetricReader({
315852
+ exporter,
315853
+ exportIntervalMillis: EXPORT_INTERVAL_MS
315854
+ });
315855
+ const resource = import_resources.resourceFromAttributes({
315856
+ "service.name": serviceName
315857
+ });
315858
+ return new import_sdk_metrics.MeterProvider({
315859
+ resource,
315860
+ readers: [reader]
315861
+ });
315862
+ }
315863
+ onProviderCreated(provider) {
315864
+ import_api2.metrics.setGlobalMeterProvider(provider);
315865
+ }
315866
+ shutdownProvider(provider) {
315867
+ return provider.shutdown();
315868
+ }
315869
+ async initialize() {
315870
+ if (this.isOtelEnabled()) {
315871
+ log.info("OTel metrics initialized (storage is handled by the collector)");
315872
+ }
315873
+ }
315874
+ async insertMetric(metric) {
315875
+ if (!this.isOtelEnabled())
315876
+ return;
315877
+ const labels = metric.labels ? JSON.parse(metric.labels) : {};
315878
+ try {
315879
+ if (metric.metric_type === "counter") {
315880
+ await this.recordCounter(metric.metric_name, metric.value, labels, metric.service);
315881
+ } else if (metric.metric_type === "gauge") {
315882
+ await this.recordGauge(metric.metric_name, metric.value, labels, metric.service);
315883
+ } else {
315884
+ await this.recordHistogram(metric.metric_name, metric.value, labels, metric.service);
315885
+ }
315886
+ } catch {}
315887
+ }
315888
+ async insertMetrics(metricsList) {
315889
+ if (!this.isOtelEnabled() || metricsList.length === 0)
315890
+ return;
315891
+ for (const m of metricsList) {
315892
+ await this.insertMetric(m);
315893
+ }
315894
+ }
315895
+ async recordCounter(metricName, value, labels, service = this.getServiceName()) {
315896
+ const provider = this.getProvider();
315897
+ if (!this.isOtelEnabled() || !provider)
315898
+ return;
315899
+ try {
315900
+ let counter = this.counters.get(metricName);
315901
+ if (!counter) {
315902
+ const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
315903
+ counter = meter.createCounter(metricName, { description: metricName });
315904
+ this.counters.set(metricName, counter);
315905
+ }
315906
+ counter.add(value, toAttributes(labels, service));
315907
+ } catch (error) {
315908
+ log.error("Failed to record counter:", error);
315909
+ }
315910
+ }
315911
+ async recordGauge(metricName, value, labels, service = this.getServiceName()) {
315912
+ const provider = this.getProvider();
315913
+ if (!this.isOtelEnabled() || !provider)
315914
+ return;
315915
+ try {
315916
+ let hist = this.histograms.get(`gauge_${metricName}`);
315917
+ if (!hist) {
315918
+ const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
315919
+ hist = meter.createHistogram(`${metricName}_gauge`, {
315920
+ description: metricName
315921
+ });
315922
+ this.histograms.set(`gauge_${metricName}`, hist);
315923
+ }
315924
+ hist.record(value, toAttributes(labels, service));
315925
+ } catch (error) {
315926
+ log.error("Failed to record gauge:", error);
315927
+ }
315928
+ }
315929
+ async recordHistogram(metricName, value, labels, service = this.getServiceName()) {
315930
+ const provider = this.getProvider();
315931
+ if (!this.isOtelEnabled() || !provider)
315932
+ return;
315933
+ try {
315934
+ let hist = this.histograms.get(metricName);
315935
+ if (!hist) {
315936
+ const meter = provider.getMeter("cex-broker-metrics", "1.0.0");
315937
+ hist = meter.createHistogram(metricName, { description: metricName });
315938
+ this.histograms.set(metricName, hist);
315939
+ }
315940
+ hist.record(value, toAttributes(labels, service));
315941
+ } catch (error) {
315942
+ log.error("Failed to record histogram:", error);
315943
+ }
315944
+ }
315945
+ }
315946
+
315947
+ class OtelLogs extends BaseOtelSignal {
315948
+ logger = null;
315949
+ constructor(config) {
315950
+ super(config, "logs");
315951
+ }
315952
+ createProvider(endpoint, serviceName, appendSignalPath) {
315953
+ const exporter = new import_exporter_logs_otlp_http.OTLPLogExporter({
315954
+ url: appendOtlpPath(endpoint, "logs", appendSignalPath)
315955
+ });
315956
+ const processor = new import_sdk_logs.BatchLogRecordProcessor(exporter);
315957
+ const resource = import_resources.resourceFromAttributes({
315958
+ "service.name": serviceName
315959
+ });
315960
+ return new import_sdk_logs.LoggerProvider({
315961
+ resource,
315962
+ processors: [processor]
315963
+ });
315964
+ }
315965
+ onProviderCreated(provider) {
315966
+ import_api_logs3.logs.setGlobalLoggerProvider(provider);
315967
+ this.logger = provider.getLogger("cex-broker-logs", "1.0.0");
315968
+ }
315969
+ shutdownProvider(provider) {
315970
+ return provider.forceFlush().then(() => provider.shutdown());
315971
+ }
315972
+ onProviderClosed() {
315973
+ this.logger = null;
315974
+ }
315975
+ emit(record) {
315976
+ if (!this.isOtelEnabled() || !this.logger) {
315977
+ return;
315978
+ }
315979
+ this.logger.emit(record);
315980
+ }
315981
+ }
315982
+ function resolveOtlpEndpoint(signal, config) {
315983
+ if (config?.otlpEndpoint) {
315984
+ return {
315985
+ endpoint: normalizeOtlpEndpoint(config.otlpEndpoint),
315986
+ appendSignalPath: true
315987
+ };
315988
+ }
315989
+ const signalEndpoint = signal === "metrics" ? process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT : process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
315990
+ if (signalEndpoint) {
315991
+ return {
315992
+ endpoint: signalEndpoint,
315993
+ appendSignalPath: false
315994
+ };
315995
+ }
315996
+ if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) {
315997
+ return {
315998
+ endpoint: normalizeOtlpEndpoint(process.env.OTEL_EXPORTER_OTLP_ENDPOINT),
315999
+ appendSignalPath: true
316000
+ };
316001
+ }
316002
+ if (config?.host) {
316003
+ const protocol = config.protocol || "http";
316004
+ const port = config.port ?? DEFAULT_OTLP_PORT;
316005
+ return {
316006
+ endpoint: `${protocol}://${config.host}:${port}`,
316007
+ appendSignalPath: true
316008
+ };
316009
+ }
316010
+ return null;
316011
+ }
316012
+ function appendOtlpPath(endpoint, signal, appendSignalPath) {
316013
+ if (!appendSignalPath) {
316014
+ return endpoint;
316015
+ }
316016
+ const baseEndpoint = normalizeOtlpEndpoint(endpoint);
316017
+ return `${baseEndpoint}/v1/${signal}`;
316018
+ }
316019
+ function normalizeOtlpEndpoint(endpoint) {
316020
+ return endpoint.replace(/\/v1\/(metrics|logs)\/?$/, "").replace(/\/+$/, "");
316021
+ }
316022
+ function getOtelHostFromEnv() {
316023
+ return process.env.CEX_BROKER_OTEL_HOST ?? process.env.CEX_BROKER_CLICKHOUSE_HOST;
315255
316024
  }
315256
316025
  function getOtelPortFromEnv() {
315257
316026
  const port = process.env.CEX_BROKER_OTEL_PORT ?? process.env.CEX_BROKER_CLICKHOUSE_PORT;
@@ -315328,14 +316097,6 @@ function safeLogError(context2, error) {
315328
316097
  }
315329
316098
  }
315330
316099
 
315331
- // src/helpers/shared/guards.ts
315332
- function isRecord(value) {
315333
- return value !== null && typeof value === "object" && !Array.isArray(value);
315334
- }
315335
- function asRecord(value) {
315336
- return isRecord(value) ? value : undefined;
315337
- }
315338
-
315339
316100
  // src/helpers/treasury-discovery.ts
315340
316101
  function callArgs(args, params) {
315341
316102
  const argsArray = Array.isArray(args) ? [...args] : [];
@@ -329734,215 +330495,6 @@ async function handleInternalTransfer(ctx) {
329734
330495
 
329735
330496
  // src/handlers/execute-action/orders.ts
329736
330497
  var grpc6 = __toESM(require_src3(), 1);
329737
-
329738
- // src/helpers/order-telemetry.ts
329739
- var NUMERIC_METRICS = [
329740
- ["requestedQuantity", "cex_market_action_requested_quantity"],
329741
- ["requestedNotional", "cex_market_action_requested_notional"],
329742
- ["executedBaseQuantity", "cex_market_action_executed_base_quantity"],
329743
- ["executedQuoteQuantity", "cex_market_action_executed_quote_quantity"],
329744
- ["averageExecutionPrice", "cex_market_action_average_execution_price"],
329745
- ["filledAmount", "cex_market_action_filled_amount"],
329746
- ["remainingAmount", "cex_market_action_remaining_amount"],
329747
- ["feeAmount", "cex_market_action_fee_amount"],
329748
- ["feeRate", "cex_market_action_fee_rate"]
329749
- ];
329750
- var REDACTED_ERROR_MESSAGE = "redacted_error";
329751
- async function emitOrderExecutionTelemetry(otelMetrics, context2, order, error48) {
329752
- try {
329753
- const telemetry = buildOrderExecutionTelemetry(context2, order, error48);
329754
- log.info("CEX market action execution telemetry", telemetry);
329755
- const labels = {
329756
- action: telemetry.action,
329757
- cex: telemetry.cex,
329758
- account: telemetry.accountLabel,
329759
- symbol: telemetry.symbol,
329760
- side: telemetry.side,
329761
- order_type: telemetry.orderType,
329762
- status: telemetry.status
329763
- };
329764
- await otelMetrics?.recordCounter("cex_market_action_executions_total", 1, {
329765
- ...labels,
329766
- result: error48 ? "error" : "ok"
329767
- });
329768
- for (const [key, metricName] of NUMERIC_METRICS) {
329769
- const value = telemetry[key];
329770
- if (typeof value === "number" && Number.isFinite(value)) {
329771
- await otelMetrics?.recordHistogram(metricName, value, labels);
329772
- }
329773
- }
329774
- return telemetry;
329775
- } catch (telemetryError) {
329776
- try {
329777
- log.error("Failed to emit CEX order telemetry", {
329778
- error: telemetryError
329779
- });
329780
- } catch {}
329781
- return;
329782
- }
329783
- }
329784
- function buildOrderExecutionTelemetry(context2, order, error48) {
329785
- const record2 = asRecord(order);
329786
- const info = asRecord(record2?.info);
329787
- const fees = getFees(record2, info);
329788
- const fee = summarizeFees(fees);
329789
- const executedBaseQuantity = firstNumber(record2?.filled, info?.executedQty, info?.cumExecQty) ?? computeFilledFromAmount(record2);
329790
- const executedQuoteQuantity = firstNumber(record2?.cost, info?.cummulativeQuoteQty, info?.cumQuote, info?.cumExecValue);
329791
- const averageExecutionPrice = firstNumber(record2?.average, info?.avgPrice) ?? computeAveragePrice(executedBaseQuantity, executedQuoteQuantity);
329792
- const exchangeTimestamp = normalizeTimestamp(firstValue(record2?.timestamp, info?.time, info?.transactTime, record2?.datetime));
329793
- const status6 = firstString(record2?.status, info?.status) ?? (error48 ? "failed" : "unknown");
329794
- const errorRecord = error48 instanceof Error ? error48 : undefined;
329795
- return compactUndefined({
329796
- event: "cex_market_action_execution",
329797
- action: context2.action,
329798
- cex: context2.cex.trim().toLowerCase() || "unknown",
329799
- accountLabel: context2.accountLabel ?? "unknown",
329800
- symbol: firstString(record2?.symbol, info?.symbol, context2.symbol) ?? "unknown",
329801
- side: firstString(record2?.side, info?.side, context2.side) ?? "unknown",
329802
- orderType: firstString(record2?.type, info?.type, context2.orderType) ?? "unknown",
329803
- orderId: firstString(record2?.id, info?.orderId, info?.orderID),
329804
- clientOrderId: firstString(context2.clientOrderId, record2?.clientOrderId, record2?.clientOrderID, record2?.clientOid, info?.clientOrderId, info?.clientOrderID, info?.clientOid),
329805
- idempotencyId: context2.idempotencyId,
329806
- makerActionId: context2.makerActionId,
329807
- status: status6.toLowerCase(),
329808
- requestedQuantity: context2.requestedQuantity ?? firstNumber(record2?.amount),
329809
- requestedNotional: context2.requestedNotional,
329810
- executedBaseQuantity,
329811
- executedQuoteQuantity,
329812
- averageExecutionPrice,
329813
- filledAmount: firstNumber(record2?.filled, info?.executedQty),
329814
- remainingAmount: firstNumber(record2?.remaining, info?.remainingQty),
329815
- feeAmount: fee.amount,
329816
- feeCurrency: fee.currency,
329817
- feeRate: fee.rate,
329818
- exchangeTimestamp,
329819
- brokerObservedTimestamp: context2.brokerObservedTimestamp ?? new Date().toISOString(),
329820
- errorType: errorRecord?.name,
329821
- errorMessage: errorRecord ? REDACTED_ERROR_MESSAGE : undefined
329822
- });
329823
- }
329824
- function emitOrderExecutionTelemetryInBackground(otelMetrics, context2, order, error48) {
329825
- emitOrderExecutionTelemetry(otelMetrics, context2, order, error48).catch((telemetryError) => {
329826
- try {
329827
- log.warn("Telemetry emit failed", { error: telemetryError });
329828
- } catch {
329829
- console.warn("Telemetry emit failed", telemetryError);
329830
- }
329831
- });
329832
- }
329833
- function extractOrderTelemetryIds(params) {
329834
- const record2 = params ?? {};
329835
- return {
329836
- clientOrderId: firstString(record2.clientOrderId, record2.clientOrderID, record2.newClientOrderId, record2.clientOid),
329837
- idempotencyId: firstString(record2.idempotencyId, record2.idempotencyID, record2.idempotencyKey, record2.requestId, record2.requestID),
329838
- makerActionId: firstString(record2.makerActionId, record2.maker_action_id, record2.actionId, record2.action_id)
329839
- };
329840
- }
329841
- function firstValue(...values2) {
329842
- return values2.find((value) => value !== undefined && value !== null);
329843
- }
329844
- function firstString(...values2) {
329845
- for (const value of values2) {
329846
- if (typeof value === "string" && value.trim()) {
329847
- return value.trim();
329848
- }
329849
- if (typeof value === "number" && Number.isFinite(value)) {
329850
- return String(value);
329851
- }
329852
- }
329853
- return;
329854
- }
329855
- function firstNumber(...values2) {
329856
- for (const value of values2) {
329857
- const numberValue = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
329858
- if (Number.isFinite(numberValue)) {
329859
- return numberValue;
329860
- }
329861
- }
329862
- return;
329863
- }
329864
- function computeFilledFromAmount(record2) {
329865
- const amount = firstNumber(record2?.amount);
329866
- const remaining = firstNumber(record2?.remaining);
329867
- if (amount === undefined || remaining === undefined) {
329868
- return;
329869
- }
329870
- return amount - remaining;
329871
- }
329872
- function computeAveragePrice(executedBaseQuantity, executedQuoteQuantity) {
329873
- if (executedBaseQuantity === undefined || executedQuoteQuantity === undefined || executedBaseQuantity === 0) {
329874
- return;
329875
- }
329876
- return executedQuoteQuantity / executedBaseQuantity;
329877
- }
329878
- function normalizeTimestamp(value) {
329879
- if (typeof value === "string" && value.trim()) {
329880
- return value.trim();
329881
- }
329882
- const timestamp = firstNumber(value);
329883
- if (timestamp === undefined) {
329884
- return;
329885
- }
329886
- const date5 = new Date(timestamp);
329887
- return Number.isNaN(date5.getTime()) ? undefined : date5.toISOString();
329888
- }
329889
- function getFees(record2, info) {
329890
- const fees = [];
329891
- if (record2?.fee)
329892
- fees.push(record2.fee);
329893
- if (Array.isArray(record2?.fees))
329894
- fees.push(...record2.fees);
329895
- if (Array.isArray(record2?.trades)) {
329896
- for (const trade of record2.trades) {
329897
- const tradeRecord = asRecord(trade);
329898
- if (tradeRecord?.fee)
329899
- fees.push(tradeRecord.fee);
329900
- if (Array.isArray(tradeRecord?.fees))
329901
- fees.push(...tradeRecord.fees);
329902
- }
329903
- }
329904
- if (Array.isArray(info?.fills)) {
329905
- for (const fill of info.fills) {
329906
- const fillRecord = asRecord(fill);
329907
- const commission = firstNumber(fillRecord?.commission);
329908
- if (commission !== undefined) {
329909
- fees.push({
329910
- cost: commission,
329911
- currency: firstString(fillRecord?.commissionAsset)
329912
- });
329913
- }
329914
- }
329915
- }
329916
- return fees;
329917
- }
329918
- function summarizeFees(fees) {
329919
- let amount = 0;
329920
- let amountFound = false;
329921
- let currency;
329922
- let rate;
329923
- for (const rawFee of fees) {
329924
- const fee = asRecord(rawFee);
329925
- if (!fee)
329926
- continue;
329927
- const cost = firstNumber(fee.cost, fee.amount);
329928
- if (cost !== undefined) {
329929
- amount += cost;
329930
- amountFound = true;
329931
- }
329932
- currency ??= firstString(fee.currency);
329933
- rate ??= firstNumber(fee.rate);
329934
- }
329935
- return {
329936
- amount: amountFound ? amount : undefined,
329937
- currency,
329938
- rate
329939
- };
329940
- }
329941
- function compactUndefined(record2) {
329942
- return Object.fromEntries(Object.entries(record2).filter(([, value]) => value !== undefined));
329943
- }
329944
-
329945
- // src/handlers/execute-action/orders.ts
329946
330498
  async function handleCreateOrder(ctx) {
329947
330499
  const {
329948
330500
  call,
@@ -329959,7 +330511,8 @@ async function handleCreateOrder(ctx) {
329959
330511
  applyVerityToBroker,
329960
330512
  useVerity,
329961
330513
  verityProverUrl,
329962
- otelMetrics
330514
+ otelMetrics,
330515
+ brokerArchiver
329963
330516
  } = ctx;
329964
330517
  const verityProof = verity.proof;
329965
330518
  const orderValue = parsePayloadForAction(ctx, CreateOrderPayloadSchema);
@@ -329985,8 +330538,18 @@ async function handleCreateOrder(ctx) {
329985
330538
  side: resolution.side,
329986
330539
  requestedQuantity: resolution.amountBase ?? orderValue.amount
329987
330540
  };
330541
+ const telemetryIds = extractOrderTelemetryIds(orderValue.params);
330542
+ const submissionTimestamp = new Date().toISOString();
330543
+ const marketMetadataHash = await captureMarketMetadataSnapshot(brokerArchiver, broker, {
330544
+ exchange: cex3,
330545
+ accountSelector: selectedBrokerAccount?.label,
330546
+ symbol: resolution.symbol,
330547
+ action: "CreateOrder",
330548
+ brokerObservedTimestamp: submissionTimestamp,
330549
+ ...telemetryIds
330550
+ });
329988
330551
  const order = await broker.createOrder(resolution.symbol, orderValue.orderType, resolution.side, resolution.amountBase ?? orderValue.amount, orderValue.price, orderValue.params ?? {});
329989
- emitOrderExecutionTelemetryInBackground(otelMetrics, {
330552
+ const createOrderContext = {
329990
330553
  action: "CreateOrder",
329991
330554
  cex: cex3,
329992
330555
  accountLabel: selectedBrokerAccount?.label,
@@ -329995,12 +330558,15 @@ async function handleCreateOrder(ctx) {
329995
330558
  orderType: orderValue.orderType,
329996
330559
  requestedQuantity: resolvedOrderTelemetry.requestedQuantity,
329997
330560
  requestedNotional: orderValue.amount * orderValue.price,
329998
- ...extractOrderTelemetryIds(orderValue.params)
329999
- }, order);
330561
+ brokerObservedTimestamp: submissionTimestamp,
330562
+ ...telemetryIds
330563
+ };
330564
+ emitOrderExecutionTelemetryInBackground(otelMetrics, createOrderContext, order);
330565
+ archiveOrderExecutionInBackground(brokerArchiver, createOrderContext, order, undefined, { marketMetadataHash });
330000
330566
  ctx.wrappedCallback(null, { result: JSON.stringify({ ...order }) });
330001
330567
  } catch (error48) {
330002
330568
  safeLogError("Order Creation failed", error48);
330003
- emitOrderExecutionTelemetryInBackground(otelMetrics, {
330569
+ const failedCreateContext = {
330004
330570
  action: "CreateOrder",
330005
330571
  cex: cex3,
330006
330572
  accountLabel: selectedBrokerAccount?.label,
@@ -330010,7 +330576,9 @@ async function handleCreateOrder(ctx) {
330010
330576
  requestedQuantity: resolvedOrderTelemetry.requestedQuantity ?? orderValue.amount,
330011
330577
  requestedNotional: orderValue.amount * orderValue.price,
330012
330578
  ...extractOrderTelemetryIds(orderValue.params)
330013
- }, undefined, error48);
330579
+ };
330580
+ emitOrderExecutionTelemetryInBackground(otelMetrics, failedCreateContext, undefined, error48);
330581
+ archiveOrderExecutionInBackground(brokerArchiver, failedCreateContext, undefined, error48);
330014
330582
  ctx.wrappedCallback({
330015
330583
  code: grpc6.status.INTERNAL,
330016
330584
  message: "Order Creation failed"
@@ -330033,7 +330601,8 @@ async function handleGetOrderDetails(ctx) {
330033
330601
  applyVerityToBroker,
330034
330602
  useVerity,
330035
330603
  verityProverUrl,
330036
- otelMetrics
330604
+ otelMetrics,
330605
+ brokerArchiver
330037
330606
  } = ctx;
330038
330607
  const verityProof = verity.proof;
330039
330608
  const getOrderValue = parsePayloadForAction(ctx, GetOrderDetailsPayloadSchema);
@@ -330047,13 +330616,15 @@ async function handleGetOrderDetails(ctx) {
330047
330616
  }, null);
330048
330617
  }
330049
330618
  const orderDetails = await broker.fetchOrder(getOrderValue.orderId, symbol2, { ...getOrderValue.params });
330050
- emitOrderExecutionTelemetryInBackground(otelMetrics, {
330619
+ const getOrderContext = {
330051
330620
  action: "GetOrderDetails",
330052
330621
  cex: cex3,
330053
330622
  accountLabel: selectedBrokerAccount?.label,
330054
330623
  symbol: symbol2,
330055
330624
  ...extractOrderTelemetryIds(getOrderValue.params)
330056
- }, orderDetails);
330625
+ };
330626
+ emitOrderExecutionTelemetryInBackground(otelMetrics, getOrderContext, orderDetails);
330627
+ archiveOrderExecutionInBackground(brokerArchiver, getOrderContext, orderDetails);
330057
330628
  ctx.wrappedCallback(null, {
330058
330629
  result: JSON.stringify({
330059
330630
  orderId: orderDetails.id,
@@ -330068,6 +330639,15 @@ async function handleGetOrderDetails(ctx) {
330068
330639
  });
330069
330640
  } catch (error48) {
330070
330641
  safeLogError(`Error fetching order details from ${cex3}`, error48);
330642
+ const failedGetOrderContext = {
330643
+ action: "GetOrderDetails",
330644
+ cex: cex3,
330645
+ accountLabel: selectedBrokerAccount?.label,
330646
+ symbol: symbol2,
330647
+ ...extractOrderTelemetryIds(getOrderValue.params)
330648
+ };
330649
+ emitOrderExecutionTelemetryInBackground(otelMetrics, failedGetOrderContext, undefined, error48);
330650
+ archiveOrderExecutionInBackground(brokerArchiver, failedGetOrderContext, undefined, error48);
330071
330651
  ctx.wrappedCallback({
330072
330652
  code: grpc6.status.INTERNAL,
330073
330653
  message: `Failed to fetch order details from ${cex3}`
@@ -330090,19 +330670,44 @@ async function handleCancelOrder(ctx) {
330090
330670
  applyVerityToBroker,
330091
330671
  useVerity,
330092
330672
  verityProverUrl,
330093
- otelMetrics
330673
+ otelMetrics,
330674
+ brokerArchiver
330094
330675
  } = ctx;
330095
330676
  const verityProof = verity.proof;
330096
330677
  const cancelOrderValue = parsePayloadForAction(ctx, CancelOrderPayloadSchema);
330097
330678
  if (cancelOrderValue === null)
330098
330679
  return;
330099
330680
  try {
330681
+ if (!broker) {
330682
+ return ctx.wrappedCallback({
330683
+ code: grpc6.status.INVALID_ARGUMENT,
330684
+ message: `Invalid CEX key: ${cex3}. Supported keys: ${Object.keys(brokers).join(", ")}`
330685
+ }, null);
330686
+ }
330687
+ const cancelOrderContext = {
330688
+ action: "CancelOrder",
330689
+ cex: cex3,
330690
+ accountLabel: selectedBrokerAccount?.label,
330691
+ symbol: symbol2,
330692
+ ...extractOrderTelemetryIds(cancelOrderValue.params)
330693
+ };
330100
330694
  const cancelledOrder = await broker.cancelOrder(cancelOrderValue.orderId, symbol2, cancelOrderValue.params ?? {});
330695
+ emitOrderExecutionTelemetryInBackground(otelMetrics, cancelOrderContext, cancelledOrder);
330696
+ archiveOrderExecutionInBackground(brokerArchiver, cancelOrderContext, cancelledOrder);
330101
330697
  ctx.wrappedCallback(null, {
330102
330698
  result: JSON.stringify({ ...cancelledOrder })
330103
330699
  });
330104
330700
  } catch (error48) {
330105
330701
  safeLogError(`Error cancelling order from ${cex3}`, error48);
330702
+ const failedCancelContext = {
330703
+ action: "CancelOrder",
330704
+ cex: cex3,
330705
+ accountLabel: selectedBrokerAccount?.label,
330706
+ symbol: symbol2,
330707
+ ...extractOrderTelemetryIds(cancelOrderValue.params)
330708
+ };
330709
+ emitOrderExecutionTelemetryInBackground(otelMetrics, failedCancelContext, undefined, error48);
330710
+ archiveOrderExecutionInBackground(brokerArchiver, failedCancelContext, undefined, error48);
330106
330711
  ctx.wrappedCallback({
330107
330712
  code: grpc6.status.INTERNAL,
330108
330713
  message: `Failed to cancel order from ${cex3}`
@@ -330823,7 +331428,8 @@ function createExecuteActionHandler(deps) {
330823
331428
  whitelistIps,
330824
331429
  useVerity,
330825
331430
  verityProverUrl,
330826
- otelMetrics
331431
+ otelMetrics,
331432
+ brokerArchiver
330827
331433
  } = deps;
330828
331434
  return async (call, callback) => {
330829
331435
  const startTime = Date.now();
@@ -330901,7 +331507,8 @@ function createExecuteActionHandler(deps) {
330901
331507
  applyVerityToBroker,
330902
331508
  useVerity,
330903
331509
  verityProverUrl,
330904
- otelMetrics
331510
+ otelMetrics,
331511
+ brokerArchiver
330905
331512
  };
330906
331513
  if (action === Action.Call) {
330907
331514
  const handled = await handleOrderBookCall(preludeCtx);
@@ -331038,157 +331645,799 @@ function decodeMessageData(data) {
331038
331645
  return data;
331039
331646
  }
331040
331647
 
331041
- class BinanceSpotUserDataStream {
331042
- exchange;
331043
- ws;
331044
- secretValues;
331045
- requestId = `user-data-${Date.now()}-${Math.random()}`;
331046
- maxBufferedEvents;
331047
- queue = [];
331048
- waiters = [];
331049
- closed = false;
331050
- closeError = null;
331051
- subscriptionId = null;
331052
- constructor(exchange, options = {}) {
331053
- this.exchange = exchange;
331054
- this.maxBufferedEvents = options.maxBufferedEvents ?? DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS;
331055
- this.secretValues = [
331056
- getOptionalExchangeString(exchange, "apiKey"),
331057
- getOptionalExchangeString(exchange, "secret")
331058
- ].filter((value) => value !== null);
331059
- this.ws = createWebSocket(getBinanceSpotWsApiUrl(exchange));
331060
- this.ws.on("open", () => this.subscribe());
331061
- this.ws.on("message", (data) => this.handleMessage(data));
331062
- this.ws.on("error", (error48) => this.fail(formatBinanceUserDataWebSocketError(error48, this.secretValues)));
331063
- this.ws.on("close", (code, reason) => this.handleClose(code, reason));
331648
+ class BinanceSpotUserDataStream {
331649
+ exchange;
331650
+ ws;
331651
+ secretValues;
331652
+ requestId = `user-data-${Date.now()}-${Math.random()}`;
331653
+ maxBufferedEvents;
331654
+ queue = [];
331655
+ waiters = [];
331656
+ closed = false;
331657
+ closeError = null;
331658
+ subscriptionId = null;
331659
+ constructor(exchange, options = {}) {
331660
+ this.exchange = exchange;
331661
+ this.maxBufferedEvents = options.maxBufferedEvents ?? DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS;
331662
+ this.secretValues = [
331663
+ getOptionalExchangeString(exchange, "apiKey"),
331664
+ getOptionalExchangeString(exchange, "secret")
331665
+ ].filter((value) => value !== null);
331666
+ this.ws = createWebSocket(getBinanceSpotWsApiUrl(exchange));
331667
+ this.ws.on("open", () => this.subscribe());
331668
+ this.ws.on("message", (data) => this.handleMessage(data));
331669
+ this.ws.on("error", (error48) => this.fail(formatBinanceUserDataWebSocketError(error48, this.secretValues)));
331670
+ this.ws.on("close", (code, reason) => this.handleClose(code, reason));
331671
+ }
331672
+ async* [Symbol.asyncIterator]() {
331673
+ while (true) {
331674
+ const event = await this.nextEvent();
331675
+ if (!event) {
331676
+ break;
331677
+ }
331678
+ yield event;
331679
+ }
331680
+ }
331681
+ close() {
331682
+ if (this.closed) {
331683
+ return;
331684
+ }
331685
+ this.closed = true;
331686
+ this.queue.length = 0;
331687
+ try {
331688
+ this.ws.close();
331689
+ } catch {}
331690
+ this.flushWaiters();
331691
+ }
331692
+ handleClose(code, reason) {
331693
+ if (this.closed) {
331694
+ return;
331695
+ }
331696
+ this.fail(formatBinanceUserDataWebSocketClose(code, reason, this.secretValues));
331697
+ }
331698
+ subscribe() {
331699
+ const apiKey = getExchangeString(this.exchange, "apiKey");
331700
+ const signedParams = signUserDataStreamParams(this.exchange, {
331701
+ apiKey,
331702
+ timestamp: Date.now()
331703
+ });
331704
+ this.ws.send(JSON.stringify({
331705
+ id: this.requestId,
331706
+ method: "userDataStream.subscribe.signature",
331707
+ params: signedParams
331708
+ }));
331709
+ }
331710
+ handleMessage(data) {
331711
+ if (this.closed) {
331712
+ return;
331713
+ }
331714
+ let message;
331715
+ try {
331716
+ const decodedData = decodeMessageData(data);
331717
+ message = typeof decodedData === "string" ? JSON.parse(decodedData) : decodedData;
331718
+ } catch (error48) {
331719
+ this.fail(error48 instanceof Error ? error48 : new Error("Invalid Binance user-data message"));
331720
+ return;
331721
+ }
331722
+ if ("id" in message && message.id === this.requestId) {
331723
+ if (message.status !== 200) {
331724
+ this.fail(new Error(message.error?.msg ?? message.error?.message ?? `Binance user-data subscription failed with status ${message.status}`));
331725
+ return;
331726
+ }
331727
+ this.subscriptionId = message.result?.subscriptionId ?? null;
331728
+ return;
331729
+ }
331730
+ if (!("event" in message) || !message.event) {
331731
+ return;
331732
+ }
331733
+ const subscriptionId = message.subscriptionId ?? this.subscriptionId;
331734
+ if (subscriptionId === null || subscriptionId === undefined) {
331735
+ return;
331736
+ }
331737
+ this.push({ subscriptionId, event: message.event });
331738
+ }
331739
+ push(event) {
331740
+ if (this.closed) {
331741
+ return;
331742
+ }
331743
+ const waiter = this.waiters.shift();
331744
+ if (waiter) {
331745
+ waiter.resolve(event);
331746
+ return;
331747
+ }
331748
+ if (this.queue.length >= this.maxBufferedEvents) {
331749
+ this.fail(new Error(`Binance user-data stream buffered event limit exceeded (${this.maxBufferedEvents}); downstream consumer is not keeping up`));
331750
+ return;
331751
+ }
331752
+ this.queue.push(event);
331753
+ }
331754
+ nextEvent() {
331755
+ const event = this.queue.shift();
331756
+ if (event) {
331757
+ return Promise.resolve(event);
331758
+ }
331759
+ if (this.closeError) {
331760
+ return Promise.reject(this.closeError);
331761
+ }
331762
+ if (this.closed) {
331763
+ return Promise.resolve(null);
331764
+ }
331765
+ return new Promise((resolve, reject) => {
331766
+ this.waiters.push({ resolve, reject });
331767
+ });
331768
+ }
331769
+ fail(error48) {
331770
+ if (this.closeError) {
331771
+ return;
331772
+ }
331773
+ this.closeError = error48;
331774
+ this.closed = true;
331775
+ this.queue.length = 0;
331776
+ this.flushWaiters();
331777
+ try {
331778
+ this.ws.close();
331779
+ } catch {}
331780
+ }
331781
+ flushWaiters() {
331782
+ const error48 = this.closeError;
331783
+ for (const waiter of this.waiters.splice(0)) {
331784
+ if (error48) {
331785
+ waiter.reject(error48);
331786
+ } else {
331787
+ waiter.resolve(null);
331788
+ }
331789
+ }
331790
+ }
331791
+ }
331792
+ function isBinanceBalanceUserDataEvent(event) {
331793
+ return event.e === "outboundAccountPosition" || event.e === "balanceUpdate" || event.e === "externalLockUpdate";
331794
+ }
331795
+ function isBinanceOrderUserDataEvent(event) {
331796
+ return event.e === "executionReport" || event.e === "listStatus";
331797
+ }
331798
+
331799
+ // src/helpers/market-data-archive/ohlcv-bar-tracker.ts
331800
+ function isFiniteNumber(value) {
331801
+ return typeof value === "number" && Number.isFinite(value);
331802
+ }
331803
+ function parseOhlcvBar(value) {
331804
+ if (!Array.isArray(value) || value.length < 6) {
331805
+ return null;
331806
+ }
331807
+ const [openTimeMs, open, high, low, close, volume, quoteVolume] = value;
331808
+ if (!isFiniteNumber(openTimeMs) || !isFiniteNumber(open) || !isFiniteNumber(high) || !isFiniteNumber(low) || !isFiniteNumber(close) || !isFiniteNumber(volume)) {
331809
+ return null;
331810
+ }
331811
+ const bar = {
331812
+ openTimeMs,
331813
+ open,
331814
+ high,
331815
+ low,
331816
+ close,
331817
+ volume
331818
+ };
331819
+ if (isFiniteNumber(quoteVolume)) {
331820
+ bar.quoteVolume = quoteVolume;
331821
+ }
331822
+ return bar;
331823
+ }
331824
+ function extractOhlcvBars(payload) {
331825
+ if (!Array.isArray(payload) || payload.length === 0) {
331826
+ return [];
331827
+ }
331828
+ const rawBars = Array.isArray(payload[0]) ? payload : [payload];
331829
+ const byOpenTime = new Map;
331830
+ for (const entry of rawBars) {
331831
+ const bar = parseOhlcvBar(entry);
331832
+ if (bar) {
331833
+ byOpenTime.set(bar.openTimeMs, bar);
331834
+ }
331835
+ }
331836
+ return [...byOpenTime.values()].sort((a, b2) => a.openTimeMs - b2.openTimeMs);
331837
+ }
331838
+ class OhlcvBarTracker {
331839
+ lastOpenTimeMs = null;
331840
+ lastBar = null;
331841
+ process(payload, brokerVersion) {
331842
+ const bars = extractOhlcvBars(payload);
331843
+ if (bars.length === 0) {
331844
+ return [];
331845
+ }
331846
+ if (bars.length === 1) {
331847
+ const [bar] = bars;
331848
+ return bar ? this.processSingleBar(bar, brokerVersion) : [];
331849
+ }
331850
+ return this.processBatch(bars, brokerVersion);
331851
+ }
331852
+ processSingleBar(currentBar, brokerVersion) {
331853
+ if (this.lastOpenTimeMs !== null && currentBar.openTimeMs < this.lastOpenTimeMs) {
331854
+ return [];
331855
+ }
331856
+ const candidates = [];
331857
+ if (this.lastOpenTimeMs !== null && this.lastBar !== null && currentBar.openTimeMs !== this.lastOpenTimeMs) {
331858
+ candidates.push({
331859
+ bar: this.lastBar,
331860
+ isClosed: true,
331861
+ brokerVersion
331862
+ });
331863
+ }
331864
+ candidates.push({
331865
+ bar: currentBar,
331866
+ isClosed: false,
331867
+ brokerVersion
331868
+ });
331869
+ this.lastOpenTimeMs = currentBar.openTimeMs;
331870
+ this.lastBar = currentBar;
331871
+ return candidates;
331872
+ }
331873
+ processBatch(bars, brokerVersion) {
331874
+ const firstBar = bars[0];
331875
+ const lastBar = bars[bars.length - 1];
331876
+ if (!firstBar || !lastBar) {
331877
+ return [];
331878
+ }
331879
+ if (this.lastOpenTimeMs !== null && lastBar.openTimeMs < this.lastOpenTimeMs) {
331880
+ return [];
331881
+ }
331882
+ const candidates = [];
331883
+ if (this.lastBar !== null && this.lastOpenTimeMs !== null && !bars.some((bar) => bar.openTimeMs === this.lastOpenTimeMs) && this.lastOpenTimeMs < firstBar.openTimeMs) {
331884
+ candidates.push({
331885
+ bar: this.lastBar,
331886
+ isClosed: true,
331887
+ brokerVersion
331888
+ });
331889
+ }
331890
+ for (let index2 = 0;index2 < bars.length - 1; index2 += 1) {
331891
+ const bar = bars[index2];
331892
+ if (bar) {
331893
+ candidates.push({
331894
+ bar,
331895
+ isClosed: true,
331896
+ brokerVersion
331897
+ });
331898
+ }
331899
+ }
331900
+ candidates.push({
331901
+ bar: lastBar,
331902
+ isClosed: false,
331903
+ brokerVersion
331904
+ });
331905
+ this.lastOpenTimeMs = lastBar.openTimeMs;
331906
+ this.lastBar = lastBar;
331907
+ return candidates;
331908
+ }
331909
+ }
331910
+
331911
+ // src/helpers/market-data-archive/orderbook-sampler.ts
331912
+ var DEFAULT_ORDERBOOK_INTERVAL_MS = 1000;
331913
+ function getOrderbookIntervalMs() {
331914
+ const raw = process.env.CEX_BROKER_ORDERBOOK_INTERVAL_MS ?? process.env.CEX_BROKER_ORDERBOOK_TOB_INTERVAL_MS;
331915
+ if (!raw) {
331916
+ return DEFAULT_ORDERBOOK_INTERVAL_MS;
331917
+ }
331918
+ const parsed = Number.parseInt(raw, 10);
331919
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_ORDERBOOK_INTERVAL_MS;
331920
+ }
331921
+ function isMarketArchiveEnabled() {
331922
+ return process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED !== "false";
331923
+ }
331924
+
331925
+ class OrderbookSampler {
331926
+ intervalMs;
331927
+ lastEmitMs = null;
331928
+ constructor(intervalMs = getOrderbookIntervalMs()) {
331929
+ this.intervalMs = intervalMs;
331930
+ }
331931
+ shouldEmit(nowMs = Date.now()) {
331932
+ if (this.lastEmitMs !== null && nowMs < this.lastEmitMs) {
331933
+ this.lastEmitMs = nowMs;
331934
+ return true;
331935
+ }
331936
+ if (this.lastEmitMs !== null && nowMs - this.lastEmitMs < this.intervalMs) {
331937
+ return false;
331938
+ }
331939
+ this.lastEmitMs = nowMs;
331940
+ return true;
331941
+ }
331942
+ }
331943
+
331944
+ // src/helpers/market-data-archive/parse-stream.ts
331945
+ function isFiniteNumber2(value) {
331946
+ return typeof value === "number" && Number.isFinite(value);
331947
+ }
331948
+ function toNumber2(value) {
331949
+ if (isFiniteNumber2(value)) {
331950
+ return value;
331951
+ }
331952
+ if (typeof value === "string") {
331953
+ const parsed = Number.parseFloat(value);
331954
+ if (Number.isFinite(parsed)) {
331955
+ return parsed;
331956
+ }
331957
+ }
331958
+ return;
331959
+ }
331960
+ function toStringId(value) {
331961
+ if (typeof value === "string" && value.trim()) {
331962
+ return value.trim();
331963
+ }
331964
+ if (typeof value === "number" && Number.isFinite(value)) {
331965
+ return String(value);
331966
+ }
331967
+ return;
331968
+ }
331969
+ function scalarTimestampMs(value, fallbackMs) {
331970
+ const numeric = toNumber2(value);
331971
+ if (numeric !== undefined) {
331972
+ return numeric < 1000000000000 ? numeric * 1000 : numeric;
331973
+ }
331974
+ if (typeof value === "string") {
331975
+ const parsed = Date.parse(value);
331976
+ if (Number.isFinite(parsed)) {
331977
+ return parsed;
331978
+ }
331979
+ }
331980
+ return fallbackMs;
331981
+ }
331982
+ function parseTrade(value, fallbackMs = Date.now()) {
331983
+ const record2 = asRecord(value);
331984
+ if (!record2) {
331985
+ return null;
331986
+ }
331987
+ const tradeId = toStringId(record2.id);
331988
+ const price = toNumber2(record2.price);
331989
+ const amount = toNumber2(record2.amount);
331990
+ const side = typeof record2.side === "string" ? record2.side.toLowerCase() : undefined;
331991
+ if (!tradeId || price === undefined || amount === undefined || !side) {
331992
+ return null;
331993
+ }
331994
+ const parsed = {
331995
+ tradeId,
331996
+ eventTimeMs: scalarTimestampMs(record2.timestamp, fallbackMs),
331997
+ side,
331998
+ price,
331999
+ amount
332000
+ };
332001
+ const cost = toNumber2(record2.cost);
332002
+ if (cost !== undefined) {
332003
+ parsed.cost = cost;
332004
+ }
332005
+ if (typeof record2.takerOrMaker === "string") {
332006
+ parsed.takerOrMaker = record2.takerOrMaker;
332007
+ }
332008
+ return parsed;
332009
+ }
332010
+ function extractTrades(payload, fallbackMs = Date.now()) {
332011
+ if (Array.isArray(payload)) {
332012
+ return payload.map((entry) => parseTrade(entry, fallbackMs)).filter((entry) => entry !== null);
332013
+ }
332014
+ const single = parseTrade(payload, fallbackMs);
332015
+ return single ? [single] : [];
332016
+ }
332017
+ function parseTicker(value, fallbackMs) {
332018
+ const record2 = asRecord(value);
332019
+ if (!record2) {
332020
+ return null;
332021
+ }
332022
+ const parsed = {
332023
+ eventTimeMs: scalarTimestampMs(record2.timestamp, fallbackMs)
332024
+ };
332025
+ const fields = [
332026
+ ["last", record2.last],
332027
+ ["bid", record2.bid],
332028
+ ["ask", record2.ask],
332029
+ ["high", record2.high],
332030
+ ["low", record2.low],
332031
+ ["open", record2.open],
332032
+ ["close", record2.close],
332033
+ ["baseVolume", record2.baseVolume],
332034
+ ["quoteVolume", record2.quoteVolume],
332035
+ ["change", record2.change],
332036
+ ["percentage", record2.percentage]
332037
+ ];
332038
+ for (const [key, rawValue] of fields) {
332039
+ const numeric = toNumber2(rawValue);
332040
+ if (numeric !== undefined) {
332041
+ parsed[key] = numeric;
332042
+ }
332043
+ }
332044
+ return parsed;
332045
+ }
332046
+
332047
+ // src/helpers/market-data-archive/orderbook-depth.ts
332048
+ var DEFAULT_ORDERBOOK_ARCHIVE_DEPTH_LIMIT = 25;
332049
+ var MAX_ORDERBOOK_ARCHIVE_DEPTH_LIMIT = 500;
332050
+ function getOrderbookArchiveDepthLimit() {
332051
+ const raw = process.env.CEX_BROKER_ORDERBOOK_ARCHIVE_DEPTH_LIMIT;
332052
+ if (!raw) {
332053
+ return DEFAULT_ORDERBOOK_ARCHIVE_DEPTH_LIMIT;
332054
+ }
332055
+ const parsed = Number.parseInt(raw, 10);
332056
+ if (!Number.isFinite(parsed) || parsed <= 0) {
332057
+ return DEFAULT_ORDERBOOK_ARCHIVE_DEPTH_LIMIT;
332058
+ }
332059
+ return Math.min(parsed, MAX_ORDERBOOK_ARCHIVE_DEPTH_LIMIT);
332060
+ }
332061
+ function splitOrderBookSide(levels, limit) {
332062
+ const prices = [];
332063
+ const sizes = [];
332064
+ for (const level of levels.slice(0, limit)) {
332065
+ if (!Array.isArray(level) || level.length < 2) {
332066
+ continue;
332067
+ }
332068
+ const price = level[0];
332069
+ const size = level[1];
332070
+ if (price === undefined || size === undefined || !Number.isFinite(price) || !Number.isFinite(size)) {
332071
+ continue;
332072
+ }
332073
+ prices.push(price);
332074
+ sizes.push(size);
332075
+ }
332076
+ return { prices, sizes };
332077
+ }
332078
+
332079
+ // src/helpers/market-data-archive/rows.ts
332080
+ function compactUndefined3(record2) {
332081
+ return Object.fromEntries(Object.entries(record2).filter(([, value]) => value !== undefined));
332082
+ }
332083
+ function scalarTimestampMs2(value) {
332084
+ if (typeof value === "number" && Number.isFinite(value)) {
332085
+ return value;
332086
+ }
332087
+ if (typeof value === "string") {
332088
+ if (/^\d+$/.test(value)) {
332089
+ const numeric = Number.parseInt(value, 10);
332090
+ if (Number.isFinite(numeric)) {
332091
+ return numeric;
332092
+ }
332093
+ }
332094
+ const parsed = Date.parse(value);
332095
+ if (Number.isFinite(parsed)) {
332096
+ return parsed;
332097
+ }
332098
+ }
332099
+ return Date.now();
332100
+ }
332101
+ function parseSequence(value) {
332102
+ if (typeof value === "number" && Number.isFinite(value)) {
332103
+ return value;
332104
+ }
332105
+ if (typeof value === "string" && /^\d+$/.test(value)) {
332106
+ return Number.parseInt(value, 10);
332107
+ }
332108
+ return;
332109
+ }
332110
+ function topOfBookLevel(levels) {
332111
+ const level = levels[0];
332112
+ if (!level || level.length < 2) {
332113
+ return null;
332114
+ }
332115
+ const price = level[0];
332116
+ const size = level[1];
332117
+ if (price === undefined || size === undefined || !Number.isFinite(price) || !Number.isFinite(size)) {
332118
+ return null;
332119
+ }
332120
+ return { price, size };
332121
+ }
332122
+ function computeSpreadBps(bestBid, bestAsk) {
332123
+ if (bestBid <= 0 || bestAsk <= 0 || bestAsk < bestBid) {
332124
+ return 0;
332125
+ }
332126
+ const mid = (bestBid + bestAsk) / 2;
332127
+ if (mid <= 0) {
332128
+ return 0;
332129
+ }
332130
+ return (bestAsk - bestBid) / mid * 1e4;
332131
+ }
332132
+ function buildOrderbookArchiveTags(input, receivedTimeMs) {
332133
+ return buildCommonArchiveTags({
332134
+ deploymentId: input.deploymentId,
332135
+ accountSelector: input.accountSelector,
332136
+ exchange: input.exchange,
332137
+ symbol: input.symbol,
332138
+ brokerObservedTimestamp: new Date(receivedTimeMs).toISOString()
332139
+ });
332140
+ }
332141
+ function buildOrderbookSnapshotRow(input) {
332142
+ const bid = topOfBookLevel(input.snapshot.bids);
332143
+ const ask = topOfBookLevel(input.snapshot.asks);
332144
+ if (!bid || !ask) {
332145
+ return null;
332146
+ }
332147
+ const archiveDepthLimit = getOrderbookArchiveDepthLimit();
332148
+ const bids = splitOrderBookSide(input.snapshot.bids, archiveDepthLimit);
332149
+ const asks = splitOrderBookSide(input.snapshot.asks, archiveDepthLimit);
332150
+ if (bids.prices.length === 0 || asks.prices.length === 0) {
332151
+ return null;
332152
+ }
332153
+ const eventTimeMs = scalarTimestampMs2(input.snapshot.timestamp);
332154
+ const receivedTimeMs = input.snapshot.receivedTimestamp;
332155
+ const mid = (bid.price + ask.price) / 2;
332156
+ const sequence = parseSequence(input.snapshot.sequence);
332157
+ return {
332158
+ table: "market_data.orderbook_snapshots",
332159
+ row: compactUndefined3({
332160
+ ...buildOrderbookArchiveTags(input, receivedTimeMs),
332161
+ asset_type: input.assetType,
332162
+ event_time_ms: eventTimeMs,
332163
+ received_time_ms: receivedTimeMs,
332164
+ best_bid: bid.price,
332165
+ best_ask: ask.price,
332166
+ bid_size: bid.size,
332167
+ ask_size: ask.size,
332168
+ mid,
332169
+ spread_bps: computeSpreadBps(bid.price, ask.price),
332170
+ depth_limit: archiveDepthLimit,
332171
+ bid_levels: bids.prices.length,
332172
+ ask_levels: asks.prices.length,
332173
+ bids_price: bids.prices,
332174
+ bids_size: bids.sizes,
332175
+ asks_price: asks.prices,
332176
+ asks_size: asks.sizes,
332177
+ sequence
332178
+ })
332179
+ };
332180
+ }
332181
+ function buildCandleRow(input) {
332182
+ const { context: context2, bar, isClosed, brokerVersion, receivedTimestamp } = input;
332183
+ const tags = buildCommonArchiveTags({
332184
+ deploymentId: context2.deploymentId,
332185
+ accountSelector: context2.accountSelector,
332186
+ exchange: context2.exchange,
332187
+ symbol: context2.symbol,
332188
+ brokerObservedTimestamp: new Date(receivedTimestamp).toISOString()
332189
+ });
332190
+ return {
332191
+ table: "market_data.candles",
332192
+ row: compactUndefined3({
332193
+ ...tags,
332194
+ asset_type: context2.assetType,
332195
+ timeframe: context2.timeframe ?? "1m",
332196
+ open_time_ms: bar.openTimeMs,
332197
+ open: bar.open,
332198
+ high: bar.high,
332199
+ low: bar.low,
332200
+ close: bar.close,
332201
+ volume: bar.volume,
332202
+ quote_volume: bar.quoteVolume,
332203
+ is_closed: isClosed ? 1 : 0,
332204
+ broker_version: brokerVersion
332205
+ })
332206
+ };
332207
+ }
332208
+ function buildCexStreamEventRow(input) {
332209
+ const receivedTimeMs = input.receivedTimestamp;
332210
+ const redactedPayload = redactStreamPayload(input.payload);
332211
+ const tags = buildCommonArchiveTags({
332212
+ deploymentId: input.deploymentId,
332213
+ accountSelector: input.accountSelector,
332214
+ exchange: input.exchange,
332215
+ symbol: input.symbol,
332216
+ brokerObservedTimestamp: new Date(receivedTimeMs).toISOString()
332217
+ });
332218
+ return {
332219
+ table: "market_data.cex_stream_events",
332220
+ row: compactUndefined3({
332221
+ ...tags,
332222
+ asset_type: input.assetType,
332223
+ stream_type: input.streamType,
332224
+ event_time_ms: input.eventTimeMs ?? receivedTimeMs,
332225
+ received_time_ms: receivedTimeMs,
332226
+ payload_json: JSON.stringify(redactedPayload)
332227
+ })
332228
+ };
332229
+ }
332230
+ function buildCexTickerEventRow(input, ticker) {
332231
+ const receivedTimeMs = input.receivedTimestamp;
332232
+ const tags = buildCommonArchiveTags({
332233
+ deploymentId: input.deploymentId,
332234
+ accountSelector: input.accountSelector,
332235
+ exchange: input.exchange,
332236
+ symbol: input.symbol,
332237
+ brokerObservedTimestamp: new Date(receivedTimeMs).toISOString()
332238
+ });
332239
+ return {
332240
+ table: "market_data.cex_ticker_events",
332241
+ row: compactUndefined3({
332242
+ ...tags,
332243
+ asset_type: input.assetType,
332244
+ event_time_ms: ticker.eventTimeMs,
332245
+ received_time_ms: receivedTimeMs,
332246
+ last: ticker.last,
332247
+ bid: ticker.bid,
332248
+ ask: ticker.ask,
332249
+ high: ticker.high,
332250
+ low: ticker.low,
332251
+ open: ticker.open,
332252
+ close: ticker.close,
332253
+ base_volume: ticker.baseVolume,
332254
+ quote_volume: ticker.quoteVolume,
332255
+ change: ticker.change,
332256
+ percentage: ticker.percentage,
332257
+ payload_json: JSON.stringify(redactStreamPayload(input.payload))
332258
+ })
332259
+ };
332260
+ }
332261
+ function buildCexTradeRow(input, trade) {
332262
+ const receivedTimeMs = input.receivedTimestamp;
332263
+ const tags = buildCommonArchiveTags({
332264
+ deploymentId: input.deploymentId,
332265
+ accountSelector: input.accountSelector,
332266
+ exchange: input.exchange,
332267
+ symbol: input.symbol,
332268
+ brokerObservedTimestamp: new Date(receivedTimeMs).toISOString()
332269
+ });
332270
+ return {
332271
+ table: "market_data.cex_trades",
332272
+ row: compactUndefined3({
332273
+ ...tags,
332274
+ asset_type: input.assetType,
332275
+ trade_id: trade.tradeId,
332276
+ event_time_ms: trade.eventTimeMs,
332277
+ received_time_ms: receivedTimeMs,
332278
+ side: trade.side,
332279
+ price: trade.price,
332280
+ amount: trade.amount,
332281
+ cost: trade.cost,
332282
+ taker_or_maker: trade.takerOrMaker
332283
+ })
332284
+ };
332285
+ }
332286
+
332287
+ // src/helpers/market-data-archive/capture.ts
332288
+ async function recordWatchMetric(otelMetrics, metricName, labels) {
332289
+ try {
332290
+ await otelMetrics?.recordCounter(metricName, 1, labels);
332291
+ } catch {}
332292
+ }
332293
+ function watchLabels(stream4, input) {
332294
+ return {
332295
+ stream: stream4,
332296
+ exchange: input.exchange,
332297
+ symbol: input.symbol
332298
+ };
332299
+ }
332300
+ function archiveOrderbookInBackground(archiver, otelMetrics, input, options) {
332301
+ const labels = watchLabels("orderbook", input);
332302
+ recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
332303
+ if (!isMarketArchiveEnabled() || !archiver?.isEnabled()) {
332304
+ return;
331064
332305
  }
331065
- async* [Symbol.asyncIterator]() {
331066
- while (true) {
331067
- const event = await this.nextEvent();
331068
- if (!event) {
331069
- break;
331070
- }
331071
- yield event;
331072
- }
332306
+ if (options?.sampledOut) {
332307
+ recordWatchMetric(otelMetrics, "cex_watch_frames_sampled_out_total", labels);
332308
+ return;
331073
332309
  }
331074
- close() {
331075
- if (this.closed) {
331076
- return;
331077
- }
331078
- this.closed = true;
331079
- this.queue.length = 0;
332310
+ queueMicrotask(() => {
331080
332311
  try {
331081
- this.ws.close();
331082
- } catch {}
331083
- this.flushWaiters();
331084
- }
331085
- handleClose(code, reason) {
331086
- if (this.closed) {
331087
- return;
332312
+ const row = buildOrderbookSnapshotRow(input);
332313
+ if (row) {
332314
+ archiver.enqueue(row);
332315
+ recordWatchMetric(otelMetrics, "cex_watch_frames_archived_total", labels);
332316
+ }
332317
+ } catch (error48) {
332318
+ log.warn("Failed to archive orderbook snapshot", { error: error48 });
331088
332319
  }
331089
- this.fail(formatBinanceUserDataWebSocketClose(code, reason, this.secretValues));
331090
- }
331091
- subscribe() {
331092
- const apiKey = getExchangeString(this.exchange, "apiKey");
331093
- const signedParams = signUserDataStreamParams(this.exchange, {
331094
- apiKey,
331095
- timestamp: Date.now()
331096
- });
331097
- this.ws.send(JSON.stringify({
331098
- id: this.requestId,
331099
- method: "userDataStream.subscribe.signature",
331100
- params: signedParams
331101
- }));
332320
+ });
332321
+ }
332322
+ function archiveOhlcvInBackground(archiver, otelMetrics, tracker, input) {
332323
+ const labels = watchLabels("ohlcv", input);
332324
+ recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
332325
+ if (!isMarketArchiveEnabled() || !archiver?.isEnabled()) {
332326
+ return;
331102
332327
  }
331103
- handleMessage(data) {
331104
- if (this.closed) {
331105
- return;
331106
- }
331107
- let message;
332328
+ queueMicrotask(() => {
331108
332329
  try {
331109
- const decodedData = decodeMessageData(data);
331110
- message = typeof decodedData === "string" ? JSON.parse(decodedData) : decodedData;
332330
+ const candidates = tracker.process(input.payload, input.receivedTimestamp);
332331
+ for (const candidate of candidates) {
332332
+ const row = buildCandleRow({
332333
+ context: input,
332334
+ bar: candidate.bar,
332335
+ isClosed: candidate.isClosed,
332336
+ brokerVersion: candidate.brokerVersion,
332337
+ receivedTimestamp: input.receivedTimestamp
332338
+ });
332339
+ archiver.enqueue(row);
332340
+ }
332341
+ if (candidates.length > 0) {
332342
+ recordWatchMetric(otelMetrics, "cex_watch_frames_archived_total", labels);
332343
+ }
331111
332344
  } catch (error48) {
331112
- this.fail(error48 instanceof Error ? error48 : new Error("Invalid Binance user-data message"));
331113
- return;
332345
+ log.warn("Failed to archive OHLCV candle", { error: error48 });
331114
332346
  }
331115
- if ("id" in message && message.id === this.requestId) {
331116
- if (message.status !== 200) {
331117
- this.fail(new Error(message.error?.msg ?? message.error?.message ?? `Binance user-data subscription failed with status ${message.status}`));
331118
- return;
332347
+ });
332348
+ }
332349
+ function createOrderbookSampler() {
332350
+ return new OrderbookSampler;
332351
+ }
332352
+ function createOhlcvBarTracker() {
332353
+ return new OhlcvBarTracker;
332354
+ }
332355
+ function archiveMarketRowsInBackground(archiver, otelMetrics, stream4, input, enqueueRows) {
332356
+ const labels = watchLabels(stream4, input);
332357
+ recordWatchMetric(otelMetrics, "cex_watch_frames_received_total", labels);
332358
+ if (!isMarketArchiveEnabled() || !archiver?.isEnabled()) {
332359
+ return;
332360
+ }
332361
+ queueMicrotask(() => {
332362
+ try {
332363
+ const rows = enqueueRows();
332364
+ for (const row of rows) {
332365
+ archiver.enqueue(row);
331119
332366
  }
331120
- this.subscriptionId = message.result?.subscriptionId ?? null;
331121
- return;
331122
- }
331123
- if (!("event" in message) || !message.event) {
331124
- return;
331125
- }
331126
- const subscriptionId = message.subscriptionId ?? this.subscriptionId;
331127
- if (subscriptionId === null || subscriptionId === undefined) {
331128
- return;
332367
+ if (rows.length > 0) {
332368
+ recordWatchMetric(otelMetrics, "cex_watch_frames_archived_total", labels);
332369
+ }
332370
+ } catch (error48) {
332371
+ log.warn(`Failed to archive ${stream4} market data`, { error: error48 });
331129
332372
  }
331130
- this.push({ subscriptionId, event: message.event });
332373
+ });
332374
+ }
332375
+ function archiveTradesInBackground(archiver, otelMetrics, input) {
332376
+ archiveMarketRowsInBackground(archiver, otelMetrics, "trades", input, () => extractTrades(input.payload, input.receivedTimestamp).map((trade) => buildCexTradeRow(input, trade)));
332377
+ }
332378
+ function archiveTickerInBackground(archiver, otelMetrics, input) {
332379
+ archiveMarketRowsInBackground(archiver, otelMetrics, "ticker", input, () => {
332380
+ const ticker = parseTicker(input.payload, input.receivedTimestamp);
332381
+ return ticker ? [buildCexTickerEventRow(input, ticker)] : [];
332382
+ });
332383
+ }
332384
+ function archiveCexStreamEventInBackground(archiver, otelMetrics, input) {
332385
+ archiveMarketRowsInBackground(archiver, otelMetrics, "stream", input, () => [
332386
+ buildCexStreamEventRow(input)
332387
+ ]);
332388
+ }
332389
+ // src/helpers/market-data-archive/ohlcv-bootstrap.ts
332390
+ var DEFAULT_OHLCV_BOOTSTRAP_LIMIT = 100;
332391
+ var MAX_OHLCV_BOOTSTRAP_LIMIT = 1000;
332392
+ function resolveOhlcvBootstrapLimit(optionValue) {
332393
+ const raw = optionValue?.trim() || process.env.CEX_BROKER_OHLCV_ARCHIVE_BOOTSTRAP_LIMIT?.trim();
332394
+ if (raw === "0" || raw?.toLowerCase() === "false") {
332395
+ return 0;
331131
332396
  }
331132
- push(event) {
331133
- if (this.closed) {
331134
- return;
331135
- }
331136
- const waiter = this.waiters.shift();
331137
- if (waiter) {
331138
- waiter.resolve(event);
331139
- return;
331140
- }
331141
- if (this.queue.length >= this.maxBufferedEvents) {
331142
- this.fail(new Error(`Binance user-data stream buffered event limit exceeded (${this.maxBufferedEvents}); downstream consumer is not keeping up`));
331143
- return;
331144
- }
331145
- this.queue.push(event);
332397
+ if (!raw) {
332398
+ return DEFAULT_OHLCV_BOOTSTRAP_LIMIT;
331146
332399
  }
331147
- nextEvent() {
331148
- const event = this.queue.shift();
331149
- if (event) {
331150
- return Promise.resolve(event);
331151
- }
331152
- if (this.closeError) {
331153
- return Promise.reject(this.closeError);
331154
- }
331155
- if (this.closed) {
331156
- return Promise.resolve(null);
331157
- }
331158
- return new Promise((resolve, reject) => {
331159
- this.waiters.push({ resolve, reject });
331160
- });
332400
+ const parsed = Number.parseInt(raw, 10);
332401
+ if (!Number.isFinite(parsed) || parsed <= 0) {
332402
+ return DEFAULT_OHLCV_BOOTSTRAP_LIMIT;
331161
332403
  }
331162
- fail(error48) {
331163
- if (this.closeError) {
331164
- return;
331165
- }
331166
- this.closeError = error48;
331167
- this.closed = true;
331168
- this.queue.length = 0;
331169
- this.flushWaiters();
331170
- try {
331171
- this.ws.close();
331172
- } catch {}
332404
+ return Math.min(parsed, MAX_OHLCV_BOOTSTRAP_LIMIT);
332405
+ }
332406
+
332407
+ // src/helpers/market-data-archive/ohlcv-history.ts
332408
+ function supportsFetchOhlcv(broker) {
332409
+ const fetchOHLCV = broker.fetchOHLCV;
332410
+ const hasValue = broker.has?.fetchOHLCV;
332411
+ return typeof fetchOHLCV === "function" && hasValue !== false;
332412
+ }
332413
+ async function bootstrapOhlcvHistory(broker, archiver, otelMetrics, tracker, input, options) {
332414
+ const limit = resolveOhlcvBootstrapLimit(options?.bootstrapLimit);
332415
+ if (limit <= 0 || !supportsFetchOhlcv(broker)) {
332416
+ return null;
331173
332417
  }
331174
- flushWaiters() {
331175
- const error48 = this.closeError;
331176
- for (const waiter of this.waiters.splice(0)) {
331177
- if (error48) {
331178
- waiter.reject(error48);
331179
- } else {
331180
- waiter.resolve(null);
331181
- }
332418
+ try {
332419
+ const fetchOHLCV = broker.fetchOHLCV.bind(broker);
332420
+ const payload = await fetchOHLCV(input.symbol, input.timeframe ?? "1m", undefined, limit);
332421
+ if (!Array.isArray(payload) || payload.length === 0) {
332422
+ return null;
331182
332423
  }
332424
+ const receivedTimestamp = Date.now();
332425
+ archiveOhlcvInBackground(archiver, otelMetrics, tracker, {
332426
+ ...input,
332427
+ payload,
332428
+ receivedTimestamp
332429
+ });
332430
+ return payload;
332431
+ } catch (error48) {
332432
+ log.warn("OHLCV bootstrap fetch failed", {
332433
+ error: error48,
332434
+ symbol: input.symbol,
332435
+ exchange: input.exchange,
332436
+ timeframe: input.timeframe
332437
+ });
332438
+ return null;
331183
332439
  }
331184
332440
  }
331185
- function isBinanceBalanceUserDataEvent(event) {
331186
- return event.e === "outboundAccountPosition" || event.e === "balanceUpdate" || event.e === "externalLockUpdate";
331187
- }
331188
- function isBinanceOrderUserDataEvent(event) {
331189
- return event.e === "executionReport" || event.e === "listStatus";
331190
- }
331191
-
331192
332441
  // src/handlers/subscribe/handler.ts
331193
332442
  function isBinanceSpotAccountSubscription(cex3, subscriptionType, marketTypeInput) {
331194
332443
  return cex3 === "binance" && parseMarketType(marketTypeInput) === "spot" && (subscriptionType === SubscriptionType.BALANCE || subscriptionType === SubscriptionType.ORDERS);
@@ -331257,7 +332506,7 @@ async function getBinanceMarketId(broker, symbol2) {
331257
332506
  }
331258
332507
  return symbol2.replace("/", "").toUpperCase();
331259
332508
  }
331260
- async function streamBinanceUserData(call, broker, symbol2, subscriptionType, isClosed) {
332509
+ async function streamBinanceUserData(call, broker, symbol2, subscriptionType, isClosed, archiveContext) {
331261
332510
  const userDataStream = new BinanceSpotUserDataStream(broker);
331262
332511
  call.once("close", () => userDataStream.close());
331263
332512
  call.once("cancelled", () => userDataStream.close());
@@ -331282,37 +332531,78 @@ async function streamBinanceUserData(call, broker, symbol2, subscriptionType, is
331282
332531
  continue;
331283
332532
  }
331284
332533
  }
332534
+ const receivedTimestamp = Date.now();
331285
332535
  if (!await writeSubscribeFrame(call, isClosed, {
331286
332536
  data: JSON.stringify({
331287
332537
  subscriptionId: message.subscriptionId,
331288
332538
  event
331289
332539
  }),
331290
- timestamp: Date.now(),
332540
+ timestamp: receivedTimestamp,
331291
332541
  symbol: symbol2,
331292
332542
  type: subscriptionType
331293
332543
  })) {
331294
332544
  break;
331295
332545
  }
332546
+ const archiveSubscriptionType = subscriptionType === SubscriptionType.BALANCE ? "BALANCE" : "ORDERS";
332547
+ archiveSubscribeStreamInBackground(archiveContext?.archiver, {
332548
+ exchange: archiveContext?.exchange ?? "binance",
332549
+ accountSelector: archiveContext?.accountSelector,
332550
+ symbol: symbol2,
332551
+ subscriptionType: archiveSubscriptionType,
332552
+ streamPayload: event
332553
+ });
332554
+ if (archiveContext) {
332555
+ archiveCexStreamEventInBackground(archiveContext.archiver, archiveContext.otelMetrics, {
332556
+ deploymentId: archiveContext.deploymentId,
332557
+ exchange: archiveContext.exchange,
332558
+ symbol: symbol2,
332559
+ assetType: archiveContext.assetType,
332560
+ accountSelector: archiveContext.accountSelector,
332561
+ streamType: archiveSubscriptionType,
332562
+ payload: event,
332563
+ receivedTimestamp
332564
+ });
332565
+ }
331296
332566
  }
331297
332567
  } finally {
331298
332568
  userDataStream.close();
331299
332569
  }
331300
332570
  }
331301
- async function runCcxtSubscribeLoop(call, isClosed, symbol2, subscriptionType, watch) {
332571
+ async function runCcxtSubscribeLoop(call, isClosed, symbol2, subscriptionType, watch, archiveContext) {
331302
332572
  while (!isClosed()) {
331303
332573
  const data = await watch();
332574
+ const receivedTimestamp = Date.now();
331304
332575
  if (!await writeSubscribeFrame(call, isClosed, {
331305
332576
  data: JSON.stringify(data),
331306
- timestamp: Date.now(),
332577
+ timestamp: receivedTimestamp,
331307
332578
  symbol: symbol2,
331308
332579
  type: subscriptionType
331309
332580
  })) {
331310
332581
  break;
331311
332582
  }
332583
+ if (archiveContext?.archiveSubscriptionType) {
332584
+ archiveSubscribeStreamInBackground(archiveContext.archiver, {
332585
+ exchange: archiveContext.exchange,
332586
+ accountSelector: archiveContext.accountSelector,
332587
+ symbol: symbol2,
332588
+ subscriptionType: archiveContext.archiveSubscriptionType,
332589
+ streamPayload: data
332590
+ });
332591
+ archiveCexStreamEventInBackground(archiveContext.archiver, archiveContext.otelMetrics, {
332592
+ deploymentId: archiveContext.deploymentId,
332593
+ exchange: archiveContext.exchange,
332594
+ symbol: symbol2,
332595
+ assetType: archiveContext.assetType,
332596
+ accountSelector: archiveContext.accountSelector,
332597
+ streamType: archiveContext.archiveSubscriptionType,
332598
+ payload: data,
332599
+ receivedTimestamp
332600
+ });
332601
+ }
331312
332602
  }
331313
332603
  }
331314
332604
  function createSubscribeHandler(deps) {
331315
- const { brokers, whitelistIps, otelMetrics } = deps;
332605
+ const { brokers, whitelistIps, otelMetrics, brokerArchiver } = deps;
331316
332606
  return async (call) => {
331317
332607
  const subscribeStartTime = Date.now();
331318
332608
  let streamClosed = false;
@@ -331378,7 +332668,9 @@ function createSubscribeHandler(deps) {
331378
332668
  return;
331379
332669
  }
331380
332670
  const normalizedCex = cex3.trim().toLowerCase();
331381
- const selectedBroker = selectBroker(brokers[normalizedCex], metadata) ?? createBroker(normalizedCex, metadata);
332671
+ const brokerPool = brokers[normalizedCex];
332672
+ const selectedBrokerAccount = selectBrokerAccount(brokerPool, metadata);
332673
+ const selectedBroker = selectedBrokerAccount?.exchange ?? createBroker(normalizedCex, metadata);
331382
332674
  const broker = selectedBroker ?? createPublicBroker(normalizedCex);
331383
332675
  if (!broker) {
331384
332676
  await writeSubscribeError(call, isStreamClosed, {
@@ -331392,8 +332684,19 @@ function createSubscribeHandler(deps) {
331392
332684
  return;
331393
332685
  }
331394
332686
  const resolvedSymbol = await resolveSubscriptionSymbol(broker, symbol2, options?.marketType);
332687
+ const assetType = parseMarketType(options?.marketType);
332688
+ const deploymentId = brokerArchiver?.getDeploymentId() ?? "unknown";
332689
+ const streamArchiveContext = {
332690
+ archiver: brokerArchiver,
332691
+ otelMetrics,
332692
+ exchange: normalizedCex,
332693
+ accountSelector: selectedBrokerAccount?.label,
332694
+ deploymentId,
332695
+ assetType
332696
+ };
331395
332697
  if (isBinanceSpotAccountSubscription(normalizedCex, subscriptionType, options?.marketType)) {
331396
- if (!selectedBroker) {
332698
+ const accountBroker = selectedBrokerAccount?.exchange ?? selectedBroker;
332699
+ if (!accountBroker) {
331397
332700
  await writeSubscribeError(call, isStreamClosed, {
331398
332701
  data: JSON.stringify({
331399
332702
  error: "Binance account subscriptions require API credentials"
@@ -331404,29 +332707,42 @@ function createSubscribeHandler(deps) {
331404
332707
  });
331405
332708
  return;
331406
332709
  }
331407
- await streamBinanceUserData(call, selectedBroker, resolvedSymbol, subscriptionType, isStreamClosed);
332710
+ await streamBinanceUserData(call, accountBroker, resolvedSymbol, subscriptionType, isStreamClosed, streamArchiveContext);
331408
332711
  return;
331409
332712
  }
331410
332713
  switch (subscriptionType) {
331411
332714
  case SubscriptionType.ORDERBOOK:
331412
332715
  try {
332716
+ const orderbookSampler = createOrderbookSampler();
331413
332717
  while (!isStreamClosed()) {
331414
332718
  const depthLimit = parseOptionalDepthLimit(options?.depthLimit ?? options?.limit);
331415
332719
  const orderbook = depthLimit === undefined ? await broker.watchOrderBook(resolvedSymbol) : await broker.watchOrderBook(resolvedSymbol, depthLimit);
331416
332720
  const receivedTimestamp = Date.now();
332721
+ const normalizedSnapshot = normalizeOrderBookSnapshot(orderbook, {
332722
+ exchange: normalizedCex,
332723
+ symbol: resolvedSymbol,
332724
+ depthLimit: depthLimit ?? Math.max(Array.isArray(orderbook?.bids) ? orderbook.bids.length : 0, Array.isArray(orderbook?.asks) ? orderbook.asks.length : 0),
332725
+ receivedTimestamp
332726
+ });
331417
332727
  if (!await writeSubscribeFrame(call, isStreamClosed, {
331418
- data: JSON.stringify(normalizeOrderBookSnapshot(orderbook, {
331419
- exchange: normalizedCex,
331420
- symbol: resolvedSymbol,
331421
- depthLimit: depthLimit ?? Math.max(Array.isArray(orderbook?.bids) ? orderbook.bids.length : 0, Array.isArray(orderbook?.asks) ? orderbook.asks.length : 0),
331422
- receivedTimestamp
331423
- })),
332728
+ data: JSON.stringify(normalizedSnapshot),
331424
332729
  timestamp: receivedTimestamp,
331425
332730
  symbol: resolvedSymbol,
331426
332731
  type: subscriptionType
331427
332732
  })) {
331428
332733
  break;
331429
332734
  }
332735
+ const shouldArchive = orderbookSampler.shouldEmit(receivedTimestamp);
332736
+ archiveOrderbookInBackground(brokerArchiver, otelMetrics, {
332737
+ deploymentId,
332738
+ exchange: normalizedCex,
332739
+ symbol: resolvedSymbol,
332740
+ assetType,
332741
+ accountSelector: selectedBrokerAccount?.label,
332742
+ snapshot: normalizedSnapshot
332743
+ }, {
332744
+ sampledOut: !shouldArchive
332745
+ });
331430
332746
  }
331431
332747
  } catch (error48) {
331432
332748
  log.error(`Error fetching orderbook for ${resolvedSymbol} on ${cex3}:`, error48);
@@ -331443,7 +332759,27 @@ function createSubscribeHandler(deps) {
331443
332759
  break;
331444
332760
  case SubscriptionType.TRADES:
331445
332761
  try {
331446
- await runCcxtSubscribeLoop(call, isStreamClosed, resolvedSymbol, subscriptionType, () => broker.watchTrades(resolvedSymbol));
332762
+ while (!isStreamClosed()) {
332763
+ const data = await broker.watchTrades(resolvedSymbol);
332764
+ const receivedTimestamp = Date.now();
332765
+ if (!await writeSubscribeFrame(call, isStreamClosed, {
332766
+ data: JSON.stringify(data),
332767
+ timestamp: receivedTimestamp,
332768
+ symbol: resolvedSymbol,
332769
+ type: subscriptionType
332770
+ })) {
332771
+ break;
332772
+ }
332773
+ archiveTradesInBackground(brokerArchiver, otelMetrics, {
332774
+ deploymentId,
332775
+ exchange: normalizedCex,
332776
+ symbol: resolvedSymbol,
332777
+ assetType,
332778
+ accountSelector: selectedBrokerAccount?.label,
332779
+ payload: data,
332780
+ receivedTimestamp
332781
+ });
332782
+ }
331447
332783
  } catch (error48) {
331448
332784
  const message = getErrorMessage(error48);
331449
332785
  log.error(`Error fetching trades for ${resolvedSymbol} on ${cex3}:`, error48);
@@ -331459,7 +332795,27 @@ function createSubscribeHandler(deps) {
331459
332795
  break;
331460
332796
  case SubscriptionType.TICKER:
331461
332797
  try {
331462
- await runCcxtSubscribeLoop(call, isStreamClosed, resolvedSymbol, subscriptionType, () => broker.watchTicker(resolvedSymbol));
332798
+ while (!isStreamClosed()) {
332799
+ const data = await broker.watchTicker(resolvedSymbol);
332800
+ const receivedTimestamp = Date.now();
332801
+ if (!await writeSubscribeFrame(call, isStreamClosed, {
332802
+ data: JSON.stringify(data),
332803
+ timestamp: receivedTimestamp,
332804
+ symbol: resolvedSymbol,
332805
+ type: subscriptionType
332806
+ })) {
332807
+ break;
332808
+ }
332809
+ archiveTickerInBackground(brokerArchiver, otelMetrics, {
332810
+ deploymentId,
332811
+ exchange: normalizedCex,
332812
+ symbol: resolvedSymbol,
332813
+ assetType,
332814
+ accountSelector: selectedBrokerAccount?.label,
332815
+ payload: data,
332816
+ receivedTimestamp
332817
+ });
332818
+ }
331463
332819
  } catch (error48) {
331464
332820
  const message = getErrorMessage(error48);
331465
332821
  log.error(`Error fetching ticker for ${resolvedSymbol} on ${cex3}:`, error48);
@@ -331476,7 +332832,53 @@ function createSubscribeHandler(deps) {
331476
332832
  case SubscriptionType.OHLCV:
331477
332833
  try {
331478
332834
  const timeframe = options?.timeframe || "1m";
331479
- await runCcxtSubscribeLoop(call, isStreamClosed, resolvedSymbol, subscriptionType, () => broker.fetchOHLCVWs(resolvedSymbol, timeframe));
332835
+ const ohlcvBarTracker = createOhlcvBarTracker();
332836
+ const ohlcvArchiveInput = {
332837
+ deploymentId,
332838
+ exchange: normalizedCex,
332839
+ symbol: resolvedSymbol,
332840
+ assetType,
332841
+ timeframe,
332842
+ accountSelector: selectedBrokerAccount?.label,
332843
+ receivedTimestamp: Date.now(),
332844
+ payload: []
332845
+ };
332846
+ const bootstrapPayload = await bootstrapOhlcvHistory(broker, brokerArchiver, otelMetrics, ohlcvBarTracker, ohlcvArchiveInput, { bootstrapLimit: options?.bootstrapLimit });
332847
+ let ohlcvStreamActive = true;
332848
+ if (bootstrapPayload) {
332849
+ const bootstrapTimestamp = Date.now();
332850
+ const bootstrapSent = await writeSubscribeFrame(call, isStreamClosed, {
332851
+ data: JSON.stringify(bootstrapPayload),
332852
+ timestamp: bootstrapTimestamp,
332853
+ symbol: resolvedSymbol,
332854
+ type: subscriptionType
332855
+ });
332856
+ if (!bootstrapSent) {
332857
+ ohlcvStreamActive = false;
332858
+ }
332859
+ }
332860
+ while (ohlcvStreamActive && !isStreamClosed()) {
332861
+ const data = await broker.fetchOHLCVWs(resolvedSymbol, timeframe);
332862
+ const receivedTimestamp = Date.now();
332863
+ if (!await writeSubscribeFrame(call, isStreamClosed, {
332864
+ data: JSON.stringify(data),
332865
+ timestamp: receivedTimestamp,
332866
+ symbol: resolvedSymbol,
332867
+ type: subscriptionType
332868
+ })) {
332869
+ break;
332870
+ }
332871
+ archiveOhlcvInBackground(brokerArchiver, otelMetrics, ohlcvBarTracker, {
332872
+ deploymentId,
332873
+ exchange: normalizedCex,
332874
+ symbol: resolvedSymbol,
332875
+ assetType,
332876
+ timeframe,
332877
+ accountSelector: selectedBrokerAccount?.label,
332878
+ payload: data,
332879
+ receivedTimestamp
332880
+ });
332881
+ }
331480
332882
  } catch (error48) {
331481
332883
  log.error(`Error fetching OHLCV for ${resolvedSymbol} on ${cex3}:`, error48);
331482
332884
  const message = getErrorMessage(error48);
@@ -331492,7 +332894,10 @@ function createSubscribeHandler(deps) {
331492
332894
  break;
331493
332895
  case SubscriptionType.BALANCE:
331494
332896
  try {
331495
- await runCcxtSubscribeLoop(call, isStreamClosed, symbol2, subscriptionType, () => broker.watchBalance());
332897
+ await runCcxtSubscribeLoop(call, isStreamClosed, symbol2, subscriptionType, () => broker.watchBalance(), {
332898
+ ...streamArchiveContext,
332899
+ archiveSubscriptionType: "BALANCE"
332900
+ });
331496
332901
  } catch (error48) {
331497
332902
  const message = getErrorMessage(error48);
331498
332903
  log.error(`Error fetching balance for ${cex3}:`, error48);
@@ -331508,7 +332913,10 @@ function createSubscribeHandler(deps) {
331508
332913
  break;
331509
332914
  case SubscriptionType.ORDERS:
331510
332915
  try {
331511
- await runCcxtSubscribeLoop(call, isStreamClosed, resolvedSymbol, subscriptionType, () => broker.watchOrders(resolvedSymbol));
332916
+ await runCcxtSubscribeLoop(call, isStreamClosed, resolvedSymbol, subscriptionType, () => broker.watchOrders(resolvedSymbol), {
332917
+ ...streamArchiveContext,
332918
+ archiveSubscriptionType: "ORDERS"
332919
+ });
331512
332920
  } catch (error48) {
331513
332921
  log.error(`Error fetching orders for ${resolvedSymbol} on ${cex3}:`, error48);
331514
332922
  const message = getErrorMessage(error48);
@@ -331689,7 +333097,7 @@ var CEX_BROKER_PACKAGE_DEFINITION = protoLoader.fromJSON(node_descriptor_default
331689
333097
  // src/server.ts
331690
333098
  var grpcObj = grpc14.loadPackageDefinition(CEX_BROKER_PACKAGE_DEFINITION);
331691
333099
  var cexNode = grpcObj.cex_broker;
331692
- function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics) {
333100
+ function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver) {
331693
333101
  const server = new grpc14.Server;
331694
333102
  server.addService(cexNode.cex_service.service, {
331695
333103
  ExecuteAction: createExecuteActionHandler({
@@ -331698,12 +333106,14 @@ function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, ot
331698
333106
  whitelistIps,
331699
333107
  useVerity,
331700
333108
  verityProverUrl,
331701
- otelMetrics
333109
+ otelMetrics,
333110
+ brokerArchiver
331702
333111
  }),
331703
333112
  Subscribe: createSubscribeHandler({
331704
333113
  brokers,
331705
333114
  whitelistIps,
331706
- otelMetrics
333115
+ otelMetrics,
333116
+ brokerArchiver
331707
333117
  })
331708
333118
  });
331709
333119
  return server;
@@ -331840,6 +333250,7 @@ class CEXBroker {
331840
333250
  useVerity = false;
331841
333251
  otelMetrics;
331842
333252
  otelLogs;
333253
+ brokerArchiver;
331843
333254
  depositReconciler;
331844
333255
  loadEnvConfig() {
331845
333256
  log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
@@ -331951,6 +333362,7 @@ class CEXBroker {
331951
333362
  this.otelMetrics = createOtelMetricsFromEnv();
331952
333363
  this.otelLogs = createOtelLogsFromEnv();
331953
333364
  }
333365
+ this.brokerArchiver = createBrokerExecutionArchiverFromEnv(this.otelLogs, this.otelMetrics);
331954
333366
  this.loadExchangeCredentials(apiCredentials);
331955
333367
  this.whitelistIps = [
331956
333368
  ...(config2 ?? { whitelistIps: [] }).whitelistIps ?? [],
@@ -331983,6 +333395,9 @@ class CEXBroker {
331983
333395
  if (this.server) {
331984
333396
  await this.server.forceShutdown();
331985
333397
  }
333398
+ if (this.brokerArchiver) {
333399
+ await this.brokerArchiver.close();
333400
+ }
331986
333401
  if (this.otelMetrics) {
331987
333402
  await this.otelMetrics.close();
331988
333403
  }
@@ -332002,7 +333417,7 @@ class CEXBroker {
332002
333417
  if (this.otelMetrics?.isOtelEnabled()) {
332003
333418
  await this.otelMetrics.initialize();
332004
333419
  }
332005
- this.server = getServer(this.policy, this.brokers, this.whitelistIps, this.useVerity, this.#verityProverUrl, this.otelMetrics);
333420
+ this.server = getServer(this.policy, this.brokers, this.whitelistIps, this.useVerity, this.#verityProverUrl, this.otelMetrics, this.brokerArchiver);
332006
333421
  this.server.bindAsync(`0.0.0.0:${this.port}`, grpc15.ServerCredentials.createInsecure(), (err2, port) => {
332007
333422
  if (err2) {
332008
333423
  log.error(err2);