@usherlabs/cex-broker 0.2.27 → 0.2.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -314993,6 +314993,65 @@ function validateDeposit(policy, exchange, network, ticker) {
314993
314993
  return { valid: true };
314994
314994
  }
314995
314995
 
314996
+ // src/helpers/shared/errors.ts
314997
+ var MAX_ERROR_DETAIL_LENGTH = 512;
314998
+ var REDACTED_ERROR_MESSAGE = "redacted_error";
314999
+ function getErrorMessage(error) {
315000
+ return error instanceof Error ? error.message : typeof error === "string" ? error : "Unknown error";
315001
+ }
315002
+ function errorClassName(error) {
315003
+ if (!(error instanceof Error)) {
315004
+ return;
315005
+ }
315006
+ const name = error.constructor?.name || error.name;
315007
+ return name && name !== "Error" ? name : undefined;
315008
+ }
315009
+ function errorCode(error) {
315010
+ if (error === null || typeof error !== "object") {
315011
+ return;
315012
+ }
315013
+ const code = error.code;
315014
+ if (typeof code === "string") {
315015
+ return code.trim() || undefined;
315016
+ }
315017
+ if (typeof code === "number" && Number.isFinite(code)) {
315018
+ return String(code);
315019
+ }
315020
+ if (typeof code === "bigint") {
315021
+ return String(code);
315022
+ }
315023
+ return;
315024
+ }
315025
+ function sanitizeErrorDetail(error, options = {}) {
315026
+ const className = errorClassName(error);
315027
+ const message = getErrorMessage(error);
315028
+ const code = options.includeCode ? errorCode(error) : undefined;
315029
+ const prefix = [className, code ? `[code=${code}]` : undefined].filter((part) => Boolean(part)).join(" ");
315030
+ const detail = prefix ? `${prefix}: ${message}` : message;
315031
+ return detail.replace(/\s+/g, " ").trim().slice(0, MAX_ERROR_DETAIL_LENGTH);
315032
+ }
315033
+ function safeLogError(context2, error) {
315034
+ try {
315035
+ log.error(context2, { error });
315036
+ } catch {
315037
+ console.error(context2, error);
315038
+ }
315039
+ }
315040
+ function safeLogRedactedError(context2, error) {
315041
+ const errorType = errorClassName(error) ?? (error instanceof Error ? error.name : typeof error);
315042
+ try {
315043
+ log.error(context2, {
315044
+ error_type: errorType,
315045
+ error_message: REDACTED_ERROR_MESSAGE
315046
+ });
315047
+ } catch {
315048
+ console.error(context2, {
315049
+ error_type: errorType,
315050
+ error_message: REDACTED_ERROR_MESSAGE
315051
+ });
315052
+ }
315053
+ }
315054
+
314996
315055
  // src/helpers/shared/guards.ts
314997
315056
  function isRecord(value) {
314998
315057
  return value !== null && typeof value === "object" && !Array.isArray(value);
@@ -315013,7 +315072,6 @@ var NUMERIC_METRICS = [
315013
315072
  ["feeAmount", "cex_market_action_fee_amount"],
315014
315073
  ["feeRate", "cex_market_action_fee_rate"]
315015
315074
  ];
315016
- var REDACTED_ERROR_MESSAGE = "redacted_error";
315017
315075
  async function emitOrderExecutionTelemetry(otelMetrics, context2, order, error) {
315018
315076
  try {
315019
315077
  const telemetry = buildOrderExecutionTelemetry(context2, order, error);
@@ -315505,7 +315563,7 @@ function buildOrderEventArchiveRow(input) {
315505
315563
  fee_rate: telemetry.feeRate,
315506
315564
  exchange_timestamp: telemetry.exchangeTimestamp,
315507
315565
  error_type: telemetry.errorType,
315508
- error_message: telemetry.errorMessage,
315566
+ error_message: input.errorDetail ?? telemetry.errorMessage,
315509
315567
  payload_json: JSON.stringify(telemetry)
315510
315568
  })
315511
315569
  };
@@ -315637,429 +315695,163 @@ function buildMarketMetadataSnapshotRow(input) {
315637
315695
  };
315638
315696
  }
315639
315697
 
315640
- // src/helpers/broker-execution-archive/capture.ts
315641
- function archiveOrderExecutionInBackground(archiver, context2, order, error, options) {
315642
- if (!archiver?.isEnabled()) {
315643
- return;
315698
+ // src/helpers/broker-execution-archive/writer.ts
315699
+ var import_api_logs2 = __toESM(require_src7(), 1);
315700
+ import { closeSync, fsyncSync, openSync, writeSync } from "node:fs";
315701
+ import { request as httpRequest2 } from "node:http";
315702
+ import { request as httpsRequest2 } from "node:https";
315703
+ var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
315704
+ "broker_execution.order_events",
315705
+ "broker_execution.market_metadata_snapshots",
315706
+ "broker_execution.transfer_events",
315707
+ "broker_execution.fill_events"
315708
+ ]);
315709
+ function isBrokerExecutionArchiveTable(table) {
315710
+ return BROKER_EXECUTION_ARCHIVE_TABLES.has(table);
315711
+ }
315712
+
315713
+ class BrokerExecutionArchiveDurabilityError extends Error {
315714
+ constructor(message, options) {
315715
+ super(message, options);
315716
+ this.name = "BrokerExecutionArchiveDurabilityError";
315644
315717
  }
315645
- queueMicrotask(() => {
315646
- try {
315647
- const telemetry = buildOrderExecutionTelemetry(context2, order, error);
315648
- const tags = buildCommonArchiveTags({
315649
- deploymentId: archiver.getDeploymentId(),
315650
- accountSelector: context2.accountLabel,
315651
- exchange: context2.cex,
315652
- symbol: telemetry.symbol,
315653
- brokerObservedTimestamp: telemetry.brokerObservedTimestamp
315654
- });
315655
- archiver.enqueue(buildOrderEventArchiveRow({
315656
- tags,
315657
- action: context2.action,
315658
- telemetry,
315659
- marketMetadataHash: options?.marketMetadataHash
315660
- }));
315661
- } catch (archiveError) {
315662
- log.warn("Failed to archive order execution", { error: archiveError });
315663
- }
315664
- });
315665
315718
  }
315666
- function archiveSubscribeStreamInBackground(archiver, input) {
315667
- if (!archiver?.isEnabled()) {
315668
- return;
315719
+ function rethrowArchiveDurabilityError(error) {
315720
+ if (error instanceof BrokerExecutionArchiveDurabilityError) {
315721
+ throw error;
315669
315722
  }
315670
- queueMicrotask(() => {
315723
+ }
315724
+ var DEFAULT_MAX_QUEUE_SIZE = 1e4;
315725
+ var DEFAULT_BATCH_SIZE = 10;
315726
+ var DEFAULT_FLUSH_INTERVAL_MS = 1000;
315727
+ var DEFAULT_FORWARDER_TIMEOUT_MS = 3000;
315728
+ var SHED_WARN_INTERVAL_MS = 60000;
315729
+ function isArchiveOtelLogsEnabled() {
315730
+ return process.env.CEX_BROKER_ARCHIVE_OTEL_LOGS_ENABLED === "true";
315731
+ }
315732
+ function resolveArchiveForwarderUrlFromEnv() {
315733
+ return process.env.CEX_BROKER_ARCHIVE_FORWARDER_URL?.trim() || undefined;
315734
+ }
315735
+
315736
+ class BrokerExecutionArchiver {
315737
+ deploymentId;
315738
+ otelLogs;
315739
+ otelMetrics;
315740
+ forwarderUrl;
315741
+ deadLetterPath;
315742
+ deadLetterFd;
315743
+ maxQueueSize;
315744
+ batchSize;
315745
+ flushIntervalMs;
315746
+ forwarderTimeoutMs;
315747
+ queue = [];
315748
+ stats = {
315749
+ enqueued: 0,
315750
+ shed: 0,
315751
+ flushed: 0,
315752
+ forwarderFailures: 0
315753
+ };
315754
+ flushTimer = null;
315755
+ flushInFlight = null;
315756
+ lastShedWarnAtMs = 0;
315757
+ closed = false;
315758
+ enabled;
315759
+ forwarderAuthToken;
315760
+ constructor(options) {
315761
+ this.deploymentId = options.deploymentId?.trim() || process.env.CEX_BROKER_DEPLOYMENT_ID?.trim() || "unknown";
315762
+ this.otelLogs = options.otelLogs;
315763
+ this.otelMetrics = options.otelMetrics;
315764
+ this.forwarderUrl = options.forwarderUrl?.trim();
315765
+ this.deadLetterPath = options.deadLetterPath?.trim();
315766
+ this.maxQueueSize = options.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE;
315767
+ this.batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
315768
+ this.flushIntervalMs = options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS;
315769
+ this.forwarderTimeoutMs = options.forwarderTimeoutMs ?? DEFAULT_FORWARDER_TIMEOUT_MS;
315770
+ this.enabled = options.enabled;
315771
+ this.forwarderAuthToken = process.env.CEX_BROKER_ARCHIVE_FORWARDER_TOKEN?.trim() || undefined;
315772
+ if (!this.enabled) {
315773
+ log.info("Broker execution archive disabled", { enabled: false });
315774
+ return;
315775
+ }
315776
+ validateForwarderUrl(this.forwarderUrl);
315777
+ if (!this.deadLetterPath) {
315778
+ throw new Error("Broker execution archive is enabled but CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH is missing");
315779
+ }
315671
315780
  try {
315672
- const tags = buildCommonArchiveTags({
315673
- deploymentId: archiver.getDeploymentId(),
315674
- accountSelector: input.accountSelector,
315675
- exchange: input.exchange,
315676
- symbol: input.symbol
315677
- });
315678
- archiver.enqueue(buildSubscribeStreamArchiveRow({
315679
- tags,
315680
- subscriptionType: input.subscriptionType,
315681
- streamPayload: input.streamPayload,
315682
- secretLiterals: input.secretLiterals
315683
- }));
315684
- } catch (archiveError) {
315685
- log.warn("Failed to archive subscribe stream event", {
315686
- error: archiveError
315687
- });
315781
+ this.deadLetterFd = openSync(this.deadLetterPath, "a", 384);
315782
+ } catch {
315783
+ throw new Error("Broker execution archive cannot open CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH for append");
315688
315784
  }
315689
- });
315690
- }
315691
- function archiveTransferEventInBackground(archiver, input) {
315692
- if (!archiver?.isEnabled()) {
315693
- return;
315694
- }
315695
- queueMicrotask(() => {
315696
315785
  try {
315697
- const tags = buildCommonArchiveTags({
315698
- deploymentId: archiver.getDeploymentId(),
315699
- accountSelector: input.accountSelector,
315700
- exchange: input.exchange,
315701
- symbol: input.assetSymbol,
315702
- brokerObservedTimestamp: input.brokerObservedTimestamp
315786
+ this.flushTimer = setInterval(() => {
315787
+ this.flush();
315788
+ }, this.flushIntervalMs);
315789
+ this.flushTimer.unref?.();
315790
+ log.info("Broker execution archive enabled", {
315791
+ enabled: true,
315792
+ otel_mirror_enabled: Boolean(this.otelLogs?.isOtelEnabled())
315703
315793
  });
315704
- archiver.enqueue(buildTransferEventArchiveRow({ tags, transfer: input.transfer }));
315705
- } catch (archiveError) {
315706
- log.warn("Failed to archive transfer event", { error: archiveError });
315794
+ } catch (error) {
315795
+ this.closeLossJournal();
315796
+ throw error;
315707
315797
  }
315708
- });
315709
- }
315710
- function archiveWithdrawalObservationsInBackground(archiver, tracker, input) {
315711
- try {
315712
- if (!archiver?.isEnabled() || !Array.isArray(input.transactions)) {
315798
+ }
315799
+ static disabled() {
315800
+ return new BrokerExecutionArchiver({ enabled: false });
315801
+ }
315802
+ static create(options) {
315803
+ return new BrokerExecutionArchiver({ ...options, enabled: true });
315804
+ }
315805
+ getDeploymentId() {
315806
+ return this.deploymentId;
315807
+ }
315808
+ isEnabled() {
315809
+ return this.enabled && !this.closed;
315810
+ }
315811
+ canPersistMarketMetadataSnapshot() {
315812
+ return this.isEnabled();
315813
+ }
315814
+ canPersistAccountBalanceSnapshots() {
315815
+ return this.isEnabled() && Boolean(this.forwarderUrl);
315816
+ }
315817
+ enqueue(row) {
315818
+ if (!this.enabled || this.closed) {
315713
315819
  return;
315714
315820
  }
315715
- const brokerObservedTimestamp = new Date().toISOString();
315716
- for (const [resultIndex, transaction] of input.transactions.entries()) {
315717
- const normalized = normalizeCcxtTransactionForArchive(transaction);
315718
- const assetSymbol = normalized.assetSymbol;
315719
- if (!tracker.shouldArchive({
315720
- exchange: input.exchange,
315721
- accountSelector: input.accountSelector,
315722
- assetSymbol,
315723
- transaction,
315724
- normalized
315725
- })) {
315726
- continue;
315821
+ if (this.queue.length >= this.maxQueueSize) {
315822
+ const shedRow = this.queue[0];
315823
+ if (shedRow) {
315824
+ this.appendLossRecords([shedRow], "queue_shed");
315825
+ this.queue.shift();
315727
315826
  }
315728
- archiveTransferEventInBackground(archiver, {
315729
- exchange: input.exchange,
315730
- accountSelector: input.accountSelector,
315731
- assetSymbol,
315732
- brokerObservedTimestamp,
315733
- transfer: {
315734
- eventKind: "withdrawal",
315735
- lifecycleAction: "observe_withdrawal",
315736
- status: normalized.status,
315737
- amount: normalized.amount,
315738
- address: normalized.address,
315739
- network: normalized.network,
315740
- externalId: normalized.externalId,
315741
- txid: normalized.txid,
315742
- resultIndex,
315743
- feeAmount: normalized.feeAmount,
315744
- feeCurrency: normalized.feeCurrency,
315745
- exchangeTimestamp: normalized.exchangeTimestamp,
315746
- payload: transaction
315747
- }
315827
+ this.stats.shed += 1;
315828
+ this.recordArchiveMetric("cex_archive_rows_shed_total", {
315829
+ table: shedRow?.table ?? "unknown"
315748
315830
  });
315831
+ const now3 = Date.now();
315832
+ if (now3 - this.lastShedWarnAtMs >= SHED_WARN_INTERVAL_MS) {
315833
+ log.warn("Archive queue full: shedding oldest rows", {
315834
+ shed_total: this.stats.shed,
315835
+ queue_max: this.maxQueueSize,
315836
+ table: shedRow?.table ?? "unknown"
315837
+ });
315838
+ this.lastShedWarnAtMs = now3;
315839
+ }
315749
315840
  }
315750
- } catch (archiveError) {
315751
- log.warn("Failed to archive withdrawal observations", {
315752
- error: archiveError
315841
+ this.queue.push(row);
315842
+ this.stats.enqueued += 1;
315843
+ this.recordArchiveMetric("cex_archive_rows_enqueued_total", {
315844
+ table: row.table
315753
315845
  });
315846
+ if (this.queue.length >= this.batchSize) {
315847
+ this.flush();
315848
+ }
315754
315849
  }
315755
- }
315756
- async function captureMarketMetadataSnapshot(archiver, broker, input) {
315757
- if (!archiver?.canPersistMarketMetadataSnapshot()) {
315758
- return;
315850
+ enqueueInBackground(row) {
315851
+ queueMicrotask(() => this.enqueue(row));
315759
315852
  }
315760
- try {
315761
- const fetchOrderBook = broker.fetchOrderBook;
315762
- if (typeof fetchOrderBook !== "function") {
315763
- return;
315764
- }
315765
- const orderBook = await fetchOrderBook.call(broker, input.symbol, 5);
315766
- const record = asRecord(orderBook);
315767
- const snapshot = {
315768
- action: input.action,
315769
- symbol: input.symbol,
315770
- bids: record?.bids,
315771
- asks: record?.asks,
315772
- timestamp: record?.timestamp ?? Date.now(),
315773
- datetime: record?.datetime,
315774
- nonce: record?.nonce
315775
- };
315776
- const tags = buildCommonArchiveTags({
315777
- deploymentId: archiver.getDeploymentId(),
315778
- accountSelector: input.accountSelector,
315779
- exchange: input.exchange,
315780
- symbol: input.symbol,
315781
- brokerObservedTimestamp: input.brokerObservedTimestamp
315782
- });
315783
- const row = buildMarketMetadataSnapshotRow({
315784
- tags,
315785
- clientOrderId: input.clientOrderId,
315786
- orderId: input.orderId,
315787
- makerActionId: input.makerActionId,
315788
- idempotencyId: input.idempotencyId,
315789
- marketSnapshot: snapshot
315790
- });
315791
- archiver.enqueue(row);
315792
- const hash4 = row.row.market_metadata_hash;
315793
- return typeof hash4 === "string" ? hash4 : undefined;
315794
- } catch (archiveError) {
315795
- log.warn("Failed to capture market metadata snapshot", {
315796
- error: archiveError
315797
- });
315798
- return;
315799
- }
315800
- }
315801
- // src/helpers/broker-execution-archive/withdrawal-observation-tracker.ts
315802
- var DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES = 1e4;
315803
- var VENUE_LIFECYCLE_EVIDENCE_FIELDS = [
315804
- "updated",
315805
- "updatedAt",
315806
- "updated_at",
315807
- "updateTime",
315808
- "update_time",
315809
- "updateTimestamp",
315810
- "lastUpdateTimestamp",
315811
- "completed",
315812
- "completedAt",
315813
- "completed_at",
315814
- "completeTime",
315815
- "complete_time",
315816
- "completionTime",
315817
- "successTime"
315818
- ];
315819
- function fingerprintEvidenceValue(value) {
315820
- if (value === undefined || value === null) {
315821
- return value;
315822
- }
315823
- if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
315824
- return value;
315825
- }
315826
- if (typeof value === "bigint") {
315827
- return value.toString();
315828
- }
315829
- try {
315830
- return JSON.stringify(value);
315831
- } catch {
315832
- return String(value);
315833
- }
315834
- }
315835
- function venueLifecycleEvidence(transaction) {
315836
- const record = asRecord(transaction);
315837
- const info = asRecord(record?.info);
315838
- const evidence = [];
315839
- for (const [scope, source] of [
315840
- ["transaction", record],
315841
- ["info", info]
315842
- ]) {
315843
- for (const field of VENUE_LIFECYCLE_EVIDENCE_FIELDS) {
315844
- const value = source?.[field];
315845
- if (value !== undefined && value !== null) {
315846
- evidence.push([scope, field, fingerprintEvidenceValue(value)]);
315847
- }
315848
- }
315849
- }
315850
- return evidence;
315851
- }
315852
- function observationFingerprint(observation) {
315853
- const { normalized, transaction } = observation;
315854
- return JSON.stringify({
315855
- status: normalized.status ?? null,
315856
- txid: normalized.txid ?? null,
315857
- amount: normalized.amount ?? null,
315858
- feeAmount: normalized.feeAmount ?? null,
315859
- feeCurrency: normalized.feeCurrency ?? null,
315860
- address: normalized.address ?? null,
315861
- network: normalized.network ?? null,
315862
- exchangeTimestamp: normalized.exchangeTimestamp ?? null,
315863
- venueLifecycleEvidence: venueLifecycleEvidence(transaction)
315864
- });
315865
- }
315866
-
315867
- class WithdrawalObservationTracker {
315868
- #fingerprints = new Map;
315869
- #maxEntries;
315870
- #missingIdSequence = 0n;
315871
- constructor(options) {
315872
- const maxEntries = options?.maxEntries ?? DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES;
315873
- this.#maxEntries = Number.isFinite(maxEntries) ? Math.max(1, Math.floor(maxEntries)) : DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES;
315874
- }
315875
- shouldArchive(observation) {
315876
- const externalId = observation.normalized.externalId?.trim();
315877
- const identity = JSON.stringify([
315878
- observation.exchange.trim().toLowerCase() || "unknown",
315879
- observation.accountSelector?.trim() || "unknown",
315880
- observation.assetSymbol?.trim().toUpperCase() || "unknown",
315881
- externalId || `missing:${++this.#missingIdSequence}`
315882
- ]);
315883
- const fingerprint = observationFingerprint(observation);
315884
- if (this.#fingerprints.get(identity) === fingerprint) {
315885
- return false;
315886
- }
315887
- this.#fingerprints.delete(identity);
315888
- this.#fingerprints.set(identity, fingerprint);
315889
- while (this.#fingerprints.size > this.#maxEntries) {
315890
- const oldest = this.#fingerprints.keys().next().value;
315891
- if (oldest === undefined)
315892
- break;
315893
- this.#fingerprints.delete(oldest);
315894
- }
315895
- return true;
315896
- }
315897
- getSize() {
315898
- return this.#fingerprints.size;
315899
- }
315900
- }
315901
- // src/helpers/broker-execution-archive/writer.ts
315902
- var import_api_logs2 = __toESM(require_src7(), 1);
315903
- import { request as httpRequest2 } from "node:http";
315904
- import { request as httpsRequest2 } from "node:https";
315905
-
315906
- // src/helpers/market-data-archive/types.ts
315907
- var MARKET_ARCHIVE_TABLES = new Set([
315908
- "market_data.orderbook_snapshots",
315909
- "market_data.candles",
315910
- "market_data.cex_stream_events",
315911
- "market_data.cex_ticker_events",
315912
- "market_data.cex_trades"
315913
- ]);
315914
- function isMarketArchiveTable(table) {
315915
- return MARKET_ARCHIVE_TABLES.has(table);
315916
- }
315917
-
315918
- // src/helpers/broker-execution-archive/writer.ts
315919
- var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
315920
- "broker_execution.order_events",
315921
- "broker_execution.market_metadata_snapshots",
315922
- "broker_execution.transfer_events",
315923
- "broker_execution.fill_events"
315924
- ]);
315925
- function isBrokerExecutionArchiveTable(table) {
315926
- return BROKER_EXECUTION_ARCHIVE_TABLES.has(table);
315927
- }
315928
- var DEFAULT_MAX_QUEUE_SIZE = 1e4;
315929
- var DEFAULT_BATCH_SIZE = 10;
315930
- var DEFAULT_FLUSH_INTERVAL_MS = 1000;
315931
- var DEFAULT_FORWARDER_TIMEOUT_MS = 3000;
315932
- var SHED_WARN_INTERVAL_MS = 60000;
315933
- var DEFAULT_ARCHIVE_FORWARDER_PATH = "/archive";
315934
- var DEFAULT_ARCHIVE_FORWARDER_PORT = 8090;
315935
- function isArchiveOtelLogsEnabled() {
315936
- return process.env.CEX_BROKER_ARCHIVE_OTEL_LOGS_ENABLED === "true";
315937
- }
315938
- function resolveArchiveForwarderUrlFromEnv() {
315939
- const explicit = process.env.CEX_BROKER_ARCHIVE_FORWARDER_URL?.trim();
315940
- if (explicit) {
315941
- return explicit;
315942
- }
315943
- const host = process.env.CEX_BROKER_ARCHIVE_FORWARDER_HOST?.trim() || process.env.CEX_BROKER_CLICKHOUSE_HOST?.trim();
315944
- if (!host) {
315945
- return;
315946
- }
315947
- const protocol = process.env.CEX_BROKER_CLICKHOUSE_PROTOCOL || "http";
315948
- const port = parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_FORWARDER_PORT, DEFAULT_ARCHIVE_FORWARDER_PORT);
315949
- const path = process.env.CEX_BROKER_ARCHIVE_FORWARDER_PATH?.trim() || DEFAULT_ARCHIVE_FORWARDER_PATH;
315950
- const normalizedPath = path.startsWith("/") ? path : `/${path}`;
315951
- return `${protocol}://${host}:${port}${normalizedPath}`;
315952
- }
315953
-
315954
- class BrokerExecutionArchiver {
315955
- deploymentId;
315956
- otelLogs;
315957
- otelMetrics;
315958
- forwarderUrl;
315959
- maxQueueSize;
315960
- batchSize;
315961
- flushIntervalMs;
315962
- forwarderTimeoutMs;
315963
- queue = [];
315964
- stats = {
315965
- enqueued: 0,
315966
- shed: 0,
315967
- flushed: 0,
315968
- forwarderFailures: 0
315969
- };
315970
- flushTimer = null;
315971
- flushInFlight = null;
315972
- lastShedWarnAtMs = 0;
315973
- closed = false;
315974
- loggedMissingMarketForwarder = false;
315975
- enabled;
315976
- forwarderAuthToken;
315977
- constructor(options) {
315978
- this.deploymentId = options.deploymentId?.trim() || process.env.CEX_BROKER_DEPLOYMENT_ID?.trim() || "unknown";
315979
- this.otelLogs = options.otelLogs;
315980
- this.otelMetrics = options.otelMetrics;
315981
- this.forwarderUrl = options.forwarderUrl?.trim();
315982
- this.maxQueueSize = options.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE;
315983
- this.batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
315984
- this.flushIntervalMs = options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS;
315985
- this.forwarderTimeoutMs = options.forwarderTimeoutMs ?? DEFAULT_FORWARDER_TIMEOUT_MS;
315986
- this.enabled = options.enabled;
315987
- this.forwarderAuthToken = process.env.CEX_BROKER_ARCHIVE_FORWARDER_TOKEN?.trim() || undefined;
315988
- if (this.enabled) {
315989
- this.flushTimer = setInterval(() => {
315990
- this.flush().catch((error) => {
315991
- log.warn("Broker execution archive flush failed", { error });
315992
- });
315993
- }, this.flushIntervalMs);
315994
- this.flushTimer.unref?.();
315995
- }
315996
- }
315997
- static disabled() {
315998
- return new BrokerExecutionArchiver({ enabled: false });
315999
- }
316000
- static create(options) {
316001
- const hasSink = Boolean(options.otelLogs?.isOtelEnabled()) || Boolean(options.forwarderUrl);
316002
- const enabled = process.env.CEX_BROKER_ARCHIVE_ENABLED !== "false" && hasSink;
316003
- return new BrokerExecutionArchiver({ ...options, enabled });
316004
- }
316005
- getDeploymentId() {
316006
- return this.deploymentId;
316007
- }
316008
- isEnabled() {
316009
- return this.enabled && !this.closed && (Boolean(this.otelLogs) || Boolean(this.forwarderUrl));
316010
- }
316011
- canPersistMarketMetadataSnapshot() {
316012
- return this.isEnabled() && (Boolean(this.otelLogs?.isOtelEnabled()) || Boolean(this.forwarderUrl));
316013
- }
316014
- canPersistAccountBalanceSnapshots() {
316015
- return this.isEnabled() && Boolean(this.forwarderUrl);
316016
- }
316017
- enqueue(row) {
316018
- if (!this.enabled || this.closed || !this.otelLogs && !this.forwarderUrl) {
316019
- return;
316020
- }
316021
- if (isMarketArchiveTable(row.table) && !this.forwarderUrl) {
316022
- if (!this.loggedMissingMarketForwarder) {
316023
- this.loggedMissingMarketForwarder = true;
316024
- log.warn("Market data archive row dropped: configure CEX_BROKER_ARCHIVE_FORWARDER_URL or CEX_BROKER_CLICKHOUSE_HOST", { table: row.table });
316025
- }
316026
- return;
316027
- }
316028
- if (row.table === "broker_account.balance_snapshots" && !this.forwarderUrl) {
316029
- return;
316030
- }
316031
- if (this.queue.length >= this.maxQueueSize) {
316032
- this.queue.shift();
316033
- this.stats.shed += 1;
316034
- this.recordArchiveMetric("cex_archive_rows_shed_total", {
316035
- table: row.table
316036
- });
316037
- const now3 = Date.now();
316038
- if (now3 - this.lastShedWarnAtMs >= SHED_WARN_INTERVAL_MS) {
316039
- log.warn("Archive queue full: shedding oldest rows", {
316040
- shed_total: this.stats.shed,
316041
- queue_max: this.maxQueueSize,
316042
- table: row.table
316043
- });
316044
- this.lastShedWarnAtMs = now3;
316045
- }
316046
- }
316047
- this.queue.push(row);
316048
- this.stats.enqueued += 1;
316049
- this.recordArchiveMetric("cex_archive_rows_enqueued_total", {
316050
- table: row.table
316051
- });
316052
- if (this.queue.length >= this.batchSize) {
316053
- this.flush().catch((error) => {
316054
- log.warn("Broker execution archive flush failed", { error });
316055
- });
316056
- }
316057
- }
316058
- enqueueInBackground(row) {
316059
- queueMicrotask(() => this.enqueue(row));
316060
- }
316061
- async flush() {
316062
- if (!this.enabled || this.closed || this.queue.length === 0) {
315853
+ async flush() {
315854
+ if (!this.enabled || this.closed || this.queue.length === 0) {
316063
315855
  return;
316064
315856
  }
316065
315857
  if (this.flushInFlight) {
@@ -316078,21 +315870,36 @@ class BrokerExecutionArchiver {
316078
315870
  clearInterval(this.flushTimer);
316079
315871
  this.flushTimer = null;
316080
315872
  }
316081
- while (this.queue.length > 0 || this.flushInFlight) {
316082
- if (this.flushInFlight) {
316083
- await this.flushInFlight;
316084
- continue;
316085
- }
316086
- const depthBefore = this.queue.length;
316087
- const flushed = await this.flushBatch();
316088
- if (!flushed && this.queue.length >= depthBefore && depthBefore > 0) {
316089
- const dropped = this.queue.length;
316090
- this.queue.length = 0;
316091
- log.warn(`Archive shutdown dropped ${dropped} undelivered row(s) after forwarder failure`);
316092
- break;
315873
+ let closeError;
315874
+ try {
315875
+ while (this.queue.length > 0 || this.flushInFlight) {
315876
+ if (this.flushInFlight) {
315877
+ await this.flushInFlight;
315878
+ continue;
315879
+ }
315880
+ const depthBefore = this.queue.length;
315881
+ const flushed = await this.flushBatch();
315882
+ if (!flushed && this.queue.length >= depthBefore && depthBefore > 0) {
315883
+ const undelivered = [...this.queue];
315884
+ this.appendLossRecords(undelivered, "shutdown_forwarder_failure");
315885
+ this.queue.length = 0;
315886
+ break;
315887
+ }
316093
315888
  }
315889
+ } catch (error) {
315890
+ closeError = error;
316094
315891
  }
316095
315892
  this.closed = true;
315893
+ if (this.deadLetterFd !== undefined) {
315894
+ try {
315895
+ this.closeLossJournal();
315896
+ } catch (error) {
315897
+ closeError ??= error;
315898
+ }
315899
+ }
315900
+ if (closeError) {
315901
+ throw closeError;
315902
+ }
316096
315903
  }
316097
315904
  getStats() {
316098
315905
  return { ...this.stats };
@@ -316100,15 +315907,60 @@ class BrokerExecutionArchiver {
316100
315907
  getQueueDepth() {
316101
315908
  return this.queue.length;
316102
315909
  }
315910
+ closeLossJournal() {
315911
+ if (this.deadLetterFd === undefined) {
315912
+ return;
315913
+ }
315914
+ const fd2 = this.deadLetterFd;
315915
+ try {
315916
+ closeSync(fd2);
315917
+ } catch (error) {
315918
+ throw new BrokerExecutionArchiveDurabilityError("Broker execution archive failed to close the configured CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH loss journal", { cause: error });
315919
+ } finally {
315920
+ this.deadLetterFd = undefined;
315921
+ }
315922
+ }
316103
315923
  enforceQueueBound() {
316104
315924
  while (this.queue.length > this.maxQueueSize) {
316105
- const dropped = this.queue.shift();
315925
+ const dropped = this.queue[0];
315926
+ if (!dropped) {
315927
+ return;
315928
+ }
315929
+ this.appendLossRecords([dropped], "queue_shed");
315930
+ this.queue.shift();
316106
315931
  this.stats.shed += 1;
316107
315932
  this.recordArchiveMetric("cex_archive_rows_shed_total", {
316108
315933
  table: dropped?.table ?? "unknown"
316109
315934
  });
316110
315935
  }
316111
315936
  }
315937
+ appendLossRecords(rows, reason) {
315938
+ if (rows.length === 0) {
315939
+ return;
315940
+ }
315941
+ if (this.deadLetterFd === undefined) {
315942
+ throw new BrokerExecutionArchiveDurabilityError(`Broker execution archive cannot record ${reason}: dead-letter file is not open`);
315943
+ }
315944
+ const timestamp = new Date().toISOString();
315945
+ const records = rows.map((payload) => ({
315946
+ timestamp,
315947
+ deployment_id: this.deploymentId,
315948
+ reason,
315949
+ payload
315950
+ }));
315951
+ try {
315952
+ const bytes2 = Buffer.from(records.map((record) => JSON.stringify(record)).join(`
315953
+ `) + `
315954
+ `);
315955
+ const written = writeSync(this.deadLetterFd, bytes2);
315956
+ if (written !== bytes2.length) {
315957
+ throw new Error(`wrote ${written} of ${bytes2.length} bytes`);
315958
+ }
315959
+ fsyncSync(this.deadLetterFd);
315960
+ } catch (error) {
315961
+ throw new BrokerExecutionArchiveDurabilityError(`Broker execution archive failed to durably record ${reason}; affected row(s) were retained`, { cause: error });
315962
+ }
315963
+ }
316112
315964
  async flushBatch() {
316113
315965
  const batch = this.queue.splice(0, this.batchSize);
316114
315966
  if (batch.length === 0) {
@@ -316145,111 +315997,407 @@ class BrokerExecutionArchiver {
316145
315997
  for (const [table, count2] of countByTable) {
316146
315998
  this.recordArchiveMetric("cex_archive_rows_flushed_total", { table }, count2);
316147
315999
  }
316148
- this.recordArchiveGauge("cex_archive_last_flush_success", Math.floor(Date.now() / 1000));
316149
- }
316150
- async recordArchiveMetric(metricName, labels, value = 1) {
316151
- try {
316152
- await this.otelMetrics?.recordCounter(metricName, value, labels);
316153
- } catch {}
316000
+ this.recordArchiveGauge("cex_archive_last_flush_success", Math.floor(Date.now() / 1000));
316001
+ }
316002
+ async recordArchiveMetric(metricName, labels, value = 1) {
316003
+ try {
316004
+ await this.otelMetrics?.recordCounter(metricName, value, labels);
316005
+ } catch {}
316006
+ }
316007
+ async recordArchiveGauge(metricName, value) {
316008
+ try {
316009
+ await this.otelMetrics?.recordGauge(metricName, value, {});
316010
+ } catch {}
316011
+ }
316012
+ emitOtelLog(entry) {
316013
+ if (!this.otelLogs?.isOtelEnabled()) {
316014
+ return;
316015
+ }
316016
+ try {
316017
+ this.otelLogs.emit({
316018
+ body: entry.table,
316019
+ severityNumber: import_api_logs2.SeverityNumber.INFO,
316020
+ severityText: "INFO",
316021
+ attributes: flattenArchiveAttributes(redactArchiveErrorForOtel(entry))
316022
+ });
316023
+ } catch (error) {
316024
+ log.warn("Broker execution archive OTLP emit failed", { error });
316025
+ }
316026
+ }
316027
+ postToForwarder(batch) {
316028
+ if (!this.forwarderUrl || batch.length === 0) {
316029
+ return Promise.resolve();
316030
+ }
316031
+ const body = JSON.stringify({
316032
+ source: "broker_write",
316033
+ deployment_id: this.deploymentId,
316034
+ rows: batch
316035
+ });
316036
+ const url2 = new URL(this.forwarderUrl);
316037
+ const doRequest = url2.protocol === "http:" ? httpRequest2 : httpsRequest2;
316038
+ const headers = {
316039
+ "content-type": "application/json",
316040
+ "content-length": Buffer.byteLength(body)
316041
+ };
316042
+ if (this.forwarderAuthToken) {
316043
+ headers.authorization = `Bearer ${this.forwarderAuthToken}`;
316044
+ }
316045
+ return new Promise((resolve, reject) => {
316046
+ const req = doRequest(url2, {
316047
+ method: "POST",
316048
+ headers,
316049
+ timeout: this.forwarderTimeoutMs
316050
+ }, (res) => {
316051
+ res.on("data", () => {});
316052
+ res.on("end", () => {
316053
+ const status = res.statusCode ?? 0;
316054
+ if (status < 200 || status >= 300) {
316055
+ reject(new Error(`Archive forwarder returned ${status} ${res.statusMessage ?? ""}`));
316056
+ return;
316057
+ }
316058
+ resolve();
316059
+ });
316060
+ });
316061
+ req.on("error", reject);
316062
+ req.on("timeout", () => {
316063
+ req.destroy(new Error("Archive forwarder request timed out"));
316064
+ });
316065
+ req.write(body);
316066
+ req.end();
316067
+ });
316068
+ }
316069
+ }
316070
+ function redactArchiveErrorForOtel(entry) {
316071
+ if (entry.table !== "broker_execution.order_events" || typeof entry.row.error_message !== "string" || entry.row.error_message.length === 0) {
316072
+ return entry;
316073
+ }
316074
+ return {
316075
+ ...entry,
316076
+ row: { ...entry.row, error_message: REDACTED_ERROR_MESSAGE }
316077
+ };
316078
+ }
316079
+ function flattenArchiveAttributes(entry) {
316080
+ const attributes = {
316081
+ ch_table: entry.table
316082
+ };
316083
+ for (const [key, value] of Object.entries(entry.row)) {
316084
+ if (value === undefined || value === null) {
316085
+ continue;
316086
+ }
316087
+ if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
316088
+ attributes[key] = value;
316089
+ } else {
316090
+ attributes[key] = JSON.stringify(value);
316091
+ }
316092
+ }
316093
+ return attributes;
316094
+ }
316095
+ function createBrokerExecutionArchiverFromEnv(otelLogs, otelMetrics) {
316096
+ if (process.env.CEX_BROKER_ARCHIVE_ENABLED !== "true") {
316097
+ return BrokerExecutionArchiver.disabled();
316098
+ }
316099
+ const forwarderUrl = resolveArchiveForwarderUrlFromEnv();
316100
+ const archiveOtelLogs = isArchiveOtelLogsEnabled() ? otelLogs : undefined;
316101
+ return BrokerExecutionArchiver.create({
316102
+ otelLogs: archiveOtelLogs,
316103
+ otelMetrics,
316104
+ forwarderUrl: forwarderUrl ?? "",
316105
+ deadLetterPath: process.env.CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH?.trim() ?? "",
316106
+ deploymentId: process.env.CEX_BROKER_DEPLOYMENT_ID,
316107
+ maxQueueSize: parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_QUEUE_MAX, DEFAULT_MAX_QUEUE_SIZE),
316108
+ batchSize: parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_BATCH_SIZE, DEFAULT_BATCH_SIZE),
316109
+ flushIntervalMs: parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_FLUSH_INTERVAL_MS, DEFAULT_FLUSH_INTERVAL_MS)
316110
+ });
316111
+ }
316112
+ function validateForwarderUrl(value) {
316113
+ if (!value) {
316114
+ throw new Error("Broker execution archive is enabled but CEX_BROKER_ARCHIVE_FORWARDER_URL is missing");
316115
+ }
316116
+ let url2;
316117
+ try {
316118
+ url2 = new URL(value);
316119
+ } catch (error) {
316120
+ throw new Error("Broker execution archive requires a valid CEX_BROKER_ARCHIVE_FORWARDER_URL", { cause: error });
316121
+ }
316122
+ if (url2.protocol !== "http:" && url2.protocol !== "https:") {
316123
+ throw new Error("Broker execution archive forwarder URL must use http or https");
316124
+ }
316125
+ return url2;
316126
+ }
316127
+ function parsePositiveInt(value, fallback) {
316128
+ if (!value) {
316129
+ return fallback;
316130
+ }
316131
+ const parsed = Number.parseInt(value, 10);
316132
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
316133
+ }
316134
+
316135
+ // src/helpers/broker-execution-archive/capture.ts
316136
+ function archiveOrderExecutionInBackground(archiver, context2, order, error, options) {
316137
+ if (!archiver?.isEnabled()) {
316138
+ return;
316139
+ }
316140
+ queueMicrotask(() => {
316141
+ try {
316142
+ const telemetry = buildOrderExecutionTelemetry(context2, order, error);
316143
+ const tags = buildCommonArchiveTags({
316144
+ deploymentId: archiver.getDeploymentId(),
316145
+ accountSelector: context2.accountLabel,
316146
+ exchange: context2.cex,
316147
+ symbol: telemetry.symbol,
316148
+ brokerObservedTimestamp: telemetry.brokerObservedTimestamp
316149
+ });
316150
+ archiver.enqueue(buildOrderEventArchiveRow({
316151
+ tags,
316152
+ action: context2.action,
316153
+ telemetry,
316154
+ errorDetail: context2.action === "CreateOrder" && error !== undefined ? sanitizeErrorDetail(error, { includeCode: true }) : undefined,
316155
+ marketMetadataHash: options?.marketMetadataHash
316156
+ }));
316157
+ } catch (archiveError) {
316158
+ rethrowArchiveDurabilityError(archiveError);
316159
+ log.warn("Failed to archive order execution", { error: archiveError });
316160
+ }
316161
+ });
316162
+ }
316163
+ function archiveSubscribeStreamInBackground(archiver, input) {
316164
+ if (!archiver?.isEnabled()) {
316165
+ return;
316166
+ }
316167
+ queueMicrotask(() => {
316168
+ try {
316169
+ const tags = buildCommonArchiveTags({
316170
+ deploymentId: archiver.getDeploymentId(),
316171
+ accountSelector: input.accountSelector,
316172
+ exchange: input.exchange,
316173
+ symbol: input.symbol
316174
+ });
316175
+ archiver.enqueue(buildSubscribeStreamArchiveRow({
316176
+ tags,
316177
+ subscriptionType: input.subscriptionType,
316178
+ streamPayload: input.streamPayload,
316179
+ secretLiterals: input.secretLiterals
316180
+ }));
316181
+ } catch (archiveError) {
316182
+ rethrowArchiveDurabilityError(archiveError);
316183
+ log.warn("Failed to archive subscribe stream event", {
316184
+ error: archiveError
316185
+ });
316186
+ }
316187
+ });
316188
+ }
316189
+ function archiveTransferEventInBackground(archiver, input) {
316190
+ if (!archiver?.isEnabled()) {
316191
+ return;
316192
+ }
316193
+ queueMicrotask(() => {
316194
+ try {
316195
+ const tags = buildCommonArchiveTags({
316196
+ deploymentId: archiver.getDeploymentId(),
316197
+ accountSelector: input.accountSelector,
316198
+ exchange: input.exchange,
316199
+ symbol: input.assetSymbol,
316200
+ brokerObservedTimestamp: input.brokerObservedTimestamp
316201
+ });
316202
+ archiver.enqueue(buildTransferEventArchiveRow({ tags, transfer: input.transfer }));
316203
+ } catch (archiveError) {
316204
+ rethrowArchiveDurabilityError(archiveError);
316205
+ log.warn("Failed to archive transfer event", { error: archiveError });
316206
+ }
316207
+ });
316208
+ }
316209
+ function archiveWithdrawalObservationsInBackground(archiver, tracker, input) {
316210
+ try {
316211
+ if (!archiver?.isEnabled() || !Array.isArray(input.transactions)) {
316212
+ return;
316213
+ }
316214
+ const brokerObservedTimestamp = new Date().toISOString();
316215
+ for (const [resultIndex, transaction] of input.transactions.entries()) {
316216
+ const normalized = normalizeCcxtTransactionForArchive(transaction);
316217
+ const assetSymbol = normalized.assetSymbol;
316218
+ if (!tracker.shouldArchive({
316219
+ exchange: input.exchange,
316220
+ accountSelector: input.accountSelector,
316221
+ assetSymbol,
316222
+ transaction,
316223
+ normalized
316224
+ })) {
316225
+ continue;
316226
+ }
316227
+ archiveTransferEventInBackground(archiver, {
316228
+ exchange: input.exchange,
316229
+ accountSelector: input.accountSelector,
316230
+ assetSymbol,
316231
+ brokerObservedTimestamp,
316232
+ transfer: {
316233
+ eventKind: "withdrawal",
316234
+ lifecycleAction: "observe_withdrawal",
316235
+ status: normalized.status,
316236
+ amount: normalized.amount,
316237
+ address: normalized.address,
316238
+ network: normalized.network,
316239
+ externalId: normalized.externalId,
316240
+ txid: normalized.txid,
316241
+ resultIndex,
316242
+ feeAmount: normalized.feeAmount,
316243
+ feeCurrency: normalized.feeCurrency,
316244
+ exchangeTimestamp: normalized.exchangeTimestamp,
316245
+ payload: transaction
316246
+ }
316247
+ });
316248
+ }
316249
+ } catch (archiveError) {
316250
+ rethrowArchiveDurabilityError(archiveError);
316251
+ log.warn("Failed to archive withdrawal observations", {
316252
+ error: archiveError
316253
+ });
316154
316254
  }
316155
- async recordArchiveGauge(metricName, value) {
316156
- try {
316157
- await this.otelMetrics?.recordGauge(metricName, value, {});
316158
- } catch {}
316255
+ }
316256
+ async function captureMarketMetadataSnapshot(archiver, broker, input) {
316257
+ if (!archiver?.canPersistMarketMetadataSnapshot()) {
316258
+ return;
316159
316259
  }
316160
- emitOtelLog(entry) {
316161
- if (!this.otelLogs?.isOtelEnabled()) {
316260
+ try {
316261
+ const fetchOrderBook = broker.fetchOrderBook;
316262
+ if (typeof fetchOrderBook !== "function") {
316162
316263
  return;
316163
316264
  }
316164
- try {
316165
- this.otelLogs.emit({
316166
- body: entry.table,
316167
- severityNumber: import_api_logs2.SeverityNumber.INFO,
316168
- severityText: "INFO",
316169
- attributes: flattenArchiveAttributes(entry)
316170
- });
316171
- } catch (error) {
316172
- log.warn("Broker execution archive OTLP emit failed", { error });
316173
- }
316174
- }
316175
- postToForwarder(batch) {
316176
- if (!this.forwarderUrl || batch.length === 0) {
316177
- return Promise.resolve();
316178
- }
316179
- const body = JSON.stringify({
316180
- source: "broker_write",
316181
- deployment_id: this.deploymentId,
316182
- rows: batch
316183
- });
316184
- const url2 = new URL(this.forwarderUrl);
316185
- const doRequest = url2.protocol === "http:" ? httpRequest2 : httpsRequest2;
316186
- const headers = {
316187
- "content-type": "application/json",
316188
- "content-length": Buffer.byteLength(body)
316265
+ const orderBook = await fetchOrderBook.call(broker, input.symbol, 5);
316266
+ const record = asRecord(orderBook);
316267
+ const snapshot = {
316268
+ action: input.action,
316269
+ symbol: input.symbol,
316270
+ bids: record?.bids,
316271
+ asks: record?.asks,
316272
+ timestamp: record?.timestamp ?? Date.now(),
316273
+ datetime: record?.datetime,
316274
+ nonce: record?.nonce
316189
316275
  };
316190
- if (this.forwarderAuthToken) {
316191
- headers.authorization = `Bearer ${this.forwarderAuthToken}`;
316192
- }
316193
- return new Promise((resolve, reject) => {
316194
- const req = doRequest(url2, {
316195
- method: "POST",
316196
- headers,
316197
- timeout: this.forwarderTimeoutMs
316198
- }, (res) => {
316199
- res.on("data", () => {});
316200
- res.on("end", () => {
316201
- const status = res.statusCode ?? 0;
316202
- if (status < 200 || status >= 300) {
316203
- reject(new Error(`Archive forwarder returned ${status} ${res.statusMessage ?? ""}`));
316204
- return;
316205
- }
316206
- resolve();
316207
- });
316208
- });
316209
- req.on("error", reject);
316210
- req.on("timeout", () => {
316211
- req.destroy(new Error("Archive forwarder request timed out"));
316212
- });
316213
- req.write(body);
316214
- req.end();
316276
+ const tags = buildCommonArchiveTags({
316277
+ deploymentId: archiver.getDeploymentId(),
316278
+ accountSelector: input.accountSelector,
316279
+ exchange: input.exchange,
316280
+ symbol: input.symbol,
316281
+ brokerObservedTimestamp: input.brokerObservedTimestamp
316282
+ });
316283
+ const row = buildMarketMetadataSnapshotRow({
316284
+ tags,
316285
+ clientOrderId: input.clientOrderId,
316286
+ orderId: input.orderId,
316287
+ makerActionId: input.makerActionId,
316288
+ idempotencyId: input.idempotencyId,
316289
+ marketSnapshot: snapshot
316290
+ });
316291
+ archiver.enqueue(row);
316292
+ const hash4 = row.row.market_metadata_hash;
316293
+ return typeof hash4 === "string" ? hash4 : undefined;
316294
+ } catch (archiveError) {
316295
+ rethrowArchiveDurabilityError(archiveError);
316296
+ log.warn("Failed to capture market metadata snapshot", {
316297
+ error: archiveError
316215
316298
  });
316299
+ return;
316216
316300
  }
316217
316301
  }
316218
- function flattenArchiveAttributes(entry) {
316219
- const attributes = {
316220
- ch_table: entry.table
316221
- };
316222
- for (const [key, value] of Object.entries(entry.row)) {
316223
- if (value === undefined || value === null) {
316224
- continue;
316225
- }
316226
- if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
316227
- attributes[key] = value;
316228
- } else {
316229
- attributes[key] = JSON.stringify(value);
316302
+ // src/helpers/broker-execution-archive/withdrawal-observation-tracker.ts
316303
+ var DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES = 1e4;
316304
+ var VENUE_LIFECYCLE_EVIDENCE_FIELDS = [
316305
+ "updated",
316306
+ "updatedAt",
316307
+ "updated_at",
316308
+ "updateTime",
316309
+ "update_time",
316310
+ "updateTimestamp",
316311
+ "lastUpdateTimestamp",
316312
+ "completed",
316313
+ "completedAt",
316314
+ "completed_at",
316315
+ "completeTime",
316316
+ "complete_time",
316317
+ "completionTime",
316318
+ "successTime"
316319
+ ];
316320
+ function fingerprintEvidenceValue(value) {
316321
+ if (value === undefined || value === null) {
316322
+ return value;
316323
+ }
316324
+ if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
316325
+ return value;
316326
+ }
316327
+ if (typeof value === "bigint") {
316328
+ return value.toString();
316329
+ }
316330
+ try {
316331
+ return JSON.stringify(value);
316332
+ } catch {
316333
+ return String(value);
316334
+ }
316335
+ }
316336
+ function venueLifecycleEvidence(transaction) {
316337
+ const record = asRecord(transaction);
316338
+ const info = asRecord(record?.info);
316339
+ const evidence = [];
316340
+ for (const [scope, source] of [
316341
+ ["transaction", record],
316342
+ ["info", info]
316343
+ ]) {
316344
+ for (const field of VENUE_LIFECYCLE_EVIDENCE_FIELDS) {
316345
+ const value = source?.[field];
316346
+ if (value !== undefined && value !== null) {
316347
+ evidence.push([scope, field, fingerprintEvidenceValue(value)]);
316348
+ }
316230
316349
  }
316231
316350
  }
316232
- return attributes;
316351
+ return evidence;
316233
316352
  }
316234
- function createBrokerExecutionArchiverFromEnv(otelLogs, otelMetrics) {
316235
- const forwarderUrl = resolveArchiveForwarderUrlFromEnv();
316236
- const archiveOtelLogs = isArchiveOtelLogsEnabled() ? otelLogs : undefined;
316237
- return BrokerExecutionArchiver.create({
316238
- otelLogs: archiveOtelLogs,
316239
- otelMetrics,
316240
- forwarderUrl,
316241
- deploymentId: process.env.CEX_BROKER_DEPLOYMENT_ID,
316242
- maxQueueSize: parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_QUEUE_MAX, DEFAULT_MAX_QUEUE_SIZE),
316243
- batchSize: parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_BATCH_SIZE, DEFAULT_BATCH_SIZE),
316244
- flushIntervalMs: parsePositiveInt(process.env.CEX_BROKER_ARCHIVE_FLUSH_INTERVAL_MS, DEFAULT_FLUSH_INTERVAL_MS)
316353
+ function observationFingerprint(observation) {
316354
+ const { normalized, transaction } = observation;
316355
+ return JSON.stringify({
316356
+ status: normalized.status ?? null,
316357
+ txid: normalized.txid ?? null,
316358
+ amount: normalized.amount ?? null,
316359
+ feeAmount: normalized.feeAmount ?? null,
316360
+ feeCurrency: normalized.feeCurrency ?? null,
316361
+ address: normalized.address ?? null,
316362
+ network: normalized.network ?? null,
316363
+ exchangeTimestamp: normalized.exchangeTimestamp ?? null,
316364
+ venueLifecycleEvidence: venueLifecycleEvidence(transaction)
316245
316365
  });
316246
316366
  }
316247
- function parsePositiveInt(value, fallback) {
316248
- if (!value) {
316249
- return fallback;
316367
+
316368
+ class WithdrawalObservationTracker {
316369
+ #fingerprints = new Map;
316370
+ #maxEntries;
316371
+ #missingIdSequence = 0n;
316372
+ constructor(options) {
316373
+ const maxEntries = options?.maxEntries ?? DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES;
316374
+ this.#maxEntries = Number.isFinite(maxEntries) ? Math.max(1, Math.floor(maxEntries)) : DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES;
316375
+ }
316376
+ shouldArchive(observation) {
316377
+ const externalId = observation.normalized.externalId?.trim();
316378
+ const identity = JSON.stringify([
316379
+ observation.exchange.trim().toLowerCase() || "unknown",
316380
+ observation.accountSelector?.trim() || "unknown",
316381
+ observation.assetSymbol?.trim().toUpperCase() || "unknown",
316382
+ externalId || `missing:${++this.#missingIdSequence}`
316383
+ ]);
316384
+ const fingerprint = observationFingerprint(observation);
316385
+ if (this.#fingerprints.get(identity) === fingerprint) {
316386
+ return false;
316387
+ }
316388
+ this.#fingerprints.delete(identity);
316389
+ this.#fingerprints.set(identity, fingerprint);
316390
+ while (this.#fingerprints.size > this.#maxEntries) {
316391
+ const oldest = this.#fingerprints.keys().next().value;
316392
+ if (oldest === undefined)
316393
+ break;
316394
+ this.#fingerprints.delete(oldest);
316395
+ }
316396
+ return true;
316397
+ }
316398
+ getSize() {
316399
+ return this.#fingerprints.size;
316250
316400
  }
316251
- const parsed = Number.parseInt(value, 10);
316252
- return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
316253
316401
  }
316254
316402
  // src/helpers/account-balance-archive-poller.ts
316255
316403
  var DEFAULT_CONFIG = {
@@ -316343,6 +316491,7 @@ class AccountBalanceArchivePoller {
316343
316491
  this.params.metrics?.recordGauge("cex_account_balance_poll_last_success_timestamp_seconds", Math.floor(successMs / 1000), labels);
316344
316492
  this.#recordFreshness(labels, successMs, successMs);
316345
316493
  } catch (error) {
316494
+ rethrowArchiveDurabilityError(error);
316346
316495
  this.params.metrics?.recordCounter("cex_account_balance_poll_failures_total", 1, labels);
316347
316496
  const now3 = Date.now();
316348
316497
  const lastSuccess = this.#lastSuccessMs.get(this.#targetKey(target));
@@ -316372,6 +316521,7 @@ class AccountBalanceArchivePoller {
316372
316521
  try {
316373
316522
  await this.pollAllOnce();
316374
316523
  } catch (error) {
316524
+ rethrowArchiveDurabilityError(error);
316375
316525
  log.error("Account balance archive poller tick failed", error);
316376
316526
  } finally {
316377
316527
  if (!this.#stopped) {
@@ -316435,6 +316585,7 @@ class FillArchivePoller {
316435
316585
  try {
316436
316586
  await this.pollTrackedOnce();
316437
316587
  } catch (error) {
316588
+ rethrowArchiveDurabilityError(error);
316438
316589
  log.error("Fill archive poller tick failed", error);
316439
316590
  } finally {
316440
316591
  this.#running = false;
@@ -316854,32 +317005,6 @@ var grpc14 = __toESM(require_src3(), 1);
316854
317005
  // src/handlers/execute-action/deposit.ts
316855
317006
  var grpc3 = __toESM(require_src3(), 1);
316856
317007
 
316857
- // src/helpers/shared/errors.ts
316858
- var MAX_ERROR_DETAIL_LENGTH = 512;
316859
- function getErrorMessage(error) {
316860
- return error instanceof Error ? error.message : typeof error === "string" ? error : "Unknown error";
316861
- }
316862
- function errorClassName(error) {
316863
- if (!(error instanceof Error)) {
316864
- return;
316865
- }
316866
- const name = error.constructor?.name || error.name;
316867
- return name && name !== "Error" ? name : undefined;
316868
- }
316869
- function sanitizeErrorDetail(error) {
316870
- const className = errorClassName(error);
316871
- const message = getErrorMessage(error);
316872
- const detail = className ? `${className}: ${message}` : message;
316873
- return detail.replace(/\s+/g, " ").trim().slice(0, MAX_ERROR_DETAIL_LENGTH);
316874
- }
316875
- function safeLogError(context2, error) {
316876
- try {
316877
- log.error(context2, { error });
316878
- } catch {
316879
- console.error(context2, error);
316880
- }
316881
- }
316882
-
316883
317008
  // src/helpers/treasury-discovery.ts
316884
317009
  function callArgs(args, params) {
316885
317010
  const argsArray = Array.isArray(args) ? [...args] : [];
@@ -331406,7 +331531,8 @@ async function handleCreateOrder(ctx) {
331406
331531
  archiveOrderExecutionInBackground(brokerArchiver, createOrderContext, order, undefined, { marketMetadataHash });
331407
331532
  ctx.wrappedCallback(null, { result: JSON.stringify({ ...order }) });
331408
331533
  } catch (error48) {
331409
- safeLogError("Order Creation failed", error48);
331534
+ rethrowArchiveDurabilityError(error48);
331535
+ safeLogRedactedError("Order Creation failed", error48);
331410
331536
  const failedCreateContext = {
331411
331537
  action: "CreateOrder",
331412
331538
  cex: cex3,
@@ -332680,8 +332806,8 @@ class BinanceSpotUserDataStream {
332680
332806
  }
332681
332807
  if ("status" in message && typeof message.status === "number" && message.status !== 200) {
332682
332808
  const errorMessage = message.error?.msg ?? message.error?.message ?? `Binance user-data request failed with status ${message.status}`;
332683
- const errorCode = message.error?.code;
332684
- this.fail(new Error(typeof errorCode === "number" ? `${errorMessage} (code ${errorCode})` : errorMessage));
332809
+ const errorCode2 = message.error?.code;
332810
+ this.fail(new Error(typeof errorCode2 === "number" ? `${errorMessage} (code ${errorCode2})` : errorMessage));
332685
332811
  return;
332686
332812
  }
332687
332813
  if (!("event" in message) || !message.event) {
@@ -333279,6 +333405,7 @@ function archiveOrderbookInBackground(archiver, otelMetrics, input, options) {
333279
333405
  recordWatchMetric(otelMetrics, "cex_watch_frames_archived_total", labels);
333280
333406
  }
333281
333407
  } catch (error48) {
333408
+ rethrowArchiveDurabilityError(error48);
333282
333409
  log.warn("Failed to archive orderbook snapshot", { error: error48 });
333283
333410
  }
333284
333411
  });
@@ -333306,6 +333433,7 @@ function archiveOhlcvInBackground(archiver, otelMetrics, tracker, input) {
333306
333433
  recordWatchMetric(otelMetrics, "cex_watch_frames_archived_total", labels);
333307
333434
  }
333308
333435
  } catch (error48) {
333436
+ rethrowArchiveDurabilityError(error48);
333309
333437
  log.warn("Failed to archive OHLCV candle", { error: error48 });
333310
333438
  }
333311
333439
  });
@@ -333332,6 +333460,7 @@ function archiveMarketRowsInBackground(archiver, otelMetrics, stream4, input, en
333332
333460
  recordWatchMetric(otelMetrics, "cex_watch_frames_archived_total", labels);
333333
333461
  }
333334
333462
  } catch (error48) {
333463
+ rethrowArchiveDurabilityError(error48);
333335
333464
  log.warn(`Failed to archive ${stream4} market data`, { error: error48 });
333336
333465
  }
333337
333466
  });
@@ -333367,7 +333496,6 @@ function resolveOhlcvBootstrapLimit(optionValue) {
333367
333496
  }
333368
333497
  return Math.min(parsed, MAX_OHLCV_BOOTSTRAP_LIMIT);
333369
333498
  }
333370
-
333371
333499
  // src/helpers/market-data-archive/ohlcv-history.ts
333372
333500
  function supportsFetchOhlcv(broker) {
333373
333501
  const fetchOHLCV = broker.fetchOHLCV;