@usherlabs/cex-broker 0.2.36 → 0.2.38

package/dist/helpers/binance-user-data-normalization.d.ts

@@ -0,0 +1,3 @@
+import type { Exchange } from "@usherlabs/ccxt";
+export declare function normalizeBinanceSpotBalanceEvent(exchange: Exchange, event: Record<string, unknown>): Promise<unknown>;
+export declare function normalizeBinanceExecutionReport(exchange: Exchange, event: Record<string, unknown>): Record<string, unknown>;

package/dist/helpers/broker-execution-archive/index.d.ts

@@ -1,6 +1,6 @@
 export { archiveOrderExecutionInBackground, archiveSubscribeStreamInBackground, archiveTransferEventInBackground, archiveWithdrawalObservationsInBackground, captureMarketMetadataSnapshot, captureMarketMetadataSnapshotInBackground, } from "./capture";
 export { hashMarketMetadata, redactErrorForArchive, redactSecretLiterals, redactStreamPayload, } from "./redact";
-export { buildAccountBalanceSnapshotRow, buildCommonArchiveTags, buildFillEventArchiveRow, buildMarketMetadataSnapshotRow, buildOrderEventArchiveRow, buildSubscribeStreamArchiveRow, buildTransferEventArchiveRow, type FillArchiveFields, type NormalizedCcxtBalance, type NormalizedCcxtTransfer, normalizeCcxtBalanceForArchive, normalizeCcxtTradeForArchive, normalizeCcxtTransactionForArchive, type TransferArchiveFields, } from "./rows";
-export { ACCOUNT_BALANCE_PRECISION_BASIS, ACCOUNT_BALANCE_SCOPE, ARCHIVE_SCHEMA_VERSION, BROKER_WRITE_SOURCE, type BrokerArchiveCommonTags, type BrokerArchiveRow, type BrokerArchiveTable, type OrderArchiveAction, type SubscribeArchiveType, type TransferEventKind, type TransferLifecycleAction, } from "./types";
+export { buildAccountBalanceSnapshotRow, buildCommonArchiveTags, buildFillEventArchiveRow, buildMarketMetadataSnapshotRow, buildOrderEventArchiveRow, buildSubscribeStreamArchiveRow, buildTransferEventArchiveRow, extractBinanceInternalTransferId, type FillArchiveFields, type NormalizedCcxtBalance, type NormalizedCcxtTransfer, normalizeCcxtBalanceForArchive, normalizeCcxtTradeForArchive, normalizeCcxtTransactionForArchive, type TransferArchiveFields, } from "./rows";
+export { ACCOUNT_BALANCE_PRECISION_BASIS, ACCOUNT_BALANCE_SCOPE, ARCHIVE_SCHEMA_VERSION, BROKER_READ_SOURCE, BROKER_WRITE_SOURCE, type BrokerArchiveCommonTags, type BrokerArchiveRow, type BrokerArchiveSource, type BrokerArchiveTable, type OrderArchiveAction, type SubscribeArchiveType, type TransferEventKind, type TransferLifecycleAction, } from "./types";
 export { DEFAULT_WITHDRAWAL_OBSERVATION_TRACKER_MAX_ENTRIES, WithdrawalObservationTracker, } from "./withdrawal-observation-tracker";
 export { BrokerExecutionArchiveDurabilityError, BrokerExecutionArchiver, type BrokerExecutionArchiverOptions, createBrokerExecutionArchiverFromEnv, isArchiveOtelLogsEnabled, isBrokerExecutionArchiveTable, resolveArchiveForwarderUrlFromEnv, rethrowArchiveDurabilityError, } from "./writer";

package/dist/helpers/broker-execution-archive/rows.d.ts

@@ -1,5 +1,5 @@
 import type { OrderExecutionTelemetry } from "../order-telemetry";
-import { type BrokerArchiveCommonTags, type BrokerArchiveRow, type OrderArchiveAction, type SubscribeArchiveType, type TransferEventKind, type TransferLifecycleAction } from "./types";
+import { type BrokerArchiveCommonTags, type BrokerArchiveRow, type BrokerArchiveSource, type OrderArchiveAction, type SubscribeArchiveType, type TransferEventKind, type TransferLifecycleAction } from "./types";
 type BalanceQuantityMap = Record<string, string>;
 export type NormalizedCcxtBalance = {
     exchangeTimestamp?: string;
@@ -18,6 +18,7 @@ export declare function buildAccountBalanceSnapshotRow(input: {
     balance: NormalizedCcxtBalance;
 }): BrokerArchiveRow;
 export declare function buildCommonArchiveTags(input: {
+    source?: BrokerArchiveSource;
     deploymentId: string;
     accountSelector?: string;
     exchange: string;
@@ -39,6 +40,7 @@ export declare function buildSubscribeStreamArchiveRow(input: {
     streamPayload: unknown;
     secretLiterals?: readonly string[];
 }): BrokerArchiveRow;
+export declare function extractBinanceInternalTransferId(response: unknown): string | undefined;
 export type TransferArchiveFields = {
     eventKind: TransferEventKind;
     lifecycleAction: TransferLifecycleAction;
@@ -62,6 +64,7 @@ export declare function buildTransferEventArchiveRow(input: {
 }): BrokerArchiveRow;
 export type NormalizedCcxtTransfer = {
     externalId?: string;
+    clientWithdrawalId?: string;
     txid?: string;
     address?: string;
     network?: string;

package/dist/helpers/broker-execution-archive/types.d.ts

@@ -1,12 +1,14 @@
+export declare const BROKER_READ_SOURCE: "broker_read";
 export declare const BROKER_WRITE_SOURCE: "broker_write";
+export type BrokerArchiveSource = typeof BROKER_READ_SOURCE | typeof BROKER_WRITE_SOURCE;
 export declare const ARCHIVE_SCHEMA_VERSION: "1";
-export type BrokerArchiveTable = "broker_execution.order_events" | "broker_execution.market_metadata_snapshots" | "broker_execution.transfer_events" | "broker_execution.fill_events" | "broker_account.balance_snapshots" | "market_data.orderbook_snapshots" | "market_data.candles" | "market_data.cex_stream_events" | "market_data.cex_ticker_events" | "market_data.cex_trades";
+export type BrokerArchiveTable = "broker_execution.order_events" | "broker_execution.market_metadata_snapshots" | "broker_execution.transfer_events" | "broker_execution.fill_events" | "broker_account.balance_snapshots" | "market_data.orderbook_snapshots" | "market_data.candles" | "market_data.cex_stream_events" | "market_data.cex_ticker_events" | "market_data.cex_trades" | "market_data.cex_ohlcv" | "market_data.cex_order_book_levels" | "market_data.cex_order_book_depth_summary";
 export type BrokerArchiveRow = {
     table: BrokerArchiveTable;
     row: Record<string, unknown>;
 };
 export type BrokerArchiveCommonTags = {
-    source: typeof BROKER_WRITE_SOURCE;
+    source: BrokerArchiveSource;
     deployment_id: string;
     account_selector: string;
     exchange: string;

package/dist/helpers/broker-execution-archive/writer.d.ts

@@ -1,7 +1,8 @@
 import type { OtelLogs, OtelMetrics } from "../otel";
-import type { BrokerArchiveRow, BrokerArchiveTable } from "./types";
+import { type BrokerArchiveRow, type BrokerArchiveSource, type BrokerArchiveTable } from "./types";
 export declare function isBrokerExecutionArchiveTable(table: BrokerArchiveTable): boolean;
 export type BrokerExecutionArchiverOptions = {
+    source?: BrokerArchiveSource;
     deploymentId?: string;
     otelLogs?: OtelLogs;
     otelMetrics?: OtelMetrics;
@@ -24,7 +25,9 @@ type ArchiverStats = {
 };
 export declare function isArchiveOtelLogsEnabled(): boolean;
 export declare function resolveArchiveForwarderUrlFromEnv(): string | undefined;
+export declare function resolveArchiveSourceFromEnv(value?: string | undefined): BrokerArchiveSource;
 export declare class BrokerExecutionArchiver {
+    private readonly source;
     private readonly deploymentId;
     private readonly otelLogs?;
     private readonly otelMetrics?;
@@ -40,6 +43,7 @@ export declare class BrokerExecutionArchiver {
     private flushTimer;
     private flushInFlight;
     private lastShedWarnAtMs;
+    private closing;
     private closed;
     private readonly enabled;
     private readonly forwarderAuthToken?;
@@ -47,6 +51,7 @@ export declare class BrokerExecutionArchiver {
     static disabled(): BrokerExecutionArchiver;
     static create(options: BrokerExecutionArchiverOptions): BrokerExecutionArchiver;
     getDeploymentId(): string;
+    getSource(): BrokerArchiveSource;
     isEnabled(): boolean;
     canPersistMarketMetadataSnapshot(): boolean;
     canPersistAccountBalanceSnapshots(): boolean;

package/dist/helpers/market-data-archive/canonical-orderbook.d.ts

@@ -0,0 +1,20 @@
+import type { BrokerArchiveRow } from "../broker-execution-archive/types";
+import type { NormalizedOrderBookSnapshot, OrderBookConstructionMode } from "../order-book";
+import type { MarketCaptureContext, RawCapture } from "./types";
+export declare class OrderBookValidationError extends Error {
+    readonly reason: string;
+    constructor(reason: string);
+}
+export type CanonicalOrderBookRows = {
+    snapshotId: string;
+    levels: BrokerArchiveRow[];
+    summary: BrokerArchiveRow;
+};
+export declare function buildCanonicalOrderBookRows(input: {
+    context: MarketCaptureContext;
+    snapshot: NormalizedOrderBookSnapshot;
+    rawCapture: RawCapture;
+    depthLimit: number;
+    measurementBandsBps?: readonly number[];
+    constructionMode?: OrderBookConstructionMode;
+}): CanonicalOrderBookRows;

package/dist/helpers/market-data-archive/capture-context.d.ts

@@ -0,0 +1,32 @@
+import type { BrokerArchiveSource } from "../broker-execution-archive/types";
+import type { BrokerMarketType } from "../market-type";
+import type { CaptureFeed, CaptureSourceMode, MarketCaptureContext } from "./types";
+export type CaptureEnvironment = "development" | "production";
+export declare function createMarketCaptureContext(input: {
+    source: BrokerArchiveSource;
+    deploymentId: string;
+    captureBundleId?: string;
+    exchange: string;
+    symbol: string;
+    assetType: BrokerMarketType;
+    feed: CaptureFeed;
+    provider?: string;
+    sourceMode: CaptureSourceMode;
+    timeframe?: string;
+    accountSelector?: string;
+    environment?: CaptureEnvironment;
+}): MarketCaptureContext;
+export declare function validateProductionCollectorArchive(input: {
+    source: BrokerArchiveSource;
+    captureBundleId?: string;
+}): void;
+export declare function validateExternalFallbackContext(input: {
+    configuredExchange: string;
+    configuredSymbol: string;
+    rowExchange: string;
+    rowSymbol: string;
+    provider: string;
+    sourceMode: "external_ccxt_fallback_v1" | "external_hummingbot_fallback_v1";
+    fallbackReason: string;
+}): void;
+export declare function captureEnvironmentFromEnv(value?: string | undefined): CaptureEnvironment;

package/dist/helpers/market-data-archive/capture-contract.d.ts

@@ -0,0 +1,29 @@
+import type { MarketCaptureContext, RawCapture, RawCaptureScope } from "./types";
+export declare const MARKET_CAPTURE_SCHEMA_VERSION: "1.0.0";
+export declare const CHECKSUM_ALGORITHM: "sha256-canonical-json-v1";
+export declare const ARCHIVE_SOURCES: readonly ["broker_read", "broker_write"];
+export declare const CAPTURE_FEEDS: readonly ["ORDERBOOK", "TICKER", "TRADES", "OHLCV"];
+export declare const SOURCE_MODES: readonly ["broker_live_stream_v1", "broker_live_sampling_v1", "broker_current_snapshot_v1", "broker_bootstrap_fetch_v1", "external_ccxt_fallback_v1", "external_hummingbot_fallback_v1", "legacy_migration_v1"];
+export declare const CONSTRUCTION_MODES: readonly ["sampled_top_n_snapshot", "exact_l2_reconstruction"];
+export declare const GAP_POLICIES: readonly ["record_gap", "ohlcv_catch_up", "fail_fast"];
+export declare const RAW_CAPTURE_SCOPES: readonly ["ccxt_normalized_object", "broker_visible_payload", "exchange_wire_frame"];
+export type CaptureFeed = (typeof CAPTURE_FEEDS)[number];
+export type SourceMode = (typeof SOURCE_MODES)[number];
+export type ConstructionMode = (typeof CONSTRUCTION_MODES)[number];
+export type GapPolicy = (typeof GAP_POLICIES)[number];
+/**
+ * Renders a finite JavaScript number as a plain decimal. Exponent expansion and
+ * negative-zero normalization are part of checksum algorithm v1 and are mirrored
+ * by the Python fixture verifier.
+ */
+export declare function canonicalDecimal(value: number): string;
+export declare function canonicalSerialize(value: unknown): string;
+export declare function sha256Canonical(value: unknown): string;
+export declare function normalizeTimestampMs(value: unknown, field: string): number;
+export declare function createRawCapture(context: MarketCaptureContext, input: {
+    payload: unknown;
+    eventTimeMs: unknown;
+    receivedTimeMs: unknown;
+    scope: RawCaptureScope;
+}): RawCapture;
+export declare function captureCoreFields(context: MarketCaptureContext, rawCapture: RawCapture): Record<string, unknown>;

package/dist/helpers/market-data-archive/capture.d.ts

@@ -5,6 +5,8 @@ import { OrderbookSampler } from "./orderbook-sampler";
 import type { CexStreamArchiveInput, OhlcvArchiveInput, OrderbookArchiveInput, TickerArchiveInput, TradesArchiveInput } from "./types";
 export declare function archiveOrderbookInBackground(archiver: BrokerExecutionArchiver | undefined, otelMetrics: OtelMetrics | undefined, input: OrderbookArchiveInput, options?: {
     sampledOut?: boolean;
+    sourceMode?: "broker_live_sampling_v1" | "broker_current_snapshot_v1";
+    depthLimit?: number;
 }): void;
 /** @deprecated Use archiveOrderbookInBackground */
 export declare const archiveOrderbookSnapshotInBackground: typeof archiveOrderbookInBackground;

package/dist/helpers/market-data-archive/index.d.ts

@@ -1,9 +1,13 @@
+export { buildCanonicalOrderBookRows, OrderBookValidationError, } from "./canonical-orderbook";
 export { archiveCexStreamEventInBackground, archiveOhlcvInBackground, archiveOrderbookInBackground, archiveOrderbookSnapshotInBackground, archiveOrderbookTobInBackground, archiveTickerInBackground, archiveTradesInBackground, createOhlcvBarTracker, createOrderbookSampler, createOrderbookTobSampler, } from "./capture";
+export { captureEnvironmentFromEnv, createMarketCaptureContext, validateExternalFallbackContext, validateProductionCollectorArchive, } from "./capture-context";
+export { ARCHIVE_SOURCES, CAPTURE_FEEDS, CHECKSUM_ALGORITHM, CONSTRUCTION_MODES, canonicalDecimal, canonicalSerialize, createRawCapture, GAP_POLICIES, MARKET_CAPTURE_SCHEMA_VERSION, RAW_CAPTURE_SCOPES, SOURCE_MODES, sha256Canonical, } from "./capture-contract";
+export { buildLegacyOhlcvMigrationRow, buildLegacyOrderBookMigrationRows, type LegacyCandle, type LegacyOrderBookSnapshot, } from "./legacy-migration";
 export { extractLatestOhlcvBar, extractOhlcvBars, OhlcvBarTracker, parseOhlcvBar, } from "./ohlcv-bar-tracker";
 export { resolveOhlcvBootstrapLimit } from "./ohlcv-bootstrap";
 export { bootstrapOhlcvHistory } from "./ohlcv-history";
 export { getOrderbookArchiveDepthLimit, splitOrderBookSide, } from "./orderbook-depth";
 export { getOrderbookIntervalMs, getOrderbookTobIntervalMs, isMarketArchiveEnabled, OrderbookSampler, OrderbookTobSampler, } from "./orderbook-sampler";
 export { extractTrades, parseTicker, parseTrade } from "./parse-stream";
-export { buildCandleRow, buildCexStreamEventRow, buildCexTickerEventRow, buildCexTradeRow, buildOrderbookDepthRow, buildOrderbookSnapshotRow, buildOrderbookTobRow, } from "./rows";
-export type { CexStreamArchiveInput, CexStreamType, MarketArchiveContext, OhlcvArchiveCandidate, OhlcvArchiveInput, OrderbookArchiveInput, OrderbookSnapshotArchiveInput, OrderbookTobArchiveInput, ParsedOhlcvBar, TickerArchiveInput, TradesArchiveInput, } from "./types";
+export { buildCanonicalCexStreamEventRow, buildCanonicalOhlcvRow, buildCanonicalTickerEventRow, buildCanonicalTradeRow, buildCexStreamEventRow, } from "./rows";
+export type { CaptureFeed, CaptureSourceMode, CexStreamArchiveInput, CexStreamType, MarketArchiveContext, MarketCaptureContext, OhlcvArchiveCandidate, OhlcvArchiveInput, OrderbookArchiveInput, OrderbookSnapshotArchiveInput, OrderbookTobArchiveInput, ParsedOhlcvBar, RawCapture, RawCaptureScope, TickerArchiveInput, TradesArchiveInput, } from "./types";

package/dist/helpers/market-data-archive/legacy-migration.d.ts

@@ -0,0 +1,37 @@
+import type { BrokerArchiveRow, BrokerArchiveSource } from "../broker-execution-archive/types";
+export type LegacyOrderBookSnapshot = {
+    source?: BrokerArchiveSource;
+    deployment_id: string;
+    account_selector?: string;
+    exchange: string;
+    asset_type: "spot" | "swap" | "future";
+    symbol: string;
+    event_time_ms: number;
+    received_time_ms: number;
+    depth_limit: number;
+    bids_price: number[];
+    bids_size: number[];
+    asks_price: number[];
+    asks_size: number[];
+    sequence?: number;
+};
+export type LegacyCandle = {
+    source?: BrokerArchiveSource;
+    deployment_id: string;
+    account_selector?: string;
+    exchange: string;
+    asset_type: "spot" | "swap" | "future";
+    symbol: string;
+    timeframe: string;
+    open_time_ms: number;
+    open: number;
+    high: number;
+    low: number;
+    close: number;
+    volume: number;
+    quote_volume?: number;
+    is_closed: 0 | 1;
+    broker_version: number;
+};
+export declare function buildLegacyOrderBookMigrationRows(legacy: LegacyOrderBookSnapshot): BrokerArchiveRow[];
+export declare function buildLegacyOhlcvMigrationRow(legacy: LegacyCandle): BrokerArchiveRow;

package/dist/helpers/market-data-archive/rows.d.ts

@@ -1,18 +1,14 @@
 import type { BrokerArchiveRow } from "../broker-execution-archive/types";
 import type { ParsedTicker, ParsedTrade } from "./parse-stream";
-import type { CexStreamArchiveInput, MarketArchiveContext, OrderbookArchiveInput, ParsedOhlcvBar, TickerArchiveInput, TradesArchiveInput } from "./types";
-export declare function buildOrderbookSnapshotRow(input: OrderbookArchiveInput): BrokerArchiveRow | null;
-/** @deprecated Use buildOrderbookSnapshotRow */
-export declare const buildOrderbookTobRow: typeof buildOrderbookSnapshotRow;
-/** @deprecated Use buildOrderbookSnapshotRow */
-export declare const buildOrderbookDepthRow: typeof buildOrderbookSnapshotRow;
-export declare function buildCandleRow(input: {
-    context: MarketArchiveContext;
+import type { CexStreamArchiveInput, MarketCaptureContext, ParsedOhlcvBar, RawCapture } from "./types";
+export declare function buildCanonicalCexStreamEventRow(context: MarketCaptureContext, rawCapture: RawCapture): BrokerArchiveRow;
+export declare function buildCanonicalTickerEventRow(context: MarketCaptureContext, rawCapture: RawCapture, ticker: ParsedTicker): BrokerArchiveRow;
+export declare function buildCanonicalTradeRow(context: MarketCaptureContext, rawCapture: RawCapture, trade: ParsedTrade): BrokerArchiveRow;
+export declare function buildCanonicalOhlcvRow(input: {
+    context: MarketCaptureContext;
+    rawCapture: RawCapture;
     bar: ParsedOhlcvBar;
     isClosed: boolean;
     brokerVersion: number;
-    receivedTimestamp: number;
 }): BrokerArchiveRow;
 export declare function buildCexStreamEventRow(input: CexStreamArchiveInput): BrokerArchiveRow;
-export declare function buildCexTickerEventRow(input: TickerArchiveInput, ticker: ParsedTicker): BrokerArchiveRow;
-export declare function buildCexTradeRow(input: TradesArchiveInput, trade: ParsedTrade): BrokerArchiveRow;

package/dist/helpers/market-data-archive/types.d.ts

@@ -1,7 +1,7 @@
-import type { BrokerArchiveRow } from "../broker-execution-archive/types";
+import type { BrokerArchiveRow, BrokerArchiveSource } from "../broker-execution-archive/types";
 import type { BrokerMarketType } from "../market-type";
 import type { NormalizedOrderBookSnapshot } from "../order-book";
-export type MarketArchiveTable = "market_data.orderbook_snapshots" | "market_data.candles" | "market_data.cex_stream_events" | "market_data.cex_ticker_events" | "market_data.cex_trades";
+export type MarketArchiveTable = "market_data.orderbook_snapshots" | "market_data.candles" | "market_data.cex_stream_events" | "market_data.cex_ticker_events" | "market_data.cex_trades" | "market_data.cex_ohlcv" | "market_data.cex_order_book_levels" | "market_data.cex_order_book_depth_summary";
 export type CexStreamType = "BALANCE" | "ORDERS" | "ORDERBOOK" | "TRADES" | "TICKER" | "OHLCV";
 export type ParsedOhlcvBar = {
     openTimeMs: number;
@@ -18,6 +18,7 @@ export type OhlcvArchiveCandidate = {
     brokerVersion: number;
 };
 export type MarketArchiveContext = {
+    source?: BrokerArchiveSource;
     exchange: string;
     symbol: string;
     assetType: BrokerMarketType;
@@ -25,6 +26,28 @@ export type MarketArchiveContext = {
     accountSelector?: string;
     deploymentId: string;
 };
+export type CaptureFeed = "ORDERBOOK" | "TICKER" | "TRADES" | "OHLCV";
+export type CaptureSourceMode = "broker_live_stream_v1" | "broker_live_sampling_v1" | "broker_current_snapshot_v1" | "broker_bootstrap_fetch_v1" | "external_ccxt_fallback_v1" | "external_hummingbot_fallback_v1" | "legacy_migration_v1";
+export type RawCaptureScope = "ccxt_normalized_object" | "broker_visible_payload" | "exchange_wire_frame";
+export type MarketCaptureContext = MarketArchiveContext & {
+    source: BrokerArchiveSource;
+    captureBundleId: string;
+    feed: CaptureFeed;
+    provider: string;
+    sourceMode: CaptureSourceMode;
+    schemaVersion: string;
+    checksumAlgorithm: string;
+    provenanceComplete: boolean;
+};
+export type RawCapture = {
+    rawCaptureId: string;
+    rawCaptureScope: RawCaptureScope;
+    rawChecksum: string;
+    redactedPayload: unknown;
+    eventTimeMs: number;
+    receivedTimeMs: number;
+    checksumAlgorithm: string;
+};
 export type OrderbookArchiveInput = MarketArchiveContext & {
     snapshot: NormalizedOrderBookSnapshot;
 };
@@ -36,6 +59,7 @@ export type OhlcvArchiveInput = MarketArchiveContext & {
     payload: unknown;
     receivedTimestamp: number;
     timeframe: string;
+    sourceMode?: "broker_live_stream_v1" | "broker_bootstrap_fetch_v1";
 };
 export type TradesArchiveInput = MarketArchiveContext & {
     payload: unknown;