@usherlabs/cex-broker 0.2.45 → 0.2.47

package/dist/handlers/subscribe/handler.d.ts

@@ -2,6 +2,7 @@ import * as grpc from "@grpc/grpc-js";
 import { type BrokerPoolEntry } from "../../helpers/broker";
 import type { BrokerExecutionArchiver } from "../../helpers/broker-execution-archive";
 import type { OtelMetrics } from "../../helpers/otel";
+import { PublicMarketDataFeedSupervisor } from "../../helpers/public-market-data-feed";
 import type { UserDataStreamSupervisor } from "../../helpers/user-data-stream-supervisor";
 import type { SubscribeRequest, SubscribeResponse } from "../types";
 import { SubscribeBrokerLifecycle } from "./broker-lifecycle";
@@ -12,6 +13,7 @@ export type SubscribeDeps = {
     brokerArchiver?: BrokerExecutionArchiver;
     brokerLifecycle?: SubscribeBrokerLifecycle;
     userDataStreamSupervisor?: UserDataStreamSupervisor;
+    publicMarketDataFeedSupervisor?: PublicMarketDataFeedSupervisor;
 };
 type SubscribeCall = grpc.ServerWritableStream<SubscribeRequest, SubscribeResponse>;
 export declare function createSubscribeHandler(deps: SubscribeDeps): (call: SubscribeCall) => Promise<void>;

package/dist/helpers/canonical-orderbook-export/exact-selection.d.ts

@@ -0,0 +1,21 @@
+import { type CanonicalOrderBookExportQuerySegment, type CanonicalOrderBookExportRequestWire } from "../market-data-preparation/contracts";
+import { type Sha256Hex } from "../market-data-vendor-backfill/contracts";
+export type ExactOrderBookQueryValue = string | number | readonly string[];
+export type CompiledExactOrderBookExport = {
+    request: CanonicalOrderBookExportRequestWire;
+    segments: CanonicalOrderBookExportQuerySegment[];
+    querySha256: Sha256Hex;
+    parameters: Readonly<Record<string, ExactOrderBookQueryValue>>;
+    predicateSql: string;
+    physicalPredicateSql: string;
+    levelsSql: string;
+    summarySql: string;
+    conflictsSql: string;
+    rowCountsSql: string;
+    promotionReceiptsSql: string;
+};
+export declare class ExactOrderBookExportError extends Error {
+    readonly reason: string;
+    constructor(reason: string);
+}
+export declare function compileExactOrderBookExport(requestInput: CanonicalOrderBookExportRequestWire): CompiledExactOrderBookExport;

package/dist/helpers/canonical-orderbook-export/exporter.d.ts

@@ -0,0 +1,26 @@
+import type { CanonicalOrderBookExportArtifact, CanonicalOrderBookExportRequestWire } from "../market-data-preparation/contracts";
+import { type CompiledExactOrderBookExport, type ExactOrderBookQueryValue } from "./exact-selection";
+export type ExactOrderBookExportFormat = "JSONEachRow" | "Parquet";
+export type ExactOrderBookExportQueryClient = {
+    execute(sql: string, parameters: Readonly<Record<string, ExactOrderBookQueryValue>>, format: ExactOrderBookExportFormat): Promise<Uint8Array>;
+};
+export type ExactCanonicalOrderBookExport = {
+    compiled: CompiledExactOrderBookExport;
+    promotionReceiptIds: string[];
+    levelsPath: string;
+    summaryPath: string;
+    levels: CanonicalOrderBookExportArtifact;
+    summary: CanonicalOrderBookExportArtifact;
+};
+export type ClickHouseExactExportClientOptions = {
+    url: string;
+    username?: string;
+    password?: string;
+    fetch?: typeof globalThis.fetch;
+};
+export declare function createClickHouseExactOrderBookExportClient(options: ClickHouseExactExportClientOptions): ExactOrderBookExportQueryClient;
+export declare function exportExactCanonicalOrderBook(input: {
+    request: CanonicalOrderBookExportRequestWire;
+    client: ExactOrderBookExportQueryClient;
+    outputDirectory: string;
+}): Promise<ExactCanonicalOrderBookExport>;

package/dist/helpers/market-data-archive/capture-contract.d.ts

@@ -2,11 +2,12 @@ import type { MarketCaptureContext, RawCapture, RawCaptureScope } from "./types"
 export declare const MARKET_CAPTURE_SCHEMA_VERSION: "1.0.0";
 export declare const CHECKSUM_ALGORITHM: "sha256-canonical-json-v1";
 export declare const ARCHIVE_SOURCES: readonly ["broker_read", "broker_write"];
+export declare const MARKET_ARCHIVE_SOURCES: readonly ["broker_read", "broker_write", "external_backfill"];
 export declare const CAPTURE_FEEDS: readonly ["ORDERBOOK", "TICKER", "TRADES", "OHLCV"];
-export declare const SOURCE_MODES: readonly ["broker_live_stream_v1", "broker_live_sampling_v1", "broker_current_snapshot_v1", "broker_bootstrap_fetch_v1", "external_ccxt_fallback_v1", "external_hummingbot_fallback_v1", "legacy_migration_v1"];
+export declare const SOURCE_MODES: readonly ["broker_live_stream_v1", "broker_live_sampling_v1", "broker_current_snapshot_v1", "broker_bootstrap_fetch_v1", "external_ccxt_fallback_v1", "external_hummingbot_fallback_v1", "legacy_migration_v1", "historical_vendor_orderbook_v1", "vendor_historical_backfill_v1"];
 export declare const CONSTRUCTION_MODES: readonly ["sampled_top_n_snapshot", "exact_l2_reconstruction"];
 export declare const GAP_POLICIES: readonly ["record_gap", "ohlcv_catch_up", "fail_fast"];
-export declare const RAW_CAPTURE_SCOPES: readonly ["ccxt_normalized_object", "broker_visible_payload", "exchange_wire_frame"];
+export declare const RAW_CAPTURE_SCOPES: readonly ["ccxt_normalized_object", "broker_visible_payload", "exchange_wire_frame", "vendor_normalized_dataset_file"];
 export type CaptureFeed = (typeof CAPTURE_FEEDS)[number];
 export type SourceMode = (typeof SOURCE_MODES)[number];
 export type ConstructionMode = (typeof CONSTRUCTION_MODES)[number];

package/dist/helpers/market-data-archive/types.d.ts

@@ -1,6 +1,7 @@
 import type { BrokerArchiveRow, BrokerArchiveSource } from "../broker-execution-archive/types";
 import type { BrokerMarketType } from "../market-type";
 import type { NormalizedOrderBookSnapshot } from "../order-book";
+export type MarketArchiveSource = BrokerArchiveSource | "external_backfill";
 export type MarketArchiveTable = "market_data.orderbook_snapshots" | "market_data.candles" | "market_data.cex_stream_events" | "market_data.cex_ticker_events" | "market_data.cex_trades" | "market_data.cex_ohlcv" | "market_data.cex_order_book_levels" | "market_data.cex_order_book_depth_summary";
 export type CexStreamType = "BALANCE" | "ORDERS" | "ORDERBOOK" | "TRADES" | "TICKER" | "OHLCV";
 export type ParsedOhlcvBar = {
@@ -27,10 +28,11 @@ export type MarketArchiveContext = {
     deploymentId: string;
 };
 export type CaptureFeed = "ORDERBOOK" | "TICKER" | "TRADES" | "OHLCV";
-export type CaptureSourceMode = "broker_live_stream_v1" | "broker_live_sampling_v1" | "broker_current_snapshot_v1" | "broker_bootstrap_fetch_v1" | "external_ccxt_fallback_v1" | "external_hummingbot_fallback_v1" | "legacy_migration_v1";
-export type RawCaptureScope = "ccxt_normalized_object" | "broker_visible_payload" | "exchange_wire_frame";
-export type MarketCaptureContext = MarketArchiveContext & {
-    source: BrokerArchiveSource;
+export type CaptureSourceMode = "broker_live_stream_v1" | "broker_live_sampling_v1" | "broker_current_snapshot_v1" | "broker_bootstrap_fetch_v1" | "external_ccxt_fallback_v1" | "external_hummingbot_fallback_v1" | "legacy_migration_v1" | "historical_vendor_orderbook_v1" | "vendor_historical_backfill_v1";
+export type CaptureOrigin = "production_capture" | "vendor_historical_backfill";
+export type RawCaptureScope = "ccxt_normalized_object" | "broker_visible_payload" | "exchange_wire_frame" | "vendor_normalized_dataset_file";
+export type MarketCaptureContext = Omit<MarketArchiveContext, "source"> & {
+    source: MarketArchiveSource;
     captureBundleId: string;
     feed: CaptureFeed;
     provider: string;
@@ -38,6 +40,8 @@ export type MarketCaptureContext = MarketArchiveContext & {
     schemaVersion: string;
     checksumAlgorithm: string;
     provenanceComplete: boolean;
+    tradingPair?: string;
+    sourceSymbol?: string;
 };
 export type RawCapture = {
     rawCaptureId: string;

package/dist/helpers/market-data-preparation/conformance-fixtures.d.ts

@@ -0,0 +1,16 @@
+export declare const PREPARATION_CONFORMANCE_FIXTURES: Readonly<{
+    fixture_id: "cex-market-data-preparation-conformance/v2";
+    legacy_fixture_id: "market-data-vendor-backfill-conformance/v1";
+    documents: Readonly<{
+        backfill_result: import("./contracts").BackfillJobResultV2Wire;
+        canonical_orderbook_export_request: import("./contracts").CanonicalOrderBookExportRequestWire;
+        canonical_orderbook_export_result: import("./contracts").CanonicalOrderBookExportResultWire;
+        preparation_product_pin: import("./contracts").PreparationProductPinWire;
+    }>;
+    identities: Readonly<{
+        manifest_sha256: string;
+        backfill_result_sha256: string;
+        selection_sha256: string;
+        export_result_sha256: string;
+    }>;
+}>;