@usherlabs/cex-broker 0.2.49 → 0.2.51

package/dist/helpers/market-data-vendor-backfill/contracts.d.ts

@@ -3,7 +3,6 @@ export declare const BACKFILL_REQUEST_SCHEMA_ID: string;
 export declare const REQUIRED_CLOCK_SCHEMA_ID: string;
 export declare const ARCHIVE_SELECTION_SCHEMA_ID: string;
 export declare const PROMOTION_RECEIPT_SCHEMA_ID: string;
-export declare const BACKFILL_RESULT_SCHEMA_ID: string;
 export type FixedUtcTimestamp = string;
 export type Sha256Hex = string;
 export type LowercaseUuid = string;
@@ -161,7 +160,7 @@ export type PromotionReceiptWire = {
         end_at: FixedUtcTimestamp;
     };
     depth: number;
-    construction_mode: "sampled_top_n_snapshot" | "exact_l2_reconstruction";
+    construction_mode: "sampled_top_n_snapshot" | "exact_l2_reconstruction" | "policy_neutral_top_n_state_change_tape/v1";
     canonical_schema: {
         schema_id: string;
         schema_sha256: Sha256Hex;
@@ -183,56 +182,32 @@ export type PromotionReceiptWire = {
 };
 export declare const FINAL_BACKFILL_STATUSES: readonly ["request_invalid", "archive_preflight_failed", "already_covered", "promoted", "capability_unsupported", "credentials_missing", "vendor_fetch_failed", "archive_ingest_failed", "promotion_verification_failed"];
 export type FinalBackfillStatus = (typeof FINAL_BACKFILL_STATUSES)[number];
-export type BackfillJobResultWire = {
-    schema_id: typeof BACKFILL_RESULT_SCHEMA_ID;
-    result_sha256: Sha256Hex;
-    job_id: LowercaseUuid;
-    request_file_sha256: Sha256Hex | null;
-    executable_sha256: Sha256Hex;
-    schema_manifest_sha256: Sha256Hex;
-    cex_package: {
-        name: "@usherlabs/cex-broker";
-        version: string;
-        package_sha256: Sha256Hex;
-    };
-    capability_policy: {
-        policy_id: string;
-        policy_sha256: Sha256Hex;
-    };
-    resource_policy: {
-        policy_id: string;
-        policy_sha256: Sha256Hex;
-    };
-    build: {
-        fiet_tee_commit: string;
-        created_at: FixedUtcTimestamp;
-    };
-    started_at: FixedUtcTimestamp;
-    completed_at: FixedUtcTimestamp;
-    outcome: {
-        status: FinalBackfillStatus;
-        reason_code: string;
-        reason_subcode: string | null;
-        request_id: LowercaseUuid | null;
-        idempotency_key: Sha256Hex | null;
-        target: {
-            environment: string;
-            cluster: string;
-        } | null;
-        selection: ArchiveSelectionWire | null;
-        receipt: PromotionReceiptWire | null;
-        diagnostics: Record<string, string | number | boolean>;
-    };
+export type BackfillOutcomeWire = {
+    status: FinalBackfillStatus;
+    reason_code: string;
+    reason_subcode: string | null;
+    request_id: LowercaseUuid | null;
+    idempotency_key: Sha256Hex | null;
+    target: {
+        environment: string;
+        cluster: string;
+    } | null;
+    selection: ArchiveSelectionWire | null;
+    receipt: PromotionReceiptWire | null;
+    diagnostics: Record<string, string | number | boolean>;
 };
 type Codec<T> = {
     decode(value: unknown): T;
     is(value: unknown): value is T;
 };
+export declare class BackfillRequestValidationError extends Error {
+    readonly reasonSubcode: string;
+    constructor(reasonSubcode: string, message: string);
+}
 export declare const requiredClockCodec: Codec<RequiredClockWire>;
 export declare const archiveSelectionCodec: Codec<ArchiveSelectionWire>;
 export declare function promotionIdentitySha256(receipt: Omit<PromotionReceiptWire, "receipt_id" | "promotion_identity_sha256"> & Partial<Pick<PromotionReceiptWire, "receipt_id" | "promotion_identity_sha256">>): string;
 export declare const promotionReceiptCodec: Codec<PromotionReceiptWire>;
-export declare const backfillResultCodec: Codec<BackfillJobResultWire>;
 export declare const backfillRequestCodec: {
     decode(value: unknown): {
         wire: BackfillRequestWire;
@@ -241,9 +216,7 @@ export declare const backfillRequestCodec: {
 };
 export declare function finalizeRequiredClock(clock: Omit<RequiredClockWire, "clock_sha256">): RequiredClockWire;
 export declare function finalizeArchiveSelection(selection: Omit<ArchiveSelectionWire, "selection_sha256">): ArchiveSelectionWire;
-export declare function finalizeBackfillResult(result: Omit<BackfillJobResultWire, "result_sha256"> & Partial<Pick<BackfillJobResultWire, "result_sha256">>): BackfillJobResultWire;
 export declare const BACKFILL_REQUEST_SCHEMA_VERSION: "market-data-vendor-backfill-request/v1";
-export declare const BACKFILL_RESULT_SCHEMA_VERSION: "market-data-vendor-backfill-result/v1";
 export declare const BACKFILL_PROMOTION_SCHEMA_VERSION: "market-data-vendor-backfill-promotion-receipt/v1";
 export declare const EXTERNAL_BACKFILL_SOURCE: "external_backfill";
 export declare const HISTORICAL_VENDOR_SOURCE_MODE: "vendor_historical_backfill_v1";
@@ -297,7 +270,8 @@ export declare const backfillRequestSchema: z.ZodObject<{
     }, z.core.$strict>;
 }, z.core.$strict>;
 type ProvisionalBackfillDomainRequest = z.infer<typeof backfillRequestSchema>;
-export type MarketDataVendorBackfillRequest = ProvisionalBackfillDomainRequest & {
+export type MarketDataVendorBackfillRequest = Omit<ProvisionalBackfillDomainRequest, "constructionMode"> & {
+    constructionMode: ProvisionalBackfillDomainRequest["constructionMode"] | "policy_neutral_top_n_state_change_tape/v1";
     attemptId?: string;
     target?: {
         environment: string;
@@ -368,7 +342,7 @@ export type PromotionReceipt = {
     startTimeMs: number;
     endTimeMs: number;
     depth: number;
-    constructionMode: "sampled_top_n_snapshot" | "exact_l2_reconstruction";
+    constructionMode: "sampled_top_n_snapshot" | "exact_l2_reconstruction" | "policy_neutral_top_n_state_change_tape/v1";
     canonicalSchemaVersion: string;
     checksumAlgorithm: "sha256-canonical-json-v1";
     vendorSemanticDigest: string;
@@ -381,13 +355,4 @@ export type PromotionReceipt = {
     verificationTimeMs: number;
 };
 export declare const promotionReceiptSchema: z.ZodType<PromotionReceipt>;
-export type BackfillResult = {
-    schemaVersion: typeof BACKFILL_RESULT_SCHEMA_VERSION;
-    requestId: string;
-    idempotencyKey?: string;
-    status: BackfillStatus;
-    reasonCode: string;
-    receipt?: PromotionReceipt;
-    diagnostics?: Record<string, string | number | boolean>;
-};
 export {};

package/dist/helpers/market-data-vendor-backfill/core.d.ts

@@ -3,6 +3,7 @@ export type ProviderDataset<Row = unknown> = {
     objects: ProviderObjectEvidence[];
     vendorSemanticDigest: string;
     rows: Row[];
+    reconstructedBooks?: unknown[];
 };
 export type NormalizedBackfill = {
     captureBundleId: string;

package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts

@@ -1,7 +1,14 @@
-import { type MarketDataVendorBackfillRequest, type ProviderCapability } from "./contracts";
+import type { ReconstructionObservationSink } from "../market-data-source-forensics";
+import { type MarketDataVendorBackfillRequest, type ProviderCapability, type ProviderObjectEvidence } from "./contracts";
 import type { NormalizedBackfill, ProviderDataset } from "./core";
 export declare const CRYPTOHFTDATA_ADAPTER_VERSION: "cryptohftdata-orderbook/v2";
 export declare const CRYPTOHFTDATA_API_URL: "https://api.cryptohftdata.com";
+export declare const DIAGNOSTIC_LAG_THRESHOLDS_MS: readonly [1000, 2000, 5000, 10000, 30000, 60000];
+export declare const SOURCE_TAPE_MAX_STATES_PER_YIELD: 4;
+export declare const SOURCE_TAPE_MAX_BATCH_BYTES: 5242880;
+export declare const SOURCE_TAPE_MAX_IN_FLIGHT: 1;
+export declare const BACKFILL_CLOCK_DIAGNOSTIC_KEYS: readonly ["target_time_ms", "source_time_ms", "asof_lag_ms", "max_prior_asof_lag_ms", "missing_target_count", "covered_target_count", "first_missing_target_time_ms", "last_missing_target_time_ms", "max_observed_asof_lag_ms", "missing_target_dates_utc", "total_target_count", "unanchored_target_count", "future_state_target_count", ...("covered_target_count_lag_1000_ms" | "covered_target_count_lag_60000_ms" | "covered_target_count_lag_5000_ms" | "covered_target_count_lag_2000_ms" | "covered_target_count_lag_10000_ms" | "covered_target_count_lag_30000_ms")[]];
+export declare const BACKFILL_SEQUENCE_DIAGNOSTIC_KEYS: readonly ["event_time_ms", "expected_previous_sequence", "observed_previous_sequence", "observed_final_sequence", "sequence_gap_count", "first_sequence_gap_event_time_ms", "last_sequence_gap_event_time_ms"];
 export type CryptoHftDataCapabilityProfile = Readonly<{
     profileId: string;
     exchange: string;
@@ -20,9 +27,11 @@ export type CryptoHftDataCapabilityProfile = Readonly<{
 }>;
 export declare const CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE: CryptoHftDataCapabilityProfile;
 export declare const CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE: CryptoHftDataCapabilityProfile;
+export declare const CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE: CryptoHftDataCapabilityProfile;
 export declare class CryptoHftDataError extends Error {
     readonly reason: string;
-    constructor(reason: string);
+    readonly diagnostics: Readonly<Record<string, string | number | boolean>>;
+    constructor(reason: string, diagnostics?: Readonly<Record<string, string | number | boolean>>);
 }
 export type CryptoHftDataOrderBookRow = {
     received_time: string | number | bigint;
@@ -51,17 +60,42 @@ export type ReconstructedCryptoHftBook = {
     datasetObjectIdentity: string;
     datasetObjectChecksum: string;
 };
+export type PolicyNeutralCryptoHftBookState = ReconstructedCryptoHftBook & {
+    tapeState: "initialization" | "change";
+};
+export type PolicyNeutralTapeSink = {
+    writeBatch(states: readonly PolicyNeutralCryptoHftBookState[]): Promise<void>;
+    complete(input: {
+        expectedObjectIdentities: readonly string[];
+        observedObjects: readonly ProviderObjectEvidence[];
+        stateCount: number;
+    }): Promise<void>;
+    abort(input: {
+        reason: string;
+        retainedStateCount: number;
+    }): Promise<void>;
+};
 export declare function cryptoHftDataCapabilityFor(request: MarketDataVendorBackfillRequest, profiles?: readonly CryptoHftDataCapabilityProfile[]): ProviderCapability | undefined;
 export declare function enumerateCryptoHftDataObjects(request: MarketDataVendorBackfillRequest, providerExchangeId: string, resolvedSymbol: string): string[];
+export declare function enumerateCryptoHftDataWindowObjects(request: MarketDataVendorBackfillRequest, providerExchangeId: string, resolvedSymbol: string): string[];
 export declare function providerAcquisitionRequest(request: MarketDataVendorBackfillRequest): MarketDataVendorBackfillRequest;
-export declare function reconstructCryptoHftDataOrderBooks(request: MarketDataVendorBackfillRequest, inputRows: readonly CryptoHftDataOrderBookRow[], profile?: CryptoHftDataCapabilityProfile): ReconstructedCryptoHftBook[];
+export declare function reconstructCryptoHftDataOrderBooks(request: MarketDataVendorBackfillRequest, inputRows: readonly CryptoHftDataOrderBookRow[], profile?: CryptoHftDataCapabilityProfile, observer?: ReconstructionObservationSink): ReconstructedCryptoHftBook[];
+/**
+ * Qualification-only full-window policy-neutral source projection.
+ * It uses the production OKX grouping, validation, sequence and mutation path,
+ * but continues after the submitted clock is exhausted and emits no Maker
+ * policy decisions.
+ */
+export declare function reconstructCryptoHftDataPolicyNeutralTape(request: MarketDataVendorBackfillRequest, inputRows: readonly CryptoHftDataOrderBookRow[], profile: CryptoHftDataCapabilityProfile, observer?: ReconstructionObservationSink): PolicyNeutralCryptoHftBookState[];
 export declare function decodeCryptoHftParquetZstd(bytes: Uint8Array): Promise<Record<string, unknown>[]>;
-type CryptoHftDataAdapterOptions = {
+export type CryptoHftDataAdapterOptions = {
     baseUrl?: string;
     fetch?: typeof fetch;
     nowMs?: () => number;
     decode?: (bytes: Uint8Array) => Promise<Record<string, unknown>[]>;
     profiles?: readonly CryptoHftDataCapabilityProfile[];
+    observer?: ReconstructionObservationSink;
+    policyNeutralTapeSink?: PolicyNeutralTapeSink;
 };
 export declare class CryptoHftDataAdapter {
     private readonly baseUrl;
@@ -69,10 +103,16 @@ export declare class CryptoHftDataAdapter {
     private readonly nowMs;
     private readonly decode;
     private readonly profiles;
+    private readonly observer?;
+    private readonly collectPolicyNeutralTape;
+    private readonly policyNeutralTapeSink?;
     constructor(options?: CryptoHftDataAdapterOptions);
+    private observe;
     capabilityFor(request: MarketDataVendorBackfillRequest): ProviderCapability | undefined;
+    private findProfile;
+    private profileFor;
     discoverSymbols(providerExchangeId: string): Promise<string[]>;
     acquire(request: MarketDataVendorBackfillRequest, capability: ProviderCapability, credential: unknown): Promise<ProviderDataset<CryptoHftDataOrderBookRow>>;
+    private acquireInternal;
     normalize(request: MarketDataVendorBackfillRequest, capability: ProviderCapability, dataset: ProviderDataset, captureBundleId: string): Promise<NormalizedBackfill>;
 }
-export {};