@usherlabs/cex-broker 0.2.50 → 0.2.52

package/dist/helpers/market-data-vendor-backfill/promotion.d.ts

@@ -1,5 +1,6 @@
 import { type BackfillArchiveRow, type PromotionReceipt, type PromotionReceiptWire } from "./contracts";
 export type StablePromotionReceipt = Omit<PromotionReceipt, "receiptId" | "verificationTimeMs">;
+export declare function promotionReceiptMatchesCurrentPolicies(receipt: PromotionReceiptWire): boolean;
 export declare function promotionReceiptId(receipt: StablePromotionReceipt): string;
 type UnfinalizedPromotionReceiptWire = Omit<PromotionReceiptWire, "receipt_id" | "promotion_identity_sha256"> & Partial<Pick<PromotionReceiptWire, "receipt_id" | "promotion_identity_sha256">>;
 export declare function finalizePromotionReceipt(receipt: UnfinalizedPromotionReceiptWire): PromotionReceiptWire;

package/dist/helpers/source-tape-sandbox.d.ts

@@ -0,0 +1,82 @@
+import type { ArchiveQueryClient } from "./market-data-vendor-backfill/archive-reader";
+import type { ArchiveSelectionWire, ForwarderBatch, MarketDataVendorBackfillRequest, ProviderObjectEvidence } from "./market-data-vendor-backfill/contracts";
+import type { CandidateVerification } from "./market-data-vendor-backfill/core";
+import { SOURCE_TAPE_CONSTRUCTION_MODE, SOURCE_TAPE_SANDBOX_TARGET, type SourceTapeArchiveSinkResult } from "./source-tape";
+export declare function verifySourceTapeArchive(input: {
+    request: MarketDataVendorBackfillRequest;
+    sinkResult: SourceTapeArchiveSinkResult;
+    client: ArchiveQueryClient;
+}): Promise<CandidateVerification>;
+export type SourceTapeExportArtifact = {
+    file_name: string;
+    rows: number;
+    bytes: number;
+    sha256: string;
+    projection_schema_id: string;
+    projection_schema_sha256: string;
+};
+export type SourceTapeExportResult = {
+    promotionReceiptIds: string[];
+    levels: SourceTapeExportArtifact;
+    summary: SourceTapeExportArtifact;
+};
+export type SourceTapeSandboxEvidence = {
+    schema_id: "market-data-source-tape-sandbox-evidence/v1";
+    archive_target: typeof SOURCE_TAPE_SANDBOX_TARGET;
+    construction_mode: typeof SOURCE_TAPE_CONSTRUCTION_MODE;
+    normal_archive_path: true;
+    tape_capability: {
+        policy_id: string;
+        policy_sha256: string;
+    };
+    state_count: number;
+    promotion: {
+        receipt_id: string;
+        promotion_identity_sha256: string;
+        qualification_event_id: string;
+    };
+    selection: {
+        selection_sha256: string;
+        receipt_ids: string[];
+    };
+    export: {
+        levels: SourceTapeExportArtifact;
+        summary: SourceTapeExportArtifact;
+    };
+    projection_schema_pins: readonly {
+        schema_id: string;
+        schema_sha256: string;
+    }[];
+};
+export declare function promoteAndExportSourceTapeSandbox(input: {
+    request: MarketDataVendorBackfillRequest;
+    sinkResult: SourceTapeArchiveSinkResult;
+    verification: CandidateVerification;
+    datasetObjects: ProviderObjectEvidence[];
+    vendorSemanticDigest: string;
+    verifiedAt: string;
+    forwarder: {
+        submit(batch: ForwarderBatch): Promise<{
+            ok: boolean;
+            inserted: number;
+        }>;
+    };
+    archive: {
+        resolveSelection(request: MarketDataVendorBackfillRequest): Promise<ArchiveSelectionWire>;
+    };
+    exporter: {
+        export(request: {
+            schema_id: "https://schemas.usher.so/cex-canonical-orderbook-export-request/v1";
+            request_id: string;
+            target: {
+                environment: string;
+                cluster: string;
+            };
+            selection: ArchiveSelectionWire;
+            depth: 100;
+            construction_mode: typeof SOURCE_TAPE_CONSTRUCTION_MODE;
+            canonical_schema_version: string;
+            checksum_algorithm: "sha256-canonical-json-v1";
+        }): Promise<SourceTapeExportResult>;
+    };
+}): Promise<SourceTapeSandboxEvidence>;

package/dist/helpers/source-tape.d.ts

@@ -0,0 +1,165 @@
+import type { BackfillArchiveRow, ForwarderBatch } from "./market-data-vendor-backfill/contracts";
+import { type PolicyNeutralCryptoHftBookState, type PolicyNeutralTapeSink } from "./market-data-vendor-backfill/cryptohftdata";
+export declare const SOURCE_TAPE_CAPABILITY: Readonly<{
+    policy_id: string;
+    provider: string;
+    adapter_version: string;
+    acquisition_policy_id: string;
+    exchange: string;
+    market_type: string;
+    feed: string;
+    trading_pairs: string[];
+    construction_mode: string;
+    canonical_schema_id: string;
+    depth: number;
+    source_window_scan: string;
+    provider_object_inventory: string;
+    state_emission: string;
+    state_window: string;
+    archive_target: {
+        environment: string;
+        cluster: string;
+    };
+    archive_path: string;
+    submission_bounds: {
+        max_states_per_yield: number;
+        max_rows_per_batch: number;
+        max_bytes_per_batch: number;
+        max_in_flight: number;
+    };
+    clickhouse_image: string;
+    clickhouse_server_version: string;
+    projection_schema_ids: string[];
+    policy_sha256: string;
+}>;
+export declare const SOURCE_TAPE_CAPABILITY_ID: string;
+export declare const SOURCE_TAPE_CONSTRUCTION_MODE: "policy_neutral_top_n_state_change_tape/v1";
+export declare const SOURCE_TAPE_DEPTH: 100;
+export declare const SOURCE_TAPE_MAX_STATES_PER_YIELD: 4;
+export declare const SOURCE_TAPE_MAX_BATCH_ROWS = 1000;
+export declare const SOURCE_TAPE_MAX_BATCH_BYTES: number;
+export declare const SOURCE_TAPE_MAX_IN_FLIGHT: 1;
+export declare const SOURCE_TAPE_SANDBOX_TARGET: Readonly<{
+    environment: "sandbox";
+    cluster: "cex-archive-local";
+}>;
+export declare const SOURCE_TAPE_PROJECTION_PINS: readonly Readonly<{
+    schema_id: string;
+    schema_sha256: string;
+}>[];
+export type SourceTapeProviderObjectInventory = {
+    expected_identities: string[];
+    observed_identities: string[];
+    complete: boolean;
+};
+export declare function sourceTapeCaptureBundleId(input: {
+    idempotencyKey: string;
+    tradingPair: "ARB-USDC" | "ARB-USDT";
+    window: {
+        startTimeMs: number;
+        endTimeMs: number;
+    };
+    expectedObjectIdentities: readonly string[];
+}): string;
+/**
+ * `production` is the stable forwarder mutation-authorization class. It is not
+ * an environment assertion: source-tape qualification remains sandbox-local.
+ */
+export declare function assertSourceTapeSandboxAuthorization(input: {
+    requestAuthorizationId: string;
+    requestTarget: {
+        environment: string;
+        cluster: string;
+    };
+    preflight: {
+        authorizationId: string;
+        scope: "production";
+        environment: string;
+        cluster: string;
+        credentialValidated: boolean;
+    };
+}): void;
+export type SourceTapeArchiveSinkResult = {
+    capture_bundle_id: string;
+    state_count: number;
+    level_row_count: number;
+    summary_row_count: number;
+    forwarder_batch_count: number;
+    forwarder_batch_identity_sha256: string;
+    initializer: {
+        canonical_snapshot_id: string;
+        source_time_ms: number;
+        sequence: string;
+        semantic_stream_position: 0;
+    };
+    expected_semantic_digest: string;
+    first_state_descriptor_sha256: string;
+    last_state_descriptor_sha256: string;
+    provider_object_inventory_complete: true;
+    max_states_per_yield: typeof SOURCE_TAPE_MAX_STATES_PER_YIELD;
+    max_batch_rows: typeof SOURCE_TAPE_MAX_BATCH_ROWS;
+    max_batch_bytes: typeof SOURCE_TAPE_MAX_BATCH_BYTES;
+    max_in_flight_submissions: typeof SOURCE_TAPE_MAX_IN_FLIGHT;
+};
+export type SourceTapeSemanticStateDescriptor = {
+    state_kind: "initialization" | "change";
+    semantic_stream_position: number;
+    canonical_snapshot_id: string;
+    source_time_ms: number;
+    sequence: string;
+    canonical_rows_sha256: string;
+};
+export declare class SourceTapeSemanticDigestAccumulator {
+    private readonly hash;
+    private count;
+    constructor();
+    append(descriptor: SourceTapeSemanticStateDescriptor): void;
+    digest(): string;
+}
+export declare function sourceTapeSemanticStateDescriptor(input: {
+    stateKind: "initialization" | "change";
+    semanticStreamPosition: number;
+    rows: readonly BackfillArchiveRow[];
+}): SourceTapeSemanticStateDescriptor;
+export type SourceTapeArchiveSink = PolicyNeutralTapeSink & {
+    result(): SourceTapeArchiveSinkResult;
+};
+/**
+ * Streams canonical tape rows to the existing archive-forwarder admission
+ * tables. Promotion, selection and exporter-v2 finalization remain later
+ * gates; this sink never creates a receipt or a success manifest by itself.
+ */
+export declare function createSourceTapeArchiveSink(input: {
+    captureBundleId: string;
+    tradingPair: "ARB-USDC" | "ARB-USDT";
+    window: {
+        startTimeMs: number;
+        endTimeMs: number;
+    };
+    forwarder: {
+        submit(batch: ForwarderBatch): Promise<{
+            ok: boolean;
+            inserted: number;
+        }>;
+    };
+}): SourceTapeArchiveSink;
+export declare function providerObjectInventoryIsComplete(inventory: SourceTapeProviderObjectInventory): boolean;
+export declare function evaluateSourceTapeEligibility(input: {
+    source_enumeration_eligible: boolean;
+    provider_object_inventory: SourceTapeProviderObjectInventory;
+    tape_complete: boolean;
+    capability_id: string;
+    capability_sha256: string;
+    construction_mode: string;
+    projection_schema_pins: readonly {
+        schema_id: string;
+        schema_sha256: string;
+    }[];
+    artifact_sha256s: readonly string[];
+    tape_manifest_sha256: string;
+}): boolean;
+export declare function normalizeSourceTapeStates(input: {
+    tape: readonly PolicyNeutralCryptoHftBookState[];
+    capture_bundle_id: string;
+    trading_pair: "ARB-USDC" | "ARB-USDT";
+}): BackfillArchiveRow[];

package/dist/index.js

@@ -292725,6 +292725,16 @@ function withTimeout(promise, timeoutMs, label) {
   return Promise.race([promise, expiry]).finally(() => clearTimeout(timer));
 }
 var ALL_CURRENCIES_CODE = "*";
+var BINANCE_UNLOCK_PROGRESS_SOURCE = {
+  venue: "binance",
+  endpoint: "GET /sapi/v1/capital/deposit/hisrec",
+  fields: {
+    status: "info.status",
+    confirmTimes: "info.confirmTimes",
+    unlockConfirm: "info.unlockConfirm",
+    completeTime: "info.completeTime"
+  }
+};
 function depositTimestamp(record) {
   const observedAt = depositField(record, [
     "timestamp",
@@ -292737,12 +292747,185 @@ function depositTimestamp(record) {
   const timestamp = typeof observedAt === "number" ? observedAt : typeof observedAt === "string" ? Date.parse(observedAt) : Number.NaN;
   return Number.isFinite(timestamp) ? timestamp : undefined;
 }
-function archivedDepositStatus(exchangeId, record) {
-  const rawStatus = asRecord(record.info)?.status;
-  if (exchangeId?.toLowerCase() === "binance" && String(rawStatus ?? "").trim() === "6") {
-    return "credited_not_withdrawable";
+function parseNonNegativeInteger(value) {
+  if (typeof value === "number") {
+    return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
+  }
+  if (typeof value !== "string") {
+    return;
+  }
+  const trimmed = value.trim();
+  if (!/^\d+$/.test(trimmed)) {
+    return;
+  }
+  const parsed = Number(trimmed);
+  return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : undefined;
+}
+function parseConfirmTimes(value) {
+  if (value === undefined || value === null) {
+    return { reason: "missing_confirmTimes" };
+  }
+  let currentValue;
+  let creditRequiredValue;
+  if (typeof value === "string") {
+    const match = /^(\d+)\s*\/\s*(\d+)$/.exec(value.trim());
+    if (!match) {
+      return { reason: "invalid_confirmTimes" };
+    }
+    currentValue = match[1];
+    creditRequiredValue = match[2];
+  } else {
+    const record = asRecord(value);
+    if (!record) {
+      return { reason: "invalid_confirmTimes" };
+    }
+    currentValue = record.current;
+    creditRequiredValue = record.credit_required;
+  }
+  const current = parseNonNegativeInteger(currentValue);
+  const creditRequired = parseNonNegativeInteger(creditRequiredValue);
+  if (current === undefined || creditRequired === undefined) {
+    return { reason: "invalid_confirmTimes" };
+  }
+  return { current, creditRequired, reason: null };
+}
+function parseOptionalNonNegativeInteger(value, fieldName) {
+  if (value === undefined || value === null) {
+    return { value: undefined, reason: null };
+  }
+  const parsed = parseNonNegativeInteger(value);
+  return parsed === undefined ? { value: undefined, reason: `invalid_${fieldName}` } : { value: parsed, reason: null };
+}
+function parseBinanceUnlockProgress(record) {
+  const info = asRecord(record.info);
+  const nativeStatus = parseNonNegativeInteger(info?.status);
+  const confirmations = parseConfirmTimes(info?.confirmTimes);
+  const unlockRequired = parseOptionalNonNegativeInteger(info?.unlockConfirm, "unlockConfirm");
+  const completeTime = parseOptionalNonNegativeInteger(info?.completeTime, "completeTime");
+  let quality = "valid";
+  let reason = null;
+  if (nativeStatus === undefined) {
+    quality = "unknown";
+    reason = "missing_or_invalid_status";
+  } else if (confirmations.reason !== null) {
+    quality = "unknown";
+    reason = confirmations.reason;
+  } else if (unlockRequired.reason !== null) {
+    quality = "unknown";
+    reason = unlockRequired.reason;
+  } else if (completeTime.reason !== null) {
+    quality = "unknown";
+    reason = completeTime.reason;
+  }
+  let state = "unknown";
+  switch (nativeStatus) {
+    case 0:
+      state = "pending";
+      break;
+    case 1:
+      if (quality === "valid" && confirmations.current !== undefined && unlockRequired.value !== undefined && confirmations.current >= unlockRequired.value) {
+        state = "ok";
+      } else if (quality === "valid") {
+        state = "credited_not_withdrawable";
+      }
+      break;
+    case 2:
+    case 7:
+      state = "failed";
+      break;
+    case 6:
+      state = "credited_not_withdrawable";
+      break;
+    case 8:
+      state = "pending";
+      break;
+    default:
+      quality = "unknown";
+      reason = "unsupported_status";
+      break;
+  }
+  return {
+    version: 1,
+    state,
+    progress_state: quality,
+    reason,
+    native_status: nativeStatus ?? null,
+    current: confirmations.current ?? null,
+    credit_required: confirmations.creditRequired ?? null,
+    unlock_required: unlockRequired.value ?? null,
+    complete_time: completeTime.value ?? null,
+    observed_at: new Date().toISOString(),
+    source: BINANCE_UNLOCK_PROGRESS_SOURCE
+  };
+}
+function unlockProgressKey(progress) {
+  return JSON.stringify({
+    version: progress.version,
+    state: progress.state,
+    progress_state: progress.progress_state,
+    reason: progress.reason,
+    native_status: progress.native_status,
+    current: progress.current,
+    credit_required: progress.credit_required,
+    unlock_required: progress.unlock_required,
+    complete_time: progress.complete_time
+  });
+}
+function progressWatermark(progress, previous) {
+  if (progress.progress_state !== "valid" || progress.current === null || progress.credit_required === null || progress.unlock_required === null) {
+    return previous;
+  }
+  return {
+    current: Math.max(previous?.current ?? 0, progress.current),
+    credit_required: progress.credit_required,
+    unlock_required: progress.unlock_required
+  };
+}
+function classifyBinanceDeposit(record, previous) {
+  let progress = parseBinanceUnlockProgress(record);
+  let highWatermark = previous?.highWatermark;
+  if (progress.progress_state === "valid" && progress.current !== null && progress.credit_required !== null && progress.unlock_required !== null && previous?.highWatermark !== undefined && (progress.current < previous.highWatermark.current || progress.credit_required !== previous.highWatermark.credit_required || progress.unlock_required !== previous.highWatermark.unlock_required)) {
+    progress = {
+      ...progress,
+      state: "unknown",
+      progress_state: "contradictory",
+      reason: "confirmation_progress_regressed_or_requirement_changed"
+    };
+  } else {
+    highWatermark = progressWatermark(progress, previous?.highWatermark);
+  }
+  return {
+    archiveStatus: progress.state,
+    holdCursor: progress.state === "pending" || progress.state === "credited_not_withdrawable" || progress.state === "unknown",
+    unlockProgress: progress,
+    progressKey: unlockProgressKey(progress),
+    highWatermark
+  };
+}
+function classifyDeposit(exchangeId, record, previous) {
+  if (exchangeId?.toLowerCase() === "binance") {
+    return classifyBinanceDeposit(record, previous);
+  }
+  const archiveStatus = normalizeCcxtTransactionForArchive(record).status ?? "";
+  const status = normalizeDepositStatus(depositField(record, ["status", "state"]));
+  return {
+    archiveStatus,
+    holdCursor: archiveStatus === "credited_not_withdrawable" || status === "pending",
+    progressKey: JSON.stringify({ status: archiveStatus })
+  };
+}
+function depositIdentity(input) {
+  if (input.externalId === undefined && input.txid === undefined) {
+    return;
   }
-  return normalizeCcxtTransactionForArchive(record).status;
+  return JSON.stringify({
+    exchange: input.exchangeId,
+    account: input.accountSelector,
+    coin: input.coin === undefined ? "" : String(input.coin),
+    network: input.network === undefined ? "" : String(input.network),
+    external_id: input.externalId ?? "",
+    txid: input.txid ?? ""
+  });
 }
 function nextDepositCursor(deposits, currentSince, depositsLimit, exchangeId) {
   if (deposits.length >= depositsLimit) {
@@ -292756,9 +292939,8 @@ function nextDepositCursor(deposits, currentSince, depositsLimit, exchangeId) {
       return currentSince;
     }
     const timestamp = depositTimestamp(record);
-    const archiveStatus = archivedDepositStatus(exchangeId, record);
-    const status = normalizeDepositStatus(depositField(record, ["status", "state"]));
-    const isPending = archiveStatus === "credited_not_withdrawable" || status === "pending";
+    const classification = classifyDeposit(exchangeId, record);
+    const isPending = classification.holdCursor;
     if (timestamp === undefined) {
       if (isPending) {
         return currentSince;
@@ -292878,7 +293060,8 @@ class DepositArchivePoller {
       if (!record) {
         continue;
       }
-      const assetSymbol = depositField(record, ["currency", "code", "asset"]);
+      const info = asRecord(record.info);
+      const assetSymbol = depositField(record, ["currency", "code", "asset"]) ?? info?.coin;
       const amount = depositField(record, ["amount"]);
       const address = depositField(record, [
         "address",
@@ -292887,8 +293070,21 @@ class DepositArchivePoller {
         "destination"
       ]);
       const txid = depositField(record, ["txid", "txId", "tx_hash", "txHash"]);
-      const network = depositField(record, ["network", "chain"]);
-      const archiveStatus = archivedDepositStatus(target.exchangeId, record);
+      const network = depositField(record, ["network", "chain"]) ?? info?.network;
+      const depositTxid = txid === undefined ? undefined : String(txid);
+      const identity = depositIdentity({
+        exchangeId: target.exchangeId,
+        accountSelector: target.account.label,
+        coin: assetSymbol,
+        network,
+        externalId: depositTxid,
+        txid: depositTxid
+      });
+      const lastArchived = identity === undefined ? undefined : this.#lastArchivedByTarget.get(key)?.get(identity);
+      const classification = classifyDeposit(target.exchangeId, record, lastArchived);
+      if (lastArchived && lastArchived.status === classification.archiveStatus && lastArchived.progressKey === classification.progressKey) {
+        continue;
+      }
       const creditedAt = depositField(record, [
         "creditedAt",
         "credited_at",
@@ -292897,11 +293093,7 @@ class DepositArchivePoller {
         "timestamp",
         "datetime"
       ]);
-      const depositTxid = txid === undefined ? undefined : String(txid);
-      const lastArchived = depositTxid === undefined ? undefined : this.#lastArchivedByTarget.get(key)?.get(depositTxid);
-      if (lastArchived && lastArchived.status === archiveStatus) {
-        continue;
-      }
+      const payload = classification.unlockProgress === undefined ? record : { ...record, unlock_progress: classification.unlockProgress };
       this.params.archiver.enqueue(buildTransferEventArchiveRow({
         tags: buildCommonArchiveTags({
           deploymentId: this.params.archiver.getDeploymentId(),
@@ -292912,25 +293104,27 @@ class DepositArchivePoller {
         transfer: {
           eventKind: "deposit",
           lifecycleAction: "observe_deposit",
-          status: archiveStatus,
+          status: classification.archiveStatus,
           amount: amount === undefined ? undefined : String(amount),
           address: address === undefined ? undefined : String(address),
           network: network === undefined ? undefined : String(network),
           externalId: depositTxid,
           txid: depositTxid,
           exchangeTimestamp: normalizeTimestamp(creditedAt),
-          payload: record
+          payload
         }
       }));
-      if (depositTxid !== undefined) {
+      if (identity !== undefined) {
         let targetDeposits2 = this.#lastArchivedByTarget.get(key);
         if (!targetDeposits2) {
           targetDeposits2 = new Map;
           this.#lastArchivedByTarget.set(key, targetDeposits2);
         }
-        targetDeposits2.set(depositTxid, {
-          status: archiveStatus,
-          timestamp: depositTimestamp(record)
+        targetDeposits2.set(identity, {
+          status: classification.archiveStatus,
+          timestamp: depositTimestamp(record),
+          progressKey: classification.progressKey,
+          highWatermark: classification.highWatermark
         });
       }
       archived += 1;
@@ -294125,7 +294319,7 @@ function commonEvidenceFields(context2, rawCapture, input) {
     source_time_ms: input.eventTimeMs,
     received_time_ms: input.receivedTimeMs,
     snapshot_id: input.snapshotId,
-    construction_mode: "sampled_top_n_snapshot",
+    construction_mode: input.constructionMode,
     gap_policy: "record_gap",
     depth_limit: input.depthLimit,
     sequence: input.sequence,
@@ -294179,7 +294373,8 @@ function buildCanonicalOrderBookRows(input) {
     sequence,
     depthLimit: input.depthLimit,
     eventTimeMs,
-    receivedTimeMs
+    receivedTimeMs,
+    constructionMode: input.constructionMode ?? "sampled_top_n_snapshot"
   });
   const levels = [
     ["bid", bids],
@@ -313344,4 +313539,4 @@ export {
   CEXBroker as default
 };
 
-//# debugId=B0B080620F355A6764756E2164756E21
+//# debugId=72C8C3614857C3F664756E2164756E21