@usherlabs/cex-broker 0.2.49 → 0.2.50

package/dist/commands/cli.js

@@ -335154,6 +335154,24 @@ function selectBrokerAccountForCex(normalizedCex, brokers, metadata) {
   return selectBrokerAccount(brokers[normalizedCex], metadata) ?? undefined;
 }
 
+// src/helpers/trace-context.ts
+var TRACE_METADATA_KEY = "x-trace-id";
+var MAX_TRACE_METADATA_LENGTH = 128;
+var OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
+var UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
+var ZERO_OTEL_TRACE_ID = "00000000000000000000000000000000";
+function extractTraceId(metadata) {
+  const raw = metadata.get(TRACE_METADATA_KEY)[0];
+  if (typeof raw !== "string" || raw.length > MAX_TRACE_METADATA_LENGTH) {
+    return;
+  }
+  const traceId = raw.trim();
+  if (OTEL_TRACE_ID_PATTERN.test(traceId) && traceId !== ZERO_OTEL_TRACE_ID || UUID_V4_PATTERN.test(traceId)) {
+    return traceId;
+  }
+  return;
+}
+
 // src/handlers/execute-action/order-book-call.ts
 var grpc4 = __toESM(require_src3(), 1);
 async function handleOrderBookCall(ctx) {
@@ -336457,6 +336475,15 @@ function isPublicMarketDataAction(action, payload) {
     return false;
   return isOrderBookCallMethod(payload?.method ?? payload?.functionName);
 }
+function grpcStatusName(error48) {
+  if (!error48) {
+    return "OK";
+  }
+  if (typeof error48.code !== "number") {
+    return "UNKNOWN";
+  }
+  return grpc12.status[error48.code] ?? "UNKNOWN";
+}
 function createExecuteActionHandler(deps) {
   const {
     policy,
@@ -336473,12 +336500,28 @@ function createExecuteActionHandler(deps) {
     const startTime = Date.now();
     const { action: rawAction, cex: cex3, symbol: symbol2 } = call.request;
     const action = resolveAction(rawAction);
+    const actionName = getActionName(action);
+    const operationalCex = cex3?.trim().toLowerCase() || "unknown";
+    const traceId = extractTraceId(call.metadata);
+    const traceFields = traceId === undefined ? {} : { trace_id: traceId };
     let actionCompleted = false;
     const wrappedCallback = (error48, value) => {
       if (!actionCompleted) {
         actionCompleted = true;
         const latency = Date.now() - startTime;
-        const actionName = getActionName(action);
+        const terminalFields = {
+          action: actionName,
+          cex: operationalCex,
+          latency_ms: latency,
+          outcome: error48 ? "error" : "success",
+          grpc_status: grpcStatusName(error48),
+          ...traceFields
+        };
+        if (error48) {
+          log.withMetadata(terminalFields).error("ExecuteAction failed");
+        } else {
+          log.withMetadata(terminalFields).info("ExecuteAction completed");
+        }
         otelMetrics?.recordHistogram("execute_action_duration_ms", latency, {
           action: actionName,
           cex: cex3 || "unknown"
@@ -336499,9 +336542,13 @@ function createExecuteActionHandler(deps) {
       callback(error48, value);
     };
     try {
-      log.info(`Request - ExecuteAction:`, { action, cex: cex3, symbol: symbol2 });
+      log.withMetadata({
+        action: actionName,
+        cex: operationalCex,
+        ...traceFields
+      }).info("ExecuteAction started");
       otelMetrics?.recordCounter("execute_action_requests_total", 1, {
-        action: getActionName(action),
+        action: actionName,
         cex: cex3 || "unknown"
       });
       if (!authenticateRequest(call, whitelistIps)) {
@@ -336739,6 +336786,9 @@ async function writeSubscribeError(call, isClosed, frame) {
     call.end();
   }
 }
+function grpcStatusName2(status14) {
+  return typeof status14 === "number" ? grpc13.status[status14] ?? "UNKNOWN" : "UNKNOWN";
+}
 function getBinanceEventMarketId(event) {
   const value = event.s;
   return typeof value === "string" ? value : null;
@@ -336871,6 +336921,24 @@ function createSubscribeHandler(deps) {
   });
   return async (call) => {
     const subscribeStartTime = Date.now();
+    const request = call.request;
+    const { cex: cex3, symbol: symbol2, type: type2 } = request;
+    const metadata = call.metadata;
+    const traceId = extractTraceId(metadata);
+    const traceFields = traceId === undefined ? {} : { trace_id: traceId };
+    const operationalCex = cex3?.trim().toLowerCase() || "unknown";
+    const operationalSymbol = symbol2?.trim() || "unknown";
+    const subscriptionType2 = resolveSubscriptionType(type2);
+    const subscriptionTypeName = getSubscriptionTypeName(subscriptionType2);
+    const operationalFields = {
+      cex: operationalCex,
+      symbol: operationalSymbol,
+      subscription_type: subscriptionTypeName,
+      ...traceFields
+    };
+    let terminalLogged = false;
+    let terminalOutcome = "completed";
+    let terminalGrpcStatus = "OK";
     let streamClosed = false;
     let ownedBroker = null;
     let ownedBrokerClosePromise;
@@ -336893,12 +336961,41 @@ function createSubscribeHandler(deps) {
     const closeOwnedBrokerOnCallEnd = () => {
       closeOwnedBroker();
     };
-    call.once("cancelled", markStreamClosed);
+    const logTerminalOutcome = (outcome) => {
+      if (terminalLogged) {
+        return;
+      }
+      terminalLogged = true;
+      const fields = {
+        ...operationalFields,
+        duration_ms: Date.now() - subscribeStartTime,
+        outcome,
+        grpc_status: outcome === "cancelled" ? "CANCELLED" : terminalGrpcStatus
+      };
+      if (outcome === "error") {
+        log.withMetadata(fields).error("Subscribe failed");
+      } else if (outcome === "cancelled") {
+        log.withMetadata(fields).info("Subscribe cancelled");
+      } else {
+        log.withMetadata(fields).info("Subscribe ended");
+      }
+    };
+    const writeTerminalError = async (frame, status14 = grpc13.status.UNKNOWN) => {
+      terminalOutcome = "error";
+      terminalGrpcStatus = grpcStatusName2(status14);
+      await writeSubscribeError(call, isStreamClosed, frame);
+    };
+    log.withMetadata(operationalFields).info("Subscribe started");
+    call.once("cancelled", () => {
+      terminalGrpcStatus = "CANCELLED";
+      markStreamClosed();
+      logTerminalOutcome("cancelled");
+    });
     call.once("cancelled", closeOwnedBrokerOnCallEnd);
     call.once("error", closeOwnedBrokerOnCallEnd);
     call.once("end", () => {
       markStreamClosed();
-      log.info("Subscribe stream ended");
+      logTerminalOutcome(terminalOutcome);
       const duration3 = Date.now() - subscribeStartTime;
       otelMetrics?.recordHistogram("subscribe_duration_ms", duration3, {
         cex: call.request?.cex || "unknown",
@@ -336907,12 +337004,16 @@ function createSubscribeHandler(deps) {
     });
     call.once("error", (error48) => {
       markStreamClosed();
-      log.error("Subscribe stream error:", error48);
+      terminalOutcome = "error";
+      terminalGrpcStatus = grpcStatusName2(typeof error48 === "object" && error48 !== null && "code" in error48 ? error48.code : undefined);
+      logTerminalOutcome(call.cancelled ? "cancelled" : "error");
       otelMetrics?.recordCounter("subscribe_errors_total", 1, {
         error_type: error48 instanceof Error ? error48.message : "unknown"
       });
     });
     if (!authenticateRequest(call, whitelistIps)) {
+      terminalOutcome = "error";
+      terminalGrpcStatus = "PERMISSION_DENIED";
       otelMetrics?.recordCounter("subscribe_errors_total", 1, {
         error_type: "permission_denied"
       });
@@ -336923,32 +337024,22 @@ function createSubscribeHandler(deps) {
       call.destroy(new Error("Access denied: Unauthorized IP"));
       return;
     }
-    const metadata = call.metadata;
-    let subscriptionType2 = SubscriptionType.ORDERBOOK;
     try {
-      const request = call.request;
-      const { cex: cex3, symbol: symbol2, type: type2, options } = request;
-      subscriptionType2 = resolveSubscriptionType(type2);
-      log.info(`Request - Subscribe:`, {
-        cex: request.cex,
-        symbol: request.symbol,
-        type: subscriptionType2
-      });
-      const subscriptionTypeName = getSubscriptionTypeName(subscriptionType2);
+      const { options } = request;
       otelMetrics?.recordCounter("subscribe_requests_total", 1, {
         cex: cex3 || "unknown",
         symbol: symbol2 || "unknown",
         type: subscriptionTypeName
       });
       if (!cex3 || !symbol2) {
-        await writeSubscribeError(call, isStreamClosed, {
+        await writeTerminalError({
           data: JSON.stringify({
             error: "cex, symbol, and type are required"
           }),
           timestamp: Date.now(),
           symbol: symbol2 || "",
           type: subscriptionType2
-        });
+        }, grpc13.status.INVALID_ARGUMENT);
         return;
       }
       if (isPublicMarketDataSubscription(subscriptionType2)) {
@@ -336977,7 +337068,7 @@ function createSubscribeHandler(deps) {
         } catch (error48) {
           const message = getErrorMessage(error48);
           if (!isStreamClosed()) {
-            await writeSubscribeError(call, isStreamClosed, {
+            await writeTerminalError({
               data: JSON.stringify({ error: message }),
               timestamp: Date.now(),
               symbol: symbol2,
@@ -336998,14 +337089,14 @@ function createSubscribeHandler(deps) {
       const selectedBroker = selectedBrokerAccount?.exchange ?? createBroker(normalizedCex, metadata);
       const broker = selectedBroker ?? createPublicBroker(normalizedCex);
       if (!broker) {
-        await writeSubscribeError(call, isStreamClosed, {
+        await writeTerminalError({
           data: JSON.stringify({
             error: "Exchange not registered and no API metadata found"
           }),
           timestamp: Date.now(),
           symbol: symbol2,
           type: subscriptionType2
-        });
+        }, grpc13.status.NOT_FOUND);
         return;
       }
       if (!selectedBrokerAccount) {
@@ -337033,28 +337124,28 @@ function createSubscribeHandler(deps) {
       if (isBinanceSpotAccountSubscription(normalizedCex, subscriptionType2, options?.marketType)) {
         const accountBroker = selectedBrokerAccount?.exchange ?? selectedBroker;
         if (!accountBroker) {
-          await writeSubscribeError(call, isStreamClosed, {
+          await writeTerminalError({
             data: JSON.stringify({
               error: "Binance account subscriptions require API credentials"
             }),
             timestamp: Date.now(),
             symbol: resolvedSymbol,
             type: subscriptionType2
-          });
+          }, grpc13.status.FAILED_PRECONDITION);
           return;
         }
         const marketId = subscriptionType2 === SubscriptionType.ORDERS ? await getBinanceMarketId(accountBroker, resolvedSymbol) : undefined;
         let userDataSource;
         if (selectedBrokerAccount) {
           if (!userDataStreamSupervisor) {
-            await writeSubscribeError(call, isStreamClosed, {
+            await writeTerminalError({
               data: JSON.stringify({
                 error: "Configured account user-data supervisor is unavailable"
               }),
               timestamp: Date.now(),
               symbol: resolvedSymbol,
               type: subscriptionType2
-            });
+            }, grpc13.status.FAILED_PRECONDITION);
             return;
           }
           userDataSource = userDataStreamSupervisor.subscribe({
@@ -337077,7 +337168,7 @@ function createSubscribeHandler(deps) {
           } catch (error48) {
             const message = getErrorMessage(error48);
             log.error(`Error fetching balance for ${cex3}:`, error48);
-            await writeSubscribeError(call, isStreamClosed, {
+            await writeTerminalError({
               data: JSON.stringify({
                 error: `Failed to fetch balance: ${message}`
               }),
@@ -337096,7 +337187,7 @@ function createSubscribeHandler(deps) {
           } catch (error48) {
             log.error(`Error fetching orders for ${resolvedSymbol} on ${cex3}:`, error48);
             const message = getErrorMessage(error48);
-            await writeSubscribeError(call, isStreamClosed, {
+            await writeTerminalError({
               data: JSON.stringify({
                 error: `Failed to fetch orders: ${message}`
               }),
@@ -337107,26 +337198,27 @@ function createSubscribeHandler(deps) {
           }
           break;
         default:
-          await writeSubscribeError(call, isStreamClosed, {
+          await writeTerminalError({
             data: JSON.stringify({ error: "Invalid subscription type" }),
             timestamp: Date.now(),
             symbol: symbol2,
             type: subscriptionType2
-          });
+          }, grpc13.status.INVALID_ARGUMENT);
       }
     } catch (error48) {
       log.error("Error in Subscribe stream:", error48);
       const message = getErrorMessage(error48);
-      await writeSubscribeError(call, isStreamClosed, {
+      await writeTerminalError({
         data: JSON.stringify({ error: `Internal server error: ${message}` }),
         timestamp: Date.now(),
         symbol: "",
         type: subscriptionType2
-      });
+      }, grpc13.status.INTERNAL);
     } finally {
       call.off("cancelled", closeOwnedBrokerOnCallEnd);
       call.off("error", closeOwnedBrokerOnCallEnd);
       await closeOwnedBroker();
+      logTerminalOutcome(call.cancelled ? "cancelled" : terminalOutcome);
     }
   };
 }

package/dist/commands/market-data-vendor-backfill.js

@@ -25307,7 +25307,8 @@ var result_schema_default = {
 // src/helpers/market-data-vendor-backfill/manifests.ts
 var LEGACY_CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
 var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v2";
-var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
+var LEGACY_RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
+var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v2";
 var ADAPTER_POLICY_ID = "cryptohftdata-orderbook-adapter/v1";
 var ACQUISITION_POLICY_ID = "cryptohftdata-hourly-acquisition/v1";
 var legacyCapabilityPolicyContent = {
@@ -25344,17 +25345,30 @@ var LEGACY_CAPABILITY_POLICY = Object.freeze({
 var capabilityPolicyContent = {
   ...legacyCapabilityPolicyContent,
   policy_id: CAPABILITY_POLICY_ID,
-  profiles: legacyCapabilityPolicyContent.profiles.map((profile) => ({
-    ...profile,
-    source_policies: ["authoritative_window", "fill_gaps"]
-  }))
+  profiles: [
+    ...legacyCapabilityPolicyContent.profiles.map((profile) => ({
+      ...profile,
+      source_policies: ["authoritative_window", "fill_gaps"]
+    })),
+    {
+      exchange: "okx",
+      market_type: "spot",
+      feed: "ORDERBOOK",
+      canonical_trading_pair: "ARB-USDC",
+      provider_exchange_id: "okx_spot",
+      resolved_symbol: "ARB-USDC",
+      construction_modes: ["sampled_top_n_snapshot"],
+      source_policies: ["authoritative_window", "fill_gaps"],
+      max_depth: 400
+    }
+  ]
 };
 var CAPABILITY_POLICY = Object.freeze({
   ...capabilityPolicyContent,
   policy_sha256: jcsSha256(capabilityPolicyContent)
 });
-var resourcePolicyContent = {
-  policy_id: RESOURCE_POLICY_ID,
+var legacyResourcePolicyContent = {
+  policy_id: LEGACY_RESOURCE_POLICY_ID,
   limits: {
     max_files: 1e4,
     max_bytes: 100 * 1024 * 1024 * 1024,
@@ -25368,6 +25382,18 @@ var resourcePolicyContent = {
     max_required_events: 1e5
   }
 };
+var LEGACY_RESOURCE_POLICY = Object.freeze({
+  ...legacyResourcePolicyContent,
+  policy_sha256: jcsSha256(legacyResourcePolicyContent)
+});
+var resourcePolicyContent = {
+  ...legacyResourcePolicyContent,
+  policy_id: RESOURCE_POLICY_ID,
+  request_bounds: {
+    ...legacyResourcePolicyContent.request_bounds,
+    max_window_ms: 31 * 24 * 60 * 60 * 1000
+  }
+};
 var RESOURCE_POLICY = Object.freeze({
   ...resourcePolicyContent,
   policy_sha256: jcsSha256(resourcePolicyContent)
@@ -25546,7 +25572,11 @@ var backfillRequestCodec = {
       CAPABILITY_POLICY,
       LEGACY_CAPABILITY_POLICY
     ].some((policy) => request.product_pins.capability_policy.policy_id === policy.policy_id && request.product_pins.capability_policy.policy_sha256 === policy.policy_sha256);
-    if (!capabilityPolicyMatches || request.product_pins.resource_policy.policy_id !== RESOURCE_POLICY.policy_id || request.product_pins.resource_policy.policy_sha256 !== RESOURCE_POLICY.policy_sha256) {
+    const resourcePolicyMatches = [
+      RESOURCE_POLICY,
+      LEGACY_RESOURCE_POLICY
+    ].some((policy) => request.product_pins.resource_policy.policy_id === policy.policy_id && request.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
+    if (!capabilityPolicyMatches || !resourcePolicyMatches) {
       throw new Error("request policy pins do not match the effective package policies");
     }
     if (request.required_clock.clock_id !== request.initial_selection.required_clock.clock_id || request.required_clock.clock_sha256 !== request.initial_selection.required_clock.clock_sha256 || request.required_clock.event_count !== request.initial_selection.required_clock.event_count) {
@@ -25661,6 +25691,10 @@ function decodeBackfillRunDocuments(input) {
     return targetTimeMs;
   });
   const capabilityProfile = CAPABILITY_POLICY.profiles.find((profile) => profile.exchange === wire.scope.exchange && profile.market_type === wire.scope.market_type && profile.feed === wire.scope.feed && profile.canonical_trading_pair === wire.scope.trading_pair);
+  const resourcePolicy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((policy) => wire.product_pins.resource_policy.policy_id === policy.policy_id && wire.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
+  if (!resourcePolicy) {
+    throw new Error("request resource policy pin is unsupported");
+  }
   return {
     schemaVersion: BACKFILL_REQUEST_SCHEMA_VERSION,
     requestId: wire.request_id,
@@ -25685,11 +25719,11 @@ function decodeBackfillRunDocuments(input) {
     maxPriorAsOfLagMs: wire.coverage_policy.max_asof_lag_ms,
     sourcePolicy: wire.source_policy,
     budgets: {
-      maxFiles: RESOURCE_POLICY.limits.max_files,
-      maxBytes: RESOURCE_POLICY.limits.max_bytes,
-      maxRows: RESOURCE_POLICY.limits.max_rows,
-      maxDurationMs: RESOURCE_POLICY.limits.max_duration_ms,
-      maxBoundaryLookbackMs: Math.min(RESOURCE_POLICY.limits.max_boundary_lookback_ms, CAPABILITY_POLICY.acquisition_policy.initialization_lookback_ms)
+      maxFiles: resourcePolicy.limits.max_files,
+      maxBytes: resourcePolicy.limits.max_bytes,
+      maxRows: resourcePolicy.limits.max_rows,
+      maxDurationMs: resourcePolicy.limits.max_duration_ms,
+      maxBoundaryLookbackMs: Math.min(resourcePolicy.limits.max_boundary_lookback_ms, CAPABILITY_POLICY.acquisition_policy.initialization_lookback_ms)
     },
     expectedProduct: {
       packageName: "@usherlabs/cex-broker",
@@ -31766,6 +31800,22 @@ var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
   constructionModes: ["sampled_top_n_snapshot"],
   sourcePolicies: ["authoritative_window", "fill_gaps"]
 });
+var CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE = Object.freeze({
+  profileId: "cryptohftdata/okx_spot/ARB-USDC/v1",
+  exchange: "okx",
+  tradingPair: "ARB-USDC",
+  sourceSymbol: "ARB-USDC",
+  marketType: "spot",
+  providerExchangeId: "okx_spot",
+  historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
+  maxDepth: 400,
+  eventTimeUnit: "milliseconds",
+  receivedTimeUnit: "nanoseconds",
+  snapshotGrouping: "event_time_final_update_id_object",
+  sequenceSemantics: "okx_seq_id_prev_seq_id",
+  constructionModes: ["sampled_top_n_snapshot"],
+  sourcePolicies: ["authoritative_window", "fill_gaps"]
+});
 
 class CryptoHftDataError extends Error {
   reason;
@@ -32511,7 +32561,10 @@ function assertForwarderPreflight(request, resolution, nowMs) {
   }
 }
 function resourcePolicyScopeExceeded(request) {
-  return request.depth > RESOURCE_POLICY.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
+  const policy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((candidate) => request.productPins?.resource_policy.policy_id === candidate.policy_id && request.productPins.resource_policy.policy_sha256 === candidate.policy_sha256);
+  if (!policy)
+    return true;
+  return request.depth > policy.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > policy.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > policy.request_bounds.max_required_events;
 }
 function storedReceiptForSelection(resolution) {
   const receiptId = resolution.selection.receipt_ids[0];
@@ -32802,8 +32855,8 @@ function validateReleaseIdentity(release) {
   return release;
 }
 function bakedReleaseIdentity() {
-  const packageVersion = "0.2.49";
-  const gitHead = "c1ffe071e7696c5b17c8f058da01d61cb27ec8a4";
+  const packageVersion = "0.2.50";
+  const gitHead = "41dbe7cc296da1c5a7964dca8f418800522f27e5";
   return validateReleaseIdentity({ packageVersion, gitHead });
 }
 function readOptionalEnvironment(environment, name) {
@@ -32827,7 +32880,10 @@ function createBackfillDependenciesFromEnv(environment, sensitiveValues = new Se
       password: clickhousePassword
     })),
     providers: new CryptoHftDataAdapter({
-      profiles: [CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE]
+      profiles: [
+        CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE,
+        CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE
+      ]
     }),
     forwarder: createArchiveForwarderClient({
       url: forwarderUrl,
@@ -33034,4 +33090,4 @@ export {
   createBackfillDependenciesFromEnv
 };
 
-//# debugId=793AA74075FC03DE64756E2164756E21
+//# debugId=7BCBC113F3C0009864756E2164756E21