@usherlabs/cex-broker 0.2.49 → 0.2.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/dist/commands/cex-canonical-orderbook-export.js +7747 -883
  2. package/dist/commands/cex-canonical-orderbook-export.js.map +41 -10
  3. package/dist/commands/cli.js +127 -34
  4. package/dist/commands/market-data-vendor-backfill.js +8284 -6391
  5. package/dist/commands/market-data-vendor-backfill.js.map +16 -15
  6. package/dist/helpers/canonical-orderbook-export/parquet-projection.d.ts +19 -0
  7. package/dist/helpers/market-data-archive/canonical-orderbook.d.ts +1 -1
  8. package/dist/helpers/market-data-preparation/conformance-fixtures.d.ts +114 -2
  9. package/dist/helpers/market-data-preparation/contracts.d.ts +1298 -18
  10. package/dist/helpers/market-data-preparation/required-clock-qualification.d.ts +43 -0
  11. package/dist/helpers/market-data-preparation/source-tape-dependencies.d.ts +19 -0
  12. package/dist/helpers/market-data-preparation/source-tape-operation.d.ts +109 -0
  13. package/dist/helpers/market-data-source-forensics.d.ts +400 -0
  14. package/dist/helpers/market-data-vendor-backfill/conformance-fixtures.d.ts +3 -5
  15. package/dist/helpers/market-data-vendor-backfill/contracts.d.ts +21 -56
  16. package/dist/helpers/market-data-vendor-backfill/core.d.ts +1 -0
  17. package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +45 -5
  18. package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +31 -1083
  19. package/dist/helpers/market-data-vendor-backfill/promotion.d.ts +1 -0
  20. package/dist/helpers/source-tape-sandbox.d.ts +82 -0
  21. package/dist/helpers/source-tape.d.ts +165 -0
  22. package/dist/helpers/trace-context.d.ts +8 -0
  23. package/dist/index.js +128 -35
  24. package/dist/index.js.map +7 -6
  25. package/dist/market-data-preparation/fixtures/{conformance-v2.json → conformance-v3.json} +257 -60
  26. package/dist/market-data-preparation/policies/capability-policy.json +15 -4
  27. package/dist/market-data-preparation/policies/resource-policy.json +3 -3
  28. package/dist/market-data-preparation/policies/source-tape-capability-v1.json +35 -0
  29. package/dist/market-data-preparation/schema-manifest.json +30 -10
  30. package/dist/market-data-preparation/schemas/canonical-orderbook-export-request-v1.schema.json +5 -1
  31. package/dist/market-data-preparation/schemas/{canonical-orderbook-export-result-v1.schema.json → canonical-orderbook-export-result-v2.schema.json} +20 -6
  32. package/dist/market-data-preparation/schemas/order-book-depth-summary-parquet-projection-v1.schema.json +256 -0
  33. package/dist/market-data-preparation/schemas/order-book-levels-parquet-projection-v1.schema.json +220 -0
  34. package/dist/market-data-preparation/schemas/{preparation-product-pin-v1.schema.json → preparation-product-pin-v2.schema.json} +59 -9
  35. package/dist/market-data-preparation/schemas/promotion-receipt-v1.schema.json +5 -1
  36. package/dist/market-data-preparation/schemas/source-forensics-ledger-v1.schema.json +463 -0
  37. package/dist/market-data-preparation/schemas/source-qualification-record-v1.schema.json +394 -0
  38. package/dist/market-data-preparation.d.ts +7 -0
  39. package/dist/market-data-preparation.js +35443 -25203
  40. package/dist/market-data-preparation.js.map +118 -78
  41. package/dist/market-data-vendor-backfill.d.ts +8 -5
  42. package/dist/market-data-vendor-backfill.js +19444 -15553
  43. package/dist/market-data-vendor-backfill.js.map +65 -60
  44. package/package.json +1 -5
  45. package/dist/market-data-preparation/policies/capability-policy-v1.json +0 -28
  46. package/dist/market-data-vendor-backfill/fixtures/conformance-v1.json +0 -448
  47. package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +0 -28
  48. package/dist/market-data-vendor-backfill/policies/capability-policy.json +0 -28
  49. package/dist/market-data-vendor-backfill/policies/resource-policy.json +0 -16
  50. package/dist/market-data-vendor-backfill/schema-manifest.json +0 -31
  51. package/dist/market-data-vendor-backfill/schemas/archive-selection.schema.json +0 -224
  52. package/dist/market-data-vendor-backfill/schemas/promotion-receipt.schema.json +0 -171
  53. package/dist/market-data-vendor-backfill/schemas/request.schema.json +0 -166
  54. package/dist/market-data-vendor-backfill/schemas/required-clock.schema.json +0 -47
  55. package/dist/market-data-vendor-backfill/schemas/result.schema.json +0 -198
@@ -318611,7 +318611,7 @@ function commonEvidenceFields(context2, rawCapture, input) {
318611
318611
  source_time_ms: input.eventTimeMs,
318612
318612
  received_time_ms: input.receivedTimeMs,
318613
318613
  snapshot_id: input.snapshotId,
318614
- construction_mode: "sampled_top_n_snapshot",
318614
+ construction_mode: input.constructionMode,
318615
318615
  gap_policy: "record_gap",
318616
318616
  depth_limit: input.depthLimit,
318617
318617
  sequence: input.sequence,
@@ -318665,7 +318665,8 @@ function buildCanonicalOrderBookRows(input) {
318665
318665
  sequence,
318666
318666
  depthLimit: input.depthLimit,
318667
318667
  eventTimeMs,
318668
- receivedTimeMs
318668
+ receivedTimeMs,
318669
+ constructionMode: input.constructionMode ?? "sampled_top_n_snapshot"
318669
318670
  });
318670
318671
  const levels = [
318671
318672
  ["bid", bids],
@@ -335154,6 +335155,24 @@ function selectBrokerAccountForCex(normalizedCex, brokers, metadata) {
335154
335155
  return selectBrokerAccount(brokers[normalizedCex], metadata) ?? undefined;
335155
335156
  }
335156
335157
 
335158
+ // src/helpers/trace-context.ts
335159
+ var TRACE_METADATA_KEY = "x-trace-id";
335160
+ var MAX_TRACE_METADATA_LENGTH = 128;
335161
+ var OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
335162
+ var UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
335163
+ var ZERO_OTEL_TRACE_ID = "00000000000000000000000000000000";
335164
+ function extractTraceId(metadata) {
335165
+ const raw = metadata.get(TRACE_METADATA_KEY)[0];
335166
+ if (typeof raw !== "string" || raw.length > MAX_TRACE_METADATA_LENGTH) {
335167
+ return;
335168
+ }
335169
+ const traceId = raw.trim();
335170
+ if (OTEL_TRACE_ID_PATTERN.test(traceId) && traceId !== ZERO_OTEL_TRACE_ID || UUID_V4_PATTERN.test(traceId)) {
335171
+ return traceId;
335172
+ }
335173
+ return;
335174
+ }
335175
+
335157
335176
  // src/handlers/execute-action/order-book-call.ts
335158
335177
  var grpc4 = __toESM(require_src3(), 1);
335159
335178
  async function handleOrderBookCall(ctx) {
@@ -336457,6 +336476,15 @@ function isPublicMarketDataAction(action, payload) {
336457
336476
  return false;
336458
336477
  return isOrderBookCallMethod(payload?.method ?? payload?.functionName);
336459
336478
  }
336479
+ function grpcStatusName(error48) {
336480
+ if (!error48) {
336481
+ return "OK";
336482
+ }
336483
+ if (typeof error48.code !== "number") {
336484
+ return "UNKNOWN";
336485
+ }
336486
+ return grpc12.status[error48.code] ?? "UNKNOWN";
336487
+ }
336460
336488
  function createExecuteActionHandler(deps) {
336461
336489
  const {
336462
336490
  policy,
@@ -336473,12 +336501,28 @@ function createExecuteActionHandler(deps) {
336473
336501
  const startTime = Date.now();
336474
336502
  const { action: rawAction, cex: cex3, symbol: symbol2 } = call.request;
336475
336503
  const action = resolveAction(rawAction);
336504
+ const actionName = getActionName(action);
336505
+ const operationalCex = cex3?.trim().toLowerCase() || "unknown";
336506
+ const traceId = extractTraceId(call.metadata);
336507
+ const traceFields = traceId === undefined ? {} : { trace_id: traceId };
336476
336508
  let actionCompleted = false;
336477
336509
  const wrappedCallback = (error48, value) => {
336478
336510
  if (!actionCompleted) {
336479
336511
  actionCompleted = true;
336480
336512
  const latency = Date.now() - startTime;
336481
- const actionName = getActionName(action);
336513
+ const terminalFields = {
336514
+ action: actionName,
336515
+ cex: operationalCex,
336516
+ latency_ms: latency,
336517
+ outcome: error48 ? "error" : "success",
336518
+ grpc_status: grpcStatusName(error48),
336519
+ ...traceFields
336520
+ };
336521
+ if (error48) {
336522
+ log.withMetadata(terminalFields).error("ExecuteAction failed");
336523
+ } else {
336524
+ log.withMetadata(terminalFields).info("ExecuteAction completed");
336525
+ }
336482
336526
  otelMetrics?.recordHistogram("execute_action_duration_ms", latency, {
336483
336527
  action: actionName,
336484
336528
  cex: cex3 || "unknown"
@@ -336499,9 +336543,13 @@ function createExecuteActionHandler(deps) {
336499
336543
  callback(error48, value);
336500
336544
  };
336501
336545
  try {
336502
- log.info(`Request - ExecuteAction:`, { action, cex: cex3, symbol: symbol2 });
336546
+ log.withMetadata({
336547
+ action: actionName,
336548
+ cex: operationalCex,
336549
+ ...traceFields
336550
+ }).info("ExecuteAction started");
336503
336551
  otelMetrics?.recordCounter("execute_action_requests_total", 1, {
336504
- action: getActionName(action),
336552
+ action: actionName,
336505
336553
  cex: cex3 || "unknown"
336506
336554
  });
336507
336555
  if (!authenticateRequest(call, whitelistIps)) {
@@ -336739,6 +336787,9 @@ async function writeSubscribeError(call, isClosed, frame) {
336739
336787
  call.end();
336740
336788
  }
336741
336789
  }
336790
+ function grpcStatusName2(status14) {
336791
+ return typeof status14 === "number" ? grpc13.status[status14] ?? "UNKNOWN" : "UNKNOWN";
336792
+ }
336742
336793
  function getBinanceEventMarketId(event) {
336743
336794
  const value = event.s;
336744
336795
  return typeof value === "string" ? value : null;
@@ -336871,6 +336922,24 @@ function createSubscribeHandler(deps) {
336871
336922
  });
336872
336923
  return async (call) => {
336873
336924
  const subscribeStartTime = Date.now();
336925
+ const request = call.request;
336926
+ const { cex: cex3, symbol: symbol2, type: type2 } = request;
336927
+ const metadata = call.metadata;
336928
+ const traceId = extractTraceId(metadata);
336929
+ const traceFields = traceId === undefined ? {} : { trace_id: traceId };
336930
+ const operationalCex = cex3?.trim().toLowerCase() || "unknown";
336931
+ const operationalSymbol = symbol2?.trim() || "unknown";
336932
+ const subscriptionType2 = resolveSubscriptionType(type2);
336933
+ const subscriptionTypeName = getSubscriptionTypeName(subscriptionType2);
336934
+ const operationalFields = {
336935
+ cex: operationalCex,
336936
+ symbol: operationalSymbol,
336937
+ subscription_type: subscriptionTypeName,
336938
+ ...traceFields
336939
+ };
336940
+ let terminalLogged = false;
336941
+ let terminalOutcome = "completed";
336942
+ let terminalGrpcStatus = "OK";
336874
336943
  let streamClosed = false;
336875
336944
  let ownedBroker = null;
336876
336945
  let ownedBrokerClosePromise;
@@ -336893,12 +336962,41 @@ function createSubscribeHandler(deps) {
336893
336962
  const closeOwnedBrokerOnCallEnd = () => {
336894
336963
  closeOwnedBroker();
336895
336964
  };
336896
- call.once("cancelled", markStreamClosed);
336965
+ const logTerminalOutcome = (outcome) => {
336966
+ if (terminalLogged) {
336967
+ return;
336968
+ }
336969
+ terminalLogged = true;
336970
+ const fields = {
336971
+ ...operationalFields,
336972
+ duration_ms: Date.now() - subscribeStartTime,
336973
+ outcome,
336974
+ grpc_status: outcome === "cancelled" ? "CANCELLED" : terminalGrpcStatus
336975
+ };
336976
+ if (outcome === "error") {
336977
+ log.withMetadata(fields).error("Subscribe failed");
336978
+ } else if (outcome === "cancelled") {
336979
+ log.withMetadata(fields).info("Subscribe cancelled");
336980
+ } else {
336981
+ log.withMetadata(fields).info("Subscribe ended");
336982
+ }
336983
+ };
336984
+ const writeTerminalError = async (frame, status14 = grpc13.status.UNKNOWN) => {
336985
+ terminalOutcome = "error";
336986
+ terminalGrpcStatus = grpcStatusName2(status14);
336987
+ await writeSubscribeError(call, isStreamClosed, frame);
336988
+ };
336989
+ log.withMetadata(operationalFields).info("Subscribe started");
336990
+ call.once("cancelled", () => {
336991
+ terminalGrpcStatus = "CANCELLED";
336992
+ markStreamClosed();
336993
+ logTerminalOutcome("cancelled");
336994
+ });
336897
336995
  call.once("cancelled", closeOwnedBrokerOnCallEnd);
336898
336996
  call.once("error", closeOwnedBrokerOnCallEnd);
336899
336997
  call.once("end", () => {
336900
336998
  markStreamClosed();
336901
- log.info("Subscribe stream ended");
336999
+ logTerminalOutcome(terminalOutcome);
336902
337000
  const duration3 = Date.now() - subscribeStartTime;
336903
337001
  otelMetrics?.recordHistogram("subscribe_duration_ms", duration3, {
336904
337002
  cex: call.request?.cex || "unknown",
@@ -336907,12 +337005,16 @@ function createSubscribeHandler(deps) {
336907
337005
  });
336908
337006
  call.once("error", (error48) => {
336909
337007
  markStreamClosed();
336910
- log.error("Subscribe stream error:", error48);
337008
+ terminalOutcome = "error";
337009
+ terminalGrpcStatus = grpcStatusName2(typeof error48 === "object" && error48 !== null && "code" in error48 ? error48.code : undefined);
337010
+ logTerminalOutcome(call.cancelled ? "cancelled" : "error");
336911
337011
  otelMetrics?.recordCounter("subscribe_errors_total", 1, {
336912
337012
  error_type: error48 instanceof Error ? error48.message : "unknown"
336913
337013
  });
336914
337014
  });
336915
337015
  if (!authenticateRequest(call, whitelistIps)) {
337016
+ terminalOutcome = "error";
337017
+ terminalGrpcStatus = "PERMISSION_DENIED";
336916
337018
  otelMetrics?.recordCounter("subscribe_errors_total", 1, {
336917
337019
  error_type: "permission_denied"
336918
337020
  });
@@ -336923,32 +337025,22 @@ function createSubscribeHandler(deps) {
336923
337025
  call.destroy(new Error("Access denied: Unauthorized IP"));
336924
337026
  return;
336925
337027
  }
336926
- const metadata = call.metadata;
336927
- let subscriptionType2 = SubscriptionType.ORDERBOOK;
336928
337028
  try {
336929
- const request = call.request;
336930
- const { cex: cex3, symbol: symbol2, type: type2, options } = request;
336931
- subscriptionType2 = resolveSubscriptionType(type2);
336932
- log.info(`Request - Subscribe:`, {
336933
- cex: request.cex,
336934
- symbol: request.symbol,
336935
- type: subscriptionType2
336936
- });
336937
- const subscriptionTypeName = getSubscriptionTypeName(subscriptionType2);
337029
+ const { options } = request;
336938
337030
  otelMetrics?.recordCounter("subscribe_requests_total", 1, {
336939
337031
  cex: cex3 || "unknown",
336940
337032
  symbol: symbol2 || "unknown",
336941
337033
  type: subscriptionTypeName
336942
337034
  });
336943
337035
  if (!cex3 || !symbol2) {
336944
- await writeSubscribeError(call, isStreamClosed, {
337036
+ await writeTerminalError({
336945
337037
  data: JSON.stringify({
336946
337038
  error: "cex, symbol, and type are required"
336947
337039
  }),
336948
337040
  timestamp: Date.now(),
336949
337041
  symbol: symbol2 || "",
336950
337042
  type: subscriptionType2
336951
- });
337043
+ }, grpc13.status.INVALID_ARGUMENT);
336952
337044
  return;
336953
337045
  }
336954
337046
  if (isPublicMarketDataSubscription(subscriptionType2)) {
@@ -336977,7 +337069,7 @@ function createSubscribeHandler(deps) {
336977
337069
  } catch (error48) {
336978
337070
  const message = getErrorMessage(error48);
336979
337071
  if (!isStreamClosed()) {
336980
- await writeSubscribeError(call, isStreamClosed, {
337072
+ await writeTerminalError({
336981
337073
  data: JSON.stringify({ error: message }),
336982
337074
  timestamp: Date.now(),
336983
337075
  symbol: symbol2,
@@ -336998,14 +337090,14 @@ function createSubscribeHandler(deps) {
336998
337090
  const selectedBroker = selectedBrokerAccount?.exchange ?? createBroker(normalizedCex, metadata);
336999
337091
  const broker = selectedBroker ?? createPublicBroker(normalizedCex);
337000
337092
  if (!broker) {
337001
- await writeSubscribeError(call, isStreamClosed, {
337093
+ await writeTerminalError({
337002
337094
  data: JSON.stringify({
337003
337095
  error: "Exchange not registered and no API metadata found"
337004
337096
  }),
337005
337097
  timestamp: Date.now(),
337006
337098
  symbol: symbol2,
337007
337099
  type: subscriptionType2
337008
- });
337100
+ }, grpc13.status.NOT_FOUND);
337009
337101
  return;
337010
337102
  }
337011
337103
  if (!selectedBrokerAccount) {
@@ -337033,28 +337125,28 @@ function createSubscribeHandler(deps) {
337033
337125
  if (isBinanceSpotAccountSubscription(normalizedCex, subscriptionType2, options?.marketType)) {
337034
337126
  const accountBroker = selectedBrokerAccount?.exchange ?? selectedBroker;
337035
337127
  if (!accountBroker) {
337036
- await writeSubscribeError(call, isStreamClosed, {
337128
+ await writeTerminalError({
337037
337129
  data: JSON.stringify({
337038
337130
  error: "Binance account subscriptions require API credentials"
337039
337131
  }),
337040
337132
  timestamp: Date.now(),
337041
337133
  symbol: resolvedSymbol,
337042
337134
  type: subscriptionType2
337043
- });
337135
+ }, grpc13.status.FAILED_PRECONDITION);
337044
337136
  return;
337045
337137
  }
337046
337138
  const marketId = subscriptionType2 === SubscriptionType.ORDERS ? await getBinanceMarketId(accountBroker, resolvedSymbol) : undefined;
337047
337139
  let userDataSource;
337048
337140
  if (selectedBrokerAccount) {
337049
337141
  if (!userDataStreamSupervisor) {
337050
- await writeSubscribeError(call, isStreamClosed, {
337142
+ await writeTerminalError({
337051
337143
  data: JSON.stringify({
337052
337144
  error: "Configured account user-data supervisor is unavailable"
337053
337145
  }),
337054
337146
  timestamp: Date.now(),
337055
337147
  symbol: resolvedSymbol,
337056
337148
  type: subscriptionType2
337057
- });
337149
+ }, grpc13.status.FAILED_PRECONDITION);
337058
337150
  return;
337059
337151
  }
337060
337152
  userDataSource = userDataStreamSupervisor.subscribe({
@@ -337077,7 +337169,7 @@ function createSubscribeHandler(deps) {
337077
337169
  } catch (error48) {
337078
337170
  const message = getErrorMessage(error48);
337079
337171
  log.error(`Error fetching balance for ${cex3}:`, error48);
337080
- await writeSubscribeError(call, isStreamClosed, {
337172
+ await writeTerminalError({
337081
337173
  data: JSON.stringify({
337082
337174
  error: `Failed to fetch balance: ${message}`
337083
337175
  }),
@@ -337096,7 +337188,7 @@ function createSubscribeHandler(deps) {
337096
337188
  } catch (error48) {
337097
337189
  log.error(`Error fetching orders for ${resolvedSymbol} on ${cex3}:`, error48);
337098
337190
  const message = getErrorMessage(error48);
337099
- await writeSubscribeError(call, isStreamClosed, {
337191
+ await writeTerminalError({
337100
337192
  data: JSON.stringify({
337101
337193
  error: `Failed to fetch orders: ${message}`
337102
337194
  }),
@@ -337107,26 +337199,27 @@ function createSubscribeHandler(deps) {
337107
337199
  }
337108
337200
  break;
337109
337201
  default:
337110
- await writeSubscribeError(call, isStreamClosed, {
337202
+ await writeTerminalError({
337111
337203
  data: JSON.stringify({ error: "Invalid subscription type" }),
337112
337204
  timestamp: Date.now(),
337113
337205
  symbol: symbol2,
337114
337206
  type: subscriptionType2
337115
- });
337207
+ }, grpc13.status.INVALID_ARGUMENT);
337116
337208
  }
337117
337209
  } catch (error48) {
337118
337210
  log.error("Error in Subscribe stream:", error48);
337119
337211
  const message = getErrorMessage(error48);
337120
- await writeSubscribeError(call, isStreamClosed, {
337212
+ await writeTerminalError({
337121
337213
  data: JSON.stringify({ error: `Internal server error: ${message}` }),
337122
337214
  timestamp: Date.now(),
337123
337215
  symbol: "",
337124
337216
  type: subscriptionType2
337125
- });
337217
+ }, grpc13.status.INTERNAL);
337126
337218
  } finally {
337127
337219
  call.off("cancelled", closeOwnedBrokerOnCallEnd);
337128
337220
  call.off("error", closeOwnedBrokerOnCallEnd);
337129
337221
  await closeOwnedBroker();
337222
+ logTerminalOutcome(call.cancelled ? "cancelled" : terminalOutcome);
337130
337223
  }
337131
337224
  };
337132
337225
  }