@usherlabs/cex-broker 0.2.47 → 0.2.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8257,241 +8257,6 @@ var require_src = __commonJS((exports) => {
8257
8257
  };
8258
8258
  });
8259
8259
 
8260
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/types/LogRecord.js
8261
- var require_LogRecord = __commonJS((exports) => {
8262
- Object.defineProperty(exports, "__esModule", { value: true });
8263
- exports.SeverityNumber = undefined;
8264
- var SeverityNumber;
8265
- (function(SeverityNumber2) {
8266
- SeverityNumber2[SeverityNumber2["UNSPECIFIED"] = 0] = "UNSPECIFIED";
8267
- SeverityNumber2[SeverityNumber2["TRACE"] = 1] = "TRACE";
8268
- SeverityNumber2[SeverityNumber2["TRACE2"] = 2] = "TRACE2";
8269
- SeverityNumber2[SeverityNumber2["TRACE3"] = 3] = "TRACE3";
8270
- SeverityNumber2[SeverityNumber2["TRACE4"] = 4] = "TRACE4";
8271
- SeverityNumber2[SeverityNumber2["DEBUG"] = 5] = "DEBUG";
8272
- SeverityNumber2[SeverityNumber2["DEBUG2"] = 6] = "DEBUG2";
8273
- SeverityNumber2[SeverityNumber2["DEBUG3"] = 7] = "DEBUG3";
8274
- SeverityNumber2[SeverityNumber2["DEBUG4"] = 8] = "DEBUG4";
8275
- SeverityNumber2[SeverityNumber2["INFO"] = 9] = "INFO";
8276
- SeverityNumber2[SeverityNumber2["INFO2"] = 10] = "INFO2";
8277
- SeverityNumber2[SeverityNumber2["INFO3"] = 11] = "INFO3";
8278
- SeverityNumber2[SeverityNumber2["INFO4"] = 12] = "INFO4";
8279
- SeverityNumber2[SeverityNumber2["WARN"] = 13] = "WARN";
8280
- SeverityNumber2[SeverityNumber2["WARN2"] = 14] = "WARN2";
8281
- SeverityNumber2[SeverityNumber2["WARN3"] = 15] = "WARN3";
8282
- SeverityNumber2[SeverityNumber2["WARN4"] = 16] = "WARN4";
8283
- SeverityNumber2[SeverityNumber2["ERROR"] = 17] = "ERROR";
8284
- SeverityNumber2[SeverityNumber2["ERROR2"] = 18] = "ERROR2";
8285
- SeverityNumber2[SeverityNumber2["ERROR3"] = 19] = "ERROR3";
8286
- SeverityNumber2[SeverityNumber2["ERROR4"] = 20] = "ERROR4";
8287
- SeverityNumber2[SeverityNumber2["FATAL"] = 21] = "FATAL";
8288
- SeverityNumber2[SeverityNumber2["FATAL2"] = 22] = "FATAL2";
8289
- SeverityNumber2[SeverityNumber2["FATAL3"] = 23] = "FATAL3";
8290
- SeverityNumber2[SeverityNumber2["FATAL4"] = 24] = "FATAL4";
8291
- })(SeverityNumber = exports.SeverityNumber || (exports.SeverityNumber = {}));
8292
- });
8293
-
8294
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLogger.js
8295
- var require_NoopLogger = __commonJS((exports) => {
8296
- Object.defineProperty(exports, "__esModule", { value: true });
8297
- exports.NOOP_LOGGER = exports.NoopLogger = undefined;
8298
-
8299
- class NoopLogger {
8300
- emit(_logRecord) {}
8301
- }
8302
- exports.NoopLogger = NoopLogger;
8303
- exports.NOOP_LOGGER = new NoopLogger;
8304
- });
8305
-
8306
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLoggerProvider.js
8307
- var require_NoopLoggerProvider = __commonJS((exports) => {
8308
- Object.defineProperty(exports, "__esModule", { value: true });
8309
- exports.NOOP_LOGGER_PROVIDER = exports.NoopLoggerProvider = undefined;
8310
- var NoopLogger_1 = require_NoopLogger();
8311
-
8312
- class NoopLoggerProvider {
8313
- getLogger(_name, _version, _options) {
8314
- return new NoopLogger_1.NoopLogger;
8315
- }
8316
- }
8317
- exports.NoopLoggerProvider = NoopLoggerProvider;
8318
- exports.NOOP_LOGGER_PROVIDER = new NoopLoggerProvider;
8319
- });
8320
-
8321
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLogger.js
8322
- var require_ProxyLogger = __commonJS((exports) => {
8323
- Object.defineProperty(exports, "__esModule", { value: true });
8324
- exports.ProxyLogger = undefined;
8325
- var NoopLogger_1 = require_NoopLogger();
8326
-
8327
- class ProxyLogger {
8328
- constructor(_provider, name, version2, options) {
8329
- this._provider = _provider;
8330
- this.name = name;
8331
- this.version = version2;
8332
- this.options = options;
8333
- }
8334
- emit(logRecord) {
8335
- this._getLogger().emit(logRecord);
8336
- }
8337
- _getLogger() {
8338
- if (this._delegate) {
8339
- return this._delegate;
8340
- }
8341
- const logger = this._provider._getDelegateLogger(this.name, this.version, this.options);
8342
- if (!logger) {
8343
- return NoopLogger_1.NOOP_LOGGER;
8344
- }
8345
- this._delegate = logger;
8346
- return this._delegate;
8347
- }
8348
- }
8349
- exports.ProxyLogger = ProxyLogger;
8350
- });
8351
-
8352
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLoggerProvider.js
8353
- var require_ProxyLoggerProvider = __commonJS((exports) => {
8354
- Object.defineProperty(exports, "__esModule", { value: true });
8355
- exports.ProxyLoggerProvider = undefined;
8356
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
8357
- var ProxyLogger_1 = require_ProxyLogger();
8358
-
8359
- class ProxyLoggerProvider {
8360
- getLogger(name, version2, options) {
8361
- var _a2;
8362
- return (_a2 = this._getDelegateLogger(name, version2, options)) !== null && _a2 !== undefined ? _a2 : new ProxyLogger_1.ProxyLogger(this, name, version2, options);
8363
- }
8364
- _getDelegate() {
8365
- var _a2;
8366
- return (_a2 = this._delegate) !== null && _a2 !== undefined ? _a2 : NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
8367
- }
8368
- _setDelegate(delegate) {
8369
- this._delegate = delegate;
8370
- }
8371
- _getDelegateLogger(name, version2, options) {
8372
- var _a2;
8373
- return (_a2 = this._delegate) === null || _a2 === undefined ? undefined : _a2.getLogger(name, version2, options);
8374
- }
8375
- }
8376
- exports.ProxyLoggerProvider = ProxyLoggerProvider;
8377
- });
8378
-
8379
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/globalThis.js
8380
- var require_globalThis2 = __commonJS((exports) => {
8381
- Object.defineProperty(exports, "__esModule", { value: true });
8382
- exports._globalThis = undefined;
8383
- exports._globalThis = typeof globalThis === "object" ? globalThis : global;
8384
- });
8385
-
8386
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/index.js
8387
- var require_node2 = __commonJS((exports) => {
8388
- Object.defineProperty(exports, "__esModule", { value: true });
8389
- exports._globalThis = undefined;
8390
- var globalThis_1 = require_globalThis2();
8391
- Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
8392
- return globalThis_1._globalThis;
8393
- } });
8394
- });
8395
-
8396
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/index.js
8397
- var require_platform2 = __commonJS((exports) => {
8398
- Object.defineProperty(exports, "__esModule", { value: true });
8399
- exports._globalThis = undefined;
8400
- var node_1 = require_node2();
8401
- Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
8402
- return node_1._globalThis;
8403
- } });
8404
- });
8405
-
8406
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/internal/global-utils.js
8407
- var require_global_utils2 = __commonJS((exports) => {
8408
- Object.defineProperty(exports, "__esModule", { value: true });
8409
- exports.API_BACKWARDS_COMPATIBILITY_VERSION = exports.makeGetter = exports._global = exports.GLOBAL_LOGS_API_KEY = undefined;
8410
- var platform_1 = require_platform2();
8411
- exports.GLOBAL_LOGS_API_KEY = Symbol.for("io.opentelemetry.js.api.logs");
8412
- exports._global = platform_1._globalThis;
8413
- function makeGetter(requiredVersion, instance, fallback) {
8414
- return (version2) => version2 === requiredVersion ? instance : fallback;
8415
- }
8416
- exports.makeGetter = makeGetter;
8417
- exports.API_BACKWARDS_COMPATIBILITY_VERSION = 1;
8418
- });
8419
-
8420
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/api/logs.js
8421
- var require_logs = __commonJS((exports) => {
8422
- Object.defineProperty(exports, "__esModule", { value: true });
8423
- exports.LogsAPI = undefined;
8424
- var global_utils_1 = require_global_utils2();
8425
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
8426
- var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
8427
-
8428
- class LogsAPI {
8429
- constructor() {
8430
- this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
8431
- }
8432
- static getInstance() {
8433
- if (!this._instance) {
8434
- this._instance = new LogsAPI;
8435
- }
8436
- return this._instance;
8437
- }
8438
- setGlobalLoggerProvider(provider) {
8439
- if (global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) {
8440
- return this.getLoggerProvider();
8441
- }
8442
- global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY] = (0, global_utils_1.makeGetter)(global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION, provider, NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER);
8443
- this._proxyLoggerProvider._setDelegate(provider);
8444
- return provider;
8445
- }
8446
- getLoggerProvider() {
8447
- var _a2, _b;
8448
- return (_b = (_a2 = global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) === null || _a2 === undefined ? undefined : _a2.call(global_utils_1._global, global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION)) !== null && _b !== undefined ? _b : this._proxyLoggerProvider;
8449
- }
8450
- getLogger(name, version2, options) {
8451
- return this.getLoggerProvider().getLogger(name, version2, options);
8452
- }
8453
- disable() {
8454
- delete global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY];
8455
- this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
8456
- }
8457
- }
8458
- exports.LogsAPI = LogsAPI;
8459
- });
8460
-
8461
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/index.js
8462
- var require_src2 = __commonJS((exports) => {
8463
- Object.defineProperty(exports, "__esModule", { value: true });
8464
- exports.logs = exports.ProxyLoggerProvider = exports.ProxyLogger = exports.NoopLoggerProvider = exports.NOOP_LOGGER_PROVIDER = exports.NoopLogger = exports.NOOP_LOGGER = exports.SeverityNumber = undefined;
8465
- var LogRecord_1 = require_LogRecord();
8466
- Object.defineProperty(exports, "SeverityNumber", { enumerable: true, get: function() {
8467
- return LogRecord_1.SeverityNumber;
8468
- } });
8469
- var NoopLogger_1 = require_NoopLogger();
8470
- Object.defineProperty(exports, "NOOP_LOGGER", { enumerable: true, get: function() {
8471
- return NoopLogger_1.NOOP_LOGGER;
8472
- } });
8473
- Object.defineProperty(exports, "NoopLogger", { enumerable: true, get: function() {
8474
- return NoopLogger_1.NoopLogger;
8475
- } });
8476
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
8477
- Object.defineProperty(exports, "NOOP_LOGGER_PROVIDER", { enumerable: true, get: function() {
8478
- return NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
8479
- } });
8480
- Object.defineProperty(exports, "NoopLoggerProvider", { enumerable: true, get: function() {
8481
- return NoopLoggerProvider_1.NoopLoggerProvider;
8482
- } });
8483
- var ProxyLogger_1 = require_ProxyLogger();
8484
- Object.defineProperty(exports, "ProxyLogger", { enumerable: true, get: function() {
8485
- return ProxyLogger_1.ProxyLogger;
8486
- } });
8487
- var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
8488
- Object.defineProperty(exports, "ProxyLoggerProvider", { enumerable: true, get: function() {
8489
- return ProxyLoggerProvider_1.ProxyLoggerProvider;
8490
- } });
8491
- var logs_1 = require_logs();
8492
- exports.logs = logs_1.LogsAPI.getInstance();
8493
- });
8494
-
8495
8260
  // src/commands/market-data-vendor-backfill.ts
8496
8261
  import { createHash as createHash5, randomUUID as randomUUID2 } from "node:crypto";
8497
8262
  import path2 from "node:path";
@@ -22254,102 +22019,6 @@ var BaseTransport = class {
22254
22019
  return this.logger;
22255
22020
  }
22256
22021
  };
22257
- var LoggerlessTransport = class {
22258
- id;
22259
- enabled;
22260
- level;
22261
- consoleDebug;
22262
- constructor(config2) {
22263
- this.id = Date.now().toString() + Math.random().toString();
22264
- this.enabled = config2.enabled ?? true;
22265
- this.consoleDebug = config2.consoleDebug ?? false;
22266
- this.level = config2.level ?? "trace";
22267
- }
22268
- _sendToLogger(params) {
22269
- if (!this.enabled)
22270
- return;
22271
- if (LogLevelPriority[params.logLevel] < LogLevelPriority[this.level])
22272
- return;
22273
- const messages = this.shipToLogger(params);
22274
- if (this.consoleDebug)
22275
- switch (params.logLevel) {
22276
- case LogLevel.info:
22277
- console.info(...messages);
22278
- break;
22279
- case LogLevel.warn:
22280
- console.warn(...messages);
22281
- break;
22282
- case LogLevel.error:
22283
- console.error(...messages);
22284
- break;
22285
- case LogLevel.trace:
22286
- console.debug(...messages);
22287
- break;
22288
- case LogLevel.debug:
22289
- console.debug(...messages);
22290
- break;
22291
- case LogLevel.fatal:
22292
- console.debug(...messages);
22293
- break;
22294
- default:
22295
- console.log(...messages);
22296
- }
22297
- }
22298
- getLoggerInstance() {
22299
- throw new Error("This transport does not have a logger instance");
22300
- }
22301
- };
22302
-
22303
- // node_modules/@loglayer/transport-opentelemetry/dist/index.mjs
22304
- var import_api_logs = __toESM(require_src2(), 1);
22305
- var logLayerLevels = {
22306
- fatal: import_api_logs.SeverityNumber.FATAL,
22307
- error: import_api_logs.SeverityNumber.ERROR,
22308
- warn: import_api_logs.SeverityNumber.WARN,
22309
- info: import_api_logs.SeverityNumber.INFO,
22310
- debug: import_api_logs.SeverityNumber.DEBUG,
22311
- trace: import_api_logs.SeverityNumber.TRACE
22312
- };
22313
- function getSeverityNumber(level) {
22314
- return logLayerLevels[level] ?? import_api_logs.SeverityNumber.TRACE;
22315
- }
22316
- function emitLogRecord(record2, logger) {
22317
- const { message, level, ...splat } = record2;
22318
- const attributes = {};
22319
- for (const key in splat)
22320
- if (Object.hasOwn(splat, key))
22321
- attributes[key] = splat[key];
22322
- const logRecord = {
22323
- severityNumber: getSeverityNumber(level),
22324
- severityText: level,
22325
- body: message,
22326
- attributes
22327
- };
22328
- logger.emit(logRecord);
22329
- }
22330
- var OpenTelemetryTransport = class extends LoggerlessTransport {
22331
- _logger;
22332
- onError;
22333
- constructor(config2 = {}) {
22334
- super(config2);
22335
- this._logger = import_api_logs.logs.getLogger("loglayer");
22336
- this.onError = config2.onError;
22337
- }
22338
- shipToLogger({ logLevel, messages, data, hasData }) {
22339
- const assembled = {
22340
- level: logLevel,
22341
- message: messages.join(" "),
22342
- ...hasData ? data : {}
22343
- };
22344
- try {
22345
- emitLogRecord(assembled, this._logger);
22346
- } catch (error48) {
22347
- if (this.onError)
22348
- this.onError(error48);
22349
- }
22350
- return [assembled];
22351
- }
22352
- };
22353
22022
 
22354
22023
  // node_modules/@loglayer/transport-tslog/dist/index.js
22355
22024
  var TsLogTransport = class extends BaseTransport {
@@ -24521,13 +24190,7 @@ var tslogLogger = new Logger({
24521
24190
  minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
24522
24191
  });
24523
24192
  var baseLogger = new LogLayer({
24524
- transport: [
24525
- new TsLogTransport({ id: "tslog", logger: tslogLogger }),
24526
- new OpenTelemetryTransport({
24527
- id: "otel",
24528
- enabled: true
24529
- })
24530
- ],
24193
+ transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
24531
24194
  plugins: [openTelemetryPlugin()],
24532
24195
  errorSerializer: serializeError
24533
24196
  });
@@ -25642,12 +25305,13 @@ var result_schema_default = {
25642
25305
  };
25643
25306
 
25644
25307
  // src/helpers/market-data-vendor-backfill/manifests.ts
25645
- var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
25308
+ var LEGACY_CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
25309
+ var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v2";
25646
25310
  var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
25647
25311
  var ADAPTER_POLICY_ID = "cryptohftdata-orderbook-adapter/v1";
25648
25312
  var ACQUISITION_POLICY_ID = "cryptohftdata-hourly-acquisition/v1";
25649
- var capabilityPolicyContent = {
25650
- policy_id: CAPABILITY_POLICY_ID,
25313
+ var legacyCapabilityPolicyContent = {
25314
+ policy_id: LEGACY_CAPABILITY_POLICY_ID,
25651
25315
  provider: "cryptohftdata",
25652
25316
  adapter_policy: {
25653
25317
  policy_id: ADAPTER_POLICY_ID,
@@ -25673,6 +25337,18 @@ var capabilityPolicyContent = {
25673
25337
  }
25674
25338
  ]
25675
25339
  };
25340
+ var LEGACY_CAPABILITY_POLICY = Object.freeze({
25341
+ ...legacyCapabilityPolicyContent,
25342
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent)
25343
+ });
25344
+ var capabilityPolicyContent = {
25345
+ ...legacyCapabilityPolicyContent,
25346
+ policy_id: CAPABILITY_POLICY_ID,
25347
+ profiles: legacyCapabilityPolicyContent.profiles.map((profile) => ({
25348
+ ...profile,
25349
+ source_policies: ["authoritative_window", "fill_gaps"]
25350
+ }))
25351
+ };
25676
25352
  var CAPABILITY_POLICY = Object.freeze({
25677
25353
  ...capabilityPolicyContent,
25678
25354
  policy_sha256: jcsSha256(capabilityPolicyContent)
@@ -25698,11 +25374,11 @@ var RESOURCE_POLICY = Object.freeze({
25698
25374
  });
25699
25375
  var EFFECTIVE_ADAPTER_POLICY_PIN = Object.freeze({
25700
25376
  policy_id: ADAPTER_POLICY_ID,
25701
- policy_sha256: jcsSha256(capabilityPolicyContent.adapter_policy)
25377
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent.adapter_policy)
25702
25378
  });
25703
25379
  var EFFECTIVE_ACQUISITION_POLICY_PIN = Object.freeze({
25704
25380
  policy_id: ACQUISITION_POLICY_ID,
25705
- policy_sha256: jcsSha256(capabilityPolicyContent.acquisition_policy)
25381
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent.acquisition_policy)
25706
25382
  });
25707
25383
  var schemaArtifacts = [
25708
25384
  ["schemas/request.schema.json", request_schema_default],
@@ -25866,7 +25542,11 @@ var backfillRequestCodec = {
25866
25542
  if (request.idempotency_key !== createBackfillIdempotencyKey(request)) {
25867
25543
  throw new Error("idempotency_key does not match canonical business fields");
25868
25544
  }
25869
- if (request.product_pins.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request.product_pins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256 || request.product_pins.resource_policy.policy_id !== RESOURCE_POLICY.policy_id || request.product_pins.resource_policy.policy_sha256 !== RESOURCE_POLICY.policy_sha256) {
25545
+ const capabilityPolicyMatches = [
25546
+ CAPABILITY_POLICY,
25547
+ LEGACY_CAPABILITY_POLICY
25548
+ ].some((policy) => request.product_pins.capability_policy.policy_id === policy.policy_id && request.product_pins.capability_policy.policy_sha256 === policy.policy_sha256);
25549
+ if (!capabilityPolicyMatches || request.product_pins.resource_policy.policy_id !== RESOURCE_POLICY.policy_id || request.product_pins.resource_policy.policy_sha256 !== RESOURCE_POLICY.policy_sha256) {
25870
25550
  throw new Error("request policy pins do not match the effective package policies");
25871
25551
  }
25872
25552
  if (request.required_clock.clock_id !== request.initial_selection.required_clock.clock_id || request.required_clock.clock_sha256 !== request.initial_selection.required_clock.clock_sha256 || request.required_clock.event_count !== request.initial_selection.required_clock.event_count) {
@@ -27319,12 +26999,13 @@ function assertStoredSelectionMatchesRequest(request, selection) {
27319
26999
  }
27320
27000
  }
27321
27001
  function eligibleBundles(request, bundles) {
27322
- return bundles.filter((bundle) => {
27323
- if (bundle.captureOrigin === "production_capture") {
27324
- return request.sourcePolicy === "fill_gaps";
27325
- }
27326
- return bundle.qualification?.state === "qualified";
27327
- });
27002
+ const qualifiedVendorBundles = bundles.filter((bundle) => bundle.captureOrigin === "vendor_historical_backfill" && bundle.qualification?.state === "qualified");
27003
+ const productionBundles = bundles.filter((bundle) => bundle.captureOrigin === "production_capture");
27004
+ if (request.sourcePolicy === "authoritative_window") {
27005
+ const vendorSupport = chooseSupport(request, qualifiedVendorBundles);
27006
+ return vendorSupport.length === request.requiredClockTargetsMs.length ? qualifiedVendorBundles : productionBundles;
27007
+ }
27008
+ return [...productionBundles, ...qualifiedVendorBundles];
27328
27009
  }
27329
27010
  function originPriority(request, origin) {
27330
27011
  if (request.sourcePolicy === "fill_gaps") {
@@ -27512,9 +27193,8 @@ function exactCandidateMatch(expected, actual) {
27512
27193
  function requestedShapeMatches(request, rows) {
27513
27194
  return rows.every(({ row }) => row.depth_limit === request.depth && row.construction_mode === request.constructionMode && row.schema_version === request.expectedProduct.canonicalSchemaVersion);
27514
27195
  }
27515
- function requiredClockCoverage(request, rows) {
27516
- const summaryTimes = rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => {
27517
- const value = row.source_time_ms;
27196
+ function requiredClockCoverage(request, rows, coverageSourceTimesMs) {
27197
+ const summaryTimes = (coverageSourceTimesMs ?? rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => row.source_time_ms)).map((value) => {
27518
27198
  if (typeof value !== "number" && (typeof value !== "string" || !/^\d+$/.test(value))) {
27519
27199
  return;
27520
27200
  }
@@ -27587,7 +27267,7 @@ function verifySemanticPromotion(evidence) {
27587
27267
  coverageVerified: false
27588
27268
  };
27589
27269
  }
27590
- const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows);
27270
+ const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows, evidence.coverageSourceTimesMs);
27591
27271
  if (!coverageVerified) {
27592
27272
  return {
27593
27273
  ...base,
@@ -27962,7 +27642,13 @@ class QualifiedOrderBookArchiveReader {
27962
27642
  suffixDigestBefore: baseline.verificationBaseline.suffixDigest,
27963
27643
  suffixDigestAfter: timelineDigest(suffix),
27964
27644
  seamVerified: this.seamIsOrdered(queriedRows),
27965
- exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string")
27645
+ exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string"),
27646
+ ...request.sourcePolicy === "fill_gaps" ? {
27647
+ coverageSourceTimesMs: [
27648
+ ...baseline.selection.support_anchors.map((anchor) => Date.parse(anchor.source_time)),
27649
+ ...summaries.map((row) => numberField(row, "source_time_ms"))
27650
+ ]
27651
+ } : {}
27966
27652
  });
27967
27653
  return { ...semantic, captureBundleId };
27968
27654
  }
@@ -28078,972 +27764,592 @@ function buildForwarderBatches(input) {
28078
27764
  return batches;
28079
27765
  }
28080
27766
 
28081
- // src/helpers/market-data-vendor-backfill/qualification.ts
28082
- var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
28083
- var SHA256 = /^[0-9a-f]{64}$/;
28084
- var REASON = /^[a-z][a-z0-9_]{0,127}$/;
28085
- var EVENT_FIELDS = new Set([
28086
- "qualification_event_id",
28087
- "capture_bundle_id",
28088
- "state",
28089
- "receipt_id",
28090
- "promotion_identity_sha256",
28091
- "window",
28092
- "event_at",
28093
- "reason_code"
28094
- ]);
28095
- function deterministicUuid(value) {
28096
- const digest = jcsSha256(value).slice(0, 32).split("");
28097
- digest[12] = "5";
28098
- digest[16] = "8";
28099
- const hex3 = digest.join("");
28100
- return `${hex3.slice(0, 8)}-${hex3.slice(8, 12)}-${hex3.slice(12, 16)}-${hex3.slice(16, 20)}-${hex3.slice(20)}`;
28101
- }
28102
- function semanticEvent(event) {
28103
- const { qualification_event_id: _eventId, ...semantic } = event;
28104
- return semantic;
28105
- }
28106
- function assertTimestamp(value, field) {
28107
- const parsed = Date.parse(value);
28108
- if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
28109
- throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
28110
- }
28111
- return parsed;
28112
- }
28113
- function parseQualificationEvent(value) {
28114
- if (!value || typeof value !== "object" || Array.isArray(value)) {
28115
- throw new Error("qualification event must be an object");
28116
- }
28117
- const event = value;
28118
- if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
28119
- throw new Error("qualification event fields are invalid");
28120
- }
28121
- if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
28122
- throw new Error("qualification window must be increasing");
28123
- }
28124
- assertTimestamp(event.event_at, "event_at");
28125
- if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
28126
- throw new Error("qualification_event_id does not match event content");
28127
- }
28128
- return event;
28129
- }
28130
- function finalizeQualificationEvent(event) {
28131
- const semantic = semanticEvent(event);
28132
- return parseQualificationEvent({
28133
- ...semantic,
28134
- qualification_event_id: deterministicUuid(semantic)
28135
- });
28136
- }
28137
- function qualificationEventToArchiveRow(eventInput) {
28138
- const event = parseQualificationEvent(eventInput);
28139
- return {
28140
- table: "market_data.cex_order_book_capture_qualifications",
28141
- row: {
28142
- source: "external_backfill",
28143
- capture_origin: "vendor_historical_backfill",
28144
- source_mode: "vendor_historical_backfill_v1",
28145
- deployment_id: "market-data-vendor-backfill",
28146
- qualification_event_id: event.qualification_event_id,
28147
- capture_bundle_id: event.capture_bundle_id,
28148
- state: event.state,
28149
- receipt_id: event.receipt_id,
28150
- promotion_identity_sha256: event.promotion_identity_sha256,
28151
- window_start_ms: Date.parse(event.window.start_at),
28152
- window_end_ms: Date.parse(event.window.end_at),
28153
- event_at_ms: Date.parse(event.event_at),
28154
- reason_code: event.reason_code,
28155
- event_json: jcsCanonicalize(event)
28156
- }
28157
- };
28158
- }
27767
+ // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
27768
+ import { createHash as createHash4 } from "node:crypto";
28159
27769
 
28160
- // src/helpers/market-data-vendor-backfill/core.ts
28161
- function outcome(request, status, reasonCode, extra = {}) {
28162
- return {
28163
- status,
28164
- reasonCode,
28165
- ...request ? {
28166
- requestId: request.requestId,
28167
- idempotencyKey: request.idempotencyKey,
28168
- ...request.target ? { target: request.target } : {}
28169
- } : {},
28170
- ...extra
28171
- };
28172
- }
28173
- function captureBundleId(request, capability, dataset) {
28174
- return sha256Canonical({
28175
- request_business_identity: request.idempotencyKey,
28176
- provider: capability.provider,
28177
- provider_exchange_id: capability.providerExchangeId,
28178
- resolved_symbol: capability.resolvedSymbol,
28179
- adapter_version: capability.adapterVersion,
28180
- objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
28181
- identity,
28182
- checksum,
28183
- bytes,
28184
- rows
28185
- })),
28186
- canonical_schema_version: request.expectedProduct.canonicalSchemaVersion,
28187
- checksum_algorithm: request.expectedProduct.checksumAlgorithm
28188
- });
28189
- }
28190
- function buildPromotionReceipt(request, capability, normalized, verification, verificationTimeMs) {
28191
- if (!request.wire || !request.initialSelection || !request.expectedCanonicalSchema || !request.coveragePolicy || !request.productPins) {
28192
- throw new Error("decoded final-v1 request context is missing");
27770
+ // node_modules/fzstd/esm/index.mjs
27771
+ var ab = ArrayBuffer;
27772
+ var u8 = Uint8Array;
27773
+ var u16 = Uint16Array;
27774
+ var i16 = Int16Array;
27775
+ var i32 = Int32Array;
27776
+ var slc = function(v, s, e) {
27777
+ if (u8.prototype.slice)
27778
+ return u8.prototype.slice.call(v, s, e);
27779
+ if (s == null || s < 0)
27780
+ s = 0;
27781
+ if (e == null || e > v.length)
27782
+ e = v.length;
27783
+ var n = new u8(e - s);
27784
+ n.set(v.subarray(s, e));
27785
+ return n;
27786
+ };
27787
+ var fill = function(v, n, s, e) {
27788
+ if (u8.prototype.fill)
27789
+ return u8.prototype.fill.call(v, n, s, e);
27790
+ if (s == null || s < 0)
27791
+ s = 0;
27792
+ if (e == null || e > v.length)
27793
+ e = v.length;
27794
+ for (;s < e; ++s)
27795
+ v[s] = n;
27796
+ return v;
27797
+ };
27798
+ var cpw = function(v, t, s, e) {
27799
+ if (u8.prototype.copyWithin)
27800
+ return u8.prototype.copyWithin.call(v, t, s, e);
27801
+ if (s == null || s < 0)
27802
+ s = 0;
27803
+ if (e == null || e > v.length)
27804
+ e = v.length;
27805
+ while (s < e) {
27806
+ v[t++] = v[s++];
28193
27807
  }
28194
- return finalizePromotionReceipt({
28195
- schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
28196
- verified_at: new Date(verificationTimeMs).toISOString(),
28197
- request_id: request.requestId,
28198
- idempotency_key: request.idempotencyKey,
28199
- source: EXTERNAL_BACKFILL_SOURCE,
28200
- capture_origin: "vendor_historical_backfill",
28201
- source_mode: "vendor_historical_backfill_v1",
28202
- provider: capability.provider,
28203
- adapter_version: capability.adapterVersion,
28204
- effective_policies: {
28205
- capability_policy: request.productPins.capability_policy,
28206
- resource_policy: request.productPins.resource_policy,
28207
- adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
28208
- acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
28209
- },
28210
- capture_bundle_id: normalized.captureBundleId,
28211
- scope: request.wire.scope,
28212
- window: request.wire.window,
28213
- depth: request.depth,
28214
- construction_mode: request.constructionMode,
28215
- canonical_schema: request.expectedCanonicalSchema,
28216
- coverage_policy: request.coveragePolicy,
28217
- selection_sha256: request.initialSelection.selection_sha256,
28218
- vendor_semantic_digest: normalized.vendorSemanticDigest,
28219
- canonical_semantic_digest: verification.canonicalSemanticDigest,
28220
- prefix_digest: verification.prefixDigest,
28221
- suffix_digest: verification.suffixDigest,
28222
- seam_verified: true,
28223
- coverage_verified: true,
28224
- dataset_objects: normalized.objects
28225
- });
28226
- }
28227
- async function submitAll(dependencies, batches) {
28228
- const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
28229
- for (const batch of batches) {
28230
- let accepted = false;
28231
- for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
28232
- try {
28233
- const response = await dependencies.forwarder.submit(batch);
28234
- accepted = response.ok && response.inserted === batch.rows.length;
28235
- } catch {
28236
- accepted = false;
28237
- }
28238
- if (accepted)
28239
- break;
28240
- if (attempt < maxAttempts)
28241
- await dependencies.retry?.wait?.(attempt);
27808
+ };
27809
+ var ec = [
27810
+ "invalid zstd data",
27811
+ "window size too large (>2046MB)",
27812
+ "invalid block type",
27813
+ "FSE accuracy too high",
27814
+ "match distance too far back",
27815
+ "unexpected EOF"
27816
+ ];
27817
+ var err = function(ind, msg, nt) {
27818
+ var e = new Error(msg || ec[ind]);
27819
+ e.code = ind;
27820
+ if (Error.captureStackTrace)
27821
+ Error.captureStackTrace(e, err);
27822
+ if (!nt)
27823
+ throw e;
27824
+ return e;
27825
+ };
27826
+ var rb = function(d, b, n) {
27827
+ var i = 0, o = 0;
27828
+ for (;i < n; ++i)
27829
+ o |= d[b++] << (i << 3);
27830
+ return o;
27831
+ };
27832
+ var b4 = function(d, b) {
27833
+ return (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0;
27834
+ };
27835
+ var rzfh = function(dat, w) {
27836
+ var n3 = dat[0] | dat[1] << 8 | dat[2] << 16;
27837
+ if (n3 == 3126568 && dat[3] == 253) {
27838
+ var flg = dat[4];
27839
+ var ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6;
27840
+ if (flg & 8)
27841
+ err(0);
27842
+ var bt = 6 - ss;
27843
+ var db = df == 3 ? 4 : df;
27844
+ var di = rb(dat, bt, db);
27845
+ bt += db;
27846
+ var fsb = fcf ? 1 << fcf : ss;
27847
+ var fss = rb(dat, bt, fsb) + (fcf == 1 && 256);
27848
+ var ws = fss;
27849
+ if (!ss) {
27850
+ var wb = 1 << 10 + (dat[5] >> 3);
27851
+ ws = wb + (wb >> 3) * (dat[5] & 7);
28242
27852
  }
28243
- if (!accepted)
28244
- return false;
27853
+ if (ws > 2145386496)
27854
+ err(1);
27855
+ var buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
27856
+ buf[0] = 1, buf[4] = 4, buf[8] = 8;
27857
+ return {
27858
+ b: bt + fsb,
27859
+ y: 0,
27860
+ l: 0,
27861
+ d: di,
27862
+ w: w && w != 1 ? w : buf.subarray(12),
27863
+ e: ws,
27864
+ o: new i32(buf.buffer, 0, 3),
27865
+ u: fss,
27866
+ c: cc,
27867
+ m: Math.min(131072, ws)
27868
+ };
27869
+ } else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
27870
+ return b4(dat, 4) + 8;
28245
27871
  }
28246
- return true;
28247
- }
28248
- function stableFailureReason(error48, fallback) {
28249
- if (error48 && typeof error48 === "object") {
28250
- const reason = error48.reason;
28251
- if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
28252
- return reason;
27872
+ err(0);
27873
+ };
27874
+ var msb = function(val) {
27875
+ var bits = 0;
27876
+ for (;1 << bits <= val; ++bits)
27877
+ ;
27878
+ return bits - 1;
27879
+ };
27880
+ var rfse = function(dat, bt, mal) {
27881
+ var tpos = (bt << 3) + 4;
27882
+ var al = (dat[bt] & 15) + 5;
27883
+ if (al > mal)
27884
+ err(3);
27885
+ var sz = 1 << al;
27886
+ var probs = sz, sym = -1, re = -1, i = -1, ht = sz;
27887
+ var buf = new ab(512 + (sz << 2));
27888
+ var freq = new i16(buf, 0, 256);
27889
+ var dstate = new u16(buf, 0, 256);
27890
+ var nstate = new u16(buf, 512, sz);
27891
+ var bb1 = 512 + (sz << 1);
27892
+ var syms = new u8(buf, bb1, sz);
27893
+ var nbits = new u8(buf, bb1 + sz);
27894
+ while (sym < 255 && probs > 0) {
27895
+ var bits = msb(probs + 1);
27896
+ var cbt = tpos >> 3;
27897
+ var msk = (1 << bits + 1) - 1;
27898
+ var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
27899
+ var msk1fb = (1 << bits) - 1;
27900
+ var msv = msk - probs - 1;
27901
+ var sval = val & msk1fb;
27902
+ if (sval < msv)
27903
+ tpos += bits, val = sval;
27904
+ else {
27905
+ tpos += bits + 1;
27906
+ if (val > msk1fb)
27907
+ val -= msv;
28253
27908
  }
28254
- }
28255
- return fallback;
28256
- }
28257
- function assertArchivePreflight(request, resolution) {
28258
- if (!request.target || !request.productionAuthorizationId) {
28259
- throw new Error("request target or production authorization ID is missing");
28260
- }
28261
- const selection = archiveSelectionCodec.decode(resolution.selection);
28262
- const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
28263
- const receiptById = new Map;
28264
- for (const receipt of receipts) {
28265
- const existing = receiptById.get(receipt.receipt_id);
28266
- if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
28267
- throw new Error("stored receipt identity has conflicting content");
27909
+ freq[++sym] = --val;
27910
+ if (val == -1) {
27911
+ probs += val;
27912
+ syms[--ht] = sym;
27913
+ } else
27914
+ probs -= val;
27915
+ if (!val) {
27916
+ do {
27917
+ var rbt = tpos >> 3;
27918
+ re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
27919
+ tpos += 2;
27920
+ sym += re;
27921
+ } while (re == 3);
28268
27922
  }
28269
- receiptById.set(receipt.receipt_id, receipt);
28270
27923
  }
28271
- for (const bundle of selection.bundles) {
28272
- if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
28273
- throw new Error("vendor selection lacks its validated stored receipt");
27924
+ if (sym > 255 || probs)
27925
+ err(0);
27926
+ var sympos = 0;
27927
+ var sstep = (sz >> 1) + (sz >> 3) + 3;
27928
+ var smask = sz - 1;
27929
+ for (var s = 0;s <= sym; ++s) {
27930
+ var sf = freq[s];
27931
+ if (sf < 1) {
27932
+ dstate[s] = -sf;
27933
+ continue;
27934
+ }
27935
+ for (i = 0;i < sf; ++i) {
27936
+ syms[sympos] = s;
27937
+ do {
27938
+ sympos = sympos + sstep & smask;
27939
+ } while (sympos >= ht);
28274
27940
  }
28275
27941
  }
28276
- if (resolution.readerIdentity.environment !== request.target.environment || resolution.readerIdentity.cluster !== request.target.cluster) {
28277
- throw new Error("archive reader cluster identity mismatch");
28278
- }
28279
- }
28280
- function assertForwarderPreflight(request, resolution, nowMs) {
28281
- if (!request.target || !request.productionAuthorizationId) {
28282
- throw new Error("request target or production authorization ID is missing");
28283
- }
28284
- if (resolution.forwarderIdentity.environment !== request.target.environment || resolution.forwarderIdentity.cluster !== request.target.cluster) {
28285
- throw new Error("archive forwarder cluster identity mismatch");
28286
- }
28287
- const authorization = resolution.authorization;
28288
- const expiresAtMs = Date.parse(authorization.expiresAt);
28289
- if (authorization.authorizationId !== request.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request.target.environment || authorization.cluster !== request.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
28290
- throw new Error("production forwarder authorization is invalid");
28291
- }
28292
- }
28293
- function resourcePolicyScopeExceeded(request) {
28294
- return request.depth > RESOURCE_POLICY.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
28295
- }
28296
- function storedReceiptForSelection(resolution) {
28297
- const receiptId = resolution.selection.receipt_ids[0];
28298
- return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
28299
- }
28300
- async function runMarketDataVendorBackfill(input, dependencies) {
28301
- let request;
28302
- try {
28303
- const documents = input;
28304
- request = decodeBackfillRunDocuments({
28305
- request: documents.request,
28306
- requiredClock: documents.requiredClock
28307
- });
28308
- } catch {
28309
- return outcome(undefined, "request_invalid", "request_invalid");
28310
- }
28311
- let initialResolution;
28312
- try {
28313
- initialResolution = await dependencies.archive.resolveSelection(request);
28314
- assertArchivePreflight(request, initialResolution);
28315
- const forwarderPreflight = await dependencies.forwarder.preflight({
28316
- authorizationId: request.productionAuthorizationId,
28317
- target: request.target
28318
- });
28319
- assertForwarderPreflight(request, forwarderPreflight, dependencies.clock.nowMs());
28320
- } catch {
28321
- return outcome(request, "archive_preflight_failed", "archive_preflight_failed");
28322
- }
28323
- if (initialResolution.selection.coverage_class === "complete") {
28324
- const receipt2 = storedReceiptForSelection(initialResolution);
28325
- return outcome(request, "already_covered", "qualified_coverage_complete", {
28326
- selection: initialResolution.selection,
28327
- ...receipt2 ? { receipt: receipt2 } : {}
28328
- });
28329
- }
28330
- if (resourcePolicyScopeExceeded(request)) {
28331
- return outcome(request, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
28332
- }
28333
- let capability;
28334
- try {
28335
- capability = dependencies.providers.capabilityFor(request);
28336
- } catch {
28337
- return outcome(request, "capability_unsupported", "capability_probe_failed");
28338
- }
28339
- if (!capability) {
28340
- return outcome(request, "capability_unsupported", "scope_unsupported");
28341
- }
28342
- if (!request.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
28343
- return outcome(request, "capability_unsupported", "adapter_version_unpinned");
28344
- }
28345
- let credential;
28346
- try {
28347
- credential = await dependencies.credentials.resolve(capability.provider);
28348
- } catch {
28349
- return outcome(request, "credentials_missing", "credential_resolution_failed");
28350
- }
28351
- if (credential === undefined || credential === null) {
28352
- return outcome(request, "credentials_missing", "provider_credentials_missing");
28353
- }
28354
- let dataset;
28355
- let normalized;
28356
- try {
28357
- dataset = await dependencies.providers.acquire(request, capability, credential);
28358
- const bundleId = captureBundleId(request, capability, dataset);
28359
- normalized = await dependencies.providers.normalize(request, capability, dataset, bundleId);
28360
- if (normalized.captureBundleId !== bundleId) {
28361
- return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
28362
- reasonSubcode: "capture_identity_mismatch"
28363
- });
28364
- }
28365
- } catch (error48) {
28366
- const reason = stableFailureReason(error48, "provider_dataset_invalid");
28367
- return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
28368
- reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
28369
- });
28370
- }
28371
- try {
28372
- const candidateBatches = buildForwarderBatches({
28373
- captureBundleId: normalized.captureBundleId,
28374
- deploymentId: "market-data-vendor-backfill",
28375
- rows: normalized.rows
28376
- });
28377
- if (!await submitAll(dependencies, candidateBatches)) {
28378
- return outcome(request, "archive_ingest_failed", "candidate_batch_rejected");
28379
- }
28380
- } catch {
28381
- return outcome(request, "archive_ingest_failed", "candidate_submission_failed");
28382
- }
28383
- let verification;
28384
- try {
28385
- verification = await dependencies.archive.verifyCandidate(request, normalized, normalized.captureBundleId, initialResolution);
28386
- } catch {
28387
- return outcome(request, "promotion_verification_failed", "candidate_query_failed");
28388
- }
28389
- if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
28390
- return outcome(request, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
27942
+ if (sympos)
27943
+ err(0);
27944
+ for (i = 0;i < sz; ++i) {
27945
+ var ns = dstate[syms[i]]++;
27946
+ var nb = nbits[i] = al - msb(ns);
27947
+ nstate[i] = (ns << nb) - sz;
28391
27948
  }
28392
- let receipt;
28393
- try {
28394
- receipt = buildPromotionReceipt(request, capability, normalized, verification, dependencies.clock.nowMs());
28395
- const [batch] = buildForwarderBatches({
28396
- captureBundleId: normalized.captureBundleId,
28397
- deploymentId: "market-data-vendor-backfill",
28398
- rows: [promotionReceiptToArchiveRow(receipt)]
28399
- });
28400
- if (!batch || !await submitAll(dependencies, [batch])) {
28401
- return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
27949
+ return [tpos + 7 >> 3, {
27950
+ b: al,
27951
+ s: syms,
27952
+ n: nbits,
27953
+ t: nstate
27954
+ }];
27955
+ };
27956
+ var rhu = function(dat, bt) {
27957
+ var i = 0, wc = -1;
27958
+ var buf = new u8(292), hb = dat[bt];
27959
+ var hw = buf.subarray(0, 256);
27960
+ var rc = buf.subarray(256, 268);
27961
+ var ri = new u16(buf.buffer, 268);
27962
+ if (hb < 128) {
27963
+ var _a2 = rfse(dat, bt + 1, 6), ebt = _a2[0], fdt = _a2[1];
27964
+ bt += hb;
27965
+ var epos = ebt << 3;
27966
+ var lb = dat[bt];
27967
+ if (!lb)
27968
+ err(0);
27969
+ var st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
27970
+ var fpos = (++bt << 3) - 8 + msb(lb);
27971
+ for (;; ) {
27972
+ fpos -= btr1;
27973
+ if (fpos < epos)
27974
+ break;
27975
+ var cbt = fpos >> 3;
27976
+ st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
27977
+ hw[++wc] = fdt.s[st1];
27978
+ fpos -= btr2;
27979
+ if (fpos < epos)
27980
+ break;
27981
+ cbt = fpos >> 3;
27982
+ st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
27983
+ hw[++wc] = fdt.s[st2];
27984
+ btr1 = fdt.n[st1];
27985
+ st1 = fdt.t[st1];
27986
+ btr2 = fdt.n[st2];
27987
+ st2 = fdt.t[st2];
28402
27988
  }
28403
- const qualification = finalizeQualificationEvent({
28404
- capture_bundle_id: receipt.capture_bundle_id,
28405
- state: "qualified",
28406
- receipt_id: receipt.receipt_id,
28407
- promotion_identity_sha256: receipt.promotion_identity_sha256,
28408
- window: receipt.window,
28409
- event_at: receipt.verified_at,
28410
- reason_code: "promotion_verified"
28411
- });
28412
- const [qualificationBatch] = buildForwarderBatches({
28413
- captureBundleId: normalized.captureBundleId,
28414
- deploymentId: "market-data-vendor-backfill",
28415
- rows: [qualificationEventToArchiveRow(qualification)]
28416
- });
28417
- if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
28418
- return outcome(request, "archive_ingest_failed", "qualification_commit_failed", { receipt });
27989
+ if (++wc > 255)
27990
+ err(0);
27991
+ } else {
27992
+ wc = hb - 127;
27993
+ for (;i < wc; i += 2) {
27994
+ var byte = dat[++bt];
27995
+ hw[i] = byte >> 4;
27996
+ hw[i + 1] = byte & 15;
28419
27997
  }
28420
- } catch {
28421
- return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
27998
+ ++bt;
28422
27999
  }
28423
- let finalResolution;
28424
- try {
28425
- finalResolution = await dependencies.archive.resolveSelection(request);
28426
- assertArchivePreflight(request, finalResolution);
28427
- } catch (error48) {
28428
- return outcome(request, "promotion_verification_failed", "post_promotion_selection_failed", {
28429
- receipt,
28430
- reasonSubcode: stableFailureReason(error48, "archive_selection_resolution_failed")
28431
- });
28000
+ var wes = 0;
28001
+ for (i = 0;i < wc; ++i) {
28002
+ var wt = hw[i];
28003
+ if (wt > 11)
28004
+ err(0);
28005
+ wes += wt && 1 << wt - 1;
28432
28006
  }
28433
- if (finalResolution.selection.coverage_class !== "complete") {
28434
- return outcome(request, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
28007
+ var mb = msb(wes) + 1;
28008
+ var ts = 1 << mb;
28009
+ var rem = ts - wes;
28010
+ if (rem & rem - 1)
28011
+ err(0);
28012
+ hw[wc++] = msb(rem) + 1;
28013
+ for (i = 0;i < wc; ++i) {
28014
+ var wt = hw[i];
28015
+ ++rc[hw[i] = wt && mb + 1 - wt];
28435
28016
  }
28436
- if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
28437
- return outcome(request, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
28017
+ var hbuf = new u8(ts << 1);
28018
+ var syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
28019
+ ri[mb] = 0;
28020
+ for (i = mb;i > 0; --i) {
28021
+ var pv = ri[i];
28022
+ fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
28438
28023
  }
28439
- try {
28440
- if (!request.wire)
28441
- throw new Error("decoded request wire is missing");
28442
- const [selectionBatch] = buildForwarderBatches({
28443
- captureBundleId: receipt.capture_bundle_id,
28444
- deploymentId: "market-data-vendor-backfill",
28445
- rows: [
28446
- archiveSelectionToArchiveRow(request.wire, finalResolution.selection)
28447
- ]
28448
- });
28449
- if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
28450
- return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
28024
+ if (ri[0] != ts)
28025
+ err(0);
28026
+ for (i = 0;i < wc; ++i) {
28027
+ var bits = hw[i];
28028
+ if (bits) {
28029
+ var code = ri[bits];
28030
+ fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
28451
28031
  }
28452
- } catch {
28453
- return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
28454
28032
  }
28455
- return outcome(request, "promoted", "promotion_qualified", {
28456
- receipt,
28457
- selection: finalResolution.selection
28458
- });
28459
- }
28460
-
28461
- // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
28462
- import { createHash as createHash4 } from "node:crypto";
28463
-
28464
- // node_modules/fzstd/esm/index.mjs
28465
- var ab = ArrayBuffer;
28466
- var u8 = Uint8Array;
28467
- var u16 = Uint16Array;
28468
- var i16 = Int16Array;
28469
- var i32 = Int32Array;
28470
- var slc = function(v, s, e) {
28471
- if (u8.prototype.slice)
28472
- return u8.prototype.slice.call(v, s, e);
28473
- if (s == null || s < 0)
28474
- s = 0;
28475
- if (e == null || e > v.length)
28476
- e = v.length;
28477
- var n = new u8(e - s);
28478
- n.set(v.subarray(s, e));
28479
- return n;
28480
- };
28481
- var fill = function(v, n, s, e) {
28482
- if (u8.prototype.fill)
28483
- return u8.prototype.fill.call(v, n, s, e);
28484
- if (s == null || s < 0)
28485
- s = 0;
28486
- if (e == null || e > v.length)
28487
- e = v.length;
28488
- for (;s < e; ++s)
28489
- v[s] = n;
28490
- return v;
28033
+ return [bt, {
28034
+ n: nb,
28035
+ b: mb,
28036
+ s: syms
28037
+ }];
28491
28038
  };
28492
- var cpw = function(v, t, s, e) {
28493
- if (u8.prototype.copyWithin)
28494
- return u8.prototype.copyWithin.call(v, t, s, e);
28495
- if (s == null || s < 0)
28496
- s = 0;
28497
- if (e == null || e > v.length)
28498
- e = v.length;
28499
- while (s < e) {
28500
- v[t++] = v[s++];
28039
+ var dllt = rfse(/* @__PURE__ */ new u8([
28040
+ 81,
28041
+ 16,
28042
+ 99,
28043
+ 140,
28044
+ 49,
28045
+ 198,
28046
+ 24,
28047
+ 99,
28048
+ 12,
28049
+ 33,
28050
+ 196,
28051
+ 24,
28052
+ 99,
28053
+ 102,
28054
+ 102,
28055
+ 134,
28056
+ 70,
28057
+ 146,
28058
+ 4
28059
+ ]), 0, 6)[1];
28060
+ var dmlt = rfse(/* @__PURE__ */ new u8([
28061
+ 33,
28062
+ 20,
28063
+ 196,
28064
+ 24,
28065
+ 99,
28066
+ 140,
28067
+ 33,
28068
+ 132,
28069
+ 16,
28070
+ 66,
28071
+ 8,
28072
+ 33,
28073
+ 132,
28074
+ 16,
28075
+ 66,
28076
+ 8,
28077
+ 33,
28078
+ 68,
28079
+ 68,
28080
+ 68,
28081
+ 68,
28082
+ 68,
28083
+ 68,
28084
+ 68,
28085
+ 68,
28086
+ 36,
28087
+ 9
28088
+ ]), 0, 6)[1];
28089
+ var doct = rfse(/* @__PURE__ */ new u8([
28090
+ 32,
28091
+ 132,
28092
+ 16,
28093
+ 66,
28094
+ 102,
28095
+ 70,
28096
+ 68,
28097
+ 68,
28098
+ 68,
28099
+ 68,
28100
+ 36,
28101
+ 73,
28102
+ 2
28103
+ ]), 0, 5)[1];
28104
+ var b2bl = function(b, s) {
28105
+ var len = b.length, bl = new i32(len);
28106
+ for (var i = 0;i < len; ++i) {
28107
+ bl[i] = s;
28108
+ s += 1 << b[i];
28501
28109
  }
28110
+ return bl;
28502
28111
  };
28503
- var ec = [
28504
- "invalid zstd data",
28505
- "window size too large (>2046MB)",
28506
- "invalid block type",
28507
- "FSE accuracy too high",
28508
- "match distance too far back",
28509
- "unexpected EOF"
28510
- ];
28511
- var err = function(ind, msg, nt) {
28512
- var e = new Error(msg || ec[ind]);
28513
- e.code = ind;
28514
- if (Error.captureStackTrace)
28515
- Error.captureStackTrace(e, err);
28516
- if (!nt)
28517
- throw e;
28518
- return e;
28519
- };
28520
- var rb = function(d, b, n) {
28521
- var i = 0, o = 0;
28522
- for (;i < n; ++i)
28523
- o |= d[b++] << (i << 3);
28524
- return o;
28112
+ var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
28113
+ 0,
28114
+ 0,
28115
+ 0,
28116
+ 0,
28117
+ 16843009,
28118
+ 50528770,
28119
+ 134678020,
28120
+ 202050057,
28121
+ 269422093
28122
+ ])).buffer, 0, 36);
28123
+ var llbl = /* @__PURE__ */ b2bl(llb, 0);
28124
+ var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
28125
+ 0,
28126
+ 0,
28127
+ 0,
28128
+ 0,
28129
+ 0,
28130
+ 0,
28131
+ 0,
28132
+ 0,
28133
+ 16843009,
28134
+ 50528770,
28135
+ 117769220,
28136
+ 185207048,
28137
+ 252579084,
28138
+ 16
28139
+ ])).buffer, 0, 53);
28140
+ var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
28141
+ var dhu = function(dat, out, hu) {
28142
+ var len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
28143
+ if (!lb)
28144
+ err(0);
28145
+ var st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
28146
+ for (;pos > eb && i < ss; ) {
28147
+ var cbt = pos >> 3;
28148
+ var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
28149
+ st = (st << btr | val) & msk;
28150
+ out[++i] = hu.s[st];
28151
+ pos -= btr = hu.n[st];
28152
+ }
28153
+ if (pos != eb || i + 1 != ss)
28154
+ err(0);
28525
28155
  };
28526
- var b4 = function(d, b) {
28527
- return (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0;
28156
+ var dhu4 = function(dat, out, hu) {
28157
+ var bt = 6;
28158
+ var ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
28159
+ dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
28160
+ dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
28161
+ dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
28162
+ dhu(dat.subarray(bt), out.subarray(sz3), hu);
28528
28163
  };
28529
- var rzfh = function(dat, w) {
28530
- var n3 = dat[0] | dat[1] << 8 | dat[2] << 16;
28531
- if (n3 == 3126568 && dat[3] == 253) {
28532
- var flg = dat[4];
28533
- var ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6;
28534
- if (flg & 8)
28535
- err(0);
28536
- var bt = 6 - ss;
28537
- var db = df == 3 ? 4 : df;
28538
- var di = rb(dat, bt, db);
28539
- bt += db;
28540
- var fsb = fcf ? 1 << fcf : ss;
28541
- var fss = rb(dat, bt, fsb) + (fcf == 1 && 256);
28542
- var ws = fss;
28543
- if (!ss) {
28544
- var wb = 1 << 10 + (dat[5] >> 3);
28545
- ws = wb + (wb >> 3) * (dat[5] & 7);
28164
+ var rzb = function(dat, st, out) {
28165
+ var _a2;
28166
+ var bt = st.b;
28167
+ var b0 = dat[bt], btype = b0 >> 1 & 3;
28168
+ st.l = b0 & 1;
28169
+ var sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
28170
+ var ebt = (bt += 3) + sz;
28171
+ if (btype == 1) {
28172
+ if (bt >= dat.length)
28173
+ return;
28174
+ st.b = bt + 1;
28175
+ if (out) {
28176
+ fill(out, dat[bt], st.y, st.y += sz);
28177
+ return out;
28546
28178
  }
28547
- if (ws > 2145386496)
28548
- err(1);
28549
- var buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
28550
- buf[0] = 1, buf[4] = 4, buf[8] = 8;
28551
- return {
28552
- b: bt + fsb,
28553
- y: 0,
28554
- l: 0,
28555
- d: di,
28556
- w: w && w != 1 ? w : buf.subarray(12),
28557
- e: ws,
28558
- o: new i32(buf.buffer, 0, 3),
28559
- u: fss,
28560
- c: cc,
28561
- m: Math.min(131072, ws)
28562
- };
28563
- } else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
28564
- return b4(dat, 4) + 8;
28179
+ return fill(new u8(sz), dat[bt]);
28565
28180
  }
28566
- err(0);
28567
- };
28568
- var msb = function(val) {
28569
- var bits = 0;
28570
- for (;1 << bits <= val; ++bits)
28571
- ;
28572
- return bits - 1;
28573
- };
28574
- var rfse = function(dat, bt, mal) {
28575
- var tpos = (bt << 3) + 4;
28576
- var al = (dat[bt] & 15) + 5;
28577
- if (al > mal)
28578
- err(3);
28579
- var sz = 1 << al;
28580
- var probs = sz, sym = -1, re = -1, i = -1, ht = sz;
28581
- var buf = new ab(512 + (sz << 2));
28582
- var freq = new i16(buf, 0, 256);
28583
- var dstate = new u16(buf, 0, 256);
28584
- var nstate = new u16(buf, 512, sz);
28585
- var bb1 = 512 + (sz << 1);
28586
- var syms = new u8(buf, bb1, sz);
28587
- var nbits = new u8(buf, bb1 + sz);
28588
- while (sym < 255 && probs > 0) {
28589
- var bits = msb(probs + 1);
28590
- var cbt = tpos >> 3;
28591
- var msk = (1 << bits + 1) - 1;
28592
- var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
28593
- var msk1fb = (1 << bits) - 1;
28594
- var msv = msk - probs - 1;
28595
- var sval = val & msk1fb;
28596
- if (sval < msv)
28597
- tpos += bits, val = sval;
28598
- else {
28599
- tpos += bits + 1;
28600
- if (val > msk1fb)
28601
- val -= msv;
28602
- }
28603
- freq[++sym] = --val;
28604
- if (val == -1) {
28605
- probs += val;
28606
- syms[--ht] = sym;
28607
- } else
28608
- probs -= val;
28609
- if (!val) {
28610
- do {
28611
- var rbt = tpos >> 3;
28612
- re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
28613
- tpos += 2;
28614
- sym += re;
28615
- } while (re == 3);
28181
+ if (ebt > dat.length)
28182
+ return;
28183
+ if (btype == 0) {
28184
+ st.b = ebt;
28185
+ if (out) {
28186
+ out.set(dat.subarray(bt, ebt), st.y);
28187
+ st.y += sz;
28188
+ return out;
28616
28189
  }
28190
+ return slc(dat, bt, ebt);
28617
28191
  }
28618
- if (sym > 255 || probs)
28619
- err(0);
28620
- var sympos = 0;
28621
- var sstep = (sz >> 1) + (sz >> 3) + 3;
28622
- var smask = sz - 1;
28623
- for (var s = 0;s <= sym; ++s) {
28624
- var sf = freq[s];
28625
- if (sf < 1) {
28626
- dstate[s] = -sf;
28627
- continue;
28192
+ if (btype == 2) {
28193
+ var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
28194
+ var lss = b3 >> 4, lcs = 0, s4 = 0;
28195
+ if (lbt < 2) {
28196
+ if (sf & 1)
28197
+ lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
28198
+ else
28199
+ lss = b3 >> 3;
28200
+ } else {
28201
+ s4 = sf;
28202
+ if (sf < 2)
28203
+ lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
28204
+ else if (sf == 2)
28205
+ lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
28206
+ else
28207
+ lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
28628
28208
  }
28629
- for (i = 0;i < sf; ++i) {
28630
- syms[sympos] = s;
28631
- do {
28632
- sympos = sympos + sstep & smask;
28633
- } while (sympos >= ht);
28209
+ ++bt;
28210
+ var buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
28211
+ var spl = buf.length - lss;
28212
+ if (lbt == 0)
28213
+ buf.set(dat.subarray(bt, bt += lss), spl);
28214
+ else if (lbt == 1)
28215
+ fill(buf, dat[bt++], spl);
28216
+ else {
28217
+ var hu = st.h;
28218
+ if (lbt == 2) {
28219
+ var hud = rhu(dat, bt);
28220
+ lcs += bt - (bt = hud[0]);
28221
+ st.h = hu = hud[1];
28222
+ } else if (!hu)
28223
+ err(0);
28224
+ (s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
28634
28225
  }
28226
+ var ns = dat[bt++];
28227
+ if (ns) {
28228
+ if (ns == 255)
28229
+ ns = (dat[bt++] | dat[bt++] << 8) + 32512;
28230
+ else if (ns > 127)
28231
+ ns = ns - 128 << 8 | dat[bt++];
28232
+ var scm = dat[bt++];
28233
+ if (scm & 3)
28234
+ err(0);
28235
+ var dts = [dmlt, doct, dllt];
28236
+ for (var i = 2;i > -1; --i) {
28237
+ var md = scm >> (i << 1) + 2 & 3;
28238
+ if (md == 1) {
28239
+ var rbuf = new u8([0, 0, dat[bt++]]);
28240
+ dts[i] = {
28241
+ s: rbuf.subarray(2, 3),
28242
+ n: rbuf.subarray(0, 1),
28243
+ t: new u16(rbuf.buffer, 0, 1),
28244
+ b: 0
28245
+ };
28246
+ } else if (md == 2) {
28247
+ _a2 = rfse(dat, bt, 9 - (i & 1)), bt = _a2[0], dts[i] = _a2[1];
28248
+ } else if (md == 3) {
28249
+ if (!st.t)
28250
+ err(0);
28251
+ dts[i] = st.t[i];
28252
+ }
28253
+ }
28254
+ var _b = st.t = dts, mlt = _b[0], oct = _b[1], llt = _b[2];
28255
+ var lb = dat[ebt - 1];
28256
+ if (!lb)
28257
+ err(0);
28258
+ var spos = (ebt << 3) - 8 + msb(lb) - llt.b, cbt = spos >> 3, oubt = 0;
28259
+ var lst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << llt.b) - 1;
28260
+ cbt = (spos -= oct.b) >> 3;
28261
+ var ost = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << oct.b) - 1;
28262
+ cbt = (spos -= mlt.b) >> 3;
28263
+ var mst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mlt.b) - 1;
28264
+ for (++ns;--ns; ) {
28265
+ var llc = llt.s[lst];
28266
+ var lbtr = llt.n[lst];
28267
+ var mlc = mlt.s[mst];
28268
+ var mbtr = mlt.n[mst];
28269
+ var ofc = oct.s[ost];
28270
+ var obtr = oct.n[ost];
28271
+ cbt = (spos -= ofc) >> 3;
28272
+ var ofp = 1 << ofc;
28273
+ var off = ofp + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16 | dat[cbt + 3] << 24) >>> (spos & 7) & ofp - 1);
28274
+ cbt = (spos -= mlb[mlc]) >> 3;
28275
+ var ml = mlbl[mlc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << mlb[mlc]) - 1);
28276
+ cbt = (spos -= llb[llc]) >> 3;
28277
+ var ll = llbl[llc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << llb[llc]) - 1);
28278
+ cbt = (spos -= lbtr) >> 3;
28279
+ lst = llt.t[lst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << lbtr) - 1);
28280
+ cbt = (spos -= mbtr) >> 3;
28281
+ mst = mlt.t[mst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mbtr) - 1);
28282
+ cbt = (spos -= obtr) >> 3;
28283
+ ost = oct.t[ost] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << obtr) - 1);
28284
+ if (off > 3) {
28285
+ st.o[2] = st.o[1];
28286
+ st.o[1] = st.o[0];
28287
+ st.o[0] = off -= 3;
28288
+ } else {
28289
+ var idx = off - (ll != 0);
28290
+ if (idx) {
28291
+ off = idx == 3 ? st.o[0] - 1 : st.o[idx];
28292
+ if (idx > 1)
28293
+ st.o[2] = st.o[1];
28294
+ st.o[1] = st.o[0];
28295
+ st.o[0] = off;
28296
+ } else
28297
+ off = st.o[0];
28298
+ }
28299
+ for (var i = 0;i < ll; ++i) {
28300
+ buf[oubt + i] = buf[spl + i];
28301
+ }
28302
+ oubt += ll, spl += ll;
28303
+ var stin = oubt - off;
28304
+ if (stin < 0) {
28305
+ var len = -stin;
28306
+ var bs = st.e + stin;
28307
+ if (len > ml)
28308
+ len = ml;
28309
+ for (var i = 0;i < len; ++i) {
28310
+ buf[oubt + i] = st.w[bs + i];
28311
+ }
28312
+ oubt += len, ml -= len, stin = 0;
28313
+ }
28314
+ for (var i = 0;i < ml; ++i) {
28315
+ buf[oubt + i] = buf[stin + i];
28316
+ }
28317
+ oubt += ml;
28318
+ }
28319
+ if (oubt != spl) {
28320
+ while (spl < buf.length) {
28321
+ buf[oubt++] = buf[spl++];
28322
+ }
28323
+ } else
28324
+ oubt = buf.length;
28325
+ if (out)
28326
+ st.y += oubt;
28327
+ else
28328
+ buf = slc(buf, 0, oubt);
28329
+ } else if (out) {
28330
+ st.y += lss;
28331
+ if (spl) {
28332
+ for (var i = 0;i < lss; ++i) {
28333
+ buf[i] = buf[spl + i];
28334
+ }
28335
+ }
28336
+ } else if (spl)
28337
+ buf = slc(buf, spl);
28338
+ st.b = ebt;
28339
+ return buf;
28635
28340
  }
28636
- if (sympos)
28637
- err(0);
28638
- for (i = 0;i < sz; ++i) {
28639
- var ns = dstate[syms[i]]++;
28640
- var nb = nbits[i] = al - msb(ns);
28641
- nstate[i] = (ns << nb) - sz;
28341
+ err(2);
28342
+ };
28343
+ var cct = function(bufs, ol) {
28344
+ if (bufs.length == 1)
28345
+ return bufs[0];
28346
+ var buf = new u8(ol);
28347
+ for (var i = 0, b = 0;i < bufs.length; ++i) {
28348
+ var chk = bufs[i];
28349
+ buf.set(chk, b);
28350
+ b += chk.length;
28642
28351
  }
28643
- return [tpos + 7 >> 3, {
28644
- b: al,
28645
- s: syms,
28646
- n: nbits,
28647
- t: nstate
28648
- }];
28649
- };
28650
- var rhu = function(dat, bt) {
28651
- var i = 0, wc = -1;
28652
- var buf = new u8(292), hb = dat[bt];
28653
- var hw = buf.subarray(0, 256);
28654
- var rc = buf.subarray(256, 268);
28655
- var ri = new u16(buf.buffer, 268);
28656
- if (hb < 128) {
28657
- var _a2 = rfse(dat, bt + 1, 6), ebt = _a2[0], fdt = _a2[1];
28658
- bt += hb;
28659
- var epos = ebt << 3;
28660
- var lb = dat[bt];
28661
- if (!lb)
28662
- err(0);
28663
- var st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
28664
- var fpos = (++bt << 3) - 8 + msb(lb);
28665
- for (;; ) {
28666
- fpos -= btr1;
28667
- if (fpos < epos)
28668
- break;
28669
- var cbt = fpos >> 3;
28670
- st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
28671
- hw[++wc] = fdt.s[st1];
28672
- fpos -= btr2;
28673
- if (fpos < epos)
28674
- break;
28675
- cbt = fpos >> 3;
28676
- st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
28677
- hw[++wc] = fdt.s[st2];
28678
- btr1 = fdt.n[st1];
28679
- st1 = fdt.t[st1];
28680
- btr2 = fdt.n[st2];
28681
- st2 = fdt.t[st2];
28682
- }
28683
- if (++wc > 255)
28684
- err(0);
28685
- } else {
28686
- wc = hb - 127;
28687
- for (;i < wc; i += 2) {
28688
- var byte = dat[++bt];
28689
- hw[i] = byte >> 4;
28690
- hw[i + 1] = byte & 15;
28691
- }
28692
- ++bt;
28693
- }
28694
- var wes = 0;
28695
- for (i = 0;i < wc; ++i) {
28696
- var wt = hw[i];
28697
- if (wt > 11)
28698
- err(0);
28699
- wes += wt && 1 << wt - 1;
28700
- }
28701
- var mb = msb(wes) + 1;
28702
- var ts = 1 << mb;
28703
- var rem = ts - wes;
28704
- if (rem & rem - 1)
28705
- err(0);
28706
- hw[wc++] = msb(rem) + 1;
28707
- for (i = 0;i < wc; ++i) {
28708
- var wt = hw[i];
28709
- ++rc[hw[i] = wt && mb + 1 - wt];
28710
- }
28711
- var hbuf = new u8(ts << 1);
28712
- var syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
28713
- ri[mb] = 0;
28714
- for (i = mb;i > 0; --i) {
28715
- var pv = ri[i];
28716
- fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
28717
- }
28718
- if (ri[0] != ts)
28719
- err(0);
28720
- for (i = 0;i < wc; ++i) {
28721
- var bits = hw[i];
28722
- if (bits) {
28723
- var code = ri[bits];
28724
- fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
28725
- }
28726
- }
28727
- return [bt, {
28728
- n: nb,
28729
- b: mb,
28730
- s: syms
28731
- }];
28732
- };
28733
- var dllt = rfse(/* @__PURE__ */ new u8([
28734
- 81,
28735
- 16,
28736
- 99,
28737
- 140,
28738
- 49,
28739
- 198,
28740
- 24,
28741
- 99,
28742
- 12,
28743
- 33,
28744
- 196,
28745
- 24,
28746
- 99,
28747
- 102,
28748
- 102,
28749
- 134,
28750
- 70,
28751
- 146,
28752
- 4
28753
- ]), 0, 6)[1];
28754
- var dmlt = rfse(/* @__PURE__ */ new u8([
28755
- 33,
28756
- 20,
28757
- 196,
28758
- 24,
28759
- 99,
28760
- 140,
28761
- 33,
28762
- 132,
28763
- 16,
28764
- 66,
28765
- 8,
28766
- 33,
28767
- 132,
28768
- 16,
28769
- 66,
28770
- 8,
28771
- 33,
28772
- 68,
28773
- 68,
28774
- 68,
28775
- 68,
28776
- 68,
28777
- 68,
28778
- 68,
28779
- 68,
28780
- 36,
28781
- 9
28782
- ]), 0, 6)[1];
28783
- var doct = rfse(/* @__PURE__ */ new u8([
28784
- 32,
28785
- 132,
28786
- 16,
28787
- 66,
28788
- 102,
28789
- 70,
28790
- 68,
28791
- 68,
28792
- 68,
28793
- 68,
28794
- 36,
28795
- 73,
28796
- 2
28797
- ]), 0, 5)[1];
28798
- var b2bl = function(b, s) {
28799
- var len = b.length, bl = new i32(len);
28800
- for (var i = 0;i < len; ++i) {
28801
- bl[i] = s;
28802
- s += 1 << b[i];
28803
- }
28804
- return bl;
28805
- };
28806
- var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
28807
- 0,
28808
- 0,
28809
- 0,
28810
- 0,
28811
- 16843009,
28812
- 50528770,
28813
- 134678020,
28814
- 202050057,
28815
- 269422093
28816
- ])).buffer, 0, 36);
28817
- var llbl = /* @__PURE__ */ b2bl(llb, 0);
28818
- var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
28819
- 0,
28820
- 0,
28821
- 0,
28822
- 0,
28823
- 0,
28824
- 0,
28825
- 0,
28826
- 0,
28827
- 16843009,
28828
- 50528770,
28829
- 117769220,
28830
- 185207048,
28831
- 252579084,
28832
- 16
28833
- ])).buffer, 0, 53);
28834
- var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
28835
- var dhu = function(dat, out, hu) {
28836
- var len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
28837
- if (!lb)
28838
- err(0);
28839
- var st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
28840
- for (;pos > eb && i < ss; ) {
28841
- var cbt = pos >> 3;
28842
- var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
28843
- st = (st << btr | val) & msk;
28844
- out[++i] = hu.s[st];
28845
- pos -= btr = hu.n[st];
28846
- }
28847
- if (pos != eb || i + 1 != ss)
28848
- err(0);
28849
- };
28850
- var dhu4 = function(dat, out, hu) {
28851
- var bt = 6;
28852
- var ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
28853
- dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
28854
- dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
28855
- dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
28856
- dhu(dat.subarray(bt), out.subarray(sz3), hu);
28857
- };
28858
- var rzb = function(dat, st, out) {
28859
- var _a2;
28860
- var bt = st.b;
28861
- var b0 = dat[bt], btype = b0 >> 1 & 3;
28862
- st.l = b0 & 1;
28863
- var sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
28864
- var ebt = (bt += 3) + sz;
28865
- if (btype == 1) {
28866
- if (bt >= dat.length)
28867
- return;
28868
- st.b = bt + 1;
28869
- if (out) {
28870
- fill(out, dat[bt], st.y, st.y += sz);
28871
- return out;
28872
- }
28873
- return fill(new u8(sz), dat[bt]);
28874
- }
28875
- if (ebt > dat.length)
28876
- return;
28877
- if (btype == 0) {
28878
- st.b = ebt;
28879
- if (out) {
28880
- out.set(dat.subarray(bt, ebt), st.y);
28881
- st.y += sz;
28882
- return out;
28883
- }
28884
- return slc(dat, bt, ebt);
28885
- }
28886
- if (btype == 2) {
28887
- var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
28888
- var lss = b3 >> 4, lcs = 0, s4 = 0;
28889
- if (lbt < 2) {
28890
- if (sf & 1)
28891
- lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
28892
- else
28893
- lss = b3 >> 3;
28894
- } else {
28895
- s4 = sf;
28896
- if (sf < 2)
28897
- lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
28898
- else if (sf == 2)
28899
- lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
28900
- else
28901
- lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
28902
- }
28903
- ++bt;
28904
- var buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
28905
- var spl = buf.length - lss;
28906
- if (lbt == 0)
28907
- buf.set(dat.subarray(bt, bt += lss), spl);
28908
- else if (lbt == 1)
28909
- fill(buf, dat[bt++], spl);
28910
- else {
28911
- var hu = st.h;
28912
- if (lbt == 2) {
28913
- var hud = rhu(dat, bt);
28914
- lcs += bt - (bt = hud[0]);
28915
- st.h = hu = hud[1];
28916
- } else if (!hu)
28917
- err(0);
28918
- (s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
28919
- }
28920
- var ns = dat[bt++];
28921
- if (ns) {
28922
- if (ns == 255)
28923
- ns = (dat[bt++] | dat[bt++] << 8) + 32512;
28924
- else if (ns > 127)
28925
- ns = ns - 128 << 8 | dat[bt++];
28926
- var scm = dat[bt++];
28927
- if (scm & 3)
28928
- err(0);
28929
- var dts = [dmlt, doct, dllt];
28930
- for (var i = 2;i > -1; --i) {
28931
- var md = scm >> (i << 1) + 2 & 3;
28932
- if (md == 1) {
28933
- var rbuf = new u8([0, 0, dat[bt++]]);
28934
- dts[i] = {
28935
- s: rbuf.subarray(2, 3),
28936
- n: rbuf.subarray(0, 1),
28937
- t: new u16(rbuf.buffer, 0, 1),
28938
- b: 0
28939
- };
28940
- } else if (md == 2) {
28941
- _a2 = rfse(dat, bt, 9 - (i & 1)), bt = _a2[0], dts[i] = _a2[1];
28942
- } else if (md == 3) {
28943
- if (!st.t)
28944
- err(0);
28945
- dts[i] = st.t[i];
28946
- }
28947
- }
28948
- var _b = st.t = dts, mlt = _b[0], oct = _b[1], llt = _b[2];
28949
- var lb = dat[ebt - 1];
28950
- if (!lb)
28951
- err(0);
28952
- var spos = (ebt << 3) - 8 + msb(lb) - llt.b, cbt = spos >> 3, oubt = 0;
28953
- var lst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << llt.b) - 1;
28954
- cbt = (spos -= oct.b) >> 3;
28955
- var ost = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << oct.b) - 1;
28956
- cbt = (spos -= mlt.b) >> 3;
28957
- var mst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mlt.b) - 1;
28958
- for (++ns;--ns; ) {
28959
- var llc = llt.s[lst];
28960
- var lbtr = llt.n[lst];
28961
- var mlc = mlt.s[mst];
28962
- var mbtr = mlt.n[mst];
28963
- var ofc = oct.s[ost];
28964
- var obtr = oct.n[ost];
28965
- cbt = (spos -= ofc) >> 3;
28966
- var ofp = 1 << ofc;
28967
- var off = ofp + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16 | dat[cbt + 3] << 24) >>> (spos & 7) & ofp - 1);
28968
- cbt = (spos -= mlb[mlc]) >> 3;
28969
- var ml = mlbl[mlc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << mlb[mlc]) - 1);
28970
- cbt = (spos -= llb[llc]) >> 3;
28971
- var ll = llbl[llc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << llb[llc]) - 1);
28972
- cbt = (spos -= lbtr) >> 3;
28973
- lst = llt.t[lst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << lbtr) - 1);
28974
- cbt = (spos -= mbtr) >> 3;
28975
- mst = mlt.t[mst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mbtr) - 1);
28976
- cbt = (spos -= obtr) >> 3;
28977
- ost = oct.t[ost] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << obtr) - 1);
28978
- if (off > 3) {
28979
- st.o[2] = st.o[1];
28980
- st.o[1] = st.o[0];
28981
- st.o[0] = off -= 3;
28982
- } else {
28983
- var idx = off - (ll != 0);
28984
- if (idx) {
28985
- off = idx == 3 ? st.o[0] - 1 : st.o[idx];
28986
- if (idx > 1)
28987
- st.o[2] = st.o[1];
28988
- st.o[1] = st.o[0];
28989
- st.o[0] = off;
28990
- } else
28991
- off = st.o[0];
28992
- }
28993
- for (var i = 0;i < ll; ++i) {
28994
- buf[oubt + i] = buf[spl + i];
28995
- }
28996
- oubt += ll, spl += ll;
28997
- var stin = oubt - off;
28998
- if (stin < 0) {
28999
- var len = -stin;
29000
- var bs = st.e + stin;
29001
- if (len > ml)
29002
- len = ml;
29003
- for (var i = 0;i < len; ++i) {
29004
- buf[oubt + i] = st.w[bs + i];
29005
- }
29006
- oubt += len, ml -= len, stin = 0;
29007
- }
29008
- for (var i = 0;i < ml; ++i) {
29009
- buf[oubt + i] = buf[stin + i];
29010
- }
29011
- oubt += ml;
29012
- }
29013
- if (oubt != spl) {
29014
- while (spl < buf.length) {
29015
- buf[oubt++] = buf[spl++];
29016
- }
29017
- } else
29018
- oubt = buf.length;
29019
- if (out)
29020
- st.y += oubt;
29021
- else
29022
- buf = slc(buf, 0, oubt);
29023
- } else if (out) {
29024
- st.y += lss;
29025
- if (spl) {
29026
- for (var i = 0;i < lss; ++i) {
29027
- buf[i] = buf[spl + i];
29028
- }
29029
- }
29030
- } else if (spl)
29031
- buf = slc(buf, spl);
29032
- st.b = ebt;
29033
- return buf;
29034
- }
29035
- err(2);
29036
- };
29037
- var cct = function(bufs, ol) {
29038
- if (bufs.length == 1)
29039
- return bufs[0];
29040
- var buf = new u8(ol);
29041
- for (var i = 0, b = 0;i < bufs.length; ++i) {
29042
- var chk = bufs[i];
29043
- buf.set(chk, b);
29044
- b += chk.length;
29045
- }
29046
- return buf;
28352
+ return buf;
29047
28353
  };
29048
28354
  function decompress(dat, buf) {
29049
28355
  var bufs = [], nb = +!buf;
@@ -32458,7 +31764,7 @@ var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
32458
31764
  snapshotGrouping: "event_time_final_update_id_object",
32459
31765
  sequenceSemantics: "okx_seq_id_prev_seq_id",
32460
31766
  constructionModes: ["sampled_top_n_snapshot"],
32461
- sourcePolicies: ["authoritative_window"]
31767
+ sourcePolicies: ["authoritative_window", "fill_gaps"]
32462
31768
  });
32463
31769
 
32464
31770
  class CryptoHftDataError extends Error {
@@ -32469,498 +31775,913 @@ class CryptoHftDataError extends Error {
32469
31775
  this.name = "CryptoHftDataError";
32470
31776
  }
32471
31777
  }
32472
- function cryptoHftDataCapabilityFor(request, profiles = []) {
32473
- if (request.providerPolicy.provider !== "cryptohftdata" || request.scope.feed !== "ORDERBOOK" || request.scope.exchange.trim().toLowerCase() === "mexc") {
32474
- return;
32475
- }
32476
- const profile = profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.constructionModes.includes(request.constructionMode) && candidate.sourcePolicies.includes(request.sourcePolicy) && request.window.startTimeMs >= candidate.historyStartMs && request.depth <= candidate.maxDepth);
32477
- if (!profile)
32478
- return;
32479
- return {
32480
- provider: "cryptohftdata",
32481
- adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
32482
- providerExchangeId: profile.providerExchangeId,
32483
- resolvedSymbol: profile.sourceSymbol
32484
- };
31778
+ function cryptoHftDataCapabilityFor(request, profiles = []) {
31779
+ if (request.providerPolicy.provider !== "cryptohftdata" || request.scope.feed !== "ORDERBOOK" || request.scope.exchange.trim().toLowerCase() === "mexc") {
31780
+ return;
31781
+ }
31782
+ const profile = profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.constructionModes.includes(request.constructionMode) && candidate.sourcePolicies.includes(request.sourcePolicy) && request.window.startTimeMs >= candidate.historyStartMs && request.depth <= candidate.maxDepth);
31783
+ if (!profile)
31784
+ return;
31785
+ return {
31786
+ provider: "cryptohftdata",
31787
+ adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
31788
+ providerExchangeId: profile.providerExchangeId,
31789
+ resolvedSymbol: profile.sourceSymbol
31790
+ };
31791
+ }
31792
+ function utcHourPath(hourMs) {
31793
+ const date5 = new Date(hourMs);
31794
+ return {
31795
+ date: `${date5.getUTCFullYear()}-${String(date5.getUTCMonth() + 1).padStart(2, "0")}-${String(date5.getUTCDate()).padStart(2, "0")}`,
31796
+ hour: String(date5.getUTCHours()).padStart(2, "0")
31797
+ };
31798
+ }
31799
+ function enumerateCryptoHftDataObjects(request, providerExchangeId, resolvedSymbol) {
31800
+ if (request.sourcePolicy === "fill_gaps") {
31801
+ const hours = new Set;
31802
+ for (const targetTimeMs of request.requiredClockTargetsMs) {
31803
+ const firstHour2 = Math.floor(Math.max(0, targetTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
31804
+ const lastHour = Math.floor(targetTimeMs / HOUR_MS) * HOUR_MS;
31805
+ for (let hourMs = firstHour2;hourMs <= lastHour; hourMs += HOUR_MS) {
31806
+ hours.add(hourMs);
31807
+ }
31808
+ }
31809
+ return [...hours].sort((left, right) => left - right).map((hourMs) => {
31810
+ const utc = utcHourPath(hourMs);
31811
+ return `${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`;
31812
+ });
31813
+ }
31814
+ const firstHour = Math.floor((request.window.startTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
31815
+ const objects = [];
31816
+ for (let hourMs = Math.max(0, firstHour);hourMs < request.window.endTimeMs; hourMs += HOUR_MS) {
31817
+ const utc = utcHourPath(hourMs);
31818
+ objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
31819
+ }
31820
+ return objects;
31821
+ }
31822
+ function initialArchiveCoversTarget(request, targetTimeMs) {
31823
+ return (request.initialSelection?.support_anchors ?? []).some((anchor) => {
31824
+ const sourceTimeMs = Date.parse(anchor.source_time);
31825
+ return Number.isSafeInteger(sourceTimeMs) && sourceTimeMs <= targetTimeMs && targetTimeMs - sourceTimeMs <= request.maxPriorAsOfLagMs;
31826
+ });
31827
+ }
31828
+ function providerAcquisitionRequest(request) {
31829
+ if (request.sourcePolicy !== "fill_gaps")
31830
+ return request;
31831
+ const requiredClockTargetsMs = request.requiredClockTargetsMs.filter((targetTimeMs) => !initialArchiveCoversTarget(request, targetTimeMs));
31832
+ if (requiredClockTargetsMs.length === 0) {
31833
+ throw new CryptoHftDataError("fill_gaps_has_no_uncovered_clock_targets");
31834
+ }
31835
+ return { ...request, requiredClockTargetsMs };
31836
+ }
31837
+ function unsignedString(value, field, required2 = false) {
31838
+ if (value === null || value === undefined) {
31839
+ if (required2)
31840
+ throw new CryptoHftDataError(`schema_${field}_missing`);
31841
+ return;
31842
+ }
31843
+ const rendered = String(value);
31844
+ if (!/^\d+$/.test(rendered)) {
31845
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
31846
+ }
31847
+ const parsed = BigInt(rendered);
31848
+ if (parsed > 18446744073709551615n) {
31849
+ throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
31850
+ }
31851
+ return parsed.toString(10);
31852
+ }
31853
+ function timestampMs2(value, field) {
31854
+ const rendered = String(value);
31855
+ if (!/^\d+$/.test(rendered)) {
31856
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
31857
+ }
31858
+ const raw = BigInt(rendered);
31859
+ const milliseconds = field === "received_time" ? raw / 1000000n : raw;
31860
+ if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
31861
+ throw new CryptoHftDataError(`schema_${field}_unsafe`);
31862
+ }
31863
+ return Number(milliseconds);
31864
+ }
31865
+ function decimal(value, field, allowZero) {
31866
+ if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
31867
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
31868
+ }
31869
+ const parsed = Number(value);
31870
+ if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
31871
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
31872
+ }
31873
+ return parsed;
31874
+ }
31875
+ function validatedRow(request, row) {
31876
+ if (row.symbol !== request.scope.sourceSymbol) {
31877
+ throw new CryptoHftDataError("schema_symbol_mismatch");
31878
+ }
31879
+ if (row.event_type !== "snapshot" && row.event_type !== "update") {
31880
+ throw new CryptoHftDataError("schema_event_type_invalid");
31881
+ }
31882
+ if (row.side !== "bid" && row.side !== "ask") {
31883
+ throw new CryptoHftDataError("schema_side_invalid");
31884
+ }
31885
+ decimal(row.price, "price", false);
31886
+ decimal(row.quantity, "quantity", true);
31887
+ if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
31888
+ throw new CryptoHftDataError("schema_object_checksum_invalid");
31889
+ }
31890
+ const eventTimeMs = timestampMs2(row.event_time, "event_time");
31891
+ const receivedTimeMs = timestampMs2(row.received_time, "received_time");
31892
+ if (receivedTimeMs < eventTimeMs) {
31893
+ throw new CryptoHftDataError("received_time_precedes_event_time");
31894
+ }
31895
+ return { ...row, eventTimeMs, receivedTimeMs };
31896
+ }
31897
+ function groupKey(row, profile) {
31898
+ const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
31899
+ return [
31900
+ row.event_type,
31901
+ row.eventTimeMs,
31902
+ sequence,
31903
+ row.dataset_object_identity
31904
+ ].join("\x00");
31905
+ }
31906
+ function absent(value) {
31907
+ return value === null || value === undefined;
31908
+ }
31909
+ function snapshotSequence(row, profile) {
31910
+ if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
31911
+ if (String(row.last_update_id) !== "-1") {
31912
+ throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
31913
+ }
31914
+ if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
31915
+ throw new CryptoHftDataError("ambiguous_snapshot_group");
31916
+ }
31917
+ return unsignedString(row.final_update_id, "final_update_id", true);
31918
+ }
31919
+ return unsignedString(row.last_update_id, "last_update_id", true);
31920
+ }
31921
+ function sortedSide(levels, side) {
31922
+ return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
31923
+ }
31924
+ function applyRows(state, rows) {
31925
+ for (const row of rows) {
31926
+ const levels = row.side === "bid" ? state.bids : state.asks;
31927
+ const quantity = decimal(row.quantity, "quantity", true);
31928
+ if (quantity === 0)
31929
+ levels.delete(row.price);
31930
+ else
31931
+ levels.set(row.price, quantity);
31932
+ }
31933
+ }
31934
+ function reconstructCryptoHftDataOrderBooks(request, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
31935
+ const rows = inputRows.map((row, index) => ({
31936
+ ...validatedRow(request, row),
31937
+ originalIndex: index
31938
+ })).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
31939
+ const groups = [];
31940
+ for (const row of rows) {
31941
+ const current = groups.at(-1);
31942
+ if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
31943
+ groups.push([row]);
31944
+ } else {
31945
+ current.push(row);
31946
+ }
31947
+ }
31948
+ const earliestTargetTimeMs = Math.min(...request.requiredClockTargetsMs);
31949
+ const latestTargetTimeMs = Math.max(...request.requiredClockTargetsMs);
31950
+ const anchorIndex = groups.findIndex((group) => {
31951
+ const first = group[0];
31952
+ return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
31953
+ });
31954
+ if (anchorIndex < 0) {
31955
+ throw new CryptoHftDataError("update_before_snapshot");
31956
+ }
31957
+ let state;
31958
+ let previousFinalUpdateId;
31959
+ const states = [];
31960
+ for (const group of groups.slice(anchorIndex)) {
31961
+ const first = group[0];
31962
+ if (first.eventTimeMs > latestTargetTimeMs)
31963
+ break;
31964
+ if (first.event_type === "snapshot") {
31965
+ const sequence = snapshotSequence(first, profile);
31966
+ for (const row of group) {
31967
+ if (snapshotSequence(row, profile) !== sequence) {
31968
+ throw new CryptoHftDataError("ambiguous_snapshot_group");
31969
+ }
31970
+ }
31971
+ if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
31972
+ throw new CryptoHftDataError("snapshot_sequence_regression");
31973
+ }
31974
+ state = {
31975
+ bids: new Map,
31976
+ asks: new Map,
31977
+ sequence,
31978
+ sourceTimeMs: first.eventTimeMs,
31979
+ receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
31980
+ datasetObjectIdentity: first.dataset_object_identity,
31981
+ datasetObjectChecksum: first.dataset_object_checksum
31982
+ };
31983
+ applyRows(state, group);
31984
+ previousFinalUpdateId = BigInt(sequence);
31985
+ } else {
31986
+ if (!state || previousFinalUpdateId === undefined) {
31987
+ throw new CryptoHftDataError("update_before_snapshot");
31988
+ }
31989
+ const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
31990
+ if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
31991
+ if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
31992
+ throw new CryptoHftDataError("ambiguous_update_group");
31993
+ }
31994
+ const previous = unsignedString(first.last_update_id, "last_update_id", true);
31995
+ for (const row of group) {
31996
+ if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
31997
+ throw new CryptoHftDataError("ambiguous_update_group");
31998
+ }
31999
+ }
32000
+ if (BigInt(previous) !== previousFinalUpdateId) {
32001
+ throw new CryptoHftDataError("update_chain_gap");
32002
+ }
32003
+ } else {
32004
+ const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
32005
+ const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
32006
+ for (const row of group) {
32007
+ if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
32008
+ throw new CryptoHftDataError("ambiguous_update_group");
32009
+ }
32010
+ }
32011
+ const expected = previousFinalUpdateId + 1n;
32012
+ if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
32013
+ throw new CryptoHftDataError("update_chain_gap");
32014
+ }
32015
+ }
32016
+ applyRows(state, group);
32017
+ state.sequence = finalUpdate.toString();
32018
+ state.sourceTimeMs = first.eventTimeMs;
32019
+ state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
32020
+ state.datasetObjectIdentity = first.dataset_object_identity;
32021
+ state.datasetObjectChecksum = first.dataset_object_checksum;
32022
+ previousFinalUpdateId = finalUpdate;
32023
+ }
32024
+ if (state) {
32025
+ states.push({
32026
+ ...state,
32027
+ bids: new Map(state.bids),
32028
+ asks: new Map(state.asks)
32029
+ });
32030
+ }
32031
+ }
32032
+ const samples = [];
32033
+ for (const targetTimeMs of request.requiredClockTargetsMs) {
32034
+ let prior;
32035
+ for (const candidate of states) {
32036
+ if (candidate.sourceTimeMs > targetTimeMs)
32037
+ break;
32038
+ prior = candidate;
32039
+ }
32040
+ if (!prior || targetTimeMs - prior.sourceTimeMs > request.maxPriorAsOfLagMs) {
32041
+ throw new CryptoHftDataError("required_clock_coverage_insufficient");
32042
+ }
32043
+ const bids = sortedSide(prior.bids, "bid").slice(0, request.depth);
32044
+ const asks = sortedSide(prior.asks, "ask").slice(0, request.depth);
32045
+ if (bids.length === 0 || asks.length === 0) {
32046
+ throw new CryptoHftDataError("book_side_missing");
32047
+ }
32048
+ if (bids[0]?.[0] >= asks[0]?.[0]) {
32049
+ throw new CryptoHftDataError("book_crossed_or_locked");
32050
+ }
32051
+ samples.push({
32052
+ targetTimeMs,
32053
+ sourceTimeMs: prior.sourceTimeMs,
32054
+ receivedTimeMs: prior.receivedTimeMs,
32055
+ sequence: prior.sequence,
32056
+ bids,
32057
+ asks,
32058
+ datasetObjectIdentity: prior.datasetObjectIdentity,
32059
+ datasetObjectChecksum: prior.datasetObjectChecksum
32060
+ });
32061
+ }
32062
+ return samples;
32063
+ }
32064
+ function sha256Bytes(bytes) {
32065
+ return createHash4("sha256").update(bytes).digest("hex");
32485
32066
  }
32486
- function utcHourPath(hourMs) {
32487
- const date5 = new Date(hourMs);
32067
+ async function decodeCryptoHftParquetZstd(bytes) {
32068
+ const parquet = decompress(bytes);
32069
+ const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
32070
+ return parquetReadObjects({ file: buffer });
32071
+ }
32072
+ function parsedDatasetRow(value, object2) {
32488
32073
  return {
32489
- date: `${date5.getUTCFullYear()}-${String(date5.getUTCMonth() + 1).padStart(2, "0")}-${String(date5.getUTCDate()).padStart(2, "0")}`,
32490
- hour: String(date5.getUTCHours()).padStart(2, "0")
32074
+ received_time: value.received_time,
32075
+ event_time: value.event_time,
32076
+ transaction_time: value.transaction_time,
32077
+ symbol: value.symbol,
32078
+ event_type: value.event_type,
32079
+ first_update_id: value.first_update_id,
32080
+ final_update_id: value.final_update_id,
32081
+ prev_final_update_id: value.prev_final_update_id,
32082
+ last_update_id: value.last_update_id,
32083
+ side: value.side,
32084
+ price: value.price,
32085
+ quantity: value.quantity,
32086
+ order_count: value.order_count,
32087
+ dataset_object_identity: object2.identity,
32088
+ dataset_object_checksum: object2.checksum
32491
32089
  };
32492
32090
  }
32493
- function enumerateCryptoHftDataObjects(request, providerExchangeId, resolvedSymbol) {
32494
- const firstHour = Math.floor((request.window.startTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
32495
- const objects = [];
32496
- for (let hourMs = Math.max(0, firstHour);hourMs < request.window.endTimeMs; hourMs += HOUR_MS) {
32497
- const utc = utcHourPath(hourMs);
32498
- objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
32091
+ async function readBoundedObject(response, remainingBytes) {
32092
+ if (!response.body)
32093
+ return new Uint8Array;
32094
+ const reader = response.body.getReader();
32095
+ const chunks = [];
32096
+ let total = 0;
32097
+ while (true) {
32098
+ const { done, value } = await reader.read();
32099
+ if (done)
32100
+ break;
32101
+ total += value.byteLength;
32102
+ if (total > remainingBytes) {
32103
+ await reader.cancel();
32104
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
32105
+ }
32106
+ chunks.push(value);
32107
+ }
32108
+ const bytes = new Uint8Array(total);
32109
+ let offset = 0;
32110
+ for (const chunk of chunks) {
32111
+ bytes.set(chunk, offset);
32112
+ offset += chunk.byteLength;
32113
+ }
32114
+ return bytes;
32115
+ }
32116
+
32117
+ class CryptoHftDataAdapter {
32118
+ baseUrl;
32119
+ request;
32120
+ nowMs;
32121
+ decode;
32122
+ profiles;
32123
+ constructor(options = {}) {
32124
+ this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
32125
+ this.request = options.fetch ?? fetch;
32126
+ this.nowMs = options.nowMs ?? Date.now;
32127
+ this.decode = options.decode ?? decodeCryptoHftParquetZstd;
32128
+ this.profiles = options.profiles ?? [];
32129
+ }
32130
+ capabilityFor(request) {
32131
+ return cryptoHftDataCapabilityFor(request, this.profiles);
32132
+ }
32133
+ async discoverSymbols(providerExchangeId) {
32134
+ if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
32135
+ throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
32136
+ }
32137
+ const endpoint = new URL("/symbols", this.baseUrl);
32138
+ endpoint.searchParams.set("exchange", providerExchangeId);
32139
+ endpoint.searchParams.set("data_type", "orderbook");
32140
+ let response;
32141
+ try {
32142
+ response = await this.request(endpoint);
32143
+ } catch {
32144
+ throw new CryptoHftDataError("symbol_discovery_failed");
32145
+ }
32146
+ if (!response.ok) {
32147
+ throw new CryptoHftDataError("symbol_discovery_failed");
32148
+ }
32149
+ let body;
32150
+ try {
32151
+ body = await response.json();
32152
+ } catch {
32153
+ throw new CryptoHftDataError("symbol_discovery_response_invalid");
32154
+ }
32155
+ const record2 = body;
32156
+ if (!body || typeof body !== "object" || Array.isArray(body) || String(record2.exchange).toLowerCase() !== providerExchangeId || String(record2.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record2.symbols) || !record2.symbols.every((symbol2) => typeof symbol2 === "string" && /^[A-Z0-9_-]+$/.test(symbol2))) {
32157
+ throw new CryptoHftDataError("symbol_discovery_response_invalid");
32158
+ }
32159
+ return [...new Set(record2.symbols)].sort();
32160
+ }
32161
+ async acquire(request, capability, credential) {
32162
+ const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
32163
+ if (typeof apiKey !== "string" || apiKey.length === 0) {
32164
+ throw new CryptoHftDataError("credentials_invalid");
32165
+ }
32166
+ const paths = enumerateCryptoHftDataObjects(request, capability.providerExchangeId, capability.resolvedSymbol);
32167
+ if (paths.length > request.budgets.maxFiles) {
32168
+ throw new CryptoHftDataError("budget_max_files_exceeded");
32169
+ }
32170
+ const started = this.nowMs();
32171
+ const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
32172
+ method: "POST",
32173
+ headers: { "content-type": "application/json", "X-API-Key": apiKey }
32174
+ });
32175
+ if (!tokenResponse.ok)
32176
+ throw new CryptoHftDataError("jwt_issuance_failed");
32177
+ const tokenBody = await tokenResponse.json();
32178
+ if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
32179
+ throw new CryptoHftDataError("jwt_response_invalid");
32180
+ }
32181
+ let totalBytes = 0;
32182
+ let totalRows = 0;
32183
+ const objects = [];
32184
+ const rows = [];
32185
+ for (const path2 of paths) {
32186
+ if (this.nowMs() - started > request.budgets.maxDurationMs) {
32187
+ throw new CryptoHftDataError("budget_max_duration_exceeded");
32188
+ }
32189
+ const endpoint = new URL("/download", this.baseUrl);
32190
+ endpoint.searchParams.set("file", path2);
32191
+ const response = await this.request(endpoint, {
32192
+ headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
32193
+ });
32194
+ if (!response.ok)
32195
+ throw new CryptoHftDataError("object_download_failed");
32196
+ const declaredBytes = Number(response.headers.get("content-length"));
32197
+ if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request.budgets.maxBytes) {
32198
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
32199
+ }
32200
+ const bytes = await readBoundedObject(response, request.budgets.maxBytes - totalBytes);
32201
+ totalBytes += bytes.byteLength;
32202
+ if (totalBytes > request.budgets.maxBytes) {
32203
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
32204
+ }
32205
+ const decoded = await this.decode(bytes);
32206
+ totalRows += decoded.length;
32207
+ if (totalRows > request.budgets.maxRows) {
32208
+ throw new CryptoHftDataError("budget_max_rows_exceeded");
32209
+ }
32210
+ const object2 = {
32211
+ identity: path2,
32212
+ checksum: sha256Bytes(bytes),
32213
+ bytes: bytes.byteLength,
32214
+ rows: decoded.length
32215
+ };
32216
+ objects.push(object2);
32217
+ for (const decodedRow of decoded) {
32218
+ const parsed = parsedDatasetRow(decodedRow, object2);
32219
+ validatedRow(request, parsed);
32220
+ rows.push(parsed);
32221
+ }
32222
+ }
32223
+ if (this.nowMs() - started > request.budgets.maxDurationMs) {
32224
+ throw new CryptoHftDataError("budget_max_duration_exceeded");
32225
+ }
32226
+ return {
32227
+ objects,
32228
+ rows,
32229
+ vendorSemanticDigest: sha256Canonical(rows)
32230
+ };
32231
+ }
32232
+ async normalize(request, capability, dataset, captureBundleId) {
32233
+ const profile = this.profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
32234
+ if (!profile) {
32235
+ throw new CryptoHftDataError("profile_semantics_unavailable");
32236
+ }
32237
+ const samples = reconstructCryptoHftDataOrderBooks(request, dataset.rows, profile);
32238
+ const context2 = {
32239
+ source: EXTERNAL_BACKFILL_SOURCE,
32240
+ deploymentId: "market-data-vendor-backfill",
32241
+ captureBundleId,
32242
+ exchange: request.scope.exchange,
32243
+ symbol: request.scope.tradingPair,
32244
+ tradingPair: request.scope.tradingPair,
32245
+ sourceSymbol: request.scope.sourceSymbol,
32246
+ assetType: request.scope.marketType,
32247
+ feed: "ORDERBOOK",
32248
+ provider: "cryptohftdata",
32249
+ sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
32250
+ schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
32251
+ checksumAlgorithm: CHECKSUM_ALGORITHM,
32252
+ provenanceComplete: true
32253
+ };
32254
+ const rows = samples.flatMap((sample) => {
32255
+ const rawCapture = {
32256
+ rawCaptureId: sha256Canonical({
32257
+ capture_bundle_id: captureBundleId,
32258
+ dataset_object_identity: sample.datasetObjectIdentity,
32259
+ dataset_object_checksum: sample.datasetObjectChecksum,
32260
+ source_time_ms: sample.sourceTimeMs,
32261
+ target_time_ms: sample.targetTimeMs
32262
+ }),
32263
+ rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
32264
+ rawChecksum: sample.datasetObjectChecksum,
32265
+ redactedPayload: {
32266
+ dataset_object_identity: sample.datasetObjectIdentity,
32267
+ dataset_object_checksum: sample.datasetObjectChecksum
32268
+ },
32269
+ eventTimeMs: sample.sourceTimeMs,
32270
+ receivedTimeMs: sample.receivedTimeMs,
32271
+ checksumAlgorithm: CHECKSUM_ALGORITHM
32272
+ };
32273
+ const canonical = buildCanonicalOrderBookRows({
32274
+ context: context2,
32275
+ rawCapture,
32276
+ depthLimit: request.depth,
32277
+ constructionMode: request.constructionMode,
32278
+ snapshot: {
32279
+ bids: sample.bids,
32280
+ asks: sample.asks,
32281
+ timestamp: sample.sourceTimeMs,
32282
+ receivedTimestamp: sample.receivedTimeMs,
32283
+ exchange: request.scope.exchange,
32284
+ symbol: request.scope.tradingPair,
32285
+ depthLimit: request.depth,
32286
+ sequence: sample.sequence
32287
+ }
32288
+ });
32289
+ return [...canonical.levels, canonical.summary];
32290
+ });
32291
+ return {
32292
+ captureBundleId,
32293
+ objects: dataset.objects,
32294
+ rows,
32295
+ vendorSemanticDigest: dataset.vendorSemanticDigest,
32296
+ canonicalSemanticDigest: semanticDigest(rows)
32297
+ };
32499
32298
  }
32500
- return objects;
32501
32299
  }
32502
- function unsignedString(value, field, required2 = false) {
32503
- if (value === null || value === undefined) {
32504
- if (required2)
32505
- throw new CryptoHftDataError(`schema_${field}_missing`);
32506
- return;
32507
- }
32508
- const rendered = String(value);
32509
- if (!/^\d+$/.test(rendered)) {
32510
- throw new CryptoHftDataError(`schema_${field}_invalid`);
32511
- }
32512
- const parsed = BigInt(rendered);
32513
- if (parsed > 18446744073709551615n) {
32514
- throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
32515
- }
32516
- return parsed.toString(10);
32300
+
32301
+ // src/helpers/market-data-vendor-backfill/qualification.ts
32302
+ var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
32303
+ var SHA256 = /^[0-9a-f]{64}$/;
32304
+ var REASON = /^[a-z][a-z0-9_]{0,127}$/;
32305
+ var EVENT_FIELDS = new Set([
32306
+ "qualification_event_id",
32307
+ "capture_bundle_id",
32308
+ "state",
32309
+ "receipt_id",
32310
+ "promotion_identity_sha256",
32311
+ "window",
32312
+ "event_at",
32313
+ "reason_code"
32314
+ ]);
32315
+ function deterministicUuid(value) {
32316
+ const digest = jcsSha256(value).slice(0, 32).split("");
32317
+ digest[12] = "5";
32318
+ digest[16] = "8";
32319
+ const hex3 = digest.join("");
32320
+ return `${hex3.slice(0, 8)}-${hex3.slice(8, 12)}-${hex3.slice(12, 16)}-${hex3.slice(16, 20)}-${hex3.slice(20)}`;
32517
32321
  }
32518
- function timestampMs2(value, field) {
32519
- const rendered = String(value);
32520
- if (!/^\d+$/.test(rendered)) {
32521
- throw new CryptoHftDataError(`schema_${field}_invalid`);
32522
- }
32523
- const raw = BigInt(rendered);
32524
- const milliseconds = field === "received_time" ? raw / 1000000n : raw;
32525
- if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
32526
- throw new CryptoHftDataError(`schema_${field}_unsafe`);
32527
- }
32528
- return Number(milliseconds);
32322
+ function semanticEvent(event) {
32323
+ const { qualification_event_id: _eventId, ...semantic } = event;
32324
+ return semantic;
32529
32325
  }
32530
- function decimal(value, field, allowZero) {
32531
- if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
32532
- throw new CryptoHftDataError(`schema_${field}_invalid`);
32533
- }
32534
- const parsed = Number(value);
32535
- if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
32536
- throw new CryptoHftDataError(`schema_${field}_invalid`);
32326
+ function assertTimestamp(value, field) {
32327
+ const parsed = Date.parse(value);
32328
+ if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
32329
+ throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
32537
32330
  }
32538
32331
  return parsed;
32539
32332
  }
32540
- function validatedRow(request, row) {
32541
- if (row.symbol !== request.scope.sourceSymbol) {
32542
- throw new CryptoHftDataError("schema_symbol_mismatch");
32543
- }
32544
- if (row.event_type !== "snapshot" && row.event_type !== "update") {
32545
- throw new CryptoHftDataError("schema_event_type_invalid");
32333
+ function parseQualificationEvent(value) {
32334
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
32335
+ throw new Error("qualification event must be an object");
32546
32336
  }
32547
- if (row.side !== "bid" && row.side !== "ask") {
32548
- throw new CryptoHftDataError("schema_side_invalid");
32337
+ const event = value;
32338
+ if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
32339
+ throw new Error("qualification event fields are invalid");
32549
32340
  }
32550
- decimal(row.price, "price", false);
32551
- decimal(row.quantity, "quantity", true);
32552
- if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
32553
- throw new CryptoHftDataError("schema_object_checksum_invalid");
32341
+ if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
32342
+ throw new Error("qualification window must be increasing");
32554
32343
  }
32555
- const eventTimeMs = timestampMs2(row.event_time, "event_time");
32556
- const receivedTimeMs = timestampMs2(row.received_time, "received_time");
32557
- if (receivedTimeMs < eventTimeMs) {
32558
- throw new CryptoHftDataError("received_time_precedes_event_time");
32344
+ assertTimestamp(event.event_at, "event_at");
32345
+ if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
32346
+ throw new Error("qualification_event_id does not match event content");
32559
32347
  }
32560
- return { ...row, eventTimeMs, receivedTimeMs };
32561
- }
32562
- function groupKey(row, profile) {
32563
- const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
32564
- return [
32565
- row.event_type,
32566
- row.eventTimeMs,
32567
- sequence,
32568
- row.dataset_object_identity
32569
- ].join("\x00");
32348
+ return event;
32570
32349
  }
32571
- function absent(value) {
32572
- return value === null || value === undefined;
32350
+ function finalizeQualificationEvent(event) {
32351
+ const semantic = semanticEvent(event);
32352
+ return parseQualificationEvent({
32353
+ ...semantic,
32354
+ qualification_event_id: deterministicUuid(semantic)
32355
+ });
32573
32356
  }
32574
- function snapshotSequence(row, profile) {
32575
- if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
32576
- if (String(row.last_update_id) !== "-1") {
32577
- throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
32578
- }
32579
- if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
32580
- throw new CryptoHftDataError("ambiguous_snapshot_group");
32357
+ function qualificationEventToArchiveRow(eventInput) {
32358
+ const event = parseQualificationEvent(eventInput);
32359
+ return {
32360
+ table: "market_data.cex_order_book_capture_qualifications",
32361
+ row: {
32362
+ source: "external_backfill",
32363
+ capture_origin: "vendor_historical_backfill",
32364
+ source_mode: "vendor_historical_backfill_v1",
32365
+ deployment_id: "market-data-vendor-backfill",
32366
+ qualification_event_id: event.qualification_event_id,
32367
+ capture_bundle_id: event.capture_bundle_id,
32368
+ state: event.state,
32369
+ receipt_id: event.receipt_id,
32370
+ promotion_identity_sha256: event.promotion_identity_sha256,
32371
+ window_start_ms: Date.parse(event.window.start_at),
32372
+ window_end_ms: Date.parse(event.window.end_at),
32373
+ event_at_ms: Date.parse(event.event_at),
32374
+ reason_code: event.reason_code,
32375
+ event_json: jcsCanonicalize(event)
32581
32376
  }
32582
- return unsignedString(row.final_update_id, "final_update_id", true);
32583
- }
32584
- return unsignedString(row.last_update_id, "last_update_id", true);
32585
- }
32586
- function sortedSide(levels, side) {
32587
- return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
32588
- }
32589
- function applyRows(state, rows) {
32590
- for (const row of rows) {
32591
- const levels = row.side === "bid" ? state.bids : state.asks;
32592
- const quantity = decimal(row.quantity, "quantity", true);
32593
- if (quantity === 0)
32594
- levels.delete(row.price);
32595
- else
32596
- levels.set(row.price, quantity);
32597
- }
32377
+ };
32598
32378
  }
32599
- function reconstructCryptoHftDataOrderBooks(request, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
32600
- const rows = inputRows.map((row, index) => ({
32601
- ...validatedRow(request, row),
32602
- originalIndex: index
32603
- })).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
32604
- const groups = [];
32605
- for (const row of rows) {
32606
- const current = groups.at(-1);
32607
- if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
32608
- groups.push([row]);
32609
- } else {
32610
- current.push(row);
32611
- }
32612
- }
32613
- const earliestTargetTimeMs = Math.min(...request.requiredClockTargetsMs);
32614
- const latestTargetTimeMs = Math.max(...request.requiredClockTargetsMs);
32615
- const anchorIndex = groups.findIndex((group) => {
32616
- const first = group[0];
32617
- return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
32618
- });
32619
- if (anchorIndex < 0) {
32620
- throw new CryptoHftDataError("update_before_snapshot");
32621
- }
32622
- let state;
32623
- let previousFinalUpdateId;
32624
- const states = [];
32625
- for (const group of groups.slice(anchorIndex)) {
32626
- const first = group[0];
32627
- if (first.eventTimeMs > latestTargetTimeMs)
32628
- break;
32629
- if (first.event_type === "snapshot") {
32630
- const sequence = snapshotSequence(first, profile);
32631
- for (const row of group) {
32632
- if (snapshotSequence(row, profile) !== sequence) {
32633
- throw new CryptoHftDataError("ambiguous_snapshot_group");
32634
- }
32635
- }
32636
- if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
32637
- throw new CryptoHftDataError("snapshot_sequence_regression");
32638
- }
32639
- state = {
32640
- bids: new Map,
32641
- asks: new Map,
32642
- sequence,
32643
- sourceTimeMs: first.eventTimeMs,
32644
- receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
32645
- datasetObjectIdentity: first.dataset_object_identity,
32646
- datasetObjectChecksum: first.dataset_object_checksum
32647
- };
32648
- applyRows(state, group);
32649
- previousFinalUpdateId = BigInt(sequence);
32650
- } else {
32651
- if (!state || previousFinalUpdateId === undefined) {
32652
- throw new CryptoHftDataError("update_before_snapshot");
32653
- }
32654
- const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
32655
- if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
32656
- if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
32657
- throw new CryptoHftDataError("ambiguous_update_group");
32658
- }
32659
- const previous = unsignedString(first.last_update_id, "last_update_id", true);
32660
- for (const row of group) {
32661
- if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
32662
- throw new CryptoHftDataError("ambiguous_update_group");
32663
- }
32664
- }
32665
- if (BigInt(previous) !== previousFinalUpdateId) {
32666
- throw new CryptoHftDataError("update_chain_gap");
32667
- }
32668
- } else {
32669
- const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
32670
- const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
32671
- for (const row of group) {
32672
- if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
32673
- throw new CryptoHftDataError("ambiguous_update_group");
32674
- }
32675
- }
32676
- const expected = previousFinalUpdateId + 1n;
32677
- if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
32678
- throw new CryptoHftDataError("update_chain_gap");
32679
- }
32680
- }
32681
- applyRows(state, group);
32682
- state.sequence = finalUpdate.toString();
32683
- state.sourceTimeMs = first.eventTimeMs;
32684
- state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
32685
- state.datasetObjectIdentity = first.dataset_object_identity;
32686
- state.datasetObjectChecksum = first.dataset_object_checksum;
32687
- previousFinalUpdateId = finalUpdate;
32688
- }
32689
- if (state) {
32690
- states.push({
32691
- ...state,
32692
- bids: new Map(state.bids),
32693
- asks: new Map(state.asks)
32694
- });
32695
- }
32379
+
32380
+ // src/helpers/market-data-vendor-backfill/core.ts
32381
+ function outcome(request, status, reasonCode, extra = {}) {
32382
+ return {
32383
+ status,
32384
+ reasonCode,
32385
+ ...request ? {
32386
+ requestId: request.requestId,
32387
+ idempotencyKey: request.idempotencyKey,
32388
+ ...request.target ? { target: request.target } : {}
32389
+ } : {},
32390
+ ...extra
32391
+ };
32392
+ }
32393
+ function captureBundleId(request, capability, dataset) {
32394
+ return sha256Canonical({
32395
+ request_business_identity: request.idempotencyKey,
32396
+ provider: capability.provider,
32397
+ provider_exchange_id: capability.providerExchangeId,
32398
+ resolved_symbol: capability.resolvedSymbol,
32399
+ adapter_version: capability.adapterVersion,
32400
+ objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
32401
+ identity,
32402
+ checksum,
32403
+ bytes,
32404
+ rows
32405
+ })),
32406
+ canonical_schema_version: request.expectedProduct.canonicalSchemaVersion,
32407
+ checksum_algorithm: request.expectedProduct.checksumAlgorithm
32408
+ });
32409
+ }
32410
+ function buildPromotionReceipt(request, capability, normalized, verification, verificationTimeMs) {
32411
+ if (!request.wire || !request.initialSelection || !request.expectedCanonicalSchema || !request.coveragePolicy || !request.productPins) {
32412
+ throw new Error("decoded final-v1 request context is missing");
32696
32413
  }
32697
- const samples = [];
32698
- for (const targetTimeMs of request.requiredClockTargetsMs) {
32699
- let prior;
32700
- for (const candidate of states) {
32701
- if (candidate.sourceTimeMs > targetTimeMs)
32414
+ return finalizePromotionReceipt({
32415
+ schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
32416
+ verified_at: new Date(verificationTimeMs).toISOString(),
32417
+ request_id: request.requestId,
32418
+ idempotency_key: request.idempotencyKey,
32419
+ source: EXTERNAL_BACKFILL_SOURCE,
32420
+ capture_origin: "vendor_historical_backfill",
32421
+ source_mode: "vendor_historical_backfill_v1",
32422
+ provider: capability.provider,
32423
+ adapter_version: capability.adapterVersion,
32424
+ effective_policies: {
32425
+ capability_policy: request.productPins.capability_policy,
32426
+ resource_policy: request.productPins.resource_policy,
32427
+ adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
32428
+ acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
32429
+ },
32430
+ capture_bundle_id: normalized.captureBundleId,
32431
+ scope: request.wire.scope,
32432
+ window: request.wire.window,
32433
+ depth: request.depth,
32434
+ construction_mode: request.constructionMode,
32435
+ canonical_schema: request.expectedCanonicalSchema,
32436
+ coverage_policy: request.coveragePolicy,
32437
+ selection_sha256: request.initialSelection.selection_sha256,
32438
+ vendor_semantic_digest: normalized.vendorSemanticDigest,
32439
+ canonical_semantic_digest: verification.canonicalSemanticDigest,
32440
+ prefix_digest: verification.prefixDigest,
32441
+ suffix_digest: verification.suffixDigest,
32442
+ seam_verified: true,
32443
+ coverage_verified: true,
32444
+ dataset_objects: normalized.objects
32445
+ });
32446
+ }
32447
+ async function submitAll(dependencies, batches) {
32448
+ const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
32449
+ for (const batch of batches) {
32450
+ let accepted = false;
32451
+ for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
32452
+ try {
32453
+ const response = await dependencies.forwarder.submit(batch);
32454
+ accepted = response.ok && response.inserted === batch.rows.length;
32455
+ } catch {
32456
+ accepted = false;
32457
+ }
32458
+ if (accepted)
32702
32459
  break;
32703
- prior = candidate;
32460
+ if (attempt < maxAttempts)
32461
+ await dependencies.retry?.wait?.(attempt);
32704
32462
  }
32705
- if (!prior || targetTimeMs - prior.sourceTimeMs > request.maxPriorAsOfLagMs) {
32706
- throw new CryptoHftDataError("required_clock_coverage_insufficient");
32463
+ if (!accepted)
32464
+ return false;
32465
+ }
32466
+ return true;
32467
+ }
32468
+ function stableFailureReason(error48, fallback) {
32469
+ if (error48 && typeof error48 === "object") {
32470
+ const reason = error48.reason;
32471
+ if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
32472
+ return reason;
32707
32473
  }
32708
- const bids = sortedSide(prior.bids, "bid").slice(0, request.depth);
32709
- const asks = sortedSide(prior.asks, "ask").slice(0, request.depth);
32710
- if (bids.length === 0 || asks.length === 0) {
32711
- throw new CryptoHftDataError("book_side_missing");
32474
+ }
32475
+ return fallback;
32476
+ }
32477
+ function assertArchivePreflight(request, resolution) {
32478
+ if (!request.target || !request.productionAuthorizationId) {
32479
+ throw new Error("request target or production authorization ID is missing");
32480
+ }
32481
+ const selection = archiveSelectionCodec.decode(resolution.selection);
32482
+ const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
32483
+ const receiptById = new Map;
32484
+ for (const receipt of receipts) {
32485
+ const existing = receiptById.get(receipt.receipt_id);
32486
+ if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
32487
+ throw new Error("stored receipt identity has conflicting content");
32712
32488
  }
32713
- if (bids[0]?.[0] >= asks[0]?.[0]) {
32714
- throw new CryptoHftDataError("book_crossed_or_locked");
32489
+ receiptById.set(receipt.receipt_id, receipt);
32490
+ }
32491
+ for (const bundle of selection.bundles) {
32492
+ if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
32493
+ throw new Error("vendor selection lacks its validated stored receipt");
32715
32494
  }
32716
- samples.push({
32717
- targetTimeMs,
32718
- sourceTimeMs: prior.sourceTimeMs,
32719
- receivedTimeMs: prior.receivedTimeMs,
32720
- sequence: prior.sequence,
32721
- bids,
32722
- asks,
32723
- datasetObjectIdentity: prior.datasetObjectIdentity,
32724
- datasetObjectChecksum: prior.datasetObjectChecksum
32725
- });
32726
32495
  }
32727
- return samples;
32496
+ if (resolution.readerIdentity.environment !== request.target.environment || resolution.readerIdentity.cluster !== request.target.cluster) {
32497
+ throw new Error("archive reader cluster identity mismatch");
32498
+ }
32728
32499
  }
32729
- function sha256Bytes(bytes) {
32730
- return createHash4("sha256").update(bytes).digest("hex");
32500
+ function assertForwarderPreflight(request, resolution, nowMs) {
32501
+ if (!request.target || !request.productionAuthorizationId) {
32502
+ throw new Error("request target or production authorization ID is missing");
32503
+ }
32504
+ if (resolution.forwarderIdentity.environment !== request.target.environment || resolution.forwarderIdentity.cluster !== request.target.cluster) {
32505
+ throw new Error("archive forwarder cluster identity mismatch");
32506
+ }
32507
+ const authorization = resolution.authorization;
32508
+ const expiresAtMs = Date.parse(authorization.expiresAt);
32509
+ if (authorization.authorizationId !== request.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request.target.environment || authorization.cluster !== request.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
32510
+ throw new Error("production forwarder authorization is invalid");
32511
+ }
32731
32512
  }
32732
- async function decodeCryptoHftParquetZstd(bytes) {
32733
- const parquet = decompress(bytes);
32734
- const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
32735
- return parquetReadObjects({ file: buffer });
32513
+ function resourcePolicyScopeExceeded(request) {
32514
+ return request.depth > RESOURCE_POLICY.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
32736
32515
  }
32737
- function parsedDatasetRow(value, object2) {
32738
- return {
32739
- received_time: value.received_time,
32740
- event_time: value.event_time,
32741
- transaction_time: value.transaction_time,
32742
- symbol: value.symbol,
32743
- event_type: value.event_type,
32744
- first_update_id: value.first_update_id,
32745
- final_update_id: value.final_update_id,
32746
- prev_final_update_id: value.prev_final_update_id,
32747
- last_update_id: value.last_update_id,
32748
- side: value.side,
32749
- price: value.price,
32750
- quantity: value.quantity,
32751
- order_count: value.order_count,
32752
- dataset_object_identity: object2.identity,
32753
- dataset_object_checksum: object2.checksum
32754
- };
32516
+ function storedReceiptForSelection(resolution) {
32517
+ const receiptId = resolution.selection.receipt_ids[0];
32518
+ return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
32755
32519
  }
32756
- async function readBoundedObject(response, remainingBytes) {
32757
- if (!response.body)
32758
- return new Uint8Array;
32759
- const reader = response.body.getReader();
32760
- const chunks = [];
32761
- let total = 0;
32762
- while (true) {
32763
- const { done, value } = await reader.read();
32764
- if (done)
32765
- break;
32766
- total += value.byteLength;
32767
- if (total > remainingBytes) {
32768
- await reader.cancel();
32769
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
32770
- }
32771
- chunks.push(value);
32520
+ async function runMarketDataVendorBackfill(input, dependencies) {
32521
+ let request;
32522
+ try {
32523
+ const documents = input;
32524
+ request = decodeBackfillRunDocuments({
32525
+ request: documents.request,
32526
+ requiredClock: documents.requiredClock
32527
+ });
32528
+ } catch {
32529
+ return outcome(undefined, "request_invalid", "request_invalid");
32772
32530
  }
32773
- const bytes = new Uint8Array(total);
32774
- let offset = 0;
32775
- for (const chunk of chunks) {
32776
- bytes.set(chunk, offset);
32777
- offset += chunk.byteLength;
32531
+ let initialResolution;
32532
+ try {
32533
+ initialResolution = await dependencies.archive.resolveSelection(request);
32534
+ assertArchivePreflight(request, initialResolution);
32535
+ const forwarderPreflight = await dependencies.forwarder.preflight({
32536
+ authorizationId: request.productionAuthorizationId,
32537
+ target: request.target
32538
+ });
32539
+ assertForwarderPreflight(request, forwarderPreflight, dependencies.clock.nowMs());
32540
+ } catch {
32541
+ return outcome(request, "archive_preflight_failed", "archive_preflight_failed");
32778
32542
  }
32779
- return bytes;
32780
- }
32781
-
32782
- class CryptoHftDataAdapter {
32783
- baseUrl;
32784
- request;
32785
- nowMs;
32786
- decode;
32787
- profiles;
32788
- constructor(options = {}) {
32789
- this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
32790
- this.request = options.fetch ?? fetch;
32791
- this.nowMs = options.nowMs ?? Date.now;
32792
- this.decode = options.decode ?? decodeCryptoHftParquetZstd;
32793
- this.profiles = options.profiles ?? [];
32543
+ if (initialResolution.selection.coverage_class === "complete") {
32544
+ const receipt2 = storedReceiptForSelection(initialResolution);
32545
+ return outcome(request, "already_covered", "qualified_coverage_complete", {
32546
+ selection: initialResolution.selection,
32547
+ ...receipt2 ? { receipt: receipt2 } : {}
32548
+ });
32549
+ }
32550
+ if (resourcePolicyScopeExceeded(request)) {
32551
+ return outcome(request, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
32552
+ }
32553
+ if (request.sourcePolicy === "fill_gaps" && (request.productPins?.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request.productPins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256)) {
32554
+ return outcome(request, "capability_unsupported", "scope_unsupported", {
32555
+ reasonSubcode: "fill_gaps_requires_capability_policy_v2"
32556
+ });
32557
+ }
32558
+ let capability;
32559
+ try {
32560
+ capability = dependencies.providers.capabilityFor(request);
32561
+ } catch {
32562
+ return outcome(request, "capability_unsupported", "capability_probe_failed");
32563
+ }
32564
+ if (!capability) {
32565
+ return outcome(request, "capability_unsupported", "scope_unsupported");
32566
+ }
32567
+ if (!request.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
32568
+ return outcome(request, "capability_unsupported", "adapter_version_unpinned");
32794
32569
  }
32795
- capabilityFor(request) {
32796
- return cryptoHftDataCapabilityFor(request, this.profiles);
32570
+ let credential;
32571
+ try {
32572
+ credential = await dependencies.credentials.resolve(capability.provider);
32573
+ } catch {
32574
+ return outcome(request, "credentials_missing", "credential_resolution_failed");
32797
32575
  }
32798
- async discoverSymbols(providerExchangeId) {
32799
- if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
32800
- throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
32801
- }
32802
- const endpoint = new URL("/symbols", this.baseUrl);
32803
- endpoint.searchParams.set("exchange", providerExchangeId);
32804
- endpoint.searchParams.set("data_type", "orderbook");
32805
- let response;
32806
- try {
32807
- response = await this.request(endpoint);
32808
- } catch {
32809
- throw new CryptoHftDataError("symbol_discovery_failed");
32810
- }
32811
- if (!response.ok) {
32812
- throw new CryptoHftDataError("symbol_discovery_failed");
32813
- }
32814
- let body;
32815
- try {
32816
- body = await response.json();
32817
- } catch {
32818
- throw new CryptoHftDataError("symbol_discovery_response_invalid");
32819
- }
32820
- const record2 = body;
32821
- if (!body || typeof body !== "object" || Array.isArray(body) || String(record2.exchange).toLowerCase() !== providerExchangeId || String(record2.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record2.symbols) || !record2.symbols.every((symbol2) => typeof symbol2 === "string" && /^[A-Z0-9_-]+$/.test(symbol2))) {
32822
- throw new CryptoHftDataError("symbol_discovery_response_invalid");
32823
- }
32824
- return [...new Set(record2.symbols)].sort();
32576
+ if (credential === undefined || credential === null) {
32577
+ return outcome(request, "credentials_missing", "provider_credentials_missing");
32825
32578
  }
32826
- async acquire(request, capability, credential) {
32827
- const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
32828
- if (typeof apiKey !== "string" || apiKey.length === 0) {
32829
- throw new CryptoHftDataError("credentials_invalid");
32830
- }
32831
- const paths = enumerateCryptoHftDataObjects(request, capability.providerExchangeId, capability.resolvedSymbol);
32832
- if (paths.length > request.budgets.maxFiles) {
32833
- throw new CryptoHftDataError("budget_max_files_exceeded");
32579
+ let dataset;
32580
+ let normalized;
32581
+ try {
32582
+ const acquisitionRequest = providerAcquisitionRequest(request);
32583
+ dataset = await dependencies.providers.acquire(acquisitionRequest, capability, credential);
32584
+ const bundleId = captureBundleId(request, capability, dataset);
32585
+ normalized = await dependencies.providers.normalize(acquisitionRequest, capability, dataset, bundleId);
32586
+ if (normalized.captureBundleId !== bundleId) {
32587
+ return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
32588
+ reasonSubcode: "capture_identity_mismatch"
32589
+ });
32834
32590
  }
32835
- const started = this.nowMs();
32836
- const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
32837
- method: "POST",
32838
- headers: { "content-type": "application/json", "X-API-Key": apiKey }
32591
+ } catch (error48) {
32592
+ const reason = stableFailureReason(error48, "provider_dataset_invalid");
32593
+ return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
32594
+ reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
32839
32595
  });
32840
- if (!tokenResponse.ok)
32841
- throw new CryptoHftDataError("jwt_issuance_failed");
32842
- const tokenBody = await tokenResponse.json();
32843
- if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
32844
- throw new CryptoHftDataError("jwt_response_invalid");
32596
+ }
32597
+ try {
32598
+ const candidateBatches = buildForwarderBatches({
32599
+ captureBundleId: normalized.captureBundleId,
32600
+ deploymentId: "market-data-vendor-backfill",
32601
+ rows: normalized.rows
32602
+ });
32603
+ if (!await submitAll(dependencies, candidateBatches)) {
32604
+ return outcome(request, "archive_ingest_failed", "candidate_batch_rejected");
32845
32605
  }
32846
- let totalBytes = 0;
32847
- let totalRows = 0;
32848
- const objects = [];
32849
- const rows = [];
32850
- for (const path2 of paths) {
32851
- if (this.nowMs() - started > request.budgets.maxDurationMs) {
32852
- throw new CryptoHftDataError("budget_max_duration_exceeded");
32853
- }
32854
- const endpoint = new URL("/download", this.baseUrl);
32855
- endpoint.searchParams.set("file", path2);
32856
- const response = await this.request(endpoint, {
32857
- headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
32858
- });
32859
- if (!response.ok)
32860
- throw new CryptoHftDataError("object_download_failed");
32861
- const declaredBytes = Number(response.headers.get("content-length"));
32862
- if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request.budgets.maxBytes) {
32863
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
32864
- }
32865
- const bytes = await readBoundedObject(response, request.budgets.maxBytes - totalBytes);
32866
- totalBytes += bytes.byteLength;
32867
- if (totalBytes > request.budgets.maxBytes) {
32868
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
32869
- }
32870
- const decoded = await this.decode(bytes);
32871
- totalRows += decoded.length;
32872
- if (totalRows > request.budgets.maxRows) {
32873
- throw new CryptoHftDataError("budget_max_rows_exceeded");
32874
- }
32875
- const object2 = {
32876
- identity: path2,
32877
- checksum: sha256Bytes(bytes),
32878
- bytes: bytes.byteLength,
32879
- rows: decoded.length
32880
- };
32881
- objects.push(object2);
32882
- for (const decodedRow of decoded) {
32883
- const parsed = parsedDatasetRow(decodedRow, object2);
32884
- validatedRow(request, parsed);
32885
- rows.push(parsed);
32886
- }
32606
+ } catch {
32607
+ return outcome(request, "archive_ingest_failed", "candidate_submission_failed");
32608
+ }
32609
+ let verification;
32610
+ try {
32611
+ verification = await dependencies.archive.verifyCandidate(request, normalized, normalized.captureBundleId, initialResolution);
32612
+ } catch {
32613
+ return outcome(request, "promotion_verification_failed", "candidate_query_failed");
32614
+ }
32615
+ if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
32616
+ return outcome(request, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
32617
+ }
32618
+ let receipt;
32619
+ try {
32620
+ receipt = buildPromotionReceipt(request, capability, normalized, verification, dependencies.clock.nowMs());
32621
+ const [batch] = buildForwarderBatches({
32622
+ captureBundleId: normalized.captureBundleId,
32623
+ deploymentId: "market-data-vendor-backfill",
32624
+ rows: [promotionReceiptToArchiveRow(receipt)]
32625
+ });
32626
+ if (!batch || !await submitAll(dependencies, [batch])) {
32627
+ return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
32887
32628
  }
32888
- if (this.nowMs() - started > request.budgets.maxDurationMs) {
32889
- throw new CryptoHftDataError("budget_max_duration_exceeded");
32629
+ const qualification = finalizeQualificationEvent({
32630
+ capture_bundle_id: receipt.capture_bundle_id,
32631
+ state: "qualified",
32632
+ receipt_id: receipt.receipt_id,
32633
+ promotion_identity_sha256: receipt.promotion_identity_sha256,
32634
+ window: receipt.window,
32635
+ event_at: receipt.verified_at,
32636
+ reason_code: "promotion_verified"
32637
+ });
32638
+ const [qualificationBatch] = buildForwarderBatches({
32639
+ captureBundleId: normalized.captureBundleId,
32640
+ deploymentId: "market-data-vendor-backfill",
32641
+ rows: [qualificationEventToArchiveRow(qualification)]
32642
+ });
32643
+ if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
32644
+ return outcome(request, "archive_ingest_failed", "qualification_commit_failed", { receipt });
32890
32645
  }
32891
- return {
32892
- objects,
32893
- rows,
32894
- vendorSemanticDigest: sha256Canonical(rows)
32895
- };
32646
+ } catch {
32647
+ return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
32896
32648
  }
32897
- async normalize(request, capability, dataset, captureBundleId2) {
32898
- const profile = this.profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
32899
- if (!profile) {
32900
- throw new CryptoHftDataError("profile_semantics_unavailable");
32901
- }
32902
- const samples = reconstructCryptoHftDataOrderBooks(request, dataset.rows, profile);
32903
- const context2 = {
32904
- source: EXTERNAL_BACKFILL_SOURCE,
32649
+ let finalResolution;
32650
+ try {
32651
+ finalResolution = await dependencies.archive.resolveSelection(request);
32652
+ assertArchivePreflight(request, finalResolution);
32653
+ } catch (error48) {
32654
+ return outcome(request, "promotion_verification_failed", "post_promotion_selection_failed", {
32655
+ receipt,
32656
+ reasonSubcode: stableFailureReason(error48, "archive_selection_resolution_failed")
32657
+ });
32658
+ }
32659
+ if (finalResolution.selection.coverage_class !== "complete") {
32660
+ return outcome(request, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
32661
+ }
32662
+ if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
32663
+ return outcome(request, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
32664
+ }
32665
+ try {
32666
+ if (!request.wire)
32667
+ throw new Error("decoded request wire is missing");
32668
+ const [selectionBatch] = buildForwarderBatches({
32669
+ captureBundleId: receipt.capture_bundle_id,
32905
32670
  deploymentId: "market-data-vendor-backfill",
32906
- captureBundleId: captureBundleId2,
32907
- exchange: request.scope.exchange,
32908
- symbol: request.scope.tradingPair,
32909
- tradingPair: request.scope.tradingPair,
32910
- sourceSymbol: request.scope.sourceSymbol,
32911
- assetType: request.scope.marketType,
32912
- feed: "ORDERBOOK",
32913
- provider: "cryptohftdata",
32914
- sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
32915
- schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
32916
- checksumAlgorithm: CHECKSUM_ALGORITHM,
32917
- provenanceComplete: true
32918
- };
32919
- const rows = samples.flatMap((sample) => {
32920
- const rawCapture = {
32921
- rawCaptureId: sha256Canonical({
32922
- capture_bundle_id: captureBundleId2,
32923
- dataset_object_identity: sample.datasetObjectIdentity,
32924
- dataset_object_checksum: sample.datasetObjectChecksum,
32925
- source_time_ms: sample.sourceTimeMs,
32926
- target_time_ms: sample.targetTimeMs
32927
- }),
32928
- rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
32929
- rawChecksum: sample.datasetObjectChecksum,
32930
- redactedPayload: {
32931
- dataset_object_identity: sample.datasetObjectIdentity,
32932
- dataset_object_checksum: sample.datasetObjectChecksum
32933
- },
32934
- eventTimeMs: sample.sourceTimeMs,
32935
- receivedTimeMs: sample.receivedTimeMs,
32936
- checksumAlgorithm: CHECKSUM_ALGORITHM
32937
- };
32938
- const canonical = buildCanonicalOrderBookRows({
32939
- context: context2,
32940
- rawCapture,
32941
- depthLimit: request.depth,
32942
- constructionMode: request.constructionMode,
32943
- snapshot: {
32944
- bids: sample.bids,
32945
- asks: sample.asks,
32946
- timestamp: sample.sourceTimeMs,
32947
- receivedTimestamp: sample.receivedTimeMs,
32948
- exchange: request.scope.exchange,
32949
- symbol: request.scope.tradingPair,
32950
- depthLimit: request.depth,
32951
- sequence: sample.sequence
32952
- }
32953
- });
32954
- return [...canonical.levels, canonical.summary];
32671
+ rows: [
32672
+ archiveSelectionToArchiveRow(request.wire, finalResolution.selection)
32673
+ ]
32955
32674
  });
32956
- return {
32957
- captureBundleId: captureBundleId2,
32958
- objects: dataset.objects,
32959
- rows,
32960
- vendorSemanticDigest: dataset.vendorSemanticDigest,
32961
- canonicalSemanticDigest: semanticDigest(rows)
32962
- };
32675
+ if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
32676
+ return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
32677
+ }
32678
+ } catch {
32679
+ return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
32963
32680
  }
32681
+ return outcome(request, "promoted", "promotion_qualified", {
32682
+ receipt,
32683
+ selection: finalResolution.selection
32684
+ });
32964
32685
  }
32965
32686
 
32966
32687
  // src/helpers/market-data-vendor-backfill/forwarder-client.ts
@@ -33081,8 +32802,8 @@ function validateReleaseIdentity(release) {
33081
32802
  return release;
33082
32803
  }
33083
32804
  function bakedReleaseIdentity() {
33084
- const packageVersion = "0.2.47";
33085
- const gitHead = "f1d81afe22d3e750317f55b04fb4dcdf712dca36";
32805
+ const packageVersion = "0.2.49";
32806
+ const gitHead = "c1ffe071e7696c5b17c8f058da01d61cb27ec8a4";
33086
32807
  return validateReleaseIdentity({ packageVersion, gitHead });
33087
32808
  }
33088
32809
  function readOptionalEnvironment(environment, name) {
@@ -33313,4 +33034,4 @@ export {
33313
33034
  createBackfillDependenciesFromEnv
33314
33035
  };
33315
33036
 
33316
- //# debugId=A982A2E4437AC4DE64756E2164756E21
33037
+ //# debugId=793AA74075FC03DE64756E2164756E21