@usherlabs/cex-broker 0.2.48 → 0.2.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/dist/commands/cex-canonical-orderbook-export.js +72 -358
  2. package/dist/commands/cex-canonical-orderbook-export.js.map +7 -19
  3. package/dist/commands/cli.js +224 -67
  4. package/dist/commands/market-data-vendor-backfill.js +975 -1198
  5. package/dist/commands/market-data-vendor-backfill.js.map +13 -25
  6. package/dist/helpers/logger.d.ts +4 -0
  7. package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +3 -1
  8. package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +58 -4
  9. package/dist/helpers/market-data-vendor-backfill/semantic-verification.d.ts +1 -0
  10. package/dist/helpers/otel.d.ts +1 -0
  11. package/dist/helpers/trace-context.d.ts +8 -0
  12. package/dist/index.js +183 -63
  13. package/dist/index.js.map +9 -8
  14. package/dist/market-data-preparation/fixtures/conformance-v2.json +8 -8
  15. package/dist/market-data-preparation/policies/capability-policy-v1.json +28 -0
  16. package/dist/market-data-preparation/policies/capability-policy.json +14 -3
  17. package/dist/market-data-preparation/policies/resource-policy-v1.json +16 -0
  18. package/dist/market-data-preparation/policies/resource-policy.json +3 -3
  19. package/dist/market-data-preparation.js +70 -359
  20. package/dist/market-data-preparation.js.map +9 -21
  21. package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +39 -0
  22. package/dist/market-data-vendor-backfill/policies/resource-policy-v2.json +16 -0
  23. package/dist/market-data-vendor-backfill.d.ts +2 -2
  24. package/dist/market-data-vendor-backfill.js +1528 -1750
  25. package/dist/market-data-vendor-backfill.js.map +13 -25
  26. package/package.json +1 -1
@@ -8257,241 +8257,6 @@ var require_src = __commonJS((exports) => {
8257
8257
  };
8258
8258
  });
8259
8259
 
8260
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/types/LogRecord.js
8261
- var require_LogRecord = __commonJS((exports) => {
8262
- Object.defineProperty(exports, "__esModule", { value: true });
8263
- exports.SeverityNumber = undefined;
8264
- var SeverityNumber;
8265
- (function(SeverityNumber2) {
8266
- SeverityNumber2[SeverityNumber2["UNSPECIFIED"] = 0] = "UNSPECIFIED";
8267
- SeverityNumber2[SeverityNumber2["TRACE"] = 1] = "TRACE";
8268
- SeverityNumber2[SeverityNumber2["TRACE2"] = 2] = "TRACE2";
8269
- SeverityNumber2[SeverityNumber2["TRACE3"] = 3] = "TRACE3";
8270
- SeverityNumber2[SeverityNumber2["TRACE4"] = 4] = "TRACE4";
8271
- SeverityNumber2[SeverityNumber2["DEBUG"] = 5] = "DEBUG";
8272
- SeverityNumber2[SeverityNumber2["DEBUG2"] = 6] = "DEBUG2";
8273
- SeverityNumber2[SeverityNumber2["DEBUG3"] = 7] = "DEBUG3";
8274
- SeverityNumber2[SeverityNumber2["DEBUG4"] = 8] = "DEBUG4";
8275
- SeverityNumber2[SeverityNumber2["INFO"] = 9] = "INFO";
8276
- SeverityNumber2[SeverityNumber2["INFO2"] = 10] = "INFO2";
8277
- SeverityNumber2[SeverityNumber2["INFO3"] = 11] = "INFO3";
8278
- SeverityNumber2[SeverityNumber2["INFO4"] = 12] = "INFO4";
8279
- SeverityNumber2[SeverityNumber2["WARN"] = 13] = "WARN";
8280
- SeverityNumber2[SeverityNumber2["WARN2"] = 14] = "WARN2";
8281
- SeverityNumber2[SeverityNumber2["WARN3"] = 15] = "WARN3";
8282
- SeverityNumber2[SeverityNumber2["WARN4"] = 16] = "WARN4";
8283
- SeverityNumber2[SeverityNumber2["ERROR"] = 17] = "ERROR";
8284
- SeverityNumber2[SeverityNumber2["ERROR2"] = 18] = "ERROR2";
8285
- SeverityNumber2[SeverityNumber2["ERROR3"] = 19] = "ERROR3";
8286
- SeverityNumber2[SeverityNumber2["ERROR4"] = 20] = "ERROR4";
8287
- SeverityNumber2[SeverityNumber2["FATAL"] = 21] = "FATAL";
8288
- SeverityNumber2[SeverityNumber2["FATAL2"] = 22] = "FATAL2";
8289
- SeverityNumber2[SeverityNumber2["FATAL3"] = 23] = "FATAL3";
8290
- SeverityNumber2[SeverityNumber2["FATAL4"] = 24] = "FATAL4";
8291
- })(SeverityNumber = exports.SeverityNumber || (exports.SeverityNumber = {}));
8292
- });
8293
-
8294
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLogger.js
8295
- var require_NoopLogger = __commonJS((exports) => {
8296
- Object.defineProperty(exports, "__esModule", { value: true });
8297
- exports.NOOP_LOGGER = exports.NoopLogger = undefined;
8298
-
8299
- class NoopLogger {
8300
- emit(_logRecord) {}
8301
- }
8302
- exports.NoopLogger = NoopLogger;
8303
- exports.NOOP_LOGGER = new NoopLogger;
8304
- });
8305
-
8306
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLoggerProvider.js
8307
- var require_NoopLoggerProvider = __commonJS((exports) => {
8308
- Object.defineProperty(exports, "__esModule", { value: true });
8309
- exports.NOOP_LOGGER_PROVIDER = exports.NoopLoggerProvider = undefined;
8310
- var NoopLogger_1 = require_NoopLogger();
8311
-
8312
- class NoopLoggerProvider {
8313
- getLogger(_name, _version, _options) {
8314
- return new NoopLogger_1.NoopLogger;
8315
- }
8316
- }
8317
- exports.NoopLoggerProvider = NoopLoggerProvider;
8318
- exports.NOOP_LOGGER_PROVIDER = new NoopLoggerProvider;
8319
- });
8320
-
8321
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLogger.js
8322
- var require_ProxyLogger = __commonJS((exports) => {
8323
- Object.defineProperty(exports, "__esModule", { value: true });
8324
- exports.ProxyLogger = undefined;
8325
- var NoopLogger_1 = require_NoopLogger();
8326
-
8327
- class ProxyLogger {
8328
- constructor(_provider, name, version2, options) {
8329
- this._provider = _provider;
8330
- this.name = name;
8331
- this.version = version2;
8332
- this.options = options;
8333
- }
8334
- emit(logRecord) {
8335
- this._getLogger().emit(logRecord);
8336
- }
8337
- _getLogger() {
8338
- if (this._delegate) {
8339
- return this._delegate;
8340
- }
8341
- const logger = this._provider._getDelegateLogger(this.name, this.version, this.options);
8342
- if (!logger) {
8343
- return NoopLogger_1.NOOP_LOGGER;
8344
- }
8345
- this._delegate = logger;
8346
- return this._delegate;
8347
- }
8348
- }
8349
- exports.ProxyLogger = ProxyLogger;
8350
- });
8351
-
8352
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLoggerProvider.js
8353
- var require_ProxyLoggerProvider = __commonJS((exports) => {
8354
- Object.defineProperty(exports, "__esModule", { value: true });
8355
- exports.ProxyLoggerProvider = undefined;
8356
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
8357
- var ProxyLogger_1 = require_ProxyLogger();
8358
-
8359
- class ProxyLoggerProvider {
8360
- getLogger(name, version2, options) {
8361
- var _a2;
8362
- return (_a2 = this._getDelegateLogger(name, version2, options)) !== null && _a2 !== undefined ? _a2 : new ProxyLogger_1.ProxyLogger(this, name, version2, options);
8363
- }
8364
- _getDelegate() {
8365
- var _a2;
8366
- return (_a2 = this._delegate) !== null && _a2 !== undefined ? _a2 : NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
8367
- }
8368
- _setDelegate(delegate) {
8369
- this._delegate = delegate;
8370
- }
8371
- _getDelegateLogger(name, version2, options) {
8372
- var _a2;
8373
- return (_a2 = this._delegate) === null || _a2 === undefined ? undefined : _a2.getLogger(name, version2, options);
8374
- }
8375
- }
8376
- exports.ProxyLoggerProvider = ProxyLoggerProvider;
8377
- });
8378
-
8379
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/globalThis.js
8380
- var require_globalThis2 = __commonJS((exports) => {
8381
- Object.defineProperty(exports, "__esModule", { value: true });
8382
- exports._globalThis = undefined;
8383
- exports._globalThis = typeof globalThis === "object" ? globalThis : global;
8384
- });
8385
-
8386
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/index.js
8387
- var require_node2 = __commonJS((exports) => {
8388
- Object.defineProperty(exports, "__esModule", { value: true });
8389
- exports._globalThis = undefined;
8390
- var globalThis_1 = require_globalThis2();
8391
- Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
8392
- return globalThis_1._globalThis;
8393
- } });
8394
- });
8395
-
8396
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/index.js
8397
- var require_platform2 = __commonJS((exports) => {
8398
- Object.defineProperty(exports, "__esModule", { value: true });
8399
- exports._globalThis = undefined;
8400
- var node_1 = require_node2();
8401
- Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
8402
- return node_1._globalThis;
8403
- } });
8404
- });
8405
-
8406
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/internal/global-utils.js
8407
- var require_global_utils2 = __commonJS((exports) => {
8408
- Object.defineProperty(exports, "__esModule", { value: true });
8409
- exports.API_BACKWARDS_COMPATIBILITY_VERSION = exports.makeGetter = exports._global = exports.GLOBAL_LOGS_API_KEY = undefined;
8410
- var platform_1 = require_platform2();
8411
- exports.GLOBAL_LOGS_API_KEY = Symbol.for("io.opentelemetry.js.api.logs");
8412
- exports._global = platform_1._globalThis;
8413
- function makeGetter(requiredVersion, instance, fallback) {
8414
- return (version2) => version2 === requiredVersion ? instance : fallback;
8415
- }
8416
- exports.makeGetter = makeGetter;
8417
- exports.API_BACKWARDS_COMPATIBILITY_VERSION = 1;
8418
- });
8419
-
8420
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/api/logs.js
8421
- var require_logs = __commonJS((exports) => {
8422
- Object.defineProperty(exports, "__esModule", { value: true });
8423
- exports.LogsAPI = undefined;
8424
- var global_utils_1 = require_global_utils2();
8425
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
8426
- var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
8427
-
8428
- class LogsAPI {
8429
- constructor() {
8430
- this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
8431
- }
8432
- static getInstance() {
8433
- if (!this._instance) {
8434
- this._instance = new LogsAPI;
8435
- }
8436
- return this._instance;
8437
- }
8438
- setGlobalLoggerProvider(provider) {
8439
- if (global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) {
8440
- return this.getLoggerProvider();
8441
- }
8442
- global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY] = (0, global_utils_1.makeGetter)(global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION, provider, NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER);
8443
- this._proxyLoggerProvider._setDelegate(provider);
8444
- return provider;
8445
- }
8446
- getLoggerProvider() {
8447
- var _a2, _b;
8448
- return (_b = (_a2 = global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) === null || _a2 === undefined ? undefined : _a2.call(global_utils_1._global, global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION)) !== null && _b !== undefined ? _b : this._proxyLoggerProvider;
8449
- }
8450
- getLogger(name, version2, options) {
8451
- return this.getLoggerProvider().getLogger(name, version2, options);
8452
- }
8453
- disable() {
8454
- delete global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY];
8455
- this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
8456
- }
8457
- }
8458
- exports.LogsAPI = LogsAPI;
8459
- });
8460
-
8461
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/index.js
8462
- var require_src2 = __commonJS((exports) => {
8463
- Object.defineProperty(exports, "__esModule", { value: true });
8464
- exports.logs = exports.ProxyLoggerProvider = exports.ProxyLogger = exports.NoopLoggerProvider = exports.NOOP_LOGGER_PROVIDER = exports.NoopLogger = exports.NOOP_LOGGER = exports.SeverityNumber = undefined;
8465
- var LogRecord_1 = require_LogRecord();
8466
- Object.defineProperty(exports, "SeverityNumber", { enumerable: true, get: function() {
8467
- return LogRecord_1.SeverityNumber;
8468
- } });
8469
- var NoopLogger_1 = require_NoopLogger();
8470
- Object.defineProperty(exports, "NOOP_LOGGER", { enumerable: true, get: function() {
8471
- return NoopLogger_1.NOOP_LOGGER;
8472
- } });
8473
- Object.defineProperty(exports, "NoopLogger", { enumerable: true, get: function() {
8474
- return NoopLogger_1.NoopLogger;
8475
- } });
8476
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
8477
- Object.defineProperty(exports, "NOOP_LOGGER_PROVIDER", { enumerable: true, get: function() {
8478
- return NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
8479
- } });
8480
- Object.defineProperty(exports, "NoopLoggerProvider", { enumerable: true, get: function() {
8481
- return NoopLoggerProvider_1.NoopLoggerProvider;
8482
- } });
8483
- var ProxyLogger_1 = require_ProxyLogger();
8484
- Object.defineProperty(exports, "ProxyLogger", { enumerable: true, get: function() {
8485
- return ProxyLogger_1.ProxyLogger;
8486
- } });
8487
- var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
8488
- Object.defineProperty(exports, "ProxyLoggerProvider", { enumerable: true, get: function() {
8489
- return ProxyLoggerProvider_1.ProxyLoggerProvider;
8490
- } });
8491
- var logs_1 = require_logs();
8492
- exports.logs = logs_1.LogsAPI.getInstance();
8493
- });
8494
-
8495
8260
  // src/commands/market-data-vendor-backfill.ts
8496
8261
  import { createHash as createHash5, randomUUID as randomUUID2 } from "node:crypto";
8497
8262
  import path2 from "node:path";
@@ -22254,102 +22019,6 @@ var BaseTransport = class {
22254
22019
  return this.logger;
22255
22020
  }
22256
22021
  };
22257
- var LoggerlessTransport = class {
22258
- id;
22259
- enabled;
22260
- level;
22261
- consoleDebug;
22262
- constructor(config2) {
22263
- this.id = Date.now().toString() + Math.random().toString();
22264
- this.enabled = config2.enabled ?? true;
22265
- this.consoleDebug = config2.consoleDebug ?? false;
22266
- this.level = config2.level ?? "trace";
22267
- }
22268
- _sendToLogger(params) {
22269
- if (!this.enabled)
22270
- return;
22271
- if (LogLevelPriority[params.logLevel] < LogLevelPriority[this.level])
22272
- return;
22273
- const messages = this.shipToLogger(params);
22274
- if (this.consoleDebug)
22275
- switch (params.logLevel) {
22276
- case LogLevel.info:
22277
- console.info(...messages);
22278
- break;
22279
- case LogLevel.warn:
22280
- console.warn(...messages);
22281
- break;
22282
- case LogLevel.error:
22283
- console.error(...messages);
22284
- break;
22285
- case LogLevel.trace:
22286
- console.debug(...messages);
22287
- break;
22288
- case LogLevel.debug:
22289
- console.debug(...messages);
22290
- break;
22291
- case LogLevel.fatal:
22292
- console.debug(...messages);
22293
- break;
22294
- default:
22295
- console.log(...messages);
22296
- }
22297
- }
22298
- getLoggerInstance() {
22299
- throw new Error("This transport does not have a logger instance");
22300
- }
22301
- };
22302
-
22303
- // node_modules/@loglayer/transport-opentelemetry/dist/index.mjs
22304
- var import_api_logs = __toESM(require_src2(), 1);
22305
- var logLayerLevels = {
22306
- fatal: import_api_logs.SeverityNumber.FATAL,
22307
- error: import_api_logs.SeverityNumber.ERROR,
22308
- warn: import_api_logs.SeverityNumber.WARN,
22309
- info: import_api_logs.SeverityNumber.INFO,
22310
- debug: import_api_logs.SeverityNumber.DEBUG,
22311
- trace: import_api_logs.SeverityNumber.TRACE
22312
- };
22313
- function getSeverityNumber(level) {
22314
- return logLayerLevels[level] ?? import_api_logs.SeverityNumber.TRACE;
22315
- }
22316
- function emitLogRecord(record2, logger) {
22317
- const { message, level, ...splat } = record2;
22318
- const attributes = {};
22319
- for (const key in splat)
22320
- if (Object.hasOwn(splat, key))
22321
- attributes[key] = splat[key];
22322
- const logRecord = {
22323
- severityNumber: getSeverityNumber(level),
22324
- severityText: level,
22325
- body: message,
22326
- attributes
22327
- };
22328
- logger.emit(logRecord);
22329
- }
22330
- var OpenTelemetryTransport = class extends LoggerlessTransport {
22331
- _logger;
22332
- onError;
22333
- constructor(config2 = {}) {
22334
- super(config2);
22335
- this._logger = import_api_logs.logs.getLogger("loglayer");
22336
- this.onError = config2.onError;
22337
- }
22338
- shipToLogger({ logLevel, messages, data, hasData }) {
22339
- const assembled = {
22340
- level: logLevel,
22341
- message: messages.join(" "),
22342
- ...hasData ? data : {}
22343
- };
22344
- try {
22345
- emitLogRecord(assembled, this._logger);
22346
- } catch (error48) {
22347
- if (this.onError)
22348
- this.onError(error48);
22349
- }
22350
- return [assembled];
22351
- }
22352
- };
22353
22022
 
22354
22023
  // node_modules/@loglayer/transport-tslog/dist/index.js
22355
22024
  var TsLogTransport = class extends BaseTransport {
@@ -24521,13 +24190,7 @@ var tslogLogger = new Logger({
24521
24190
  minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
24522
24191
  });
24523
24192
  var baseLogger = new LogLayer({
24524
- transport: [
24525
- new TsLogTransport({ id: "tslog", logger: tslogLogger }),
24526
- new OpenTelemetryTransport({
24527
- id: "otel",
24528
- enabled: true
24529
- })
24530
- ],
24193
+ transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
24531
24194
  plugins: [openTelemetryPlugin()],
24532
24195
  errorSerializer: serializeError
24533
24196
  });
@@ -25642,12 +25305,14 @@ var result_schema_default = {
25642
25305
  };
25643
25306
 
25644
25307
  // src/helpers/market-data-vendor-backfill/manifests.ts
25645
- var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
25646
- var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
25308
+ var LEGACY_CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
25309
+ var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v2";
25310
+ var LEGACY_RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
25311
+ var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v2";
25647
25312
  var ADAPTER_POLICY_ID = "cryptohftdata-orderbook-adapter/v1";
25648
25313
  var ACQUISITION_POLICY_ID = "cryptohftdata-hourly-acquisition/v1";
25649
- var capabilityPolicyContent = {
25650
- policy_id: CAPABILITY_POLICY_ID,
25314
+ var legacyCapabilityPolicyContent = {
25315
+ policy_id: LEGACY_CAPABILITY_POLICY_ID,
25651
25316
  provider: "cryptohftdata",
25652
25317
  adapter_policy: {
25653
25318
  policy_id: ADAPTER_POLICY_ID,
@@ -25673,12 +25338,37 @@ var capabilityPolicyContent = {
25673
25338
  }
25674
25339
  ]
25675
25340
  };
25341
+ var LEGACY_CAPABILITY_POLICY = Object.freeze({
25342
+ ...legacyCapabilityPolicyContent,
25343
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent)
25344
+ });
25345
+ var capabilityPolicyContent = {
25346
+ ...legacyCapabilityPolicyContent,
25347
+ policy_id: CAPABILITY_POLICY_ID,
25348
+ profiles: [
25349
+ ...legacyCapabilityPolicyContent.profiles.map((profile) => ({
25350
+ ...profile,
25351
+ source_policies: ["authoritative_window", "fill_gaps"]
25352
+ })),
25353
+ {
25354
+ exchange: "okx",
25355
+ market_type: "spot",
25356
+ feed: "ORDERBOOK",
25357
+ canonical_trading_pair: "ARB-USDC",
25358
+ provider_exchange_id: "okx_spot",
25359
+ resolved_symbol: "ARB-USDC",
25360
+ construction_modes: ["sampled_top_n_snapshot"],
25361
+ source_policies: ["authoritative_window", "fill_gaps"],
25362
+ max_depth: 400
25363
+ }
25364
+ ]
25365
+ };
25676
25366
  var CAPABILITY_POLICY = Object.freeze({
25677
25367
  ...capabilityPolicyContent,
25678
25368
  policy_sha256: jcsSha256(capabilityPolicyContent)
25679
25369
  });
25680
- var resourcePolicyContent = {
25681
- policy_id: RESOURCE_POLICY_ID,
25370
+ var legacyResourcePolicyContent = {
25371
+ policy_id: LEGACY_RESOURCE_POLICY_ID,
25682
25372
  limits: {
25683
25373
  max_files: 1e4,
25684
25374
  max_bytes: 100 * 1024 * 1024 * 1024,
@@ -25692,17 +25382,29 @@ var resourcePolicyContent = {
25692
25382
  max_required_events: 1e5
25693
25383
  }
25694
25384
  };
25385
+ var LEGACY_RESOURCE_POLICY = Object.freeze({
25386
+ ...legacyResourcePolicyContent,
25387
+ policy_sha256: jcsSha256(legacyResourcePolicyContent)
25388
+ });
25389
+ var resourcePolicyContent = {
25390
+ ...legacyResourcePolicyContent,
25391
+ policy_id: RESOURCE_POLICY_ID,
25392
+ request_bounds: {
25393
+ ...legacyResourcePolicyContent.request_bounds,
25394
+ max_window_ms: 31 * 24 * 60 * 60 * 1000
25395
+ }
25396
+ };
25695
25397
  var RESOURCE_POLICY = Object.freeze({
25696
25398
  ...resourcePolicyContent,
25697
25399
  policy_sha256: jcsSha256(resourcePolicyContent)
25698
25400
  });
25699
25401
  var EFFECTIVE_ADAPTER_POLICY_PIN = Object.freeze({
25700
25402
  policy_id: ADAPTER_POLICY_ID,
25701
- policy_sha256: jcsSha256(capabilityPolicyContent.adapter_policy)
25403
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent.adapter_policy)
25702
25404
  });
25703
25405
  var EFFECTIVE_ACQUISITION_POLICY_PIN = Object.freeze({
25704
25406
  policy_id: ACQUISITION_POLICY_ID,
25705
- policy_sha256: jcsSha256(capabilityPolicyContent.acquisition_policy)
25407
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent.acquisition_policy)
25706
25408
  });
25707
25409
  var schemaArtifacts = [
25708
25410
  ["schemas/request.schema.json", request_schema_default],
@@ -25866,7 +25568,15 @@ var backfillRequestCodec = {
25866
25568
  if (request.idempotency_key !== createBackfillIdempotencyKey(request)) {
25867
25569
  throw new Error("idempotency_key does not match canonical business fields");
25868
25570
  }
25869
- if (request.product_pins.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request.product_pins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256 || request.product_pins.resource_policy.policy_id !== RESOURCE_POLICY.policy_id || request.product_pins.resource_policy.policy_sha256 !== RESOURCE_POLICY.policy_sha256) {
25571
+ const capabilityPolicyMatches = [
25572
+ CAPABILITY_POLICY,
25573
+ LEGACY_CAPABILITY_POLICY
25574
+ ].some((policy) => request.product_pins.capability_policy.policy_id === policy.policy_id && request.product_pins.capability_policy.policy_sha256 === policy.policy_sha256);
25575
+ const resourcePolicyMatches = [
25576
+ RESOURCE_POLICY,
25577
+ LEGACY_RESOURCE_POLICY
25578
+ ].some((policy) => request.product_pins.resource_policy.policy_id === policy.policy_id && request.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
25579
+ if (!capabilityPolicyMatches || !resourcePolicyMatches) {
25870
25580
  throw new Error("request policy pins do not match the effective package policies");
25871
25581
  }
25872
25582
  if (request.required_clock.clock_id !== request.initial_selection.required_clock.clock_id || request.required_clock.clock_sha256 !== request.initial_selection.required_clock.clock_sha256 || request.required_clock.event_count !== request.initial_selection.required_clock.event_count) {
@@ -25981,6 +25691,10 @@ function decodeBackfillRunDocuments(input) {
25981
25691
  return targetTimeMs;
25982
25692
  });
25983
25693
  const capabilityProfile = CAPABILITY_POLICY.profiles.find((profile) => profile.exchange === wire.scope.exchange && profile.market_type === wire.scope.market_type && profile.feed === wire.scope.feed && profile.canonical_trading_pair === wire.scope.trading_pair);
25694
+ const resourcePolicy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((policy) => wire.product_pins.resource_policy.policy_id === policy.policy_id && wire.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
25695
+ if (!resourcePolicy) {
25696
+ throw new Error("request resource policy pin is unsupported");
25697
+ }
25984
25698
  return {
25985
25699
  schemaVersion: BACKFILL_REQUEST_SCHEMA_VERSION,
25986
25700
  requestId: wire.request_id,
@@ -26005,11 +25719,11 @@ function decodeBackfillRunDocuments(input) {
26005
25719
  maxPriorAsOfLagMs: wire.coverage_policy.max_asof_lag_ms,
26006
25720
  sourcePolicy: wire.source_policy,
26007
25721
  budgets: {
26008
- maxFiles: RESOURCE_POLICY.limits.max_files,
26009
- maxBytes: RESOURCE_POLICY.limits.max_bytes,
26010
- maxRows: RESOURCE_POLICY.limits.max_rows,
26011
- maxDurationMs: RESOURCE_POLICY.limits.max_duration_ms,
26012
- maxBoundaryLookbackMs: Math.min(RESOURCE_POLICY.limits.max_boundary_lookback_ms, CAPABILITY_POLICY.acquisition_policy.initialization_lookback_ms)
25722
+ maxFiles: resourcePolicy.limits.max_files,
25723
+ maxBytes: resourcePolicy.limits.max_bytes,
25724
+ maxRows: resourcePolicy.limits.max_rows,
25725
+ maxDurationMs: resourcePolicy.limits.max_duration_ms,
25726
+ maxBoundaryLookbackMs: Math.min(resourcePolicy.limits.max_boundary_lookback_ms, CAPABILITY_POLICY.acquisition_policy.initialization_lookback_ms)
26013
25727
  },
26014
25728
  expectedProduct: {
26015
25729
  packageName: "@usherlabs/cex-broker",
@@ -27319,12 +27033,13 @@ function assertStoredSelectionMatchesRequest(request, selection) {
27319
27033
  }
27320
27034
  }
27321
27035
  function eligibleBundles(request, bundles) {
27322
- return bundles.filter((bundle) => {
27323
- if (bundle.captureOrigin === "production_capture") {
27324
- return request.sourcePolicy === "fill_gaps";
27325
- }
27326
- return bundle.qualification?.state === "qualified";
27327
- });
27036
+ const qualifiedVendorBundles = bundles.filter((bundle) => bundle.captureOrigin === "vendor_historical_backfill" && bundle.qualification?.state === "qualified");
27037
+ const productionBundles = bundles.filter((bundle) => bundle.captureOrigin === "production_capture");
27038
+ if (request.sourcePolicy === "authoritative_window") {
27039
+ const vendorSupport = chooseSupport(request, qualifiedVendorBundles);
27040
+ return vendorSupport.length === request.requiredClockTargetsMs.length ? qualifiedVendorBundles : productionBundles;
27041
+ }
27042
+ return [...productionBundles, ...qualifiedVendorBundles];
27328
27043
  }
27329
27044
  function originPriority(request, origin) {
27330
27045
  if (request.sourcePolicy === "fill_gaps") {
@@ -27512,9 +27227,8 @@ function exactCandidateMatch(expected, actual) {
27512
27227
  function requestedShapeMatches(request, rows) {
27513
27228
  return rows.every(({ row }) => row.depth_limit === request.depth && row.construction_mode === request.constructionMode && row.schema_version === request.expectedProduct.canonicalSchemaVersion);
27514
27229
  }
27515
- function requiredClockCoverage(request, rows) {
27516
- const summaryTimes = rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => {
27517
- const value = row.source_time_ms;
27230
+ function requiredClockCoverage(request, rows, coverageSourceTimesMs) {
27231
+ const summaryTimes = (coverageSourceTimesMs ?? rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => row.source_time_ms)).map((value) => {
27518
27232
  if (typeof value !== "number" && (typeof value !== "string" || !/^\d+$/.test(value))) {
27519
27233
  return;
27520
27234
  }
@@ -27587,7 +27301,7 @@ function verifySemanticPromotion(evidence) {
27587
27301
  coverageVerified: false
27588
27302
  };
27589
27303
  }
27590
- const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows);
27304
+ const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows, evidence.coverageSourceTimesMs);
27591
27305
  if (!coverageVerified) {
27592
27306
  return {
27593
27307
  ...base,
@@ -27962,7 +27676,13 @@ class QualifiedOrderBookArchiveReader {
27962
27676
  suffixDigestBefore: baseline.verificationBaseline.suffixDigest,
27963
27677
  suffixDigestAfter: timelineDigest(suffix),
27964
27678
  seamVerified: this.seamIsOrdered(queriedRows),
27965
- exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string")
27679
+ exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string"),
27680
+ ...request.sourcePolicy === "fill_gaps" ? {
27681
+ coverageSourceTimesMs: [
27682
+ ...baseline.selection.support_anchors.map((anchor) => Date.parse(anchor.source_time)),
27683
+ ...summaries.map((row) => numberField(row, "source_time_ms"))
27684
+ ]
27685
+ } : {}
27966
27686
  });
27967
27687
  return { ...semantic, captureBundleId };
27968
27688
  }
@@ -28078,388 +27798,8 @@ function buildForwarderBatches(input) {
28078
27798
  return batches;
28079
27799
  }
28080
27800
 
28081
- // src/helpers/market-data-vendor-backfill/qualification.ts
28082
- var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
28083
- var SHA256 = /^[0-9a-f]{64}$/;
28084
- var REASON = /^[a-z][a-z0-9_]{0,127}$/;
28085
- var EVENT_FIELDS = new Set([
28086
- "qualification_event_id",
28087
- "capture_bundle_id",
28088
- "state",
28089
- "receipt_id",
28090
- "promotion_identity_sha256",
28091
- "window",
28092
- "event_at",
28093
- "reason_code"
28094
- ]);
28095
- function deterministicUuid(value) {
28096
- const digest = jcsSha256(value).slice(0, 32).split("");
28097
- digest[12] = "5";
28098
- digest[16] = "8";
28099
- const hex3 = digest.join("");
28100
- return `${hex3.slice(0, 8)}-${hex3.slice(8, 12)}-${hex3.slice(12, 16)}-${hex3.slice(16, 20)}-${hex3.slice(20)}`;
28101
- }
28102
- function semanticEvent(event) {
28103
- const { qualification_event_id: _eventId, ...semantic } = event;
28104
- return semantic;
28105
- }
28106
- function assertTimestamp(value, field) {
28107
- const parsed = Date.parse(value);
28108
- if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
28109
- throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
28110
- }
28111
- return parsed;
28112
- }
28113
- function parseQualificationEvent(value) {
28114
- if (!value || typeof value !== "object" || Array.isArray(value)) {
28115
- throw new Error("qualification event must be an object");
28116
- }
28117
- const event = value;
28118
- if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
28119
- throw new Error("qualification event fields are invalid");
28120
- }
28121
- if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
28122
- throw new Error("qualification window must be increasing");
28123
- }
28124
- assertTimestamp(event.event_at, "event_at");
28125
- if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
28126
- throw new Error("qualification_event_id does not match event content");
28127
- }
28128
- return event;
28129
- }
28130
- function finalizeQualificationEvent(event) {
28131
- const semantic = semanticEvent(event);
28132
- return parseQualificationEvent({
28133
- ...semantic,
28134
- qualification_event_id: deterministicUuid(semantic)
28135
- });
28136
- }
28137
- function qualificationEventToArchiveRow(eventInput) {
28138
- const event = parseQualificationEvent(eventInput);
28139
- return {
28140
- table: "market_data.cex_order_book_capture_qualifications",
28141
- row: {
28142
- source: "external_backfill",
28143
- capture_origin: "vendor_historical_backfill",
28144
- source_mode: "vendor_historical_backfill_v1",
28145
- deployment_id: "market-data-vendor-backfill",
28146
- qualification_event_id: event.qualification_event_id,
28147
- capture_bundle_id: event.capture_bundle_id,
28148
- state: event.state,
28149
- receipt_id: event.receipt_id,
28150
- promotion_identity_sha256: event.promotion_identity_sha256,
28151
- window_start_ms: Date.parse(event.window.start_at),
28152
- window_end_ms: Date.parse(event.window.end_at),
28153
- event_at_ms: Date.parse(event.event_at),
28154
- reason_code: event.reason_code,
28155
- event_json: jcsCanonicalize(event)
28156
- }
28157
- };
28158
- }
28159
-
28160
- // src/helpers/market-data-vendor-backfill/core.ts
28161
- function outcome(request, status, reasonCode, extra = {}) {
28162
- return {
28163
- status,
28164
- reasonCode,
28165
- ...request ? {
28166
- requestId: request.requestId,
28167
- idempotencyKey: request.idempotencyKey,
28168
- ...request.target ? { target: request.target } : {}
28169
- } : {},
28170
- ...extra
28171
- };
28172
- }
28173
- function captureBundleId(request, capability, dataset) {
28174
- return sha256Canonical({
28175
- request_business_identity: request.idempotencyKey,
28176
- provider: capability.provider,
28177
- provider_exchange_id: capability.providerExchangeId,
28178
- resolved_symbol: capability.resolvedSymbol,
28179
- adapter_version: capability.adapterVersion,
28180
- objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
28181
- identity,
28182
- checksum,
28183
- bytes,
28184
- rows
28185
- })),
28186
- canonical_schema_version: request.expectedProduct.canonicalSchemaVersion,
28187
- checksum_algorithm: request.expectedProduct.checksumAlgorithm
28188
- });
28189
- }
28190
- function buildPromotionReceipt(request, capability, normalized, verification, verificationTimeMs) {
28191
- if (!request.wire || !request.initialSelection || !request.expectedCanonicalSchema || !request.coveragePolicy || !request.productPins) {
28192
- throw new Error("decoded final-v1 request context is missing");
28193
- }
28194
- return finalizePromotionReceipt({
28195
- schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
28196
- verified_at: new Date(verificationTimeMs).toISOString(),
28197
- request_id: request.requestId,
28198
- idempotency_key: request.idempotencyKey,
28199
- source: EXTERNAL_BACKFILL_SOURCE,
28200
- capture_origin: "vendor_historical_backfill",
28201
- source_mode: "vendor_historical_backfill_v1",
28202
- provider: capability.provider,
28203
- adapter_version: capability.adapterVersion,
28204
- effective_policies: {
28205
- capability_policy: request.productPins.capability_policy,
28206
- resource_policy: request.productPins.resource_policy,
28207
- adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
28208
- acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
28209
- },
28210
- capture_bundle_id: normalized.captureBundleId,
28211
- scope: request.wire.scope,
28212
- window: request.wire.window,
28213
- depth: request.depth,
28214
- construction_mode: request.constructionMode,
28215
- canonical_schema: request.expectedCanonicalSchema,
28216
- coverage_policy: request.coveragePolicy,
28217
- selection_sha256: request.initialSelection.selection_sha256,
28218
- vendor_semantic_digest: normalized.vendorSemanticDigest,
28219
- canonical_semantic_digest: verification.canonicalSemanticDigest,
28220
- prefix_digest: verification.prefixDigest,
28221
- suffix_digest: verification.suffixDigest,
28222
- seam_verified: true,
28223
- coverage_verified: true,
28224
- dataset_objects: normalized.objects
28225
- });
28226
- }
28227
- async function submitAll(dependencies, batches) {
28228
- const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
28229
- for (const batch of batches) {
28230
- let accepted = false;
28231
- for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
28232
- try {
28233
- const response = await dependencies.forwarder.submit(batch);
28234
- accepted = response.ok && response.inserted === batch.rows.length;
28235
- } catch {
28236
- accepted = false;
28237
- }
28238
- if (accepted)
28239
- break;
28240
- if (attempt < maxAttempts)
28241
- await dependencies.retry?.wait?.(attempt);
28242
- }
28243
- if (!accepted)
28244
- return false;
28245
- }
28246
- return true;
28247
- }
28248
- function stableFailureReason(error48, fallback) {
28249
- if (error48 && typeof error48 === "object") {
28250
- const reason = error48.reason;
28251
- if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
28252
- return reason;
28253
- }
28254
- }
28255
- return fallback;
28256
- }
28257
- function assertArchivePreflight(request, resolution) {
28258
- if (!request.target || !request.productionAuthorizationId) {
28259
- throw new Error("request target or production authorization ID is missing");
28260
- }
28261
- const selection = archiveSelectionCodec.decode(resolution.selection);
28262
- const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
28263
- const receiptById = new Map;
28264
- for (const receipt of receipts) {
28265
- const existing = receiptById.get(receipt.receipt_id);
28266
- if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
28267
- throw new Error("stored receipt identity has conflicting content");
28268
- }
28269
- receiptById.set(receipt.receipt_id, receipt);
28270
- }
28271
- for (const bundle of selection.bundles) {
28272
- if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
28273
- throw new Error("vendor selection lacks its validated stored receipt");
28274
- }
28275
- }
28276
- if (resolution.readerIdentity.environment !== request.target.environment || resolution.readerIdentity.cluster !== request.target.cluster) {
28277
- throw new Error("archive reader cluster identity mismatch");
28278
- }
28279
- }
28280
- function assertForwarderPreflight(request, resolution, nowMs) {
28281
- if (!request.target || !request.productionAuthorizationId) {
28282
- throw new Error("request target or production authorization ID is missing");
28283
- }
28284
- if (resolution.forwarderIdentity.environment !== request.target.environment || resolution.forwarderIdentity.cluster !== request.target.cluster) {
28285
- throw new Error("archive forwarder cluster identity mismatch");
28286
- }
28287
- const authorization = resolution.authorization;
28288
- const expiresAtMs = Date.parse(authorization.expiresAt);
28289
- if (authorization.authorizationId !== request.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request.target.environment || authorization.cluster !== request.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
28290
- throw new Error("production forwarder authorization is invalid");
28291
- }
28292
- }
28293
- function resourcePolicyScopeExceeded(request) {
28294
- return request.depth > RESOURCE_POLICY.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
28295
- }
28296
- function storedReceiptForSelection(resolution) {
28297
- const receiptId = resolution.selection.receipt_ids[0];
28298
- return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
28299
- }
28300
- async function runMarketDataVendorBackfill(input, dependencies) {
28301
- let request;
28302
- try {
28303
- const documents = input;
28304
- request = decodeBackfillRunDocuments({
28305
- request: documents.request,
28306
- requiredClock: documents.requiredClock
28307
- });
28308
- } catch {
28309
- return outcome(undefined, "request_invalid", "request_invalid");
28310
- }
28311
- let initialResolution;
28312
- try {
28313
- initialResolution = await dependencies.archive.resolveSelection(request);
28314
- assertArchivePreflight(request, initialResolution);
28315
- const forwarderPreflight = await dependencies.forwarder.preflight({
28316
- authorizationId: request.productionAuthorizationId,
28317
- target: request.target
28318
- });
28319
- assertForwarderPreflight(request, forwarderPreflight, dependencies.clock.nowMs());
28320
- } catch {
28321
- return outcome(request, "archive_preflight_failed", "archive_preflight_failed");
28322
- }
28323
- if (initialResolution.selection.coverage_class === "complete") {
28324
- const receipt2 = storedReceiptForSelection(initialResolution);
28325
- return outcome(request, "already_covered", "qualified_coverage_complete", {
28326
- selection: initialResolution.selection,
28327
- ...receipt2 ? { receipt: receipt2 } : {}
28328
- });
28329
- }
28330
- if (resourcePolicyScopeExceeded(request)) {
28331
- return outcome(request, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
28332
- }
28333
- let capability;
28334
- try {
28335
- capability = dependencies.providers.capabilityFor(request);
28336
- } catch {
28337
- return outcome(request, "capability_unsupported", "capability_probe_failed");
28338
- }
28339
- if (!capability) {
28340
- return outcome(request, "capability_unsupported", "scope_unsupported");
28341
- }
28342
- if (!request.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
28343
- return outcome(request, "capability_unsupported", "adapter_version_unpinned");
28344
- }
28345
- let credential;
28346
- try {
28347
- credential = await dependencies.credentials.resolve(capability.provider);
28348
- } catch {
28349
- return outcome(request, "credentials_missing", "credential_resolution_failed");
28350
- }
28351
- if (credential === undefined || credential === null) {
28352
- return outcome(request, "credentials_missing", "provider_credentials_missing");
28353
- }
28354
- let dataset;
28355
- let normalized;
28356
- try {
28357
- dataset = await dependencies.providers.acquire(request, capability, credential);
28358
- const bundleId = captureBundleId(request, capability, dataset);
28359
- normalized = await dependencies.providers.normalize(request, capability, dataset, bundleId);
28360
- if (normalized.captureBundleId !== bundleId) {
28361
- return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
28362
- reasonSubcode: "capture_identity_mismatch"
28363
- });
28364
- }
28365
- } catch (error48) {
28366
- const reason = stableFailureReason(error48, "provider_dataset_invalid");
28367
- return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
28368
- reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
28369
- });
28370
- }
28371
- try {
28372
- const candidateBatches = buildForwarderBatches({
28373
- captureBundleId: normalized.captureBundleId,
28374
- deploymentId: "market-data-vendor-backfill",
28375
- rows: normalized.rows
28376
- });
28377
- if (!await submitAll(dependencies, candidateBatches)) {
28378
- return outcome(request, "archive_ingest_failed", "candidate_batch_rejected");
28379
- }
28380
- } catch {
28381
- return outcome(request, "archive_ingest_failed", "candidate_submission_failed");
28382
- }
28383
- let verification;
28384
- try {
28385
- verification = await dependencies.archive.verifyCandidate(request, normalized, normalized.captureBundleId, initialResolution);
28386
- } catch {
28387
- return outcome(request, "promotion_verification_failed", "candidate_query_failed");
28388
- }
28389
- if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
28390
- return outcome(request, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
28391
- }
28392
- let receipt;
28393
- try {
28394
- receipt = buildPromotionReceipt(request, capability, normalized, verification, dependencies.clock.nowMs());
28395
- const [batch] = buildForwarderBatches({
28396
- captureBundleId: normalized.captureBundleId,
28397
- deploymentId: "market-data-vendor-backfill",
28398
- rows: [promotionReceiptToArchiveRow(receipt)]
28399
- });
28400
- if (!batch || !await submitAll(dependencies, [batch])) {
28401
- return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
28402
- }
28403
- const qualification = finalizeQualificationEvent({
28404
- capture_bundle_id: receipt.capture_bundle_id,
28405
- state: "qualified",
28406
- receipt_id: receipt.receipt_id,
28407
- promotion_identity_sha256: receipt.promotion_identity_sha256,
28408
- window: receipt.window,
28409
- event_at: receipt.verified_at,
28410
- reason_code: "promotion_verified"
28411
- });
28412
- const [qualificationBatch] = buildForwarderBatches({
28413
- captureBundleId: normalized.captureBundleId,
28414
- deploymentId: "market-data-vendor-backfill",
28415
- rows: [qualificationEventToArchiveRow(qualification)]
28416
- });
28417
- if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
28418
- return outcome(request, "archive_ingest_failed", "qualification_commit_failed", { receipt });
28419
- }
28420
- } catch {
28421
- return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
28422
- }
28423
- let finalResolution;
28424
- try {
28425
- finalResolution = await dependencies.archive.resolveSelection(request);
28426
- assertArchivePreflight(request, finalResolution);
28427
- } catch (error48) {
28428
- return outcome(request, "promotion_verification_failed", "post_promotion_selection_failed", {
28429
- receipt,
28430
- reasonSubcode: stableFailureReason(error48, "archive_selection_resolution_failed")
28431
- });
28432
- }
28433
- if (finalResolution.selection.coverage_class !== "complete") {
28434
- return outcome(request, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
28435
- }
28436
- if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
28437
- return outcome(request, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
28438
- }
28439
- try {
28440
- if (!request.wire)
28441
- throw new Error("decoded request wire is missing");
28442
- const [selectionBatch] = buildForwarderBatches({
28443
- captureBundleId: receipt.capture_bundle_id,
28444
- deploymentId: "market-data-vendor-backfill",
28445
- rows: [
28446
- archiveSelectionToArchiveRow(request.wire, finalResolution.selection)
28447
- ]
28448
- });
28449
- if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
28450
- return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
28451
- }
28452
- } catch {
28453
- return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
28454
- }
28455
- return outcome(request, "promoted", "promotion_qualified", {
28456
- receipt,
28457
- selection: finalResolution.selection
28458
- });
28459
- }
28460
-
28461
- // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
28462
- import { createHash as createHash4 } from "node:crypto";
27801
+ // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
27802
+ import { createHash as createHash4 } from "node:crypto";
28463
27803
 
28464
27804
  // node_modules/fzstd/esm/index.mjs
28465
27805
  var ab = ArrayBuffer;
@@ -32458,7 +31798,23 @@ var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
32458
31798
  snapshotGrouping: "event_time_final_update_id_object",
32459
31799
  sequenceSemantics: "okx_seq_id_prev_seq_id",
32460
31800
  constructionModes: ["sampled_top_n_snapshot"],
32461
- sourcePolicies: ["authoritative_window"]
31801
+ sourcePolicies: ["authoritative_window", "fill_gaps"]
31802
+ });
31803
+ var CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE = Object.freeze({
31804
+ profileId: "cryptohftdata/okx_spot/ARB-USDC/v1",
31805
+ exchange: "okx",
31806
+ tradingPair: "ARB-USDC",
31807
+ sourceSymbol: "ARB-USDC",
31808
+ marketType: "spot",
31809
+ providerExchangeId: "okx_spot",
31810
+ historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
31811
+ maxDepth: 400,
31812
+ eventTimeUnit: "milliseconds",
31813
+ receivedTimeUnit: "nanoseconds",
31814
+ snapshotGrouping: "event_time_final_update_id_object",
31815
+ sequenceSemantics: "okx_seq_id_prev_seq_id",
31816
+ constructionModes: ["sampled_top_n_snapshot"],
31817
+ sourcePolicies: ["authoritative_window", "fill_gaps"]
32462
31818
  });
32463
31819
 
32464
31820
  class CryptoHftDataError extends Error {
@@ -32469,498 +31825,916 @@ class CryptoHftDataError extends Error {
32469
31825
  this.name = "CryptoHftDataError";
32470
31826
  }
32471
31827
  }
32472
- function cryptoHftDataCapabilityFor(request, profiles = []) {
32473
- if (request.providerPolicy.provider !== "cryptohftdata" || request.scope.feed !== "ORDERBOOK" || request.scope.exchange.trim().toLowerCase() === "mexc") {
32474
- return;
32475
- }
32476
- const profile = profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.constructionModes.includes(request.constructionMode) && candidate.sourcePolicies.includes(request.sourcePolicy) && request.window.startTimeMs >= candidate.historyStartMs && request.depth <= candidate.maxDepth);
32477
- if (!profile)
32478
- return;
32479
- return {
32480
- provider: "cryptohftdata",
32481
- adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
32482
- providerExchangeId: profile.providerExchangeId,
32483
- resolvedSymbol: profile.sourceSymbol
32484
- };
31828
+ function cryptoHftDataCapabilityFor(request, profiles = []) {
31829
+ if (request.providerPolicy.provider !== "cryptohftdata" || request.scope.feed !== "ORDERBOOK" || request.scope.exchange.trim().toLowerCase() === "mexc") {
31830
+ return;
31831
+ }
31832
+ const profile = profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.constructionModes.includes(request.constructionMode) && candidate.sourcePolicies.includes(request.sourcePolicy) && request.window.startTimeMs >= candidate.historyStartMs && request.depth <= candidate.maxDepth);
31833
+ if (!profile)
31834
+ return;
31835
+ return {
31836
+ provider: "cryptohftdata",
31837
+ adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
31838
+ providerExchangeId: profile.providerExchangeId,
31839
+ resolvedSymbol: profile.sourceSymbol
31840
+ };
31841
+ }
31842
+ function utcHourPath(hourMs) {
31843
+ const date5 = new Date(hourMs);
31844
+ return {
31845
+ date: `${date5.getUTCFullYear()}-${String(date5.getUTCMonth() + 1).padStart(2, "0")}-${String(date5.getUTCDate()).padStart(2, "0")}`,
31846
+ hour: String(date5.getUTCHours()).padStart(2, "0")
31847
+ };
31848
+ }
31849
+ function enumerateCryptoHftDataObjects(request, providerExchangeId, resolvedSymbol) {
31850
+ if (request.sourcePolicy === "fill_gaps") {
31851
+ const hours = new Set;
31852
+ for (const targetTimeMs of request.requiredClockTargetsMs) {
31853
+ const firstHour2 = Math.floor(Math.max(0, targetTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
31854
+ const lastHour = Math.floor(targetTimeMs / HOUR_MS) * HOUR_MS;
31855
+ for (let hourMs = firstHour2;hourMs <= lastHour; hourMs += HOUR_MS) {
31856
+ hours.add(hourMs);
31857
+ }
31858
+ }
31859
+ return [...hours].sort((left, right) => left - right).map((hourMs) => {
31860
+ const utc = utcHourPath(hourMs);
31861
+ return `${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`;
31862
+ });
31863
+ }
31864
+ const firstHour = Math.floor((request.window.startTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
31865
+ const objects = [];
31866
+ for (let hourMs = Math.max(0, firstHour);hourMs < request.window.endTimeMs; hourMs += HOUR_MS) {
31867
+ const utc = utcHourPath(hourMs);
31868
+ objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
31869
+ }
31870
+ return objects;
31871
+ }
31872
+ function initialArchiveCoversTarget(request, targetTimeMs) {
31873
+ return (request.initialSelection?.support_anchors ?? []).some((anchor) => {
31874
+ const sourceTimeMs = Date.parse(anchor.source_time);
31875
+ return Number.isSafeInteger(sourceTimeMs) && sourceTimeMs <= targetTimeMs && targetTimeMs - sourceTimeMs <= request.maxPriorAsOfLagMs;
31876
+ });
31877
+ }
31878
+ function providerAcquisitionRequest(request) {
31879
+ if (request.sourcePolicy !== "fill_gaps")
31880
+ return request;
31881
+ const requiredClockTargetsMs = request.requiredClockTargetsMs.filter((targetTimeMs) => !initialArchiveCoversTarget(request, targetTimeMs));
31882
+ if (requiredClockTargetsMs.length === 0) {
31883
+ throw new CryptoHftDataError("fill_gaps_has_no_uncovered_clock_targets");
31884
+ }
31885
+ return { ...request, requiredClockTargetsMs };
31886
+ }
31887
+ function unsignedString(value, field, required2 = false) {
31888
+ if (value === null || value === undefined) {
31889
+ if (required2)
31890
+ throw new CryptoHftDataError(`schema_${field}_missing`);
31891
+ return;
31892
+ }
31893
+ const rendered = String(value);
31894
+ if (!/^\d+$/.test(rendered)) {
31895
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
31896
+ }
31897
+ const parsed = BigInt(rendered);
31898
+ if (parsed > 18446744073709551615n) {
31899
+ throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
31900
+ }
31901
+ return parsed.toString(10);
31902
+ }
31903
+ function timestampMs2(value, field) {
31904
+ const rendered = String(value);
31905
+ if (!/^\d+$/.test(rendered)) {
31906
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
31907
+ }
31908
+ const raw = BigInt(rendered);
31909
+ const milliseconds = field === "received_time" ? raw / 1000000n : raw;
31910
+ if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
31911
+ throw new CryptoHftDataError(`schema_${field}_unsafe`);
31912
+ }
31913
+ return Number(milliseconds);
31914
+ }
31915
+ function decimal(value, field, allowZero) {
31916
+ if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
31917
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
31918
+ }
31919
+ const parsed = Number(value);
31920
+ if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
31921
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
31922
+ }
31923
+ return parsed;
31924
+ }
31925
+ function validatedRow(request, row) {
31926
+ if (row.symbol !== request.scope.sourceSymbol) {
31927
+ throw new CryptoHftDataError("schema_symbol_mismatch");
31928
+ }
31929
+ if (row.event_type !== "snapshot" && row.event_type !== "update") {
31930
+ throw new CryptoHftDataError("schema_event_type_invalid");
31931
+ }
31932
+ if (row.side !== "bid" && row.side !== "ask") {
31933
+ throw new CryptoHftDataError("schema_side_invalid");
31934
+ }
31935
+ decimal(row.price, "price", false);
31936
+ decimal(row.quantity, "quantity", true);
31937
+ if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
31938
+ throw new CryptoHftDataError("schema_object_checksum_invalid");
31939
+ }
31940
+ const eventTimeMs = timestampMs2(row.event_time, "event_time");
31941
+ const receivedTimeMs = timestampMs2(row.received_time, "received_time");
31942
+ if (receivedTimeMs < eventTimeMs) {
31943
+ throw new CryptoHftDataError("received_time_precedes_event_time");
31944
+ }
31945
+ return { ...row, eventTimeMs, receivedTimeMs };
31946
+ }
31947
+ function groupKey(row, profile) {
31948
+ const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
31949
+ return [
31950
+ row.event_type,
31951
+ row.eventTimeMs,
31952
+ sequence,
31953
+ row.dataset_object_identity
31954
+ ].join("\x00");
31955
+ }
31956
+ function absent(value) {
31957
+ return value === null || value === undefined;
31958
+ }
31959
+ function snapshotSequence(row, profile) {
31960
+ if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
31961
+ if (String(row.last_update_id) !== "-1") {
31962
+ throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
31963
+ }
31964
+ if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
31965
+ throw new CryptoHftDataError("ambiguous_snapshot_group");
31966
+ }
31967
+ return unsignedString(row.final_update_id, "final_update_id", true);
31968
+ }
31969
+ return unsignedString(row.last_update_id, "last_update_id", true);
31970
+ }
31971
+ function sortedSide(levels, side) {
31972
+ return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
31973
+ }
31974
+ function applyRows(state, rows) {
31975
+ for (const row of rows) {
31976
+ const levels = row.side === "bid" ? state.bids : state.asks;
31977
+ const quantity = decimal(row.quantity, "quantity", true);
31978
+ if (quantity === 0)
31979
+ levels.delete(row.price);
31980
+ else
31981
+ levels.set(row.price, quantity);
31982
+ }
31983
+ }
31984
+ function reconstructCryptoHftDataOrderBooks(request, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
31985
+ const rows = inputRows.map((row, index) => ({
31986
+ ...validatedRow(request, row),
31987
+ originalIndex: index
31988
+ })).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
31989
+ const groups = [];
31990
+ for (const row of rows) {
31991
+ const current = groups.at(-1);
31992
+ if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
31993
+ groups.push([row]);
31994
+ } else {
31995
+ current.push(row);
31996
+ }
31997
+ }
31998
+ const earliestTargetTimeMs = Math.min(...request.requiredClockTargetsMs);
31999
+ const latestTargetTimeMs = Math.max(...request.requiredClockTargetsMs);
32000
+ const anchorIndex = groups.findIndex((group) => {
32001
+ const first = group[0];
32002
+ return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
32003
+ });
32004
+ if (anchorIndex < 0) {
32005
+ throw new CryptoHftDataError("update_before_snapshot");
32006
+ }
32007
+ let state;
32008
+ let previousFinalUpdateId;
32009
+ const states = [];
32010
+ for (const group of groups.slice(anchorIndex)) {
32011
+ const first = group[0];
32012
+ if (first.eventTimeMs > latestTargetTimeMs)
32013
+ break;
32014
+ if (first.event_type === "snapshot") {
32015
+ const sequence = snapshotSequence(first, profile);
32016
+ for (const row of group) {
32017
+ if (snapshotSequence(row, profile) !== sequence) {
32018
+ throw new CryptoHftDataError("ambiguous_snapshot_group");
32019
+ }
32020
+ }
32021
+ if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
32022
+ throw new CryptoHftDataError("snapshot_sequence_regression");
32023
+ }
32024
+ state = {
32025
+ bids: new Map,
32026
+ asks: new Map,
32027
+ sequence,
32028
+ sourceTimeMs: first.eventTimeMs,
32029
+ receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
32030
+ datasetObjectIdentity: first.dataset_object_identity,
32031
+ datasetObjectChecksum: first.dataset_object_checksum
32032
+ };
32033
+ applyRows(state, group);
32034
+ previousFinalUpdateId = BigInt(sequence);
32035
+ } else {
32036
+ if (!state || previousFinalUpdateId === undefined) {
32037
+ throw new CryptoHftDataError("update_before_snapshot");
32038
+ }
32039
+ const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
32040
+ if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
32041
+ if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
32042
+ throw new CryptoHftDataError("ambiguous_update_group");
32043
+ }
32044
+ const previous = unsignedString(first.last_update_id, "last_update_id", true);
32045
+ for (const row of group) {
32046
+ if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
32047
+ throw new CryptoHftDataError("ambiguous_update_group");
32048
+ }
32049
+ }
32050
+ if (BigInt(previous) !== previousFinalUpdateId) {
32051
+ throw new CryptoHftDataError("update_chain_gap");
32052
+ }
32053
+ } else {
32054
+ const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
32055
+ const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
32056
+ for (const row of group) {
32057
+ if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
32058
+ throw new CryptoHftDataError("ambiguous_update_group");
32059
+ }
32060
+ }
32061
+ const expected = previousFinalUpdateId + 1n;
32062
+ if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
32063
+ throw new CryptoHftDataError("update_chain_gap");
32064
+ }
32065
+ }
32066
+ applyRows(state, group);
32067
+ state.sequence = finalUpdate.toString();
32068
+ state.sourceTimeMs = first.eventTimeMs;
32069
+ state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
32070
+ state.datasetObjectIdentity = first.dataset_object_identity;
32071
+ state.datasetObjectChecksum = first.dataset_object_checksum;
32072
+ previousFinalUpdateId = finalUpdate;
32073
+ }
32074
+ if (state) {
32075
+ states.push({
32076
+ ...state,
32077
+ bids: new Map(state.bids),
32078
+ asks: new Map(state.asks)
32079
+ });
32080
+ }
32081
+ }
32082
+ const samples = [];
32083
+ for (const targetTimeMs of request.requiredClockTargetsMs) {
32084
+ let prior;
32085
+ for (const candidate of states) {
32086
+ if (candidate.sourceTimeMs > targetTimeMs)
32087
+ break;
32088
+ prior = candidate;
32089
+ }
32090
+ if (!prior || targetTimeMs - prior.sourceTimeMs > request.maxPriorAsOfLagMs) {
32091
+ throw new CryptoHftDataError("required_clock_coverage_insufficient");
32092
+ }
32093
+ const bids = sortedSide(prior.bids, "bid").slice(0, request.depth);
32094
+ const asks = sortedSide(prior.asks, "ask").slice(0, request.depth);
32095
+ if (bids.length === 0 || asks.length === 0) {
32096
+ throw new CryptoHftDataError("book_side_missing");
32097
+ }
32098
+ if (bids[0]?.[0] >= asks[0]?.[0]) {
32099
+ throw new CryptoHftDataError("book_crossed_or_locked");
32100
+ }
32101
+ samples.push({
32102
+ targetTimeMs,
32103
+ sourceTimeMs: prior.sourceTimeMs,
32104
+ receivedTimeMs: prior.receivedTimeMs,
32105
+ sequence: prior.sequence,
32106
+ bids,
32107
+ asks,
32108
+ datasetObjectIdentity: prior.datasetObjectIdentity,
32109
+ datasetObjectChecksum: prior.datasetObjectChecksum
32110
+ });
32111
+ }
32112
+ return samples;
32113
+ }
32114
+ function sha256Bytes(bytes) {
32115
+ return createHash4("sha256").update(bytes).digest("hex");
32485
32116
  }
32486
- function utcHourPath(hourMs) {
32487
- const date5 = new Date(hourMs);
32117
+ async function decodeCryptoHftParquetZstd(bytes) {
32118
+ const parquet = decompress(bytes);
32119
+ const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
32120
+ return parquetReadObjects({ file: buffer });
32121
+ }
32122
+ function parsedDatasetRow(value, object2) {
32488
32123
  return {
32489
- date: `${date5.getUTCFullYear()}-${String(date5.getUTCMonth() + 1).padStart(2, "0")}-${String(date5.getUTCDate()).padStart(2, "0")}`,
32490
- hour: String(date5.getUTCHours()).padStart(2, "0")
32124
+ received_time: value.received_time,
32125
+ event_time: value.event_time,
32126
+ transaction_time: value.transaction_time,
32127
+ symbol: value.symbol,
32128
+ event_type: value.event_type,
32129
+ first_update_id: value.first_update_id,
32130
+ final_update_id: value.final_update_id,
32131
+ prev_final_update_id: value.prev_final_update_id,
32132
+ last_update_id: value.last_update_id,
32133
+ side: value.side,
32134
+ price: value.price,
32135
+ quantity: value.quantity,
32136
+ order_count: value.order_count,
32137
+ dataset_object_identity: object2.identity,
32138
+ dataset_object_checksum: object2.checksum
32491
32139
  };
32492
32140
  }
32493
- function enumerateCryptoHftDataObjects(request, providerExchangeId, resolvedSymbol) {
32494
- const firstHour = Math.floor((request.window.startTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
32495
- const objects = [];
32496
- for (let hourMs = Math.max(0, firstHour);hourMs < request.window.endTimeMs; hourMs += HOUR_MS) {
32497
- const utc = utcHourPath(hourMs);
32498
- objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
32141
+ async function readBoundedObject(response, remainingBytes) {
32142
+ if (!response.body)
32143
+ return new Uint8Array;
32144
+ const reader = response.body.getReader();
32145
+ const chunks = [];
32146
+ let total = 0;
32147
+ while (true) {
32148
+ const { done, value } = await reader.read();
32149
+ if (done)
32150
+ break;
32151
+ total += value.byteLength;
32152
+ if (total > remainingBytes) {
32153
+ await reader.cancel();
32154
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
32155
+ }
32156
+ chunks.push(value);
32157
+ }
32158
+ const bytes = new Uint8Array(total);
32159
+ let offset = 0;
32160
+ for (const chunk of chunks) {
32161
+ bytes.set(chunk, offset);
32162
+ offset += chunk.byteLength;
32163
+ }
32164
+ return bytes;
32165
+ }
32166
+
32167
+ class CryptoHftDataAdapter {
32168
+ baseUrl;
32169
+ request;
32170
+ nowMs;
32171
+ decode;
32172
+ profiles;
32173
+ constructor(options = {}) {
32174
+ this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
32175
+ this.request = options.fetch ?? fetch;
32176
+ this.nowMs = options.nowMs ?? Date.now;
32177
+ this.decode = options.decode ?? decodeCryptoHftParquetZstd;
32178
+ this.profiles = options.profiles ?? [];
32179
+ }
32180
+ capabilityFor(request) {
32181
+ return cryptoHftDataCapabilityFor(request, this.profiles);
32182
+ }
32183
+ async discoverSymbols(providerExchangeId) {
32184
+ if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
32185
+ throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
32186
+ }
32187
+ const endpoint = new URL("/symbols", this.baseUrl);
32188
+ endpoint.searchParams.set("exchange", providerExchangeId);
32189
+ endpoint.searchParams.set("data_type", "orderbook");
32190
+ let response;
32191
+ try {
32192
+ response = await this.request(endpoint);
32193
+ } catch {
32194
+ throw new CryptoHftDataError("symbol_discovery_failed");
32195
+ }
32196
+ if (!response.ok) {
32197
+ throw new CryptoHftDataError("symbol_discovery_failed");
32198
+ }
32199
+ let body;
32200
+ try {
32201
+ body = await response.json();
32202
+ } catch {
32203
+ throw new CryptoHftDataError("symbol_discovery_response_invalid");
32204
+ }
32205
+ const record2 = body;
32206
+ if (!body || typeof body !== "object" || Array.isArray(body) || String(record2.exchange).toLowerCase() !== providerExchangeId || String(record2.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record2.symbols) || !record2.symbols.every((symbol2) => typeof symbol2 === "string" && /^[A-Z0-9_-]+$/.test(symbol2))) {
32207
+ throw new CryptoHftDataError("symbol_discovery_response_invalid");
32208
+ }
32209
+ return [...new Set(record2.symbols)].sort();
32210
+ }
32211
+ async acquire(request, capability, credential) {
32212
+ const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
32213
+ if (typeof apiKey !== "string" || apiKey.length === 0) {
32214
+ throw new CryptoHftDataError("credentials_invalid");
32215
+ }
32216
+ const paths = enumerateCryptoHftDataObjects(request, capability.providerExchangeId, capability.resolvedSymbol);
32217
+ if (paths.length > request.budgets.maxFiles) {
32218
+ throw new CryptoHftDataError("budget_max_files_exceeded");
32219
+ }
32220
+ const started = this.nowMs();
32221
+ const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
32222
+ method: "POST",
32223
+ headers: { "content-type": "application/json", "X-API-Key": apiKey }
32224
+ });
32225
+ if (!tokenResponse.ok)
32226
+ throw new CryptoHftDataError("jwt_issuance_failed");
32227
+ const tokenBody = await tokenResponse.json();
32228
+ if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
32229
+ throw new CryptoHftDataError("jwt_response_invalid");
32230
+ }
32231
+ let totalBytes = 0;
32232
+ let totalRows = 0;
32233
+ const objects = [];
32234
+ const rows = [];
32235
+ for (const path2 of paths) {
32236
+ if (this.nowMs() - started > request.budgets.maxDurationMs) {
32237
+ throw new CryptoHftDataError("budget_max_duration_exceeded");
32238
+ }
32239
+ const endpoint = new URL("/download", this.baseUrl);
32240
+ endpoint.searchParams.set("file", path2);
32241
+ const response = await this.request(endpoint, {
32242
+ headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
32243
+ });
32244
+ if (!response.ok)
32245
+ throw new CryptoHftDataError("object_download_failed");
32246
+ const declaredBytes = Number(response.headers.get("content-length"));
32247
+ if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request.budgets.maxBytes) {
32248
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
32249
+ }
32250
+ const bytes = await readBoundedObject(response, request.budgets.maxBytes - totalBytes);
32251
+ totalBytes += bytes.byteLength;
32252
+ if (totalBytes > request.budgets.maxBytes) {
32253
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
32254
+ }
32255
+ const decoded = await this.decode(bytes);
32256
+ totalRows += decoded.length;
32257
+ if (totalRows > request.budgets.maxRows) {
32258
+ throw new CryptoHftDataError("budget_max_rows_exceeded");
32259
+ }
32260
+ const object2 = {
32261
+ identity: path2,
32262
+ checksum: sha256Bytes(bytes),
32263
+ bytes: bytes.byteLength,
32264
+ rows: decoded.length
32265
+ };
32266
+ objects.push(object2);
32267
+ for (const decodedRow of decoded) {
32268
+ const parsed = parsedDatasetRow(decodedRow, object2);
32269
+ validatedRow(request, parsed);
32270
+ rows.push(parsed);
32271
+ }
32272
+ }
32273
+ if (this.nowMs() - started > request.budgets.maxDurationMs) {
32274
+ throw new CryptoHftDataError("budget_max_duration_exceeded");
32275
+ }
32276
+ return {
32277
+ objects,
32278
+ rows,
32279
+ vendorSemanticDigest: sha256Canonical(rows)
32280
+ };
32281
+ }
32282
+ async normalize(request, capability, dataset, captureBundleId) {
32283
+ const profile = this.profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
32284
+ if (!profile) {
32285
+ throw new CryptoHftDataError("profile_semantics_unavailable");
32286
+ }
32287
+ const samples = reconstructCryptoHftDataOrderBooks(request, dataset.rows, profile);
32288
+ const context2 = {
32289
+ source: EXTERNAL_BACKFILL_SOURCE,
32290
+ deploymentId: "market-data-vendor-backfill",
32291
+ captureBundleId,
32292
+ exchange: request.scope.exchange,
32293
+ symbol: request.scope.tradingPair,
32294
+ tradingPair: request.scope.tradingPair,
32295
+ sourceSymbol: request.scope.sourceSymbol,
32296
+ assetType: request.scope.marketType,
32297
+ feed: "ORDERBOOK",
32298
+ provider: "cryptohftdata",
32299
+ sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
32300
+ schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
32301
+ checksumAlgorithm: CHECKSUM_ALGORITHM,
32302
+ provenanceComplete: true
32303
+ };
32304
+ const rows = samples.flatMap((sample) => {
32305
+ const rawCapture = {
32306
+ rawCaptureId: sha256Canonical({
32307
+ capture_bundle_id: captureBundleId,
32308
+ dataset_object_identity: sample.datasetObjectIdentity,
32309
+ dataset_object_checksum: sample.datasetObjectChecksum,
32310
+ source_time_ms: sample.sourceTimeMs,
32311
+ target_time_ms: sample.targetTimeMs
32312
+ }),
32313
+ rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
32314
+ rawChecksum: sample.datasetObjectChecksum,
32315
+ redactedPayload: {
32316
+ dataset_object_identity: sample.datasetObjectIdentity,
32317
+ dataset_object_checksum: sample.datasetObjectChecksum
32318
+ },
32319
+ eventTimeMs: sample.sourceTimeMs,
32320
+ receivedTimeMs: sample.receivedTimeMs,
32321
+ checksumAlgorithm: CHECKSUM_ALGORITHM
32322
+ };
32323
+ const canonical = buildCanonicalOrderBookRows({
32324
+ context: context2,
32325
+ rawCapture,
32326
+ depthLimit: request.depth,
32327
+ constructionMode: request.constructionMode,
32328
+ snapshot: {
32329
+ bids: sample.bids,
32330
+ asks: sample.asks,
32331
+ timestamp: sample.sourceTimeMs,
32332
+ receivedTimestamp: sample.receivedTimeMs,
32333
+ exchange: request.scope.exchange,
32334
+ symbol: request.scope.tradingPair,
32335
+ depthLimit: request.depth,
32336
+ sequence: sample.sequence
32337
+ }
32338
+ });
32339
+ return [...canonical.levels, canonical.summary];
32340
+ });
32341
+ return {
32342
+ captureBundleId,
32343
+ objects: dataset.objects,
32344
+ rows,
32345
+ vendorSemanticDigest: dataset.vendorSemanticDigest,
32346
+ canonicalSemanticDigest: semanticDigest(rows)
32347
+ };
32499
32348
  }
32500
- return objects;
32501
32349
  }
32502
- function unsignedString(value, field, required2 = false) {
32503
- if (value === null || value === undefined) {
32504
- if (required2)
32505
- throw new CryptoHftDataError(`schema_${field}_missing`);
32506
- return;
32507
- }
32508
- const rendered = String(value);
32509
- if (!/^\d+$/.test(rendered)) {
32510
- throw new CryptoHftDataError(`schema_${field}_invalid`);
32511
- }
32512
- const parsed = BigInt(rendered);
32513
- if (parsed > 18446744073709551615n) {
32514
- throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
32515
- }
32516
- return parsed.toString(10);
32350
+
32351
+ // src/helpers/market-data-vendor-backfill/qualification.ts
32352
+ var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
32353
+ var SHA256 = /^[0-9a-f]{64}$/;
32354
+ var REASON = /^[a-z][a-z0-9_]{0,127}$/;
32355
+ var EVENT_FIELDS = new Set([
32356
+ "qualification_event_id",
32357
+ "capture_bundle_id",
32358
+ "state",
32359
+ "receipt_id",
32360
+ "promotion_identity_sha256",
32361
+ "window",
32362
+ "event_at",
32363
+ "reason_code"
32364
+ ]);
32365
+ function deterministicUuid(value) {
32366
+ const digest = jcsSha256(value).slice(0, 32).split("");
32367
+ digest[12] = "5";
32368
+ digest[16] = "8";
32369
+ const hex3 = digest.join("");
32370
+ return `${hex3.slice(0, 8)}-${hex3.slice(8, 12)}-${hex3.slice(12, 16)}-${hex3.slice(16, 20)}-${hex3.slice(20)}`;
32517
32371
  }
32518
- function timestampMs2(value, field) {
32519
- const rendered = String(value);
32520
- if (!/^\d+$/.test(rendered)) {
32521
- throw new CryptoHftDataError(`schema_${field}_invalid`);
32522
- }
32523
- const raw = BigInt(rendered);
32524
- const milliseconds = field === "received_time" ? raw / 1000000n : raw;
32525
- if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
32526
- throw new CryptoHftDataError(`schema_${field}_unsafe`);
32527
- }
32528
- return Number(milliseconds);
32372
+ function semanticEvent(event) {
32373
+ const { qualification_event_id: _eventId, ...semantic } = event;
32374
+ return semantic;
32529
32375
  }
32530
- function decimal(value, field, allowZero) {
32531
- if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
32532
- throw new CryptoHftDataError(`schema_${field}_invalid`);
32533
- }
32534
- const parsed = Number(value);
32535
- if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
32536
- throw new CryptoHftDataError(`schema_${field}_invalid`);
32376
+ function assertTimestamp(value, field) {
32377
+ const parsed = Date.parse(value);
32378
+ if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
32379
+ throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
32537
32380
  }
32538
32381
  return parsed;
32539
32382
  }
32540
- function validatedRow(request, row) {
32541
- if (row.symbol !== request.scope.sourceSymbol) {
32542
- throw new CryptoHftDataError("schema_symbol_mismatch");
32543
- }
32544
- if (row.event_type !== "snapshot" && row.event_type !== "update") {
32545
- throw new CryptoHftDataError("schema_event_type_invalid");
32383
+ function parseQualificationEvent(value) {
32384
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
32385
+ throw new Error("qualification event must be an object");
32546
32386
  }
32547
- if (row.side !== "bid" && row.side !== "ask") {
32548
- throw new CryptoHftDataError("schema_side_invalid");
32387
+ const event = value;
32388
+ if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
32389
+ throw new Error("qualification event fields are invalid");
32549
32390
  }
32550
- decimal(row.price, "price", false);
32551
- decimal(row.quantity, "quantity", true);
32552
- if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
32553
- throw new CryptoHftDataError("schema_object_checksum_invalid");
32391
+ if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
32392
+ throw new Error("qualification window must be increasing");
32554
32393
  }
32555
- const eventTimeMs = timestampMs2(row.event_time, "event_time");
32556
- const receivedTimeMs = timestampMs2(row.received_time, "received_time");
32557
- if (receivedTimeMs < eventTimeMs) {
32558
- throw new CryptoHftDataError("received_time_precedes_event_time");
32394
+ assertTimestamp(event.event_at, "event_at");
32395
+ if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
32396
+ throw new Error("qualification_event_id does not match event content");
32559
32397
  }
32560
- return { ...row, eventTimeMs, receivedTimeMs };
32561
- }
32562
- function groupKey(row, profile) {
32563
- const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
32564
- return [
32565
- row.event_type,
32566
- row.eventTimeMs,
32567
- sequence,
32568
- row.dataset_object_identity
32569
- ].join("\x00");
32398
+ return event;
32570
32399
  }
32571
- function absent(value) {
32572
- return value === null || value === undefined;
32400
+ function finalizeQualificationEvent(event) {
32401
+ const semantic = semanticEvent(event);
32402
+ return parseQualificationEvent({
32403
+ ...semantic,
32404
+ qualification_event_id: deterministicUuid(semantic)
32405
+ });
32573
32406
  }
32574
- function snapshotSequence(row, profile) {
32575
- if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
32576
- if (String(row.last_update_id) !== "-1") {
32577
- throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
32578
- }
32579
- if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
32580
- throw new CryptoHftDataError("ambiguous_snapshot_group");
32407
+ function qualificationEventToArchiveRow(eventInput) {
32408
+ const event = parseQualificationEvent(eventInput);
32409
+ return {
32410
+ table: "market_data.cex_order_book_capture_qualifications",
32411
+ row: {
32412
+ source: "external_backfill",
32413
+ capture_origin: "vendor_historical_backfill",
32414
+ source_mode: "vendor_historical_backfill_v1",
32415
+ deployment_id: "market-data-vendor-backfill",
32416
+ qualification_event_id: event.qualification_event_id,
32417
+ capture_bundle_id: event.capture_bundle_id,
32418
+ state: event.state,
32419
+ receipt_id: event.receipt_id,
32420
+ promotion_identity_sha256: event.promotion_identity_sha256,
32421
+ window_start_ms: Date.parse(event.window.start_at),
32422
+ window_end_ms: Date.parse(event.window.end_at),
32423
+ event_at_ms: Date.parse(event.event_at),
32424
+ reason_code: event.reason_code,
32425
+ event_json: jcsCanonicalize(event)
32581
32426
  }
32582
- return unsignedString(row.final_update_id, "final_update_id", true);
32583
- }
32584
- return unsignedString(row.last_update_id, "last_update_id", true);
32585
- }
32586
- function sortedSide(levels, side) {
32587
- return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
32588
- }
32589
- function applyRows(state, rows) {
32590
- for (const row of rows) {
32591
- const levels = row.side === "bid" ? state.bids : state.asks;
32592
- const quantity = decimal(row.quantity, "quantity", true);
32593
- if (quantity === 0)
32594
- levels.delete(row.price);
32595
- else
32596
- levels.set(row.price, quantity);
32597
- }
32427
+ };
32598
32428
  }
32599
- function reconstructCryptoHftDataOrderBooks(request, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
32600
- const rows = inputRows.map((row, index) => ({
32601
- ...validatedRow(request, row),
32602
- originalIndex: index
32603
- })).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
32604
- const groups = [];
32605
- for (const row of rows) {
32606
- const current = groups.at(-1);
32607
- if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
32608
- groups.push([row]);
32609
- } else {
32610
- current.push(row);
32611
- }
32612
- }
32613
- const earliestTargetTimeMs = Math.min(...request.requiredClockTargetsMs);
32614
- const latestTargetTimeMs = Math.max(...request.requiredClockTargetsMs);
32615
- const anchorIndex = groups.findIndex((group) => {
32616
- const first = group[0];
32617
- return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
32618
- });
32619
- if (anchorIndex < 0) {
32620
- throw new CryptoHftDataError("update_before_snapshot");
32621
- }
32622
- let state;
32623
- let previousFinalUpdateId;
32624
- const states = [];
32625
- for (const group of groups.slice(anchorIndex)) {
32626
- const first = group[0];
32627
- if (first.eventTimeMs > latestTargetTimeMs)
32628
- break;
32629
- if (first.event_type === "snapshot") {
32630
- const sequence = snapshotSequence(first, profile);
32631
- for (const row of group) {
32632
- if (snapshotSequence(row, profile) !== sequence) {
32633
- throw new CryptoHftDataError("ambiguous_snapshot_group");
32634
- }
32635
- }
32636
- if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
32637
- throw new CryptoHftDataError("snapshot_sequence_regression");
32638
- }
32639
- state = {
32640
- bids: new Map,
32641
- asks: new Map,
32642
- sequence,
32643
- sourceTimeMs: first.eventTimeMs,
32644
- receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
32645
- datasetObjectIdentity: first.dataset_object_identity,
32646
- datasetObjectChecksum: first.dataset_object_checksum
32647
- };
32648
- applyRows(state, group);
32649
- previousFinalUpdateId = BigInt(sequence);
32650
- } else {
32651
- if (!state || previousFinalUpdateId === undefined) {
32652
- throw new CryptoHftDataError("update_before_snapshot");
32653
- }
32654
- const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
32655
- if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
32656
- if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
32657
- throw new CryptoHftDataError("ambiguous_update_group");
32658
- }
32659
- const previous = unsignedString(first.last_update_id, "last_update_id", true);
32660
- for (const row of group) {
32661
- if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
32662
- throw new CryptoHftDataError("ambiguous_update_group");
32663
- }
32664
- }
32665
- if (BigInt(previous) !== previousFinalUpdateId) {
32666
- throw new CryptoHftDataError("update_chain_gap");
32667
- }
32668
- } else {
32669
- const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
32670
- const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
32671
- for (const row of group) {
32672
- if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
32673
- throw new CryptoHftDataError("ambiguous_update_group");
32674
- }
32675
- }
32676
- const expected = previousFinalUpdateId + 1n;
32677
- if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
32678
- throw new CryptoHftDataError("update_chain_gap");
32679
- }
32680
- }
32681
- applyRows(state, group);
32682
- state.sequence = finalUpdate.toString();
32683
- state.sourceTimeMs = first.eventTimeMs;
32684
- state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
32685
- state.datasetObjectIdentity = first.dataset_object_identity;
32686
- state.datasetObjectChecksum = first.dataset_object_checksum;
32687
- previousFinalUpdateId = finalUpdate;
32688
- }
32689
- if (state) {
32690
- states.push({
32691
- ...state,
32692
- bids: new Map(state.bids),
32693
- asks: new Map(state.asks)
32694
- });
32695
- }
32429
+
32430
+ // src/helpers/market-data-vendor-backfill/core.ts
32431
+ function outcome(request, status, reasonCode, extra = {}) {
32432
+ return {
32433
+ status,
32434
+ reasonCode,
32435
+ ...request ? {
32436
+ requestId: request.requestId,
32437
+ idempotencyKey: request.idempotencyKey,
32438
+ ...request.target ? { target: request.target } : {}
32439
+ } : {},
32440
+ ...extra
32441
+ };
32442
+ }
32443
+ function captureBundleId(request, capability, dataset) {
32444
+ return sha256Canonical({
32445
+ request_business_identity: request.idempotencyKey,
32446
+ provider: capability.provider,
32447
+ provider_exchange_id: capability.providerExchangeId,
32448
+ resolved_symbol: capability.resolvedSymbol,
32449
+ adapter_version: capability.adapterVersion,
32450
+ objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
32451
+ identity,
32452
+ checksum,
32453
+ bytes,
32454
+ rows
32455
+ })),
32456
+ canonical_schema_version: request.expectedProduct.canonicalSchemaVersion,
32457
+ checksum_algorithm: request.expectedProduct.checksumAlgorithm
32458
+ });
32459
+ }
32460
+ function buildPromotionReceipt(request, capability, normalized, verification, verificationTimeMs) {
32461
+ if (!request.wire || !request.initialSelection || !request.expectedCanonicalSchema || !request.coveragePolicy || !request.productPins) {
32462
+ throw new Error("decoded final-v1 request context is missing");
32696
32463
  }
32697
- const samples = [];
32698
- for (const targetTimeMs of request.requiredClockTargetsMs) {
32699
- let prior;
32700
- for (const candidate of states) {
32701
- if (candidate.sourceTimeMs > targetTimeMs)
32464
+ return finalizePromotionReceipt({
32465
+ schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
32466
+ verified_at: new Date(verificationTimeMs).toISOString(),
32467
+ request_id: request.requestId,
32468
+ idempotency_key: request.idempotencyKey,
32469
+ source: EXTERNAL_BACKFILL_SOURCE,
32470
+ capture_origin: "vendor_historical_backfill",
32471
+ source_mode: "vendor_historical_backfill_v1",
32472
+ provider: capability.provider,
32473
+ adapter_version: capability.adapterVersion,
32474
+ effective_policies: {
32475
+ capability_policy: request.productPins.capability_policy,
32476
+ resource_policy: request.productPins.resource_policy,
32477
+ adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
32478
+ acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
32479
+ },
32480
+ capture_bundle_id: normalized.captureBundleId,
32481
+ scope: request.wire.scope,
32482
+ window: request.wire.window,
32483
+ depth: request.depth,
32484
+ construction_mode: request.constructionMode,
32485
+ canonical_schema: request.expectedCanonicalSchema,
32486
+ coverage_policy: request.coveragePolicy,
32487
+ selection_sha256: request.initialSelection.selection_sha256,
32488
+ vendor_semantic_digest: normalized.vendorSemanticDigest,
32489
+ canonical_semantic_digest: verification.canonicalSemanticDigest,
32490
+ prefix_digest: verification.prefixDigest,
32491
+ suffix_digest: verification.suffixDigest,
32492
+ seam_verified: true,
32493
+ coverage_verified: true,
32494
+ dataset_objects: normalized.objects
32495
+ });
32496
+ }
32497
+ async function submitAll(dependencies, batches) {
32498
+ const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
32499
+ for (const batch of batches) {
32500
+ let accepted = false;
32501
+ for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
32502
+ try {
32503
+ const response = await dependencies.forwarder.submit(batch);
32504
+ accepted = response.ok && response.inserted === batch.rows.length;
32505
+ } catch {
32506
+ accepted = false;
32507
+ }
32508
+ if (accepted)
32702
32509
  break;
32703
- prior = candidate;
32510
+ if (attempt < maxAttempts)
32511
+ await dependencies.retry?.wait?.(attempt);
32704
32512
  }
32705
- if (!prior || targetTimeMs - prior.sourceTimeMs > request.maxPriorAsOfLagMs) {
32706
- throw new CryptoHftDataError("required_clock_coverage_insufficient");
32513
+ if (!accepted)
32514
+ return false;
32515
+ }
32516
+ return true;
32517
+ }
32518
+ function stableFailureReason(error48, fallback) {
32519
+ if (error48 && typeof error48 === "object") {
32520
+ const reason = error48.reason;
32521
+ if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
32522
+ return reason;
32707
32523
  }
32708
- const bids = sortedSide(prior.bids, "bid").slice(0, request.depth);
32709
- const asks = sortedSide(prior.asks, "ask").slice(0, request.depth);
32710
- if (bids.length === 0 || asks.length === 0) {
32711
- throw new CryptoHftDataError("book_side_missing");
32524
+ }
32525
+ return fallback;
32526
+ }
32527
+ function assertArchivePreflight(request, resolution) {
32528
+ if (!request.target || !request.productionAuthorizationId) {
32529
+ throw new Error("request target or production authorization ID is missing");
32530
+ }
32531
+ const selection = archiveSelectionCodec.decode(resolution.selection);
32532
+ const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
32533
+ const receiptById = new Map;
32534
+ for (const receipt of receipts) {
32535
+ const existing = receiptById.get(receipt.receipt_id);
32536
+ if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
32537
+ throw new Error("stored receipt identity has conflicting content");
32712
32538
  }
32713
- if (bids[0]?.[0] >= asks[0]?.[0]) {
32714
- throw new CryptoHftDataError("book_crossed_or_locked");
32539
+ receiptById.set(receipt.receipt_id, receipt);
32540
+ }
32541
+ for (const bundle of selection.bundles) {
32542
+ if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
32543
+ throw new Error("vendor selection lacks its validated stored receipt");
32715
32544
  }
32716
- samples.push({
32717
- targetTimeMs,
32718
- sourceTimeMs: prior.sourceTimeMs,
32719
- receivedTimeMs: prior.receivedTimeMs,
32720
- sequence: prior.sequence,
32721
- bids,
32722
- asks,
32723
- datasetObjectIdentity: prior.datasetObjectIdentity,
32724
- datasetObjectChecksum: prior.datasetObjectChecksum
32725
- });
32726
32545
  }
32727
- return samples;
32546
+ if (resolution.readerIdentity.environment !== request.target.environment || resolution.readerIdentity.cluster !== request.target.cluster) {
32547
+ throw new Error("archive reader cluster identity mismatch");
32548
+ }
32728
32549
  }
32729
- function sha256Bytes(bytes) {
32730
- return createHash4("sha256").update(bytes).digest("hex");
32550
+ function assertForwarderPreflight(request, resolution, nowMs) {
32551
+ if (!request.target || !request.productionAuthorizationId) {
32552
+ throw new Error("request target or production authorization ID is missing");
32553
+ }
32554
+ if (resolution.forwarderIdentity.environment !== request.target.environment || resolution.forwarderIdentity.cluster !== request.target.cluster) {
32555
+ throw new Error("archive forwarder cluster identity mismatch");
32556
+ }
32557
+ const authorization = resolution.authorization;
32558
+ const expiresAtMs = Date.parse(authorization.expiresAt);
32559
+ if (authorization.authorizationId !== request.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request.target.environment || authorization.cluster !== request.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
32560
+ throw new Error("production forwarder authorization is invalid");
32561
+ }
32731
32562
  }
32732
- async function decodeCryptoHftParquetZstd(bytes) {
32733
- const parquet = decompress(bytes);
32734
- const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
32735
- return parquetReadObjects({ file: buffer });
32563
+ function resourcePolicyScopeExceeded(request) {
32564
+ const policy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((candidate) => request.productPins?.resource_policy.policy_id === candidate.policy_id && request.productPins.resource_policy.policy_sha256 === candidate.policy_sha256);
32565
+ if (!policy)
32566
+ return true;
32567
+ return request.depth > policy.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > policy.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > policy.request_bounds.max_required_events;
32736
32568
  }
32737
- function parsedDatasetRow(value, object2) {
32738
- return {
32739
- received_time: value.received_time,
32740
- event_time: value.event_time,
32741
- transaction_time: value.transaction_time,
32742
- symbol: value.symbol,
32743
- event_type: value.event_type,
32744
- first_update_id: value.first_update_id,
32745
- final_update_id: value.final_update_id,
32746
- prev_final_update_id: value.prev_final_update_id,
32747
- last_update_id: value.last_update_id,
32748
- side: value.side,
32749
- price: value.price,
32750
- quantity: value.quantity,
32751
- order_count: value.order_count,
32752
- dataset_object_identity: object2.identity,
32753
- dataset_object_checksum: object2.checksum
32754
- };
32569
+ function storedReceiptForSelection(resolution) {
32570
+ const receiptId = resolution.selection.receipt_ids[0];
32571
+ return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
32755
32572
  }
32756
- async function readBoundedObject(response, remainingBytes) {
32757
- if (!response.body)
32758
- return new Uint8Array;
32759
- const reader = response.body.getReader();
32760
- const chunks = [];
32761
- let total = 0;
32762
- while (true) {
32763
- const { done, value } = await reader.read();
32764
- if (done)
32765
- break;
32766
- total += value.byteLength;
32767
- if (total > remainingBytes) {
32768
- await reader.cancel();
32769
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
32770
- }
32771
- chunks.push(value);
32573
+ async function runMarketDataVendorBackfill(input, dependencies) {
32574
+ let request;
32575
+ try {
32576
+ const documents = input;
32577
+ request = decodeBackfillRunDocuments({
32578
+ request: documents.request,
32579
+ requiredClock: documents.requiredClock
32580
+ });
32581
+ } catch {
32582
+ return outcome(undefined, "request_invalid", "request_invalid");
32772
32583
  }
32773
- const bytes = new Uint8Array(total);
32774
- let offset = 0;
32775
- for (const chunk of chunks) {
32776
- bytes.set(chunk, offset);
32777
- offset += chunk.byteLength;
32584
+ let initialResolution;
32585
+ try {
32586
+ initialResolution = await dependencies.archive.resolveSelection(request);
32587
+ assertArchivePreflight(request, initialResolution);
32588
+ const forwarderPreflight = await dependencies.forwarder.preflight({
32589
+ authorizationId: request.productionAuthorizationId,
32590
+ target: request.target
32591
+ });
32592
+ assertForwarderPreflight(request, forwarderPreflight, dependencies.clock.nowMs());
32593
+ } catch {
32594
+ return outcome(request, "archive_preflight_failed", "archive_preflight_failed");
32778
32595
  }
32779
- return bytes;
32780
- }
32781
-
32782
- class CryptoHftDataAdapter {
32783
- baseUrl;
32784
- request;
32785
- nowMs;
32786
- decode;
32787
- profiles;
32788
- constructor(options = {}) {
32789
- this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
32790
- this.request = options.fetch ?? fetch;
32791
- this.nowMs = options.nowMs ?? Date.now;
32792
- this.decode = options.decode ?? decodeCryptoHftParquetZstd;
32793
- this.profiles = options.profiles ?? [];
32596
+ if (initialResolution.selection.coverage_class === "complete") {
32597
+ const receipt2 = storedReceiptForSelection(initialResolution);
32598
+ return outcome(request, "already_covered", "qualified_coverage_complete", {
32599
+ selection: initialResolution.selection,
32600
+ ...receipt2 ? { receipt: receipt2 } : {}
32601
+ });
32602
+ }
32603
+ if (resourcePolicyScopeExceeded(request)) {
32604
+ return outcome(request, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
32605
+ }
32606
+ if (request.sourcePolicy === "fill_gaps" && (request.productPins?.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request.productPins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256)) {
32607
+ return outcome(request, "capability_unsupported", "scope_unsupported", {
32608
+ reasonSubcode: "fill_gaps_requires_capability_policy_v2"
32609
+ });
32610
+ }
32611
+ let capability;
32612
+ try {
32613
+ capability = dependencies.providers.capabilityFor(request);
32614
+ } catch {
32615
+ return outcome(request, "capability_unsupported", "capability_probe_failed");
32616
+ }
32617
+ if (!capability) {
32618
+ return outcome(request, "capability_unsupported", "scope_unsupported");
32619
+ }
32620
+ if (!request.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
32621
+ return outcome(request, "capability_unsupported", "adapter_version_unpinned");
32794
32622
  }
32795
- capabilityFor(request) {
32796
- return cryptoHftDataCapabilityFor(request, this.profiles);
32623
+ let credential;
32624
+ try {
32625
+ credential = await dependencies.credentials.resolve(capability.provider);
32626
+ } catch {
32627
+ return outcome(request, "credentials_missing", "credential_resolution_failed");
32797
32628
  }
32798
- async discoverSymbols(providerExchangeId) {
32799
- if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
32800
- throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
32801
- }
32802
- const endpoint = new URL("/symbols", this.baseUrl);
32803
- endpoint.searchParams.set("exchange", providerExchangeId);
32804
- endpoint.searchParams.set("data_type", "orderbook");
32805
- let response;
32806
- try {
32807
- response = await this.request(endpoint);
32808
- } catch {
32809
- throw new CryptoHftDataError("symbol_discovery_failed");
32810
- }
32811
- if (!response.ok) {
32812
- throw new CryptoHftDataError("symbol_discovery_failed");
32813
- }
32814
- let body;
32815
- try {
32816
- body = await response.json();
32817
- } catch {
32818
- throw new CryptoHftDataError("symbol_discovery_response_invalid");
32819
- }
32820
- const record2 = body;
32821
- if (!body || typeof body !== "object" || Array.isArray(body) || String(record2.exchange).toLowerCase() !== providerExchangeId || String(record2.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record2.symbols) || !record2.symbols.every((symbol2) => typeof symbol2 === "string" && /^[A-Z0-9_-]+$/.test(symbol2))) {
32822
- throw new CryptoHftDataError("symbol_discovery_response_invalid");
32823
- }
32824
- return [...new Set(record2.symbols)].sort();
32629
+ if (credential === undefined || credential === null) {
32630
+ return outcome(request, "credentials_missing", "provider_credentials_missing");
32825
32631
  }
32826
- async acquire(request, capability, credential) {
32827
- const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
32828
- if (typeof apiKey !== "string" || apiKey.length === 0) {
32829
- throw new CryptoHftDataError("credentials_invalid");
32830
- }
32831
- const paths = enumerateCryptoHftDataObjects(request, capability.providerExchangeId, capability.resolvedSymbol);
32832
- if (paths.length > request.budgets.maxFiles) {
32833
- throw new CryptoHftDataError("budget_max_files_exceeded");
32632
+ let dataset;
32633
+ let normalized;
32634
+ try {
32635
+ const acquisitionRequest = providerAcquisitionRequest(request);
32636
+ dataset = await dependencies.providers.acquire(acquisitionRequest, capability, credential);
32637
+ const bundleId = captureBundleId(request, capability, dataset);
32638
+ normalized = await dependencies.providers.normalize(acquisitionRequest, capability, dataset, bundleId);
32639
+ if (normalized.captureBundleId !== bundleId) {
32640
+ return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
32641
+ reasonSubcode: "capture_identity_mismatch"
32642
+ });
32834
32643
  }
32835
- const started = this.nowMs();
32836
- const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
32837
- method: "POST",
32838
- headers: { "content-type": "application/json", "X-API-Key": apiKey }
32644
+ } catch (error48) {
32645
+ const reason = stableFailureReason(error48, "provider_dataset_invalid");
32646
+ return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
32647
+ reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
32839
32648
  });
32840
- if (!tokenResponse.ok)
32841
- throw new CryptoHftDataError("jwt_issuance_failed");
32842
- const tokenBody = await tokenResponse.json();
32843
- if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
32844
- throw new CryptoHftDataError("jwt_response_invalid");
32649
+ }
32650
+ try {
32651
+ const candidateBatches = buildForwarderBatches({
32652
+ captureBundleId: normalized.captureBundleId,
32653
+ deploymentId: "market-data-vendor-backfill",
32654
+ rows: normalized.rows
32655
+ });
32656
+ if (!await submitAll(dependencies, candidateBatches)) {
32657
+ return outcome(request, "archive_ingest_failed", "candidate_batch_rejected");
32845
32658
  }
32846
- let totalBytes = 0;
32847
- let totalRows = 0;
32848
- const objects = [];
32849
- const rows = [];
32850
- for (const path2 of paths) {
32851
- if (this.nowMs() - started > request.budgets.maxDurationMs) {
32852
- throw new CryptoHftDataError("budget_max_duration_exceeded");
32853
- }
32854
- const endpoint = new URL("/download", this.baseUrl);
32855
- endpoint.searchParams.set("file", path2);
32856
- const response = await this.request(endpoint, {
32857
- headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
32858
- });
32859
- if (!response.ok)
32860
- throw new CryptoHftDataError("object_download_failed");
32861
- const declaredBytes = Number(response.headers.get("content-length"));
32862
- if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request.budgets.maxBytes) {
32863
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
32864
- }
32865
- const bytes = await readBoundedObject(response, request.budgets.maxBytes - totalBytes);
32866
- totalBytes += bytes.byteLength;
32867
- if (totalBytes > request.budgets.maxBytes) {
32868
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
32869
- }
32870
- const decoded = await this.decode(bytes);
32871
- totalRows += decoded.length;
32872
- if (totalRows > request.budgets.maxRows) {
32873
- throw new CryptoHftDataError("budget_max_rows_exceeded");
32874
- }
32875
- const object2 = {
32876
- identity: path2,
32877
- checksum: sha256Bytes(bytes),
32878
- bytes: bytes.byteLength,
32879
- rows: decoded.length
32880
- };
32881
- objects.push(object2);
32882
- for (const decodedRow of decoded) {
32883
- const parsed = parsedDatasetRow(decodedRow, object2);
32884
- validatedRow(request, parsed);
32885
- rows.push(parsed);
32886
- }
32659
+ } catch {
32660
+ return outcome(request, "archive_ingest_failed", "candidate_submission_failed");
32661
+ }
32662
+ let verification;
32663
+ try {
32664
+ verification = await dependencies.archive.verifyCandidate(request, normalized, normalized.captureBundleId, initialResolution);
32665
+ } catch {
32666
+ return outcome(request, "promotion_verification_failed", "candidate_query_failed");
32667
+ }
32668
+ if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
32669
+ return outcome(request, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
32670
+ }
32671
+ let receipt;
32672
+ try {
32673
+ receipt = buildPromotionReceipt(request, capability, normalized, verification, dependencies.clock.nowMs());
32674
+ const [batch] = buildForwarderBatches({
32675
+ captureBundleId: normalized.captureBundleId,
32676
+ deploymentId: "market-data-vendor-backfill",
32677
+ rows: [promotionReceiptToArchiveRow(receipt)]
32678
+ });
32679
+ if (!batch || !await submitAll(dependencies, [batch])) {
32680
+ return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
32887
32681
  }
32888
- if (this.nowMs() - started > request.budgets.maxDurationMs) {
32889
- throw new CryptoHftDataError("budget_max_duration_exceeded");
32682
+ const qualification = finalizeQualificationEvent({
32683
+ capture_bundle_id: receipt.capture_bundle_id,
32684
+ state: "qualified",
32685
+ receipt_id: receipt.receipt_id,
32686
+ promotion_identity_sha256: receipt.promotion_identity_sha256,
32687
+ window: receipt.window,
32688
+ event_at: receipt.verified_at,
32689
+ reason_code: "promotion_verified"
32690
+ });
32691
+ const [qualificationBatch] = buildForwarderBatches({
32692
+ captureBundleId: normalized.captureBundleId,
32693
+ deploymentId: "market-data-vendor-backfill",
32694
+ rows: [qualificationEventToArchiveRow(qualification)]
32695
+ });
32696
+ if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
32697
+ return outcome(request, "archive_ingest_failed", "qualification_commit_failed", { receipt });
32890
32698
  }
32891
- return {
32892
- objects,
32893
- rows,
32894
- vendorSemanticDigest: sha256Canonical(rows)
32895
- };
32699
+ } catch {
32700
+ return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
32896
32701
  }
32897
- async normalize(request, capability, dataset, captureBundleId2) {
32898
- const profile = this.profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
32899
- if (!profile) {
32900
- throw new CryptoHftDataError("profile_semantics_unavailable");
32901
- }
32902
- const samples = reconstructCryptoHftDataOrderBooks(request, dataset.rows, profile);
32903
- const context2 = {
32904
- source: EXTERNAL_BACKFILL_SOURCE,
32702
+ let finalResolution;
32703
+ try {
32704
+ finalResolution = await dependencies.archive.resolveSelection(request);
32705
+ assertArchivePreflight(request, finalResolution);
32706
+ } catch (error48) {
32707
+ return outcome(request, "promotion_verification_failed", "post_promotion_selection_failed", {
32708
+ receipt,
32709
+ reasonSubcode: stableFailureReason(error48, "archive_selection_resolution_failed")
32710
+ });
32711
+ }
32712
+ if (finalResolution.selection.coverage_class !== "complete") {
32713
+ return outcome(request, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
32714
+ }
32715
+ if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
32716
+ return outcome(request, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
32717
+ }
32718
+ try {
32719
+ if (!request.wire)
32720
+ throw new Error("decoded request wire is missing");
32721
+ const [selectionBatch] = buildForwarderBatches({
32722
+ captureBundleId: receipt.capture_bundle_id,
32905
32723
  deploymentId: "market-data-vendor-backfill",
32906
- captureBundleId: captureBundleId2,
32907
- exchange: request.scope.exchange,
32908
- symbol: request.scope.tradingPair,
32909
- tradingPair: request.scope.tradingPair,
32910
- sourceSymbol: request.scope.sourceSymbol,
32911
- assetType: request.scope.marketType,
32912
- feed: "ORDERBOOK",
32913
- provider: "cryptohftdata",
32914
- sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
32915
- schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
32916
- checksumAlgorithm: CHECKSUM_ALGORITHM,
32917
- provenanceComplete: true
32918
- };
32919
- const rows = samples.flatMap((sample) => {
32920
- const rawCapture = {
32921
- rawCaptureId: sha256Canonical({
32922
- capture_bundle_id: captureBundleId2,
32923
- dataset_object_identity: sample.datasetObjectIdentity,
32924
- dataset_object_checksum: sample.datasetObjectChecksum,
32925
- source_time_ms: sample.sourceTimeMs,
32926
- target_time_ms: sample.targetTimeMs
32927
- }),
32928
- rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
32929
- rawChecksum: sample.datasetObjectChecksum,
32930
- redactedPayload: {
32931
- dataset_object_identity: sample.datasetObjectIdentity,
32932
- dataset_object_checksum: sample.datasetObjectChecksum
32933
- },
32934
- eventTimeMs: sample.sourceTimeMs,
32935
- receivedTimeMs: sample.receivedTimeMs,
32936
- checksumAlgorithm: CHECKSUM_ALGORITHM
32937
- };
32938
- const canonical = buildCanonicalOrderBookRows({
32939
- context: context2,
32940
- rawCapture,
32941
- depthLimit: request.depth,
32942
- constructionMode: request.constructionMode,
32943
- snapshot: {
32944
- bids: sample.bids,
32945
- asks: sample.asks,
32946
- timestamp: sample.sourceTimeMs,
32947
- receivedTimestamp: sample.receivedTimeMs,
32948
- exchange: request.scope.exchange,
32949
- symbol: request.scope.tradingPair,
32950
- depthLimit: request.depth,
32951
- sequence: sample.sequence
32952
- }
32953
- });
32954
- return [...canonical.levels, canonical.summary];
32724
+ rows: [
32725
+ archiveSelectionToArchiveRow(request.wire, finalResolution.selection)
32726
+ ]
32955
32727
  });
32956
- return {
32957
- captureBundleId: captureBundleId2,
32958
- objects: dataset.objects,
32959
- rows,
32960
- vendorSemanticDigest: dataset.vendorSemanticDigest,
32961
- canonicalSemanticDigest: semanticDigest(rows)
32962
- };
32728
+ if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
32729
+ return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
32730
+ }
32731
+ } catch {
32732
+ return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
32963
32733
  }
32734
+ return outcome(request, "promoted", "promotion_qualified", {
32735
+ receipt,
32736
+ selection: finalResolution.selection
32737
+ });
32964
32738
  }
32965
32739
 
32966
32740
  // src/helpers/market-data-vendor-backfill/forwarder-client.ts
@@ -33081,8 +32855,8 @@ function validateReleaseIdentity(release) {
33081
32855
  return release;
33082
32856
  }
33083
32857
  function bakedReleaseIdentity() {
33084
- const packageVersion = "0.2.48";
33085
- const gitHead = "3b42f32422c587afbdfab6ed16b0969980eaaddb";
32858
+ const packageVersion = "0.2.50";
32859
+ const gitHead = "41dbe7cc296da1c5a7964dca8f418800522f27e5";
33086
32860
  return validateReleaseIdentity({ packageVersion, gitHead });
33087
32861
  }
33088
32862
  function readOptionalEnvironment(environment, name) {
@@ -33106,7 +32880,10 @@ function createBackfillDependenciesFromEnv(environment, sensitiveValues = new Se
33106
32880
  password: clickhousePassword
33107
32881
  })),
33108
32882
  providers: new CryptoHftDataAdapter({
33109
- profiles: [CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE]
32883
+ profiles: [
32884
+ CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE,
32885
+ CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE
32886
+ ]
33110
32887
  }),
33111
32888
  forwarder: createArchiveForwarderClient({
33112
32889
  url: forwarderUrl,
@@ -33313,4 +33090,4 @@ export {
33313
33090
  createBackfillDependenciesFromEnv
33314
33091
  };
33315
33092
 
33316
- //# debugId=D04AAF122F979D2864756E2164756E21
33093
+ //# debugId=7BCBC113F3C0009864756E2164756E21