@usherlabs/cex-broker 0.2.47 → 0.2.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1431,241 +1431,6 @@ var require_src = __commonJS((exports) => {
1431
1431
  };
1432
1432
  });
1433
1433
 
1434
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/types/LogRecord.js
1435
- var require_LogRecord = __commonJS((exports) => {
1436
- Object.defineProperty(exports, "__esModule", { value: true });
1437
- exports.SeverityNumber = undefined;
1438
- var SeverityNumber;
1439
- (function(SeverityNumber2) {
1440
- SeverityNumber2[SeverityNumber2["UNSPECIFIED"] = 0] = "UNSPECIFIED";
1441
- SeverityNumber2[SeverityNumber2["TRACE"] = 1] = "TRACE";
1442
- SeverityNumber2[SeverityNumber2["TRACE2"] = 2] = "TRACE2";
1443
- SeverityNumber2[SeverityNumber2["TRACE3"] = 3] = "TRACE3";
1444
- SeverityNumber2[SeverityNumber2["TRACE4"] = 4] = "TRACE4";
1445
- SeverityNumber2[SeverityNumber2["DEBUG"] = 5] = "DEBUG";
1446
- SeverityNumber2[SeverityNumber2["DEBUG2"] = 6] = "DEBUG2";
1447
- SeverityNumber2[SeverityNumber2["DEBUG3"] = 7] = "DEBUG3";
1448
- SeverityNumber2[SeverityNumber2["DEBUG4"] = 8] = "DEBUG4";
1449
- SeverityNumber2[SeverityNumber2["INFO"] = 9] = "INFO";
1450
- SeverityNumber2[SeverityNumber2["INFO2"] = 10] = "INFO2";
1451
- SeverityNumber2[SeverityNumber2["INFO3"] = 11] = "INFO3";
1452
- SeverityNumber2[SeverityNumber2["INFO4"] = 12] = "INFO4";
1453
- SeverityNumber2[SeverityNumber2["WARN"] = 13] = "WARN";
1454
- SeverityNumber2[SeverityNumber2["WARN2"] = 14] = "WARN2";
1455
- SeverityNumber2[SeverityNumber2["WARN3"] = 15] = "WARN3";
1456
- SeverityNumber2[SeverityNumber2["WARN4"] = 16] = "WARN4";
1457
- SeverityNumber2[SeverityNumber2["ERROR"] = 17] = "ERROR";
1458
- SeverityNumber2[SeverityNumber2["ERROR2"] = 18] = "ERROR2";
1459
- SeverityNumber2[SeverityNumber2["ERROR3"] = 19] = "ERROR3";
1460
- SeverityNumber2[SeverityNumber2["ERROR4"] = 20] = "ERROR4";
1461
- SeverityNumber2[SeverityNumber2["FATAL"] = 21] = "FATAL";
1462
- SeverityNumber2[SeverityNumber2["FATAL2"] = 22] = "FATAL2";
1463
- SeverityNumber2[SeverityNumber2["FATAL3"] = 23] = "FATAL3";
1464
- SeverityNumber2[SeverityNumber2["FATAL4"] = 24] = "FATAL4";
1465
- })(SeverityNumber = exports.SeverityNumber || (exports.SeverityNumber = {}));
1466
- });
1467
-
1468
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLogger.js
1469
- var require_NoopLogger = __commonJS((exports) => {
1470
- Object.defineProperty(exports, "__esModule", { value: true });
1471
- exports.NOOP_LOGGER = exports.NoopLogger = undefined;
1472
-
1473
- class NoopLogger {
1474
- emit(_logRecord) {}
1475
- }
1476
- exports.NoopLogger = NoopLogger;
1477
- exports.NOOP_LOGGER = new NoopLogger;
1478
- });
1479
-
1480
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLoggerProvider.js
1481
- var require_NoopLoggerProvider = __commonJS((exports) => {
1482
- Object.defineProperty(exports, "__esModule", { value: true });
1483
- exports.NOOP_LOGGER_PROVIDER = exports.NoopLoggerProvider = undefined;
1484
- var NoopLogger_1 = require_NoopLogger();
1485
-
1486
- class NoopLoggerProvider {
1487
- getLogger(_name, _version, _options) {
1488
- return new NoopLogger_1.NoopLogger;
1489
- }
1490
- }
1491
- exports.NoopLoggerProvider = NoopLoggerProvider;
1492
- exports.NOOP_LOGGER_PROVIDER = new NoopLoggerProvider;
1493
- });
1494
-
1495
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLogger.js
1496
- var require_ProxyLogger = __commonJS((exports) => {
1497
- Object.defineProperty(exports, "__esModule", { value: true });
1498
- exports.ProxyLogger = undefined;
1499
- var NoopLogger_1 = require_NoopLogger();
1500
-
1501
- class ProxyLogger {
1502
- constructor(_provider, name, version, options) {
1503
- this._provider = _provider;
1504
- this.name = name;
1505
- this.version = version;
1506
- this.options = options;
1507
- }
1508
- emit(logRecord) {
1509
- this._getLogger().emit(logRecord);
1510
- }
1511
- _getLogger() {
1512
- if (this._delegate) {
1513
- return this._delegate;
1514
- }
1515
- const logger = this._provider._getDelegateLogger(this.name, this.version, this.options);
1516
- if (!logger) {
1517
- return NoopLogger_1.NOOP_LOGGER;
1518
- }
1519
- this._delegate = logger;
1520
- return this._delegate;
1521
- }
1522
- }
1523
- exports.ProxyLogger = ProxyLogger;
1524
- });
1525
-
1526
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLoggerProvider.js
1527
- var require_ProxyLoggerProvider = __commonJS((exports) => {
1528
- Object.defineProperty(exports, "__esModule", { value: true });
1529
- exports.ProxyLoggerProvider = undefined;
1530
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
1531
- var ProxyLogger_1 = require_ProxyLogger();
1532
-
1533
- class ProxyLoggerProvider {
1534
- getLogger(name, version, options) {
1535
- var _a;
1536
- return (_a = this._getDelegateLogger(name, version, options)) !== null && _a !== undefined ? _a : new ProxyLogger_1.ProxyLogger(this, name, version, options);
1537
- }
1538
- _getDelegate() {
1539
- var _a;
1540
- return (_a = this._delegate) !== null && _a !== undefined ? _a : NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
1541
- }
1542
- _setDelegate(delegate) {
1543
- this._delegate = delegate;
1544
- }
1545
- _getDelegateLogger(name, version, options) {
1546
- var _a;
1547
- return (_a = this._delegate) === null || _a === undefined ? undefined : _a.getLogger(name, version, options);
1548
- }
1549
- }
1550
- exports.ProxyLoggerProvider = ProxyLoggerProvider;
1551
- });
1552
-
1553
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/globalThis.js
1554
- var require_globalThis2 = __commonJS((exports) => {
1555
- Object.defineProperty(exports, "__esModule", { value: true });
1556
- exports._globalThis = undefined;
1557
- exports._globalThis = typeof globalThis === "object" ? globalThis : global;
1558
- });
1559
-
1560
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/index.js
1561
- var require_node2 = __commonJS((exports) => {
1562
- Object.defineProperty(exports, "__esModule", { value: true });
1563
- exports._globalThis = undefined;
1564
- var globalThis_1 = require_globalThis2();
1565
- Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
1566
- return globalThis_1._globalThis;
1567
- } });
1568
- });
1569
-
1570
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/index.js
1571
- var require_platform2 = __commonJS((exports) => {
1572
- Object.defineProperty(exports, "__esModule", { value: true });
1573
- exports._globalThis = undefined;
1574
- var node_1 = require_node2();
1575
- Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
1576
- return node_1._globalThis;
1577
- } });
1578
- });
1579
-
1580
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/internal/global-utils.js
1581
- var require_global_utils2 = __commonJS((exports) => {
1582
- Object.defineProperty(exports, "__esModule", { value: true });
1583
- exports.API_BACKWARDS_COMPATIBILITY_VERSION = exports.makeGetter = exports._global = exports.GLOBAL_LOGS_API_KEY = undefined;
1584
- var platform_1 = require_platform2();
1585
- exports.GLOBAL_LOGS_API_KEY = Symbol.for("io.opentelemetry.js.api.logs");
1586
- exports._global = platform_1._globalThis;
1587
- function makeGetter(requiredVersion, instance, fallback) {
1588
- return (version) => version === requiredVersion ? instance : fallback;
1589
- }
1590
- exports.makeGetter = makeGetter;
1591
- exports.API_BACKWARDS_COMPATIBILITY_VERSION = 1;
1592
- });
1593
-
1594
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/api/logs.js
1595
- var require_logs = __commonJS((exports) => {
1596
- Object.defineProperty(exports, "__esModule", { value: true });
1597
- exports.LogsAPI = undefined;
1598
- var global_utils_1 = require_global_utils2();
1599
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
1600
- var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
1601
-
1602
- class LogsAPI {
1603
- constructor() {
1604
- this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
1605
- }
1606
- static getInstance() {
1607
- if (!this._instance) {
1608
- this._instance = new LogsAPI;
1609
- }
1610
- return this._instance;
1611
- }
1612
- setGlobalLoggerProvider(provider) {
1613
- if (global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) {
1614
- return this.getLoggerProvider();
1615
- }
1616
- global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY] = (0, global_utils_1.makeGetter)(global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION, provider, NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER);
1617
- this._proxyLoggerProvider._setDelegate(provider);
1618
- return provider;
1619
- }
1620
- getLoggerProvider() {
1621
- var _a, _b;
1622
- return (_b = (_a = global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) === null || _a === undefined ? undefined : _a.call(global_utils_1._global, global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION)) !== null && _b !== undefined ? _b : this._proxyLoggerProvider;
1623
- }
1624
- getLogger(name, version, options) {
1625
- return this.getLoggerProvider().getLogger(name, version, options);
1626
- }
1627
- disable() {
1628
- delete global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY];
1629
- this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
1630
- }
1631
- }
1632
- exports.LogsAPI = LogsAPI;
1633
- });
1634
-
1635
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/index.js
1636
- var require_src2 = __commonJS((exports) => {
1637
- Object.defineProperty(exports, "__esModule", { value: true });
1638
- exports.logs = exports.ProxyLoggerProvider = exports.ProxyLogger = exports.NoopLoggerProvider = exports.NOOP_LOGGER_PROVIDER = exports.NoopLogger = exports.NOOP_LOGGER = exports.SeverityNumber = undefined;
1639
- var LogRecord_1 = require_LogRecord();
1640
- Object.defineProperty(exports, "SeverityNumber", { enumerable: true, get: function() {
1641
- return LogRecord_1.SeverityNumber;
1642
- } });
1643
- var NoopLogger_1 = require_NoopLogger();
1644
- Object.defineProperty(exports, "NOOP_LOGGER", { enumerable: true, get: function() {
1645
- return NoopLogger_1.NOOP_LOGGER;
1646
- } });
1647
- Object.defineProperty(exports, "NoopLogger", { enumerable: true, get: function() {
1648
- return NoopLogger_1.NoopLogger;
1649
- } });
1650
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
1651
- Object.defineProperty(exports, "NOOP_LOGGER_PROVIDER", { enumerable: true, get: function() {
1652
- return NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
1653
- } });
1654
- Object.defineProperty(exports, "NoopLoggerProvider", { enumerable: true, get: function() {
1655
- return NoopLoggerProvider_1.NoopLoggerProvider;
1656
- } });
1657
- var ProxyLogger_1 = require_ProxyLogger();
1658
- Object.defineProperty(exports, "ProxyLogger", { enumerable: true, get: function() {
1659
- return ProxyLogger_1.ProxyLogger;
1660
- } });
1661
- var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
1662
- Object.defineProperty(exports, "ProxyLoggerProvider", { enumerable: true, get: function() {
1663
- return ProxyLoggerProvider_1.ProxyLoggerProvider;
1664
- } });
1665
- var logs_1 = require_logs();
1666
- exports.logs = logs_1.LogsAPI.getInstance();
1667
- });
1668
-
1669
1434
  // node_modules/ajv/dist/compile/codegen/code.js
1670
1435
  var require_code = __commonJS((exports) => {
1671
1436
  Object.defineProperty(exports, "__esModule", { value: true });
@@ -8697,102 +8462,6 @@ var BaseTransport = class {
8697
8462
  return this.logger;
8698
8463
  }
8699
8464
  };
8700
- var LoggerlessTransport = class {
8701
- id;
8702
- enabled;
8703
- level;
8704
- consoleDebug;
8705
- constructor(config) {
8706
- this.id = Date.now().toString() + Math.random().toString();
8707
- this.enabled = config.enabled ?? true;
8708
- this.consoleDebug = config.consoleDebug ?? false;
8709
- this.level = config.level ?? "trace";
8710
- }
8711
- _sendToLogger(params) {
8712
- if (!this.enabled)
8713
- return;
8714
- if (LogLevelPriority[params.logLevel] < LogLevelPriority[this.level])
8715
- return;
8716
- const messages = this.shipToLogger(params);
8717
- if (this.consoleDebug)
8718
- switch (params.logLevel) {
8719
- case LogLevel.info:
8720
- console.info(...messages);
8721
- break;
8722
- case LogLevel.warn:
8723
- console.warn(...messages);
8724
- break;
8725
- case LogLevel.error:
8726
- console.error(...messages);
8727
- break;
8728
- case LogLevel.trace:
8729
- console.debug(...messages);
8730
- break;
8731
- case LogLevel.debug:
8732
- console.debug(...messages);
8733
- break;
8734
- case LogLevel.fatal:
8735
- console.debug(...messages);
8736
- break;
8737
- default:
8738
- console.log(...messages);
8739
- }
8740
- }
8741
- getLoggerInstance() {
8742
- throw new Error("This transport does not have a logger instance");
8743
- }
8744
- };
8745
-
8746
- // node_modules/@loglayer/transport-opentelemetry/dist/index.mjs
8747
- var import_api_logs = __toESM(require_src2(), 1);
8748
- var logLayerLevels = {
8749
- fatal: import_api_logs.SeverityNumber.FATAL,
8750
- error: import_api_logs.SeverityNumber.ERROR,
8751
- warn: import_api_logs.SeverityNumber.WARN,
8752
- info: import_api_logs.SeverityNumber.INFO,
8753
- debug: import_api_logs.SeverityNumber.DEBUG,
8754
- trace: import_api_logs.SeverityNumber.TRACE
8755
- };
8756
- function getSeverityNumber(level) {
8757
- return logLayerLevels[level] ?? import_api_logs.SeverityNumber.TRACE;
8758
- }
8759
- function emitLogRecord(record, logger) {
8760
- const { message, level, ...splat } = record;
8761
- const attributes = {};
8762
- for (const key in splat)
8763
- if (Object.hasOwn(splat, key))
8764
- attributes[key] = splat[key];
8765
- const logRecord = {
8766
- severityNumber: getSeverityNumber(level),
8767
- severityText: level,
8768
- body: message,
8769
- attributes
8770
- };
8771
- logger.emit(logRecord);
8772
- }
8773
- var OpenTelemetryTransport = class extends LoggerlessTransport {
8774
- _logger;
8775
- onError;
8776
- constructor(config = {}) {
8777
- super(config);
8778
- this._logger = import_api_logs.logs.getLogger("loglayer");
8779
- this.onError = config.onError;
8780
- }
8781
- shipToLogger({ logLevel, messages, data, hasData }) {
8782
- const assembled = {
8783
- level: logLevel,
8784
- message: messages.join(" "),
8785
- ...hasData ? data : {}
8786
- };
8787
- try {
8788
- emitLogRecord(assembled, this._logger);
8789
- } catch (error) {
8790
- if (this.onError)
8791
- this.onError(error);
8792
- }
8793
- return [assembled];
8794
- }
8795
- };
8796
8465
 
8797
8466
  // node_modules/@loglayer/transport-tslog/dist/index.js
8798
8467
  var TsLogTransport = class extends BaseTransport {
@@ -10964,13 +10633,7 @@ var tslogLogger = new Logger({
10964
10633
  minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
10965
10634
  });
10966
10635
  var baseLogger = new LogLayer({
10967
- transport: [
10968
- new TsLogTransport({ id: "tslog", logger: tslogLogger }),
10969
- new OpenTelemetryTransport({
10970
- id: "otel",
10971
- enabled: true
10972
- })
10973
- ],
10636
+ transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
10974
10637
  plugins: [openTelemetryPlugin()],
10975
10638
  errorSerializer: serializeError
10976
10639
  });
@@ -12089,12 +11752,13 @@ var result_schema_default = {
12089
11752
  };
12090
11753
 
12091
11754
  // src/helpers/market-data-vendor-backfill/manifests.ts
12092
- var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
11755
+ var LEGACY_CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
11756
+ var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v2";
12093
11757
  var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
12094
11758
  var ADAPTER_POLICY_ID = "cryptohftdata-orderbook-adapter/v1";
12095
11759
  var ACQUISITION_POLICY_ID = "cryptohftdata-hourly-acquisition/v1";
12096
- var capabilityPolicyContent = {
12097
- policy_id: CAPABILITY_POLICY_ID,
11760
+ var legacyCapabilityPolicyContent = {
11761
+ policy_id: LEGACY_CAPABILITY_POLICY_ID,
12098
11762
  provider: "cryptohftdata",
12099
11763
  adapter_policy: {
12100
11764
  policy_id: ADAPTER_POLICY_ID,
@@ -12120,6 +11784,18 @@ var capabilityPolicyContent = {
12120
11784
  }
12121
11785
  ]
12122
11786
  };
11787
+ var LEGACY_CAPABILITY_POLICY = Object.freeze({
11788
+ ...legacyCapabilityPolicyContent,
11789
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent)
11790
+ });
11791
+ var capabilityPolicyContent = {
11792
+ ...legacyCapabilityPolicyContent,
11793
+ policy_id: CAPABILITY_POLICY_ID,
11794
+ profiles: legacyCapabilityPolicyContent.profiles.map((profile) => ({
11795
+ ...profile,
11796
+ source_policies: ["authoritative_window", "fill_gaps"]
11797
+ }))
11798
+ };
12123
11799
  var CAPABILITY_POLICY = Object.freeze({
12124
11800
  ...capabilityPolicyContent,
12125
11801
  policy_sha256: jcsSha256(capabilityPolicyContent)
@@ -12145,11 +11821,11 @@ var RESOURCE_POLICY = Object.freeze({
12145
11821
  });
12146
11822
  var EFFECTIVE_ADAPTER_POLICY_PIN = Object.freeze({
12147
11823
  policy_id: ADAPTER_POLICY_ID,
12148
- policy_sha256: jcsSha256(capabilityPolicyContent.adapter_policy)
11824
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent.adapter_policy)
12149
11825
  });
12150
11826
  var EFFECTIVE_ACQUISITION_POLICY_PIN = Object.freeze({
12151
11827
  policy_id: ACQUISITION_POLICY_ID,
12152
- policy_sha256: jcsSha256(capabilityPolicyContent.acquisition_policy)
11828
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent.acquisition_policy)
12153
11829
  });
12154
11830
  var schemaArtifacts = [
12155
11831
  ["schemas/request.schema.json", request_schema_default],
@@ -12329,7 +12005,11 @@ var backfillRequestCodec = {
12329
12005
  if (request.idempotency_key !== createBackfillIdempotencyKey(request)) {
12330
12006
  throw new Error("idempotency_key does not match canonical business fields");
12331
12007
  }
12332
- if (request.product_pins.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request.product_pins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256 || request.product_pins.resource_policy.policy_id !== RESOURCE_POLICY.policy_id || request.product_pins.resource_policy.policy_sha256 !== RESOURCE_POLICY.policy_sha256) {
12008
+ const capabilityPolicyMatches = [
12009
+ CAPABILITY_POLICY,
12010
+ LEGACY_CAPABILITY_POLICY
12011
+ ].some((policy) => request.product_pins.capability_policy.policy_id === policy.policy_id && request.product_pins.capability_policy.policy_sha256 === policy.policy_sha256);
12012
+ if (!capabilityPolicyMatches || request.product_pins.resource_policy.policy_id !== RESOURCE_POLICY.policy_id || request.product_pins.resource_policy.policy_sha256 !== RESOURCE_POLICY.policy_sha256) {
12333
12013
  throw new Error("request policy pins do not match the effective package policies");
12334
12014
  }
12335
12015
  if (request.required_clock.clock_id !== request.initial_selection.required_clock.clock_id || request.required_clock.clock_sha256 !== request.initial_selection.required_clock.clock_sha256 || request.required_clock.event_count !== request.initial_selection.required_clock.event_count) {
@@ -12763,12 +12443,13 @@ function assertStoredSelectionMatchesRequest(request, selection) {
12763
12443
  }
12764
12444
  }
12765
12445
  function eligibleBundles(request, bundles) {
12766
- return bundles.filter((bundle) => {
12767
- if (bundle.captureOrigin === "production_capture") {
12768
- return request.sourcePolicy === "fill_gaps";
12769
- }
12770
- return bundle.qualification?.state === "qualified";
12771
- });
12446
+ const qualifiedVendorBundles = bundles.filter((bundle) => bundle.captureOrigin === "vendor_historical_backfill" && bundle.qualification?.state === "qualified");
12447
+ const productionBundles = bundles.filter((bundle) => bundle.captureOrigin === "production_capture");
12448
+ if (request.sourcePolicy === "authoritative_window") {
12449
+ const vendorSupport = chooseSupport(request, qualifiedVendorBundles);
12450
+ return vendorSupport.length === request.requiredClockTargetsMs.length ? qualifiedVendorBundles : productionBundles;
12451
+ }
12452
+ return [...productionBundles, ...qualifiedVendorBundles];
12772
12453
  }
12773
12454
  function originPriority(request, origin) {
12774
12455
  if (request.sourcePolicy === "fill_gaps") {
@@ -12973,9 +12654,8 @@ function exactCandidateMatch(expected, actual) {
12973
12654
  function requestedShapeMatches(request, rows) {
12974
12655
  return rows.every(({ row }) => row.depth_limit === request.depth && row.construction_mode === request.constructionMode && row.schema_version === request.expectedProduct.canonicalSchemaVersion);
12975
12656
  }
12976
- function requiredClockCoverage(request, rows) {
12977
- const summaryTimes = rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => {
12978
- const value = row.source_time_ms;
12657
+ function requiredClockCoverage(request, rows, coverageSourceTimesMs) {
12658
+ const summaryTimes = (coverageSourceTimesMs ?? rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => row.source_time_ms)).map((value) => {
12979
12659
  if (typeof value !== "number" && (typeof value !== "string" || !/^\d+$/.test(value))) {
12980
12660
  return;
12981
12661
  }
@@ -13048,7 +12728,7 @@ function verifySemanticPromotion(evidence) {
13048
12728
  coverageVerified: false
13049
12729
  };
13050
12730
  }
13051
- const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows);
12731
+ const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows, evidence.coverageSourceTimesMs);
13052
12732
  if (!coverageVerified) {
13053
12733
  return {
13054
12734
  ...base,
@@ -13423,7 +13103,13 @@ class QualifiedOrderBookArchiveReader {
13423
13103
  suffixDigestBefore: baseline.verificationBaseline.suffixDigest,
13424
13104
  suffixDigestAfter: timelineDigest(suffix),
13425
13105
  seamVerified: this.seamIsOrdered(queriedRows),
13426
- exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string")
13106
+ exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string"),
13107
+ ...request.sourcePolicy === "fill_gaps" ? {
13108
+ coverageSourceTimesMs: [
13109
+ ...baseline.selection.support_anchors.map((anchor) => Date.parse(anchor.source_time)),
13110
+ ...summaries.map((row) => numberField(row, "source_time_ms"))
13111
+ ]
13112
+ } : {}
13427
13113
  });
13428
13114
  return { ...semantic, captureBundleId };
13429
13115
  }
@@ -13612,8 +13298,8 @@ var requestContent = {
13612
13298
  },
13613
13299
  product_pins: {
13614
13300
  capability_policy: {
13615
- policy_id: CAPABILITY_POLICY.policy_id,
13616
- policy_sha256: CAPABILITY_POLICY.policy_sha256
13301
+ policy_id: LEGACY_CAPABILITY_POLICY.policy_id,
13302
+ policy_sha256: LEGACY_CAPABILITY_POLICY.policy_sha256
13617
13303
  },
13618
13304
  resource_policy: {
13619
13305
  policy_id: RESOURCE_POLICY.policy_id,
@@ -13766,422 +13452,14 @@ var CONFORMANCE_FIXTURES = Object.freeze({
13766
13452
  promotion_identity_sha256: promotionReceipt.promotion_identity_sha256,
13767
13453
  receipt_id: promotionReceipt.receipt_id,
13768
13454
  result_sha256: result.result_sha256,
13769
- capability_policy_sha256: CAPABILITY_POLICY.policy_sha256,
13455
+ capability_policy_sha256: LEGACY_CAPABILITY_POLICY.policy_sha256,
13770
13456
  resource_policy_sha256: RESOURCE_POLICY.policy_sha256
13771
13457
  },
13772
13458
  jcs_edge_vector: jcsEdgeVector,
13773
13459
  hashes: { jcs_edge_vector_sha256: jcsSha256(jcsEdgeVector) }
13774
13460
  });
13775
- // src/helpers/market-data-vendor-backfill/qualification.ts
13776
- var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
13777
- var SHA256 = /^[0-9a-f]{64}$/;
13778
- var REASON = /^[a-z][a-z0-9_]{0,127}$/;
13779
- var EVENT_FIELDS = new Set([
13780
- "qualification_event_id",
13781
- "capture_bundle_id",
13782
- "state",
13783
- "receipt_id",
13784
- "promotion_identity_sha256",
13785
- "window",
13786
- "event_at",
13787
- "reason_code"
13788
- ]);
13789
- function deterministicUuid(value) {
13790
- const digest = jcsSha256(value).slice(0, 32).split("");
13791
- digest[12] = "5";
13792
- digest[16] = "8";
13793
- const hex = digest.join("");
13794
- return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
13795
- }
13796
- function semanticEvent(event) {
13797
- const { qualification_event_id: _eventId, ...semantic } = event;
13798
- return semantic;
13799
- }
13800
- function assertTimestamp(value, field) {
13801
- const parsed = Date.parse(value);
13802
- if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
13803
- throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
13804
- }
13805
- return parsed;
13806
- }
13807
- function parseQualificationEvent(value) {
13808
- if (!value || typeof value !== "object" || Array.isArray(value)) {
13809
- throw new Error("qualification event must be an object");
13810
- }
13811
- const event = value;
13812
- if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
13813
- throw new Error("qualification event fields are invalid");
13814
- }
13815
- if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
13816
- throw new Error("qualification window must be increasing");
13817
- }
13818
- assertTimestamp(event.event_at, "event_at");
13819
- if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
13820
- throw new Error("qualification_event_id does not match event content");
13821
- }
13822
- return event;
13823
- }
13824
- function finalizeQualificationEvent(event) {
13825
- const semantic = semanticEvent(event);
13826
- return parseQualificationEvent({
13827
- ...semantic,
13828
- qualification_event_id: deterministicUuid(semantic)
13829
- });
13830
- }
13831
- function qualificationEventToArchiveRow(eventInput) {
13832
- const event = parseQualificationEvent(eventInput);
13833
- return {
13834
- table: "market_data.cex_order_book_capture_qualifications",
13835
- row: {
13836
- source: "external_backfill",
13837
- capture_origin: "vendor_historical_backfill",
13838
- source_mode: "vendor_historical_backfill_v1",
13839
- deployment_id: "market-data-vendor-backfill",
13840
- qualification_event_id: event.qualification_event_id,
13841
- capture_bundle_id: event.capture_bundle_id,
13842
- state: event.state,
13843
- receipt_id: event.receipt_id,
13844
- promotion_identity_sha256: event.promotion_identity_sha256,
13845
- window_start_ms: Date.parse(event.window.start_at),
13846
- window_end_ms: Date.parse(event.window.end_at),
13847
- event_at_ms: Date.parse(event.event_at),
13848
- reason_code: event.reason_code,
13849
- event_json: jcsCanonicalize(event)
13850
- }
13851
- };
13852
- }
13853
-
13854
- // src/helpers/market-data-vendor-backfill/core.ts
13855
- function createMarketDataVendorBackfillDependencies(dependencies) {
13856
- for (const [name, value] of Object.entries({
13857
- archive: dependencies.archive,
13858
- providers: dependencies.providers,
13859
- credentials: dependencies.credentials,
13860
- forwarder: dependencies.forwarder,
13861
- clock: dependencies.clock
13862
- })) {
13863
- if (!value || typeof value !== "object") {
13864
- throw new TypeError(`Backfill dependency ${name} is required`);
13865
- }
13866
- }
13867
- for (const [name, method] of Object.entries({
13868
- "archive.resolveSelection": dependencies.archive.resolveSelection,
13869
- "archive.verifyCandidate": dependencies.archive.verifyCandidate,
13870
- "providers.capabilityFor": dependencies.providers.capabilityFor,
13871
- "providers.acquire": dependencies.providers.acquire,
13872
- "providers.normalize": dependencies.providers.normalize,
13873
- "credentials.resolve": dependencies.credentials.resolve,
13874
- "forwarder.preflight": dependencies.forwarder.preflight,
13875
- "forwarder.submit": dependencies.forwarder.submit,
13876
- "clock.nowMs": dependencies.clock.nowMs
13877
- })) {
13878
- if (typeof method !== "function") {
13879
- throw new TypeError(`Backfill dependency ${name} is required`);
13880
- }
13881
- }
13882
- return dependencies;
13883
- }
13884
- function outcome(request2, status, reasonCode, extra = {}) {
13885
- return {
13886
- status,
13887
- reasonCode,
13888
- ...request2 ? {
13889
- requestId: request2.requestId,
13890
- idempotencyKey: request2.idempotencyKey,
13891
- ...request2.target ? { target: request2.target } : {}
13892
- } : {},
13893
- ...extra
13894
- };
13895
- }
13896
- function captureBundleId2(request2, capability, dataset) {
13897
- return sha256Canonical({
13898
- request_business_identity: request2.idempotencyKey,
13899
- provider: capability.provider,
13900
- provider_exchange_id: capability.providerExchangeId,
13901
- resolved_symbol: capability.resolvedSymbol,
13902
- adapter_version: capability.adapterVersion,
13903
- objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
13904
- identity,
13905
- checksum,
13906
- bytes,
13907
- rows
13908
- })),
13909
- canonical_schema_version: request2.expectedProduct.canonicalSchemaVersion,
13910
- checksum_algorithm: request2.expectedProduct.checksumAlgorithm
13911
- });
13912
- }
13913
- function buildPromotionReceipt(request2, capability, normalized, verification, verificationTimeMs) {
13914
- if (!request2.wire || !request2.initialSelection || !request2.expectedCanonicalSchema || !request2.coveragePolicy || !request2.productPins) {
13915
- throw new Error("decoded final-v1 request context is missing");
13916
- }
13917
- return finalizePromotionReceipt({
13918
- schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
13919
- verified_at: new Date(verificationTimeMs).toISOString(),
13920
- request_id: request2.requestId,
13921
- idempotency_key: request2.idempotencyKey,
13922
- source: EXTERNAL_BACKFILL_SOURCE,
13923
- capture_origin: "vendor_historical_backfill",
13924
- source_mode: "vendor_historical_backfill_v1",
13925
- provider: capability.provider,
13926
- adapter_version: capability.adapterVersion,
13927
- effective_policies: {
13928
- capability_policy: request2.productPins.capability_policy,
13929
- resource_policy: request2.productPins.resource_policy,
13930
- adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
13931
- acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
13932
- },
13933
- capture_bundle_id: normalized.captureBundleId,
13934
- scope: request2.wire.scope,
13935
- window: request2.wire.window,
13936
- depth: request2.depth,
13937
- construction_mode: request2.constructionMode,
13938
- canonical_schema: request2.expectedCanonicalSchema,
13939
- coverage_policy: request2.coveragePolicy,
13940
- selection_sha256: request2.initialSelection.selection_sha256,
13941
- vendor_semantic_digest: normalized.vendorSemanticDigest,
13942
- canonical_semantic_digest: verification.canonicalSemanticDigest,
13943
- prefix_digest: verification.prefixDigest,
13944
- suffix_digest: verification.suffixDigest,
13945
- seam_verified: true,
13946
- coverage_verified: true,
13947
- dataset_objects: normalized.objects
13948
- });
13949
- }
13950
- async function submitAll(dependencies, batches) {
13951
- const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
13952
- for (const batch of batches) {
13953
- let accepted = false;
13954
- for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
13955
- try {
13956
- const response = await dependencies.forwarder.submit(batch);
13957
- accepted = response.ok && response.inserted === batch.rows.length;
13958
- } catch {
13959
- accepted = false;
13960
- }
13961
- if (accepted)
13962
- break;
13963
- if (attempt < maxAttempts)
13964
- await dependencies.retry?.wait?.(attempt);
13965
- }
13966
- if (!accepted)
13967
- return false;
13968
- }
13969
- return true;
13970
- }
13971
- function stableFailureReason(error, fallback) {
13972
- if (error && typeof error === "object") {
13973
- const reason = error.reason;
13974
- if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
13975
- return reason;
13976
- }
13977
- }
13978
- return fallback;
13979
- }
13980
- function assertArchivePreflight(request2, resolution) {
13981
- if (!request2.target || !request2.productionAuthorizationId) {
13982
- throw new Error("request target or production authorization ID is missing");
13983
- }
13984
- const selection = archiveSelectionCodec.decode(resolution.selection);
13985
- const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
13986
- const receiptById = new Map;
13987
- for (const receipt of receipts) {
13988
- const existing = receiptById.get(receipt.receipt_id);
13989
- if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
13990
- throw new Error("stored receipt identity has conflicting content");
13991
- }
13992
- receiptById.set(receipt.receipt_id, receipt);
13993
- }
13994
- for (const bundle of selection.bundles) {
13995
- if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
13996
- throw new Error("vendor selection lacks its validated stored receipt");
13997
- }
13998
- }
13999
- if (resolution.readerIdentity.environment !== request2.target.environment || resolution.readerIdentity.cluster !== request2.target.cluster) {
14000
- throw new Error("archive reader cluster identity mismatch");
14001
- }
14002
- }
14003
- function assertForwarderPreflight(request2, resolution, nowMs) {
14004
- if (!request2.target || !request2.productionAuthorizationId) {
14005
- throw new Error("request target or production authorization ID is missing");
14006
- }
14007
- if (resolution.forwarderIdentity.environment !== request2.target.environment || resolution.forwarderIdentity.cluster !== request2.target.cluster) {
14008
- throw new Error("archive forwarder cluster identity mismatch");
14009
- }
14010
- const authorization = resolution.authorization;
14011
- const expiresAtMs = Date.parse(authorization.expiresAt);
14012
- if (authorization.authorizationId !== request2.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request2.target.environment || authorization.cluster !== request2.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
14013
- throw new Error("production forwarder authorization is invalid");
14014
- }
14015
- }
14016
- function resourcePolicyScopeExceeded(request2) {
14017
- return request2.depth > RESOURCE_POLICY.request_bounds.max_depth || request2.window.endTimeMs - request2.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request2.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
14018
- }
14019
- function storedReceiptForSelection(resolution) {
14020
- const receiptId = resolution.selection.receipt_ids[0];
14021
- return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
14022
- }
14023
- async function runMarketDataVendorBackfill(input, dependencies) {
14024
- let request2;
14025
- try {
14026
- const documents = input;
14027
- request2 = decodeBackfillRunDocuments({
14028
- request: documents.request,
14029
- requiredClock: documents.requiredClock
14030
- });
14031
- } catch {
14032
- return outcome(undefined, "request_invalid", "request_invalid");
14033
- }
14034
- let initialResolution;
14035
- try {
14036
- initialResolution = await dependencies.archive.resolveSelection(request2);
14037
- assertArchivePreflight(request2, initialResolution);
14038
- const forwarderPreflight = await dependencies.forwarder.preflight({
14039
- authorizationId: request2.productionAuthorizationId,
14040
- target: request2.target
14041
- });
14042
- assertForwarderPreflight(request2, forwarderPreflight, dependencies.clock.nowMs());
14043
- } catch {
14044
- return outcome(request2, "archive_preflight_failed", "archive_preflight_failed");
14045
- }
14046
- if (initialResolution.selection.coverage_class === "complete") {
14047
- const receipt2 = storedReceiptForSelection(initialResolution);
14048
- return outcome(request2, "already_covered", "qualified_coverage_complete", {
14049
- selection: initialResolution.selection,
14050
- ...receipt2 ? { receipt: receipt2 } : {}
14051
- });
14052
- }
14053
- if (resourcePolicyScopeExceeded(request2)) {
14054
- return outcome(request2, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
14055
- }
14056
- let capability;
14057
- try {
14058
- capability = dependencies.providers.capabilityFor(request2);
14059
- } catch {
14060
- return outcome(request2, "capability_unsupported", "capability_probe_failed");
14061
- }
14062
- if (!capability) {
14063
- return outcome(request2, "capability_unsupported", "scope_unsupported");
14064
- }
14065
- if (!request2.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
14066
- return outcome(request2, "capability_unsupported", "adapter_version_unpinned");
14067
- }
14068
- let credential;
14069
- try {
14070
- credential = await dependencies.credentials.resolve(capability.provider);
14071
- } catch {
14072
- return outcome(request2, "credentials_missing", "credential_resolution_failed");
14073
- }
14074
- if (credential === undefined || credential === null) {
14075
- return outcome(request2, "credentials_missing", "provider_credentials_missing");
14076
- }
14077
- let dataset;
14078
- let normalized;
14079
- try {
14080
- dataset = await dependencies.providers.acquire(request2, capability, credential);
14081
- const bundleId = captureBundleId2(request2, capability, dataset);
14082
- normalized = await dependencies.providers.normalize(request2, capability, dataset, bundleId);
14083
- if (normalized.captureBundleId !== bundleId) {
14084
- return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
14085
- reasonSubcode: "capture_identity_mismatch"
14086
- });
14087
- }
14088
- } catch (error) {
14089
- const reason = stableFailureReason(error, "provider_dataset_invalid");
14090
- return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
14091
- reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
14092
- });
14093
- }
14094
- try {
14095
- const candidateBatches = buildForwarderBatches({
14096
- captureBundleId: normalized.captureBundleId,
14097
- deploymentId: "market-data-vendor-backfill",
14098
- rows: normalized.rows
14099
- });
14100
- if (!await submitAll(dependencies, candidateBatches)) {
14101
- return outcome(request2, "archive_ingest_failed", "candidate_batch_rejected");
14102
- }
14103
- } catch {
14104
- return outcome(request2, "archive_ingest_failed", "candidate_submission_failed");
14105
- }
14106
- let verification;
14107
- try {
14108
- verification = await dependencies.archive.verifyCandidate(request2, normalized, normalized.captureBundleId, initialResolution);
14109
- } catch {
14110
- return outcome(request2, "promotion_verification_failed", "candidate_query_failed");
14111
- }
14112
- if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
14113
- return outcome(request2, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
14114
- }
14115
- let receipt;
14116
- try {
14117
- receipt = buildPromotionReceipt(request2, capability, normalized, verification, dependencies.clock.nowMs());
14118
- const [batch] = buildForwarderBatches({
14119
- captureBundleId: normalized.captureBundleId,
14120
- deploymentId: "market-data-vendor-backfill",
14121
- rows: [promotionReceiptToArchiveRow(receipt)]
14122
- });
14123
- if (!batch || !await submitAll(dependencies, [batch])) {
14124
- return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
14125
- }
14126
- const qualification = finalizeQualificationEvent({
14127
- capture_bundle_id: receipt.capture_bundle_id,
14128
- state: "qualified",
14129
- receipt_id: receipt.receipt_id,
14130
- promotion_identity_sha256: receipt.promotion_identity_sha256,
14131
- window: receipt.window,
14132
- event_at: receipt.verified_at,
14133
- reason_code: "promotion_verified"
14134
- });
14135
- const [qualificationBatch] = buildForwarderBatches({
14136
- captureBundleId: normalized.captureBundleId,
14137
- deploymentId: "market-data-vendor-backfill",
14138
- rows: [qualificationEventToArchiveRow(qualification)]
14139
- });
14140
- if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
14141
- return outcome(request2, "archive_ingest_failed", "qualification_commit_failed", { receipt });
14142
- }
14143
- } catch {
14144
- return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
14145
- }
14146
- let finalResolution;
14147
- try {
14148
- finalResolution = await dependencies.archive.resolveSelection(request2);
14149
- assertArchivePreflight(request2, finalResolution);
14150
- } catch (error) {
14151
- return outcome(request2, "promotion_verification_failed", "post_promotion_selection_failed", {
14152
- receipt,
14153
- reasonSubcode: stableFailureReason(error, "archive_selection_resolution_failed")
14154
- });
14155
- }
14156
- if (finalResolution.selection.coverage_class !== "complete") {
14157
- return outcome(request2, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
14158
- }
14159
- if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
14160
- return outcome(request2, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
14161
- }
14162
- try {
14163
- if (!request2.wire)
14164
- throw new Error("decoded request wire is missing");
14165
- const [selectionBatch] = buildForwarderBatches({
14166
- captureBundleId: receipt.capture_bundle_id,
14167
- deploymentId: "market-data-vendor-backfill",
14168
- rows: [
14169
- archiveSelectionToArchiveRow(request2.wire, finalResolution.selection)
14170
- ]
14171
- });
14172
- if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
14173
- return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
14174
- }
14175
- } catch {
14176
- return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
14177
- }
14178
- return outcome(request2, "promoted", "promotion_qualified", {
14179
- receipt,
14180
- selection: finalResolution.selection
14181
- });
14182
- }
14183
- // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
14184
- import { createHash as createHash3 } from "node:crypto";
13461
+ // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
13462
+ import { createHash as createHash3 } from "node:crypto";
14185
13463
 
14186
13464
  // node_modules/fzstd/esm/index.mjs
14187
13465
  var ab = ArrayBuffer;
@@ -18090,599 +17368,1043 @@ function buildCanonicalOrderBookRows(input) {
18090
17368
  const common = commonEvidenceFields(input.context, input.rawCapture, {
18091
17369
  snapshotId,
18092
17370
  sequence,
18093
- depthLimit: input.depthLimit,
18094
- eventTimeMs,
18095
- receivedTimeMs
18096
- });
18097
- const levels = [
18098
- ["bid", bids],
18099
- ["ask", asks]
18100
- ].flatMap(([side, sideLevels]) => sideLevels.map(({ price, amount }, levelIndex) => {
18101
- const row = checksumRow({
18102
- ...common,
18103
- side,
18104
- level_index: levelIndex,
18105
- price,
18106
- amount,
18107
- notional: price * amount,
18108
- mid_price: midPrice,
18109
- spread_from_mid_bps: Math.abs((price - midPrice) / midPrice) * 1e4
18110
- });
18111
- return { table: "market_data.cex_order_book_levels", row };
18112
- }));
18113
- const bands = normalizedBands(input.measurementBandsBps);
18114
- const bidDepth = bands.map((band) => {
18115
- const minimumPrice = bestBid.price * (1 - band / 1e4);
18116
- return bids.filter(({ price }) => price >= minimumPrice).reduce((sum, { amount }) => sum + amount, 0);
18117
- });
18118
- const askDepth = bands.map((band) => {
18119
- const maximumPrice = bestAsk.price * (1 + band / 1e4);
18120
- return asks.filter(({ price }) => price <= maximumPrice).reduce((sum, { amount }) => sum + amount, 0);
18121
- });
18122
- const summaryRow = checksumRow({
18123
- ...common,
18124
- best_bid: bestBid.price,
18125
- best_ask: bestAsk.price,
18126
- best_bid_amount: bestBid.amount,
18127
- best_ask_amount: bestAsk.amount,
18128
- mid_price: midPrice,
18129
- spread,
18130
- spread_bps: spreadBps,
18131
- staleness_ms: receivedTimeMs - eventTimeMs,
18132
- bid_level_count: bids.length,
18133
- ask_level_count: asks.length,
18134
- measurement_bands_bps: bands,
18135
- bid_depth_by_band: bidDepth,
18136
- ask_depth_by_band: askDepth
18137
- });
18138
- return {
18139
- snapshotId,
18140
- levels,
18141
- summary: {
18142
- table: "market_data.cex_order_book_depth_summary",
18143
- row: summaryRow
17371
+ depthLimit: input.depthLimit,
17372
+ eventTimeMs,
17373
+ receivedTimeMs
17374
+ });
17375
+ const levels = [
17376
+ ["bid", bids],
17377
+ ["ask", asks]
17378
+ ].flatMap(([side, sideLevels]) => sideLevels.map(({ price, amount }, levelIndex) => {
17379
+ const row = checksumRow({
17380
+ ...common,
17381
+ side,
17382
+ level_index: levelIndex,
17383
+ price,
17384
+ amount,
17385
+ notional: price * amount,
17386
+ mid_price: midPrice,
17387
+ spread_from_mid_bps: Math.abs((price - midPrice) / midPrice) * 1e4
17388
+ });
17389
+ return { table: "market_data.cex_order_book_levels", row };
17390
+ }));
17391
+ const bands = normalizedBands(input.measurementBandsBps);
17392
+ const bidDepth = bands.map((band) => {
17393
+ const minimumPrice = bestBid.price * (1 - band / 1e4);
17394
+ return bids.filter(({ price }) => price >= minimumPrice).reduce((sum, { amount }) => sum + amount, 0);
17395
+ });
17396
+ const askDepth = bands.map((band) => {
17397
+ const maximumPrice = bestAsk.price * (1 + band / 1e4);
17398
+ return asks.filter(({ price }) => price <= maximumPrice).reduce((sum, { amount }) => sum + amount, 0);
17399
+ });
17400
+ const summaryRow = checksumRow({
17401
+ ...common,
17402
+ best_bid: bestBid.price,
17403
+ best_ask: bestAsk.price,
17404
+ best_bid_amount: bestBid.amount,
17405
+ best_ask_amount: bestAsk.amount,
17406
+ mid_price: midPrice,
17407
+ spread,
17408
+ spread_bps: spreadBps,
17409
+ staleness_ms: receivedTimeMs - eventTimeMs,
17410
+ bid_level_count: bids.length,
17411
+ ask_level_count: asks.length,
17412
+ measurement_bands_bps: bands,
17413
+ bid_depth_by_band: bidDepth,
17414
+ ask_depth_by_band: askDepth
17415
+ });
17416
+ return {
17417
+ snapshotId,
17418
+ levels,
17419
+ summary: {
17420
+ table: "market_data.cex_order_book_depth_summary",
17421
+ row: summaryRow
17422
+ }
17423
+ };
17424
+ }
17425
+
17426
+ // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
17427
+ var CRYPTOHFTDATA_ADAPTER_VERSION = "cryptohftdata-orderbook/v2";
17428
+ var CRYPTOHFTDATA_API_URL = "https://api.cryptohftdata.com";
17429
+ var CRYPTOHFTDATA_HISTORY_START_MS = Date.UTC(2025, 5, 28);
17430
+ var HOUR_MS = 60 * 60 * 1000;
17431
+ var CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE = Object.freeze({
17432
+ profileId: "cryptohftdata/binance_spot/BTCUSDT/v1",
17433
+ exchange: "binance",
17434
+ tradingPair: "BTC-USDT",
17435
+ sourceSymbol: "BTCUSDT",
17436
+ marketType: "spot",
17437
+ providerExchangeId: "binance_spot",
17438
+ historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
17439
+ maxDepth: 500,
17440
+ eventTimeUnit: "milliseconds",
17441
+ receivedTimeUnit: "nanoseconds",
17442
+ snapshotGrouping: "event_time_last_update_id_object",
17443
+ sequenceSemantics: "binance_u_U_pu",
17444
+ constructionModes: ["sampled_top_n_snapshot"],
17445
+ sourcePolicies: ["authoritative_window"]
17446
+ });
17447
+ var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
17448
+ profileId: "cryptohftdata/okx_spot/ARB-USDT/v1",
17449
+ exchange: "okx",
17450
+ tradingPair: "ARB-USDT",
17451
+ sourceSymbol: "ARB-USDT",
17452
+ marketType: "spot",
17453
+ providerExchangeId: "okx_spot",
17454
+ historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
17455
+ maxDepth: 400,
17456
+ eventTimeUnit: "milliseconds",
17457
+ receivedTimeUnit: "nanoseconds",
17458
+ snapshotGrouping: "event_time_final_update_id_object",
17459
+ sequenceSemantics: "okx_seq_id_prev_seq_id",
17460
+ constructionModes: ["sampled_top_n_snapshot"],
17461
+ sourcePolicies: ["authoritative_window", "fill_gaps"]
17462
+ });
17463
+
17464
+ class CryptoHftDataError extends Error {
17465
+ reason;
17466
+ constructor(reason) {
17467
+ super(`CryptoHFTData backfill failed: ${reason}`);
17468
+ this.reason = reason;
17469
+ this.name = "CryptoHftDataError";
17470
+ }
17471
+ }
17472
+ function cryptoHftDataCapabilityFor(request2, profiles = []) {
17473
+ if (request2.providerPolicy.provider !== "cryptohftdata" || request2.scope.feed !== "ORDERBOOK" || request2.scope.exchange.trim().toLowerCase() === "mexc") {
17474
+ return;
17475
+ }
17476
+ const profile = profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.constructionModes.includes(request2.constructionMode) && candidate.sourcePolicies.includes(request2.sourcePolicy) && request2.window.startTimeMs >= candidate.historyStartMs && request2.depth <= candidate.maxDepth);
17477
+ if (!profile)
17478
+ return;
17479
+ return {
17480
+ provider: "cryptohftdata",
17481
+ adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
17482
+ providerExchangeId: profile.providerExchangeId,
17483
+ resolvedSymbol: profile.sourceSymbol
17484
+ };
17485
+ }
17486
+ function utcHourPath(hourMs) {
17487
+ const date = new Date(hourMs);
17488
+ return {
17489
+ date: `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String(date.getUTCDate()).padStart(2, "0")}`,
17490
+ hour: String(date.getUTCHours()).padStart(2, "0")
17491
+ };
17492
+ }
17493
+ function enumerateCryptoHftDataObjects(request2, providerExchangeId, resolvedSymbol) {
17494
+ if (request2.sourcePolicy === "fill_gaps") {
17495
+ const hours = new Set;
17496
+ for (const targetTimeMs of request2.requiredClockTargetsMs) {
17497
+ const firstHour2 = Math.floor(Math.max(0, targetTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
17498
+ const lastHour = Math.floor(targetTimeMs / HOUR_MS) * HOUR_MS;
17499
+ for (let hourMs = firstHour2;hourMs <= lastHour; hourMs += HOUR_MS) {
17500
+ hours.add(hourMs);
17501
+ }
17502
+ }
17503
+ return [...hours].sort((left, right) => left - right).map((hourMs) => {
17504
+ const utc = utcHourPath(hourMs);
17505
+ return `${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`;
17506
+ });
17507
+ }
17508
+ const firstHour = Math.floor((request2.window.startTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
17509
+ const objects = [];
17510
+ for (let hourMs = Math.max(0, firstHour);hourMs < request2.window.endTimeMs; hourMs += HOUR_MS) {
17511
+ const utc = utcHourPath(hourMs);
17512
+ objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
17513
+ }
17514
+ return objects;
17515
+ }
17516
+ function initialArchiveCoversTarget(request2, targetTimeMs) {
17517
+ return (request2.initialSelection?.support_anchors ?? []).some((anchor) => {
17518
+ const sourceTimeMs = Date.parse(anchor.source_time);
17519
+ return Number.isSafeInteger(sourceTimeMs) && sourceTimeMs <= targetTimeMs && targetTimeMs - sourceTimeMs <= request2.maxPriorAsOfLagMs;
17520
+ });
17521
+ }
17522
+ function providerAcquisitionRequest(request2) {
17523
+ if (request2.sourcePolicy !== "fill_gaps")
17524
+ return request2;
17525
+ const requiredClockTargetsMs = request2.requiredClockTargetsMs.filter((targetTimeMs) => !initialArchiveCoversTarget(request2, targetTimeMs));
17526
+ if (requiredClockTargetsMs.length === 0) {
17527
+ throw new CryptoHftDataError("fill_gaps_has_no_uncovered_clock_targets");
17528
+ }
17529
+ return { ...request2, requiredClockTargetsMs };
17530
+ }
17531
+ function unsignedString(value, field, required = false) {
17532
+ if (value === null || value === undefined) {
17533
+ if (required)
17534
+ throw new CryptoHftDataError(`schema_${field}_missing`);
17535
+ return;
17536
+ }
17537
+ const rendered = String(value);
17538
+ if (!/^\d+$/.test(rendered)) {
17539
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
17540
+ }
17541
+ const parsed = BigInt(rendered);
17542
+ if (parsed > 18446744073709551615n) {
17543
+ throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
17544
+ }
17545
+ return parsed.toString(10);
17546
+ }
17547
+ function timestampMs2(value, field) {
17548
+ const rendered = String(value);
17549
+ if (!/^\d+$/.test(rendered)) {
17550
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
17551
+ }
17552
+ const raw = BigInt(rendered);
17553
+ const milliseconds = field === "received_time" ? raw / 1000000n : raw;
17554
+ if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
17555
+ throw new CryptoHftDataError(`schema_${field}_unsafe`);
17556
+ }
17557
+ return Number(milliseconds);
17558
+ }
17559
+ function decimal(value, field, allowZero) {
17560
+ if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
17561
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
17562
+ }
17563
+ const parsed = Number(value);
17564
+ if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
17565
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
17566
+ }
17567
+ return parsed;
17568
+ }
17569
+ function validatedRow(request2, row) {
17570
+ if (row.symbol !== request2.scope.sourceSymbol) {
17571
+ throw new CryptoHftDataError("schema_symbol_mismatch");
17572
+ }
17573
+ if (row.event_type !== "snapshot" && row.event_type !== "update") {
17574
+ throw new CryptoHftDataError("schema_event_type_invalid");
17575
+ }
17576
+ if (row.side !== "bid" && row.side !== "ask") {
17577
+ throw new CryptoHftDataError("schema_side_invalid");
17578
+ }
17579
+ decimal(row.price, "price", false);
17580
+ decimal(row.quantity, "quantity", true);
17581
+ if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
17582
+ throw new CryptoHftDataError("schema_object_checksum_invalid");
17583
+ }
17584
+ const eventTimeMs = timestampMs2(row.event_time, "event_time");
17585
+ const receivedTimeMs = timestampMs2(row.received_time, "received_time");
17586
+ if (receivedTimeMs < eventTimeMs) {
17587
+ throw new CryptoHftDataError("received_time_precedes_event_time");
17588
+ }
17589
+ return { ...row, eventTimeMs, receivedTimeMs };
17590
+ }
17591
+ function groupKey(row, profile) {
17592
+ const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
17593
+ return [
17594
+ row.event_type,
17595
+ row.eventTimeMs,
17596
+ sequence,
17597
+ row.dataset_object_identity
17598
+ ].join("\x00");
17599
+ }
17600
+ function absent(value) {
17601
+ return value === null || value === undefined;
17602
+ }
17603
+ function snapshotSequence(row, profile) {
17604
+ if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
17605
+ if (String(row.last_update_id) !== "-1") {
17606
+ throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
18144
17607
  }
18145
- };
17608
+ if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
17609
+ throw new CryptoHftDataError("ambiguous_snapshot_group");
17610
+ }
17611
+ return unsignedString(row.final_update_id, "final_update_id", true);
17612
+ }
17613
+ return unsignedString(row.last_update_id, "last_update_id", true);
18146
17614
  }
18147
-
18148
- // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
18149
- var CRYPTOHFTDATA_ADAPTER_VERSION = "cryptohftdata-orderbook/v2";
18150
- var CRYPTOHFTDATA_API_URL = "https://api.cryptohftdata.com";
18151
- var CRYPTOHFTDATA_HISTORY_START_MS = Date.UTC(2025, 5, 28);
18152
- var HOUR_MS = 60 * 60 * 1000;
18153
- var CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE = Object.freeze({
18154
- profileId: "cryptohftdata/binance_spot/BTCUSDT/v1",
18155
- exchange: "binance",
18156
- tradingPair: "BTC-USDT",
18157
- sourceSymbol: "BTCUSDT",
18158
- marketType: "spot",
18159
- providerExchangeId: "binance_spot",
18160
- historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
18161
- maxDepth: 500,
18162
- eventTimeUnit: "milliseconds",
18163
- receivedTimeUnit: "nanoseconds",
18164
- snapshotGrouping: "event_time_last_update_id_object",
18165
- sequenceSemantics: "binance_u_U_pu",
18166
- constructionModes: ["sampled_top_n_snapshot"],
18167
- sourcePolicies: ["authoritative_window"]
18168
- });
18169
- var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
18170
- profileId: "cryptohftdata/okx_spot/ARB-USDT/v1",
18171
- exchange: "okx",
18172
- tradingPair: "ARB-USDT",
18173
- sourceSymbol: "ARB-USDT",
18174
- marketType: "spot",
18175
- providerExchangeId: "okx_spot",
18176
- historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
18177
- maxDepth: 400,
18178
- eventTimeUnit: "milliseconds",
18179
- receivedTimeUnit: "nanoseconds",
18180
- snapshotGrouping: "event_time_final_update_id_object",
18181
- sequenceSemantics: "okx_seq_id_prev_seq_id",
18182
- constructionModes: ["sampled_top_n_snapshot"],
18183
- sourcePolicies: ["authoritative_window"]
18184
- });
18185
-
18186
- class CryptoHftDataError extends Error {
18187
- reason;
18188
- constructor(reason) {
18189
- super(`CryptoHFTData backfill failed: ${reason}`);
18190
- this.reason = reason;
18191
- this.name = "CryptoHftDataError";
17615
+ function sortedSide(levels, side) {
17616
+ return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
17617
+ }
17618
+ function applyRows(state, rows) {
17619
+ for (const row of rows) {
17620
+ const levels = row.side === "bid" ? state.bids : state.asks;
17621
+ const quantity = decimal(row.quantity, "quantity", true);
17622
+ if (quantity === 0)
17623
+ levels.delete(row.price);
17624
+ else
17625
+ levels.set(row.price, quantity);
18192
17626
  }
18193
17627
  }
18194
- function cryptoHftDataCapabilityFor(request2, profiles = []) {
18195
- if (request2.providerPolicy.provider !== "cryptohftdata" || request2.scope.feed !== "ORDERBOOK" || request2.scope.exchange.trim().toLowerCase() === "mexc") {
18196
- return;
17628
+ function reconstructCryptoHftDataOrderBooks(request2, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
17629
+ const rows = inputRows.map((row, index) => ({
17630
+ ...validatedRow(request2, row),
17631
+ originalIndex: index
17632
+ })).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
17633
+ const groups = [];
17634
+ for (const row of rows) {
17635
+ const current = groups.at(-1);
17636
+ if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
17637
+ groups.push([row]);
17638
+ } else {
17639
+ current.push(row);
17640
+ }
17641
+ }
17642
+ const earliestTargetTimeMs = Math.min(...request2.requiredClockTargetsMs);
17643
+ const latestTargetTimeMs = Math.max(...request2.requiredClockTargetsMs);
17644
+ const anchorIndex = groups.findIndex((group) => {
17645
+ const first = group[0];
17646
+ return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
17647
+ });
17648
+ if (anchorIndex < 0) {
17649
+ throw new CryptoHftDataError("update_before_snapshot");
17650
+ }
17651
+ let state;
17652
+ let previousFinalUpdateId;
17653
+ const states = [];
17654
+ for (const group of groups.slice(anchorIndex)) {
17655
+ const first = group[0];
17656
+ if (first.eventTimeMs > latestTargetTimeMs)
17657
+ break;
17658
+ if (first.event_type === "snapshot") {
17659
+ const sequence = snapshotSequence(first, profile);
17660
+ for (const row of group) {
17661
+ if (snapshotSequence(row, profile) !== sequence) {
17662
+ throw new CryptoHftDataError("ambiguous_snapshot_group");
17663
+ }
17664
+ }
17665
+ if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
17666
+ throw new CryptoHftDataError("snapshot_sequence_regression");
17667
+ }
17668
+ state = {
17669
+ bids: new Map,
17670
+ asks: new Map,
17671
+ sequence,
17672
+ sourceTimeMs: first.eventTimeMs,
17673
+ receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
17674
+ datasetObjectIdentity: first.dataset_object_identity,
17675
+ datasetObjectChecksum: first.dataset_object_checksum
17676
+ };
17677
+ applyRows(state, group);
17678
+ previousFinalUpdateId = BigInt(sequence);
17679
+ } else {
17680
+ if (!state || previousFinalUpdateId === undefined) {
17681
+ throw new CryptoHftDataError("update_before_snapshot");
17682
+ }
17683
+ const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
17684
+ if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
17685
+ if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
17686
+ throw new CryptoHftDataError("ambiguous_update_group");
17687
+ }
17688
+ const previous = unsignedString(first.last_update_id, "last_update_id", true);
17689
+ for (const row of group) {
17690
+ if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
17691
+ throw new CryptoHftDataError("ambiguous_update_group");
17692
+ }
17693
+ }
17694
+ if (BigInt(previous) !== previousFinalUpdateId) {
17695
+ throw new CryptoHftDataError("update_chain_gap");
17696
+ }
17697
+ } else {
17698
+ const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
17699
+ const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
17700
+ for (const row of group) {
17701
+ if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
17702
+ throw new CryptoHftDataError("ambiguous_update_group");
17703
+ }
17704
+ }
17705
+ const expected = previousFinalUpdateId + 1n;
17706
+ if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
17707
+ throw new CryptoHftDataError("update_chain_gap");
17708
+ }
17709
+ }
17710
+ applyRows(state, group);
17711
+ state.sequence = finalUpdate.toString();
17712
+ state.sourceTimeMs = first.eventTimeMs;
17713
+ state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
17714
+ state.datasetObjectIdentity = first.dataset_object_identity;
17715
+ state.datasetObjectChecksum = first.dataset_object_checksum;
17716
+ previousFinalUpdateId = finalUpdate;
17717
+ }
17718
+ if (state) {
17719
+ states.push({
17720
+ ...state,
17721
+ bids: new Map(state.bids),
17722
+ asks: new Map(state.asks)
17723
+ });
17724
+ }
17725
+ }
17726
+ const samples = [];
17727
+ for (const targetTimeMs of request2.requiredClockTargetsMs) {
17728
+ let prior;
17729
+ for (const candidate of states) {
17730
+ if (candidate.sourceTimeMs > targetTimeMs)
17731
+ break;
17732
+ prior = candidate;
17733
+ }
17734
+ if (!prior || targetTimeMs - prior.sourceTimeMs > request2.maxPriorAsOfLagMs) {
17735
+ throw new CryptoHftDataError("required_clock_coverage_insufficient");
17736
+ }
17737
+ const bids = sortedSide(prior.bids, "bid").slice(0, request2.depth);
17738
+ const asks = sortedSide(prior.asks, "ask").slice(0, request2.depth);
17739
+ if (bids.length === 0 || asks.length === 0) {
17740
+ throw new CryptoHftDataError("book_side_missing");
17741
+ }
17742
+ if (bids[0]?.[0] >= asks[0]?.[0]) {
17743
+ throw new CryptoHftDataError("book_crossed_or_locked");
17744
+ }
17745
+ samples.push({
17746
+ targetTimeMs,
17747
+ sourceTimeMs: prior.sourceTimeMs,
17748
+ receivedTimeMs: prior.receivedTimeMs,
17749
+ sequence: prior.sequence,
17750
+ bids,
17751
+ asks,
17752
+ datasetObjectIdentity: prior.datasetObjectIdentity,
17753
+ datasetObjectChecksum: prior.datasetObjectChecksum
17754
+ });
18197
17755
  }
18198
- const profile = profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.constructionModes.includes(request2.constructionMode) && candidate.sourcePolicies.includes(request2.sourcePolicy) && request2.window.startTimeMs >= candidate.historyStartMs && request2.depth <= candidate.maxDepth);
18199
- if (!profile)
18200
- return;
18201
- return {
18202
- provider: "cryptohftdata",
18203
- adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
18204
- providerExchangeId: profile.providerExchangeId,
18205
- resolvedSymbol: profile.sourceSymbol
18206
- };
17756
+ return samples;
18207
17757
  }
18208
- function utcHourPath(hourMs) {
18209
- const date = new Date(hourMs);
17758
+ function sha256Bytes(bytes) {
17759
+ return createHash3("sha256").update(bytes).digest("hex");
17760
+ }
17761
+ async function decodeCryptoHftParquetZstd(bytes) {
17762
+ const parquet = decompress(bytes);
17763
+ const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
17764
+ return parquetReadObjects({ file: buffer });
17765
+ }
17766
+ function parsedDatasetRow(value, object) {
18210
17767
  return {
18211
- date: `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String(date.getUTCDate()).padStart(2, "0")}`,
18212
- hour: String(date.getUTCHours()).padStart(2, "0")
17768
+ received_time: value.received_time,
17769
+ event_time: value.event_time,
17770
+ transaction_time: value.transaction_time,
17771
+ symbol: value.symbol,
17772
+ event_type: value.event_type,
17773
+ first_update_id: value.first_update_id,
17774
+ final_update_id: value.final_update_id,
17775
+ prev_final_update_id: value.prev_final_update_id,
17776
+ last_update_id: value.last_update_id,
17777
+ side: value.side,
17778
+ price: value.price,
17779
+ quantity: value.quantity,
17780
+ order_count: value.order_count,
17781
+ dataset_object_identity: object.identity,
17782
+ dataset_object_checksum: object.checksum
18213
17783
  };
18214
17784
  }
18215
- function enumerateCryptoHftDataObjects(request2, providerExchangeId, resolvedSymbol) {
18216
- const firstHour = Math.floor((request2.window.startTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
18217
- const objects = [];
18218
- for (let hourMs = Math.max(0, firstHour);hourMs < request2.window.endTimeMs; hourMs += HOUR_MS) {
18219
- const utc = utcHourPath(hourMs);
18220
- objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
17785
+ async function readBoundedObject(response, remainingBytes) {
17786
+ if (!response.body)
17787
+ return new Uint8Array;
17788
+ const reader = response.body.getReader();
17789
+ const chunks = [];
17790
+ let total = 0;
17791
+ while (true) {
17792
+ const { done, value } = await reader.read();
17793
+ if (done)
17794
+ break;
17795
+ total += value.byteLength;
17796
+ if (total > remainingBytes) {
17797
+ await reader.cancel();
17798
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
17799
+ }
17800
+ chunks.push(value);
18221
17801
  }
18222
- return objects;
17802
+ const bytes = new Uint8Array(total);
17803
+ let offset = 0;
17804
+ for (const chunk of chunks) {
17805
+ bytes.set(chunk, offset);
17806
+ offset += chunk.byteLength;
17807
+ }
17808
+ return bytes;
18223
17809
  }
18224
- function unsignedString(value, field, required = false) {
18225
- if (value === null || value === undefined) {
18226
- if (required)
18227
- throw new CryptoHftDataError(`schema_${field}_missing`);
18228
- return;
17810
+
17811
+ class CryptoHftDataAdapter {
17812
+ baseUrl;
17813
+ request;
17814
+ nowMs;
17815
+ decode;
17816
+ profiles;
17817
+ constructor(options = {}) {
17818
+ this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
17819
+ this.request = options.fetch ?? fetch;
17820
+ this.nowMs = options.nowMs ?? Date.now;
17821
+ this.decode = options.decode ?? decodeCryptoHftParquetZstd;
17822
+ this.profiles = options.profiles ?? [];
18229
17823
  }
18230
- const rendered = String(value);
18231
- if (!/^\d+$/.test(rendered)) {
18232
- throw new CryptoHftDataError(`schema_${field}_invalid`);
17824
+ capabilityFor(request2) {
17825
+ return cryptoHftDataCapabilityFor(request2, this.profiles);
18233
17826
  }
18234
- const parsed = BigInt(rendered);
18235
- if (parsed > 18446744073709551615n) {
18236
- throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
17827
+ async discoverSymbols(providerExchangeId) {
17828
+ if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
17829
+ throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
17830
+ }
17831
+ const endpoint = new URL("/symbols", this.baseUrl);
17832
+ endpoint.searchParams.set("exchange", providerExchangeId);
17833
+ endpoint.searchParams.set("data_type", "orderbook");
17834
+ let response;
17835
+ try {
17836
+ response = await this.request(endpoint);
17837
+ } catch {
17838
+ throw new CryptoHftDataError("symbol_discovery_failed");
17839
+ }
17840
+ if (!response.ok) {
17841
+ throw new CryptoHftDataError("symbol_discovery_failed");
17842
+ }
17843
+ let body;
17844
+ try {
17845
+ body = await response.json();
17846
+ } catch {
17847
+ throw new CryptoHftDataError("symbol_discovery_response_invalid");
17848
+ }
17849
+ const record = body;
17850
+ if (!body || typeof body !== "object" || Array.isArray(body) || String(record.exchange).toLowerCase() !== providerExchangeId || String(record.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record.symbols) || !record.symbols.every((symbol) => typeof symbol === "string" && /^[A-Z0-9_-]+$/.test(symbol))) {
17851
+ throw new CryptoHftDataError("symbol_discovery_response_invalid");
17852
+ }
17853
+ return [...new Set(record.symbols)].sort();
18237
17854
  }
18238
- return parsed.toString(10);
18239
- }
18240
- function timestampMs2(value, field) {
18241
- const rendered = String(value);
18242
- if (!/^\d+$/.test(rendered)) {
18243
- throw new CryptoHftDataError(`schema_${field}_invalid`);
17855
+ async acquire(request2, capability, credential) {
17856
+ const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
17857
+ if (typeof apiKey !== "string" || apiKey.length === 0) {
17858
+ throw new CryptoHftDataError("credentials_invalid");
17859
+ }
17860
+ const paths = enumerateCryptoHftDataObjects(request2, capability.providerExchangeId, capability.resolvedSymbol);
17861
+ if (paths.length > request2.budgets.maxFiles) {
17862
+ throw new CryptoHftDataError("budget_max_files_exceeded");
17863
+ }
17864
+ const started = this.nowMs();
17865
+ const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
17866
+ method: "POST",
17867
+ headers: { "content-type": "application/json", "X-API-Key": apiKey }
17868
+ });
17869
+ if (!tokenResponse.ok)
17870
+ throw new CryptoHftDataError("jwt_issuance_failed");
17871
+ const tokenBody = await tokenResponse.json();
17872
+ if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
17873
+ throw new CryptoHftDataError("jwt_response_invalid");
17874
+ }
17875
+ let totalBytes = 0;
17876
+ let totalRows = 0;
17877
+ const objects = [];
17878
+ const rows = [];
17879
+ for (const path of paths) {
17880
+ if (this.nowMs() - started > request2.budgets.maxDurationMs) {
17881
+ throw new CryptoHftDataError("budget_max_duration_exceeded");
17882
+ }
17883
+ const endpoint = new URL("/download", this.baseUrl);
17884
+ endpoint.searchParams.set("file", path);
17885
+ const response = await this.request(endpoint, {
17886
+ headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
17887
+ });
17888
+ if (!response.ok)
17889
+ throw new CryptoHftDataError("object_download_failed");
17890
+ const declaredBytes = Number(response.headers.get("content-length"));
17891
+ if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request2.budgets.maxBytes) {
17892
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
17893
+ }
17894
+ const bytes = await readBoundedObject(response, request2.budgets.maxBytes - totalBytes);
17895
+ totalBytes += bytes.byteLength;
17896
+ if (totalBytes > request2.budgets.maxBytes) {
17897
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
17898
+ }
17899
+ const decoded = await this.decode(bytes);
17900
+ totalRows += decoded.length;
17901
+ if (totalRows > request2.budgets.maxRows) {
17902
+ throw new CryptoHftDataError("budget_max_rows_exceeded");
17903
+ }
17904
+ const object = {
17905
+ identity: path,
17906
+ checksum: sha256Bytes(bytes),
17907
+ bytes: bytes.byteLength,
17908
+ rows: decoded.length
17909
+ };
17910
+ objects.push(object);
17911
+ for (const decodedRow of decoded) {
17912
+ const parsed = parsedDatasetRow(decodedRow, object);
17913
+ validatedRow(request2, parsed);
17914
+ rows.push(parsed);
17915
+ }
17916
+ }
17917
+ if (this.nowMs() - started > request2.budgets.maxDurationMs) {
17918
+ throw new CryptoHftDataError("budget_max_duration_exceeded");
17919
+ }
17920
+ return {
17921
+ objects,
17922
+ rows,
17923
+ vendorSemanticDigest: sha256Canonical(rows)
17924
+ };
18244
17925
  }
18245
- const raw = BigInt(rendered);
18246
- const milliseconds = field === "received_time" ? raw / 1000000n : raw;
18247
- if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
18248
- throw new CryptoHftDataError(`schema_${field}_unsafe`);
17926
+ async normalize(request2, capability, dataset, captureBundleId2) {
17927
+ const profile = this.profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
17928
+ if (!profile) {
17929
+ throw new CryptoHftDataError("profile_semantics_unavailable");
17930
+ }
17931
+ const samples = reconstructCryptoHftDataOrderBooks(request2, dataset.rows, profile);
17932
+ const context2 = {
17933
+ source: EXTERNAL_BACKFILL_SOURCE,
17934
+ deploymentId: "market-data-vendor-backfill",
17935
+ captureBundleId: captureBundleId2,
17936
+ exchange: request2.scope.exchange,
17937
+ symbol: request2.scope.tradingPair,
17938
+ tradingPair: request2.scope.tradingPair,
17939
+ sourceSymbol: request2.scope.sourceSymbol,
17940
+ assetType: request2.scope.marketType,
17941
+ feed: "ORDERBOOK",
17942
+ provider: "cryptohftdata",
17943
+ sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
17944
+ schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
17945
+ checksumAlgorithm: CHECKSUM_ALGORITHM,
17946
+ provenanceComplete: true
17947
+ };
17948
+ const rows = samples.flatMap((sample) => {
17949
+ const rawCapture = {
17950
+ rawCaptureId: sha256Canonical({
17951
+ capture_bundle_id: captureBundleId2,
17952
+ dataset_object_identity: sample.datasetObjectIdentity,
17953
+ dataset_object_checksum: sample.datasetObjectChecksum,
17954
+ source_time_ms: sample.sourceTimeMs,
17955
+ target_time_ms: sample.targetTimeMs
17956
+ }),
17957
+ rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
17958
+ rawChecksum: sample.datasetObjectChecksum,
17959
+ redactedPayload: {
17960
+ dataset_object_identity: sample.datasetObjectIdentity,
17961
+ dataset_object_checksum: sample.datasetObjectChecksum
17962
+ },
17963
+ eventTimeMs: sample.sourceTimeMs,
17964
+ receivedTimeMs: sample.receivedTimeMs,
17965
+ checksumAlgorithm: CHECKSUM_ALGORITHM
17966
+ };
17967
+ const canonical = buildCanonicalOrderBookRows({
17968
+ context: context2,
17969
+ rawCapture,
17970
+ depthLimit: request2.depth,
17971
+ constructionMode: request2.constructionMode,
17972
+ snapshot: {
17973
+ bids: sample.bids,
17974
+ asks: sample.asks,
17975
+ timestamp: sample.sourceTimeMs,
17976
+ receivedTimestamp: sample.receivedTimeMs,
17977
+ exchange: request2.scope.exchange,
17978
+ symbol: request2.scope.tradingPair,
17979
+ depthLimit: request2.depth,
17980
+ sequence: sample.sequence
17981
+ }
17982
+ });
17983
+ return [...canonical.levels, canonical.summary];
17984
+ });
17985
+ return {
17986
+ captureBundleId: captureBundleId2,
17987
+ objects: dataset.objects,
17988
+ rows,
17989
+ vendorSemanticDigest: dataset.vendorSemanticDigest,
17990
+ canonicalSemanticDigest: semanticDigest(rows)
17991
+ };
18249
17992
  }
18250
- return Number(milliseconds);
18251
17993
  }
18252
- function decimal(value, field, allowZero) {
18253
- if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
18254
- throw new CryptoHftDataError(`schema_${field}_invalid`);
18255
- }
18256
- const parsed = Number(value);
18257
- if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
18258
- throw new CryptoHftDataError(`schema_${field}_invalid`);
17994
+
17995
+ // src/helpers/market-data-vendor-backfill/qualification.ts
17996
+ var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
17997
+ var SHA256 = /^[0-9a-f]{64}$/;
17998
+ var REASON = /^[a-z][a-z0-9_]{0,127}$/;
17999
+ var EVENT_FIELDS = new Set([
18000
+ "qualification_event_id",
18001
+ "capture_bundle_id",
18002
+ "state",
18003
+ "receipt_id",
18004
+ "promotion_identity_sha256",
18005
+ "window",
18006
+ "event_at",
18007
+ "reason_code"
18008
+ ]);
18009
+ function deterministicUuid(value) {
18010
+ const digest = jcsSha256(value).slice(0, 32).split("");
18011
+ digest[12] = "5";
18012
+ digest[16] = "8";
18013
+ const hex = digest.join("");
18014
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
18015
+ }
18016
+ function semanticEvent(event) {
18017
+ const { qualification_event_id: _eventId, ...semantic } = event;
18018
+ return semantic;
18019
+ }
18020
+ function assertTimestamp(value, field) {
18021
+ const parsed = Date.parse(value);
18022
+ if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
18023
+ throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
18259
18024
  }
18260
18025
  return parsed;
18261
18026
  }
18262
- function validatedRow(request2, row) {
18263
- if (row.symbol !== request2.scope.sourceSymbol) {
18264
- throw new CryptoHftDataError("schema_symbol_mismatch");
18265
- }
18266
- if (row.event_type !== "snapshot" && row.event_type !== "update") {
18267
- throw new CryptoHftDataError("schema_event_type_invalid");
18027
+ function parseQualificationEvent(value) {
18028
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
18029
+ throw new Error("qualification event must be an object");
18268
18030
  }
18269
- if (row.side !== "bid" && row.side !== "ask") {
18270
- throw new CryptoHftDataError("schema_side_invalid");
18031
+ const event = value;
18032
+ if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
18033
+ throw new Error("qualification event fields are invalid");
18271
18034
  }
18272
- decimal(row.price, "price", false);
18273
- decimal(row.quantity, "quantity", true);
18274
- if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
18275
- throw new CryptoHftDataError("schema_object_checksum_invalid");
18035
+ if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
18036
+ throw new Error("qualification window must be increasing");
18276
18037
  }
18277
- const eventTimeMs = timestampMs2(row.event_time, "event_time");
18278
- const receivedTimeMs = timestampMs2(row.received_time, "received_time");
18279
- if (receivedTimeMs < eventTimeMs) {
18280
- throw new CryptoHftDataError("received_time_precedes_event_time");
18038
+ assertTimestamp(event.event_at, "event_at");
18039
+ if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
18040
+ throw new Error("qualification_event_id does not match event content");
18281
18041
  }
18282
- return { ...row, eventTimeMs, receivedTimeMs };
18283
- }
18284
- function groupKey(row, profile) {
18285
- const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
18286
- return [
18287
- row.event_type,
18288
- row.eventTimeMs,
18289
- sequence,
18290
- row.dataset_object_identity
18291
- ].join("\x00");
18042
+ return event;
18292
18043
  }
18293
- function absent(value) {
18294
- return value === null || value === undefined;
18044
+ function finalizeQualificationEvent(event) {
18045
+ const semantic = semanticEvent(event);
18046
+ return parseQualificationEvent({
18047
+ ...semantic,
18048
+ qualification_event_id: deterministicUuid(semantic)
18049
+ });
18295
18050
  }
18296
- function snapshotSequence(row, profile) {
18297
- if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
18298
- if (String(row.last_update_id) !== "-1") {
18299
- throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
18300
- }
18301
- if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
18302
- throw new CryptoHftDataError("ambiguous_snapshot_group");
18051
+ function qualificationEventToArchiveRow(eventInput) {
18052
+ const event = parseQualificationEvent(eventInput);
18053
+ return {
18054
+ table: "market_data.cex_order_book_capture_qualifications",
18055
+ row: {
18056
+ source: "external_backfill",
18057
+ capture_origin: "vendor_historical_backfill",
18058
+ source_mode: "vendor_historical_backfill_v1",
18059
+ deployment_id: "market-data-vendor-backfill",
18060
+ qualification_event_id: event.qualification_event_id,
18061
+ capture_bundle_id: event.capture_bundle_id,
18062
+ state: event.state,
18063
+ receipt_id: event.receipt_id,
18064
+ promotion_identity_sha256: event.promotion_identity_sha256,
18065
+ window_start_ms: Date.parse(event.window.start_at),
18066
+ window_end_ms: Date.parse(event.window.end_at),
18067
+ event_at_ms: Date.parse(event.event_at),
18068
+ reason_code: event.reason_code,
18069
+ event_json: jcsCanonicalize(event)
18303
18070
  }
18304
- return unsignedString(row.final_update_id, "final_update_id", true);
18305
- }
18306
- return unsignedString(row.last_update_id, "last_update_id", true);
18307
- }
18308
- function sortedSide(levels, side) {
18309
- return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
18310
- }
18311
- function applyRows(state, rows) {
18312
- for (const row of rows) {
18313
- const levels = row.side === "bid" ? state.bids : state.asks;
18314
- const quantity = decimal(row.quantity, "quantity", true);
18315
- if (quantity === 0)
18316
- levels.delete(row.price);
18317
- else
18318
- levels.set(row.price, quantity);
18319
- }
18071
+ };
18320
18072
  }
18321
- function reconstructCryptoHftDataOrderBooks(request2, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
18322
- const rows = inputRows.map((row, index) => ({
18323
- ...validatedRow(request2, row),
18324
- originalIndex: index
18325
- })).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
18326
- const groups = [];
18327
- for (const row of rows) {
18328
- const current = groups.at(-1);
18329
- if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
18330
- groups.push([row]);
18331
- } else {
18332
- current.push(row);
18073
+
18074
+ // src/helpers/market-data-vendor-backfill/core.ts
18075
+ function createMarketDataVendorBackfillDependencies(dependencies) {
18076
+ for (const [name, value] of Object.entries({
18077
+ archive: dependencies.archive,
18078
+ providers: dependencies.providers,
18079
+ credentials: dependencies.credentials,
18080
+ forwarder: dependencies.forwarder,
18081
+ clock: dependencies.clock
18082
+ })) {
18083
+ if (!value || typeof value !== "object") {
18084
+ throw new TypeError(`Backfill dependency ${name} is required`);
18333
18085
  }
18334
18086
  }
18335
- const earliestTargetTimeMs = Math.min(...request2.requiredClockTargetsMs);
18336
- const latestTargetTimeMs = Math.max(...request2.requiredClockTargetsMs);
18337
- const anchorIndex = groups.findIndex((group) => {
18338
- const first = group[0];
18339
- return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
18340
- });
18341
- if (anchorIndex < 0) {
18342
- throw new CryptoHftDataError("update_before_snapshot");
18343
- }
18344
- let state;
18345
- let previousFinalUpdateId;
18346
- const states = [];
18347
- for (const group of groups.slice(anchorIndex)) {
18348
- const first = group[0];
18349
- if (first.eventTimeMs > latestTargetTimeMs)
18350
- break;
18351
- if (first.event_type === "snapshot") {
18352
- const sequence = snapshotSequence(first, profile);
18353
- for (const row of group) {
18354
- if (snapshotSequence(row, profile) !== sequence) {
18355
- throw new CryptoHftDataError("ambiguous_snapshot_group");
18356
- }
18357
- }
18358
- if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
18359
- throw new CryptoHftDataError("snapshot_sequence_regression");
18360
- }
18361
- state = {
18362
- bids: new Map,
18363
- asks: new Map,
18364
- sequence,
18365
- sourceTimeMs: first.eventTimeMs,
18366
- receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
18367
- datasetObjectIdentity: first.dataset_object_identity,
18368
- datasetObjectChecksum: first.dataset_object_checksum
18369
- };
18370
- applyRows(state, group);
18371
- previousFinalUpdateId = BigInt(sequence);
18372
- } else {
18373
- if (!state || previousFinalUpdateId === undefined) {
18374
- throw new CryptoHftDataError("update_before_snapshot");
18375
- }
18376
- const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
18377
- if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
18378
- if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
18379
- throw new CryptoHftDataError("ambiguous_update_group");
18380
- }
18381
- const previous = unsignedString(first.last_update_id, "last_update_id", true);
18382
- for (const row of group) {
18383
- if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
18384
- throw new CryptoHftDataError("ambiguous_update_group");
18385
- }
18386
- }
18387
- if (BigInt(previous) !== previousFinalUpdateId) {
18388
- throw new CryptoHftDataError("update_chain_gap");
18389
- }
18390
- } else {
18391
- const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
18392
- const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
18393
- for (const row of group) {
18394
- if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
18395
- throw new CryptoHftDataError("ambiguous_update_group");
18396
- }
18397
- }
18398
- const expected = previousFinalUpdateId + 1n;
18399
- if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
18400
- throw new CryptoHftDataError("update_chain_gap");
18401
- }
18402
- }
18403
- applyRows(state, group);
18404
- state.sequence = finalUpdate.toString();
18405
- state.sourceTimeMs = first.eventTimeMs;
18406
- state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
18407
- state.datasetObjectIdentity = first.dataset_object_identity;
18408
- state.datasetObjectChecksum = first.dataset_object_checksum;
18409
- previousFinalUpdateId = finalUpdate;
18410
- }
18411
- if (state) {
18412
- states.push({
18413
- ...state,
18414
- bids: new Map(state.bids),
18415
- asks: new Map(state.asks)
18416
- });
18087
+ for (const [name, method] of Object.entries({
18088
+ "archive.resolveSelection": dependencies.archive.resolveSelection,
18089
+ "archive.verifyCandidate": dependencies.archive.verifyCandidate,
18090
+ "providers.capabilityFor": dependencies.providers.capabilityFor,
18091
+ "providers.acquire": dependencies.providers.acquire,
18092
+ "providers.normalize": dependencies.providers.normalize,
18093
+ "credentials.resolve": dependencies.credentials.resolve,
18094
+ "forwarder.preflight": dependencies.forwarder.preflight,
18095
+ "forwarder.submit": dependencies.forwarder.submit,
18096
+ "clock.nowMs": dependencies.clock.nowMs
18097
+ })) {
18098
+ if (typeof method !== "function") {
18099
+ throw new TypeError(`Backfill dependency ${name} is required`);
18417
18100
  }
18418
18101
  }
18419
- const samples = [];
18420
- for (const targetTimeMs of request2.requiredClockTargetsMs) {
18421
- let prior;
18422
- for (const candidate of states) {
18423
- if (candidate.sourceTimeMs > targetTimeMs)
18102
+ return dependencies;
18103
+ }
18104
+ function outcome(request2, status, reasonCode, extra = {}) {
18105
+ return {
18106
+ status,
18107
+ reasonCode,
18108
+ ...request2 ? {
18109
+ requestId: request2.requestId,
18110
+ idempotencyKey: request2.idempotencyKey,
18111
+ ...request2.target ? { target: request2.target } : {}
18112
+ } : {},
18113
+ ...extra
18114
+ };
18115
+ }
18116
+ function captureBundleId2(request2, capability, dataset) {
18117
+ return sha256Canonical({
18118
+ request_business_identity: request2.idempotencyKey,
18119
+ provider: capability.provider,
18120
+ provider_exchange_id: capability.providerExchangeId,
18121
+ resolved_symbol: capability.resolvedSymbol,
18122
+ adapter_version: capability.adapterVersion,
18123
+ objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
18124
+ identity,
18125
+ checksum,
18126
+ bytes,
18127
+ rows
18128
+ })),
18129
+ canonical_schema_version: request2.expectedProduct.canonicalSchemaVersion,
18130
+ checksum_algorithm: request2.expectedProduct.checksumAlgorithm
18131
+ });
18132
+ }
18133
+ function buildPromotionReceipt(request2, capability, normalized, verification, verificationTimeMs) {
18134
+ if (!request2.wire || !request2.initialSelection || !request2.expectedCanonicalSchema || !request2.coveragePolicy || !request2.productPins) {
18135
+ throw new Error("decoded final-v1 request context is missing");
18136
+ }
18137
+ return finalizePromotionReceipt({
18138
+ schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
18139
+ verified_at: new Date(verificationTimeMs).toISOString(),
18140
+ request_id: request2.requestId,
18141
+ idempotency_key: request2.idempotencyKey,
18142
+ source: EXTERNAL_BACKFILL_SOURCE,
18143
+ capture_origin: "vendor_historical_backfill",
18144
+ source_mode: "vendor_historical_backfill_v1",
18145
+ provider: capability.provider,
18146
+ adapter_version: capability.adapterVersion,
18147
+ effective_policies: {
18148
+ capability_policy: request2.productPins.capability_policy,
18149
+ resource_policy: request2.productPins.resource_policy,
18150
+ adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
18151
+ acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
18152
+ },
18153
+ capture_bundle_id: normalized.captureBundleId,
18154
+ scope: request2.wire.scope,
18155
+ window: request2.wire.window,
18156
+ depth: request2.depth,
18157
+ construction_mode: request2.constructionMode,
18158
+ canonical_schema: request2.expectedCanonicalSchema,
18159
+ coverage_policy: request2.coveragePolicy,
18160
+ selection_sha256: request2.initialSelection.selection_sha256,
18161
+ vendor_semantic_digest: normalized.vendorSemanticDigest,
18162
+ canonical_semantic_digest: verification.canonicalSemanticDigest,
18163
+ prefix_digest: verification.prefixDigest,
18164
+ suffix_digest: verification.suffixDigest,
18165
+ seam_verified: true,
18166
+ coverage_verified: true,
18167
+ dataset_objects: normalized.objects
18168
+ });
18169
+ }
18170
+ async function submitAll(dependencies, batches) {
18171
+ const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
18172
+ for (const batch of batches) {
18173
+ let accepted = false;
18174
+ for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
18175
+ try {
18176
+ const response = await dependencies.forwarder.submit(batch);
18177
+ accepted = response.ok && response.inserted === batch.rows.length;
18178
+ } catch {
18179
+ accepted = false;
18180
+ }
18181
+ if (accepted)
18424
18182
  break;
18425
- prior = candidate;
18183
+ if (attempt < maxAttempts)
18184
+ await dependencies.retry?.wait?.(attempt);
18426
18185
  }
18427
- if (!prior || targetTimeMs - prior.sourceTimeMs > request2.maxPriorAsOfLagMs) {
18428
- throw new CryptoHftDataError("required_clock_coverage_insufficient");
18186
+ if (!accepted)
18187
+ return false;
18188
+ }
18189
+ return true;
18190
+ }
18191
+ function stableFailureReason(error, fallback) {
18192
+ if (error && typeof error === "object") {
18193
+ const reason = error.reason;
18194
+ if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
18195
+ return reason;
18429
18196
  }
18430
- const bids = sortedSide(prior.bids, "bid").slice(0, request2.depth);
18431
- const asks = sortedSide(prior.asks, "ask").slice(0, request2.depth);
18432
- if (bids.length === 0 || asks.length === 0) {
18433
- throw new CryptoHftDataError("book_side_missing");
18197
+ }
18198
+ return fallback;
18199
+ }
18200
+ function assertArchivePreflight(request2, resolution) {
18201
+ if (!request2.target || !request2.productionAuthorizationId) {
18202
+ throw new Error("request target or production authorization ID is missing");
18203
+ }
18204
+ const selection = archiveSelectionCodec.decode(resolution.selection);
18205
+ const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
18206
+ const receiptById = new Map;
18207
+ for (const receipt of receipts) {
18208
+ const existing = receiptById.get(receipt.receipt_id);
18209
+ if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
18210
+ throw new Error("stored receipt identity has conflicting content");
18434
18211
  }
18435
- if (bids[0]?.[0] >= asks[0]?.[0]) {
18436
- throw new CryptoHftDataError("book_crossed_or_locked");
18212
+ receiptById.set(receipt.receipt_id, receipt);
18213
+ }
18214
+ for (const bundle of selection.bundles) {
18215
+ if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
18216
+ throw new Error("vendor selection lacks its validated stored receipt");
18437
18217
  }
18438
- samples.push({
18439
- targetTimeMs,
18440
- sourceTimeMs: prior.sourceTimeMs,
18441
- receivedTimeMs: prior.receivedTimeMs,
18442
- sequence: prior.sequence,
18443
- bids,
18444
- asks,
18445
- datasetObjectIdentity: prior.datasetObjectIdentity,
18446
- datasetObjectChecksum: prior.datasetObjectChecksum
18447
- });
18448
18218
  }
18449
- return samples;
18219
+ if (resolution.readerIdentity.environment !== request2.target.environment || resolution.readerIdentity.cluster !== request2.target.cluster) {
18220
+ throw new Error("archive reader cluster identity mismatch");
18221
+ }
18450
18222
  }
18451
- function sha256Bytes(bytes) {
18452
- return createHash3("sha256").update(bytes).digest("hex");
18223
+ function assertForwarderPreflight(request2, resolution, nowMs) {
18224
+ if (!request2.target || !request2.productionAuthorizationId) {
18225
+ throw new Error("request target or production authorization ID is missing");
18226
+ }
18227
+ if (resolution.forwarderIdentity.environment !== request2.target.environment || resolution.forwarderIdentity.cluster !== request2.target.cluster) {
18228
+ throw new Error("archive forwarder cluster identity mismatch");
18229
+ }
18230
+ const authorization = resolution.authorization;
18231
+ const expiresAtMs = Date.parse(authorization.expiresAt);
18232
+ if (authorization.authorizationId !== request2.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request2.target.environment || authorization.cluster !== request2.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
18233
+ throw new Error("production forwarder authorization is invalid");
18234
+ }
18453
18235
  }
18454
- async function decodeCryptoHftParquetZstd(bytes) {
18455
- const parquet = decompress(bytes);
18456
- const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
18457
- return parquetReadObjects({ file: buffer });
18236
+ function resourcePolicyScopeExceeded(request2) {
18237
+ return request2.depth > RESOURCE_POLICY.request_bounds.max_depth || request2.window.endTimeMs - request2.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request2.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
18458
18238
  }
18459
- function parsedDatasetRow(value, object) {
18460
- return {
18461
- received_time: value.received_time,
18462
- event_time: value.event_time,
18463
- transaction_time: value.transaction_time,
18464
- symbol: value.symbol,
18465
- event_type: value.event_type,
18466
- first_update_id: value.first_update_id,
18467
- final_update_id: value.final_update_id,
18468
- prev_final_update_id: value.prev_final_update_id,
18469
- last_update_id: value.last_update_id,
18470
- side: value.side,
18471
- price: value.price,
18472
- quantity: value.quantity,
18473
- order_count: value.order_count,
18474
- dataset_object_identity: object.identity,
18475
- dataset_object_checksum: object.checksum
18476
- };
18239
+ function storedReceiptForSelection(resolution) {
18240
+ const receiptId = resolution.selection.receipt_ids[0];
18241
+ return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
18477
18242
  }
18478
- async function readBoundedObject(response, remainingBytes) {
18479
- if (!response.body)
18480
- return new Uint8Array;
18481
- const reader = response.body.getReader();
18482
- const chunks = [];
18483
- let total = 0;
18484
- while (true) {
18485
- const { done, value } = await reader.read();
18486
- if (done)
18487
- break;
18488
- total += value.byteLength;
18489
- if (total > remainingBytes) {
18490
- await reader.cancel();
18491
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
18492
- }
18493
- chunks.push(value);
18243
+ async function runMarketDataVendorBackfill(input, dependencies) {
18244
+ let request2;
18245
+ try {
18246
+ const documents = input;
18247
+ request2 = decodeBackfillRunDocuments({
18248
+ request: documents.request,
18249
+ requiredClock: documents.requiredClock
18250
+ });
18251
+ } catch {
18252
+ return outcome(undefined, "request_invalid", "request_invalid");
18494
18253
  }
18495
- const bytes = new Uint8Array(total);
18496
- let offset = 0;
18497
- for (const chunk of chunks) {
18498
- bytes.set(chunk, offset);
18499
- offset += chunk.byteLength;
18254
+ let initialResolution;
18255
+ try {
18256
+ initialResolution = await dependencies.archive.resolveSelection(request2);
18257
+ assertArchivePreflight(request2, initialResolution);
18258
+ const forwarderPreflight = await dependencies.forwarder.preflight({
18259
+ authorizationId: request2.productionAuthorizationId,
18260
+ target: request2.target
18261
+ });
18262
+ assertForwarderPreflight(request2, forwarderPreflight, dependencies.clock.nowMs());
18263
+ } catch {
18264
+ return outcome(request2, "archive_preflight_failed", "archive_preflight_failed");
18500
18265
  }
18501
- return bytes;
18502
- }
18503
-
18504
- class CryptoHftDataAdapter {
18505
- baseUrl;
18506
- request;
18507
- nowMs;
18508
- decode;
18509
- profiles;
18510
- constructor(options = {}) {
18511
- this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
18512
- this.request = options.fetch ?? fetch;
18513
- this.nowMs = options.nowMs ?? Date.now;
18514
- this.decode = options.decode ?? decodeCryptoHftParquetZstd;
18515
- this.profiles = options.profiles ?? [];
18266
+ if (initialResolution.selection.coverage_class === "complete") {
18267
+ const receipt2 = storedReceiptForSelection(initialResolution);
18268
+ return outcome(request2, "already_covered", "qualified_coverage_complete", {
18269
+ selection: initialResolution.selection,
18270
+ ...receipt2 ? { receipt: receipt2 } : {}
18271
+ });
18272
+ }
18273
+ if (resourcePolicyScopeExceeded(request2)) {
18274
+ return outcome(request2, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
18275
+ }
18276
+ if (request2.sourcePolicy === "fill_gaps" && (request2.productPins?.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request2.productPins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256)) {
18277
+ return outcome(request2, "capability_unsupported", "scope_unsupported", {
18278
+ reasonSubcode: "fill_gaps_requires_capability_policy_v2"
18279
+ });
18280
+ }
18281
+ let capability;
18282
+ try {
18283
+ capability = dependencies.providers.capabilityFor(request2);
18284
+ } catch {
18285
+ return outcome(request2, "capability_unsupported", "capability_probe_failed");
18286
+ }
18287
+ if (!capability) {
18288
+ return outcome(request2, "capability_unsupported", "scope_unsupported");
18289
+ }
18290
+ if (!request2.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
18291
+ return outcome(request2, "capability_unsupported", "adapter_version_unpinned");
18516
18292
  }
18517
- capabilityFor(request2) {
18518
- return cryptoHftDataCapabilityFor(request2, this.profiles);
18293
+ let credential;
18294
+ try {
18295
+ credential = await dependencies.credentials.resolve(capability.provider);
18296
+ } catch {
18297
+ return outcome(request2, "credentials_missing", "credential_resolution_failed");
18519
18298
  }
18520
- async discoverSymbols(providerExchangeId) {
18521
- if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
18522
- throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
18523
- }
18524
- const endpoint = new URL("/symbols", this.baseUrl);
18525
- endpoint.searchParams.set("exchange", providerExchangeId);
18526
- endpoint.searchParams.set("data_type", "orderbook");
18527
- let response;
18528
- try {
18529
- response = await this.request(endpoint);
18530
- } catch {
18531
- throw new CryptoHftDataError("symbol_discovery_failed");
18532
- }
18533
- if (!response.ok) {
18534
- throw new CryptoHftDataError("symbol_discovery_failed");
18535
- }
18536
- let body;
18537
- try {
18538
- body = await response.json();
18539
- } catch {
18540
- throw new CryptoHftDataError("symbol_discovery_response_invalid");
18541
- }
18542
- const record = body;
18543
- if (!body || typeof body !== "object" || Array.isArray(body) || String(record.exchange).toLowerCase() !== providerExchangeId || String(record.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record.symbols) || !record.symbols.every((symbol) => typeof symbol === "string" && /^[A-Z0-9_-]+$/.test(symbol))) {
18544
- throw new CryptoHftDataError("symbol_discovery_response_invalid");
18545
- }
18546
- return [...new Set(record.symbols)].sort();
18299
+ if (credential === undefined || credential === null) {
18300
+ return outcome(request2, "credentials_missing", "provider_credentials_missing");
18547
18301
  }
18548
- async acquire(request2, capability, credential) {
18549
- const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
18550
- if (typeof apiKey !== "string" || apiKey.length === 0) {
18551
- throw new CryptoHftDataError("credentials_invalid");
18552
- }
18553
- const paths = enumerateCryptoHftDataObjects(request2, capability.providerExchangeId, capability.resolvedSymbol);
18554
- if (paths.length > request2.budgets.maxFiles) {
18555
- throw new CryptoHftDataError("budget_max_files_exceeded");
18302
+ let dataset;
18303
+ let normalized;
18304
+ try {
18305
+ const acquisitionRequest = providerAcquisitionRequest(request2);
18306
+ dataset = await dependencies.providers.acquire(acquisitionRequest, capability, credential);
18307
+ const bundleId = captureBundleId2(request2, capability, dataset);
18308
+ normalized = await dependencies.providers.normalize(acquisitionRequest, capability, dataset, bundleId);
18309
+ if (normalized.captureBundleId !== bundleId) {
18310
+ return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
18311
+ reasonSubcode: "capture_identity_mismatch"
18312
+ });
18556
18313
  }
18557
- const started = this.nowMs();
18558
- const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
18559
- method: "POST",
18560
- headers: { "content-type": "application/json", "X-API-Key": apiKey }
18314
+ } catch (error) {
18315
+ const reason = stableFailureReason(error, "provider_dataset_invalid");
18316
+ return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
18317
+ reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
18561
18318
  });
18562
- if (!tokenResponse.ok)
18563
- throw new CryptoHftDataError("jwt_issuance_failed");
18564
- const tokenBody = await tokenResponse.json();
18565
- if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
18566
- throw new CryptoHftDataError("jwt_response_invalid");
18319
+ }
18320
+ try {
18321
+ const candidateBatches = buildForwarderBatches({
18322
+ captureBundleId: normalized.captureBundleId,
18323
+ deploymentId: "market-data-vendor-backfill",
18324
+ rows: normalized.rows
18325
+ });
18326
+ if (!await submitAll(dependencies, candidateBatches)) {
18327
+ return outcome(request2, "archive_ingest_failed", "candidate_batch_rejected");
18567
18328
  }
18568
- let totalBytes = 0;
18569
- let totalRows = 0;
18570
- const objects = [];
18571
- const rows = [];
18572
- for (const path of paths) {
18573
- if (this.nowMs() - started > request2.budgets.maxDurationMs) {
18574
- throw new CryptoHftDataError("budget_max_duration_exceeded");
18575
- }
18576
- const endpoint = new URL("/download", this.baseUrl);
18577
- endpoint.searchParams.set("file", path);
18578
- const response = await this.request(endpoint, {
18579
- headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
18580
- });
18581
- if (!response.ok)
18582
- throw new CryptoHftDataError("object_download_failed");
18583
- const declaredBytes = Number(response.headers.get("content-length"));
18584
- if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request2.budgets.maxBytes) {
18585
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
18586
- }
18587
- const bytes = await readBoundedObject(response, request2.budgets.maxBytes - totalBytes);
18588
- totalBytes += bytes.byteLength;
18589
- if (totalBytes > request2.budgets.maxBytes) {
18590
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
18591
- }
18592
- const decoded = await this.decode(bytes);
18593
- totalRows += decoded.length;
18594
- if (totalRows > request2.budgets.maxRows) {
18595
- throw new CryptoHftDataError("budget_max_rows_exceeded");
18596
- }
18597
- const object = {
18598
- identity: path,
18599
- checksum: sha256Bytes(bytes),
18600
- bytes: bytes.byteLength,
18601
- rows: decoded.length
18602
- };
18603
- objects.push(object);
18604
- for (const decodedRow of decoded) {
18605
- const parsed = parsedDatasetRow(decodedRow, object);
18606
- validatedRow(request2, parsed);
18607
- rows.push(parsed);
18608
- }
18329
+ } catch {
18330
+ return outcome(request2, "archive_ingest_failed", "candidate_submission_failed");
18331
+ }
18332
+ let verification;
18333
+ try {
18334
+ verification = await dependencies.archive.verifyCandidate(request2, normalized, normalized.captureBundleId, initialResolution);
18335
+ } catch {
18336
+ return outcome(request2, "promotion_verification_failed", "candidate_query_failed");
18337
+ }
18338
+ if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
18339
+ return outcome(request2, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
18340
+ }
18341
+ let receipt;
18342
+ try {
18343
+ receipt = buildPromotionReceipt(request2, capability, normalized, verification, dependencies.clock.nowMs());
18344
+ const [batch] = buildForwarderBatches({
18345
+ captureBundleId: normalized.captureBundleId,
18346
+ deploymentId: "market-data-vendor-backfill",
18347
+ rows: [promotionReceiptToArchiveRow(receipt)]
18348
+ });
18349
+ if (!batch || !await submitAll(dependencies, [batch])) {
18350
+ return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
18609
18351
  }
18610
- if (this.nowMs() - started > request2.budgets.maxDurationMs) {
18611
- throw new CryptoHftDataError("budget_max_duration_exceeded");
18352
+ const qualification = finalizeQualificationEvent({
18353
+ capture_bundle_id: receipt.capture_bundle_id,
18354
+ state: "qualified",
18355
+ receipt_id: receipt.receipt_id,
18356
+ promotion_identity_sha256: receipt.promotion_identity_sha256,
18357
+ window: receipt.window,
18358
+ event_at: receipt.verified_at,
18359
+ reason_code: "promotion_verified"
18360
+ });
18361
+ const [qualificationBatch] = buildForwarderBatches({
18362
+ captureBundleId: normalized.captureBundleId,
18363
+ deploymentId: "market-data-vendor-backfill",
18364
+ rows: [qualificationEventToArchiveRow(qualification)]
18365
+ });
18366
+ if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
18367
+ return outcome(request2, "archive_ingest_failed", "qualification_commit_failed", { receipt });
18612
18368
  }
18613
- return {
18614
- objects,
18615
- rows,
18616
- vendorSemanticDigest: sha256Canonical(rows)
18617
- };
18369
+ } catch {
18370
+ return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
18618
18371
  }
18619
- async normalize(request2, capability, dataset, captureBundleId3) {
18620
- const profile = this.profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
18621
- if (!profile) {
18622
- throw new CryptoHftDataError("profile_semantics_unavailable");
18623
- }
18624
- const samples = reconstructCryptoHftDataOrderBooks(request2, dataset.rows, profile);
18625
- const context2 = {
18626
- source: EXTERNAL_BACKFILL_SOURCE,
18372
+ let finalResolution;
18373
+ try {
18374
+ finalResolution = await dependencies.archive.resolveSelection(request2);
18375
+ assertArchivePreflight(request2, finalResolution);
18376
+ } catch (error) {
18377
+ return outcome(request2, "promotion_verification_failed", "post_promotion_selection_failed", {
18378
+ receipt,
18379
+ reasonSubcode: stableFailureReason(error, "archive_selection_resolution_failed")
18380
+ });
18381
+ }
18382
+ if (finalResolution.selection.coverage_class !== "complete") {
18383
+ return outcome(request2, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
18384
+ }
18385
+ if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
18386
+ return outcome(request2, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
18387
+ }
18388
+ try {
18389
+ if (!request2.wire)
18390
+ throw new Error("decoded request wire is missing");
18391
+ const [selectionBatch] = buildForwarderBatches({
18392
+ captureBundleId: receipt.capture_bundle_id,
18627
18393
  deploymentId: "market-data-vendor-backfill",
18628
- captureBundleId: captureBundleId3,
18629
- exchange: request2.scope.exchange,
18630
- symbol: request2.scope.tradingPair,
18631
- tradingPair: request2.scope.tradingPair,
18632
- sourceSymbol: request2.scope.sourceSymbol,
18633
- assetType: request2.scope.marketType,
18634
- feed: "ORDERBOOK",
18635
- provider: "cryptohftdata",
18636
- sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
18637
- schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
18638
- checksumAlgorithm: CHECKSUM_ALGORITHM,
18639
- provenanceComplete: true
18640
- };
18641
- const rows = samples.flatMap((sample) => {
18642
- const rawCapture = {
18643
- rawCaptureId: sha256Canonical({
18644
- capture_bundle_id: captureBundleId3,
18645
- dataset_object_identity: sample.datasetObjectIdentity,
18646
- dataset_object_checksum: sample.datasetObjectChecksum,
18647
- source_time_ms: sample.sourceTimeMs,
18648
- target_time_ms: sample.targetTimeMs
18649
- }),
18650
- rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
18651
- rawChecksum: sample.datasetObjectChecksum,
18652
- redactedPayload: {
18653
- dataset_object_identity: sample.datasetObjectIdentity,
18654
- dataset_object_checksum: sample.datasetObjectChecksum
18655
- },
18656
- eventTimeMs: sample.sourceTimeMs,
18657
- receivedTimeMs: sample.receivedTimeMs,
18658
- checksumAlgorithm: CHECKSUM_ALGORITHM
18659
- };
18660
- const canonical = buildCanonicalOrderBookRows({
18661
- context: context2,
18662
- rawCapture,
18663
- depthLimit: request2.depth,
18664
- constructionMode: request2.constructionMode,
18665
- snapshot: {
18666
- bids: sample.bids,
18667
- asks: sample.asks,
18668
- timestamp: sample.sourceTimeMs,
18669
- receivedTimestamp: sample.receivedTimeMs,
18670
- exchange: request2.scope.exchange,
18671
- symbol: request2.scope.tradingPair,
18672
- depthLimit: request2.depth,
18673
- sequence: sample.sequence
18674
- }
18675
- });
18676
- return [...canonical.levels, canonical.summary];
18394
+ rows: [
18395
+ archiveSelectionToArchiveRow(request2.wire, finalResolution.selection)
18396
+ ]
18677
18397
  });
18678
- return {
18679
- captureBundleId: captureBundleId3,
18680
- objects: dataset.objects,
18681
- rows,
18682
- vendorSemanticDigest: dataset.vendorSemanticDigest,
18683
- canonicalSemanticDigest: semanticDigest(rows)
18684
- };
18398
+ if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
18399
+ return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
18400
+ }
18401
+ } catch {
18402
+ return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
18685
18403
  }
18404
+ return outcome(request2, "promoted", "promotion_qualified", {
18405
+ receipt,
18406
+ selection: finalResolution.selection
18407
+ });
18686
18408
  }
18687
18409
  // src/helpers/market-data-vendor-backfill/forwarder-client.ts
18688
18410
  class ArchiveForwarderSubmissionError extends Error {
@@ -18847,4 +18569,4 @@ export {
18847
18569
  ACQUISITION_POLICY_ID
18848
18570
  };
18849
18571
 
18850
- //# debugId=75C87110D96E1B9B64756E2164756E21
18572
+ //# debugId=E83C0385A80F908D64756E2164756E21