@usherlabs/cex-broker 0.2.48 → 0.2.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/dist/commands/cex-canonical-orderbook-export.js +72 -358
  2. package/dist/commands/cex-canonical-orderbook-export.js.map +7 -19
  3. package/dist/commands/cli.js +224 -67
  4. package/dist/commands/market-data-vendor-backfill.js +975 -1198
  5. package/dist/commands/market-data-vendor-backfill.js.map +13 -25
  6. package/dist/helpers/logger.d.ts +4 -0
  7. package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +3 -1
  8. package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +58 -4
  9. package/dist/helpers/market-data-vendor-backfill/semantic-verification.d.ts +1 -0
  10. package/dist/helpers/otel.d.ts +1 -0
  11. package/dist/helpers/trace-context.d.ts +8 -0
  12. package/dist/index.js +183 -63
  13. package/dist/index.js.map +9 -8
  14. package/dist/market-data-preparation/fixtures/conformance-v2.json +8 -8
  15. package/dist/market-data-preparation/policies/capability-policy-v1.json +28 -0
  16. package/dist/market-data-preparation/policies/capability-policy.json +14 -3
  17. package/dist/market-data-preparation/policies/resource-policy-v1.json +16 -0
  18. package/dist/market-data-preparation/policies/resource-policy.json +3 -3
  19. package/dist/market-data-preparation.js +70 -359
  20. package/dist/market-data-preparation.js.map +9 -21
  21. package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +39 -0
  22. package/dist/market-data-vendor-backfill/policies/resource-policy-v2.json +16 -0
  23. package/dist/market-data-vendor-backfill.d.ts +2 -2
  24. package/dist/market-data-vendor-backfill.js +1528 -1750
  25. package/dist/market-data-vendor-backfill.js.map +13 -25
  26. package/package.json +1 -1
@@ -1431,241 +1431,6 @@ var require_src = __commonJS((exports) => {
1431
1431
  };
1432
1432
  });
1433
1433
 
1434
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/types/LogRecord.js
1435
- var require_LogRecord = __commonJS((exports) => {
1436
- Object.defineProperty(exports, "__esModule", { value: true });
1437
- exports.SeverityNumber = undefined;
1438
- var SeverityNumber;
1439
- (function(SeverityNumber2) {
1440
- SeverityNumber2[SeverityNumber2["UNSPECIFIED"] = 0] = "UNSPECIFIED";
1441
- SeverityNumber2[SeverityNumber2["TRACE"] = 1] = "TRACE";
1442
- SeverityNumber2[SeverityNumber2["TRACE2"] = 2] = "TRACE2";
1443
- SeverityNumber2[SeverityNumber2["TRACE3"] = 3] = "TRACE3";
1444
- SeverityNumber2[SeverityNumber2["TRACE4"] = 4] = "TRACE4";
1445
- SeverityNumber2[SeverityNumber2["DEBUG"] = 5] = "DEBUG";
1446
- SeverityNumber2[SeverityNumber2["DEBUG2"] = 6] = "DEBUG2";
1447
- SeverityNumber2[SeverityNumber2["DEBUG3"] = 7] = "DEBUG3";
1448
- SeverityNumber2[SeverityNumber2["DEBUG4"] = 8] = "DEBUG4";
1449
- SeverityNumber2[SeverityNumber2["INFO"] = 9] = "INFO";
1450
- SeverityNumber2[SeverityNumber2["INFO2"] = 10] = "INFO2";
1451
- SeverityNumber2[SeverityNumber2["INFO3"] = 11] = "INFO3";
1452
- SeverityNumber2[SeverityNumber2["INFO4"] = 12] = "INFO4";
1453
- SeverityNumber2[SeverityNumber2["WARN"] = 13] = "WARN";
1454
- SeverityNumber2[SeverityNumber2["WARN2"] = 14] = "WARN2";
1455
- SeverityNumber2[SeverityNumber2["WARN3"] = 15] = "WARN3";
1456
- SeverityNumber2[SeverityNumber2["WARN4"] = 16] = "WARN4";
1457
- SeverityNumber2[SeverityNumber2["ERROR"] = 17] = "ERROR";
1458
- SeverityNumber2[SeverityNumber2["ERROR2"] = 18] = "ERROR2";
1459
- SeverityNumber2[SeverityNumber2["ERROR3"] = 19] = "ERROR3";
1460
- SeverityNumber2[SeverityNumber2["ERROR4"] = 20] = "ERROR4";
1461
- SeverityNumber2[SeverityNumber2["FATAL"] = 21] = "FATAL";
1462
- SeverityNumber2[SeverityNumber2["FATAL2"] = 22] = "FATAL2";
1463
- SeverityNumber2[SeverityNumber2["FATAL3"] = 23] = "FATAL3";
1464
- SeverityNumber2[SeverityNumber2["FATAL4"] = 24] = "FATAL4";
1465
- })(SeverityNumber = exports.SeverityNumber || (exports.SeverityNumber = {}));
1466
- });
1467
-
1468
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLogger.js
1469
- var require_NoopLogger = __commonJS((exports) => {
1470
- Object.defineProperty(exports, "__esModule", { value: true });
1471
- exports.NOOP_LOGGER = exports.NoopLogger = undefined;
1472
-
1473
- class NoopLogger {
1474
- emit(_logRecord) {}
1475
- }
1476
- exports.NoopLogger = NoopLogger;
1477
- exports.NOOP_LOGGER = new NoopLogger;
1478
- });
1479
-
1480
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLoggerProvider.js
1481
- var require_NoopLoggerProvider = __commonJS((exports) => {
1482
- Object.defineProperty(exports, "__esModule", { value: true });
1483
- exports.NOOP_LOGGER_PROVIDER = exports.NoopLoggerProvider = undefined;
1484
- var NoopLogger_1 = require_NoopLogger();
1485
-
1486
- class NoopLoggerProvider {
1487
- getLogger(_name, _version, _options) {
1488
- return new NoopLogger_1.NoopLogger;
1489
- }
1490
- }
1491
- exports.NoopLoggerProvider = NoopLoggerProvider;
1492
- exports.NOOP_LOGGER_PROVIDER = new NoopLoggerProvider;
1493
- });
1494
-
1495
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLogger.js
1496
- var require_ProxyLogger = __commonJS((exports) => {
1497
- Object.defineProperty(exports, "__esModule", { value: true });
1498
- exports.ProxyLogger = undefined;
1499
- var NoopLogger_1 = require_NoopLogger();
1500
-
1501
- class ProxyLogger {
1502
- constructor(_provider, name, version, options) {
1503
- this._provider = _provider;
1504
- this.name = name;
1505
- this.version = version;
1506
- this.options = options;
1507
- }
1508
- emit(logRecord) {
1509
- this._getLogger().emit(logRecord);
1510
- }
1511
- _getLogger() {
1512
- if (this._delegate) {
1513
- return this._delegate;
1514
- }
1515
- const logger = this._provider._getDelegateLogger(this.name, this.version, this.options);
1516
- if (!logger) {
1517
- return NoopLogger_1.NOOP_LOGGER;
1518
- }
1519
- this._delegate = logger;
1520
- return this._delegate;
1521
- }
1522
- }
1523
- exports.ProxyLogger = ProxyLogger;
1524
- });
1525
-
1526
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLoggerProvider.js
1527
- var require_ProxyLoggerProvider = __commonJS((exports) => {
1528
- Object.defineProperty(exports, "__esModule", { value: true });
1529
- exports.ProxyLoggerProvider = undefined;
1530
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
1531
- var ProxyLogger_1 = require_ProxyLogger();
1532
-
1533
- class ProxyLoggerProvider {
1534
- getLogger(name, version, options) {
1535
- var _a;
1536
- return (_a = this._getDelegateLogger(name, version, options)) !== null && _a !== undefined ? _a : new ProxyLogger_1.ProxyLogger(this, name, version, options);
1537
- }
1538
- _getDelegate() {
1539
- var _a;
1540
- return (_a = this._delegate) !== null && _a !== undefined ? _a : NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
1541
- }
1542
- _setDelegate(delegate) {
1543
- this._delegate = delegate;
1544
- }
1545
- _getDelegateLogger(name, version, options) {
1546
- var _a;
1547
- return (_a = this._delegate) === null || _a === undefined ? undefined : _a.getLogger(name, version, options);
1548
- }
1549
- }
1550
- exports.ProxyLoggerProvider = ProxyLoggerProvider;
1551
- });
1552
-
1553
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/globalThis.js
1554
- var require_globalThis2 = __commonJS((exports) => {
1555
- Object.defineProperty(exports, "__esModule", { value: true });
1556
- exports._globalThis = undefined;
1557
- exports._globalThis = typeof globalThis === "object" ? globalThis : global;
1558
- });
1559
-
1560
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/index.js
1561
- var require_node2 = __commonJS((exports) => {
1562
- Object.defineProperty(exports, "__esModule", { value: true });
1563
- exports._globalThis = undefined;
1564
- var globalThis_1 = require_globalThis2();
1565
- Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
1566
- return globalThis_1._globalThis;
1567
- } });
1568
- });
1569
-
1570
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/index.js
1571
- var require_platform2 = __commonJS((exports) => {
1572
- Object.defineProperty(exports, "__esModule", { value: true });
1573
- exports._globalThis = undefined;
1574
- var node_1 = require_node2();
1575
- Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
1576
- return node_1._globalThis;
1577
- } });
1578
- });
1579
-
1580
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/internal/global-utils.js
1581
- var require_global_utils2 = __commonJS((exports) => {
1582
- Object.defineProperty(exports, "__esModule", { value: true });
1583
- exports.API_BACKWARDS_COMPATIBILITY_VERSION = exports.makeGetter = exports._global = exports.GLOBAL_LOGS_API_KEY = undefined;
1584
- var platform_1 = require_platform2();
1585
- exports.GLOBAL_LOGS_API_KEY = Symbol.for("io.opentelemetry.js.api.logs");
1586
- exports._global = platform_1._globalThis;
1587
- function makeGetter(requiredVersion, instance, fallback) {
1588
- return (version) => version === requiredVersion ? instance : fallback;
1589
- }
1590
- exports.makeGetter = makeGetter;
1591
- exports.API_BACKWARDS_COMPATIBILITY_VERSION = 1;
1592
- });
1593
-
1594
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/api/logs.js
1595
- var require_logs = __commonJS((exports) => {
1596
- Object.defineProperty(exports, "__esModule", { value: true });
1597
- exports.LogsAPI = undefined;
1598
- var global_utils_1 = require_global_utils2();
1599
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
1600
- var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
1601
-
1602
- class LogsAPI {
1603
- constructor() {
1604
- this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
1605
- }
1606
- static getInstance() {
1607
- if (!this._instance) {
1608
- this._instance = new LogsAPI;
1609
- }
1610
- return this._instance;
1611
- }
1612
- setGlobalLoggerProvider(provider) {
1613
- if (global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) {
1614
- return this.getLoggerProvider();
1615
- }
1616
- global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY] = (0, global_utils_1.makeGetter)(global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION, provider, NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER);
1617
- this._proxyLoggerProvider._setDelegate(provider);
1618
- return provider;
1619
- }
1620
- getLoggerProvider() {
1621
- var _a, _b;
1622
- return (_b = (_a = global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) === null || _a === undefined ? undefined : _a.call(global_utils_1._global, global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION)) !== null && _b !== undefined ? _b : this._proxyLoggerProvider;
1623
- }
1624
- getLogger(name, version, options) {
1625
- return this.getLoggerProvider().getLogger(name, version, options);
1626
- }
1627
- disable() {
1628
- delete global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY];
1629
- this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
1630
- }
1631
- }
1632
- exports.LogsAPI = LogsAPI;
1633
- });
1634
-
1635
- // node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/index.js
1636
- var require_src2 = __commonJS((exports) => {
1637
- Object.defineProperty(exports, "__esModule", { value: true });
1638
- exports.logs = exports.ProxyLoggerProvider = exports.ProxyLogger = exports.NoopLoggerProvider = exports.NOOP_LOGGER_PROVIDER = exports.NoopLogger = exports.NOOP_LOGGER = exports.SeverityNumber = undefined;
1639
- var LogRecord_1 = require_LogRecord();
1640
- Object.defineProperty(exports, "SeverityNumber", { enumerable: true, get: function() {
1641
- return LogRecord_1.SeverityNumber;
1642
- } });
1643
- var NoopLogger_1 = require_NoopLogger();
1644
- Object.defineProperty(exports, "NOOP_LOGGER", { enumerable: true, get: function() {
1645
- return NoopLogger_1.NOOP_LOGGER;
1646
- } });
1647
- Object.defineProperty(exports, "NoopLogger", { enumerable: true, get: function() {
1648
- return NoopLogger_1.NoopLogger;
1649
- } });
1650
- var NoopLoggerProvider_1 = require_NoopLoggerProvider();
1651
- Object.defineProperty(exports, "NOOP_LOGGER_PROVIDER", { enumerable: true, get: function() {
1652
- return NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
1653
- } });
1654
- Object.defineProperty(exports, "NoopLoggerProvider", { enumerable: true, get: function() {
1655
- return NoopLoggerProvider_1.NoopLoggerProvider;
1656
- } });
1657
- var ProxyLogger_1 = require_ProxyLogger();
1658
- Object.defineProperty(exports, "ProxyLogger", { enumerable: true, get: function() {
1659
- return ProxyLogger_1.ProxyLogger;
1660
- } });
1661
- var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
1662
- Object.defineProperty(exports, "ProxyLoggerProvider", { enumerable: true, get: function() {
1663
- return ProxyLoggerProvider_1.ProxyLoggerProvider;
1664
- } });
1665
- var logs_1 = require_logs();
1666
- exports.logs = logs_1.LogsAPI.getInstance();
1667
- });
1668
-
1669
1434
  // node_modules/ajv/dist/compile/codegen/code.js
1670
1435
  var require_code = __commonJS((exports) => {
1671
1436
  Object.defineProperty(exports, "__esModule", { value: true });
@@ -8697,102 +8462,6 @@ var BaseTransport = class {
8697
8462
  return this.logger;
8698
8463
  }
8699
8464
  };
8700
- var LoggerlessTransport = class {
8701
- id;
8702
- enabled;
8703
- level;
8704
- consoleDebug;
8705
- constructor(config) {
8706
- this.id = Date.now().toString() + Math.random().toString();
8707
- this.enabled = config.enabled ?? true;
8708
- this.consoleDebug = config.consoleDebug ?? false;
8709
- this.level = config.level ?? "trace";
8710
- }
8711
- _sendToLogger(params) {
8712
- if (!this.enabled)
8713
- return;
8714
- if (LogLevelPriority[params.logLevel] < LogLevelPriority[this.level])
8715
- return;
8716
- const messages = this.shipToLogger(params);
8717
- if (this.consoleDebug)
8718
- switch (params.logLevel) {
8719
- case LogLevel.info:
8720
- console.info(...messages);
8721
- break;
8722
- case LogLevel.warn:
8723
- console.warn(...messages);
8724
- break;
8725
- case LogLevel.error:
8726
- console.error(...messages);
8727
- break;
8728
- case LogLevel.trace:
8729
- console.debug(...messages);
8730
- break;
8731
- case LogLevel.debug:
8732
- console.debug(...messages);
8733
- break;
8734
- case LogLevel.fatal:
8735
- console.debug(...messages);
8736
- break;
8737
- default:
8738
- console.log(...messages);
8739
- }
8740
- }
8741
- getLoggerInstance() {
8742
- throw new Error("This transport does not have a logger instance");
8743
- }
8744
- };
8745
-
8746
- // node_modules/@loglayer/transport-opentelemetry/dist/index.mjs
8747
- var import_api_logs = __toESM(require_src2(), 1);
8748
- var logLayerLevels = {
8749
- fatal: import_api_logs.SeverityNumber.FATAL,
8750
- error: import_api_logs.SeverityNumber.ERROR,
8751
- warn: import_api_logs.SeverityNumber.WARN,
8752
- info: import_api_logs.SeverityNumber.INFO,
8753
- debug: import_api_logs.SeverityNumber.DEBUG,
8754
- trace: import_api_logs.SeverityNumber.TRACE
8755
- };
8756
- function getSeverityNumber(level) {
8757
- return logLayerLevels[level] ?? import_api_logs.SeverityNumber.TRACE;
8758
- }
8759
- function emitLogRecord(record, logger) {
8760
- const { message, level, ...splat } = record;
8761
- const attributes = {};
8762
- for (const key in splat)
8763
- if (Object.hasOwn(splat, key))
8764
- attributes[key] = splat[key];
8765
- const logRecord = {
8766
- severityNumber: getSeverityNumber(level),
8767
- severityText: level,
8768
- body: message,
8769
- attributes
8770
- };
8771
- logger.emit(logRecord);
8772
- }
8773
- var OpenTelemetryTransport = class extends LoggerlessTransport {
8774
- _logger;
8775
- onError;
8776
- constructor(config = {}) {
8777
- super(config);
8778
- this._logger = import_api_logs.logs.getLogger("loglayer");
8779
- this.onError = config.onError;
8780
- }
8781
- shipToLogger({ logLevel, messages, data, hasData }) {
8782
- const assembled = {
8783
- level: logLevel,
8784
- message: messages.join(" "),
8785
- ...hasData ? data : {}
8786
- };
8787
- try {
8788
- emitLogRecord(assembled, this._logger);
8789
- } catch (error) {
8790
- if (this.onError)
8791
- this.onError(error);
8792
- }
8793
- return [assembled];
8794
- }
8795
- };
8796
8465
 
8797
8466
  // node_modules/@loglayer/transport-tslog/dist/index.js
8798
8467
  var TsLogTransport = class extends BaseTransport {
@@ -10964,13 +10633,7 @@ var tslogLogger = new Logger({
10964
10633
  minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
10965
10634
  });
10966
10635
  var baseLogger = new LogLayer({
10967
- transport: [
10968
- new TsLogTransport({ id: "tslog", logger: tslogLogger }),
10969
- new OpenTelemetryTransport({
10970
- id: "otel",
10971
- enabled: true
10972
- })
10973
- ],
10636
+ transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
10974
10637
  plugins: [openTelemetryPlugin()],
10975
10638
  errorSerializer: serializeError
10976
10639
  });
@@ -12089,12 +11752,14 @@ var result_schema_default = {
12089
11752
  };
12090
11753
 
12091
11754
  // src/helpers/market-data-vendor-backfill/manifests.ts
12092
- var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
12093
- var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
11755
+ var LEGACY_CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
11756
+ var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v2";
11757
+ var LEGACY_RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
11758
+ var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v2";
12094
11759
  var ADAPTER_POLICY_ID = "cryptohftdata-orderbook-adapter/v1";
12095
11760
  var ACQUISITION_POLICY_ID = "cryptohftdata-hourly-acquisition/v1";
12096
- var capabilityPolicyContent = {
12097
- policy_id: CAPABILITY_POLICY_ID,
11761
+ var legacyCapabilityPolicyContent = {
11762
+ policy_id: LEGACY_CAPABILITY_POLICY_ID,
12098
11763
  provider: "cryptohftdata",
12099
11764
  adapter_policy: {
12100
11765
  policy_id: ADAPTER_POLICY_ID,
@@ -12120,12 +11785,37 @@ var capabilityPolicyContent = {
12120
11785
  }
12121
11786
  ]
12122
11787
  };
11788
+ var LEGACY_CAPABILITY_POLICY = Object.freeze({
11789
+ ...legacyCapabilityPolicyContent,
11790
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent)
11791
+ });
11792
+ var capabilityPolicyContent = {
11793
+ ...legacyCapabilityPolicyContent,
11794
+ policy_id: CAPABILITY_POLICY_ID,
11795
+ profiles: [
11796
+ ...legacyCapabilityPolicyContent.profiles.map((profile) => ({
11797
+ ...profile,
11798
+ source_policies: ["authoritative_window", "fill_gaps"]
11799
+ })),
11800
+ {
11801
+ exchange: "okx",
11802
+ market_type: "spot",
11803
+ feed: "ORDERBOOK",
11804
+ canonical_trading_pair: "ARB-USDC",
11805
+ provider_exchange_id: "okx_spot",
11806
+ resolved_symbol: "ARB-USDC",
11807
+ construction_modes: ["sampled_top_n_snapshot"],
11808
+ source_policies: ["authoritative_window", "fill_gaps"],
11809
+ max_depth: 400
11810
+ }
11811
+ ]
11812
+ };
12123
11813
  var CAPABILITY_POLICY = Object.freeze({
12124
11814
  ...capabilityPolicyContent,
12125
11815
  policy_sha256: jcsSha256(capabilityPolicyContent)
12126
11816
  });
12127
- var resourcePolicyContent = {
12128
- policy_id: RESOURCE_POLICY_ID,
11817
+ var legacyResourcePolicyContent = {
11818
+ policy_id: LEGACY_RESOURCE_POLICY_ID,
12129
11819
  limits: {
12130
11820
  max_files: 1e4,
12131
11821
  max_bytes: 100 * 1024 * 1024 * 1024,
@@ -12139,17 +11829,29 @@ var resourcePolicyContent = {
12139
11829
  max_required_events: 1e5
12140
11830
  }
12141
11831
  };
11832
+ var LEGACY_RESOURCE_POLICY = Object.freeze({
11833
+ ...legacyResourcePolicyContent,
11834
+ policy_sha256: jcsSha256(legacyResourcePolicyContent)
11835
+ });
11836
+ var resourcePolicyContent = {
11837
+ ...legacyResourcePolicyContent,
11838
+ policy_id: RESOURCE_POLICY_ID,
11839
+ request_bounds: {
11840
+ ...legacyResourcePolicyContent.request_bounds,
11841
+ max_window_ms: 31 * 24 * 60 * 60 * 1000
11842
+ }
11843
+ };
12142
11844
  var RESOURCE_POLICY = Object.freeze({
12143
11845
  ...resourcePolicyContent,
12144
11846
  policy_sha256: jcsSha256(resourcePolicyContent)
12145
11847
  });
12146
11848
  var EFFECTIVE_ADAPTER_POLICY_PIN = Object.freeze({
12147
11849
  policy_id: ADAPTER_POLICY_ID,
12148
- policy_sha256: jcsSha256(capabilityPolicyContent.adapter_policy)
11850
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent.adapter_policy)
12149
11851
  });
12150
11852
  var EFFECTIVE_ACQUISITION_POLICY_PIN = Object.freeze({
12151
11853
  policy_id: ACQUISITION_POLICY_ID,
12152
- policy_sha256: jcsSha256(capabilityPolicyContent.acquisition_policy)
11854
+ policy_sha256: jcsSha256(legacyCapabilityPolicyContent.acquisition_policy)
12153
11855
  });
12154
11856
  var schemaArtifacts = [
12155
11857
  ["schemas/request.schema.json", request_schema_default],
@@ -12329,7 +12031,15 @@ var backfillRequestCodec = {
12329
12031
  if (request.idempotency_key !== createBackfillIdempotencyKey(request)) {
12330
12032
  throw new Error("idempotency_key does not match canonical business fields");
12331
12033
  }
12332
- if (request.product_pins.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request.product_pins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256 || request.product_pins.resource_policy.policy_id !== RESOURCE_POLICY.policy_id || request.product_pins.resource_policy.policy_sha256 !== RESOURCE_POLICY.policy_sha256) {
12034
+ const capabilityPolicyMatches = [
12035
+ CAPABILITY_POLICY,
12036
+ LEGACY_CAPABILITY_POLICY
12037
+ ].some((policy) => request.product_pins.capability_policy.policy_id === policy.policy_id && request.product_pins.capability_policy.policy_sha256 === policy.policy_sha256);
12038
+ const resourcePolicyMatches = [
12039
+ RESOURCE_POLICY,
12040
+ LEGACY_RESOURCE_POLICY
12041
+ ].some((policy) => request.product_pins.resource_policy.policy_id === policy.policy_id && request.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
12042
+ if (!capabilityPolicyMatches || !resourcePolicyMatches) {
12333
12043
  throw new Error("request policy pins do not match the effective package policies");
12334
12044
  }
12335
12045
  if (request.required_clock.clock_id !== request.initial_selection.required_clock.clock_id || request.required_clock.clock_sha256 !== request.initial_selection.required_clock.clock_sha256 || request.required_clock.event_count !== request.initial_selection.required_clock.event_count) {
@@ -12457,6 +12167,10 @@ function decodeBackfillRunDocuments(input) {
12457
12167
  return targetTimeMs;
12458
12168
  });
12459
12169
  const capabilityProfile = CAPABILITY_POLICY.profiles.find((profile) => profile.exchange === wire.scope.exchange && profile.market_type === wire.scope.market_type && profile.feed === wire.scope.feed && profile.canonical_trading_pair === wire.scope.trading_pair);
12170
+ const resourcePolicy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((policy) => wire.product_pins.resource_policy.policy_id === policy.policy_id && wire.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
12171
+ if (!resourcePolicy) {
12172
+ throw new Error("request resource policy pin is unsupported");
12173
+ }
12460
12174
  return {
12461
12175
  schemaVersion: BACKFILL_REQUEST_SCHEMA_VERSION,
12462
12176
  requestId: wire.request_id,
@@ -12481,11 +12195,11 @@ function decodeBackfillRunDocuments(input) {
12481
12195
  maxPriorAsOfLagMs: wire.coverage_policy.max_asof_lag_ms,
12482
12196
  sourcePolicy: wire.source_policy,
12483
12197
  budgets: {
12484
- maxFiles: RESOURCE_POLICY.limits.max_files,
12485
- maxBytes: RESOURCE_POLICY.limits.max_bytes,
12486
- maxRows: RESOURCE_POLICY.limits.max_rows,
12487
- maxDurationMs: RESOURCE_POLICY.limits.max_duration_ms,
12488
- maxBoundaryLookbackMs: Math.min(RESOURCE_POLICY.limits.max_boundary_lookback_ms, CAPABILITY_POLICY.acquisition_policy.initialization_lookback_ms)
12198
+ maxFiles: resourcePolicy.limits.max_files,
12199
+ maxBytes: resourcePolicy.limits.max_bytes,
12200
+ maxRows: resourcePolicy.limits.max_rows,
12201
+ maxDurationMs: resourcePolicy.limits.max_duration_ms,
12202
+ maxBoundaryLookbackMs: Math.min(resourcePolicy.limits.max_boundary_lookback_ms, CAPABILITY_POLICY.acquisition_policy.initialization_lookback_ms)
12489
12203
  },
12490
12204
  expectedProduct: {
12491
12205
  packageName: "@usherlabs/cex-broker",
@@ -12763,12 +12477,13 @@ function assertStoredSelectionMatchesRequest(request, selection) {
12763
12477
  }
12764
12478
  }
12765
12479
  function eligibleBundles(request, bundles) {
12766
- return bundles.filter((bundle) => {
12767
- if (bundle.captureOrigin === "production_capture") {
12768
- return request.sourcePolicy === "fill_gaps";
12769
- }
12770
- return bundle.qualification?.state === "qualified";
12771
- });
12480
+ const qualifiedVendorBundles = bundles.filter((bundle) => bundle.captureOrigin === "vendor_historical_backfill" && bundle.qualification?.state === "qualified");
12481
+ const productionBundles = bundles.filter((bundle) => bundle.captureOrigin === "production_capture");
12482
+ if (request.sourcePolicy === "authoritative_window") {
12483
+ const vendorSupport = chooseSupport(request, qualifiedVendorBundles);
12484
+ return vendorSupport.length === request.requiredClockTargetsMs.length ? qualifiedVendorBundles : productionBundles;
12485
+ }
12486
+ return [...productionBundles, ...qualifiedVendorBundles];
12772
12487
  }
12773
12488
  function originPriority(request, origin) {
12774
12489
  if (request.sourcePolicy === "fill_gaps") {
@@ -12973,9 +12688,8 @@ function exactCandidateMatch(expected, actual) {
12973
12688
  function requestedShapeMatches(request, rows) {
12974
12689
  return rows.every(({ row }) => row.depth_limit === request.depth && row.construction_mode === request.constructionMode && row.schema_version === request.expectedProduct.canonicalSchemaVersion);
12975
12690
  }
12976
- function requiredClockCoverage(request, rows) {
12977
- const summaryTimes = rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => {
12978
- const value = row.source_time_ms;
12691
+ function requiredClockCoverage(request, rows, coverageSourceTimesMs) {
12692
+ const summaryTimes = (coverageSourceTimesMs ?? rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => row.source_time_ms)).map((value) => {
12979
12693
  if (typeof value !== "number" && (typeof value !== "string" || !/^\d+$/.test(value))) {
12980
12694
  return;
12981
12695
  }
@@ -13048,7 +12762,7 @@ function verifySemanticPromotion(evidence) {
13048
12762
  coverageVerified: false
13049
12763
  };
13050
12764
  }
13051
- const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows);
12765
+ const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows, evidence.coverageSourceTimesMs);
13052
12766
  if (!coverageVerified) {
13053
12767
  return {
13054
12768
  ...base,
@@ -13423,7 +13137,13 @@ class QualifiedOrderBookArchiveReader {
13423
13137
  suffixDigestBefore: baseline.verificationBaseline.suffixDigest,
13424
13138
  suffixDigestAfter: timelineDigest(suffix),
13425
13139
  seamVerified: this.seamIsOrdered(queriedRows),
13426
- exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string")
13140
+ exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string"),
13141
+ ...request.sourcePolicy === "fill_gaps" ? {
13142
+ coverageSourceTimesMs: [
13143
+ ...baseline.selection.support_anchors.map((anchor) => Date.parse(anchor.source_time)),
13144
+ ...summaries.map((row) => numberField(row, "source_time_ms"))
13145
+ ]
13146
+ } : {}
13427
13147
  });
13428
13148
  return { ...semantic, captureBundleId };
13429
13149
  }
@@ -13612,12 +13332,12 @@ var requestContent = {
13612
13332
  },
13613
13333
  product_pins: {
13614
13334
  capability_policy: {
13615
- policy_id: CAPABILITY_POLICY.policy_id,
13616
- policy_sha256: CAPABILITY_POLICY.policy_sha256
13335
+ policy_id: LEGACY_CAPABILITY_POLICY.policy_id,
13336
+ policy_sha256: LEGACY_CAPABILITY_POLICY.policy_sha256
13617
13337
  },
13618
13338
  resource_policy: {
13619
- policy_id: RESOURCE_POLICY.policy_id,
13620
- policy_sha256: RESOURCE_POLICY.policy_sha256
13339
+ policy_id: LEGACY_RESOURCE_POLICY.policy_id,
13340
+ policy_sha256: LEGACY_RESOURCE_POLICY.policy_sha256
13621
13341
  }
13622
13342
  },
13623
13343
  production_authorization_id: authorizationId
@@ -13766,878 +13486,470 @@ var CONFORMANCE_FIXTURES = Object.freeze({
13766
13486
  promotion_identity_sha256: promotionReceipt.promotion_identity_sha256,
13767
13487
  receipt_id: promotionReceipt.receipt_id,
13768
13488
  result_sha256: result.result_sha256,
13769
- capability_policy_sha256: CAPABILITY_POLICY.policy_sha256,
13770
- resource_policy_sha256: RESOURCE_POLICY.policy_sha256
13489
+ capability_policy_sha256: LEGACY_CAPABILITY_POLICY.policy_sha256,
13490
+ resource_policy_sha256: LEGACY_RESOURCE_POLICY.policy_sha256
13771
13491
  },
13772
13492
  jcs_edge_vector: jcsEdgeVector,
13773
13493
  hashes: { jcs_edge_vector_sha256: jcsSha256(jcsEdgeVector) }
13774
13494
  });
13775
- // src/helpers/market-data-vendor-backfill/qualification.ts
13776
- var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
13777
- var SHA256 = /^[0-9a-f]{64}$/;
13778
- var REASON = /^[a-z][a-z0-9_]{0,127}$/;
13779
- var EVENT_FIELDS = new Set([
13780
- "qualification_event_id",
13781
- "capture_bundle_id",
13782
- "state",
13783
- "receipt_id",
13784
- "promotion_identity_sha256",
13785
- "window",
13786
- "event_at",
13787
- "reason_code"
13788
- ]);
13789
- function deterministicUuid(value) {
13790
- const digest = jcsSha256(value).slice(0, 32).split("");
13791
- digest[12] = "5";
13792
- digest[16] = "8";
13793
- const hex = digest.join("");
13794
- return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
13795
- }
13796
- function semanticEvent(event) {
13797
- const { qualification_event_id: _eventId, ...semantic } = event;
13798
- return semantic;
13799
- }
13800
- function assertTimestamp(value, field) {
13801
- const parsed = Date.parse(value);
13802
- if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
13803
- throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
13804
- }
13805
- return parsed;
13806
- }
13807
- function parseQualificationEvent(value) {
13808
- if (!value || typeof value !== "object" || Array.isArray(value)) {
13809
- throw new Error("qualification event must be an object");
13495
+ // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
13496
+ import { createHash as createHash3 } from "node:crypto";
13497
+
13498
+ // node_modules/fzstd/esm/index.mjs
13499
+ var ab = ArrayBuffer;
13500
+ var u8 = Uint8Array;
13501
+ var u16 = Uint16Array;
13502
+ var i16 = Int16Array;
13503
+ var i32 = Int32Array;
13504
+ var slc = function(v, s, e) {
13505
+ if (u8.prototype.slice)
13506
+ return u8.prototype.slice.call(v, s, e);
13507
+ if (s == null || s < 0)
13508
+ s = 0;
13509
+ if (e == null || e > v.length)
13510
+ e = v.length;
13511
+ var n = new u8(e - s);
13512
+ n.set(v.subarray(s, e));
13513
+ return n;
13514
+ };
13515
+ var fill = function(v, n, s, e) {
13516
+ if (u8.prototype.fill)
13517
+ return u8.prototype.fill.call(v, n, s, e);
13518
+ if (s == null || s < 0)
13519
+ s = 0;
13520
+ if (e == null || e > v.length)
13521
+ e = v.length;
13522
+ for (;s < e; ++s)
13523
+ v[s] = n;
13524
+ return v;
13525
+ };
13526
+ var cpw = function(v, t, s, e) {
13527
+ if (u8.prototype.copyWithin)
13528
+ return u8.prototype.copyWithin.call(v, t, s, e);
13529
+ if (s == null || s < 0)
13530
+ s = 0;
13531
+ if (e == null || e > v.length)
13532
+ e = v.length;
13533
+ while (s < e) {
13534
+ v[t++] = v[s++];
13810
13535
  }
13811
- const event = value;
13812
- if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
13813
- throw new Error("qualification event fields are invalid");
13814
- }
13815
- if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
13816
- throw new Error("qualification window must be increasing");
13817
- }
13818
- assertTimestamp(event.event_at, "event_at");
13819
- if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
13820
- throw new Error("qualification_event_id does not match event content");
13536
+ };
13537
+ var ec = [
13538
+ "invalid zstd data",
13539
+ "window size too large (>2046MB)",
13540
+ "invalid block type",
13541
+ "FSE accuracy too high",
13542
+ "match distance too far back",
13543
+ "unexpected EOF"
13544
+ ];
13545
+ var err = function(ind, msg, nt) {
13546
+ var e = new Error(msg || ec[ind]);
13547
+ e.code = ind;
13548
+ if (Error.captureStackTrace)
13549
+ Error.captureStackTrace(e, err);
13550
+ if (!nt)
13551
+ throw e;
13552
+ return e;
13553
+ };
13554
+ var rb = function(d, b, n) {
13555
+ var i = 0, o = 0;
13556
+ for (;i < n; ++i)
13557
+ o |= d[b++] << (i << 3);
13558
+ return o;
13559
+ };
13560
+ var b4 = function(d, b) {
13561
+ return (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0;
13562
+ };
13563
+ var rzfh = function(dat, w) {
13564
+ var n3 = dat[0] | dat[1] << 8 | dat[2] << 16;
13565
+ if (n3 == 3126568 && dat[3] == 253) {
13566
+ var flg = dat[4];
13567
+ var ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6;
13568
+ if (flg & 8)
13569
+ err(0);
13570
+ var bt = 6 - ss;
13571
+ var db = df == 3 ? 4 : df;
13572
+ var di = rb(dat, bt, db);
13573
+ bt += db;
13574
+ var fsb = fcf ? 1 << fcf : ss;
13575
+ var fss = rb(dat, bt, fsb) + (fcf == 1 && 256);
13576
+ var ws = fss;
13577
+ if (!ss) {
13578
+ var wb = 1 << 10 + (dat[5] >> 3);
13579
+ ws = wb + (wb >> 3) * (dat[5] & 7);
13580
+ }
13581
+ if (ws > 2145386496)
13582
+ err(1);
13583
+ var buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
13584
+ buf[0] = 1, buf[4] = 4, buf[8] = 8;
13585
+ return {
13586
+ b: bt + fsb,
13587
+ y: 0,
13588
+ l: 0,
13589
+ d: di,
13590
+ w: w && w != 1 ? w : buf.subarray(12),
13591
+ e: ws,
13592
+ o: new i32(buf.buffer, 0, 3),
13593
+ u: fss,
13594
+ c: cc,
13595
+ m: Math.min(131072, ws)
13596
+ };
13597
+ } else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
13598
+ return b4(dat, 4) + 8;
13821
13599
  }
13822
- return event;
13823
- }
13824
- function finalizeQualificationEvent(event) {
13825
- const semantic = semanticEvent(event);
13826
- return parseQualificationEvent({
13827
- ...semantic,
13828
- qualification_event_id: deterministicUuid(semantic)
13829
- });
13830
- }
13831
- function qualificationEventToArchiveRow(eventInput) {
13832
- const event = parseQualificationEvent(eventInput);
13833
- return {
13834
- table: "market_data.cex_order_book_capture_qualifications",
13835
- row: {
13836
- source: "external_backfill",
13837
- capture_origin: "vendor_historical_backfill",
13838
- source_mode: "vendor_historical_backfill_v1",
13839
- deployment_id: "market-data-vendor-backfill",
13840
- qualification_event_id: event.qualification_event_id,
13841
- capture_bundle_id: event.capture_bundle_id,
13842
- state: event.state,
13843
- receipt_id: event.receipt_id,
13844
- promotion_identity_sha256: event.promotion_identity_sha256,
13845
- window_start_ms: Date.parse(event.window.start_at),
13846
- window_end_ms: Date.parse(event.window.end_at),
13847
- event_at_ms: Date.parse(event.event_at),
13848
- reason_code: event.reason_code,
13849
- event_json: jcsCanonicalize(event)
13600
+ err(0);
13601
+ };
13602
+ var msb = function(val) {
13603
+ var bits = 0;
13604
+ for (;1 << bits <= val; ++bits)
13605
+ ;
13606
+ return bits - 1;
13607
+ };
13608
+ var rfse = function(dat, bt, mal) {
13609
+ var tpos = (bt << 3) + 4;
13610
+ var al = (dat[bt] & 15) + 5;
13611
+ if (al > mal)
13612
+ err(3);
13613
+ var sz = 1 << al;
13614
+ var probs = sz, sym = -1, re = -1, i = -1, ht = sz;
13615
+ var buf = new ab(512 + (sz << 2));
13616
+ var freq = new i16(buf, 0, 256);
13617
+ var dstate = new u16(buf, 0, 256);
13618
+ var nstate = new u16(buf, 512, sz);
13619
+ var bb1 = 512 + (sz << 1);
13620
+ var syms = new u8(buf, bb1, sz);
13621
+ var nbits = new u8(buf, bb1 + sz);
13622
+ while (sym < 255 && probs > 0) {
13623
+ var bits = msb(probs + 1);
13624
+ var cbt = tpos >> 3;
13625
+ var msk = (1 << bits + 1) - 1;
13626
+ var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
13627
+ var msk1fb = (1 << bits) - 1;
13628
+ var msv = msk - probs - 1;
13629
+ var sval = val & msk1fb;
13630
+ if (sval < msv)
13631
+ tpos += bits, val = sval;
13632
+ else {
13633
+ tpos += bits + 1;
13634
+ if (val > msk1fb)
13635
+ val -= msv;
13850
13636
  }
13851
- };
13852
- }
13853
-
13854
- // src/helpers/market-data-vendor-backfill/core.ts
13855
- function createMarketDataVendorBackfillDependencies(dependencies) {
13856
- for (const [name, value] of Object.entries({
13857
- archive: dependencies.archive,
13858
- providers: dependencies.providers,
13859
- credentials: dependencies.credentials,
13860
- forwarder: dependencies.forwarder,
13861
- clock: dependencies.clock
13862
- })) {
13863
- if (!value || typeof value !== "object") {
13864
- throw new TypeError(`Backfill dependency ${name} is required`);
13637
+ freq[++sym] = --val;
13638
+ if (val == -1) {
13639
+ probs += val;
13640
+ syms[--ht] = sym;
13641
+ } else
13642
+ probs -= val;
13643
+ if (!val) {
13644
+ do {
13645
+ var rbt = tpos >> 3;
13646
+ re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
13647
+ tpos += 2;
13648
+ sym += re;
13649
+ } while (re == 3);
13865
13650
  }
13866
13651
  }
13867
- for (const [name, method] of Object.entries({
13868
- "archive.resolveSelection": dependencies.archive.resolveSelection,
13869
- "archive.verifyCandidate": dependencies.archive.verifyCandidate,
13870
- "providers.capabilityFor": dependencies.providers.capabilityFor,
13871
- "providers.acquire": dependencies.providers.acquire,
13872
- "providers.normalize": dependencies.providers.normalize,
13873
- "credentials.resolve": dependencies.credentials.resolve,
13874
- "forwarder.preflight": dependencies.forwarder.preflight,
13875
- "forwarder.submit": dependencies.forwarder.submit,
13876
- "clock.nowMs": dependencies.clock.nowMs
13877
- })) {
13878
- if (typeof method !== "function") {
13879
- throw new TypeError(`Backfill dependency ${name} is required`);
13652
+ if (sym > 255 || probs)
13653
+ err(0);
13654
+ var sympos = 0;
13655
+ var sstep = (sz >> 1) + (sz >> 3) + 3;
13656
+ var smask = sz - 1;
13657
+ for (var s = 0;s <= sym; ++s) {
13658
+ var sf = freq[s];
13659
+ if (sf < 1) {
13660
+ dstate[s] = -sf;
13661
+ continue;
13662
+ }
13663
+ for (i = 0;i < sf; ++i) {
13664
+ syms[sympos] = s;
13665
+ do {
13666
+ sympos = sympos + sstep & smask;
13667
+ } while (sympos >= ht);
13880
13668
  }
13881
13669
  }
13882
- return dependencies;
13883
- }
13884
- function outcome(request2, status, reasonCode, extra = {}) {
13885
- return {
13886
- status,
13887
- reasonCode,
13888
- ...request2 ? {
13889
- requestId: request2.requestId,
13890
- idempotencyKey: request2.idempotencyKey,
13891
- ...request2.target ? { target: request2.target } : {}
13892
- } : {},
13893
- ...extra
13894
- };
13895
- }
13896
- function captureBundleId2(request2, capability, dataset) {
13897
- return sha256Canonical({
13898
- request_business_identity: request2.idempotencyKey,
13899
- provider: capability.provider,
13900
- provider_exchange_id: capability.providerExchangeId,
13901
- resolved_symbol: capability.resolvedSymbol,
13902
- adapter_version: capability.adapterVersion,
13903
- objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
13904
- identity,
13905
- checksum,
13906
- bytes,
13907
- rows
13908
- })),
13909
- canonical_schema_version: request2.expectedProduct.canonicalSchemaVersion,
13910
- checksum_algorithm: request2.expectedProduct.checksumAlgorithm
13911
- });
13912
- }
13913
- function buildPromotionReceipt(request2, capability, normalized, verification, verificationTimeMs) {
13914
- if (!request2.wire || !request2.initialSelection || !request2.expectedCanonicalSchema || !request2.coveragePolicy || !request2.productPins) {
13915
- throw new Error("decoded final-v1 request context is missing");
13670
+ if (sympos)
13671
+ err(0);
13672
+ for (i = 0;i < sz; ++i) {
13673
+ var ns = dstate[syms[i]]++;
13674
+ var nb = nbits[i] = al - msb(ns);
13675
+ nstate[i] = (ns << nb) - sz;
13916
13676
  }
13917
- return finalizePromotionReceipt({
13918
- schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
13919
- verified_at: new Date(verificationTimeMs).toISOString(),
13920
- request_id: request2.requestId,
13921
- idempotency_key: request2.idempotencyKey,
13922
- source: EXTERNAL_BACKFILL_SOURCE,
13923
- capture_origin: "vendor_historical_backfill",
13924
- source_mode: "vendor_historical_backfill_v1",
13925
- provider: capability.provider,
13926
- adapter_version: capability.adapterVersion,
13927
- effective_policies: {
13928
- capability_policy: request2.productPins.capability_policy,
13929
- resource_policy: request2.productPins.resource_policy,
13930
- adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
13931
- acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
13932
- },
13933
- capture_bundle_id: normalized.captureBundleId,
13934
- scope: request2.wire.scope,
13935
- window: request2.wire.window,
13936
- depth: request2.depth,
13937
- construction_mode: request2.constructionMode,
13938
- canonical_schema: request2.expectedCanonicalSchema,
13939
- coverage_policy: request2.coveragePolicy,
13940
- selection_sha256: request2.initialSelection.selection_sha256,
13941
- vendor_semantic_digest: normalized.vendorSemanticDigest,
13942
- canonical_semantic_digest: verification.canonicalSemanticDigest,
13943
- prefix_digest: verification.prefixDigest,
13944
- suffix_digest: verification.suffixDigest,
13945
- seam_verified: true,
13946
- coverage_verified: true,
13947
- dataset_objects: normalized.objects
13948
- });
13949
- }
13950
- async function submitAll(dependencies, batches) {
13951
- const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
13952
- for (const batch of batches) {
13953
- let accepted = false;
13954
- for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
13955
- try {
13956
- const response = await dependencies.forwarder.submit(batch);
13957
- accepted = response.ok && response.inserted === batch.rows.length;
13958
- } catch {
13959
- accepted = false;
13960
- }
13961
- if (accepted)
13677
+ return [tpos + 7 >> 3, {
13678
+ b: al,
13679
+ s: syms,
13680
+ n: nbits,
13681
+ t: nstate
13682
+ }];
13683
+ };
13684
+ var rhu = function(dat, bt) {
13685
+ var i = 0, wc = -1;
13686
+ var buf = new u8(292), hb = dat[bt];
13687
+ var hw = buf.subarray(0, 256);
13688
+ var rc = buf.subarray(256, 268);
13689
+ var ri = new u16(buf.buffer, 268);
13690
+ if (hb < 128) {
13691
+ var _a = rfse(dat, bt + 1, 6), ebt = _a[0], fdt = _a[1];
13692
+ bt += hb;
13693
+ var epos = ebt << 3;
13694
+ var lb = dat[bt];
13695
+ if (!lb)
13696
+ err(0);
13697
+ var st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
13698
+ var fpos = (++bt << 3) - 8 + msb(lb);
13699
+ for (;; ) {
13700
+ fpos -= btr1;
13701
+ if (fpos < epos)
13962
13702
  break;
13963
- if (attempt < maxAttempts)
13964
- await dependencies.retry?.wait?.(attempt);
13703
+ var cbt = fpos >> 3;
13704
+ st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
13705
+ hw[++wc] = fdt.s[st1];
13706
+ fpos -= btr2;
13707
+ if (fpos < epos)
13708
+ break;
13709
+ cbt = fpos >> 3;
13710
+ st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
13711
+ hw[++wc] = fdt.s[st2];
13712
+ btr1 = fdt.n[st1];
13713
+ st1 = fdt.t[st1];
13714
+ btr2 = fdt.n[st2];
13715
+ st2 = fdt.t[st2];
13965
13716
  }
13966
- if (!accepted)
13967
- return false;
13968
- }
13969
- return true;
13970
- }
13971
- function stableFailureReason(error, fallback) {
13972
- if (error && typeof error === "object") {
13973
- const reason = error.reason;
13974
- if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
13975
- return reason;
13717
+ if (++wc > 255)
13718
+ err(0);
13719
+ } else {
13720
+ wc = hb - 127;
13721
+ for (;i < wc; i += 2) {
13722
+ var byte = dat[++bt];
13723
+ hw[i] = byte >> 4;
13724
+ hw[i + 1] = byte & 15;
13976
13725
  }
13726
+ ++bt;
13977
13727
  }
13978
- return fallback;
13979
- }
13980
- function assertArchivePreflight(request2, resolution) {
13981
- if (!request2.target || !request2.productionAuthorizationId) {
13982
- throw new Error("request target or production authorization ID is missing");
13728
+ var wes = 0;
13729
+ for (i = 0;i < wc; ++i) {
13730
+ var wt = hw[i];
13731
+ if (wt > 11)
13732
+ err(0);
13733
+ wes += wt && 1 << wt - 1;
13983
13734
  }
13984
- const selection = archiveSelectionCodec.decode(resolution.selection);
13985
- const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
13986
- const receiptById = new Map;
13987
- for (const receipt of receipts) {
13988
- const existing = receiptById.get(receipt.receipt_id);
13989
- if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
13990
- throw new Error("stored receipt identity has conflicting content");
13991
- }
13992
- receiptById.set(receipt.receipt_id, receipt);
13735
+ var mb = msb(wes) + 1;
13736
+ var ts = 1 << mb;
13737
+ var rem = ts - wes;
13738
+ if (rem & rem - 1)
13739
+ err(0);
13740
+ hw[wc++] = msb(rem) + 1;
13741
+ for (i = 0;i < wc; ++i) {
13742
+ var wt = hw[i];
13743
+ ++rc[hw[i] = wt && mb + 1 - wt];
13993
13744
  }
13994
- for (const bundle of selection.bundles) {
13995
- if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
13996
- throw new Error("vendor selection lacks its validated stored receipt");
13997
- }
13745
+ var hbuf = new u8(ts << 1);
13746
+ var syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
13747
+ ri[mb] = 0;
13748
+ for (i = mb;i > 0; --i) {
13749
+ var pv = ri[i];
13750
+ fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
13998
13751
  }
13999
- if (resolution.readerIdentity.environment !== request2.target.environment || resolution.readerIdentity.cluster !== request2.target.cluster) {
14000
- throw new Error("archive reader cluster identity mismatch");
13752
+ if (ri[0] != ts)
13753
+ err(0);
13754
+ for (i = 0;i < wc; ++i) {
13755
+ var bits = hw[i];
13756
+ if (bits) {
13757
+ var code = ri[bits];
13758
+ fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
13759
+ }
14001
13760
  }
14002
- }
14003
- function assertForwarderPreflight(request2, resolution, nowMs) {
14004
- if (!request2.target || !request2.productionAuthorizationId) {
14005
- throw new Error("request target or production authorization ID is missing");
13761
+ return [bt, {
13762
+ n: nb,
13763
+ b: mb,
13764
+ s: syms
13765
+ }];
13766
+ };
13767
+ var dllt = rfse(/* @__PURE__ */ new u8([
13768
+ 81,
13769
+ 16,
13770
+ 99,
13771
+ 140,
13772
+ 49,
13773
+ 198,
13774
+ 24,
13775
+ 99,
13776
+ 12,
13777
+ 33,
13778
+ 196,
13779
+ 24,
13780
+ 99,
13781
+ 102,
13782
+ 102,
13783
+ 134,
13784
+ 70,
13785
+ 146,
13786
+ 4
13787
+ ]), 0, 6)[1];
13788
+ var dmlt = rfse(/* @__PURE__ */ new u8([
13789
+ 33,
13790
+ 20,
13791
+ 196,
13792
+ 24,
13793
+ 99,
13794
+ 140,
13795
+ 33,
13796
+ 132,
13797
+ 16,
13798
+ 66,
13799
+ 8,
13800
+ 33,
13801
+ 132,
13802
+ 16,
13803
+ 66,
13804
+ 8,
13805
+ 33,
13806
+ 68,
13807
+ 68,
13808
+ 68,
13809
+ 68,
13810
+ 68,
13811
+ 68,
13812
+ 68,
13813
+ 68,
13814
+ 36,
13815
+ 9
13816
+ ]), 0, 6)[1];
13817
+ var doct = rfse(/* @__PURE__ */ new u8([
13818
+ 32,
13819
+ 132,
13820
+ 16,
13821
+ 66,
13822
+ 102,
13823
+ 70,
13824
+ 68,
13825
+ 68,
13826
+ 68,
13827
+ 68,
13828
+ 36,
13829
+ 73,
13830
+ 2
13831
+ ]), 0, 5)[1];
13832
+ var b2bl = function(b, s) {
13833
+ var len = b.length, bl = new i32(len);
13834
+ for (var i = 0;i < len; ++i) {
13835
+ bl[i] = s;
13836
+ s += 1 << b[i];
14006
13837
  }
14007
- if (resolution.forwarderIdentity.environment !== request2.target.environment || resolution.forwarderIdentity.cluster !== request2.target.cluster) {
14008
- throw new Error("archive forwarder cluster identity mismatch");
13838
+ return bl;
13839
+ };
13840
+ var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
13841
+ 0,
13842
+ 0,
13843
+ 0,
13844
+ 0,
13845
+ 16843009,
13846
+ 50528770,
13847
+ 134678020,
13848
+ 202050057,
13849
+ 269422093
13850
+ ])).buffer, 0, 36);
13851
+ var llbl = /* @__PURE__ */ b2bl(llb, 0);
13852
+ var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
13853
+ 0,
13854
+ 0,
13855
+ 0,
13856
+ 0,
13857
+ 0,
13858
+ 0,
13859
+ 0,
13860
+ 0,
13861
+ 16843009,
13862
+ 50528770,
13863
+ 117769220,
13864
+ 185207048,
13865
+ 252579084,
13866
+ 16
13867
+ ])).buffer, 0, 53);
13868
+ var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
13869
+ var dhu = function(dat, out, hu) {
13870
+ var len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
13871
+ if (!lb)
13872
+ err(0);
13873
+ var st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
13874
+ for (;pos > eb && i < ss; ) {
13875
+ var cbt = pos >> 3;
13876
+ var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
13877
+ st = (st << btr | val) & msk;
13878
+ out[++i] = hu.s[st];
13879
+ pos -= btr = hu.n[st];
14009
13880
  }
14010
- const authorization = resolution.authorization;
14011
- const expiresAtMs = Date.parse(authorization.expiresAt);
14012
- if (authorization.authorizationId !== request2.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request2.target.environment || authorization.cluster !== request2.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
14013
- throw new Error("production forwarder authorization is invalid");
14014
- }
14015
- }
14016
- function resourcePolicyScopeExceeded(request2) {
14017
- return request2.depth > RESOURCE_POLICY.request_bounds.max_depth || request2.window.endTimeMs - request2.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request2.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
14018
- }
14019
- function storedReceiptForSelection(resolution) {
14020
- const receiptId = resolution.selection.receipt_ids[0];
14021
- return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
14022
- }
14023
- async function runMarketDataVendorBackfill(input, dependencies) {
14024
- let request2;
14025
- try {
14026
- const documents = input;
14027
- request2 = decodeBackfillRunDocuments({
14028
- request: documents.request,
14029
- requiredClock: documents.requiredClock
14030
- });
14031
- } catch {
14032
- return outcome(undefined, "request_invalid", "request_invalid");
14033
- }
14034
- let initialResolution;
14035
- try {
14036
- initialResolution = await dependencies.archive.resolveSelection(request2);
14037
- assertArchivePreflight(request2, initialResolution);
14038
- const forwarderPreflight = await dependencies.forwarder.preflight({
14039
- authorizationId: request2.productionAuthorizationId,
14040
- target: request2.target
14041
- });
14042
- assertForwarderPreflight(request2, forwarderPreflight, dependencies.clock.nowMs());
14043
- } catch {
14044
- return outcome(request2, "archive_preflight_failed", "archive_preflight_failed");
14045
- }
14046
- if (initialResolution.selection.coverage_class === "complete") {
14047
- const receipt2 = storedReceiptForSelection(initialResolution);
14048
- return outcome(request2, "already_covered", "qualified_coverage_complete", {
14049
- selection: initialResolution.selection,
14050
- ...receipt2 ? { receipt: receipt2 } : {}
14051
- });
14052
- }
14053
- if (resourcePolicyScopeExceeded(request2)) {
14054
- return outcome(request2, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
14055
- }
14056
- let capability;
14057
- try {
14058
- capability = dependencies.providers.capabilityFor(request2);
14059
- } catch {
14060
- return outcome(request2, "capability_unsupported", "capability_probe_failed");
14061
- }
14062
- if (!capability) {
14063
- return outcome(request2, "capability_unsupported", "scope_unsupported");
14064
- }
14065
- if (!request2.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
14066
- return outcome(request2, "capability_unsupported", "adapter_version_unpinned");
14067
- }
14068
- let credential;
14069
- try {
14070
- credential = await dependencies.credentials.resolve(capability.provider);
14071
- } catch {
14072
- return outcome(request2, "credentials_missing", "credential_resolution_failed");
14073
- }
14074
- if (credential === undefined || credential === null) {
14075
- return outcome(request2, "credentials_missing", "provider_credentials_missing");
14076
- }
14077
- let dataset;
14078
- let normalized;
14079
- try {
14080
- dataset = await dependencies.providers.acquire(request2, capability, credential);
14081
- const bundleId = captureBundleId2(request2, capability, dataset);
14082
- normalized = await dependencies.providers.normalize(request2, capability, dataset, bundleId);
14083
- if (normalized.captureBundleId !== bundleId) {
14084
- return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
14085
- reasonSubcode: "capture_identity_mismatch"
14086
- });
13881
+ if (pos != eb || i + 1 != ss)
13882
+ err(0);
13883
+ };
13884
+ var dhu4 = function(dat, out, hu) {
13885
+ var bt = 6;
13886
+ var ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
13887
+ dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
13888
+ dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
13889
+ dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
13890
+ dhu(dat.subarray(bt), out.subarray(sz3), hu);
13891
+ };
13892
+ var rzb = function(dat, st, out) {
13893
+ var _a;
13894
+ var bt = st.b;
13895
+ var b0 = dat[bt], btype = b0 >> 1 & 3;
13896
+ st.l = b0 & 1;
13897
+ var sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
13898
+ var ebt = (bt += 3) + sz;
13899
+ if (btype == 1) {
13900
+ if (bt >= dat.length)
13901
+ return;
13902
+ st.b = bt + 1;
13903
+ if (out) {
13904
+ fill(out, dat[bt], st.y, st.y += sz);
13905
+ return out;
14087
13906
  }
14088
- } catch (error) {
14089
- const reason = stableFailureReason(error, "provider_dataset_invalid");
14090
- return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
14091
- reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
14092
- });
13907
+ return fill(new u8(sz), dat[bt]);
14093
13908
  }
14094
- try {
14095
- const candidateBatches = buildForwarderBatches({
14096
- captureBundleId: normalized.captureBundleId,
14097
- deploymentId: "market-data-vendor-backfill",
14098
- rows: normalized.rows
14099
- });
14100
- if (!await submitAll(dependencies, candidateBatches)) {
14101
- return outcome(request2, "archive_ingest_failed", "candidate_batch_rejected");
13909
+ if (ebt > dat.length)
13910
+ return;
13911
+ if (btype == 0) {
13912
+ st.b = ebt;
13913
+ if (out) {
13914
+ out.set(dat.subarray(bt, ebt), st.y);
13915
+ st.y += sz;
13916
+ return out;
14102
13917
  }
14103
- } catch {
14104
- return outcome(request2, "archive_ingest_failed", "candidate_submission_failed");
14105
- }
14106
- let verification;
14107
- try {
14108
- verification = await dependencies.archive.verifyCandidate(request2, normalized, normalized.captureBundleId, initialResolution);
14109
- } catch {
14110
- return outcome(request2, "promotion_verification_failed", "candidate_query_failed");
14111
- }
14112
- if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
14113
- return outcome(request2, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
13918
+ return slc(dat, bt, ebt);
14114
13919
  }
14115
- let receipt;
14116
- try {
14117
- receipt = buildPromotionReceipt(request2, capability, normalized, verification, dependencies.clock.nowMs());
14118
- const [batch] = buildForwarderBatches({
14119
- captureBundleId: normalized.captureBundleId,
14120
- deploymentId: "market-data-vendor-backfill",
14121
- rows: [promotionReceiptToArchiveRow(receipt)]
14122
- });
14123
- if (!batch || !await submitAll(dependencies, [batch])) {
14124
- return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
14125
- }
14126
- const qualification = finalizeQualificationEvent({
14127
- capture_bundle_id: receipt.capture_bundle_id,
14128
- state: "qualified",
14129
- receipt_id: receipt.receipt_id,
14130
- promotion_identity_sha256: receipt.promotion_identity_sha256,
14131
- window: receipt.window,
14132
- event_at: receipt.verified_at,
14133
- reason_code: "promotion_verified"
14134
- });
14135
- const [qualificationBatch] = buildForwarderBatches({
14136
- captureBundleId: normalized.captureBundleId,
14137
- deploymentId: "market-data-vendor-backfill",
14138
- rows: [qualificationEventToArchiveRow(qualification)]
14139
- });
14140
- if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
14141
- return outcome(request2, "archive_ingest_failed", "qualification_commit_failed", { receipt });
13920
+ if (btype == 2) {
13921
+ var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
13922
+ var lss = b3 >> 4, lcs = 0, s4 = 0;
13923
+ if (lbt < 2) {
13924
+ if (sf & 1)
13925
+ lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
13926
+ else
13927
+ lss = b3 >> 3;
13928
+ } else {
13929
+ s4 = sf;
13930
+ if (sf < 2)
13931
+ lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
13932
+ else if (sf == 2)
13933
+ lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
13934
+ else
13935
+ lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
14142
13936
  }
14143
- } catch {
14144
- return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
14145
- }
14146
- let finalResolution;
14147
- try {
14148
- finalResolution = await dependencies.archive.resolveSelection(request2);
14149
- assertArchivePreflight(request2, finalResolution);
14150
- } catch (error) {
14151
- return outcome(request2, "promotion_verification_failed", "post_promotion_selection_failed", {
14152
- receipt,
14153
- reasonSubcode: stableFailureReason(error, "archive_selection_resolution_failed")
14154
- });
14155
- }
14156
- if (finalResolution.selection.coverage_class !== "complete") {
14157
- return outcome(request2, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
14158
- }
14159
- if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
14160
- return outcome(request2, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
14161
- }
14162
- try {
14163
- if (!request2.wire)
14164
- throw new Error("decoded request wire is missing");
14165
- const [selectionBatch] = buildForwarderBatches({
14166
- captureBundleId: receipt.capture_bundle_id,
14167
- deploymentId: "market-data-vendor-backfill",
14168
- rows: [
14169
- archiveSelectionToArchiveRow(request2.wire, finalResolution.selection)
14170
- ]
14171
- });
14172
- if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
14173
- return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
14174
- }
14175
- } catch {
14176
- return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
14177
- }
14178
- return outcome(request2, "promoted", "promotion_qualified", {
14179
- receipt,
14180
- selection: finalResolution.selection
14181
- });
14182
- }
14183
- // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
14184
- import { createHash as createHash3 } from "node:crypto";
14185
-
14186
- // node_modules/fzstd/esm/index.mjs
14187
- var ab = ArrayBuffer;
14188
- var u8 = Uint8Array;
14189
- var u16 = Uint16Array;
14190
- var i16 = Int16Array;
14191
- var i32 = Int32Array;
14192
- var slc = function(v, s, e) {
14193
- if (u8.prototype.slice)
14194
- return u8.prototype.slice.call(v, s, e);
14195
- if (s == null || s < 0)
14196
- s = 0;
14197
- if (e == null || e > v.length)
14198
- e = v.length;
14199
- var n = new u8(e - s);
14200
- n.set(v.subarray(s, e));
14201
- return n;
14202
- };
14203
- var fill = function(v, n, s, e) {
14204
- if (u8.prototype.fill)
14205
- return u8.prototype.fill.call(v, n, s, e);
14206
- if (s == null || s < 0)
14207
- s = 0;
14208
- if (e == null || e > v.length)
14209
- e = v.length;
14210
- for (;s < e; ++s)
14211
- v[s] = n;
14212
- return v;
14213
- };
14214
- var cpw = function(v, t, s, e) {
14215
- if (u8.prototype.copyWithin)
14216
- return u8.prototype.copyWithin.call(v, t, s, e);
14217
- if (s == null || s < 0)
14218
- s = 0;
14219
- if (e == null || e > v.length)
14220
- e = v.length;
14221
- while (s < e) {
14222
- v[t++] = v[s++];
14223
- }
14224
- };
14225
- var ec = [
14226
- "invalid zstd data",
14227
- "window size too large (>2046MB)",
14228
- "invalid block type",
14229
- "FSE accuracy too high",
14230
- "match distance too far back",
14231
- "unexpected EOF"
14232
- ];
14233
- var err = function(ind, msg, nt) {
14234
- var e = new Error(msg || ec[ind]);
14235
- e.code = ind;
14236
- if (Error.captureStackTrace)
14237
- Error.captureStackTrace(e, err);
14238
- if (!nt)
14239
- throw e;
14240
- return e;
14241
- };
14242
- var rb = function(d, b, n) {
14243
- var i = 0, o = 0;
14244
- for (;i < n; ++i)
14245
- o |= d[b++] << (i << 3);
14246
- return o;
14247
- };
14248
- var b4 = function(d, b) {
14249
- return (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0;
14250
- };
14251
- var rzfh = function(dat, w) {
14252
- var n3 = dat[0] | dat[1] << 8 | dat[2] << 16;
14253
- if (n3 == 3126568 && dat[3] == 253) {
14254
- var flg = dat[4];
14255
- var ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6;
14256
- if (flg & 8)
14257
- err(0);
14258
- var bt = 6 - ss;
14259
- var db = df == 3 ? 4 : df;
14260
- var di = rb(dat, bt, db);
14261
- bt += db;
14262
- var fsb = fcf ? 1 << fcf : ss;
14263
- var fss = rb(dat, bt, fsb) + (fcf == 1 && 256);
14264
- var ws = fss;
14265
- if (!ss) {
14266
- var wb = 1 << 10 + (dat[5] >> 3);
14267
- ws = wb + (wb >> 3) * (dat[5] & 7);
14268
- }
14269
- if (ws > 2145386496)
14270
- err(1);
14271
- var buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
14272
- buf[0] = 1, buf[4] = 4, buf[8] = 8;
14273
- return {
14274
- b: bt + fsb,
14275
- y: 0,
14276
- l: 0,
14277
- d: di,
14278
- w: w && w != 1 ? w : buf.subarray(12),
14279
- e: ws,
14280
- o: new i32(buf.buffer, 0, 3),
14281
- u: fss,
14282
- c: cc,
14283
- m: Math.min(131072, ws)
14284
- };
14285
- } else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
14286
- return b4(dat, 4) + 8;
14287
- }
14288
- err(0);
14289
- };
14290
- var msb = function(val) {
14291
- var bits = 0;
14292
- for (;1 << bits <= val; ++bits)
14293
- ;
14294
- return bits - 1;
14295
- };
14296
- var rfse = function(dat, bt, mal) {
14297
- var tpos = (bt << 3) + 4;
14298
- var al = (dat[bt] & 15) + 5;
14299
- if (al > mal)
14300
- err(3);
14301
- var sz = 1 << al;
14302
- var probs = sz, sym = -1, re = -1, i = -1, ht = sz;
14303
- var buf = new ab(512 + (sz << 2));
14304
- var freq = new i16(buf, 0, 256);
14305
- var dstate = new u16(buf, 0, 256);
14306
- var nstate = new u16(buf, 512, sz);
14307
- var bb1 = 512 + (sz << 1);
14308
- var syms = new u8(buf, bb1, sz);
14309
- var nbits = new u8(buf, bb1 + sz);
14310
- while (sym < 255 && probs > 0) {
14311
- var bits = msb(probs + 1);
14312
- var cbt = tpos >> 3;
14313
- var msk = (1 << bits + 1) - 1;
14314
- var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
14315
- var msk1fb = (1 << bits) - 1;
14316
- var msv = msk - probs - 1;
14317
- var sval = val & msk1fb;
14318
- if (sval < msv)
14319
- tpos += bits, val = sval;
14320
- else {
14321
- tpos += bits + 1;
14322
- if (val > msk1fb)
14323
- val -= msv;
14324
- }
14325
- freq[++sym] = --val;
14326
- if (val == -1) {
14327
- probs += val;
14328
- syms[--ht] = sym;
14329
- } else
14330
- probs -= val;
14331
- if (!val) {
14332
- do {
14333
- var rbt = tpos >> 3;
14334
- re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
14335
- tpos += 2;
14336
- sym += re;
14337
- } while (re == 3);
14338
- }
14339
- }
14340
- if (sym > 255 || probs)
14341
- err(0);
14342
- var sympos = 0;
14343
- var sstep = (sz >> 1) + (sz >> 3) + 3;
14344
- var smask = sz - 1;
14345
- for (var s = 0;s <= sym; ++s) {
14346
- var sf = freq[s];
14347
- if (sf < 1) {
14348
- dstate[s] = -sf;
14349
- continue;
14350
- }
14351
- for (i = 0;i < sf; ++i) {
14352
- syms[sympos] = s;
14353
- do {
14354
- sympos = sympos + sstep & smask;
14355
- } while (sympos >= ht);
14356
- }
14357
- }
14358
- if (sympos)
14359
- err(0);
14360
- for (i = 0;i < sz; ++i) {
14361
- var ns = dstate[syms[i]]++;
14362
- var nb = nbits[i] = al - msb(ns);
14363
- nstate[i] = (ns << nb) - sz;
14364
- }
14365
- return [tpos + 7 >> 3, {
14366
- b: al,
14367
- s: syms,
14368
- n: nbits,
14369
- t: nstate
14370
- }];
14371
- };
14372
- var rhu = function(dat, bt) {
14373
- var i = 0, wc = -1;
14374
- var buf = new u8(292), hb = dat[bt];
14375
- var hw = buf.subarray(0, 256);
14376
- var rc = buf.subarray(256, 268);
14377
- var ri = new u16(buf.buffer, 268);
14378
- if (hb < 128) {
14379
- var _a = rfse(dat, bt + 1, 6), ebt = _a[0], fdt = _a[1];
14380
- bt += hb;
14381
- var epos = ebt << 3;
14382
- var lb = dat[bt];
14383
- if (!lb)
14384
- err(0);
14385
- var st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
14386
- var fpos = (++bt << 3) - 8 + msb(lb);
14387
- for (;; ) {
14388
- fpos -= btr1;
14389
- if (fpos < epos)
14390
- break;
14391
- var cbt = fpos >> 3;
14392
- st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
14393
- hw[++wc] = fdt.s[st1];
14394
- fpos -= btr2;
14395
- if (fpos < epos)
14396
- break;
14397
- cbt = fpos >> 3;
14398
- st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
14399
- hw[++wc] = fdt.s[st2];
14400
- btr1 = fdt.n[st1];
14401
- st1 = fdt.t[st1];
14402
- btr2 = fdt.n[st2];
14403
- st2 = fdt.t[st2];
14404
- }
14405
- if (++wc > 255)
14406
- err(0);
14407
- } else {
14408
- wc = hb - 127;
14409
- for (;i < wc; i += 2) {
14410
- var byte = dat[++bt];
14411
- hw[i] = byte >> 4;
14412
- hw[i + 1] = byte & 15;
14413
- }
14414
- ++bt;
14415
- }
14416
- var wes = 0;
14417
- for (i = 0;i < wc; ++i) {
14418
- var wt = hw[i];
14419
- if (wt > 11)
14420
- err(0);
14421
- wes += wt && 1 << wt - 1;
14422
- }
14423
- var mb = msb(wes) + 1;
14424
- var ts = 1 << mb;
14425
- var rem = ts - wes;
14426
- if (rem & rem - 1)
14427
- err(0);
14428
- hw[wc++] = msb(rem) + 1;
14429
- for (i = 0;i < wc; ++i) {
14430
- var wt = hw[i];
14431
- ++rc[hw[i] = wt && mb + 1 - wt];
14432
- }
14433
- var hbuf = new u8(ts << 1);
14434
- var syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
14435
- ri[mb] = 0;
14436
- for (i = mb;i > 0; --i) {
14437
- var pv = ri[i];
14438
- fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
14439
- }
14440
- if (ri[0] != ts)
14441
- err(0);
14442
- for (i = 0;i < wc; ++i) {
14443
- var bits = hw[i];
14444
- if (bits) {
14445
- var code = ri[bits];
14446
- fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
14447
- }
14448
- }
14449
- return [bt, {
14450
- n: nb,
14451
- b: mb,
14452
- s: syms
14453
- }];
14454
- };
14455
- var dllt = rfse(/* @__PURE__ */ new u8([
14456
- 81,
14457
- 16,
14458
- 99,
14459
- 140,
14460
- 49,
14461
- 198,
14462
- 24,
14463
- 99,
14464
- 12,
14465
- 33,
14466
- 196,
14467
- 24,
14468
- 99,
14469
- 102,
14470
- 102,
14471
- 134,
14472
- 70,
14473
- 146,
14474
- 4
14475
- ]), 0, 6)[1];
14476
- var dmlt = rfse(/* @__PURE__ */ new u8([
14477
- 33,
14478
- 20,
14479
- 196,
14480
- 24,
14481
- 99,
14482
- 140,
14483
- 33,
14484
- 132,
14485
- 16,
14486
- 66,
14487
- 8,
14488
- 33,
14489
- 132,
14490
- 16,
14491
- 66,
14492
- 8,
14493
- 33,
14494
- 68,
14495
- 68,
14496
- 68,
14497
- 68,
14498
- 68,
14499
- 68,
14500
- 68,
14501
- 68,
14502
- 36,
14503
- 9
14504
- ]), 0, 6)[1];
14505
- var doct = rfse(/* @__PURE__ */ new u8([
14506
- 32,
14507
- 132,
14508
- 16,
14509
- 66,
14510
- 102,
14511
- 70,
14512
- 68,
14513
- 68,
14514
- 68,
14515
- 68,
14516
- 36,
14517
- 73,
14518
- 2
14519
- ]), 0, 5)[1];
14520
- var b2bl = function(b, s) {
14521
- var len = b.length, bl = new i32(len);
14522
- for (var i = 0;i < len; ++i) {
14523
- bl[i] = s;
14524
- s += 1 << b[i];
14525
- }
14526
- return bl;
14527
- };
14528
- var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
14529
- 0,
14530
- 0,
14531
- 0,
14532
- 0,
14533
- 16843009,
14534
- 50528770,
14535
- 134678020,
14536
- 202050057,
14537
- 269422093
14538
- ])).buffer, 0, 36);
14539
- var llbl = /* @__PURE__ */ b2bl(llb, 0);
14540
- var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
14541
- 0,
14542
- 0,
14543
- 0,
14544
- 0,
14545
- 0,
14546
- 0,
14547
- 0,
14548
- 0,
14549
- 16843009,
14550
- 50528770,
14551
- 117769220,
14552
- 185207048,
14553
- 252579084,
14554
- 16
14555
- ])).buffer, 0, 53);
14556
- var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
14557
- var dhu = function(dat, out, hu) {
14558
- var len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
14559
- if (!lb)
14560
- err(0);
14561
- var st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
14562
- for (;pos > eb && i < ss; ) {
14563
- var cbt = pos >> 3;
14564
- var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
14565
- st = (st << btr | val) & msk;
14566
- out[++i] = hu.s[st];
14567
- pos -= btr = hu.n[st];
14568
- }
14569
- if (pos != eb || i + 1 != ss)
14570
- err(0);
14571
- };
14572
- var dhu4 = function(dat, out, hu) {
14573
- var bt = 6;
14574
- var ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
14575
- dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
14576
- dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
14577
- dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
14578
- dhu(dat.subarray(bt), out.subarray(sz3), hu);
14579
- };
14580
- var rzb = function(dat, st, out) {
14581
- var _a;
14582
- var bt = st.b;
14583
- var b0 = dat[bt], btype = b0 >> 1 & 3;
14584
- st.l = b0 & 1;
14585
- var sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
14586
- var ebt = (bt += 3) + sz;
14587
- if (btype == 1) {
14588
- if (bt >= dat.length)
14589
- return;
14590
- st.b = bt + 1;
14591
- if (out) {
14592
- fill(out, dat[bt], st.y, st.y += sz);
14593
- return out;
14594
- }
14595
- return fill(new u8(sz), dat[bt]);
14596
- }
14597
- if (ebt > dat.length)
14598
- return;
14599
- if (btype == 0) {
14600
- st.b = ebt;
14601
- if (out) {
14602
- out.set(dat.subarray(bt, ebt), st.y);
14603
- st.y += sz;
14604
- return out;
14605
- }
14606
- return slc(dat, bt, ebt);
14607
- }
14608
- if (btype == 2) {
14609
- var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
14610
- var lss = b3 >> 4, lcs = 0, s4 = 0;
14611
- if (lbt < 2) {
14612
- if (sf & 1)
14613
- lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
14614
- else
14615
- lss = b3 >> 3;
14616
- } else {
14617
- s4 = sf;
14618
- if (sf < 2)
14619
- lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
14620
- else if (sf == 2)
14621
- lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
14622
- else
14623
- lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
14624
- }
14625
- ++bt;
14626
- var buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
14627
- var spl = buf.length - lss;
14628
- if (lbt == 0)
14629
- buf.set(dat.subarray(bt, bt += lss), spl);
14630
- else if (lbt == 1)
14631
- fill(buf, dat[bt++], spl);
14632
- else {
14633
- var hu = st.h;
14634
- if (lbt == 2) {
14635
- var hud = rhu(dat, bt);
14636
- lcs += bt - (bt = hud[0]);
14637
- st.h = hu = hud[1];
14638
- } else if (!hu)
14639
- err(0);
14640
- (s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
13937
+ ++bt;
13938
+ var buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
13939
+ var spl = buf.length - lss;
13940
+ if (lbt == 0)
13941
+ buf.set(dat.subarray(bt, bt += lss), spl);
13942
+ else if (lbt == 1)
13943
+ fill(buf, dat[bt++], spl);
13944
+ else {
13945
+ var hu = st.h;
13946
+ if (lbt == 2) {
13947
+ var hud = rhu(dat, bt);
13948
+ lcs += bt - (bt = hud[0]);
13949
+ st.h = hu = hud[1];
13950
+ } else if (!hu)
13951
+ err(0);
13952
+ (s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
14641
13953
  }
14642
13954
  var ns = dat[bt++];
14643
13955
  if (ns) {
@@ -18090,599 +17402,1062 @@ function buildCanonicalOrderBookRows(input) {
18090
17402
  const common = commonEvidenceFields(input.context, input.rawCapture, {
18091
17403
  snapshotId,
18092
17404
  sequence,
18093
- depthLimit: input.depthLimit,
18094
- eventTimeMs,
18095
- receivedTimeMs
18096
- });
18097
- const levels = [
18098
- ["bid", bids],
18099
- ["ask", asks]
18100
- ].flatMap(([side, sideLevels]) => sideLevels.map(({ price, amount }, levelIndex) => {
18101
- const row = checksumRow({
18102
- ...common,
18103
- side,
18104
- level_index: levelIndex,
18105
- price,
18106
- amount,
18107
- notional: price * amount,
18108
- mid_price: midPrice,
18109
- spread_from_mid_bps: Math.abs((price - midPrice) / midPrice) * 1e4
18110
- });
18111
- return { table: "market_data.cex_order_book_levels", row };
18112
- }));
18113
- const bands = normalizedBands(input.measurementBandsBps);
18114
- const bidDepth = bands.map((band) => {
18115
- const minimumPrice = bestBid.price * (1 - band / 1e4);
18116
- return bids.filter(({ price }) => price >= minimumPrice).reduce((sum, { amount }) => sum + amount, 0);
18117
- });
18118
- const askDepth = bands.map((band) => {
18119
- const maximumPrice = bestAsk.price * (1 + band / 1e4);
18120
- return asks.filter(({ price }) => price <= maximumPrice).reduce((sum, { amount }) => sum + amount, 0);
18121
- });
18122
- const summaryRow = checksumRow({
18123
- ...common,
18124
- best_bid: bestBid.price,
18125
- best_ask: bestAsk.price,
18126
- best_bid_amount: bestBid.amount,
18127
- best_ask_amount: bestAsk.amount,
18128
- mid_price: midPrice,
18129
- spread,
18130
- spread_bps: spreadBps,
18131
- staleness_ms: receivedTimeMs - eventTimeMs,
18132
- bid_level_count: bids.length,
18133
- ask_level_count: asks.length,
18134
- measurement_bands_bps: bands,
18135
- bid_depth_by_band: bidDepth,
18136
- ask_depth_by_band: askDepth
18137
- });
18138
- return {
18139
- snapshotId,
18140
- levels,
18141
- summary: {
18142
- table: "market_data.cex_order_book_depth_summary",
18143
- row: summaryRow
17405
+ depthLimit: input.depthLimit,
17406
+ eventTimeMs,
17407
+ receivedTimeMs
17408
+ });
17409
+ const levels = [
17410
+ ["bid", bids],
17411
+ ["ask", asks]
17412
+ ].flatMap(([side, sideLevels]) => sideLevels.map(({ price, amount }, levelIndex) => {
17413
+ const row = checksumRow({
17414
+ ...common,
17415
+ side,
17416
+ level_index: levelIndex,
17417
+ price,
17418
+ amount,
17419
+ notional: price * amount,
17420
+ mid_price: midPrice,
17421
+ spread_from_mid_bps: Math.abs((price - midPrice) / midPrice) * 1e4
17422
+ });
17423
+ return { table: "market_data.cex_order_book_levels", row };
17424
+ }));
17425
+ const bands = normalizedBands(input.measurementBandsBps);
17426
+ const bidDepth = bands.map((band) => {
17427
+ const minimumPrice = bestBid.price * (1 - band / 1e4);
17428
+ return bids.filter(({ price }) => price >= minimumPrice).reduce((sum, { amount }) => sum + amount, 0);
17429
+ });
17430
+ const askDepth = bands.map((band) => {
17431
+ const maximumPrice = bestAsk.price * (1 + band / 1e4);
17432
+ return asks.filter(({ price }) => price <= maximumPrice).reduce((sum, { amount }) => sum + amount, 0);
17433
+ });
17434
+ const summaryRow = checksumRow({
17435
+ ...common,
17436
+ best_bid: bestBid.price,
17437
+ best_ask: bestAsk.price,
17438
+ best_bid_amount: bestBid.amount,
17439
+ best_ask_amount: bestAsk.amount,
17440
+ mid_price: midPrice,
17441
+ spread,
17442
+ spread_bps: spreadBps,
17443
+ staleness_ms: receivedTimeMs - eventTimeMs,
17444
+ bid_level_count: bids.length,
17445
+ ask_level_count: asks.length,
17446
+ measurement_bands_bps: bands,
17447
+ bid_depth_by_band: bidDepth,
17448
+ ask_depth_by_band: askDepth
17449
+ });
17450
+ return {
17451
+ snapshotId,
17452
+ levels,
17453
+ summary: {
17454
+ table: "market_data.cex_order_book_depth_summary",
17455
+ row: summaryRow
17456
+ }
17457
+ };
17458
+ }
17459
+
17460
+ // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
17461
+ var CRYPTOHFTDATA_ADAPTER_VERSION = "cryptohftdata-orderbook/v2";
17462
+ var CRYPTOHFTDATA_API_URL = "https://api.cryptohftdata.com";
17463
+ var CRYPTOHFTDATA_HISTORY_START_MS = Date.UTC(2025, 5, 28);
17464
+ var HOUR_MS = 60 * 60 * 1000;
17465
+ var CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE = Object.freeze({
17466
+ profileId: "cryptohftdata/binance_spot/BTCUSDT/v1",
17467
+ exchange: "binance",
17468
+ tradingPair: "BTC-USDT",
17469
+ sourceSymbol: "BTCUSDT",
17470
+ marketType: "spot",
17471
+ providerExchangeId: "binance_spot",
17472
+ historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
17473
+ maxDepth: 500,
17474
+ eventTimeUnit: "milliseconds",
17475
+ receivedTimeUnit: "nanoseconds",
17476
+ snapshotGrouping: "event_time_last_update_id_object",
17477
+ sequenceSemantics: "binance_u_U_pu",
17478
+ constructionModes: ["sampled_top_n_snapshot"],
17479
+ sourcePolicies: ["authoritative_window"]
17480
+ });
17481
+ var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
17482
+ profileId: "cryptohftdata/okx_spot/ARB-USDT/v1",
17483
+ exchange: "okx",
17484
+ tradingPair: "ARB-USDT",
17485
+ sourceSymbol: "ARB-USDT",
17486
+ marketType: "spot",
17487
+ providerExchangeId: "okx_spot",
17488
+ historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
17489
+ maxDepth: 400,
17490
+ eventTimeUnit: "milliseconds",
17491
+ receivedTimeUnit: "nanoseconds",
17492
+ snapshotGrouping: "event_time_final_update_id_object",
17493
+ sequenceSemantics: "okx_seq_id_prev_seq_id",
17494
+ constructionModes: ["sampled_top_n_snapshot"],
17495
+ sourcePolicies: ["authoritative_window", "fill_gaps"]
17496
+ });
17497
+ var CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE = Object.freeze({
17498
+ profileId: "cryptohftdata/okx_spot/ARB-USDC/v1",
17499
+ exchange: "okx",
17500
+ tradingPair: "ARB-USDC",
17501
+ sourceSymbol: "ARB-USDC",
17502
+ marketType: "spot",
17503
+ providerExchangeId: "okx_spot",
17504
+ historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
17505
+ maxDepth: 400,
17506
+ eventTimeUnit: "milliseconds",
17507
+ receivedTimeUnit: "nanoseconds",
17508
+ snapshotGrouping: "event_time_final_update_id_object",
17509
+ sequenceSemantics: "okx_seq_id_prev_seq_id",
17510
+ constructionModes: ["sampled_top_n_snapshot"],
17511
+ sourcePolicies: ["authoritative_window", "fill_gaps"]
17512
+ });
17513
+
17514
+ class CryptoHftDataError extends Error {
17515
+ reason;
17516
+ constructor(reason) {
17517
+ super(`CryptoHFTData backfill failed: ${reason}`);
17518
+ this.reason = reason;
17519
+ this.name = "CryptoHftDataError";
17520
+ }
17521
+ }
17522
+ function cryptoHftDataCapabilityFor(request2, profiles = []) {
17523
+ if (request2.providerPolicy.provider !== "cryptohftdata" || request2.scope.feed !== "ORDERBOOK" || request2.scope.exchange.trim().toLowerCase() === "mexc") {
17524
+ return;
17525
+ }
17526
+ const profile = profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.constructionModes.includes(request2.constructionMode) && candidate.sourcePolicies.includes(request2.sourcePolicy) && request2.window.startTimeMs >= candidate.historyStartMs && request2.depth <= candidate.maxDepth);
17527
+ if (!profile)
17528
+ return;
17529
+ return {
17530
+ provider: "cryptohftdata",
17531
+ adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
17532
+ providerExchangeId: profile.providerExchangeId,
17533
+ resolvedSymbol: profile.sourceSymbol
17534
+ };
17535
+ }
17536
+ function utcHourPath(hourMs) {
17537
+ const date = new Date(hourMs);
17538
+ return {
17539
+ date: `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String(date.getUTCDate()).padStart(2, "0")}`,
17540
+ hour: String(date.getUTCHours()).padStart(2, "0")
17541
+ };
17542
+ }
17543
+ function enumerateCryptoHftDataObjects(request2, providerExchangeId, resolvedSymbol) {
17544
+ if (request2.sourcePolicy === "fill_gaps") {
17545
+ const hours = new Set;
17546
+ for (const targetTimeMs of request2.requiredClockTargetsMs) {
17547
+ const firstHour2 = Math.floor(Math.max(0, targetTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
17548
+ const lastHour = Math.floor(targetTimeMs / HOUR_MS) * HOUR_MS;
17549
+ for (let hourMs = firstHour2;hourMs <= lastHour; hourMs += HOUR_MS) {
17550
+ hours.add(hourMs);
17551
+ }
17552
+ }
17553
+ return [...hours].sort((left, right) => left - right).map((hourMs) => {
17554
+ const utc = utcHourPath(hourMs);
17555
+ return `${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`;
17556
+ });
17557
+ }
17558
+ const firstHour = Math.floor((request2.window.startTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
17559
+ const objects = [];
17560
+ for (let hourMs = Math.max(0, firstHour);hourMs < request2.window.endTimeMs; hourMs += HOUR_MS) {
17561
+ const utc = utcHourPath(hourMs);
17562
+ objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
17563
+ }
17564
+ return objects;
17565
+ }
17566
+ function initialArchiveCoversTarget(request2, targetTimeMs) {
17567
+ return (request2.initialSelection?.support_anchors ?? []).some((anchor) => {
17568
+ const sourceTimeMs = Date.parse(anchor.source_time);
17569
+ return Number.isSafeInteger(sourceTimeMs) && sourceTimeMs <= targetTimeMs && targetTimeMs - sourceTimeMs <= request2.maxPriorAsOfLagMs;
17570
+ });
17571
+ }
17572
+ function providerAcquisitionRequest(request2) {
17573
+ if (request2.sourcePolicy !== "fill_gaps")
17574
+ return request2;
17575
+ const requiredClockTargetsMs = request2.requiredClockTargetsMs.filter((targetTimeMs) => !initialArchiveCoversTarget(request2, targetTimeMs));
17576
+ if (requiredClockTargetsMs.length === 0) {
17577
+ throw new CryptoHftDataError("fill_gaps_has_no_uncovered_clock_targets");
17578
+ }
17579
+ return { ...request2, requiredClockTargetsMs };
17580
+ }
17581
+ function unsignedString(value, field, required = false) {
17582
+ if (value === null || value === undefined) {
17583
+ if (required)
17584
+ throw new CryptoHftDataError(`schema_${field}_missing`);
17585
+ return;
17586
+ }
17587
+ const rendered = String(value);
17588
+ if (!/^\d+$/.test(rendered)) {
17589
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
17590
+ }
17591
+ const parsed = BigInt(rendered);
17592
+ if (parsed > 18446744073709551615n) {
17593
+ throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
17594
+ }
17595
+ return parsed.toString(10);
17596
+ }
17597
+ function timestampMs2(value, field) {
17598
+ const rendered = String(value);
17599
+ if (!/^\d+$/.test(rendered)) {
17600
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
17601
+ }
17602
+ const raw = BigInt(rendered);
17603
+ const milliseconds = field === "received_time" ? raw / 1000000n : raw;
17604
+ if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
17605
+ throw new CryptoHftDataError(`schema_${field}_unsafe`);
17606
+ }
17607
+ return Number(milliseconds);
17608
+ }
17609
+ function decimal(value, field, allowZero) {
17610
+ if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
17611
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
17612
+ }
17613
+ const parsed = Number(value);
17614
+ if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
17615
+ throw new CryptoHftDataError(`schema_${field}_invalid`);
17616
+ }
17617
+ return parsed;
17618
+ }
17619
+ function validatedRow(request2, row) {
17620
+ if (row.symbol !== request2.scope.sourceSymbol) {
17621
+ throw new CryptoHftDataError("schema_symbol_mismatch");
17622
+ }
17623
+ if (row.event_type !== "snapshot" && row.event_type !== "update") {
17624
+ throw new CryptoHftDataError("schema_event_type_invalid");
17625
+ }
17626
+ if (row.side !== "bid" && row.side !== "ask") {
17627
+ throw new CryptoHftDataError("schema_side_invalid");
17628
+ }
17629
+ decimal(row.price, "price", false);
17630
+ decimal(row.quantity, "quantity", true);
17631
+ if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
17632
+ throw new CryptoHftDataError("schema_object_checksum_invalid");
17633
+ }
17634
+ const eventTimeMs = timestampMs2(row.event_time, "event_time");
17635
+ const receivedTimeMs = timestampMs2(row.received_time, "received_time");
17636
+ if (receivedTimeMs < eventTimeMs) {
17637
+ throw new CryptoHftDataError("received_time_precedes_event_time");
17638
+ }
17639
+ return { ...row, eventTimeMs, receivedTimeMs };
17640
+ }
17641
+ function groupKey(row, profile) {
17642
+ const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
17643
+ return [
17644
+ row.event_type,
17645
+ row.eventTimeMs,
17646
+ sequence,
17647
+ row.dataset_object_identity
17648
+ ].join("\x00");
17649
+ }
17650
+ function absent(value) {
17651
+ return value === null || value === undefined;
17652
+ }
17653
+ function snapshotSequence(row, profile) {
17654
+ if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
17655
+ if (String(row.last_update_id) !== "-1") {
17656
+ throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
18144
17657
  }
18145
- };
17658
+ if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
17659
+ throw new CryptoHftDataError("ambiguous_snapshot_group");
17660
+ }
17661
+ return unsignedString(row.final_update_id, "final_update_id", true);
17662
+ }
17663
+ return unsignedString(row.last_update_id, "last_update_id", true);
18146
17664
  }
18147
-
18148
- // src/helpers/market-data-vendor-backfill/cryptohftdata.ts
18149
- var CRYPTOHFTDATA_ADAPTER_VERSION = "cryptohftdata-orderbook/v2";
18150
- var CRYPTOHFTDATA_API_URL = "https://api.cryptohftdata.com";
18151
- var CRYPTOHFTDATA_HISTORY_START_MS = Date.UTC(2025, 5, 28);
18152
- var HOUR_MS = 60 * 60 * 1000;
18153
- var CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE = Object.freeze({
18154
- profileId: "cryptohftdata/binance_spot/BTCUSDT/v1",
18155
- exchange: "binance",
18156
- tradingPair: "BTC-USDT",
18157
- sourceSymbol: "BTCUSDT",
18158
- marketType: "spot",
18159
- providerExchangeId: "binance_spot",
18160
- historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
18161
- maxDepth: 500,
18162
- eventTimeUnit: "milliseconds",
18163
- receivedTimeUnit: "nanoseconds",
18164
- snapshotGrouping: "event_time_last_update_id_object",
18165
- sequenceSemantics: "binance_u_U_pu",
18166
- constructionModes: ["sampled_top_n_snapshot"],
18167
- sourcePolicies: ["authoritative_window"]
18168
- });
18169
- var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
18170
- profileId: "cryptohftdata/okx_spot/ARB-USDT/v1",
18171
- exchange: "okx",
18172
- tradingPair: "ARB-USDT",
18173
- sourceSymbol: "ARB-USDT",
18174
- marketType: "spot",
18175
- providerExchangeId: "okx_spot",
18176
- historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
18177
- maxDepth: 400,
18178
- eventTimeUnit: "milliseconds",
18179
- receivedTimeUnit: "nanoseconds",
18180
- snapshotGrouping: "event_time_final_update_id_object",
18181
- sequenceSemantics: "okx_seq_id_prev_seq_id",
18182
- constructionModes: ["sampled_top_n_snapshot"],
18183
- sourcePolicies: ["authoritative_window"]
18184
- });
18185
-
18186
- class CryptoHftDataError extends Error {
18187
- reason;
18188
- constructor(reason) {
18189
- super(`CryptoHFTData backfill failed: ${reason}`);
18190
- this.reason = reason;
18191
- this.name = "CryptoHftDataError";
17665
+ function sortedSide(levels, side) {
17666
+ return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
17667
+ }
17668
+ function applyRows(state, rows) {
17669
+ for (const row of rows) {
17670
+ const levels = row.side === "bid" ? state.bids : state.asks;
17671
+ const quantity = decimal(row.quantity, "quantity", true);
17672
+ if (quantity === 0)
17673
+ levels.delete(row.price);
17674
+ else
17675
+ levels.set(row.price, quantity);
18192
17676
  }
18193
17677
  }
18194
- function cryptoHftDataCapabilityFor(request2, profiles = []) {
18195
- if (request2.providerPolicy.provider !== "cryptohftdata" || request2.scope.feed !== "ORDERBOOK" || request2.scope.exchange.trim().toLowerCase() === "mexc") {
18196
- return;
17678
+ function reconstructCryptoHftDataOrderBooks(request2, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
17679
+ const rows = inputRows.map((row, index) => ({
17680
+ ...validatedRow(request2, row),
17681
+ originalIndex: index
17682
+ })).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
17683
+ const groups = [];
17684
+ for (const row of rows) {
17685
+ const current = groups.at(-1);
17686
+ if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
17687
+ groups.push([row]);
17688
+ } else {
17689
+ current.push(row);
17690
+ }
17691
+ }
17692
+ const earliestTargetTimeMs = Math.min(...request2.requiredClockTargetsMs);
17693
+ const latestTargetTimeMs = Math.max(...request2.requiredClockTargetsMs);
17694
+ const anchorIndex = groups.findIndex((group) => {
17695
+ const first = group[0];
17696
+ return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
17697
+ });
17698
+ if (anchorIndex < 0) {
17699
+ throw new CryptoHftDataError("update_before_snapshot");
17700
+ }
17701
+ let state;
17702
+ let previousFinalUpdateId;
17703
+ const states = [];
17704
+ for (const group of groups.slice(anchorIndex)) {
17705
+ const first = group[0];
17706
+ if (first.eventTimeMs > latestTargetTimeMs)
17707
+ break;
17708
+ if (first.event_type === "snapshot") {
17709
+ const sequence = snapshotSequence(first, profile);
17710
+ for (const row of group) {
17711
+ if (snapshotSequence(row, profile) !== sequence) {
17712
+ throw new CryptoHftDataError("ambiguous_snapshot_group");
17713
+ }
17714
+ }
17715
+ if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
17716
+ throw new CryptoHftDataError("snapshot_sequence_regression");
17717
+ }
17718
+ state = {
17719
+ bids: new Map,
17720
+ asks: new Map,
17721
+ sequence,
17722
+ sourceTimeMs: first.eventTimeMs,
17723
+ receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
17724
+ datasetObjectIdentity: first.dataset_object_identity,
17725
+ datasetObjectChecksum: first.dataset_object_checksum
17726
+ };
17727
+ applyRows(state, group);
17728
+ previousFinalUpdateId = BigInt(sequence);
17729
+ } else {
17730
+ if (!state || previousFinalUpdateId === undefined) {
17731
+ throw new CryptoHftDataError("update_before_snapshot");
17732
+ }
17733
+ const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
17734
+ if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
17735
+ if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
17736
+ throw new CryptoHftDataError("ambiguous_update_group");
17737
+ }
17738
+ const previous = unsignedString(first.last_update_id, "last_update_id", true);
17739
+ for (const row of group) {
17740
+ if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
17741
+ throw new CryptoHftDataError("ambiguous_update_group");
17742
+ }
17743
+ }
17744
+ if (BigInt(previous) !== previousFinalUpdateId) {
17745
+ throw new CryptoHftDataError("update_chain_gap");
17746
+ }
17747
+ } else {
17748
+ const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
17749
+ const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
17750
+ for (const row of group) {
17751
+ if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
17752
+ throw new CryptoHftDataError("ambiguous_update_group");
17753
+ }
17754
+ }
17755
+ const expected = previousFinalUpdateId + 1n;
17756
+ if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
17757
+ throw new CryptoHftDataError("update_chain_gap");
17758
+ }
17759
+ }
17760
+ applyRows(state, group);
17761
+ state.sequence = finalUpdate.toString();
17762
+ state.sourceTimeMs = first.eventTimeMs;
17763
+ state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
17764
+ state.datasetObjectIdentity = first.dataset_object_identity;
17765
+ state.datasetObjectChecksum = first.dataset_object_checksum;
17766
+ previousFinalUpdateId = finalUpdate;
17767
+ }
17768
+ if (state) {
17769
+ states.push({
17770
+ ...state,
17771
+ bids: new Map(state.bids),
17772
+ asks: new Map(state.asks)
17773
+ });
17774
+ }
17775
+ }
17776
+ const samples = [];
17777
+ for (const targetTimeMs of request2.requiredClockTargetsMs) {
17778
+ let prior;
17779
+ for (const candidate of states) {
17780
+ if (candidate.sourceTimeMs > targetTimeMs)
17781
+ break;
17782
+ prior = candidate;
17783
+ }
17784
+ if (!prior || targetTimeMs - prior.sourceTimeMs > request2.maxPriorAsOfLagMs) {
17785
+ throw new CryptoHftDataError("required_clock_coverage_insufficient");
17786
+ }
17787
+ const bids = sortedSide(prior.bids, "bid").slice(0, request2.depth);
17788
+ const asks = sortedSide(prior.asks, "ask").slice(0, request2.depth);
17789
+ if (bids.length === 0 || asks.length === 0) {
17790
+ throw new CryptoHftDataError("book_side_missing");
17791
+ }
17792
+ if (bids[0]?.[0] >= asks[0]?.[0]) {
17793
+ throw new CryptoHftDataError("book_crossed_or_locked");
17794
+ }
17795
+ samples.push({
17796
+ targetTimeMs,
17797
+ sourceTimeMs: prior.sourceTimeMs,
17798
+ receivedTimeMs: prior.receivedTimeMs,
17799
+ sequence: prior.sequence,
17800
+ bids,
17801
+ asks,
17802
+ datasetObjectIdentity: prior.datasetObjectIdentity,
17803
+ datasetObjectChecksum: prior.datasetObjectChecksum
17804
+ });
18197
17805
  }
18198
- const profile = profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.constructionModes.includes(request2.constructionMode) && candidate.sourcePolicies.includes(request2.sourcePolicy) && request2.window.startTimeMs >= candidate.historyStartMs && request2.depth <= candidate.maxDepth);
18199
- if (!profile)
18200
- return;
18201
- return {
18202
- provider: "cryptohftdata",
18203
- adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
18204
- providerExchangeId: profile.providerExchangeId,
18205
- resolvedSymbol: profile.sourceSymbol
18206
- };
17806
+ return samples;
18207
17807
  }
18208
- function utcHourPath(hourMs) {
18209
- const date = new Date(hourMs);
17808
+ function sha256Bytes(bytes) {
17809
+ return createHash3("sha256").update(bytes).digest("hex");
17810
+ }
17811
+ async function decodeCryptoHftParquetZstd(bytes) {
17812
+ const parquet = decompress(bytes);
17813
+ const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
17814
+ return parquetReadObjects({ file: buffer });
17815
+ }
17816
+ function parsedDatasetRow(value, object) {
18210
17817
  return {
18211
- date: `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String(date.getUTCDate()).padStart(2, "0")}`,
18212
- hour: String(date.getUTCHours()).padStart(2, "0")
17818
+ received_time: value.received_time,
17819
+ event_time: value.event_time,
17820
+ transaction_time: value.transaction_time,
17821
+ symbol: value.symbol,
17822
+ event_type: value.event_type,
17823
+ first_update_id: value.first_update_id,
17824
+ final_update_id: value.final_update_id,
17825
+ prev_final_update_id: value.prev_final_update_id,
17826
+ last_update_id: value.last_update_id,
17827
+ side: value.side,
17828
+ price: value.price,
17829
+ quantity: value.quantity,
17830
+ order_count: value.order_count,
17831
+ dataset_object_identity: object.identity,
17832
+ dataset_object_checksum: object.checksum
18213
17833
  };
18214
17834
  }
18215
- function enumerateCryptoHftDataObjects(request2, providerExchangeId, resolvedSymbol) {
18216
- const firstHour = Math.floor((request2.window.startTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
18217
- const objects = [];
18218
- for (let hourMs = Math.max(0, firstHour);hourMs < request2.window.endTimeMs; hourMs += HOUR_MS) {
18219
- const utc = utcHourPath(hourMs);
18220
- objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
17835
+ async function readBoundedObject(response, remainingBytes) {
17836
+ if (!response.body)
17837
+ return new Uint8Array;
17838
+ const reader = response.body.getReader();
17839
+ const chunks = [];
17840
+ let total = 0;
17841
+ while (true) {
17842
+ const { done, value } = await reader.read();
17843
+ if (done)
17844
+ break;
17845
+ total += value.byteLength;
17846
+ if (total > remainingBytes) {
17847
+ await reader.cancel();
17848
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
17849
+ }
17850
+ chunks.push(value);
18221
17851
  }
18222
- return objects;
17852
+ const bytes = new Uint8Array(total);
17853
+ let offset = 0;
17854
+ for (const chunk of chunks) {
17855
+ bytes.set(chunk, offset);
17856
+ offset += chunk.byteLength;
17857
+ }
17858
+ return bytes;
18223
17859
  }
18224
- function unsignedString(value, field, required = false) {
18225
- if (value === null || value === undefined) {
18226
- if (required)
18227
- throw new CryptoHftDataError(`schema_${field}_missing`);
18228
- return;
17860
+
17861
+ class CryptoHftDataAdapter {
17862
+ baseUrl;
17863
+ request;
17864
+ nowMs;
17865
+ decode;
17866
+ profiles;
17867
+ constructor(options = {}) {
17868
+ this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
17869
+ this.request = options.fetch ?? fetch;
17870
+ this.nowMs = options.nowMs ?? Date.now;
17871
+ this.decode = options.decode ?? decodeCryptoHftParquetZstd;
17872
+ this.profiles = options.profiles ?? [];
18229
17873
  }
18230
- const rendered = String(value);
18231
- if (!/^\d+$/.test(rendered)) {
18232
- throw new CryptoHftDataError(`schema_${field}_invalid`);
17874
+ capabilityFor(request2) {
17875
+ return cryptoHftDataCapabilityFor(request2, this.profiles);
18233
17876
  }
18234
- const parsed = BigInt(rendered);
18235
- if (parsed > 18446744073709551615n) {
18236
- throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
17877
+ async discoverSymbols(providerExchangeId) {
17878
+ if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
17879
+ throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
17880
+ }
17881
+ const endpoint = new URL("/symbols", this.baseUrl);
17882
+ endpoint.searchParams.set("exchange", providerExchangeId);
17883
+ endpoint.searchParams.set("data_type", "orderbook");
17884
+ let response;
17885
+ try {
17886
+ response = await this.request(endpoint);
17887
+ } catch {
17888
+ throw new CryptoHftDataError("symbol_discovery_failed");
17889
+ }
17890
+ if (!response.ok) {
17891
+ throw new CryptoHftDataError("symbol_discovery_failed");
17892
+ }
17893
+ let body;
17894
+ try {
17895
+ body = await response.json();
17896
+ } catch {
17897
+ throw new CryptoHftDataError("symbol_discovery_response_invalid");
17898
+ }
17899
+ const record = body;
17900
+ if (!body || typeof body !== "object" || Array.isArray(body) || String(record.exchange).toLowerCase() !== providerExchangeId || String(record.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record.symbols) || !record.symbols.every((symbol) => typeof symbol === "string" && /^[A-Z0-9_-]+$/.test(symbol))) {
17901
+ throw new CryptoHftDataError("symbol_discovery_response_invalid");
17902
+ }
17903
+ return [...new Set(record.symbols)].sort();
18237
17904
  }
18238
- return parsed.toString(10);
18239
- }
18240
- function timestampMs2(value, field) {
18241
- const rendered = String(value);
18242
- if (!/^\d+$/.test(rendered)) {
18243
- throw new CryptoHftDataError(`schema_${field}_invalid`);
17905
+ async acquire(request2, capability, credential) {
17906
+ const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
17907
+ if (typeof apiKey !== "string" || apiKey.length === 0) {
17908
+ throw new CryptoHftDataError("credentials_invalid");
17909
+ }
17910
+ const paths = enumerateCryptoHftDataObjects(request2, capability.providerExchangeId, capability.resolvedSymbol);
17911
+ if (paths.length > request2.budgets.maxFiles) {
17912
+ throw new CryptoHftDataError("budget_max_files_exceeded");
17913
+ }
17914
+ const started = this.nowMs();
17915
+ const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
17916
+ method: "POST",
17917
+ headers: { "content-type": "application/json", "X-API-Key": apiKey }
17918
+ });
17919
+ if (!tokenResponse.ok)
17920
+ throw new CryptoHftDataError("jwt_issuance_failed");
17921
+ const tokenBody = await tokenResponse.json();
17922
+ if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
17923
+ throw new CryptoHftDataError("jwt_response_invalid");
17924
+ }
17925
+ let totalBytes = 0;
17926
+ let totalRows = 0;
17927
+ const objects = [];
17928
+ const rows = [];
17929
+ for (const path of paths) {
17930
+ if (this.nowMs() - started > request2.budgets.maxDurationMs) {
17931
+ throw new CryptoHftDataError("budget_max_duration_exceeded");
17932
+ }
17933
+ const endpoint = new URL("/download", this.baseUrl);
17934
+ endpoint.searchParams.set("file", path);
17935
+ const response = await this.request(endpoint, {
17936
+ headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
17937
+ });
17938
+ if (!response.ok)
17939
+ throw new CryptoHftDataError("object_download_failed");
17940
+ const declaredBytes = Number(response.headers.get("content-length"));
17941
+ if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request2.budgets.maxBytes) {
17942
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
17943
+ }
17944
+ const bytes = await readBoundedObject(response, request2.budgets.maxBytes - totalBytes);
17945
+ totalBytes += bytes.byteLength;
17946
+ if (totalBytes > request2.budgets.maxBytes) {
17947
+ throw new CryptoHftDataError("budget_max_bytes_exceeded");
17948
+ }
17949
+ const decoded = await this.decode(bytes);
17950
+ totalRows += decoded.length;
17951
+ if (totalRows > request2.budgets.maxRows) {
17952
+ throw new CryptoHftDataError("budget_max_rows_exceeded");
17953
+ }
17954
+ const object = {
17955
+ identity: path,
17956
+ checksum: sha256Bytes(bytes),
17957
+ bytes: bytes.byteLength,
17958
+ rows: decoded.length
17959
+ };
17960
+ objects.push(object);
17961
+ for (const decodedRow of decoded) {
17962
+ const parsed = parsedDatasetRow(decodedRow, object);
17963
+ validatedRow(request2, parsed);
17964
+ rows.push(parsed);
17965
+ }
17966
+ }
17967
+ if (this.nowMs() - started > request2.budgets.maxDurationMs) {
17968
+ throw new CryptoHftDataError("budget_max_duration_exceeded");
17969
+ }
17970
+ return {
17971
+ objects,
17972
+ rows,
17973
+ vendorSemanticDigest: sha256Canonical(rows)
17974
+ };
18244
17975
  }
18245
- const raw = BigInt(rendered);
18246
- const milliseconds = field === "received_time" ? raw / 1000000n : raw;
18247
- if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
18248
- throw new CryptoHftDataError(`schema_${field}_unsafe`);
17976
+ async normalize(request2, capability, dataset, captureBundleId2) {
17977
+ const profile = this.profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
17978
+ if (!profile) {
17979
+ throw new CryptoHftDataError("profile_semantics_unavailable");
17980
+ }
17981
+ const samples = reconstructCryptoHftDataOrderBooks(request2, dataset.rows, profile);
17982
+ const context2 = {
17983
+ source: EXTERNAL_BACKFILL_SOURCE,
17984
+ deploymentId: "market-data-vendor-backfill",
17985
+ captureBundleId: captureBundleId2,
17986
+ exchange: request2.scope.exchange,
17987
+ symbol: request2.scope.tradingPair,
17988
+ tradingPair: request2.scope.tradingPair,
17989
+ sourceSymbol: request2.scope.sourceSymbol,
17990
+ assetType: request2.scope.marketType,
17991
+ feed: "ORDERBOOK",
17992
+ provider: "cryptohftdata",
17993
+ sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
17994
+ schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
17995
+ checksumAlgorithm: CHECKSUM_ALGORITHM,
17996
+ provenanceComplete: true
17997
+ };
17998
+ const rows = samples.flatMap((sample) => {
17999
+ const rawCapture = {
18000
+ rawCaptureId: sha256Canonical({
18001
+ capture_bundle_id: captureBundleId2,
18002
+ dataset_object_identity: sample.datasetObjectIdentity,
18003
+ dataset_object_checksum: sample.datasetObjectChecksum,
18004
+ source_time_ms: sample.sourceTimeMs,
18005
+ target_time_ms: sample.targetTimeMs
18006
+ }),
18007
+ rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
18008
+ rawChecksum: sample.datasetObjectChecksum,
18009
+ redactedPayload: {
18010
+ dataset_object_identity: sample.datasetObjectIdentity,
18011
+ dataset_object_checksum: sample.datasetObjectChecksum
18012
+ },
18013
+ eventTimeMs: sample.sourceTimeMs,
18014
+ receivedTimeMs: sample.receivedTimeMs,
18015
+ checksumAlgorithm: CHECKSUM_ALGORITHM
18016
+ };
18017
+ const canonical = buildCanonicalOrderBookRows({
18018
+ context: context2,
18019
+ rawCapture,
18020
+ depthLimit: request2.depth,
18021
+ constructionMode: request2.constructionMode,
18022
+ snapshot: {
18023
+ bids: sample.bids,
18024
+ asks: sample.asks,
18025
+ timestamp: sample.sourceTimeMs,
18026
+ receivedTimestamp: sample.receivedTimeMs,
18027
+ exchange: request2.scope.exchange,
18028
+ symbol: request2.scope.tradingPair,
18029
+ depthLimit: request2.depth,
18030
+ sequence: sample.sequence
18031
+ }
18032
+ });
18033
+ return [...canonical.levels, canonical.summary];
18034
+ });
18035
+ return {
18036
+ captureBundleId: captureBundleId2,
18037
+ objects: dataset.objects,
18038
+ rows,
18039
+ vendorSemanticDigest: dataset.vendorSemanticDigest,
18040
+ canonicalSemanticDigest: semanticDigest(rows)
18041
+ };
18249
18042
  }
18250
- return Number(milliseconds);
18251
18043
  }
18252
- function decimal(value, field, allowZero) {
18253
- if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
18254
- throw new CryptoHftDataError(`schema_${field}_invalid`);
18255
- }
18256
- const parsed = Number(value);
18257
- if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
18258
- throw new CryptoHftDataError(`schema_${field}_invalid`);
18044
+
18045
+ // src/helpers/market-data-vendor-backfill/qualification.ts
18046
+ var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
18047
+ var SHA256 = /^[0-9a-f]{64}$/;
18048
+ var REASON = /^[a-z][a-z0-9_]{0,127}$/;
18049
+ var EVENT_FIELDS = new Set([
18050
+ "qualification_event_id",
18051
+ "capture_bundle_id",
18052
+ "state",
18053
+ "receipt_id",
18054
+ "promotion_identity_sha256",
18055
+ "window",
18056
+ "event_at",
18057
+ "reason_code"
18058
+ ]);
18059
+ function deterministicUuid(value) {
18060
+ const digest = jcsSha256(value).slice(0, 32).split("");
18061
+ digest[12] = "5";
18062
+ digest[16] = "8";
18063
+ const hex = digest.join("");
18064
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
18065
+ }
18066
+ function semanticEvent(event) {
18067
+ const { qualification_event_id: _eventId, ...semantic } = event;
18068
+ return semantic;
18069
+ }
18070
+ function assertTimestamp(value, field) {
18071
+ const parsed = Date.parse(value);
18072
+ if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
18073
+ throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
18259
18074
  }
18260
18075
  return parsed;
18261
18076
  }
18262
- function validatedRow(request2, row) {
18263
- if (row.symbol !== request2.scope.sourceSymbol) {
18264
- throw new CryptoHftDataError("schema_symbol_mismatch");
18265
- }
18266
- if (row.event_type !== "snapshot" && row.event_type !== "update") {
18267
- throw new CryptoHftDataError("schema_event_type_invalid");
18077
+ function parseQualificationEvent(value) {
18078
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
18079
+ throw new Error("qualification event must be an object");
18268
18080
  }
18269
- if (row.side !== "bid" && row.side !== "ask") {
18270
- throw new CryptoHftDataError("schema_side_invalid");
18081
+ const event = value;
18082
+ if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
18083
+ throw new Error("qualification event fields are invalid");
18271
18084
  }
18272
- decimal(row.price, "price", false);
18273
- decimal(row.quantity, "quantity", true);
18274
- if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
18275
- throw new CryptoHftDataError("schema_object_checksum_invalid");
18085
+ if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
18086
+ throw new Error("qualification window must be increasing");
18276
18087
  }
18277
- const eventTimeMs = timestampMs2(row.event_time, "event_time");
18278
- const receivedTimeMs = timestampMs2(row.received_time, "received_time");
18279
- if (receivedTimeMs < eventTimeMs) {
18280
- throw new CryptoHftDataError("received_time_precedes_event_time");
18088
+ assertTimestamp(event.event_at, "event_at");
18089
+ if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
18090
+ throw new Error("qualification_event_id does not match event content");
18281
18091
  }
18282
- return { ...row, eventTimeMs, receivedTimeMs };
18283
- }
18284
- function groupKey(row, profile) {
18285
- const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
18286
- return [
18287
- row.event_type,
18288
- row.eventTimeMs,
18289
- sequence,
18290
- row.dataset_object_identity
18291
- ].join("\x00");
18092
+ return event;
18292
18093
  }
18293
- function absent(value) {
18294
- return value === null || value === undefined;
18094
+ function finalizeQualificationEvent(event) {
18095
+ const semantic = semanticEvent(event);
18096
+ return parseQualificationEvent({
18097
+ ...semantic,
18098
+ qualification_event_id: deterministicUuid(semantic)
18099
+ });
18295
18100
  }
18296
- function snapshotSequence(row, profile) {
18297
- if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
18298
- if (String(row.last_update_id) !== "-1") {
18299
- throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
18300
- }
18301
- if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
18302
- throw new CryptoHftDataError("ambiguous_snapshot_group");
18101
+ function qualificationEventToArchiveRow(eventInput) {
18102
+ const event = parseQualificationEvent(eventInput);
18103
+ return {
18104
+ table: "market_data.cex_order_book_capture_qualifications",
18105
+ row: {
18106
+ source: "external_backfill",
18107
+ capture_origin: "vendor_historical_backfill",
18108
+ source_mode: "vendor_historical_backfill_v1",
18109
+ deployment_id: "market-data-vendor-backfill",
18110
+ qualification_event_id: event.qualification_event_id,
18111
+ capture_bundle_id: event.capture_bundle_id,
18112
+ state: event.state,
18113
+ receipt_id: event.receipt_id,
18114
+ promotion_identity_sha256: event.promotion_identity_sha256,
18115
+ window_start_ms: Date.parse(event.window.start_at),
18116
+ window_end_ms: Date.parse(event.window.end_at),
18117
+ event_at_ms: Date.parse(event.event_at),
18118
+ reason_code: event.reason_code,
18119
+ event_json: jcsCanonicalize(event)
18303
18120
  }
18304
- return unsignedString(row.final_update_id, "final_update_id", true);
18305
- }
18306
- return unsignedString(row.last_update_id, "last_update_id", true);
18307
- }
18308
- function sortedSide(levels, side) {
18309
- return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
18310
- }
18311
- function applyRows(state, rows) {
18312
- for (const row of rows) {
18313
- const levels = row.side === "bid" ? state.bids : state.asks;
18314
- const quantity = decimal(row.quantity, "quantity", true);
18315
- if (quantity === 0)
18316
- levels.delete(row.price);
18317
- else
18318
- levels.set(row.price, quantity);
18319
- }
18121
+ };
18320
18122
  }
18321
- function reconstructCryptoHftDataOrderBooks(request2, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
18322
- const rows = inputRows.map((row, index) => ({
18323
- ...validatedRow(request2, row),
18324
- originalIndex: index
18325
- })).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
18326
- const groups = [];
18327
- for (const row of rows) {
18328
- const current = groups.at(-1);
18329
- if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
18330
- groups.push([row]);
18331
- } else {
18332
- current.push(row);
18123
+
18124
+ // src/helpers/market-data-vendor-backfill/core.ts
18125
+ function createMarketDataVendorBackfillDependencies(dependencies) {
18126
+ for (const [name, value] of Object.entries({
18127
+ archive: dependencies.archive,
18128
+ providers: dependencies.providers,
18129
+ credentials: dependencies.credentials,
18130
+ forwarder: dependencies.forwarder,
18131
+ clock: dependencies.clock
18132
+ })) {
18133
+ if (!value || typeof value !== "object") {
18134
+ throw new TypeError(`Backfill dependency ${name} is required`);
18333
18135
  }
18334
18136
  }
18335
- const earliestTargetTimeMs = Math.min(...request2.requiredClockTargetsMs);
18336
- const latestTargetTimeMs = Math.max(...request2.requiredClockTargetsMs);
18337
- const anchorIndex = groups.findIndex((group) => {
18338
- const first = group[0];
18339
- return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
18340
- });
18341
- if (anchorIndex < 0) {
18342
- throw new CryptoHftDataError("update_before_snapshot");
18343
- }
18344
- let state;
18345
- let previousFinalUpdateId;
18346
- const states = [];
18347
- for (const group of groups.slice(anchorIndex)) {
18348
- const first = group[0];
18349
- if (first.eventTimeMs > latestTargetTimeMs)
18350
- break;
18351
- if (first.event_type === "snapshot") {
18352
- const sequence = snapshotSequence(first, profile);
18353
- for (const row of group) {
18354
- if (snapshotSequence(row, profile) !== sequence) {
18355
- throw new CryptoHftDataError("ambiguous_snapshot_group");
18356
- }
18357
- }
18358
- if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
18359
- throw new CryptoHftDataError("snapshot_sequence_regression");
18360
- }
18361
- state = {
18362
- bids: new Map,
18363
- asks: new Map,
18364
- sequence,
18365
- sourceTimeMs: first.eventTimeMs,
18366
- receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
18367
- datasetObjectIdentity: first.dataset_object_identity,
18368
- datasetObjectChecksum: first.dataset_object_checksum
18369
- };
18370
- applyRows(state, group);
18371
- previousFinalUpdateId = BigInt(sequence);
18372
- } else {
18373
- if (!state || previousFinalUpdateId === undefined) {
18374
- throw new CryptoHftDataError("update_before_snapshot");
18375
- }
18376
- const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
18377
- if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
18378
- if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
18379
- throw new CryptoHftDataError("ambiguous_update_group");
18380
- }
18381
- const previous = unsignedString(first.last_update_id, "last_update_id", true);
18382
- for (const row of group) {
18383
- if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
18384
- throw new CryptoHftDataError("ambiguous_update_group");
18385
- }
18386
- }
18387
- if (BigInt(previous) !== previousFinalUpdateId) {
18388
- throw new CryptoHftDataError("update_chain_gap");
18389
- }
18390
- } else {
18391
- const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
18392
- const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
18393
- for (const row of group) {
18394
- if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
18395
- throw new CryptoHftDataError("ambiguous_update_group");
18396
- }
18397
- }
18398
- const expected = previousFinalUpdateId + 1n;
18399
- if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
18400
- throw new CryptoHftDataError("update_chain_gap");
18401
- }
18402
- }
18403
- applyRows(state, group);
18404
- state.sequence = finalUpdate.toString();
18405
- state.sourceTimeMs = first.eventTimeMs;
18406
- state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
18407
- state.datasetObjectIdentity = first.dataset_object_identity;
18408
- state.datasetObjectChecksum = first.dataset_object_checksum;
18409
- previousFinalUpdateId = finalUpdate;
18410
- }
18411
- if (state) {
18412
- states.push({
18413
- ...state,
18414
- bids: new Map(state.bids),
18415
- asks: new Map(state.asks)
18416
- });
18137
+ for (const [name, method] of Object.entries({
18138
+ "archive.resolveSelection": dependencies.archive.resolveSelection,
18139
+ "archive.verifyCandidate": dependencies.archive.verifyCandidate,
18140
+ "providers.capabilityFor": dependencies.providers.capabilityFor,
18141
+ "providers.acquire": dependencies.providers.acquire,
18142
+ "providers.normalize": dependencies.providers.normalize,
18143
+ "credentials.resolve": dependencies.credentials.resolve,
18144
+ "forwarder.preflight": dependencies.forwarder.preflight,
18145
+ "forwarder.submit": dependencies.forwarder.submit,
18146
+ "clock.nowMs": dependencies.clock.nowMs
18147
+ })) {
18148
+ if (typeof method !== "function") {
18149
+ throw new TypeError(`Backfill dependency ${name} is required`);
18417
18150
  }
18418
18151
  }
18419
- const samples = [];
18420
- for (const targetTimeMs of request2.requiredClockTargetsMs) {
18421
- let prior;
18422
- for (const candidate of states) {
18423
- if (candidate.sourceTimeMs > targetTimeMs)
18152
+ return dependencies;
18153
+ }
18154
+ function outcome(request2, status, reasonCode, extra = {}) {
18155
+ return {
18156
+ status,
18157
+ reasonCode,
18158
+ ...request2 ? {
18159
+ requestId: request2.requestId,
18160
+ idempotencyKey: request2.idempotencyKey,
18161
+ ...request2.target ? { target: request2.target } : {}
18162
+ } : {},
18163
+ ...extra
18164
+ };
18165
+ }
18166
+ function captureBundleId2(request2, capability, dataset) {
18167
+ return sha256Canonical({
18168
+ request_business_identity: request2.idempotencyKey,
18169
+ provider: capability.provider,
18170
+ provider_exchange_id: capability.providerExchangeId,
18171
+ resolved_symbol: capability.resolvedSymbol,
18172
+ adapter_version: capability.adapterVersion,
18173
+ objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
18174
+ identity,
18175
+ checksum,
18176
+ bytes,
18177
+ rows
18178
+ })),
18179
+ canonical_schema_version: request2.expectedProduct.canonicalSchemaVersion,
18180
+ checksum_algorithm: request2.expectedProduct.checksumAlgorithm
18181
+ });
18182
+ }
18183
+ function buildPromotionReceipt(request2, capability, normalized, verification, verificationTimeMs) {
18184
+ if (!request2.wire || !request2.initialSelection || !request2.expectedCanonicalSchema || !request2.coveragePolicy || !request2.productPins) {
18185
+ throw new Error("decoded final-v1 request context is missing");
18186
+ }
18187
+ return finalizePromotionReceipt({
18188
+ schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
18189
+ verified_at: new Date(verificationTimeMs).toISOString(),
18190
+ request_id: request2.requestId,
18191
+ idempotency_key: request2.idempotencyKey,
18192
+ source: EXTERNAL_BACKFILL_SOURCE,
18193
+ capture_origin: "vendor_historical_backfill",
18194
+ source_mode: "vendor_historical_backfill_v1",
18195
+ provider: capability.provider,
18196
+ adapter_version: capability.adapterVersion,
18197
+ effective_policies: {
18198
+ capability_policy: request2.productPins.capability_policy,
18199
+ resource_policy: request2.productPins.resource_policy,
18200
+ adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
18201
+ acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
18202
+ },
18203
+ capture_bundle_id: normalized.captureBundleId,
18204
+ scope: request2.wire.scope,
18205
+ window: request2.wire.window,
18206
+ depth: request2.depth,
18207
+ construction_mode: request2.constructionMode,
18208
+ canonical_schema: request2.expectedCanonicalSchema,
18209
+ coverage_policy: request2.coveragePolicy,
18210
+ selection_sha256: request2.initialSelection.selection_sha256,
18211
+ vendor_semantic_digest: normalized.vendorSemanticDigest,
18212
+ canonical_semantic_digest: verification.canonicalSemanticDigest,
18213
+ prefix_digest: verification.prefixDigest,
18214
+ suffix_digest: verification.suffixDigest,
18215
+ seam_verified: true,
18216
+ coverage_verified: true,
18217
+ dataset_objects: normalized.objects
18218
+ });
18219
+ }
18220
+ async function submitAll(dependencies, batches) {
18221
+ const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
18222
+ for (const batch of batches) {
18223
+ let accepted = false;
18224
+ for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
18225
+ try {
18226
+ const response = await dependencies.forwarder.submit(batch);
18227
+ accepted = response.ok && response.inserted === batch.rows.length;
18228
+ } catch {
18229
+ accepted = false;
18230
+ }
18231
+ if (accepted)
18424
18232
  break;
18425
- prior = candidate;
18233
+ if (attempt < maxAttempts)
18234
+ await dependencies.retry?.wait?.(attempt);
18426
18235
  }
18427
- if (!prior || targetTimeMs - prior.sourceTimeMs > request2.maxPriorAsOfLagMs) {
18428
- throw new CryptoHftDataError("required_clock_coverage_insufficient");
18236
+ if (!accepted)
18237
+ return false;
18238
+ }
18239
+ return true;
18240
+ }
18241
+ function stableFailureReason(error, fallback) {
18242
+ if (error && typeof error === "object") {
18243
+ const reason = error.reason;
18244
+ if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
18245
+ return reason;
18429
18246
  }
18430
- const bids = sortedSide(prior.bids, "bid").slice(0, request2.depth);
18431
- const asks = sortedSide(prior.asks, "ask").slice(0, request2.depth);
18432
- if (bids.length === 0 || asks.length === 0) {
18433
- throw new CryptoHftDataError("book_side_missing");
18247
+ }
18248
+ return fallback;
18249
+ }
18250
+ function assertArchivePreflight(request2, resolution) {
18251
+ if (!request2.target || !request2.productionAuthorizationId) {
18252
+ throw new Error("request target or production authorization ID is missing");
18253
+ }
18254
+ const selection = archiveSelectionCodec.decode(resolution.selection);
18255
+ const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
18256
+ const receiptById = new Map;
18257
+ for (const receipt of receipts) {
18258
+ const existing = receiptById.get(receipt.receipt_id);
18259
+ if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
18260
+ throw new Error("stored receipt identity has conflicting content");
18434
18261
  }
18435
- if (bids[0]?.[0] >= asks[0]?.[0]) {
18436
- throw new CryptoHftDataError("book_crossed_or_locked");
18262
+ receiptById.set(receipt.receipt_id, receipt);
18263
+ }
18264
+ for (const bundle of selection.bundles) {
18265
+ if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
18266
+ throw new Error("vendor selection lacks its validated stored receipt");
18437
18267
  }
18438
- samples.push({
18439
- targetTimeMs,
18440
- sourceTimeMs: prior.sourceTimeMs,
18441
- receivedTimeMs: prior.receivedTimeMs,
18442
- sequence: prior.sequence,
18443
- bids,
18444
- asks,
18445
- datasetObjectIdentity: prior.datasetObjectIdentity,
18446
- datasetObjectChecksum: prior.datasetObjectChecksum
18447
- });
18448
18268
  }
18449
- return samples;
18269
+ if (resolution.readerIdentity.environment !== request2.target.environment || resolution.readerIdentity.cluster !== request2.target.cluster) {
18270
+ throw new Error("archive reader cluster identity mismatch");
18271
+ }
18450
18272
  }
18451
- function sha256Bytes(bytes) {
18452
- return createHash3("sha256").update(bytes).digest("hex");
18273
+ function assertForwarderPreflight(request2, resolution, nowMs) {
18274
+ if (!request2.target || !request2.productionAuthorizationId) {
18275
+ throw new Error("request target or production authorization ID is missing");
18276
+ }
18277
+ if (resolution.forwarderIdentity.environment !== request2.target.environment || resolution.forwarderIdentity.cluster !== request2.target.cluster) {
18278
+ throw new Error("archive forwarder cluster identity mismatch");
18279
+ }
18280
+ const authorization = resolution.authorization;
18281
+ const expiresAtMs = Date.parse(authorization.expiresAt);
18282
+ if (authorization.authorizationId !== request2.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request2.target.environment || authorization.cluster !== request2.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
18283
+ throw new Error("production forwarder authorization is invalid");
18284
+ }
18453
18285
  }
18454
- async function decodeCryptoHftParquetZstd(bytes) {
18455
- const parquet = decompress(bytes);
18456
- const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
18457
- return parquetReadObjects({ file: buffer });
18286
+ function resourcePolicyScopeExceeded(request2) {
18287
+ const policy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((candidate) => request2.productPins?.resource_policy.policy_id === candidate.policy_id && request2.productPins.resource_policy.policy_sha256 === candidate.policy_sha256);
18288
+ if (!policy)
18289
+ return true;
18290
+ return request2.depth > policy.request_bounds.max_depth || request2.window.endTimeMs - request2.window.startTimeMs > policy.request_bounds.max_window_ms || request2.requiredClockTargetsMs.length > policy.request_bounds.max_required_events;
18458
18291
  }
18459
- function parsedDatasetRow(value, object) {
18460
- return {
18461
- received_time: value.received_time,
18462
- event_time: value.event_time,
18463
- transaction_time: value.transaction_time,
18464
- symbol: value.symbol,
18465
- event_type: value.event_type,
18466
- first_update_id: value.first_update_id,
18467
- final_update_id: value.final_update_id,
18468
- prev_final_update_id: value.prev_final_update_id,
18469
- last_update_id: value.last_update_id,
18470
- side: value.side,
18471
- price: value.price,
18472
- quantity: value.quantity,
18473
- order_count: value.order_count,
18474
- dataset_object_identity: object.identity,
18475
- dataset_object_checksum: object.checksum
18476
- };
18292
+ function storedReceiptForSelection(resolution) {
18293
+ const receiptId = resolution.selection.receipt_ids[0];
18294
+ return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
18477
18295
  }
18478
- async function readBoundedObject(response, remainingBytes) {
18479
- if (!response.body)
18480
- return new Uint8Array;
18481
- const reader = response.body.getReader();
18482
- const chunks = [];
18483
- let total = 0;
18484
- while (true) {
18485
- const { done, value } = await reader.read();
18486
- if (done)
18487
- break;
18488
- total += value.byteLength;
18489
- if (total > remainingBytes) {
18490
- await reader.cancel();
18491
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
18492
- }
18493
- chunks.push(value);
18296
+ async function runMarketDataVendorBackfill(input, dependencies) {
18297
+ let request2;
18298
+ try {
18299
+ const documents = input;
18300
+ request2 = decodeBackfillRunDocuments({
18301
+ request: documents.request,
18302
+ requiredClock: documents.requiredClock
18303
+ });
18304
+ } catch {
18305
+ return outcome(undefined, "request_invalid", "request_invalid");
18494
18306
  }
18495
- const bytes = new Uint8Array(total);
18496
- let offset = 0;
18497
- for (const chunk of chunks) {
18498
- bytes.set(chunk, offset);
18499
- offset += chunk.byteLength;
18307
+ let initialResolution;
18308
+ try {
18309
+ initialResolution = await dependencies.archive.resolveSelection(request2);
18310
+ assertArchivePreflight(request2, initialResolution);
18311
+ const forwarderPreflight = await dependencies.forwarder.preflight({
18312
+ authorizationId: request2.productionAuthorizationId,
18313
+ target: request2.target
18314
+ });
18315
+ assertForwarderPreflight(request2, forwarderPreflight, dependencies.clock.nowMs());
18316
+ } catch {
18317
+ return outcome(request2, "archive_preflight_failed", "archive_preflight_failed");
18500
18318
  }
18501
- return bytes;
18502
- }
18503
-
18504
- class CryptoHftDataAdapter {
18505
- baseUrl;
18506
- request;
18507
- nowMs;
18508
- decode;
18509
- profiles;
18510
- constructor(options = {}) {
18511
- this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
18512
- this.request = options.fetch ?? fetch;
18513
- this.nowMs = options.nowMs ?? Date.now;
18514
- this.decode = options.decode ?? decodeCryptoHftParquetZstd;
18515
- this.profiles = options.profiles ?? [];
18319
+ if (initialResolution.selection.coverage_class === "complete") {
18320
+ const receipt2 = storedReceiptForSelection(initialResolution);
18321
+ return outcome(request2, "already_covered", "qualified_coverage_complete", {
18322
+ selection: initialResolution.selection,
18323
+ ...receipt2 ? { receipt: receipt2 } : {}
18324
+ });
18325
+ }
18326
+ if (resourcePolicyScopeExceeded(request2)) {
18327
+ return outcome(request2, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
18328
+ }
18329
+ if (request2.sourcePolicy === "fill_gaps" && (request2.productPins?.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request2.productPins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256)) {
18330
+ return outcome(request2, "capability_unsupported", "scope_unsupported", {
18331
+ reasonSubcode: "fill_gaps_requires_capability_policy_v2"
18332
+ });
18333
+ }
18334
+ let capability;
18335
+ try {
18336
+ capability = dependencies.providers.capabilityFor(request2);
18337
+ } catch {
18338
+ return outcome(request2, "capability_unsupported", "capability_probe_failed");
18339
+ }
18340
+ if (!capability) {
18341
+ return outcome(request2, "capability_unsupported", "scope_unsupported");
18342
+ }
18343
+ if (!request2.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
18344
+ return outcome(request2, "capability_unsupported", "adapter_version_unpinned");
18516
18345
  }
18517
- capabilityFor(request2) {
18518
- return cryptoHftDataCapabilityFor(request2, this.profiles);
18346
+ let credential;
18347
+ try {
18348
+ credential = await dependencies.credentials.resolve(capability.provider);
18349
+ } catch {
18350
+ return outcome(request2, "credentials_missing", "credential_resolution_failed");
18519
18351
  }
18520
- async discoverSymbols(providerExchangeId) {
18521
- if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
18522
- throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
18523
- }
18524
- const endpoint = new URL("/symbols", this.baseUrl);
18525
- endpoint.searchParams.set("exchange", providerExchangeId);
18526
- endpoint.searchParams.set("data_type", "orderbook");
18527
- let response;
18528
- try {
18529
- response = await this.request(endpoint);
18530
- } catch {
18531
- throw new CryptoHftDataError("symbol_discovery_failed");
18532
- }
18533
- if (!response.ok) {
18534
- throw new CryptoHftDataError("symbol_discovery_failed");
18535
- }
18536
- let body;
18537
- try {
18538
- body = await response.json();
18539
- } catch {
18540
- throw new CryptoHftDataError("symbol_discovery_response_invalid");
18541
- }
18542
- const record = body;
18543
- if (!body || typeof body !== "object" || Array.isArray(body) || String(record.exchange).toLowerCase() !== providerExchangeId || String(record.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record.symbols) || !record.symbols.every((symbol) => typeof symbol === "string" && /^[A-Z0-9_-]+$/.test(symbol))) {
18544
- throw new CryptoHftDataError("symbol_discovery_response_invalid");
18545
- }
18546
- return [...new Set(record.symbols)].sort();
18352
+ if (credential === undefined || credential === null) {
18353
+ return outcome(request2, "credentials_missing", "provider_credentials_missing");
18547
18354
  }
18548
- async acquire(request2, capability, credential) {
18549
- const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
18550
- if (typeof apiKey !== "string" || apiKey.length === 0) {
18551
- throw new CryptoHftDataError("credentials_invalid");
18552
- }
18553
- const paths = enumerateCryptoHftDataObjects(request2, capability.providerExchangeId, capability.resolvedSymbol);
18554
- if (paths.length > request2.budgets.maxFiles) {
18555
- throw new CryptoHftDataError("budget_max_files_exceeded");
18355
+ let dataset;
18356
+ let normalized;
18357
+ try {
18358
+ const acquisitionRequest = providerAcquisitionRequest(request2);
18359
+ dataset = await dependencies.providers.acquire(acquisitionRequest, capability, credential);
18360
+ const bundleId = captureBundleId2(request2, capability, dataset);
18361
+ normalized = await dependencies.providers.normalize(acquisitionRequest, capability, dataset, bundleId);
18362
+ if (normalized.captureBundleId !== bundleId) {
18363
+ return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
18364
+ reasonSubcode: "capture_identity_mismatch"
18365
+ });
18556
18366
  }
18557
- const started = this.nowMs();
18558
- const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
18559
- method: "POST",
18560
- headers: { "content-type": "application/json", "X-API-Key": apiKey }
18367
+ } catch (error) {
18368
+ const reason = stableFailureReason(error, "provider_dataset_invalid");
18369
+ return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
18370
+ reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
18561
18371
  });
18562
- if (!tokenResponse.ok)
18563
- throw new CryptoHftDataError("jwt_issuance_failed");
18564
- const tokenBody = await tokenResponse.json();
18565
- if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
18566
- throw new CryptoHftDataError("jwt_response_invalid");
18372
+ }
18373
+ try {
18374
+ const candidateBatches = buildForwarderBatches({
18375
+ captureBundleId: normalized.captureBundleId,
18376
+ deploymentId: "market-data-vendor-backfill",
18377
+ rows: normalized.rows
18378
+ });
18379
+ if (!await submitAll(dependencies, candidateBatches)) {
18380
+ return outcome(request2, "archive_ingest_failed", "candidate_batch_rejected");
18567
18381
  }
18568
- let totalBytes = 0;
18569
- let totalRows = 0;
18570
- const objects = [];
18571
- const rows = [];
18572
- for (const path of paths) {
18573
- if (this.nowMs() - started > request2.budgets.maxDurationMs) {
18574
- throw new CryptoHftDataError("budget_max_duration_exceeded");
18575
- }
18576
- const endpoint = new URL("/download", this.baseUrl);
18577
- endpoint.searchParams.set("file", path);
18578
- const response = await this.request(endpoint, {
18579
- headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
18580
- });
18581
- if (!response.ok)
18582
- throw new CryptoHftDataError("object_download_failed");
18583
- const declaredBytes = Number(response.headers.get("content-length"));
18584
- if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request2.budgets.maxBytes) {
18585
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
18586
- }
18587
- const bytes = await readBoundedObject(response, request2.budgets.maxBytes - totalBytes);
18588
- totalBytes += bytes.byteLength;
18589
- if (totalBytes > request2.budgets.maxBytes) {
18590
- throw new CryptoHftDataError("budget_max_bytes_exceeded");
18591
- }
18592
- const decoded = await this.decode(bytes);
18593
- totalRows += decoded.length;
18594
- if (totalRows > request2.budgets.maxRows) {
18595
- throw new CryptoHftDataError("budget_max_rows_exceeded");
18596
- }
18597
- const object = {
18598
- identity: path,
18599
- checksum: sha256Bytes(bytes),
18600
- bytes: bytes.byteLength,
18601
- rows: decoded.length
18602
- };
18603
- objects.push(object);
18604
- for (const decodedRow of decoded) {
18605
- const parsed = parsedDatasetRow(decodedRow, object);
18606
- validatedRow(request2, parsed);
18607
- rows.push(parsed);
18608
- }
18382
+ } catch {
18383
+ return outcome(request2, "archive_ingest_failed", "candidate_submission_failed");
18384
+ }
18385
+ let verification;
18386
+ try {
18387
+ verification = await dependencies.archive.verifyCandidate(request2, normalized, normalized.captureBundleId, initialResolution);
18388
+ } catch {
18389
+ return outcome(request2, "promotion_verification_failed", "candidate_query_failed");
18390
+ }
18391
+ if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
18392
+ return outcome(request2, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
18393
+ }
18394
+ let receipt;
18395
+ try {
18396
+ receipt = buildPromotionReceipt(request2, capability, normalized, verification, dependencies.clock.nowMs());
18397
+ const [batch] = buildForwarderBatches({
18398
+ captureBundleId: normalized.captureBundleId,
18399
+ deploymentId: "market-data-vendor-backfill",
18400
+ rows: [promotionReceiptToArchiveRow(receipt)]
18401
+ });
18402
+ if (!batch || !await submitAll(dependencies, [batch])) {
18403
+ return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
18609
18404
  }
18610
- if (this.nowMs() - started > request2.budgets.maxDurationMs) {
18611
- throw new CryptoHftDataError("budget_max_duration_exceeded");
18405
+ const qualification = finalizeQualificationEvent({
18406
+ capture_bundle_id: receipt.capture_bundle_id,
18407
+ state: "qualified",
18408
+ receipt_id: receipt.receipt_id,
18409
+ promotion_identity_sha256: receipt.promotion_identity_sha256,
18410
+ window: receipt.window,
18411
+ event_at: receipt.verified_at,
18412
+ reason_code: "promotion_verified"
18413
+ });
18414
+ const [qualificationBatch] = buildForwarderBatches({
18415
+ captureBundleId: normalized.captureBundleId,
18416
+ deploymentId: "market-data-vendor-backfill",
18417
+ rows: [qualificationEventToArchiveRow(qualification)]
18418
+ });
18419
+ if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
18420
+ return outcome(request2, "archive_ingest_failed", "qualification_commit_failed", { receipt });
18612
18421
  }
18613
- return {
18614
- objects,
18615
- rows,
18616
- vendorSemanticDigest: sha256Canonical(rows)
18617
- };
18422
+ } catch {
18423
+ return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
18618
18424
  }
18619
- async normalize(request2, capability, dataset, captureBundleId3) {
18620
- const profile = this.profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
18621
- if (!profile) {
18622
- throw new CryptoHftDataError("profile_semantics_unavailable");
18623
- }
18624
- const samples = reconstructCryptoHftDataOrderBooks(request2, dataset.rows, profile);
18625
- const context2 = {
18626
- source: EXTERNAL_BACKFILL_SOURCE,
18425
+ let finalResolution;
18426
+ try {
18427
+ finalResolution = await dependencies.archive.resolveSelection(request2);
18428
+ assertArchivePreflight(request2, finalResolution);
18429
+ } catch (error) {
18430
+ return outcome(request2, "promotion_verification_failed", "post_promotion_selection_failed", {
18431
+ receipt,
18432
+ reasonSubcode: stableFailureReason(error, "archive_selection_resolution_failed")
18433
+ });
18434
+ }
18435
+ if (finalResolution.selection.coverage_class !== "complete") {
18436
+ return outcome(request2, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
18437
+ }
18438
+ if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
18439
+ return outcome(request2, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
18440
+ }
18441
+ try {
18442
+ if (!request2.wire)
18443
+ throw new Error("decoded request wire is missing");
18444
+ const [selectionBatch] = buildForwarderBatches({
18445
+ captureBundleId: receipt.capture_bundle_id,
18627
18446
  deploymentId: "market-data-vendor-backfill",
18628
- captureBundleId: captureBundleId3,
18629
- exchange: request2.scope.exchange,
18630
- symbol: request2.scope.tradingPair,
18631
- tradingPair: request2.scope.tradingPair,
18632
- sourceSymbol: request2.scope.sourceSymbol,
18633
- assetType: request2.scope.marketType,
18634
- feed: "ORDERBOOK",
18635
- provider: "cryptohftdata",
18636
- sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
18637
- schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
18638
- checksumAlgorithm: CHECKSUM_ALGORITHM,
18639
- provenanceComplete: true
18640
- };
18641
- const rows = samples.flatMap((sample) => {
18642
- const rawCapture = {
18643
- rawCaptureId: sha256Canonical({
18644
- capture_bundle_id: captureBundleId3,
18645
- dataset_object_identity: sample.datasetObjectIdentity,
18646
- dataset_object_checksum: sample.datasetObjectChecksum,
18647
- source_time_ms: sample.sourceTimeMs,
18648
- target_time_ms: sample.targetTimeMs
18649
- }),
18650
- rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
18651
- rawChecksum: sample.datasetObjectChecksum,
18652
- redactedPayload: {
18653
- dataset_object_identity: sample.datasetObjectIdentity,
18654
- dataset_object_checksum: sample.datasetObjectChecksum
18655
- },
18656
- eventTimeMs: sample.sourceTimeMs,
18657
- receivedTimeMs: sample.receivedTimeMs,
18658
- checksumAlgorithm: CHECKSUM_ALGORITHM
18659
- };
18660
- const canonical = buildCanonicalOrderBookRows({
18661
- context: context2,
18662
- rawCapture,
18663
- depthLimit: request2.depth,
18664
- constructionMode: request2.constructionMode,
18665
- snapshot: {
18666
- bids: sample.bids,
18667
- asks: sample.asks,
18668
- timestamp: sample.sourceTimeMs,
18669
- receivedTimestamp: sample.receivedTimeMs,
18670
- exchange: request2.scope.exchange,
18671
- symbol: request2.scope.tradingPair,
18672
- depthLimit: request2.depth,
18673
- sequence: sample.sequence
18674
- }
18675
- });
18676
- return [...canonical.levels, canonical.summary];
18447
+ rows: [
18448
+ archiveSelectionToArchiveRow(request2.wire, finalResolution.selection)
18449
+ ]
18677
18450
  });
18678
- return {
18679
- captureBundleId: captureBundleId3,
18680
- objects: dataset.objects,
18681
- rows,
18682
- vendorSemanticDigest: dataset.vendorSemanticDigest,
18683
- canonicalSemanticDigest: semanticDigest(rows)
18684
- };
18451
+ if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
18452
+ return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
18453
+ }
18454
+ } catch {
18455
+ return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
18685
18456
  }
18457
+ return outcome(request2, "promoted", "promotion_qualified", {
18458
+ receipt,
18459
+ selection: finalResolution.selection
18460
+ });
18686
18461
  }
18687
18462
  // src/helpers/market-data-vendor-backfill/forwarder-client.ts
18688
18463
  class ArchiveForwarderSubmissionError extends Error {
@@ -18825,12 +18600,15 @@ export {
18825
18600
  REQUIRED_CLOCK_SCHEMA_ID,
18826
18601
  QualifiedOrderBookArchiveReader,
18827
18602
  PROMOTION_RECEIPT_SCHEMA_ID,
18603
+ LEGACY_RESOURCE_POLICY_ID,
18604
+ LEGACY_RESOURCE_POLICY,
18828
18605
  FINAL_BACKFILL_STATUSES,
18829
18606
  EFFECTIVE_ADAPTER_POLICY_PIN,
18830
18607
  EFFECTIVE_ACQUISITION_POLICY_PIN,
18831
18608
  CryptoHftDataError,
18832
18609
  CryptoHftDataAdapter,
18833
18610
  CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE,
18611
+ CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE,
18834
18612
  CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE,
18835
18613
  CRYPTOHFTDATA_API_URL,
18836
18614
  CRYPTOHFTDATA_ADAPTER_VERSION,
@@ -18847,4 +18625,4 @@ export {
18847
18625
  ACQUISITION_POLICY_ID
18848
18626
  };
18849
18627
 
18850
- //# debugId=75C87110D96E1B9B64756E2164756E21
18628
+ //# debugId=352683EF086D61EB64756E2164756E21