@usherlabs/cex-broker 0.2.48 → 0.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cex-canonical-orderbook-export.js +72 -358
- package/dist/commands/cex-canonical-orderbook-export.js.map +7 -19
- package/dist/commands/cli.js +224 -67
- package/dist/commands/market-data-vendor-backfill.js +975 -1198
- package/dist/commands/market-data-vendor-backfill.js.map +13 -25
- package/dist/helpers/logger.d.ts +4 -0
- package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +3 -1
- package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +58 -4
- package/dist/helpers/market-data-vendor-backfill/semantic-verification.d.ts +1 -0
- package/dist/helpers/otel.d.ts +1 -0
- package/dist/helpers/trace-context.d.ts +8 -0
- package/dist/index.js +183 -63
- package/dist/index.js.map +9 -8
- package/dist/market-data-preparation/fixtures/conformance-v2.json +8 -8
- package/dist/market-data-preparation/policies/capability-policy-v1.json +28 -0
- package/dist/market-data-preparation/policies/capability-policy.json +14 -3
- package/dist/market-data-preparation/policies/resource-policy-v1.json +16 -0
- package/dist/market-data-preparation/policies/resource-policy.json +3 -3
- package/dist/market-data-preparation.js +70 -359
- package/dist/market-data-preparation.js.map +9 -21
- package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +39 -0
- package/dist/market-data-vendor-backfill/policies/resource-policy-v2.json +16 -0
- package/dist/market-data-vendor-backfill.d.ts +2 -2
- package/dist/market-data-vendor-backfill.js +1528 -1750
- package/dist/market-data-vendor-backfill.js.map +13 -25
- package/package.json +1 -1
|
@@ -1431,241 +1431,6 @@ var require_src = __commonJS((exports) => {
|
|
|
1431
1431
|
};
|
|
1432
1432
|
});
|
|
1433
1433
|
|
|
1434
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/types/LogRecord.js
|
|
1435
|
-
var require_LogRecord = __commonJS((exports) => {
|
|
1436
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1437
|
-
exports.SeverityNumber = undefined;
|
|
1438
|
-
var SeverityNumber;
|
|
1439
|
-
(function(SeverityNumber2) {
|
|
1440
|
-
SeverityNumber2[SeverityNumber2["UNSPECIFIED"] = 0] = "UNSPECIFIED";
|
|
1441
|
-
SeverityNumber2[SeverityNumber2["TRACE"] = 1] = "TRACE";
|
|
1442
|
-
SeverityNumber2[SeverityNumber2["TRACE2"] = 2] = "TRACE2";
|
|
1443
|
-
SeverityNumber2[SeverityNumber2["TRACE3"] = 3] = "TRACE3";
|
|
1444
|
-
SeverityNumber2[SeverityNumber2["TRACE4"] = 4] = "TRACE4";
|
|
1445
|
-
SeverityNumber2[SeverityNumber2["DEBUG"] = 5] = "DEBUG";
|
|
1446
|
-
SeverityNumber2[SeverityNumber2["DEBUG2"] = 6] = "DEBUG2";
|
|
1447
|
-
SeverityNumber2[SeverityNumber2["DEBUG3"] = 7] = "DEBUG3";
|
|
1448
|
-
SeverityNumber2[SeverityNumber2["DEBUG4"] = 8] = "DEBUG4";
|
|
1449
|
-
SeverityNumber2[SeverityNumber2["INFO"] = 9] = "INFO";
|
|
1450
|
-
SeverityNumber2[SeverityNumber2["INFO2"] = 10] = "INFO2";
|
|
1451
|
-
SeverityNumber2[SeverityNumber2["INFO3"] = 11] = "INFO3";
|
|
1452
|
-
SeverityNumber2[SeverityNumber2["INFO4"] = 12] = "INFO4";
|
|
1453
|
-
SeverityNumber2[SeverityNumber2["WARN"] = 13] = "WARN";
|
|
1454
|
-
SeverityNumber2[SeverityNumber2["WARN2"] = 14] = "WARN2";
|
|
1455
|
-
SeverityNumber2[SeverityNumber2["WARN3"] = 15] = "WARN3";
|
|
1456
|
-
SeverityNumber2[SeverityNumber2["WARN4"] = 16] = "WARN4";
|
|
1457
|
-
SeverityNumber2[SeverityNumber2["ERROR"] = 17] = "ERROR";
|
|
1458
|
-
SeverityNumber2[SeverityNumber2["ERROR2"] = 18] = "ERROR2";
|
|
1459
|
-
SeverityNumber2[SeverityNumber2["ERROR3"] = 19] = "ERROR3";
|
|
1460
|
-
SeverityNumber2[SeverityNumber2["ERROR4"] = 20] = "ERROR4";
|
|
1461
|
-
SeverityNumber2[SeverityNumber2["FATAL"] = 21] = "FATAL";
|
|
1462
|
-
SeverityNumber2[SeverityNumber2["FATAL2"] = 22] = "FATAL2";
|
|
1463
|
-
SeverityNumber2[SeverityNumber2["FATAL3"] = 23] = "FATAL3";
|
|
1464
|
-
SeverityNumber2[SeverityNumber2["FATAL4"] = 24] = "FATAL4";
|
|
1465
|
-
})(SeverityNumber = exports.SeverityNumber || (exports.SeverityNumber = {}));
|
|
1466
|
-
});
|
|
1467
|
-
|
|
1468
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLogger.js
|
|
1469
|
-
var require_NoopLogger = __commonJS((exports) => {
|
|
1470
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1471
|
-
exports.NOOP_LOGGER = exports.NoopLogger = undefined;
|
|
1472
|
-
|
|
1473
|
-
class NoopLogger {
|
|
1474
|
-
emit(_logRecord) {}
|
|
1475
|
-
}
|
|
1476
|
-
exports.NoopLogger = NoopLogger;
|
|
1477
|
-
exports.NOOP_LOGGER = new NoopLogger;
|
|
1478
|
-
});
|
|
1479
|
-
|
|
1480
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLoggerProvider.js
|
|
1481
|
-
var require_NoopLoggerProvider = __commonJS((exports) => {
|
|
1482
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1483
|
-
exports.NOOP_LOGGER_PROVIDER = exports.NoopLoggerProvider = undefined;
|
|
1484
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
1485
|
-
|
|
1486
|
-
class NoopLoggerProvider {
|
|
1487
|
-
getLogger(_name, _version, _options) {
|
|
1488
|
-
return new NoopLogger_1.NoopLogger;
|
|
1489
|
-
}
|
|
1490
|
-
}
|
|
1491
|
-
exports.NoopLoggerProvider = NoopLoggerProvider;
|
|
1492
|
-
exports.NOOP_LOGGER_PROVIDER = new NoopLoggerProvider;
|
|
1493
|
-
});
|
|
1494
|
-
|
|
1495
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLogger.js
|
|
1496
|
-
var require_ProxyLogger = __commonJS((exports) => {
|
|
1497
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1498
|
-
exports.ProxyLogger = undefined;
|
|
1499
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
1500
|
-
|
|
1501
|
-
class ProxyLogger {
|
|
1502
|
-
constructor(_provider, name, version, options) {
|
|
1503
|
-
this._provider = _provider;
|
|
1504
|
-
this.name = name;
|
|
1505
|
-
this.version = version;
|
|
1506
|
-
this.options = options;
|
|
1507
|
-
}
|
|
1508
|
-
emit(logRecord) {
|
|
1509
|
-
this._getLogger().emit(logRecord);
|
|
1510
|
-
}
|
|
1511
|
-
_getLogger() {
|
|
1512
|
-
if (this._delegate) {
|
|
1513
|
-
return this._delegate;
|
|
1514
|
-
}
|
|
1515
|
-
const logger = this._provider._getDelegateLogger(this.name, this.version, this.options);
|
|
1516
|
-
if (!logger) {
|
|
1517
|
-
return NoopLogger_1.NOOP_LOGGER;
|
|
1518
|
-
}
|
|
1519
|
-
this._delegate = logger;
|
|
1520
|
-
return this._delegate;
|
|
1521
|
-
}
|
|
1522
|
-
}
|
|
1523
|
-
exports.ProxyLogger = ProxyLogger;
|
|
1524
|
-
});
|
|
1525
|
-
|
|
1526
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLoggerProvider.js
|
|
1527
|
-
var require_ProxyLoggerProvider = __commonJS((exports) => {
|
|
1528
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1529
|
-
exports.ProxyLoggerProvider = undefined;
|
|
1530
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
1531
|
-
var ProxyLogger_1 = require_ProxyLogger();
|
|
1532
|
-
|
|
1533
|
-
class ProxyLoggerProvider {
|
|
1534
|
-
getLogger(name, version, options) {
|
|
1535
|
-
var _a;
|
|
1536
|
-
return (_a = this._getDelegateLogger(name, version, options)) !== null && _a !== undefined ? _a : new ProxyLogger_1.ProxyLogger(this, name, version, options);
|
|
1537
|
-
}
|
|
1538
|
-
_getDelegate() {
|
|
1539
|
-
var _a;
|
|
1540
|
-
return (_a = this._delegate) !== null && _a !== undefined ? _a : NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
|
|
1541
|
-
}
|
|
1542
|
-
_setDelegate(delegate) {
|
|
1543
|
-
this._delegate = delegate;
|
|
1544
|
-
}
|
|
1545
|
-
_getDelegateLogger(name, version, options) {
|
|
1546
|
-
var _a;
|
|
1547
|
-
return (_a = this._delegate) === null || _a === undefined ? undefined : _a.getLogger(name, version, options);
|
|
1548
|
-
}
|
|
1549
|
-
}
|
|
1550
|
-
exports.ProxyLoggerProvider = ProxyLoggerProvider;
|
|
1551
|
-
});
|
|
1552
|
-
|
|
1553
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/globalThis.js
|
|
1554
|
-
var require_globalThis2 = __commonJS((exports) => {
|
|
1555
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1556
|
-
exports._globalThis = undefined;
|
|
1557
|
-
exports._globalThis = typeof globalThis === "object" ? globalThis : global;
|
|
1558
|
-
});
|
|
1559
|
-
|
|
1560
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/index.js
|
|
1561
|
-
var require_node2 = __commonJS((exports) => {
|
|
1562
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1563
|
-
exports._globalThis = undefined;
|
|
1564
|
-
var globalThis_1 = require_globalThis2();
|
|
1565
|
-
Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
|
|
1566
|
-
return globalThis_1._globalThis;
|
|
1567
|
-
} });
|
|
1568
|
-
});
|
|
1569
|
-
|
|
1570
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/index.js
|
|
1571
|
-
var require_platform2 = __commonJS((exports) => {
|
|
1572
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1573
|
-
exports._globalThis = undefined;
|
|
1574
|
-
var node_1 = require_node2();
|
|
1575
|
-
Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
|
|
1576
|
-
return node_1._globalThis;
|
|
1577
|
-
} });
|
|
1578
|
-
});
|
|
1579
|
-
|
|
1580
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/internal/global-utils.js
|
|
1581
|
-
var require_global_utils2 = __commonJS((exports) => {
|
|
1582
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1583
|
-
exports.API_BACKWARDS_COMPATIBILITY_VERSION = exports.makeGetter = exports._global = exports.GLOBAL_LOGS_API_KEY = undefined;
|
|
1584
|
-
var platform_1 = require_platform2();
|
|
1585
|
-
exports.GLOBAL_LOGS_API_KEY = Symbol.for("io.opentelemetry.js.api.logs");
|
|
1586
|
-
exports._global = platform_1._globalThis;
|
|
1587
|
-
function makeGetter(requiredVersion, instance, fallback) {
|
|
1588
|
-
return (version) => version === requiredVersion ? instance : fallback;
|
|
1589
|
-
}
|
|
1590
|
-
exports.makeGetter = makeGetter;
|
|
1591
|
-
exports.API_BACKWARDS_COMPATIBILITY_VERSION = 1;
|
|
1592
|
-
});
|
|
1593
|
-
|
|
1594
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/api/logs.js
|
|
1595
|
-
var require_logs = __commonJS((exports) => {
|
|
1596
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1597
|
-
exports.LogsAPI = undefined;
|
|
1598
|
-
var global_utils_1 = require_global_utils2();
|
|
1599
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
1600
|
-
var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
|
|
1601
|
-
|
|
1602
|
-
class LogsAPI {
|
|
1603
|
-
constructor() {
|
|
1604
|
-
this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
1605
|
-
}
|
|
1606
|
-
static getInstance() {
|
|
1607
|
-
if (!this._instance) {
|
|
1608
|
-
this._instance = new LogsAPI;
|
|
1609
|
-
}
|
|
1610
|
-
return this._instance;
|
|
1611
|
-
}
|
|
1612
|
-
setGlobalLoggerProvider(provider) {
|
|
1613
|
-
if (global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) {
|
|
1614
|
-
return this.getLoggerProvider();
|
|
1615
|
-
}
|
|
1616
|
-
global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY] = (0, global_utils_1.makeGetter)(global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION, provider, NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER);
|
|
1617
|
-
this._proxyLoggerProvider._setDelegate(provider);
|
|
1618
|
-
return provider;
|
|
1619
|
-
}
|
|
1620
|
-
getLoggerProvider() {
|
|
1621
|
-
var _a, _b;
|
|
1622
|
-
return (_b = (_a = global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) === null || _a === undefined ? undefined : _a.call(global_utils_1._global, global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION)) !== null && _b !== undefined ? _b : this._proxyLoggerProvider;
|
|
1623
|
-
}
|
|
1624
|
-
getLogger(name, version, options) {
|
|
1625
|
-
return this.getLoggerProvider().getLogger(name, version, options);
|
|
1626
|
-
}
|
|
1627
|
-
disable() {
|
|
1628
|
-
delete global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY];
|
|
1629
|
-
this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
1630
|
-
}
|
|
1631
|
-
}
|
|
1632
|
-
exports.LogsAPI = LogsAPI;
|
|
1633
|
-
});
|
|
1634
|
-
|
|
1635
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/index.js
|
|
1636
|
-
var require_src2 = __commonJS((exports) => {
|
|
1637
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1638
|
-
exports.logs = exports.ProxyLoggerProvider = exports.ProxyLogger = exports.NoopLoggerProvider = exports.NOOP_LOGGER_PROVIDER = exports.NoopLogger = exports.NOOP_LOGGER = exports.SeverityNumber = undefined;
|
|
1639
|
-
var LogRecord_1 = require_LogRecord();
|
|
1640
|
-
Object.defineProperty(exports, "SeverityNumber", { enumerable: true, get: function() {
|
|
1641
|
-
return LogRecord_1.SeverityNumber;
|
|
1642
|
-
} });
|
|
1643
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
1644
|
-
Object.defineProperty(exports, "NOOP_LOGGER", { enumerable: true, get: function() {
|
|
1645
|
-
return NoopLogger_1.NOOP_LOGGER;
|
|
1646
|
-
} });
|
|
1647
|
-
Object.defineProperty(exports, "NoopLogger", { enumerable: true, get: function() {
|
|
1648
|
-
return NoopLogger_1.NoopLogger;
|
|
1649
|
-
} });
|
|
1650
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
1651
|
-
Object.defineProperty(exports, "NOOP_LOGGER_PROVIDER", { enumerable: true, get: function() {
|
|
1652
|
-
return NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
|
|
1653
|
-
} });
|
|
1654
|
-
Object.defineProperty(exports, "NoopLoggerProvider", { enumerable: true, get: function() {
|
|
1655
|
-
return NoopLoggerProvider_1.NoopLoggerProvider;
|
|
1656
|
-
} });
|
|
1657
|
-
var ProxyLogger_1 = require_ProxyLogger();
|
|
1658
|
-
Object.defineProperty(exports, "ProxyLogger", { enumerable: true, get: function() {
|
|
1659
|
-
return ProxyLogger_1.ProxyLogger;
|
|
1660
|
-
} });
|
|
1661
|
-
var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
|
|
1662
|
-
Object.defineProperty(exports, "ProxyLoggerProvider", { enumerable: true, get: function() {
|
|
1663
|
-
return ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
1664
|
-
} });
|
|
1665
|
-
var logs_1 = require_logs();
|
|
1666
|
-
exports.logs = logs_1.LogsAPI.getInstance();
|
|
1667
|
-
});
|
|
1668
|
-
|
|
1669
1434
|
// node_modules/ajv/dist/compile/codegen/code.js
|
|
1670
1435
|
var require_code = __commonJS((exports) => {
|
|
1671
1436
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -8697,102 +8462,6 @@ var BaseTransport = class {
|
|
|
8697
8462
|
return this.logger;
|
|
8698
8463
|
}
|
|
8699
8464
|
};
|
|
8700
|
-
var LoggerlessTransport = class {
|
|
8701
|
-
id;
|
|
8702
|
-
enabled;
|
|
8703
|
-
level;
|
|
8704
|
-
consoleDebug;
|
|
8705
|
-
constructor(config) {
|
|
8706
|
-
this.id = Date.now().toString() + Math.random().toString();
|
|
8707
|
-
this.enabled = config.enabled ?? true;
|
|
8708
|
-
this.consoleDebug = config.consoleDebug ?? false;
|
|
8709
|
-
this.level = config.level ?? "trace";
|
|
8710
|
-
}
|
|
8711
|
-
_sendToLogger(params) {
|
|
8712
|
-
if (!this.enabled)
|
|
8713
|
-
return;
|
|
8714
|
-
if (LogLevelPriority[params.logLevel] < LogLevelPriority[this.level])
|
|
8715
|
-
return;
|
|
8716
|
-
const messages = this.shipToLogger(params);
|
|
8717
|
-
if (this.consoleDebug)
|
|
8718
|
-
switch (params.logLevel) {
|
|
8719
|
-
case LogLevel.info:
|
|
8720
|
-
console.info(...messages);
|
|
8721
|
-
break;
|
|
8722
|
-
case LogLevel.warn:
|
|
8723
|
-
console.warn(...messages);
|
|
8724
|
-
break;
|
|
8725
|
-
case LogLevel.error:
|
|
8726
|
-
console.error(...messages);
|
|
8727
|
-
break;
|
|
8728
|
-
case LogLevel.trace:
|
|
8729
|
-
console.debug(...messages);
|
|
8730
|
-
break;
|
|
8731
|
-
case LogLevel.debug:
|
|
8732
|
-
console.debug(...messages);
|
|
8733
|
-
break;
|
|
8734
|
-
case LogLevel.fatal:
|
|
8735
|
-
console.debug(...messages);
|
|
8736
|
-
break;
|
|
8737
|
-
default:
|
|
8738
|
-
console.log(...messages);
|
|
8739
|
-
}
|
|
8740
|
-
}
|
|
8741
|
-
getLoggerInstance() {
|
|
8742
|
-
throw new Error("This transport does not have a logger instance");
|
|
8743
|
-
}
|
|
8744
|
-
};
|
|
8745
|
-
|
|
8746
|
-
// node_modules/@loglayer/transport-opentelemetry/dist/index.mjs
|
|
8747
|
-
var import_api_logs = __toESM(require_src2(), 1);
|
|
8748
|
-
var logLayerLevels = {
|
|
8749
|
-
fatal: import_api_logs.SeverityNumber.FATAL,
|
|
8750
|
-
error: import_api_logs.SeverityNumber.ERROR,
|
|
8751
|
-
warn: import_api_logs.SeverityNumber.WARN,
|
|
8752
|
-
info: import_api_logs.SeverityNumber.INFO,
|
|
8753
|
-
debug: import_api_logs.SeverityNumber.DEBUG,
|
|
8754
|
-
trace: import_api_logs.SeverityNumber.TRACE
|
|
8755
|
-
};
|
|
8756
|
-
function getSeverityNumber(level) {
|
|
8757
|
-
return logLayerLevels[level] ?? import_api_logs.SeverityNumber.TRACE;
|
|
8758
|
-
}
|
|
8759
|
-
function emitLogRecord(record, logger) {
|
|
8760
|
-
const { message, level, ...splat } = record;
|
|
8761
|
-
const attributes = {};
|
|
8762
|
-
for (const key in splat)
|
|
8763
|
-
if (Object.hasOwn(splat, key))
|
|
8764
|
-
attributes[key] = splat[key];
|
|
8765
|
-
const logRecord = {
|
|
8766
|
-
severityNumber: getSeverityNumber(level),
|
|
8767
|
-
severityText: level,
|
|
8768
|
-
body: message,
|
|
8769
|
-
attributes
|
|
8770
|
-
};
|
|
8771
|
-
logger.emit(logRecord);
|
|
8772
|
-
}
|
|
8773
|
-
var OpenTelemetryTransport = class extends LoggerlessTransport {
|
|
8774
|
-
_logger;
|
|
8775
|
-
onError;
|
|
8776
|
-
constructor(config = {}) {
|
|
8777
|
-
super(config);
|
|
8778
|
-
this._logger = import_api_logs.logs.getLogger("loglayer");
|
|
8779
|
-
this.onError = config.onError;
|
|
8780
|
-
}
|
|
8781
|
-
shipToLogger({ logLevel, messages, data, hasData }) {
|
|
8782
|
-
const assembled = {
|
|
8783
|
-
level: logLevel,
|
|
8784
|
-
message: messages.join(" "),
|
|
8785
|
-
...hasData ? data : {}
|
|
8786
|
-
};
|
|
8787
|
-
try {
|
|
8788
|
-
emitLogRecord(assembled, this._logger);
|
|
8789
|
-
} catch (error) {
|
|
8790
|
-
if (this.onError)
|
|
8791
|
-
this.onError(error);
|
|
8792
|
-
}
|
|
8793
|
-
return [assembled];
|
|
8794
|
-
}
|
|
8795
|
-
};
|
|
8796
8465
|
|
|
8797
8466
|
// node_modules/@loglayer/transport-tslog/dist/index.js
|
|
8798
8467
|
var TsLogTransport = class extends BaseTransport {
|
|
@@ -10964,13 +10633,7 @@ var tslogLogger = new Logger({
|
|
|
10964
10633
|
minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
|
|
10965
10634
|
});
|
|
10966
10635
|
var baseLogger = new LogLayer({
|
|
10967
|
-
transport: [
|
|
10968
|
-
new TsLogTransport({ id: "tslog", logger: tslogLogger }),
|
|
10969
|
-
new OpenTelemetryTransport({
|
|
10970
|
-
id: "otel",
|
|
10971
|
-
enabled: true
|
|
10972
|
-
})
|
|
10973
|
-
],
|
|
10636
|
+
transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
|
|
10974
10637
|
plugins: [openTelemetryPlugin()],
|
|
10975
10638
|
errorSerializer: serializeError
|
|
10976
10639
|
});
|
|
@@ -12089,12 +11752,14 @@ var result_schema_default = {
|
|
|
12089
11752
|
};
|
|
12090
11753
|
|
|
12091
11754
|
// src/helpers/market-data-vendor-backfill/manifests.ts
|
|
12092
|
-
var
|
|
12093
|
-
var
|
|
11755
|
+
var LEGACY_CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
|
|
11756
|
+
var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v2";
|
|
11757
|
+
var LEGACY_RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
|
|
11758
|
+
var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v2";
|
|
12094
11759
|
var ADAPTER_POLICY_ID = "cryptohftdata-orderbook-adapter/v1";
|
|
12095
11760
|
var ACQUISITION_POLICY_ID = "cryptohftdata-hourly-acquisition/v1";
|
|
12096
|
-
var
|
|
12097
|
-
policy_id:
|
|
11761
|
+
var legacyCapabilityPolicyContent = {
|
|
11762
|
+
policy_id: LEGACY_CAPABILITY_POLICY_ID,
|
|
12098
11763
|
provider: "cryptohftdata",
|
|
12099
11764
|
adapter_policy: {
|
|
12100
11765
|
policy_id: ADAPTER_POLICY_ID,
|
|
@@ -12120,12 +11785,37 @@ var capabilityPolicyContent = {
|
|
|
12120
11785
|
}
|
|
12121
11786
|
]
|
|
12122
11787
|
};
|
|
11788
|
+
var LEGACY_CAPABILITY_POLICY = Object.freeze({
|
|
11789
|
+
...legacyCapabilityPolicyContent,
|
|
11790
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent)
|
|
11791
|
+
});
|
|
11792
|
+
var capabilityPolicyContent = {
|
|
11793
|
+
...legacyCapabilityPolicyContent,
|
|
11794
|
+
policy_id: CAPABILITY_POLICY_ID,
|
|
11795
|
+
profiles: [
|
|
11796
|
+
...legacyCapabilityPolicyContent.profiles.map((profile) => ({
|
|
11797
|
+
...profile,
|
|
11798
|
+
source_policies: ["authoritative_window", "fill_gaps"]
|
|
11799
|
+
})),
|
|
11800
|
+
{
|
|
11801
|
+
exchange: "okx",
|
|
11802
|
+
market_type: "spot",
|
|
11803
|
+
feed: "ORDERBOOK",
|
|
11804
|
+
canonical_trading_pair: "ARB-USDC",
|
|
11805
|
+
provider_exchange_id: "okx_spot",
|
|
11806
|
+
resolved_symbol: "ARB-USDC",
|
|
11807
|
+
construction_modes: ["sampled_top_n_snapshot"],
|
|
11808
|
+
source_policies: ["authoritative_window", "fill_gaps"],
|
|
11809
|
+
max_depth: 400
|
|
11810
|
+
}
|
|
11811
|
+
]
|
|
11812
|
+
};
|
|
12123
11813
|
var CAPABILITY_POLICY = Object.freeze({
|
|
12124
11814
|
...capabilityPolicyContent,
|
|
12125
11815
|
policy_sha256: jcsSha256(capabilityPolicyContent)
|
|
12126
11816
|
});
|
|
12127
|
-
var
|
|
12128
|
-
policy_id:
|
|
11817
|
+
var legacyResourcePolicyContent = {
|
|
11818
|
+
policy_id: LEGACY_RESOURCE_POLICY_ID,
|
|
12129
11819
|
limits: {
|
|
12130
11820
|
max_files: 1e4,
|
|
12131
11821
|
max_bytes: 100 * 1024 * 1024 * 1024,
|
|
@@ -12139,17 +11829,29 @@ var resourcePolicyContent = {
|
|
|
12139
11829
|
max_required_events: 1e5
|
|
12140
11830
|
}
|
|
12141
11831
|
};
|
|
11832
|
+
var LEGACY_RESOURCE_POLICY = Object.freeze({
|
|
11833
|
+
...legacyResourcePolicyContent,
|
|
11834
|
+
policy_sha256: jcsSha256(legacyResourcePolicyContent)
|
|
11835
|
+
});
|
|
11836
|
+
var resourcePolicyContent = {
|
|
11837
|
+
...legacyResourcePolicyContent,
|
|
11838
|
+
policy_id: RESOURCE_POLICY_ID,
|
|
11839
|
+
request_bounds: {
|
|
11840
|
+
...legacyResourcePolicyContent.request_bounds,
|
|
11841
|
+
max_window_ms: 31 * 24 * 60 * 60 * 1000
|
|
11842
|
+
}
|
|
11843
|
+
};
|
|
12142
11844
|
var RESOURCE_POLICY = Object.freeze({
|
|
12143
11845
|
...resourcePolicyContent,
|
|
12144
11846
|
policy_sha256: jcsSha256(resourcePolicyContent)
|
|
12145
11847
|
});
|
|
12146
11848
|
var EFFECTIVE_ADAPTER_POLICY_PIN = Object.freeze({
|
|
12147
11849
|
policy_id: ADAPTER_POLICY_ID,
|
|
12148
|
-
policy_sha256: jcsSha256(
|
|
11850
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent.adapter_policy)
|
|
12149
11851
|
});
|
|
12150
11852
|
var EFFECTIVE_ACQUISITION_POLICY_PIN = Object.freeze({
|
|
12151
11853
|
policy_id: ACQUISITION_POLICY_ID,
|
|
12152
|
-
policy_sha256: jcsSha256(
|
|
11854
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent.acquisition_policy)
|
|
12153
11855
|
});
|
|
12154
11856
|
var schemaArtifacts = [
|
|
12155
11857
|
["schemas/request.schema.json", request_schema_default],
|
|
@@ -12329,7 +12031,15 @@ var backfillRequestCodec = {
|
|
|
12329
12031
|
if (request.idempotency_key !== createBackfillIdempotencyKey(request)) {
|
|
12330
12032
|
throw new Error("idempotency_key does not match canonical business fields");
|
|
12331
12033
|
}
|
|
12332
|
-
|
|
12034
|
+
const capabilityPolicyMatches = [
|
|
12035
|
+
CAPABILITY_POLICY,
|
|
12036
|
+
LEGACY_CAPABILITY_POLICY
|
|
12037
|
+
].some((policy) => request.product_pins.capability_policy.policy_id === policy.policy_id && request.product_pins.capability_policy.policy_sha256 === policy.policy_sha256);
|
|
12038
|
+
const resourcePolicyMatches = [
|
|
12039
|
+
RESOURCE_POLICY,
|
|
12040
|
+
LEGACY_RESOURCE_POLICY
|
|
12041
|
+
].some((policy) => request.product_pins.resource_policy.policy_id === policy.policy_id && request.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
|
|
12042
|
+
if (!capabilityPolicyMatches || !resourcePolicyMatches) {
|
|
12333
12043
|
throw new Error("request policy pins do not match the effective package policies");
|
|
12334
12044
|
}
|
|
12335
12045
|
if (request.required_clock.clock_id !== request.initial_selection.required_clock.clock_id || request.required_clock.clock_sha256 !== request.initial_selection.required_clock.clock_sha256 || request.required_clock.event_count !== request.initial_selection.required_clock.event_count) {
|
|
@@ -12457,6 +12167,10 @@ function decodeBackfillRunDocuments(input) {
|
|
|
12457
12167
|
return targetTimeMs;
|
|
12458
12168
|
});
|
|
12459
12169
|
const capabilityProfile = CAPABILITY_POLICY.profiles.find((profile) => profile.exchange === wire.scope.exchange && profile.market_type === wire.scope.market_type && profile.feed === wire.scope.feed && profile.canonical_trading_pair === wire.scope.trading_pair);
|
|
12170
|
+
const resourcePolicy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((policy) => wire.product_pins.resource_policy.policy_id === policy.policy_id && wire.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
|
|
12171
|
+
if (!resourcePolicy) {
|
|
12172
|
+
throw new Error("request resource policy pin is unsupported");
|
|
12173
|
+
}
|
|
12460
12174
|
return {
|
|
12461
12175
|
schemaVersion: BACKFILL_REQUEST_SCHEMA_VERSION,
|
|
12462
12176
|
requestId: wire.request_id,
|
|
@@ -12481,11 +12195,11 @@ function decodeBackfillRunDocuments(input) {
|
|
|
12481
12195
|
maxPriorAsOfLagMs: wire.coverage_policy.max_asof_lag_ms,
|
|
12482
12196
|
sourcePolicy: wire.source_policy,
|
|
12483
12197
|
budgets: {
|
|
12484
|
-
maxFiles:
|
|
12485
|
-
maxBytes:
|
|
12486
|
-
maxRows:
|
|
12487
|
-
maxDurationMs:
|
|
12488
|
-
maxBoundaryLookbackMs: Math.min(
|
|
12198
|
+
maxFiles: resourcePolicy.limits.max_files,
|
|
12199
|
+
maxBytes: resourcePolicy.limits.max_bytes,
|
|
12200
|
+
maxRows: resourcePolicy.limits.max_rows,
|
|
12201
|
+
maxDurationMs: resourcePolicy.limits.max_duration_ms,
|
|
12202
|
+
maxBoundaryLookbackMs: Math.min(resourcePolicy.limits.max_boundary_lookback_ms, CAPABILITY_POLICY.acquisition_policy.initialization_lookback_ms)
|
|
12489
12203
|
},
|
|
12490
12204
|
expectedProduct: {
|
|
12491
12205
|
packageName: "@usherlabs/cex-broker",
|
|
@@ -12763,12 +12477,13 @@ function assertStoredSelectionMatchesRequest(request, selection) {
|
|
|
12763
12477
|
}
|
|
12764
12478
|
}
|
|
12765
12479
|
function eligibleBundles(request, bundles) {
|
|
12766
|
-
|
|
12767
|
-
|
|
12768
|
-
|
|
12769
|
-
|
|
12770
|
-
return
|
|
12771
|
-
}
|
|
12480
|
+
const qualifiedVendorBundles = bundles.filter((bundle) => bundle.captureOrigin === "vendor_historical_backfill" && bundle.qualification?.state === "qualified");
|
|
12481
|
+
const productionBundles = bundles.filter((bundle) => bundle.captureOrigin === "production_capture");
|
|
12482
|
+
if (request.sourcePolicy === "authoritative_window") {
|
|
12483
|
+
const vendorSupport = chooseSupport(request, qualifiedVendorBundles);
|
|
12484
|
+
return vendorSupport.length === request.requiredClockTargetsMs.length ? qualifiedVendorBundles : productionBundles;
|
|
12485
|
+
}
|
|
12486
|
+
return [...productionBundles, ...qualifiedVendorBundles];
|
|
12772
12487
|
}
|
|
12773
12488
|
function originPriority(request, origin) {
|
|
12774
12489
|
if (request.sourcePolicy === "fill_gaps") {
|
|
@@ -12973,9 +12688,8 @@ function exactCandidateMatch(expected, actual) {
|
|
|
12973
12688
|
function requestedShapeMatches(request, rows) {
|
|
12974
12689
|
return rows.every(({ row }) => row.depth_limit === request.depth && row.construction_mode === request.constructionMode && row.schema_version === request.expectedProduct.canonicalSchemaVersion);
|
|
12975
12690
|
}
|
|
12976
|
-
function requiredClockCoverage(request, rows) {
|
|
12977
|
-
const summaryTimes = rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => {
|
|
12978
|
-
const value = row.source_time_ms;
|
|
12691
|
+
function requiredClockCoverage(request, rows, coverageSourceTimesMs) {
|
|
12692
|
+
const summaryTimes = (coverageSourceTimesMs ?? rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => row.source_time_ms)).map((value) => {
|
|
12979
12693
|
if (typeof value !== "number" && (typeof value !== "string" || !/^\d+$/.test(value))) {
|
|
12980
12694
|
return;
|
|
12981
12695
|
}
|
|
@@ -13048,7 +12762,7 @@ function verifySemanticPromotion(evidence) {
|
|
|
13048
12762
|
coverageVerified: false
|
|
13049
12763
|
};
|
|
13050
12764
|
}
|
|
13051
|
-
const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows);
|
|
12765
|
+
const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows, evidence.coverageSourceTimesMs);
|
|
13052
12766
|
if (!coverageVerified) {
|
|
13053
12767
|
return {
|
|
13054
12768
|
...base,
|
|
@@ -13423,7 +13137,13 @@ class QualifiedOrderBookArchiveReader {
|
|
|
13423
13137
|
suffixDigestBefore: baseline.verificationBaseline.suffixDigest,
|
|
13424
13138
|
suffixDigestAfter: timelineDigest(suffix),
|
|
13425
13139
|
seamVerified: this.seamIsOrdered(queriedRows),
|
|
13426
|
-
exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string")
|
|
13140
|
+
exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string"),
|
|
13141
|
+
...request.sourcePolicy === "fill_gaps" ? {
|
|
13142
|
+
coverageSourceTimesMs: [
|
|
13143
|
+
...baseline.selection.support_anchors.map((anchor) => Date.parse(anchor.source_time)),
|
|
13144
|
+
...summaries.map((row) => numberField(row, "source_time_ms"))
|
|
13145
|
+
]
|
|
13146
|
+
} : {}
|
|
13427
13147
|
});
|
|
13428
13148
|
return { ...semantic, captureBundleId };
|
|
13429
13149
|
}
|
|
@@ -13612,12 +13332,12 @@ var requestContent = {
|
|
|
13612
13332
|
},
|
|
13613
13333
|
product_pins: {
|
|
13614
13334
|
capability_policy: {
|
|
13615
|
-
policy_id:
|
|
13616
|
-
policy_sha256:
|
|
13335
|
+
policy_id: LEGACY_CAPABILITY_POLICY.policy_id,
|
|
13336
|
+
policy_sha256: LEGACY_CAPABILITY_POLICY.policy_sha256
|
|
13617
13337
|
},
|
|
13618
13338
|
resource_policy: {
|
|
13619
|
-
policy_id:
|
|
13620
|
-
policy_sha256:
|
|
13339
|
+
policy_id: LEGACY_RESOURCE_POLICY.policy_id,
|
|
13340
|
+
policy_sha256: LEGACY_RESOURCE_POLICY.policy_sha256
|
|
13621
13341
|
}
|
|
13622
13342
|
},
|
|
13623
13343
|
production_authorization_id: authorizationId
|
|
@@ -13766,878 +13486,470 @@ var CONFORMANCE_FIXTURES = Object.freeze({
|
|
|
13766
13486
|
promotion_identity_sha256: promotionReceipt.promotion_identity_sha256,
|
|
13767
13487
|
receipt_id: promotionReceipt.receipt_id,
|
|
13768
13488
|
result_sha256: result.result_sha256,
|
|
13769
|
-
capability_policy_sha256:
|
|
13770
|
-
resource_policy_sha256:
|
|
13489
|
+
capability_policy_sha256: LEGACY_CAPABILITY_POLICY.policy_sha256,
|
|
13490
|
+
resource_policy_sha256: LEGACY_RESOURCE_POLICY.policy_sha256
|
|
13771
13491
|
},
|
|
13772
13492
|
jcs_edge_vector: jcsEdgeVector,
|
|
13773
13493
|
hashes: { jcs_edge_vector_sha256: jcsSha256(jcsEdgeVector) }
|
|
13774
13494
|
});
|
|
13775
|
-
// src/helpers/market-data-vendor-backfill/
|
|
13776
|
-
|
|
13777
|
-
|
|
13778
|
-
|
|
13779
|
-
var
|
|
13780
|
-
|
|
13781
|
-
|
|
13782
|
-
|
|
13783
|
-
|
|
13784
|
-
|
|
13785
|
-
|
|
13786
|
-
|
|
13787
|
-
|
|
13788
|
-
|
|
13789
|
-
|
|
13790
|
-
|
|
13791
|
-
|
|
13792
|
-
|
|
13793
|
-
|
|
13794
|
-
|
|
13795
|
-
|
|
13796
|
-
|
|
13797
|
-
|
|
13798
|
-
|
|
13799
|
-
|
|
13800
|
-
|
|
13801
|
-
|
|
13802
|
-
|
|
13803
|
-
|
|
13804
|
-
|
|
13805
|
-
|
|
13806
|
-
|
|
13807
|
-
|
|
13808
|
-
|
|
13809
|
-
|
|
13495
|
+
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
13496
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
13497
|
+
|
|
13498
|
+
// node_modules/fzstd/esm/index.mjs
|
|
13499
|
+
var ab = ArrayBuffer;
|
|
13500
|
+
var u8 = Uint8Array;
|
|
13501
|
+
var u16 = Uint16Array;
|
|
13502
|
+
var i16 = Int16Array;
|
|
13503
|
+
var i32 = Int32Array;
|
|
13504
|
+
var slc = function(v, s, e) {
|
|
13505
|
+
if (u8.prototype.slice)
|
|
13506
|
+
return u8.prototype.slice.call(v, s, e);
|
|
13507
|
+
if (s == null || s < 0)
|
|
13508
|
+
s = 0;
|
|
13509
|
+
if (e == null || e > v.length)
|
|
13510
|
+
e = v.length;
|
|
13511
|
+
var n = new u8(e - s);
|
|
13512
|
+
n.set(v.subarray(s, e));
|
|
13513
|
+
return n;
|
|
13514
|
+
};
|
|
13515
|
+
var fill = function(v, n, s, e) {
|
|
13516
|
+
if (u8.prototype.fill)
|
|
13517
|
+
return u8.prototype.fill.call(v, n, s, e);
|
|
13518
|
+
if (s == null || s < 0)
|
|
13519
|
+
s = 0;
|
|
13520
|
+
if (e == null || e > v.length)
|
|
13521
|
+
e = v.length;
|
|
13522
|
+
for (;s < e; ++s)
|
|
13523
|
+
v[s] = n;
|
|
13524
|
+
return v;
|
|
13525
|
+
};
|
|
13526
|
+
var cpw = function(v, t, s, e) {
|
|
13527
|
+
if (u8.prototype.copyWithin)
|
|
13528
|
+
return u8.prototype.copyWithin.call(v, t, s, e);
|
|
13529
|
+
if (s == null || s < 0)
|
|
13530
|
+
s = 0;
|
|
13531
|
+
if (e == null || e > v.length)
|
|
13532
|
+
e = v.length;
|
|
13533
|
+
while (s < e) {
|
|
13534
|
+
v[t++] = v[s++];
|
|
13810
13535
|
}
|
|
13811
|
-
|
|
13812
|
-
|
|
13813
|
-
|
|
13814
|
-
|
|
13815
|
-
|
|
13816
|
-
|
|
13817
|
-
|
|
13818
|
-
|
|
13819
|
-
|
|
13820
|
-
|
|
13536
|
+
};
|
|
13537
|
+
var ec = [
|
|
13538
|
+
"invalid zstd data",
|
|
13539
|
+
"window size too large (>2046MB)",
|
|
13540
|
+
"invalid block type",
|
|
13541
|
+
"FSE accuracy too high",
|
|
13542
|
+
"match distance too far back",
|
|
13543
|
+
"unexpected EOF"
|
|
13544
|
+
];
|
|
13545
|
+
var err = function(ind, msg, nt) {
|
|
13546
|
+
var e = new Error(msg || ec[ind]);
|
|
13547
|
+
e.code = ind;
|
|
13548
|
+
if (Error.captureStackTrace)
|
|
13549
|
+
Error.captureStackTrace(e, err);
|
|
13550
|
+
if (!nt)
|
|
13551
|
+
throw e;
|
|
13552
|
+
return e;
|
|
13553
|
+
};
|
|
13554
|
+
var rb = function(d, b, n) {
|
|
13555
|
+
var i = 0, o = 0;
|
|
13556
|
+
for (;i < n; ++i)
|
|
13557
|
+
o |= d[b++] << (i << 3);
|
|
13558
|
+
return o;
|
|
13559
|
+
};
|
|
13560
|
+
var b4 = function(d, b) {
|
|
13561
|
+
return (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0;
|
|
13562
|
+
};
|
|
13563
|
+
var rzfh = function(dat, w) {
|
|
13564
|
+
var n3 = dat[0] | dat[1] << 8 | dat[2] << 16;
|
|
13565
|
+
if (n3 == 3126568 && dat[3] == 253) {
|
|
13566
|
+
var flg = dat[4];
|
|
13567
|
+
var ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6;
|
|
13568
|
+
if (flg & 8)
|
|
13569
|
+
err(0);
|
|
13570
|
+
var bt = 6 - ss;
|
|
13571
|
+
var db = df == 3 ? 4 : df;
|
|
13572
|
+
var di = rb(dat, bt, db);
|
|
13573
|
+
bt += db;
|
|
13574
|
+
var fsb = fcf ? 1 << fcf : ss;
|
|
13575
|
+
var fss = rb(dat, bt, fsb) + (fcf == 1 && 256);
|
|
13576
|
+
var ws = fss;
|
|
13577
|
+
if (!ss) {
|
|
13578
|
+
var wb = 1 << 10 + (dat[5] >> 3);
|
|
13579
|
+
ws = wb + (wb >> 3) * (dat[5] & 7);
|
|
13580
|
+
}
|
|
13581
|
+
if (ws > 2145386496)
|
|
13582
|
+
err(1);
|
|
13583
|
+
var buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
|
|
13584
|
+
buf[0] = 1, buf[4] = 4, buf[8] = 8;
|
|
13585
|
+
return {
|
|
13586
|
+
b: bt + fsb,
|
|
13587
|
+
y: 0,
|
|
13588
|
+
l: 0,
|
|
13589
|
+
d: di,
|
|
13590
|
+
w: w && w != 1 ? w : buf.subarray(12),
|
|
13591
|
+
e: ws,
|
|
13592
|
+
o: new i32(buf.buffer, 0, 3),
|
|
13593
|
+
u: fss,
|
|
13594
|
+
c: cc,
|
|
13595
|
+
m: Math.min(131072, ws)
|
|
13596
|
+
};
|
|
13597
|
+
} else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
|
|
13598
|
+
return b4(dat, 4) + 8;
|
|
13821
13599
|
}
|
|
13822
|
-
|
|
13823
|
-
}
|
|
13824
|
-
function
|
|
13825
|
-
|
|
13826
|
-
|
|
13827
|
-
|
|
13828
|
-
|
|
13829
|
-
|
|
13830
|
-
|
|
13831
|
-
|
|
13832
|
-
|
|
13833
|
-
|
|
13834
|
-
|
|
13835
|
-
|
|
13836
|
-
|
|
13837
|
-
|
|
13838
|
-
|
|
13839
|
-
|
|
13840
|
-
|
|
13841
|
-
|
|
13842
|
-
|
|
13843
|
-
|
|
13844
|
-
|
|
13845
|
-
|
|
13846
|
-
|
|
13847
|
-
|
|
13848
|
-
|
|
13849
|
-
|
|
13600
|
+
err(0);
|
|
13601
|
+
};
|
|
13602
|
+
var msb = function(val) {
|
|
13603
|
+
var bits = 0;
|
|
13604
|
+
for (;1 << bits <= val; ++bits)
|
|
13605
|
+
;
|
|
13606
|
+
return bits - 1;
|
|
13607
|
+
};
|
|
13608
|
+
var rfse = function(dat, bt, mal) {
|
|
13609
|
+
var tpos = (bt << 3) + 4;
|
|
13610
|
+
var al = (dat[bt] & 15) + 5;
|
|
13611
|
+
if (al > mal)
|
|
13612
|
+
err(3);
|
|
13613
|
+
var sz = 1 << al;
|
|
13614
|
+
var probs = sz, sym = -1, re = -1, i = -1, ht = sz;
|
|
13615
|
+
var buf = new ab(512 + (sz << 2));
|
|
13616
|
+
var freq = new i16(buf, 0, 256);
|
|
13617
|
+
var dstate = new u16(buf, 0, 256);
|
|
13618
|
+
var nstate = new u16(buf, 512, sz);
|
|
13619
|
+
var bb1 = 512 + (sz << 1);
|
|
13620
|
+
var syms = new u8(buf, bb1, sz);
|
|
13621
|
+
var nbits = new u8(buf, bb1 + sz);
|
|
13622
|
+
while (sym < 255 && probs > 0) {
|
|
13623
|
+
var bits = msb(probs + 1);
|
|
13624
|
+
var cbt = tpos >> 3;
|
|
13625
|
+
var msk = (1 << bits + 1) - 1;
|
|
13626
|
+
var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
|
|
13627
|
+
var msk1fb = (1 << bits) - 1;
|
|
13628
|
+
var msv = msk - probs - 1;
|
|
13629
|
+
var sval = val & msk1fb;
|
|
13630
|
+
if (sval < msv)
|
|
13631
|
+
tpos += bits, val = sval;
|
|
13632
|
+
else {
|
|
13633
|
+
tpos += bits + 1;
|
|
13634
|
+
if (val > msk1fb)
|
|
13635
|
+
val -= msv;
|
|
13850
13636
|
}
|
|
13851
|
-
|
|
13852
|
-
|
|
13853
|
-
|
|
13854
|
-
|
|
13855
|
-
|
|
13856
|
-
|
|
13857
|
-
|
|
13858
|
-
|
|
13859
|
-
|
|
13860
|
-
|
|
13861
|
-
|
|
13862
|
-
|
|
13863
|
-
|
|
13864
|
-
throw new TypeError(`Backfill dependency ${name} is required`);
|
|
13637
|
+
freq[++sym] = --val;
|
|
13638
|
+
if (val == -1) {
|
|
13639
|
+
probs += val;
|
|
13640
|
+
syms[--ht] = sym;
|
|
13641
|
+
} else
|
|
13642
|
+
probs -= val;
|
|
13643
|
+
if (!val) {
|
|
13644
|
+
do {
|
|
13645
|
+
var rbt = tpos >> 3;
|
|
13646
|
+
re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
|
|
13647
|
+
tpos += 2;
|
|
13648
|
+
sym += re;
|
|
13649
|
+
} while (re == 3);
|
|
13865
13650
|
}
|
|
13866
13651
|
}
|
|
13867
|
-
|
|
13868
|
-
|
|
13869
|
-
|
|
13870
|
-
|
|
13871
|
-
|
|
13872
|
-
|
|
13873
|
-
|
|
13874
|
-
|
|
13875
|
-
|
|
13876
|
-
|
|
13877
|
-
|
|
13878
|
-
|
|
13879
|
-
|
|
13652
|
+
if (sym > 255 || probs)
|
|
13653
|
+
err(0);
|
|
13654
|
+
var sympos = 0;
|
|
13655
|
+
var sstep = (sz >> 1) + (sz >> 3) + 3;
|
|
13656
|
+
var smask = sz - 1;
|
|
13657
|
+
for (var s = 0;s <= sym; ++s) {
|
|
13658
|
+
var sf = freq[s];
|
|
13659
|
+
if (sf < 1) {
|
|
13660
|
+
dstate[s] = -sf;
|
|
13661
|
+
continue;
|
|
13662
|
+
}
|
|
13663
|
+
for (i = 0;i < sf; ++i) {
|
|
13664
|
+
syms[sympos] = s;
|
|
13665
|
+
do {
|
|
13666
|
+
sympos = sympos + sstep & smask;
|
|
13667
|
+
} while (sympos >= ht);
|
|
13880
13668
|
}
|
|
13881
13669
|
}
|
|
13882
|
-
|
|
13883
|
-
|
|
13884
|
-
|
|
13885
|
-
|
|
13886
|
-
|
|
13887
|
-
|
|
13888
|
-
...request2 ? {
|
|
13889
|
-
requestId: request2.requestId,
|
|
13890
|
-
idempotencyKey: request2.idempotencyKey,
|
|
13891
|
-
...request2.target ? { target: request2.target } : {}
|
|
13892
|
-
} : {},
|
|
13893
|
-
...extra
|
|
13894
|
-
};
|
|
13895
|
-
}
|
|
13896
|
-
function captureBundleId2(request2, capability, dataset) {
|
|
13897
|
-
return sha256Canonical({
|
|
13898
|
-
request_business_identity: request2.idempotencyKey,
|
|
13899
|
-
provider: capability.provider,
|
|
13900
|
-
provider_exchange_id: capability.providerExchangeId,
|
|
13901
|
-
resolved_symbol: capability.resolvedSymbol,
|
|
13902
|
-
adapter_version: capability.adapterVersion,
|
|
13903
|
-
objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
|
|
13904
|
-
identity,
|
|
13905
|
-
checksum,
|
|
13906
|
-
bytes,
|
|
13907
|
-
rows
|
|
13908
|
-
})),
|
|
13909
|
-
canonical_schema_version: request2.expectedProduct.canonicalSchemaVersion,
|
|
13910
|
-
checksum_algorithm: request2.expectedProduct.checksumAlgorithm
|
|
13911
|
-
});
|
|
13912
|
-
}
|
|
13913
|
-
function buildPromotionReceipt(request2, capability, normalized, verification, verificationTimeMs) {
|
|
13914
|
-
if (!request2.wire || !request2.initialSelection || !request2.expectedCanonicalSchema || !request2.coveragePolicy || !request2.productPins) {
|
|
13915
|
-
throw new Error("decoded final-v1 request context is missing");
|
|
13670
|
+
if (sympos)
|
|
13671
|
+
err(0);
|
|
13672
|
+
for (i = 0;i < sz; ++i) {
|
|
13673
|
+
var ns = dstate[syms[i]]++;
|
|
13674
|
+
var nb = nbits[i] = al - msb(ns);
|
|
13675
|
+
nstate[i] = (ns << nb) - sz;
|
|
13916
13676
|
}
|
|
13917
|
-
return
|
|
13918
|
-
|
|
13919
|
-
|
|
13920
|
-
|
|
13921
|
-
|
|
13922
|
-
|
|
13923
|
-
|
|
13924
|
-
|
|
13925
|
-
|
|
13926
|
-
|
|
13927
|
-
|
|
13928
|
-
|
|
13929
|
-
|
|
13930
|
-
|
|
13931
|
-
|
|
13932
|
-
|
|
13933
|
-
|
|
13934
|
-
|
|
13935
|
-
|
|
13936
|
-
|
|
13937
|
-
|
|
13938
|
-
|
|
13939
|
-
|
|
13940
|
-
|
|
13941
|
-
|
|
13942
|
-
canonical_semantic_digest: verification.canonicalSemanticDigest,
|
|
13943
|
-
prefix_digest: verification.prefixDigest,
|
|
13944
|
-
suffix_digest: verification.suffixDigest,
|
|
13945
|
-
seam_verified: true,
|
|
13946
|
-
coverage_verified: true,
|
|
13947
|
-
dataset_objects: normalized.objects
|
|
13948
|
-
});
|
|
13949
|
-
}
|
|
13950
|
-
async function submitAll(dependencies, batches) {
|
|
13951
|
-
const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
|
|
13952
|
-
for (const batch of batches) {
|
|
13953
|
-
let accepted = false;
|
|
13954
|
-
for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
|
|
13955
|
-
try {
|
|
13956
|
-
const response = await dependencies.forwarder.submit(batch);
|
|
13957
|
-
accepted = response.ok && response.inserted === batch.rows.length;
|
|
13958
|
-
} catch {
|
|
13959
|
-
accepted = false;
|
|
13960
|
-
}
|
|
13961
|
-
if (accepted)
|
|
13677
|
+
return [tpos + 7 >> 3, {
|
|
13678
|
+
b: al,
|
|
13679
|
+
s: syms,
|
|
13680
|
+
n: nbits,
|
|
13681
|
+
t: nstate
|
|
13682
|
+
}];
|
|
13683
|
+
};
|
|
13684
|
+
var rhu = function(dat, bt) {
|
|
13685
|
+
var i = 0, wc = -1;
|
|
13686
|
+
var buf = new u8(292), hb = dat[bt];
|
|
13687
|
+
var hw = buf.subarray(0, 256);
|
|
13688
|
+
var rc = buf.subarray(256, 268);
|
|
13689
|
+
var ri = new u16(buf.buffer, 268);
|
|
13690
|
+
if (hb < 128) {
|
|
13691
|
+
var _a = rfse(dat, bt + 1, 6), ebt = _a[0], fdt = _a[1];
|
|
13692
|
+
bt += hb;
|
|
13693
|
+
var epos = ebt << 3;
|
|
13694
|
+
var lb = dat[bt];
|
|
13695
|
+
if (!lb)
|
|
13696
|
+
err(0);
|
|
13697
|
+
var st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
|
|
13698
|
+
var fpos = (++bt << 3) - 8 + msb(lb);
|
|
13699
|
+
for (;; ) {
|
|
13700
|
+
fpos -= btr1;
|
|
13701
|
+
if (fpos < epos)
|
|
13962
13702
|
break;
|
|
13963
|
-
|
|
13964
|
-
|
|
13703
|
+
var cbt = fpos >> 3;
|
|
13704
|
+
st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
|
|
13705
|
+
hw[++wc] = fdt.s[st1];
|
|
13706
|
+
fpos -= btr2;
|
|
13707
|
+
if (fpos < epos)
|
|
13708
|
+
break;
|
|
13709
|
+
cbt = fpos >> 3;
|
|
13710
|
+
st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
|
|
13711
|
+
hw[++wc] = fdt.s[st2];
|
|
13712
|
+
btr1 = fdt.n[st1];
|
|
13713
|
+
st1 = fdt.t[st1];
|
|
13714
|
+
btr2 = fdt.n[st2];
|
|
13715
|
+
st2 = fdt.t[st2];
|
|
13965
13716
|
}
|
|
13966
|
-
if (
|
|
13967
|
-
|
|
13968
|
-
}
|
|
13969
|
-
|
|
13970
|
-
|
|
13971
|
-
|
|
13972
|
-
|
|
13973
|
-
|
|
13974
|
-
if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
|
|
13975
|
-
return reason;
|
|
13717
|
+
if (++wc > 255)
|
|
13718
|
+
err(0);
|
|
13719
|
+
} else {
|
|
13720
|
+
wc = hb - 127;
|
|
13721
|
+
for (;i < wc; i += 2) {
|
|
13722
|
+
var byte = dat[++bt];
|
|
13723
|
+
hw[i] = byte >> 4;
|
|
13724
|
+
hw[i + 1] = byte & 15;
|
|
13976
13725
|
}
|
|
13726
|
+
++bt;
|
|
13977
13727
|
}
|
|
13978
|
-
|
|
13979
|
-
|
|
13980
|
-
|
|
13981
|
-
|
|
13982
|
-
|
|
13728
|
+
var wes = 0;
|
|
13729
|
+
for (i = 0;i < wc; ++i) {
|
|
13730
|
+
var wt = hw[i];
|
|
13731
|
+
if (wt > 11)
|
|
13732
|
+
err(0);
|
|
13733
|
+
wes += wt && 1 << wt - 1;
|
|
13983
13734
|
}
|
|
13984
|
-
|
|
13985
|
-
|
|
13986
|
-
|
|
13987
|
-
|
|
13988
|
-
|
|
13989
|
-
|
|
13990
|
-
|
|
13991
|
-
|
|
13992
|
-
|
|
13735
|
+
var mb = msb(wes) + 1;
|
|
13736
|
+
var ts = 1 << mb;
|
|
13737
|
+
var rem = ts - wes;
|
|
13738
|
+
if (rem & rem - 1)
|
|
13739
|
+
err(0);
|
|
13740
|
+
hw[wc++] = msb(rem) + 1;
|
|
13741
|
+
for (i = 0;i < wc; ++i) {
|
|
13742
|
+
var wt = hw[i];
|
|
13743
|
+
++rc[hw[i] = wt && mb + 1 - wt];
|
|
13993
13744
|
}
|
|
13994
|
-
|
|
13995
|
-
|
|
13996
|
-
|
|
13997
|
-
|
|
13745
|
+
var hbuf = new u8(ts << 1);
|
|
13746
|
+
var syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
|
|
13747
|
+
ri[mb] = 0;
|
|
13748
|
+
for (i = mb;i > 0; --i) {
|
|
13749
|
+
var pv = ri[i];
|
|
13750
|
+
fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
|
|
13998
13751
|
}
|
|
13999
|
-
if (
|
|
14000
|
-
|
|
13752
|
+
if (ri[0] != ts)
|
|
13753
|
+
err(0);
|
|
13754
|
+
for (i = 0;i < wc; ++i) {
|
|
13755
|
+
var bits = hw[i];
|
|
13756
|
+
if (bits) {
|
|
13757
|
+
var code = ri[bits];
|
|
13758
|
+
fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
|
|
13759
|
+
}
|
|
14001
13760
|
}
|
|
14002
|
-
|
|
14003
|
-
|
|
14004
|
-
|
|
14005
|
-
|
|
13761
|
+
return [bt, {
|
|
13762
|
+
n: nb,
|
|
13763
|
+
b: mb,
|
|
13764
|
+
s: syms
|
|
13765
|
+
}];
|
|
13766
|
+
};
|
|
13767
|
+
var dllt = rfse(/* @__PURE__ */ new u8([
|
|
13768
|
+
81,
|
|
13769
|
+
16,
|
|
13770
|
+
99,
|
|
13771
|
+
140,
|
|
13772
|
+
49,
|
|
13773
|
+
198,
|
|
13774
|
+
24,
|
|
13775
|
+
99,
|
|
13776
|
+
12,
|
|
13777
|
+
33,
|
|
13778
|
+
196,
|
|
13779
|
+
24,
|
|
13780
|
+
99,
|
|
13781
|
+
102,
|
|
13782
|
+
102,
|
|
13783
|
+
134,
|
|
13784
|
+
70,
|
|
13785
|
+
146,
|
|
13786
|
+
4
|
|
13787
|
+
]), 0, 6)[1];
|
|
13788
|
+
var dmlt = rfse(/* @__PURE__ */ new u8([
|
|
13789
|
+
33,
|
|
13790
|
+
20,
|
|
13791
|
+
196,
|
|
13792
|
+
24,
|
|
13793
|
+
99,
|
|
13794
|
+
140,
|
|
13795
|
+
33,
|
|
13796
|
+
132,
|
|
13797
|
+
16,
|
|
13798
|
+
66,
|
|
13799
|
+
8,
|
|
13800
|
+
33,
|
|
13801
|
+
132,
|
|
13802
|
+
16,
|
|
13803
|
+
66,
|
|
13804
|
+
8,
|
|
13805
|
+
33,
|
|
13806
|
+
68,
|
|
13807
|
+
68,
|
|
13808
|
+
68,
|
|
13809
|
+
68,
|
|
13810
|
+
68,
|
|
13811
|
+
68,
|
|
13812
|
+
68,
|
|
13813
|
+
68,
|
|
13814
|
+
36,
|
|
13815
|
+
9
|
|
13816
|
+
]), 0, 6)[1];
|
|
13817
|
+
var doct = rfse(/* @__PURE__ */ new u8([
|
|
13818
|
+
32,
|
|
13819
|
+
132,
|
|
13820
|
+
16,
|
|
13821
|
+
66,
|
|
13822
|
+
102,
|
|
13823
|
+
70,
|
|
13824
|
+
68,
|
|
13825
|
+
68,
|
|
13826
|
+
68,
|
|
13827
|
+
68,
|
|
13828
|
+
36,
|
|
13829
|
+
73,
|
|
13830
|
+
2
|
|
13831
|
+
]), 0, 5)[1];
|
|
13832
|
+
var b2bl = function(b, s) {
|
|
13833
|
+
var len = b.length, bl = new i32(len);
|
|
13834
|
+
for (var i = 0;i < len; ++i) {
|
|
13835
|
+
bl[i] = s;
|
|
13836
|
+
s += 1 << b[i];
|
|
14006
13837
|
}
|
|
14007
|
-
|
|
14008
|
-
|
|
13838
|
+
return bl;
|
|
13839
|
+
};
|
|
13840
|
+
var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
13841
|
+
0,
|
|
13842
|
+
0,
|
|
13843
|
+
0,
|
|
13844
|
+
0,
|
|
13845
|
+
16843009,
|
|
13846
|
+
50528770,
|
|
13847
|
+
134678020,
|
|
13848
|
+
202050057,
|
|
13849
|
+
269422093
|
|
13850
|
+
])).buffer, 0, 36);
|
|
13851
|
+
var llbl = /* @__PURE__ */ b2bl(llb, 0);
|
|
13852
|
+
var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
13853
|
+
0,
|
|
13854
|
+
0,
|
|
13855
|
+
0,
|
|
13856
|
+
0,
|
|
13857
|
+
0,
|
|
13858
|
+
0,
|
|
13859
|
+
0,
|
|
13860
|
+
0,
|
|
13861
|
+
16843009,
|
|
13862
|
+
50528770,
|
|
13863
|
+
117769220,
|
|
13864
|
+
185207048,
|
|
13865
|
+
252579084,
|
|
13866
|
+
16
|
|
13867
|
+
])).buffer, 0, 53);
|
|
13868
|
+
var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
|
|
13869
|
+
var dhu = function(dat, out, hu) {
|
|
13870
|
+
var len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
|
|
13871
|
+
if (!lb)
|
|
13872
|
+
err(0);
|
|
13873
|
+
var st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
|
|
13874
|
+
for (;pos > eb && i < ss; ) {
|
|
13875
|
+
var cbt = pos >> 3;
|
|
13876
|
+
var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
|
|
13877
|
+
st = (st << btr | val) & msk;
|
|
13878
|
+
out[++i] = hu.s[st];
|
|
13879
|
+
pos -= btr = hu.n[st];
|
|
14009
13880
|
}
|
|
14010
|
-
|
|
14011
|
-
|
|
14012
|
-
|
|
14013
|
-
|
|
14014
|
-
|
|
14015
|
-
|
|
14016
|
-
|
|
14017
|
-
|
|
14018
|
-
|
|
14019
|
-
|
|
14020
|
-
|
|
14021
|
-
|
|
14022
|
-
|
|
14023
|
-
|
|
14024
|
-
|
|
14025
|
-
|
|
14026
|
-
|
|
14027
|
-
|
|
14028
|
-
|
|
14029
|
-
|
|
14030
|
-
|
|
14031
|
-
|
|
14032
|
-
|
|
14033
|
-
|
|
14034
|
-
|
|
14035
|
-
try {
|
|
14036
|
-
initialResolution = await dependencies.archive.resolveSelection(request2);
|
|
14037
|
-
assertArchivePreflight(request2, initialResolution);
|
|
14038
|
-
const forwarderPreflight = await dependencies.forwarder.preflight({
|
|
14039
|
-
authorizationId: request2.productionAuthorizationId,
|
|
14040
|
-
target: request2.target
|
|
14041
|
-
});
|
|
14042
|
-
assertForwarderPreflight(request2, forwarderPreflight, dependencies.clock.nowMs());
|
|
14043
|
-
} catch {
|
|
14044
|
-
return outcome(request2, "archive_preflight_failed", "archive_preflight_failed");
|
|
14045
|
-
}
|
|
14046
|
-
if (initialResolution.selection.coverage_class === "complete") {
|
|
14047
|
-
const receipt2 = storedReceiptForSelection(initialResolution);
|
|
14048
|
-
return outcome(request2, "already_covered", "qualified_coverage_complete", {
|
|
14049
|
-
selection: initialResolution.selection,
|
|
14050
|
-
...receipt2 ? { receipt: receipt2 } : {}
|
|
14051
|
-
});
|
|
14052
|
-
}
|
|
14053
|
-
if (resourcePolicyScopeExceeded(request2)) {
|
|
14054
|
-
return outcome(request2, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
|
|
14055
|
-
}
|
|
14056
|
-
let capability;
|
|
14057
|
-
try {
|
|
14058
|
-
capability = dependencies.providers.capabilityFor(request2);
|
|
14059
|
-
} catch {
|
|
14060
|
-
return outcome(request2, "capability_unsupported", "capability_probe_failed");
|
|
14061
|
-
}
|
|
14062
|
-
if (!capability) {
|
|
14063
|
-
return outcome(request2, "capability_unsupported", "scope_unsupported");
|
|
14064
|
-
}
|
|
14065
|
-
if (!request2.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
|
|
14066
|
-
return outcome(request2, "capability_unsupported", "adapter_version_unpinned");
|
|
14067
|
-
}
|
|
14068
|
-
let credential;
|
|
14069
|
-
try {
|
|
14070
|
-
credential = await dependencies.credentials.resolve(capability.provider);
|
|
14071
|
-
} catch {
|
|
14072
|
-
return outcome(request2, "credentials_missing", "credential_resolution_failed");
|
|
14073
|
-
}
|
|
14074
|
-
if (credential === undefined || credential === null) {
|
|
14075
|
-
return outcome(request2, "credentials_missing", "provider_credentials_missing");
|
|
14076
|
-
}
|
|
14077
|
-
let dataset;
|
|
14078
|
-
let normalized;
|
|
14079
|
-
try {
|
|
14080
|
-
dataset = await dependencies.providers.acquire(request2, capability, credential);
|
|
14081
|
-
const bundleId = captureBundleId2(request2, capability, dataset);
|
|
14082
|
-
normalized = await dependencies.providers.normalize(request2, capability, dataset, bundleId);
|
|
14083
|
-
if (normalized.captureBundleId !== bundleId) {
|
|
14084
|
-
return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
14085
|
-
reasonSubcode: "capture_identity_mismatch"
|
|
14086
|
-
});
|
|
13881
|
+
if (pos != eb || i + 1 != ss)
|
|
13882
|
+
err(0);
|
|
13883
|
+
};
|
|
13884
|
+
var dhu4 = function(dat, out, hu) {
|
|
13885
|
+
var bt = 6;
|
|
13886
|
+
var ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
|
|
13887
|
+
dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
|
|
13888
|
+
dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
|
|
13889
|
+
dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
|
|
13890
|
+
dhu(dat.subarray(bt), out.subarray(sz3), hu);
|
|
13891
|
+
};
|
|
13892
|
+
var rzb = function(dat, st, out) {
|
|
13893
|
+
var _a;
|
|
13894
|
+
var bt = st.b;
|
|
13895
|
+
var b0 = dat[bt], btype = b0 >> 1 & 3;
|
|
13896
|
+
st.l = b0 & 1;
|
|
13897
|
+
var sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
|
|
13898
|
+
var ebt = (bt += 3) + sz;
|
|
13899
|
+
if (btype == 1) {
|
|
13900
|
+
if (bt >= dat.length)
|
|
13901
|
+
return;
|
|
13902
|
+
st.b = bt + 1;
|
|
13903
|
+
if (out) {
|
|
13904
|
+
fill(out, dat[bt], st.y, st.y += sz);
|
|
13905
|
+
return out;
|
|
14087
13906
|
}
|
|
14088
|
-
|
|
14089
|
-
const reason = stableFailureReason(error, "provider_dataset_invalid");
|
|
14090
|
-
return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
14091
|
-
reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
|
|
14092
|
-
});
|
|
13907
|
+
return fill(new u8(sz), dat[bt]);
|
|
14093
13908
|
}
|
|
14094
|
-
|
|
14095
|
-
|
|
14096
|
-
|
|
14097
|
-
|
|
14098
|
-
|
|
14099
|
-
|
|
14100
|
-
|
|
14101
|
-
return
|
|
13909
|
+
if (ebt > dat.length)
|
|
13910
|
+
return;
|
|
13911
|
+
if (btype == 0) {
|
|
13912
|
+
st.b = ebt;
|
|
13913
|
+
if (out) {
|
|
13914
|
+
out.set(dat.subarray(bt, ebt), st.y);
|
|
13915
|
+
st.y += sz;
|
|
13916
|
+
return out;
|
|
14102
13917
|
}
|
|
14103
|
-
|
|
14104
|
-
return outcome(request2, "archive_ingest_failed", "candidate_submission_failed");
|
|
14105
|
-
}
|
|
14106
|
-
let verification;
|
|
14107
|
-
try {
|
|
14108
|
-
verification = await dependencies.archive.verifyCandidate(request2, normalized, normalized.captureBundleId, initialResolution);
|
|
14109
|
-
} catch {
|
|
14110
|
-
return outcome(request2, "promotion_verification_failed", "candidate_query_failed");
|
|
14111
|
-
}
|
|
14112
|
-
if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
|
|
14113
|
-
return outcome(request2, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
|
|
13918
|
+
return slc(dat, bt, ebt);
|
|
14114
13919
|
}
|
|
14115
|
-
|
|
14116
|
-
|
|
14117
|
-
|
|
14118
|
-
|
|
14119
|
-
|
|
14120
|
-
|
|
14121
|
-
|
|
14122
|
-
|
|
14123
|
-
|
|
14124
|
-
|
|
14125
|
-
|
|
14126
|
-
|
|
14127
|
-
|
|
14128
|
-
|
|
14129
|
-
|
|
14130
|
-
|
|
14131
|
-
window: receipt.window,
|
|
14132
|
-
event_at: receipt.verified_at,
|
|
14133
|
-
reason_code: "promotion_verified"
|
|
14134
|
-
});
|
|
14135
|
-
const [qualificationBatch] = buildForwarderBatches({
|
|
14136
|
-
captureBundleId: normalized.captureBundleId,
|
|
14137
|
-
deploymentId: "market-data-vendor-backfill",
|
|
14138
|
-
rows: [qualificationEventToArchiveRow(qualification)]
|
|
14139
|
-
});
|
|
14140
|
-
if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
|
|
14141
|
-
return outcome(request2, "archive_ingest_failed", "qualification_commit_failed", { receipt });
|
|
13920
|
+
if (btype == 2) {
|
|
13921
|
+
var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
|
|
13922
|
+
var lss = b3 >> 4, lcs = 0, s4 = 0;
|
|
13923
|
+
if (lbt < 2) {
|
|
13924
|
+
if (sf & 1)
|
|
13925
|
+
lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
|
|
13926
|
+
else
|
|
13927
|
+
lss = b3 >> 3;
|
|
13928
|
+
} else {
|
|
13929
|
+
s4 = sf;
|
|
13930
|
+
if (sf < 2)
|
|
13931
|
+
lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
|
|
13932
|
+
else if (sf == 2)
|
|
13933
|
+
lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
|
|
13934
|
+
else
|
|
13935
|
+
lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
|
|
14142
13936
|
}
|
|
14143
|
-
|
|
14144
|
-
|
|
14145
|
-
|
|
14146
|
-
|
|
14147
|
-
|
|
14148
|
-
|
|
14149
|
-
|
|
14150
|
-
|
|
14151
|
-
|
|
14152
|
-
|
|
14153
|
-
|
|
14154
|
-
|
|
14155
|
-
|
|
14156
|
-
|
|
14157
|
-
|
|
14158
|
-
|
|
14159
|
-
if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
|
|
14160
|
-
return outcome(request2, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
|
|
14161
|
-
}
|
|
14162
|
-
try {
|
|
14163
|
-
if (!request2.wire)
|
|
14164
|
-
throw new Error("decoded request wire is missing");
|
|
14165
|
-
const [selectionBatch] = buildForwarderBatches({
|
|
14166
|
-
captureBundleId: receipt.capture_bundle_id,
|
|
14167
|
-
deploymentId: "market-data-vendor-backfill",
|
|
14168
|
-
rows: [
|
|
14169
|
-
archiveSelectionToArchiveRow(request2.wire, finalResolution.selection)
|
|
14170
|
-
]
|
|
14171
|
-
});
|
|
14172
|
-
if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
|
|
14173
|
-
return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
14174
|
-
}
|
|
14175
|
-
} catch {
|
|
14176
|
-
return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
14177
|
-
}
|
|
14178
|
-
return outcome(request2, "promoted", "promotion_qualified", {
|
|
14179
|
-
receipt,
|
|
14180
|
-
selection: finalResolution.selection
|
|
14181
|
-
});
|
|
14182
|
-
}
|
|
14183
|
-
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
14184
|
-
import { createHash as createHash3 } from "node:crypto";
|
|
14185
|
-
|
|
14186
|
-
// node_modules/fzstd/esm/index.mjs
|
|
14187
|
-
var ab = ArrayBuffer;
|
|
14188
|
-
var u8 = Uint8Array;
|
|
14189
|
-
var u16 = Uint16Array;
|
|
14190
|
-
var i16 = Int16Array;
|
|
14191
|
-
var i32 = Int32Array;
|
|
14192
|
-
var slc = function(v, s, e) {
|
|
14193
|
-
if (u8.prototype.slice)
|
|
14194
|
-
return u8.prototype.slice.call(v, s, e);
|
|
14195
|
-
if (s == null || s < 0)
|
|
14196
|
-
s = 0;
|
|
14197
|
-
if (e == null || e > v.length)
|
|
14198
|
-
e = v.length;
|
|
14199
|
-
var n = new u8(e - s);
|
|
14200
|
-
n.set(v.subarray(s, e));
|
|
14201
|
-
return n;
|
|
14202
|
-
};
|
|
14203
|
-
var fill = function(v, n, s, e) {
|
|
14204
|
-
if (u8.prototype.fill)
|
|
14205
|
-
return u8.prototype.fill.call(v, n, s, e);
|
|
14206
|
-
if (s == null || s < 0)
|
|
14207
|
-
s = 0;
|
|
14208
|
-
if (e == null || e > v.length)
|
|
14209
|
-
e = v.length;
|
|
14210
|
-
for (;s < e; ++s)
|
|
14211
|
-
v[s] = n;
|
|
14212
|
-
return v;
|
|
14213
|
-
};
|
|
14214
|
-
var cpw = function(v, t, s, e) {
|
|
14215
|
-
if (u8.prototype.copyWithin)
|
|
14216
|
-
return u8.prototype.copyWithin.call(v, t, s, e);
|
|
14217
|
-
if (s == null || s < 0)
|
|
14218
|
-
s = 0;
|
|
14219
|
-
if (e == null || e > v.length)
|
|
14220
|
-
e = v.length;
|
|
14221
|
-
while (s < e) {
|
|
14222
|
-
v[t++] = v[s++];
|
|
14223
|
-
}
|
|
14224
|
-
};
|
|
14225
|
-
var ec = [
|
|
14226
|
-
"invalid zstd data",
|
|
14227
|
-
"window size too large (>2046MB)",
|
|
14228
|
-
"invalid block type",
|
|
14229
|
-
"FSE accuracy too high",
|
|
14230
|
-
"match distance too far back",
|
|
14231
|
-
"unexpected EOF"
|
|
14232
|
-
];
|
|
14233
|
-
var err = function(ind, msg, nt) {
|
|
14234
|
-
var e = new Error(msg || ec[ind]);
|
|
14235
|
-
e.code = ind;
|
|
14236
|
-
if (Error.captureStackTrace)
|
|
14237
|
-
Error.captureStackTrace(e, err);
|
|
14238
|
-
if (!nt)
|
|
14239
|
-
throw e;
|
|
14240
|
-
return e;
|
|
14241
|
-
};
|
|
14242
|
-
var rb = function(d, b, n) {
|
|
14243
|
-
var i = 0, o = 0;
|
|
14244
|
-
for (;i < n; ++i)
|
|
14245
|
-
o |= d[b++] << (i << 3);
|
|
14246
|
-
return o;
|
|
14247
|
-
};
|
|
14248
|
-
var b4 = function(d, b) {
|
|
14249
|
-
return (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0;
|
|
14250
|
-
};
|
|
14251
|
-
var rzfh = function(dat, w) {
|
|
14252
|
-
var n3 = dat[0] | dat[1] << 8 | dat[2] << 16;
|
|
14253
|
-
if (n3 == 3126568 && dat[3] == 253) {
|
|
14254
|
-
var flg = dat[4];
|
|
14255
|
-
var ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6;
|
|
14256
|
-
if (flg & 8)
|
|
14257
|
-
err(0);
|
|
14258
|
-
var bt = 6 - ss;
|
|
14259
|
-
var db = df == 3 ? 4 : df;
|
|
14260
|
-
var di = rb(dat, bt, db);
|
|
14261
|
-
bt += db;
|
|
14262
|
-
var fsb = fcf ? 1 << fcf : ss;
|
|
14263
|
-
var fss = rb(dat, bt, fsb) + (fcf == 1 && 256);
|
|
14264
|
-
var ws = fss;
|
|
14265
|
-
if (!ss) {
|
|
14266
|
-
var wb = 1 << 10 + (dat[5] >> 3);
|
|
14267
|
-
ws = wb + (wb >> 3) * (dat[5] & 7);
|
|
14268
|
-
}
|
|
14269
|
-
if (ws > 2145386496)
|
|
14270
|
-
err(1);
|
|
14271
|
-
var buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
|
|
14272
|
-
buf[0] = 1, buf[4] = 4, buf[8] = 8;
|
|
14273
|
-
return {
|
|
14274
|
-
b: bt + fsb,
|
|
14275
|
-
y: 0,
|
|
14276
|
-
l: 0,
|
|
14277
|
-
d: di,
|
|
14278
|
-
w: w && w != 1 ? w : buf.subarray(12),
|
|
14279
|
-
e: ws,
|
|
14280
|
-
o: new i32(buf.buffer, 0, 3),
|
|
14281
|
-
u: fss,
|
|
14282
|
-
c: cc,
|
|
14283
|
-
m: Math.min(131072, ws)
|
|
14284
|
-
};
|
|
14285
|
-
} else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
|
|
14286
|
-
return b4(dat, 4) + 8;
|
|
14287
|
-
}
|
|
14288
|
-
err(0);
|
|
14289
|
-
};
|
|
14290
|
-
var msb = function(val) {
|
|
14291
|
-
var bits = 0;
|
|
14292
|
-
for (;1 << bits <= val; ++bits)
|
|
14293
|
-
;
|
|
14294
|
-
return bits - 1;
|
|
14295
|
-
};
|
|
14296
|
-
var rfse = function(dat, bt, mal) {
|
|
14297
|
-
var tpos = (bt << 3) + 4;
|
|
14298
|
-
var al = (dat[bt] & 15) + 5;
|
|
14299
|
-
if (al > mal)
|
|
14300
|
-
err(3);
|
|
14301
|
-
var sz = 1 << al;
|
|
14302
|
-
var probs = sz, sym = -1, re = -1, i = -1, ht = sz;
|
|
14303
|
-
var buf = new ab(512 + (sz << 2));
|
|
14304
|
-
var freq = new i16(buf, 0, 256);
|
|
14305
|
-
var dstate = new u16(buf, 0, 256);
|
|
14306
|
-
var nstate = new u16(buf, 512, sz);
|
|
14307
|
-
var bb1 = 512 + (sz << 1);
|
|
14308
|
-
var syms = new u8(buf, bb1, sz);
|
|
14309
|
-
var nbits = new u8(buf, bb1 + sz);
|
|
14310
|
-
while (sym < 255 && probs > 0) {
|
|
14311
|
-
var bits = msb(probs + 1);
|
|
14312
|
-
var cbt = tpos >> 3;
|
|
14313
|
-
var msk = (1 << bits + 1) - 1;
|
|
14314
|
-
var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
|
|
14315
|
-
var msk1fb = (1 << bits) - 1;
|
|
14316
|
-
var msv = msk - probs - 1;
|
|
14317
|
-
var sval = val & msk1fb;
|
|
14318
|
-
if (sval < msv)
|
|
14319
|
-
tpos += bits, val = sval;
|
|
14320
|
-
else {
|
|
14321
|
-
tpos += bits + 1;
|
|
14322
|
-
if (val > msk1fb)
|
|
14323
|
-
val -= msv;
|
|
14324
|
-
}
|
|
14325
|
-
freq[++sym] = --val;
|
|
14326
|
-
if (val == -1) {
|
|
14327
|
-
probs += val;
|
|
14328
|
-
syms[--ht] = sym;
|
|
14329
|
-
} else
|
|
14330
|
-
probs -= val;
|
|
14331
|
-
if (!val) {
|
|
14332
|
-
do {
|
|
14333
|
-
var rbt = tpos >> 3;
|
|
14334
|
-
re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
|
|
14335
|
-
tpos += 2;
|
|
14336
|
-
sym += re;
|
|
14337
|
-
} while (re == 3);
|
|
14338
|
-
}
|
|
14339
|
-
}
|
|
14340
|
-
if (sym > 255 || probs)
|
|
14341
|
-
err(0);
|
|
14342
|
-
var sympos = 0;
|
|
14343
|
-
var sstep = (sz >> 1) + (sz >> 3) + 3;
|
|
14344
|
-
var smask = sz - 1;
|
|
14345
|
-
for (var s = 0;s <= sym; ++s) {
|
|
14346
|
-
var sf = freq[s];
|
|
14347
|
-
if (sf < 1) {
|
|
14348
|
-
dstate[s] = -sf;
|
|
14349
|
-
continue;
|
|
14350
|
-
}
|
|
14351
|
-
for (i = 0;i < sf; ++i) {
|
|
14352
|
-
syms[sympos] = s;
|
|
14353
|
-
do {
|
|
14354
|
-
sympos = sympos + sstep & smask;
|
|
14355
|
-
} while (sympos >= ht);
|
|
14356
|
-
}
|
|
14357
|
-
}
|
|
14358
|
-
if (sympos)
|
|
14359
|
-
err(0);
|
|
14360
|
-
for (i = 0;i < sz; ++i) {
|
|
14361
|
-
var ns = dstate[syms[i]]++;
|
|
14362
|
-
var nb = nbits[i] = al - msb(ns);
|
|
14363
|
-
nstate[i] = (ns << nb) - sz;
|
|
14364
|
-
}
|
|
14365
|
-
return [tpos + 7 >> 3, {
|
|
14366
|
-
b: al,
|
|
14367
|
-
s: syms,
|
|
14368
|
-
n: nbits,
|
|
14369
|
-
t: nstate
|
|
14370
|
-
}];
|
|
14371
|
-
};
|
|
14372
|
-
var rhu = function(dat, bt) {
|
|
14373
|
-
var i = 0, wc = -1;
|
|
14374
|
-
var buf = new u8(292), hb = dat[bt];
|
|
14375
|
-
var hw = buf.subarray(0, 256);
|
|
14376
|
-
var rc = buf.subarray(256, 268);
|
|
14377
|
-
var ri = new u16(buf.buffer, 268);
|
|
14378
|
-
if (hb < 128) {
|
|
14379
|
-
var _a = rfse(dat, bt + 1, 6), ebt = _a[0], fdt = _a[1];
|
|
14380
|
-
bt += hb;
|
|
14381
|
-
var epos = ebt << 3;
|
|
14382
|
-
var lb = dat[bt];
|
|
14383
|
-
if (!lb)
|
|
14384
|
-
err(0);
|
|
14385
|
-
var st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
|
|
14386
|
-
var fpos = (++bt << 3) - 8 + msb(lb);
|
|
14387
|
-
for (;; ) {
|
|
14388
|
-
fpos -= btr1;
|
|
14389
|
-
if (fpos < epos)
|
|
14390
|
-
break;
|
|
14391
|
-
var cbt = fpos >> 3;
|
|
14392
|
-
st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
|
|
14393
|
-
hw[++wc] = fdt.s[st1];
|
|
14394
|
-
fpos -= btr2;
|
|
14395
|
-
if (fpos < epos)
|
|
14396
|
-
break;
|
|
14397
|
-
cbt = fpos >> 3;
|
|
14398
|
-
st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
|
|
14399
|
-
hw[++wc] = fdt.s[st2];
|
|
14400
|
-
btr1 = fdt.n[st1];
|
|
14401
|
-
st1 = fdt.t[st1];
|
|
14402
|
-
btr2 = fdt.n[st2];
|
|
14403
|
-
st2 = fdt.t[st2];
|
|
14404
|
-
}
|
|
14405
|
-
if (++wc > 255)
|
|
14406
|
-
err(0);
|
|
14407
|
-
} else {
|
|
14408
|
-
wc = hb - 127;
|
|
14409
|
-
for (;i < wc; i += 2) {
|
|
14410
|
-
var byte = dat[++bt];
|
|
14411
|
-
hw[i] = byte >> 4;
|
|
14412
|
-
hw[i + 1] = byte & 15;
|
|
14413
|
-
}
|
|
14414
|
-
++bt;
|
|
14415
|
-
}
|
|
14416
|
-
var wes = 0;
|
|
14417
|
-
for (i = 0;i < wc; ++i) {
|
|
14418
|
-
var wt = hw[i];
|
|
14419
|
-
if (wt > 11)
|
|
14420
|
-
err(0);
|
|
14421
|
-
wes += wt && 1 << wt - 1;
|
|
14422
|
-
}
|
|
14423
|
-
var mb = msb(wes) + 1;
|
|
14424
|
-
var ts = 1 << mb;
|
|
14425
|
-
var rem = ts - wes;
|
|
14426
|
-
if (rem & rem - 1)
|
|
14427
|
-
err(0);
|
|
14428
|
-
hw[wc++] = msb(rem) + 1;
|
|
14429
|
-
for (i = 0;i < wc; ++i) {
|
|
14430
|
-
var wt = hw[i];
|
|
14431
|
-
++rc[hw[i] = wt && mb + 1 - wt];
|
|
14432
|
-
}
|
|
14433
|
-
var hbuf = new u8(ts << 1);
|
|
14434
|
-
var syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
|
|
14435
|
-
ri[mb] = 0;
|
|
14436
|
-
for (i = mb;i > 0; --i) {
|
|
14437
|
-
var pv = ri[i];
|
|
14438
|
-
fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
|
|
14439
|
-
}
|
|
14440
|
-
if (ri[0] != ts)
|
|
14441
|
-
err(0);
|
|
14442
|
-
for (i = 0;i < wc; ++i) {
|
|
14443
|
-
var bits = hw[i];
|
|
14444
|
-
if (bits) {
|
|
14445
|
-
var code = ri[bits];
|
|
14446
|
-
fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
|
|
14447
|
-
}
|
|
14448
|
-
}
|
|
14449
|
-
return [bt, {
|
|
14450
|
-
n: nb,
|
|
14451
|
-
b: mb,
|
|
14452
|
-
s: syms
|
|
14453
|
-
}];
|
|
14454
|
-
};
|
|
14455
|
-
var dllt = rfse(/* @__PURE__ */ new u8([
|
|
14456
|
-
81,
|
|
14457
|
-
16,
|
|
14458
|
-
99,
|
|
14459
|
-
140,
|
|
14460
|
-
49,
|
|
14461
|
-
198,
|
|
14462
|
-
24,
|
|
14463
|
-
99,
|
|
14464
|
-
12,
|
|
14465
|
-
33,
|
|
14466
|
-
196,
|
|
14467
|
-
24,
|
|
14468
|
-
99,
|
|
14469
|
-
102,
|
|
14470
|
-
102,
|
|
14471
|
-
134,
|
|
14472
|
-
70,
|
|
14473
|
-
146,
|
|
14474
|
-
4
|
|
14475
|
-
]), 0, 6)[1];
|
|
14476
|
-
var dmlt = rfse(/* @__PURE__ */ new u8([
|
|
14477
|
-
33,
|
|
14478
|
-
20,
|
|
14479
|
-
196,
|
|
14480
|
-
24,
|
|
14481
|
-
99,
|
|
14482
|
-
140,
|
|
14483
|
-
33,
|
|
14484
|
-
132,
|
|
14485
|
-
16,
|
|
14486
|
-
66,
|
|
14487
|
-
8,
|
|
14488
|
-
33,
|
|
14489
|
-
132,
|
|
14490
|
-
16,
|
|
14491
|
-
66,
|
|
14492
|
-
8,
|
|
14493
|
-
33,
|
|
14494
|
-
68,
|
|
14495
|
-
68,
|
|
14496
|
-
68,
|
|
14497
|
-
68,
|
|
14498
|
-
68,
|
|
14499
|
-
68,
|
|
14500
|
-
68,
|
|
14501
|
-
68,
|
|
14502
|
-
36,
|
|
14503
|
-
9
|
|
14504
|
-
]), 0, 6)[1];
|
|
14505
|
-
var doct = rfse(/* @__PURE__ */ new u8([
|
|
14506
|
-
32,
|
|
14507
|
-
132,
|
|
14508
|
-
16,
|
|
14509
|
-
66,
|
|
14510
|
-
102,
|
|
14511
|
-
70,
|
|
14512
|
-
68,
|
|
14513
|
-
68,
|
|
14514
|
-
68,
|
|
14515
|
-
68,
|
|
14516
|
-
36,
|
|
14517
|
-
73,
|
|
14518
|
-
2
|
|
14519
|
-
]), 0, 5)[1];
|
|
14520
|
-
var b2bl = function(b, s) {
|
|
14521
|
-
var len = b.length, bl = new i32(len);
|
|
14522
|
-
for (var i = 0;i < len; ++i) {
|
|
14523
|
-
bl[i] = s;
|
|
14524
|
-
s += 1 << b[i];
|
|
14525
|
-
}
|
|
14526
|
-
return bl;
|
|
14527
|
-
};
|
|
14528
|
-
var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
14529
|
-
0,
|
|
14530
|
-
0,
|
|
14531
|
-
0,
|
|
14532
|
-
0,
|
|
14533
|
-
16843009,
|
|
14534
|
-
50528770,
|
|
14535
|
-
134678020,
|
|
14536
|
-
202050057,
|
|
14537
|
-
269422093
|
|
14538
|
-
])).buffer, 0, 36);
|
|
14539
|
-
var llbl = /* @__PURE__ */ b2bl(llb, 0);
|
|
14540
|
-
var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
14541
|
-
0,
|
|
14542
|
-
0,
|
|
14543
|
-
0,
|
|
14544
|
-
0,
|
|
14545
|
-
0,
|
|
14546
|
-
0,
|
|
14547
|
-
0,
|
|
14548
|
-
0,
|
|
14549
|
-
16843009,
|
|
14550
|
-
50528770,
|
|
14551
|
-
117769220,
|
|
14552
|
-
185207048,
|
|
14553
|
-
252579084,
|
|
14554
|
-
16
|
|
14555
|
-
])).buffer, 0, 53);
|
|
14556
|
-
var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
|
|
14557
|
-
var dhu = function(dat, out, hu) {
|
|
14558
|
-
var len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
|
|
14559
|
-
if (!lb)
|
|
14560
|
-
err(0);
|
|
14561
|
-
var st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
|
|
14562
|
-
for (;pos > eb && i < ss; ) {
|
|
14563
|
-
var cbt = pos >> 3;
|
|
14564
|
-
var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
|
|
14565
|
-
st = (st << btr | val) & msk;
|
|
14566
|
-
out[++i] = hu.s[st];
|
|
14567
|
-
pos -= btr = hu.n[st];
|
|
14568
|
-
}
|
|
14569
|
-
if (pos != eb || i + 1 != ss)
|
|
14570
|
-
err(0);
|
|
14571
|
-
};
|
|
14572
|
-
var dhu4 = function(dat, out, hu) {
|
|
14573
|
-
var bt = 6;
|
|
14574
|
-
var ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
|
|
14575
|
-
dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
|
|
14576
|
-
dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
|
|
14577
|
-
dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
|
|
14578
|
-
dhu(dat.subarray(bt), out.subarray(sz3), hu);
|
|
14579
|
-
};
|
|
14580
|
-
var rzb = function(dat, st, out) {
|
|
14581
|
-
var _a;
|
|
14582
|
-
var bt = st.b;
|
|
14583
|
-
var b0 = dat[bt], btype = b0 >> 1 & 3;
|
|
14584
|
-
st.l = b0 & 1;
|
|
14585
|
-
var sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
|
|
14586
|
-
var ebt = (bt += 3) + sz;
|
|
14587
|
-
if (btype == 1) {
|
|
14588
|
-
if (bt >= dat.length)
|
|
14589
|
-
return;
|
|
14590
|
-
st.b = bt + 1;
|
|
14591
|
-
if (out) {
|
|
14592
|
-
fill(out, dat[bt], st.y, st.y += sz);
|
|
14593
|
-
return out;
|
|
14594
|
-
}
|
|
14595
|
-
return fill(new u8(sz), dat[bt]);
|
|
14596
|
-
}
|
|
14597
|
-
if (ebt > dat.length)
|
|
14598
|
-
return;
|
|
14599
|
-
if (btype == 0) {
|
|
14600
|
-
st.b = ebt;
|
|
14601
|
-
if (out) {
|
|
14602
|
-
out.set(dat.subarray(bt, ebt), st.y);
|
|
14603
|
-
st.y += sz;
|
|
14604
|
-
return out;
|
|
14605
|
-
}
|
|
14606
|
-
return slc(dat, bt, ebt);
|
|
14607
|
-
}
|
|
14608
|
-
if (btype == 2) {
|
|
14609
|
-
var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
|
|
14610
|
-
var lss = b3 >> 4, lcs = 0, s4 = 0;
|
|
14611
|
-
if (lbt < 2) {
|
|
14612
|
-
if (sf & 1)
|
|
14613
|
-
lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
|
|
14614
|
-
else
|
|
14615
|
-
lss = b3 >> 3;
|
|
14616
|
-
} else {
|
|
14617
|
-
s4 = sf;
|
|
14618
|
-
if (sf < 2)
|
|
14619
|
-
lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
|
|
14620
|
-
else if (sf == 2)
|
|
14621
|
-
lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
|
|
14622
|
-
else
|
|
14623
|
-
lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
|
|
14624
|
-
}
|
|
14625
|
-
++bt;
|
|
14626
|
-
var buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
|
|
14627
|
-
var spl = buf.length - lss;
|
|
14628
|
-
if (lbt == 0)
|
|
14629
|
-
buf.set(dat.subarray(bt, bt += lss), spl);
|
|
14630
|
-
else if (lbt == 1)
|
|
14631
|
-
fill(buf, dat[bt++], spl);
|
|
14632
|
-
else {
|
|
14633
|
-
var hu = st.h;
|
|
14634
|
-
if (lbt == 2) {
|
|
14635
|
-
var hud = rhu(dat, bt);
|
|
14636
|
-
lcs += bt - (bt = hud[0]);
|
|
14637
|
-
st.h = hu = hud[1];
|
|
14638
|
-
} else if (!hu)
|
|
14639
|
-
err(0);
|
|
14640
|
-
(s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
|
|
13937
|
+
++bt;
|
|
13938
|
+
var buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
|
|
13939
|
+
var spl = buf.length - lss;
|
|
13940
|
+
if (lbt == 0)
|
|
13941
|
+
buf.set(dat.subarray(bt, bt += lss), spl);
|
|
13942
|
+
else if (lbt == 1)
|
|
13943
|
+
fill(buf, dat[bt++], spl);
|
|
13944
|
+
else {
|
|
13945
|
+
var hu = st.h;
|
|
13946
|
+
if (lbt == 2) {
|
|
13947
|
+
var hud = rhu(dat, bt);
|
|
13948
|
+
lcs += bt - (bt = hud[0]);
|
|
13949
|
+
st.h = hu = hud[1];
|
|
13950
|
+
} else if (!hu)
|
|
13951
|
+
err(0);
|
|
13952
|
+
(s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
|
|
14641
13953
|
}
|
|
14642
13954
|
var ns = dat[bt++];
|
|
14643
13955
|
if (ns) {
|
|
@@ -18090,599 +17402,1062 @@ function buildCanonicalOrderBookRows(input) {
|
|
|
18090
17402
|
const common = commonEvidenceFields(input.context, input.rawCapture, {
|
|
18091
17403
|
snapshotId,
|
|
18092
17404
|
sequence,
|
|
18093
|
-
depthLimit: input.depthLimit,
|
|
18094
|
-
eventTimeMs,
|
|
18095
|
-
receivedTimeMs
|
|
18096
|
-
});
|
|
18097
|
-
const levels = [
|
|
18098
|
-
["bid", bids],
|
|
18099
|
-
["ask", asks]
|
|
18100
|
-
].flatMap(([side, sideLevels]) => sideLevels.map(({ price, amount }, levelIndex) => {
|
|
18101
|
-
const row = checksumRow({
|
|
18102
|
-
...common,
|
|
18103
|
-
side,
|
|
18104
|
-
level_index: levelIndex,
|
|
18105
|
-
price,
|
|
18106
|
-
amount,
|
|
18107
|
-
notional: price * amount,
|
|
18108
|
-
mid_price: midPrice,
|
|
18109
|
-
spread_from_mid_bps: Math.abs((price - midPrice) / midPrice) * 1e4
|
|
18110
|
-
});
|
|
18111
|
-
return { table: "market_data.cex_order_book_levels", row };
|
|
18112
|
-
}));
|
|
18113
|
-
const bands = normalizedBands(input.measurementBandsBps);
|
|
18114
|
-
const bidDepth = bands.map((band) => {
|
|
18115
|
-
const minimumPrice = bestBid.price * (1 - band / 1e4);
|
|
18116
|
-
return bids.filter(({ price }) => price >= minimumPrice).reduce((sum, { amount }) => sum + amount, 0);
|
|
18117
|
-
});
|
|
18118
|
-
const askDepth = bands.map((band) => {
|
|
18119
|
-
const maximumPrice = bestAsk.price * (1 + band / 1e4);
|
|
18120
|
-
return asks.filter(({ price }) => price <= maximumPrice).reduce((sum, { amount }) => sum + amount, 0);
|
|
18121
|
-
});
|
|
18122
|
-
const summaryRow = checksumRow({
|
|
18123
|
-
...common,
|
|
18124
|
-
best_bid: bestBid.price,
|
|
18125
|
-
best_ask: bestAsk.price,
|
|
18126
|
-
best_bid_amount: bestBid.amount,
|
|
18127
|
-
best_ask_amount: bestAsk.amount,
|
|
18128
|
-
mid_price: midPrice,
|
|
18129
|
-
spread,
|
|
18130
|
-
spread_bps: spreadBps,
|
|
18131
|
-
staleness_ms: receivedTimeMs - eventTimeMs,
|
|
18132
|
-
bid_level_count: bids.length,
|
|
18133
|
-
ask_level_count: asks.length,
|
|
18134
|
-
measurement_bands_bps: bands,
|
|
18135
|
-
bid_depth_by_band: bidDepth,
|
|
18136
|
-
ask_depth_by_band: askDepth
|
|
18137
|
-
});
|
|
18138
|
-
return {
|
|
18139
|
-
snapshotId,
|
|
18140
|
-
levels,
|
|
18141
|
-
summary: {
|
|
18142
|
-
table: "market_data.cex_order_book_depth_summary",
|
|
18143
|
-
row: summaryRow
|
|
17405
|
+
depthLimit: input.depthLimit,
|
|
17406
|
+
eventTimeMs,
|
|
17407
|
+
receivedTimeMs
|
|
17408
|
+
});
|
|
17409
|
+
const levels = [
|
|
17410
|
+
["bid", bids],
|
|
17411
|
+
["ask", asks]
|
|
17412
|
+
].flatMap(([side, sideLevels]) => sideLevels.map(({ price, amount }, levelIndex) => {
|
|
17413
|
+
const row = checksumRow({
|
|
17414
|
+
...common,
|
|
17415
|
+
side,
|
|
17416
|
+
level_index: levelIndex,
|
|
17417
|
+
price,
|
|
17418
|
+
amount,
|
|
17419
|
+
notional: price * amount,
|
|
17420
|
+
mid_price: midPrice,
|
|
17421
|
+
spread_from_mid_bps: Math.abs((price - midPrice) / midPrice) * 1e4
|
|
17422
|
+
});
|
|
17423
|
+
return { table: "market_data.cex_order_book_levels", row };
|
|
17424
|
+
}));
|
|
17425
|
+
const bands = normalizedBands(input.measurementBandsBps);
|
|
17426
|
+
const bidDepth = bands.map((band) => {
|
|
17427
|
+
const minimumPrice = bestBid.price * (1 - band / 1e4);
|
|
17428
|
+
return bids.filter(({ price }) => price >= minimumPrice).reduce((sum, { amount }) => sum + amount, 0);
|
|
17429
|
+
});
|
|
17430
|
+
const askDepth = bands.map((band) => {
|
|
17431
|
+
const maximumPrice = bestAsk.price * (1 + band / 1e4);
|
|
17432
|
+
return asks.filter(({ price }) => price <= maximumPrice).reduce((sum, { amount }) => sum + amount, 0);
|
|
17433
|
+
});
|
|
17434
|
+
const summaryRow = checksumRow({
|
|
17435
|
+
...common,
|
|
17436
|
+
best_bid: bestBid.price,
|
|
17437
|
+
best_ask: bestAsk.price,
|
|
17438
|
+
best_bid_amount: bestBid.amount,
|
|
17439
|
+
best_ask_amount: bestAsk.amount,
|
|
17440
|
+
mid_price: midPrice,
|
|
17441
|
+
spread,
|
|
17442
|
+
spread_bps: spreadBps,
|
|
17443
|
+
staleness_ms: receivedTimeMs - eventTimeMs,
|
|
17444
|
+
bid_level_count: bids.length,
|
|
17445
|
+
ask_level_count: asks.length,
|
|
17446
|
+
measurement_bands_bps: bands,
|
|
17447
|
+
bid_depth_by_band: bidDepth,
|
|
17448
|
+
ask_depth_by_band: askDepth
|
|
17449
|
+
});
|
|
17450
|
+
return {
|
|
17451
|
+
snapshotId,
|
|
17452
|
+
levels,
|
|
17453
|
+
summary: {
|
|
17454
|
+
table: "market_data.cex_order_book_depth_summary",
|
|
17455
|
+
row: summaryRow
|
|
17456
|
+
}
|
|
17457
|
+
};
|
|
17458
|
+
}
|
|
17459
|
+
|
|
17460
|
+
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
17461
|
+
var CRYPTOHFTDATA_ADAPTER_VERSION = "cryptohftdata-orderbook/v2";
|
|
17462
|
+
var CRYPTOHFTDATA_API_URL = "https://api.cryptohftdata.com";
|
|
17463
|
+
var CRYPTOHFTDATA_HISTORY_START_MS = Date.UTC(2025, 5, 28);
|
|
17464
|
+
var HOUR_MS = 60 * 60 * 1000;
|
|
17465
|
+
var CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE = Object.freeze({
|
|
17466
|
+
profileId: "cryptohftdata/binance_spot/BTCUSDT/v1",
|
|
17467
|
+
exchange: "binance",
|
|
17468
|
+
tradingPair: "BTC-USDT",
|
|
17469
|
+
sourceSymbol: "BTCUSDT",
|
|
17470
|
+
marketType: "spot",
|
|
17471
|
+
providerExchangeId: "binance_spot",
|
|
17472
|
+
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
17473
|
+
maxDepth: 500,
|
|
17474
|
+
eventTimeUnit: "milliseconds",
|
|
17475
|
+
receivedTimeUnit: "nanoseconds",
|
|
17476
|
+
snapshotGrouping: "event_time_last_update_id_object",
|
|
17477
|
+
sequenceSemantics: "binance_u_U_pu",
|
|
17478
|
+
constructionModes: ["sampled_top_n_snapshot"],
|
|
17479
|
+
sourcePolicies: ["authoritative_window"]
|
|
17480
|
+
});
|
|
17481
|
+
var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
|
|
17482
|
+
profileId: "cryptohftdata/okx_spot/ARB-USDT/v1",
|
|
17483
|
+
exchange: "okx",
|
|
17484
|
+
tradingPair: "ARB-USDT",
|
|
17485
|
+
sourceSymbol: "ARB-USDT",
|
|
17486
|
+
marketType: "spot",
|
|
17487
|
+
providerExchangeId: "okx_spot",
|
|
17488
|
+
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
17489
|
+
maxDepth: 400,
|
|
17490
|
+
eventTimeUnit: "milliseconds",
|
|
17491
|
+
receivedTimeUnit: "nanoseconds",
|
|
17492
|
+
snapshotGrouping: "event_time_final_update_id_object",
|
|
17493
|
+
sequenceSemantics: "okx_seq_id_prev_seq_id",
|
|
17494
|
+
constructionModes: ["sampled_top_n_snapshot"],
|
|
17495
|
+
sourcePolicies: ["authoritative_window", "fill_gaps"]
|
|
17496
|
+
});
|
|
17497
|
+
var CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE = Object.freeze({
|
|
17498
|
+
profileId: "cryptohftdata/okx_spot/ARB-USDC/v1",
|
|
17499
|
+
exchange: "okx",
|
|
17500
|
+
tradingPair: "ARB-USDC",
|
|
17501
|
+
sourceSymbol: "ARB-USDC",
|
|
17502
|
+
marketType: "spot",
|
|
17503
|
+
providerExchangeId: "okx_spot",
|
|
17504
|
+
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
17505
|
+
maxDepth: 400,
|
|
17506
|
+
eventTimeUnit: "milliseconds",
|
|
17507
|
+
receivedTimeUnit: "nanoseconds",
|
|
17508
|
+
snapshotGrouping: "event_time_final_update_id_object",
|
|
17509
|
+
sequenceSemantics: "okx_seq_id_prev_seq_id",
|
|
17510
|
+
constructionModes: ["sampled_top_n_snapshot"],
|
|
17511
|
+
sourcePolicies: ["authoritative_window", "fill_gaps"]
|
|
17512
|
+
});
|
|
17513
|
+
|
|
17514
|
+
class CryptoHftDataError extends Error {
|
|
17515
|
+
reason;
|
|
17516
|
+
constructor(reason) {
|
|
17517
|
+
super(`CryptoHFTData backfill failed: ${reason}`);
|
|
17518
|
+
this.reason = reason;
|
|
17519
|
+
this.name = "CryptoHftDataError";
|
|
17520
|
+
}
|
|
17521
|
+
}
|
|
17522
|
+
function cryptoHftDataCapabilityFor(request2, profiles = []) {
|
|
17523
|
+
if (request2.providerPolicy.provider !== "cryptohftdata" || request2.scope.feed !== "ORDERBOOK" || request2.scope.exchange.trim().toLowerCase() === "mexc") {
|
|
17524
|
+
return;
|
|
17525
|
+
}
|
|
17526
|
+
const profile = profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.constructionModes.includes(request2.constructionMode) && candidate.sourcePolicies.includes(request2.sourcePolicy) && request2.window.startTimeMs >= candidate.historyStartMs && request2.depth <= candidate.maxDepth);
|
|
17527
|
+
if (!profile)
|
|
17528
|
+
return;
|
|
17529
|
+
return {
|
|
17530
|
+
provider: "cryptohftdata",
|
|
17531
|
+
adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
|
|
17532
|
+
providerExchangeId: profile.providerExchangeId,
|
|
17533
|
+
resolvedSymbol: profile.sourceSymbol
|
|
17534
|
+
};
|
|
17535
|
+
}
|
|
17536
|
+
function utcHourPath(hourMs) {
|
|
17537
|
+
const date = new Date(hourMs);
|
|
17538
|
+
return {
|
|
17539
|
+
date: `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String(date.getUTCDate()).padStart(2, "0")}`,
|
|
17540
|
+
hour: String(date.getUTCHours()).padStart(2, "0")
|
|
17541
|
+
};
|
|
17542
|
+
}
|
|
17543
|
+
function enumerateCryptoHftDataObjects(request2, providerExchangeId, resolvedSymbol) {
|
|
17544
|
+
if (request2.sourcePolicy === "fill_gaps") {
|
|
17545
|
+
const hours = new Set;
|
|
17546
|
+
for (const targetTimeMs of request2.requiredClockTargetsMs) {
|
|
17547
|
+
const firstHour2 = Math.floor(Math.max(0, targetTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
|
|
17548
|
+
const lastHour = Math.floor(targetTimeMs / HOUR_MS) * HOUR_MS;
|
|
17549
|
+
for (let hourMs = firstHour2;hourMs <= lastHour; hourMs += HOUR_MS) {
|
|
17550
|
+
hours.add(hourMs);
|
|
17551
|
+
}
|
|
17552
|
+
}
|
|
17553
|
+
return [...hours].sort((left, right) => left - right).map((hourMs) => {
|
|
17554
|
+
const utc = utcHourPath(hourMs);
|
|
17555
|
+
return `${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`;
|
|
17556
|
+
});
|
|
17557
|
+
}
|
|
17558
|
+
const firstHour = Math.floor((request2.window.startTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
|
|
17559
|
+
const objects = [];
|
|
17560
|
+
for (let hourMs = Math.max(0, firstHour);hourMs < request2.window.endTimeMs; hourMs += HOUR_MS) {
|
|
17561
|
+
const utc = utcHourPath(hourMs);
|
|
17562
|
+
objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
|
|
17563
|
+
}
|
|
17564
|
+
return objects;
|
|
17565
|
+
}
|
|
17566
|
+
function initialArchiveCoversTarget(request2, targetTimeMs) {
|
|
17567
|
+
return (request2.initialSelection?.support_anchors ?? []).some((anchor) => {
|
|
17568
|
+
const sourceTimeMs = Date.parse(anchor.source_time);
|
|
17569
|
+
return Number.isSafeInteger(sourceTimeMs) && sourceTimeMs <= targetTimeMs && targetTimeMs - sourceTimeMs <= request2.maxPriorAsOfLagMs;
|
|
17570
|
+
});
|
|
17571
|
+
}
|
|
17572
|
+
function providerAcquisitionRequest(request2) {
|
|
17573
|
+
if (request2.sourcePolicy !== "fill_gaps")
|
|
17574
|
+
return request2;
|
|
17575
|
+
const requiredClockTargetsMs = request2.requiredClockTargetsMs.filter((targetTimeMs) => !initialArchiveCoversTarget(request2, targetTimeMs));
|
|
17576
|
+
if (requiredClockTargetsMs.length === 0) {
|
|
17577
|
+
throw new CryptoHftDataError("fill_gaps_has_no_uncovered_clock_targets");
|
|
17578
|
+
}
|
|
17579
|
+
return { ...request2, requiredClockTargetsMs };
|
|
17580
|
+
}
|
|
17581
|
+
function unsignedString(value, field, required = false) {
|
|
17582
|
+
if (value === null || value === undefined) {
|
|
17583
|
+
if (required)
|
|
17584
|
+
throw new CryptoHftDataError(`schema_${field}_missing`);
|
|
17585
|
+
return;
|
|
17586
|
+
}
|
|
17587
|
+
const rendered = String(value);
|
|
17588
|
+
if (!/^\d+$/.test(rendered)) {
|
|
17589
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17590
|
+
}
|
|
17591
|
+
const parsed = BigInt(rendered);
|
|
17592
|
+
if (parsed > 18446744073709551615n) {
|
|
17593
|
+
throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
|
|
17594
|
+
}
|
|
17595
|
+
return parsed.toString(10);
|
|
17596
|
+
}
|
|
17597
|
+
function timestampMs2(value, field) {
|
|
17598
|
+
const rendered = String(value);
|
|
17599
|
+
if (!/^\d+$/.test(rendered)) {
|
|
17600
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17601
|
+
}
|
|
17602
|
+
const raw = BigInt(rendered);
|
|
17603
|
+
const milliseconds = field === "received_time" ? raw / 1000000n : raw;
|
|
17604
|
+
if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
17605
|
+
throw new CryptoHftDataError(`schema_${field}_unsafe`);
|
|
17606
|
+
}
|
|
17607
|
+
return Number(milliseconds);
|
|
17608
|
+
}
|
|
17609
|
+
function decimal(value, field, allowZero) {
|
|
17610
|
+
if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
|
|
17611
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17612
|
+
}
|
|
17613
|
+
const parsed = Number(value);
|
|
17614
|
+
if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
|
|
17615
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17616
|
+
}
|
|
17617
|
+
return parsed;
|
|
17618
|
+
}
|
|
17619
|
+
function validatedRow(request2, row) {
|
|
17620
|
+
if (row.symbol !== request2.scope.sourceSymbol) {
|
|
17621
|
+
throw new CryptoHftDataError("schema_symbol_mismatch");
|
|
17622
|
+
}
|
|
17623
|
+
if (row.event_type !== "snapshot" && row.event_type !== "update") {
|
|
17624
|
+
throw new CryptoHftDataError("schema_event_type_invalid");
|
|
17625
|
+
}
|
|
17626
|
+
if (row.side !== "bid" && row.side !== "ask") {
|
|
17627
|
+
throw new CryptoHftDataError("schema_side_invalid");
|
|
17628
|
+
}
|
|
17629
|
+
decimal(row.price, "price", false);
|
|
17630
|
+
decimal(row.quantity, "quantity", true);
|
|
17631
|
+
if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
|
|
17632
|
+
throw new CryptoHftDataError("schema_object_checksum_invalid");
|
|
17633
|
+
}
|
|
17634
|
+
const eventTimeMs = timestampMs2(row.event_time, "event_time");
|
|
17635
|
+
const receivedTimeMs = timestampMs2(row.received_time, "received_time");
|
|
17636
|
+
if (receivedTimeMs < eventTimeMs) {
|
|
17637
|
+
throw new CryptoHftDataError("received_time_precedes_event_time");
|
|
17638
|
+
}
|
|
17639
|
+
return { ...row, eventTimeMs, receivedTimeMs };
|
|
17640
|
+
}
|
|
17641
|
+
function groupKey(row, profile) {
|
|
17642
|
+
const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
|
|
17643
|
+
return [
|
|
17644
|
+
row.event_type,
|
|
17645
|
+
row.eventTimeMs,
|
|
17646
|
+
sequence,
|
|
17647
|
+
row.dataset_object_identity
|
|
17648
|
+
].join("\x00");
|
|
17649
|
+
}
|
|
17650
|
+
function absent(value) {
|
|
17651
|
+
return value === null || value === undefined;
|
|
17652
|
+
}
|
|
17653
|
+
function snapshotSequence(row, profile) {
|
|
17654
|
+
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
17655
|
+
if (String(row.last_update_id) !== "-1") {
|
|
17656
|
+
throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
|
|
18144
17657
|
}
|
|
18145
|
-
|
|
17658
|
+
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
|
|
17659
|
+
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
17660
|
+
}
|
|
17661
|
+
return unsignedString(row.final_update_id, "final_update_id", true);
|
|
17662
|
+
}
|
|
17663
|
+
return unsignedString(row.last_update_id, "last_update_id", true);
|
|
18146
17664
|
}
|
|
18147
|
-
|
|
18148
|
-
|
|
18149
|
-
|
|
18150
|
-
|
|
18151
|
-
|
|
18152
|
-
|
|
18153
|
-
|
|
18154
|
-
|
|
18155
|
-
|
|
18156
|
-
|
|
18157
|
-
|
|
18158
|
-
marketType: "spot",
|
|
18159
|
-
providerExchangeId: "binance_spot",
|
|
18160
|
-
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
18161
|
-
maxDepth: 500,
|
|
18162
|
-
eventTimeUnit: "milliseconds",
|
|
18163
|
-
receivedTimeUnit: "nanoseconds",
|
|
18164
|
-
snapshotGrouping: "event_time_last_update_id_object",
|
|
18165
|
-
sequenceSemantics: "binance_u_U_pu",
|
|
18166
|
-
constructionModes: ["sampled_top_n_snapshot"],
|
|
18167
|
-
sourcePolicies: ["authoritative_window"]
|
|
18168
|
-
});
|
|
18169
|
-
var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
|
|
18170
|
-
profileId: "cryptohftdata/okx_spot/ARB-USDT/v1",
|
|
18171
|
-
exchange: "okx",
|
|
18172
|
-
tradingPair: "ARB-USDT",
|
|
18173
|
-
sourceSymbol: "ARB-USDT",
|
|
18174
|
-
marketType: "spot",
|
|
18175
|
-
providerExchangeId: "okx_spot",
|
|
18176
|
-
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
18177
|
-
maxDepth: 400,
|
|
18178
|
-
eventTimeUnit: "milliseconds",
|
|
18179
|
-
receivedTimeUnit: "nanoseconds",
|
|
18180
|
-
snapshotGrouping: "event_time_final_update_id_object",
|
|
18181
|
-
sequenceSemantics: "okx_seq_id_prev_seq_id",
|
|
18182
|
-
constructionModes: ["sampled_top_n_snapshot"],
|
|
18183
|
-
sourcePolicies: ["authoritative_window"]
|
|
18184
|
-
});
|
|
18185
|
-
|
|
18186
|
-
class CryptoHftDataError extends Error {
|
|
18187
|
-
reason;
|
|
18188
|
-
constructor(reason) {
|
|
18189
|
-
super(`CryptoHFTData backfill failed: ${reason}`);
|
|
18190
|
-
this.reason = reason;
|
|
18191
|
-
this.name = "CryptoHftDataError";
|
|
17665
|
+
function sortedSide(levels, side) {
|
|
17666
|
+
return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
|
|
17667
|
+
}
|
|
17668
|
+
function applyRows(state, rows) {
|
|
17669
|
+
for (const row of rows) {
|
|
17670
|
+
const levels = row.side === "bid" ? state.bids : state.asks;
|
|
17671
|
+
const quantity = decimal(row.quantity, "quantity", true);
|
|
17672
|
+
if (quantity === 0)
|
|
17673
|
+
levels.delete(row.price);
|
|
17674
|
+
else
|
|
17675
|
+
levels.set(row.price, quantity);
|
|
18192
17676
|
}
|
|
18193
17677
|
}
|
|
18194
|
-
function
|
|
18195
|
-
|
|
18196
|
-
|
|
17678
|
+
function reconstructCryptoHftDataOrderBooks(request2, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
|
|
17679
|
+
const rows = inputRows.map((row, index) => ({
|
|
17680
|
+
...validatedRow(request2, row),
|
|
17681
|
+
originalIndex: index
|
|
17682
|
+
})).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
|
|
17683
|
+
const groups = [];
|
|
17684
|
+
for (const row of rows) {
|
|
17685
|
+
const current = groups.at(-1);
|
|
17686
|
+
if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
|
|
17687
|
+
groups.push([row]);
|
|
17688
|
+
} else {
|
|
17689
|
+
current.push(row);
|
|
17690
|
+
}
|
|
17691
|
+
}
|
|
17692
|
+
const earliestTargetTimeMs = Math.min(...request2.requiredClockTargetsMs);
|
|
17693
|
+
const latestTargetTimeMs = Math.max(...request2.requiredClockTargetsMs);
|
|
17694
|
+
const anchorIndex = groups.findIndex((group) => {
|
|
17695
|
+
const first = group[0];
|
|
17696
|
+
return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
|
|
17697
|
+
});
|
|
17698
|
+
if (anchorIndex < 0) {
|
|
17699
|
+
throw new CryptoHftDataError("update_before_snapshot");
|
|
17700
|
+
}
|
|
17701
|
+
let state;
|
|
17702
|
+
let previousFinalUpdateId;
|
|
17703
|
+
const states = [];
|
|
17704
|
+
for (const group of groups.slice(anchorIndex)) {
|
|
17705
|
+
const first = group[0];
|
|
17706
|
+
if (first.eventTimeMs > latestTargetTimeMs)
|
|
17707
|
+
break;
|
|
17708
|
+
if (first.event_type === "snapshot") {
|
|
17709
|
+
const sequence = snapshotSequence(first, profile);
|
|
17710
|
+
for (const row of group) {
|
|
17711
|
+
if (snapshotSequence(row, profile) !== sequence) {
|
|
17712
|
+
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
17713
|
+
}
|
|
17714
|
+
}
|
|
17715
|
+
if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
|
|
17716
|
+
throw new CryptoHftDataError("snapshot_sequence_regression");
|
|
17717
|
+
}
|
|
17718
|
+
state = {
|
|
17719
|
+
bids: new Map,
|
|
17720
|
+
asks: new Map,
|
|
17721
|
+
sequence,
|
|
17722
|
+
sourceTimeMs: first.eventTimeMs,
|
|
17723
|
+
receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
|
|
17724
|
+
datasetObjectIdentity: first.dataset_object_identity,
|
|
17725
|
+
datasetObjectChecksum: first.dataset_object_checksum
|
|
17726
|
+
};
|
|
17727
|
+
applyRows(state, group);
|
|
17728
|
+
previousFinalUpdateId = BigInt(sequence);
|
|
17729
|
+
} else {
|
|
17730
|
+
if (!state || previousFinalUpdateId === undefined) {
|
|
17731
|
+
throw new CryptoHftDataError("update_before_snapshot");
|
|
17732
|
+
}
|
|
17733
|
+
const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
|
|
17734
|
+
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
17735
|
+
if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
|
|
17736
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
17737
|
+
}
|
|
17738
|
+
const previous = unsignedString(first.last_update_id, "last_update_id", true);
|
|
17739
|
+
for (const row of group) {
|
|
17740
|
+
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
|
|
17741
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
17742
|
+
}
|
|
17743
|
+
}
|
|
17744
|
+
if (BigInt(previous) !== previousFinalUpdateId) {
|
|
17745
|
+
throw new CryptoHftDataError("update_chain_gap");
|
|
17746
|
+
}
|
|
17747
|
+
} else {
|
|
17748
|
+
const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
|
|
17749
|
+
const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
|
|
17750
|
+
for (const row of group) {
|
|
17751
|
+
if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
|
|
17752
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
17753
|
+
}
|
|
17754
|
+
}
|
|
17755
|
+
const expected = previousFinalUpdateId + 1n;
|
|
17756
|
+
if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
|
|
17757
|
+
throw new CryptoHftDataError("update_chain_gap");
|
|
17758
|
+
}
|
|
17759
|
+
}
|
|
17760
|
+
applyRows(state, group);
|
|
17761
|
+
state.sequence = finalUpdate.toString();
|
|
17762
|
+
state.sourceTimeMs = first.eventTimeMs;
|
|
17763
|
+
state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
|
|
17764
|
+
state.datasetObjectIdentity = first.dataset_object_identity;
|
|
17765
|
+
state.datasetObjectChecksum = first.dataset_object_checksum;
|
|
17766
|
+
previousFinalUpdateId = finalUpdate;
|
|
17767
|
+
}
|
|
17768
|
+
if (state) {
|
|
17769
|
+
states.push({
|
|
17770
|
+
...state,
|
|
17771
|
+
bids: new Map(state.bids),
|
|
17772
|
+
asks: new Map(state.asks)
|
|
17773
|
+
});
|
|
17774
|
+
}
|
|
17775
|
+
}
|
|
17776
|
+
const samples = [];
|
|
17777
|
+
for (const targetTimeMs of request2.requiredClockTargetsMs) {
|
|
17778
|
+
let prior;
|
|
17779
|
+
for (const candidate of states) {
|
|
17780
|
+
if (candidate.sourceTimeMs > targetTimeMs)
|
|
17781
|
+
break;
|
|
17782
|
+
prior = candidate;
|
|
17783
|
+
}
|
|
17784
|
+
if (!prior || targetTimeMs - prior.sourceTimeMs > request2.maxPriorAsOfLagMs) {
|
|
17785
|
+
throw new CryptoHftDataError("required_clock_coverage_insufficient");
|
|
17786
|
+
}
|
|
17787
|
+
const bids = sortedSide(prior.bids, "bid").slice(0, request2.depth);
|
|
17788
|
+
const asks = sortedSide(prior.asks, "ask").slice(0, request2.depth);
|
|
17789
|
+
if (bids.length === 0 || asks.length === 0) {
|
|
17790
|
+
throw new CryptoHftDataError("book_side_missing");
|
|
17791
|
+
}
|
|
17792
|
+
if (bids[0]?.[0] >= asks[0]?.[0]) {
|
|
17793
|
+
throw new CryptoHftDataError("book_crossed_or_locked");
|
|
17794
|
+
}
|
|
17795
|
+
samples.push({
|
|
17796
|
+
targetTimeMs,
|
|
17797
|
+
sourceTimeMs: prior.sourceTimeMs,
|
|
17798
|
+
receivedTimeMs: prior.receivedTimeMs,
|
|
17799
|
+
sequence: prior.sequence,
|
|
17800
|
+
bids,
|
|
17801
|
+
asks,
|
|
17802
|
+
datasetObjectIdentity: prior.datasetObjectIdentity,
|
|
17803
|
+
datasetObjectChecksum: prior.datasetObjectChecksum
|
|
17804
|
+
});
|
|
18197
17805
|
}
|
|
18198
|
-
|
|
18199
|
-
if (!profile)
|
|
18200
|
-
return;
|
|
18201
|
-
return {
|
|
18202
|
-
provider: "cryptohftdata",
|
|
18203
|
-
adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
|
|
18204
|
-
providerExchangeId: profile.providerExchangeId,
|
|
18205
|
-
resolvedSymbol: profile.sourceSymbol
|
|
18206
|
-
};
|
|
17806
|
+
return samples;
|
|
18207
17807
|
}
|
|
18208
|
-
function
|
|
18209
|
-
|
|
17808
|
+
function sha256Bytes(bytes) {
|
|
17809
|
+
return createHash3("sha256").update(bytes).digest("hex");
|
|
17810
|
+
}
|
|
17811
|
+
async function decodeCryptoHftParquetZstd(bytes) {
|
|
17812
|
+
const parquet = decompress(bytes);
|
|
17813
|
+
const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
|
|
17814
|
+
return parquetReadObjects({ file: buffer });
|
|
17815
|
+
}
|
|
17816
|
+
function parsedDatasetRow(value, object) {
|
|
18210
17817
|
return {
|
|
18211
|
-
|
|
18212
|
-
|
|
17818
|
+
received_time: value.received_time,
|
|
17819
|
+
event_time: value.event_time,
|
|
17820
|
+
transaction_time: value.transaction_time,
|
|
17821
|
+
symbol: value.symbol,
|
|
17822
|
+
event_type: value.event_type,
|
|
17823
|
+
first_update_id: value.first_update_id,
|
|
17824
|
+
final_update_id: value.final_update_id,
|
|
17825
|
+
prev_final_update_id: value.prev_final_update_id,
|
|
17826
|
+
last_update_id: value.last_update_id,
|
|
17827
|
+
side: value.side,
|
|
17828
|
+
price: value.price,
|
|
17829
|
+
quantity: value.quantity,
|
|
17830
|
+
order_count: value.order_count,
|
|
17831
|
+
dataset_object_identity: object.identity,
|
|
17832
|
+
dataset_object_checksum: object.checksum
|
|
18213
17833
|
};
|
|
18214
17834
|
}
|
|
18215
|
-
function
|
|
18216
|
-
|
|
18217
|
-
|
|
18218
|
-
|
|
18219
|
-
|
|
18220
|
-
|
|
17835
|
+
async function readBoundedObject(response, remainingBytes) {
|
|
17836
|
+
if (!response.body)
|
|
17837
|
+
return new Uint8Array;
|
|
17838
|
+
const reader = response.body.getReader();
|
|
17839
|
+
const chunks = [];
|
|
17840
|
+
let total = 0;
|
|
17841
|
+
while (true) {
|
|
17842
|
+
const { done, value } = await reader.read();
|
|
17843
|
+
if (done)
|
|
17844
|
+
break;
|
|
17845
|
+
total += value.byteLength;
|
|
17846
|
+
if (total > remainingBytes) {
|
|
17847
|
+
await reader.cancel();
|
|
17848
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
17849
|
+
}
|
|
17850
|
+
chunks.push(value);
|
|
18221
17851
|
}
|
|
18222
|
-
|
|
17852
|
+
const bytes = new Uint8Array(total);
|
|
17853
|
+
let offset = 0;
|
|
17854
|
+
for (const chunk of chunks) {
|
|
17855
|
+
bytes.set(chunk, offset);
|
|
17856
|
+
offset += chunk.byteLength;
|
|
17857
|
+
}
|
|
17858
|
+
return bytes;
|
|
18223
17859
|
}
|
|
18224
|
-
|
|
18225
|
-
|
|
18226
|
-
|
|
18227
|
-
|
|
18228
|
-
|
|
17860
|
+
|
|
17861
|
+
class CryptoHftDataAdapter {
|
|
17862
|
+
baseUrl;
|
|
17863
|
+
request;
|
|
17864
|
+
nowMs;
|
|
17865
|
+
decode;
|
|
17866
|
+
profiles;
|
|
17867
|
+
constructor(options = {}) {
|
|
17868
|
+
this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
|
|
17869
|
+
this.request = options.fetch ?? fetch;
|
|
17870
|
+
this.nowMs = options.nowMs ?? Date.now;
|
|
17871
|
+
this.decode = options.decode ?? decodeCryptoHftParquetZstd;
|
|
17872
|
+
this.profiles = options.profiles ?? [];
|
|
18229
17873
|
}
|
|
18230
|
-
|
|
18231
|
-
|
|
18232
|
-
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17874
|
+
capabilityFor(request2) {
|
|
17875
|
+
return cryptoHftDataCapabilityFor(request2, this.profiles);
|
|
18233
17876
|
}
|
|
18234
|
-
|
|
18235
|
-
|
|
18236
|
-
|
|
17877
|
+
async discoverSymbols(providerExchangeId) {
|
|
17878
|
+
if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
|
|
17879
|
+
throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
|
|
17880
|
+
}
|
|
17881
|
+
const endpoint = new URL("/symbols", this.baseUrl);
|
|
17882
|
+
endpoint.searchParams.set("exchange", providerExchangeId);
|
|
17883
|
+
endpoint.searchParams.set("data_type", "orderbook");
|
|
17884
|
+
let response;
|
|
17885
|
+
try {
|
|
17886
|
+
response = await this.request(endpoint);
|
|
17887
|
+
} catch {
|
|
17888
|
+
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
17889
|
+
}
|
|
17890
|
+
if (!response.ok) {
|
|
17891
|
+
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
17892
|
+
}
|
|
17893
|
+
let body;
|
|
17894
|
+
try {
|
|
17895
|
+
body = await response.json();
|
|
17896
|
+
} catch {
|
|
17897
|
+
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
17898
|
+
}
|
|
17899
|
+
const record = body;
|
|
17900
|
+
if (!body || typeof body !== "object" || Array.isArray(body) || String(record.exchange).toLowerCase() !== providerExchangeId || String(record.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record.symbols) || !record.symbols.every((symbol) => typeof symbol === "string" && /^[A-Z0-9_-]+$/.test(symbol))) {
|
|
17901
|
+
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
17902
|
+
}
|
|
17903
|
+
return [...new Set(record.symbols)].sort();
|
|
18237
17904
|
}
|
|
18238
|
-
|
|
18239
|
-
|
|
18240
|
-
|
|
18241
|
-
|
|
18242
|
-
|
|
18243
|
-
|
|
17905
|
+
async acquire(request2, capability, credential) {
|
|
17906
|
+
const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
|
|
17907
|
+
if (typeof apiKey !== "string" || apiKey.length === 0) {
|
|
17908
|
+
throw new CryptoHftDataError("credentials_invalid");
|
|
17909
|
+
}
|
|
17910
|
+
const paths = enumerateCryptoHftDataObjects(request2, capability.providerExchangeId, capability.resolvedSymbol);
|
|
17911
|
+
if (paths.length > request2.budgets.maxFiles) {
|
|
17912
|
+
throw new CryptoHftDataError("budget_max_files_exceeded");
|
|
17913
|
+
}
|
|
17914
|
+
const started = this.nowMs();
|
|
17915
|
+
const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
|
|
17916
|
+
method: "POST",
|
|
17917
|
+
headers: { "content-type": "application/json", "X-API-Key": apiKey }
|
|
17918
|
+
});
|
|
17919
|
+
if (!tokenResponse.ok)
|
|
17920
|
+
throw new CryptoHftDataError("jwt_issuance_failed");
|
|
17921
|
+
const tokenBody = await tokenResponse.json();
|
|
17922
|
+
if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
|
|
17923
|
+
throw new CryptoHftDataError("jwt_response_invalid");
|
|
17924
|
+
}
|
|
17925
|
+
let totalBytes = 0;
|
|
17926
|
+
let totalRows = 0;
|
|
17927
|
+
const objects = [];
|
|
17928
|
+
const rows = [];
|
|
17929
|
+
for (const path of paths) {
|
|
17930
|
+
if (this.nowMs() - started > request2.budgets.maxDurationMs) {
|
|
17931
|
+
throw new CryptoHftDataError("budget_max_duration_exceeded");
|
|
17932
|
+
}
|
|
17933
|
+
const endpoint = new URL("/download", this.baseUrl);
|
|
17934
|
+
endpoint.searchParams.set("file", path);
|
|
17935
|
+
const response = await this.request(endpoint, {
|
|
17936
|
+
headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
|
|
17937
|
+
});
|
|
17938
|
+
if (!response.ok)
|
|
17939
|
+
throw new CryptoHftDataError("object_download_failed");
|
|
17940
|
+
const declaredBytes = Number(response.headers.get("content-length"));
|
|
17941
|
+
if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request2.budgets.maxBytes) {
|
|
17942
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
17943
|
+
}
|
|
17944
|
+
const bytes = await readBoundedObject(response, request2.budgets.maxBytes - totalBytes);
|
|
17945
|
+
totalBytes += bytes.byteLength;
|
|
17946
|
+
if (totalBytes > request2.budgets.maxBytes) {
|
|
17947
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
17948
|
+
}
|
|
17949
|
+
const decoded = await this.decode(bytes);
|
|
17950
|
+
totalRows += decoded.length;
|
|
17951
|
+
if (totalRows > request2.budgets.maxRows) {
|
|
17952
|
+
throw new CryptoHftDataError("budget_max_rows_exceeded");
|
|
17953
|
+
}
|
|
17954
|
+
const object = {
|
|
17955
|
+
identity: path,
|
|
17956
|
+
checksum: sha256Bytes(bytes),
|
|
17957
|
+
bytes: bytes.byteLength,
|
|
17958
|
+
rows: decoded.length
|
|
17959
|
+
};
|
|
17960
|
+
objects.push(object);
|
|
17961
|
+
for (const decodedRow of decoded) {
|
|
17962
|
+
const parsed = parsedDatasetRow(decodedRow, object);
|
|
17963
|
+
validatedRow(request2, parsed);
|
|
17964
|
+
rows.push(parsed);
|
|
17965
|
+
}
|
|
17966
|
+
}
|
|
17967
|
+
if (this.nowMs() - started > request2.budgets.maxDurationMs) {
|
|
17968
|
+
throw new CryptoHftDataError("budget_max_duration_exceeded");
|
|
17969
|
+
}
|
|
17970
|
+
return {
|
|
17971
|
+
objects,
|
|
17972
|
+
rows,
|
|
17973
|
+
vendorSemanticDigest: sha256Canonical(rows)
|
|
17974
|
+
};
|
|
18244
17975
|
}
|
|
18245
|
-
|
|
18246
|
-
|
|
18247
|
-
|
|
18248
|
-
|
|
17976
|
+
async normalize(request2, capability, dataset, captureBundleId2) {
|
|
17977
|
+
const profile = this.profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
|
|
17978
|
+
if (!profile) {
|
|
17979
|
+
throw new CryptoHftDataError("profile_semantics_unavailable");
|
|
17980
|
+
}
|
|
17981
|
+
const samples = reconstructCryptoHftDataOrderBooks(request2, dataset.rows, profile);
|
|
17982
|
+
const context2 = {
|
|
17983
|
+
source: EXTERNAL_BACKFILL_SOURCE,
|
|
17984
|
+
deploymentId: "market-data-vendor-backfill",
|
|
17985
|
+
captureBundleId: captureBundleId2,
|
|
17986
|
+
exchange: request2.scope.exchange,
|
|
17987
|
+
symbol: request2.scope.tradingPair,
|
|
17988
|
+
tradingPair: request2.scope.tradingPair,
|
|
17989
|
+
sourceSymbol: request2.scope.sourceSymbol,
|
|
17990
|
+
assetType: request2.scope.marketType,
|
|
17991
|
+
feed: "ORDERBOOK",
|
|
17992
|
+
provider: "cryptohftdata",
|
|
17993
|
+
sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
|
|
17994
|
+
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
17995
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
17996
|
+
provenanceComplete: true
|
|
17997
|
+
};
|
|
17998
|
+
const rows = samples.flatMap((sample) => {
|
|
17999
|
+
const rawCapture = {
|
|
18000
|
+
rawCaptureId: sha256Canonical({
|
|
18001
|
+
capture_bundle_id: captureBundleId2,
|
|
18002
|
+
dataset_object_identity: sample.datasetObjectIdentity,
|
|
18003
|
+
dataset_object_checksum: sample.datasetObjectChecksum,
|
|
18004
|
+
source_time_ms: sample.sourceTimeMs,
|
|
18005
|
+
target_time_ms: sample.targetTimeMs
|
|
18006
|
+
}),
|
|
18007
|
+
rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
|
|
18008
|
+
rawChecksum: sample.datasetObjectChecksum,
|
|
18009
|
+
redactedPayload: {
|
|
18010
|
+
dataset_object_identity: sample.datasetObjectIdentity,
|
|
18011
|
+
dataset_object_checksum: sample.datasetObjectChecksum
|
|
18012
|
+
},
|
|
18013
|
+
eventTimeMs: sample.sourceTimeMs,
|
|
18014
|
+
receivedTimeMs: sample.receivedTimeMs,
|
|
18015
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM
|
|
18016
|
+
};
|
|
18017
|
+
const canonical = buildCanonicalOrderBookRows({
|
|
18018
|
+
context: context2,
|
|
18019
|
+
rawCapture,
|
|
18020
|
+
depthLimit: request2.depth,
|
|
18021
|
+
constructionMode: request2.constructionMode,
|
|
18022
|
+
snapshot: {
|
|
18023
|
+
bids: sample.bids,
|
|
18024
|
+
asks: sample.asks,
|
|
18025
|
+
timestamp: sample.sourceTimeMs,
|
|
18026
|
+
receivedTimestamp: sample.receivedTimeMs,
|
|
18027
|
+
exchange: request2.scope.exchange,
|
|
18028
|
+
symbol: request2.scope.tradingPair,
|
|
18029
|
+
depthLimit: request2.depth,
|
|
18030
|
+
sequence: sample.sequence
|
|
18031
|
+
}
|
|
18032
|
+
});
|
|
18033
|
+
return [...canonical.levels, canonical.summary];
|
|
18034
|
+
});
|
|
18035
|
+
return {
|
|
18036
|
+
captureBundleId: captureBundleId2,
|
|
18037
|
+
objects: dataset.objects,
|
|
18038
|
+
rows,
|
|
18039
|
+
vendorSemanticDigest: dataset.vendorSemanticDigest,
|
|
18040
|
+
canonicalSemanticDigest: semanticDigest(rows)
|
|
18041
|
+
};
|
|
18249
18042
|
}
|
|
18250
|
-
return Number(milliseconds);
|
|
18251
18043
|
}
|
|
18252
|
-
|
|
18253
|
-
|
|
18254
|
-
|
|
18255
|
-
|
|
18256
|
-
|
|
18257
|
-
|
|
18258
|
-
|
|
18044
|
+
|
|
18045
|
+
// src/helpers/market-data-vendor-backfill/qualification.ts
|
|
18046
|
+
var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
18047
|
+
var SHA256 = /^[0-9a-f]{64}$/;
|
|
18048
|
+
var REASON = /^[a-z][a-z0-9_]{0,127}$/;
|
|
18049
|
+
var EVENT_FIELDS = new Set([
|
|
18050
|
+
"qualification_event_id",
|
|
18051
|
+
"capture_bundle_id",
|
|
18052
|
+
"state",
|
|
18053
|
+
"receipt_id",
|
|
18054
|
+
"promotion_identity_sha256",
|
|
18055
|
+
"window",
|
|
18056
|
+
"event_at",
|
|
18057
|
+
"reason_code"
|
|
18058
|
+
]);
|
|
18059
|
+
function deterministicUuid(value) {
|
|
18060
|
+
const digest = jcsSha256(value).slice(0, 32).split("");
|
|
18061
|
+
digest[12] = "5";
|
|
18062
|
+
digest[16] = "8";
|
|
18063
|
+
const hex = digest.join("");
|
|
18064
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
18065
|
+
}
|
|
18066
|
+
function semanticEvent(event) {
|
|
18067
|
+
const { qualification_event_id: _eventId, ...semantic } = event;
|
|
18068
|
+
return semantic;
|
|
18069
|
+
}
|
|
18070
|
+
function assertTimestamp(value, field) {
|
|
18071
|
+
const parsed = Date.parse(value);
|
|
18072
|
+
if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
|
|
18073
|
+
throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
|
|
18259
18074
|
}
|
|
18260
18075
|
return parsed;
|
|
18261
18076
|
}
|
|
18262
|
-
function
|
|
18263
|
-
if (
|
|
18264
|
-
throw new
|
|
18265
|
-
}
|
|
18266
|
-
if (row.event_type !== "snapshot" && row.event_type !== "update") {
|
|
18267
|
-
throw new CryptoHftDataError("schema_event_type_invalid");
|
|
18077
|
+
function parseQualificationEvent(value) {
|
|
18078
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
18079
|
+
throw new Error("qualification event must be an object");
|
|
18268
18080
|
}
|
|
18269
|
-
|
|
18270
|
-
|
|
18081
|
+
const event = value;
|
|
18082
|
+
if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
|
|
18083
|
+
throw new Error("qualification event fields are invalid");
|
|
18271
18084
|
}
|
|
18272
|
-
|
|
18273
|
-
|
|
18274
|
-
if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
|
|
18275
|
-
throw new CryptoHftDataError("schema_object_checksum_invalid");
|
|
18085
|
+
if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
|
|
18086
|
+
throw new Error("qualification window must be increasing");
|
|
18276
18087
|
}
|
|
18277
|
-
|
|
18278
|
-
|
|
18279
|
-
|
|
18280
|
-
throw new CryptoHftDataError("received_time_precedes_event_time");
|
|
18088
|
+
assertTimestamp(event.event_at, "event_at");
|
|
18089
|
+
if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
|
|
18090
|
+
throw new Error("qualification_event_id does not match event content");
|
|
18281
18091
|
}
|
|
18282
|
-
return
|
|
18283
|
-
}
|
|
18284
|
-
function groupKey(row, profile) {
|
|
18285
|
-
const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
|
|
18286
|
-
return [
|
|
18287
|
-
row.event_type,
|
|
18288
|
-
row.eventTimeMs,
|
|
18289
|
-
sequence,
|
|
18290
|
-
row.dataset_object_identity
|
|
18291
|
-
].join("\x00");
|
|
18092
|
+
return event;
|
|
18292
18093
|
}
|
|
18293
|
-
function
|
|
18294
|
-
|
|
18094
|
+
function finalizeQualificationEvent(event) {
|
|
18095
|
+
const semantic = semanticEvent(event);
|
|
18096
|
+
return parseQualificationEvent({
|
|
18097
|
+
...semantic,
|
|
18098
|
+
qualification_event_id: deterministicUuid(semantic)
|
|
18099
|
+
});
|
|
18295
18100
|
}
|
|
18296
|
-
function
|
|
18297
|
-
|
|
18298
|
-
|
|
18299
|
-
|
|
18300
|
-
|
|
18301
|
-
|
|
18302
|
-
|
|
18101
|
+
function qualificationEventToArchiveRow(eventInput) {
|
|
18102
|
+
const event = parseQualificationEvent(eventInput);
|
|
18103
|
+
return {
|
|
18104
|
+
table: "market_data.cex_order_book_capture_qualifications",
|
|
18105
|
+
row: {
|
|
18106
|
+
source: "external_backfill",
|
|
18107
|
+
capture_origin: "vendor_historical_backfill",
|
|
18108
|
+
source_mode: "vendor_historical_backfill_v1",
|
|
18109
|
+
deployment_id: "market-data-vendor-backfill",
|
|
18110
|
+
qualification_event_id: event.qualification_event_id,
|
|
18111
|
+
capture_bundle_id: event.capture_bundle_id,
|
|
18112
|
+
state: event.state,
|
|
18113
|
+
receipt_id: event.receipt_id,
|
|
18114
|
+
promotion_identity_sha256: event.promotion_identity_sha256,
|
|
18115
|
+
window_start_ms: Date.parse(event.window.start_at),
|
|
18116
|
+
window_end_ms: Date.parse(event.window.end_at),
|
|
18117
|
+
event_at_ms: Date.parse(event.event_at),
|
|
18118
|
+
reason_code: event.reason_code,
|
|
18119
|
+
event_json: jcsCanonicalize(event)
|
|
18303
18120
|
}
|
|
18304
|
-
|
|
18305
|
-
}
|
|
18306
|
-
return unsignedString(row.last_update_id, "last_update_id", true);
|
|
18307
|
-
}
|
|
18308
|
-
function sortedSide(levels, side) {
|
|
18309
|
-
return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
|
|
18310
|
-
}
|
|
18311
|
-
function applyRows(state, rows) {
|
|
18312
|
-
for (const row of rows) {
|
|
18313
|
-
const levels = row.side === "bid" ? state.bids : state.asks;
|
|
18314
|
-
const quantity = decimal(row.quantity, "quantity", true);
|
|
18315
|
-
if (quantity === 0)
|
|
18316
|
-
levels.delete(row.price);
|
|
18317
|
-
else
|
|
18318
|
-
levels.set(row.price, quantity);
|
|
18319
|
-
}
|
|
18121
|
+
};
|
|
18320
18122
|
}
|
|
18321
|
-
|
|
18322
|
-
|
|
18323
|
-
|
|
18324
|
-
|
|
18325
|
-
|
|
18326
|
-
|
|
18327
|
-
|
|
18328
|
-
|
|
18329
|
-
|
|
18330
|
-
|
|
18331
|
-
|
|
18332
|
-
|
|
18123
|
+
|
|
18124
|
+
// src/helpers/market-data-vendor-backfill/core.ts
|
|
18125
|
+
function createMarketDataVendorBackfillDependencies(dependencies) {
|
|
18126
|
+
for (const [name, value] of Object.entries({
|
|
18127
|
+
archive: dependencies.archive,
|
|
18128
|
+
providers: dependencies.providers,
|
|
18129
|
+
credentials: dependencies.credentials,
|
|
18130
|
+
forwarder: dependencies.forwarder,
|
|
18131
|
+
clock: dependencies.clock
|
|
18132
|
+
})) {
|
|
18133
|
+
if (!value || typeof value !== "object") {
|
|
18134
|
+
throw new TypeError(`Backfill dependency ${name} is required`);
|
|
18333
18135
|
}
|
|
18334
18136
|
}
|
|
18335
|
-
const
|
|
18336
|
-
|
|
18337
|
-
|
|
18338
|
-
|
|
18339
|
-
|
|
18340
|
-
|
|
18341
|
-
|
|
18342
|
-
|
|
18343
|
-
|
|
18344
|
-
|
|
18345
|
-
|
|
18346
|
-
|
|
18347
|
-
|
|
18348
|
-
const first = group[0];
|
|
18349
|
-
if (first.eventTimeMs > latestTargetTimeMs)
|
|
18350
|
-
break;
|
|
18351
|
-
if (first.event_type === "snapshot") {
|
|
18352
|
-
const sequence = snapshotSequence(first, profile);
|
|
18353
|
-
for (const row of group) {
|
|
18354
|
-
if (snapshotSequence(row, profile) !== sequence) {
|
|
18355
|
-
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
18356
|
-
}
|
|
18357
|
-
}
|
|
18358
|
-
if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
|
|
18359
|
-
throw new CryptoHftDataError("snapshot_sequence_regression");
|
|
18360
|
-
}
|
|
18361
|
-
state = {
|
|
18362
|
-
bids: new Map,
|
|
18363
|
-
asks: new Map,
|
|
18364
|
-
sequence,
|
|
18365
|
-
sourceTimeMs: first.eventTimeMs,
|
|
18366
|
-
receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
|
|
18367
|
-
datasetObjectIdentity: first.dataset_object_identity,
|
|
18368
|
-
datasetObjectChecksum: first.dataset_object_checksum
|
|
18369
|
-
};
|
|
18370
|
-
applyRows(state, group);
|
|
18371
|
-
previousFinalUpdateId = BigInt(sequence);
|
|
18372
|
-
} else {
|
|
18373
|
-
if (!state || previousFinalUpdateId === undefined) {
|
|
18374
|
-
throw new CryptoHftDataError("update_before_snapshot");
|
|
18375
|
-
}
|
|
18376
|
-
const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
|
|
18377
|
-
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
18378
|
-
if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
|
|
18379
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
18380
|
-
}
|
|
18381
|
-
const previous = unsignedString(first.last_update_id, "last_update_id", true);
|
|
18382
|
-
for (const row of group) {
|
|
18383
|
-
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
|
|
18384
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
18385
|
-
}
|
|
18386
|
-
}
|
|
18387
|
-
if (BigInt(previous) !== previousFinalUpdateId) {
|
|
18388
|
-
throw new CryptoHftDataError("update_chain_gap");
|
|
18389
|
-
}
|
|
18390
|
-
} else {
|
|
18391
|
-
const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
|
|
18392
|
-
const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
|
|
18393
|
-
for (const row of group) {
|
|
18394
|
-
if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
|
|
18395
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
18396
|
-
}
|
|
18397
|
-
}
|
|
18398
|
-
const expected = previousFinalUpdateId + 1n;
|
|
18399
|
-
if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
|
|
18400
|
-
throw new CryptoHftDataError("update_chain_gap");
|
|
18401
|
-
}
|
|
18402
|
-
}
|
|
18403
|
-
applyRows(state, group);
|
|
18404
|
-
state.sequence = finalUpdate.toString();
|
|
18405
|
-
state.sourceTimeMs = first.eventTimeMs;
|
|
18406
|
-
state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
|
|
18407
|
-
state.datasetObjectIdentity = first.dataset_object_identity;
|
|
18408
|
-
state.datasetObjectChecksum = first.dataset_object_checksum;
|
|
18409
|
-
previousFinalUpdateId = finalUpdate;
|
|
18410
|
-
}
|
|
18411
|
-
if (state) {
|
|
18412
|
-
states.push({
|
|
18413
|
-
...state,
|
|
18414
|
-
bids: new Map(state.bids),
|
|
18415
|
-
asks: new Map(state.asks)
|
|
18416
|
-
});
|
|
18137
|
+
for (const [name, method] of Object.entries({
|
|
18138
|
+
"archive.resolveSelection": dependencies.archive.resolveSelection,
|
|
18139
|
+
"archive.verifyCandidate": dependencies.archive.verifyCandidate,
|
|
18140
|
+
"providers.capabilityFor": dependencies.providers.capabilityFor,
|
|
18141
|
+
"providers.acquire": dependencies.providers.acquire,
|
|
18142
|
+
"providers.normalize": dependencies.providers.normalize,
|
|
18143
|
+
"credentials.resolve": dependencies.credentials.resolve,
|
|
18144
|
+
"forwarder.preflight": dependencies.forwarder.preflight,
|
|
18145
|
+
"forwarder.submit": dependencies.forwarder.submit,
|
|
18146
|
+
"clock.nowMs": dependencies.clock.nowMs
|
|
18147
|
+
})) {
|
|
18148
|
+
if (typeof method !== "function") {
|
|
18149
|
+
throw new TypeError(`Backfill dependency ${name} is required`);
|
|
18417
18150
|
}
|
|
18418
18151
|
}
|
|
18419
|
-
|
|
18420
|
-
|
|
18421
|
-
|
|
18422
|
-
|
|
18423
|
-
|
|
18152
|
+
return dependencies;
|
|
18153
|
+
}
|
|
18154
|
+
function outcome(request2, status, reasonCode, extra = {}) {
|
|
18155
|
+
return {
|
|
18156
|
+
status,
|
|
18157
|
+
reasonCode,
|
|
18158
|
+
...request2 ? {
|
|
18159
|
+
requestId: request2.requestId,
|
|
18160
|
+
idempotencyKey: request2.idempotencyKey,
|
|
18161
|
+
...request2.target ? { target: request2.target } : {}
|
|
18162
|
+
} : {},
|
|
18163
|
+
...extra
|
|
18164
|
+
};
|
|
18165
|
+
}
|
|
18166
|
+
function captureBundleId2(request2, capability, dataset) {
|
|
18167
|
+
return sha256Canonical({
|
|
18168
|
+
request_business_identity: request2.idempotencyKey,
|
|
18169
|
+
provider: capability.provider,
|
|
18170
|
+
provider_exchange_id: capability.providerExchangeId,
|
|
18171
|
+
resolved_symbol: capability.resolvedSymbol,
|
|
18172
|
+
adapter_version: capability.adapterVersion,
|
|
18173
|
+
objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
|
|
18174
|
+
identity,
|
|
18175
|
+
checksum,
|
|
18176
|
+
bytes,
|
|
18177
|
+
rows
|
|
18178
|
+
})),
|
|
18179
|
+
canonical_schema_version: request2.expectedProduct.canonicalSchemaVersion,
|
|
18180
|
+
checksum_algorithm: request2.expectedProduct.checksumAlgorithm
|
|
18181
|
+
});
|
|
18182
|
+
}
|
|
18183
|
+
function buildPromotionReceipt(request2, capability, normalized, verification, verificationTimeMs) {
|
|
18184
|
+
if (!request2.wire || !request2.initialSelection || !request2.expectedCanonicalSchema || !request2.coveragePolicy || !request2.productPins) {
|
|
18185
|
+
throw new Error("decoded final-v1 request context is missing");
|
|
18186
|
+
}
|
|
18187
|
+
return finalizePromotionReceipt({
|
|
18188
|
+
schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
|
|
18189
|
+
verified_at: new Date(verificationTimeMs).toISOString(),
|
|
18190
|
+
request_id: request2.requestId,
|
|
18191
|
+
idempotency_key: request2.idempotencyKey,
|
|
18192
|
+
source: EXTERNAL_BACKFILL_SOURCE,
|
|
18193
|
+
capture_origin: "vendor_historical_backfill",
|
|
18194
|
+
source_mode: "vendor_historical_backfill_v1",
|
|
18195
|
+
provider: capability.provider,
|
|
18196
|
+
adapter_version: capability.adapterVersion,
|
|
18197
|
+
effective_policies: {
|
|
18198
|
+
capability_policy: request2.productPins.capability_policy,
|
|
18199
|
+
resource_policy: request2.productPins.resource_policy,
|
|
18200
|
+
adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
|
|
18201
|
+
acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
|
|
18202
|
+
},
|
|
18203
|
+
capture_bundle_id: normalized.captureBundleId,
|
|
18204
|
+
scope: request2.wire.scope,
|
|
18205
|
+
window: request2.wire.window,
|
|
18206
|
+
depth: request2.depth,
|
|
18207
|
+
construction_mode: request2.constructionMode,
|
|
18208
|
+
canonical_schema: request2.expectedCanonicalSchema,
|
|
18209
|
+
coverage_policy: request2.coveragePolicy,
|
|
18210
|
+
selection_sha256: request2.initialSelection.selection_sha256,
|
|
18211
|
+
vendor_semantic_digest: normalized.vendorSemanticDigest,
|
|
18212
|
+
canonical_semantic_digest: verification.canonicalSemanticDigest,
|
|
18213
|
+
prefix_digest: verification.prefixDigest,
|
|
18214
|
+
suffix_digest: verification.suffixDigest,
|
|
18215
|
+
seam_verified: true,
|
|
18216
|
+
coverage_verified: true,
|
|
18217
|
+
dataset_objects: normalized.objects
|
|
18218
|
+
});
|
|
18219
|
+
}
|
|
18220
|
+
async function submitAll(dependencies, batches) {
|
|
18221
|
+
const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
|
|
18222
|
+
for (const batch of batches) {
|
|
18223
|
+
let accepted = false;
|
|
18224
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
|
|
18225
|
+
try {
|
|
18226
|
+
const response = await dependencies.forwarder.submit(batch);
|
|
18227
|
+
accepted = response.ok && response.inserted === batch.rows.length;
|
|
18228
|
+
} catch {
|
|
18229
|
+
accepted = false;
|
|
18230
|
+
}
|
|
18231
|
+
if (accepted)
|
|
18424
18232
|
break;
|
|
18425
|
-
|
|
18233
|
+
if (attempt < maxAttempts)
|
|
18234
|
+
await dependencies.retry?.wait?.(attempt);
|
|
18426
18235
|
}
|
|
18427
|
-
if (!
|
|
18428
|
-
|
|
18236
|
+
if (!accepted)
|
|
18237
|
+
return false;
|
|
18238
|
+
}
|
|
18239
|
+
return true;
|
|
18240
|
+
}
|
|
18241
|
+
function stableFailureReason(error, fallback) {
|
|
18242
|
+
if (error && typeof error === "object") {
|
|
18243
|
+
const reason = error.reason;
|
|
18244
|
+
if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
|
|
18245
|
+
return reason;
|
|
18429
18246
|
}
|
|
18430
|
-
|
|
18431
|
-
|
|
18432
|
-
|
|
18433
|
-
|
|
18247
|
+
}
|
|
18248
|
+
return fallback;
|
|
18249
|
+
}
|
|
18250
|
+
function assertArchivePreflight(request2, resolution) {
|
|
18251
|
+
if (!request2.target || !request2.productionAuthorizationId) {
|
|
18252
|
+
throw new Error("request target or production authorization ID is missing");
|
|
18253
|
+
}
|
|
18254
|
+
const selection = archiveSelectionCodec.decode(resolution.selection);
|
|
18255
|
+
const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
|
|
18256
|
+
const receiptById = new Map;
|
|
18257
|
+
for (const receipt of receipts) {
|
|
18258
|
+
const existing = receiptById.get(receipt.receipt_id);
|
|
18259
|
+
if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
|
|
18260
|
+
throw new Error("stored receipt identity has conflicting content");
|
|
18434
18261
|
}
|
|
18435
|
-
|
|
18436
|
-
|
|
18262
|
+
receiptById.set(receipt.receipt_id, receipt);
|
|
18263
|
+
}
|
|
18264
|
+
for (const bundle of selection.bundles) {
|
|
18265
|
+
if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
|
|
18266
|
+
throw new Error("vendor selection lacks its validated stored receipt");
|
|
18437
18267
|
}
|
|
18438
|
-
samples.push({
|
|
18439
|
-
targetTimeMs,
|
|
18440
|
-
sourceTimeMs: prior.sourceTimeMs,
|
|
18441
|
-
receivedTimeMs: prior.receivedTimeMs,
|
|
18442
|
-
sequence: prior.sequence,
|
|
18443
|
-
bids,
|
|
18444
|
-
asks,
|
|
18445
|
-
datasetObjectIdentity: prior.datasetObjectIdentity,
|
|
18446
|
-
datasetObjectChecksum: prior.datasetObjectChecksum
|
|
18447
|
-
});
|
|
18448
18268
|
}
|
|
18449
|
-
|
|
18269
|
+
if (resolution.readerIdentity.environment !== request2.target.environment || resolution.readerIdentity.cluster !== request2.target.cluster) {
|
|
18270
|
+
throw new Error("archive reader cluster identity mismatch");
|
|
18271
|
+
}
|
|
18450
18272
|
}
|
|
18451
|
-
function
|
|
18452
|
-
|
|
18273
|
+
function assertForwarderPreflight(request2, resolution, nowMs) {
|
|
18274
|
+
if (!request2.target || !request2.productionAuthorizationId) {
|
|
18275
|
+
throw new Error("request target or production authorization ID is missing");
|
|
18276
|
+
}
|
|
18277
|
+
if (resolution.forwarderIdentity.environment !== request2.target.environment || resolution.forwarderIdentity.cluster !== request2.target.cluster) {
|
|
18278
|
+
throw new Error("archive forwarder cluster identity mismatch");
|
|
18279
|
+
}
|
|
18280
|
+
const authorization = resolution.authorization;
|
|
18281
|
+
const expiresAtMs = Date.parse(authorization.expiresAt);
|
|
18282
|
+
if (authorization.authorizationId !== request2.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request2.target.environment || authorization.cluster !== request2.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
|
|
18283
|
+
throw new Error("production forwarder authorization is invalid");
|
|
18284
|
+
}
|
|
18453
18285
|
}
|
|
18454
|
-
|
|
18455
|
-
const
|
|
18456
|
-
|
|
18457
|
-
|
|
18286
|
+
function resourcePolicyScopeExceeded(request2) {
|
|
18287
|
+
const policy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((candidate) => request2.productPins?.resource_policy.policy_id === candidate.policy_id && request2.productPins.resource_policy.policy_sha256 === candidate.policy_sha256);
|
|
18288
|
+
if (!policy)
|
|
18289
|
+
return true;
|
|
18290
|
+
return request2.depth > policy.request_bounds.max_depth || request2.window.endTimeMs - request2.window.startTimeMs > policy.request_bounds.max_window_ms || request2.requiredClockTargetsMs.length > policy.request_bounds.max_required_events;
|
|
18458
18291
|
}
|
|
18459
|
-
function
|
|
18460
|
-
|
|
18461
|
-
|
|
18462
|
-
event_time: value.event_time,
|
|
18463
|
-
transaction_time: value.transaction_time,
|
|
18464
|
-
symbol: value.symbol,
|
|
18465
|
-
event_type: value.event_type,
|
|
18466
|
-
first_update_id: value.first_update_id,
|
|
18467
|
-
final_update_id: value.final_update_id,
|
|
18468
|
-
prev_final_update_id: value.prev_final_update_id,
|
|
18469
|
-
last_update_id: value.last_update_id,
|
|
18470
|
-
side: value.side,
|
|
18471
|
-
price: value.price,
|
|
18472
|
-
quantity: value.quantity,
|
|
18473
|
-
order_count: value.order_count,
|
|
18474
|
-
dataset_object_identity: object.identity,
|
|
18475
|
-
dataset_object_checksum: object.checksum
|
|
18476
|
-
};
|
|
18292
|
+
function storedReceiptForSelection(resolution) {
|
|
18293
|
+
const receiptId = resolution.selection.receipt_ids[0];
|
|
18294
|
+
return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
|
|
18477
18295
|
}
|
|
18478
|
-
async function
|
|
18479
|
-
|
|
18480
|
-
|
|
18481
|
-
|
|
18482
|
-
|
|
18483
|
-
|
|
18484
|
-
|
|
18485
|
-
|
|
18486
|
-
|
|
18487
|
-
|
|
18488
|
-
total += value.byteLength;
|
|
18489
|
-
if (total > remainingBytes) {
|
|
18490
|
-
await reader.cancel();
|
|
18491
|
-
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
18492
|
-
}
|
|
18493
|
-
chunks.push(value);
|
|
18296
|
+
async function runMarketDataVendorBackfill(input, dependencies) {
|
|
18297
|
+
let request2;
|
|
18298
|
+
try {
|
|
18299
|
+
const documents = input;
|
|
18300
|
+
request2 = decodeBackfillRunDocuments({
|
|
18301
|
+
request: documents.request,
|
|
18302
|
+
requiredClock: documents.requiredClock
|
|
18303
|
+
});
|
|
18304
|
+
} catch {
|
|
18305
|
+
return outcome(undefined, "request_invalid", "request_invalid");
|
|
18494
18306
|
}
|
|
18495
|
-
|
|
18496
|
-
|
|
18497
|
-
|
|
18498
|
-
|
|
18499
|
-
|
|
18307
|
+
let initialResolution;
|
|
18308
|
+
try {
|
|
18309
|
+
initialResolution = await dependencies.archive.resolveSelection(request2);
|
|
18310
|
+
assertArchivePreflight(request2, initialResolution);
|
|
18311
|
+
const forwarderPreflight = await dependencies.forwarder.preflight({
|
|
18312
|
+
authorizationId: request2.productionAuthorizationId,
|
|
18313
|
+
target: request2.target
|
|
18314
|
+
});
|
|
18315
|
+
assertForwarderPreflight(request2, forwarderPreflight, dependencies.clock.nowMs());
|
|
18316
|
+
} catch {
|
|
18317
|
+
return outcome(request2, "archive_preflight_failed", "archive_preflight_failed");
|
|
18500
18318
|
}
|
|
18501
|
-
|
|
18502
|
-
|
|
18503
|
-
|
|
18504
|
-
|
|
18505
|
-
|
|
18506
|
-
|
|
18507
|
-
|
|
18508
|
-
|
|
18509
|
-
|
|
18510
|
-
|
|
18511
|
-
|
|
18512
|
-
|
|
18513
|
-
|
|
18514
|
-
|
|
18515
|
-
|
|
18319
|
+
if (initialResolution.selection.coverage_class === "complete") {
|
|
18320
|
+
const receipt2 = storedReceiptForSelection(initialResolution);
|
|
18321
|
+
return outcome(request2, "already_covered", "qualified_coverage_complete", {
|
|
18322
|
+
selection: initialResolution.selection,
|
|
18323
|
+
...receipt2 ? { receipt: receipt2 } : {}
|
|
18324
|
+
});
|
|
18325
|
+
}
|
|
18326
|
+
if (resourcePolicyScopeExceeded(request2)) {
|
|
18327
|
+
return outcome(request2, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
|
|
18328
|
+
}
|
|
18329
|
+
if (request2.sourcePolicy === "fill_gaps" && (request2.productPins?.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request2.productPins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256)) {
|
|
18330
|
+
return outcome(request2, "capability_unsupported", "scope_unsupported", {
|
|
18331
|
+
reasonSubcode: "fill_gaps_requires_capability_policy_v2"
|
|
18332
|
+
});
|
|
18333
|
+
}
|
|
18334
|
+
let capability;
|
|
18335
|
+
try {
|
|
18336
|
+
capability = dependencies.providers.capabilityFor(request2);
|
|
18337
|
+
} catch {
|
|
18338
|
+
return outcome(request2, "capability_unsupported", "capability_probe_failed");
|
|
18339
|
+
}
|
|
18340
|
+
if (!capability) {
|
|
18341
|
+
return outcome(request2, "capability_unsupported", "scope_unsupported");
|
|
18342
|
+
}
|
|
18343
|
+
if (!request2.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
|
|
18344
|
+
return outcome(request2, "capability_unsupported", "adapter_version_unpinned");
|
|
18516
18345
|
}
|
|
18517
|
-
|
|
18518
|
-
|
|
18346
|
+
let credential;
|
|
18347
|
+
try {
|
|
18348
|
+
credential = await dependencies.credentials.resolve(capability.provider);
|
|
18349
|
+
} catch {
|
|
18350
|
+
return outcome(request2, "credentials_missing", "credential_resolution_failed");
|
|
18519
18351
|
}
|
|
18520
|
-
|
|
18521
|
-
|
|
18522
|
-
throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
|
|
18523
|
-
}
|
|
18524
|
-
const endpoint = new URL("/symbols", this.baseUrl);
|
|
18525
|
-
endpoint.searchParams.set("exchange", providerExchangeId);
|
|
18526
|
-
endpoint.searchParams.set("data_type", "orderbook");
|
|
18527
|
-
let response;
|
|
18528
|
-
try {
|
|
18529
|
-
response = await this.request(endpoint);
|
|
18530
|
-
} catch {
|
|
18531
|
-
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
18532
|
-
}
|
|
18533
|
-
if (!response.ok) {
|
|
18534
|
-
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
18535
|
-
}
|
|
18536
|
-
let body;
|
|
18537
|
-
try {
|
|
18538
|
-
body = await response.json();
|
|
18539
|
-
} catch {
|
|
18540
|
-
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
18541
|
-
}
|
|
18542
|
-
const record = body;
|
|
18543
|
-
if (!body || typeof body !== "object" || Array.isArray(body) || String(record.exchange).toLowerCase() !== providerExchangeId || String(record.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record.symbols) || !record.symbols.every((symbol) => typeof symbol === "string" && /^[A-Z0-9_-]+$/.test(symbol))) {
|
|
18544
|
-
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
18545
|
-
}
|
|
18546
|
-
return [...new Set(record.symbols)].sort();
|
|
18352
|
+
if (credential === undefined || credential === null) {
|
|
18353
|
+
return outcome(request2, "credentials_missing", "provider_credentials_missing");
|
|
18547
18354
|
}
|
|
18548
|
-
|
|
18549
|
-
|
|
18550
|
-
|
|
18551
|
-
|
|
18552
|
-
|
|
18553
|
-
const
|
|
18554
|
-
|
|
18555
|
-
|
|
18355
|
+
let dataset;
|
|
18356
|
+
let normalized;
|
|
18357
|
+
try {
|
|
18358
|
+
const acquisitionRequest = providerAcquisitionRequest(request2);
|
|
18359
|
+
dataset = await dependencies.providers.acquire(acquisitionRequest, capability, credential);
|
|
18360
|
+
const bundleId = captureBundleId2(request2, capability, dataset);
|
|
18361
|
+
normalized = await dependencies.providers.normalize(acquisitionRequest, capability, dataset, bundleId);
|
|
18362
|
+
if (normalized.captureBundleId !== bundleId) {
|
|
18363
|
+
return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
18364
|
+
reasonSubcode: "capture_identity_mismatch"
|
|
18365
|
+
});
|
|
18556
18366
|
}
|
|
18557
|
-
|
|
18558
|
-
const
|
|
18559
|
-
|
|
18560
|
-
|
|
18367
|
+
} catch (error) {
|
|
18368
|
+
const reason = stableFailureReason(error, "provider_dataset_invalid");
|
|
18369
|
+
return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
18370
|
+
reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
|
|
18561
18371
|
});
|
|
18562
|
-
|
|
18563
|
-
|
|
18564
|
-
const
|
|
18565
|
-
|
|
18566
|
-
|
|
18372
|
+
}
|
|
18373
|
+
try {
|
|
18374
|
+
const candidateBatches = buildForwarderBatches({
|
|
18375
|
+
captureBundleId: normalized.captureBundleId,
|
|
18376
|
+
deploymentId: "market-data-vendor-backfill",
|
|
18377
|
+
rows: normalized.rows
|
|
18378
|
+
});
|
|
18379
|
+
if (!await submitAll(dependencies, candidateBatches)) {
|
|
18380
|
+
return outcome(request2, "archive_ingest_failed", "candidate_batch_rejected");
|
|
18567
18381
|
}
|
|
18568
|
-
|
|
18569
|
-
|
|
18570
|
-
|
|
18571
|
-
|
|
18572
|
-
|
|
18573
|
-
|
|
18574
|
-
|
|
18575
|
-
|
|
18576
|
-
|
|
18577
|
-
|
|
18578
|
-
|
|
18579
|
-
|
|
18580
|
-
|
|
18581
|
-
|
|
18582
|
-
|
|
18583
|
-
|
|
18584
|
-
|
|
18585
|
-
|
|
18586
|
-
|
|
18587
|
-
|
|
18588
|
-
|
|
18589
|
-
|
|
18590
|
-
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
18591
|
-
}
|
|
18592
|
-
const decoded = await this.decode(bytes);
|
|
18593
|
-
totalRows += decoded.length;
|
|
18594
|
-
if (totalRows > request2.budgets.maxRows) {
|
|
18595
|
-
throw new CryptoHftDataError("budget_max_rows_exceeded");
|
|
18596
|
-
}
|
|
18597
|
-
const object = {
|
|
18598
|
-
identity: path,
|
|
18599
|
-
checksum: sha256Bytes(bytes),
|
|
18600
|
-
bytes: bytes.byteLength,
|
|
18601
|
-
rows: decoded.length
|
|
18602
|
-
};
|
|
18603
|
-
objects.push(object);
|
|
18604
|
-
for (const decodedRow of decoded) {
|
|
18605
|
-
const parsed = parsedDatasetRow(decodedRow, object);
|
|
18606
|
-
validatedRow(request2, parsed);
|
|
18607
|
-
rows.push(parsed);
|
|
18608
|
-
}
|
|
18382
|
+
} catch {
|
|
18383
|
+
return outcome(request2, "archive_ingest_failed", "candidate_submission_failed");
|
|
18384
|
+
}
|
|
18385
|
+
let verification;
|
|
18386
|
+
try {
|
|
18387
|
+
verification = await dependencies.archive.verifyCandidate(request2, normalized, normalized.captureBundleId, initialResolution);
|
|
18388
|
+
} catch {
|
|
18389
|
+
return outcome(request2, "promotion_verification_failed", "candidate_query_failed");
|
|
18390
|
+
}
|
|
18391
|
+
if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
|
|
18392
|
+
return outcome(request2, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
|
|
18393
|
+
}
|
|
18394
|
+
let receipt;
|
|
18395
|
+
try {
|
|
18396
|
+
receipt = buildPromotionReceipt(request2, capability, normalized, verification, dependencies.clock.nowMs());
|
|
18397
|
+
const [batch] = buildForwarderBatches({
|
|
18398
|
+
captureBundleId: normalized.captureBundleId,
|
|
18399
|
+
deploymentId: "market-data-vendor-backfill",
|
|
18400
|
+
rows: [promotionReceiptToArchiveRow(receipt)]
|
|
18401
|
+
});
|
|
18402
|
+
if (!batch || !await submitAll(dependencies, [batch])) {
|
|
18403
|
+
return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
|
|
18609
18404
|
}
|
|
18610
|
-
|
|
18611
|
-
|
|
18405
|
+
const qualification = finalizeQualificationEvent({
|
|
18406
|
+
capture_bundle_id: receipt.capture_bundle_id,
|
|
18407
|
+
state: "qualified",
|
|
18408
|
+
receipt_id: receipt.receipt_id,
|
|
18409
|
+
promotion_identity_sha256: receipt.promotion_identity_sha256,
|
|
18410
|
+
window: receipt.window,
|
|
18411
|
+
event_at: receipt.verified_at,
|
|
18412
|
+
reason_code: "promotion_verified"
|
|
18413
|
+
});
|
|
18414
|
+
const [qualificationBatch] = buildForwarderBatches({
|
|
18415
|
+
captureBundleId: normalized.captureBundleId,
|
|
18416
|
+
deploymentId: "market-data-vendor-backfill",
|
|
18417
|
+
rows: [qualificationEventToArchiveRow(qualification)]
|
|
18418
|
+
});
|
|
18419
|
+
if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
|
|
18420
|
+
return outcome(request2, "archive_ingest_failed", "qualification_commit_failed", { receipt });
|
|
18612
18421
|
}
|
|
18613
|
-
|
|
18614
|
-
|
|
18615
|
-
rows,
|
|
18616
|
-
vendorSemanticDigest: sha256Canonical(rows)
|
|
18617
|
-
};
|
|
18422
|
+
} catch {
|
|
18423
|
+
return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
|
|
18618
18424
|
}
|
|
18619
|
-
|
|
18620
|
-
|
|
18621
|
-
|
|
18622
|
-
|
|
18623
|
-
|
|
18624
|
-
|
|
18625
|
-
|
|
18626
|
-
|
|
18425
|
+
let finalResolution;
|
|
18426
|
+
try {
|
|
18427
|
+
finalResolution = await dependencies.archive.resolveSelection(request2);
|
|
18428
|
+
assertArchivePreflight(request2, finalResolution);
|
|
18429
|
+
} catch (error) {
|
|
18430
|
+
return outcome(request2, "promotion_verification_failed", "post_promotion_selection_failed", {
|
|
18431
|
+
receipt,
|
|
18432
|
+
reasonSubcode: stableFailureReason(error, "archive_selection_resolution_failed")
|
|
18433
|
+
});
|
|
18434
|
+
}
|
|
18435
|
+
if (finalResolution.selection.coverage_class !== "complete") {
|
|
18436
|
+
return outcome(request2, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
|
|
18437
|
+
}
|
|
18438
|
+
if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
|
|
18439
|
+
return outcome(request2, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
|
|
18440
|
+
}
|
|
18441
|
+
try {
|
|
18442
|
+
if (!request2.wire)
|
|
18443
|
+
throw new Error("decoded request wire is missing");
|
|
18444
|
+
const [selectionBatch] = buildForwarderBatches({
|
|
18445
|
+
captureBundleId: receipt.capture_bundle_id,
|
|
18627
18446
|
deploymentId: "market-data-vendor-backfill",
|
|
18628
|
-
|
|
18629
|
-
|
|
18630
|
-
|
|
18631
|
-
tradingPair: request2.scope.tradingPair,
|
|
18632
|
-
sourceSymbol: request2.scope.sourceSymbol,
|
|
18633
|
-
assetType: request2.scope.marketType,
|
|
18634
|
-
feed: "ORDERBOOK",
|
|
18635
|
-
provider: "cryptohftdata",
|
|
18636
|
-
sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
|
|
18637
|
-
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
18638
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
18639
|
-
provenanceComplete: true
|
|
18640
|
-
};
|
|
18641
|
-
const rows = samples.flatMap((sample) => {
|
|
18642
|
-
const rawCapture = {
|
|
18643
|
-
rawCaptureId: sha256Canonical({
|
|
18644
|
-
capture_bundle_id: captureBundleId3,
|
|
18645
|
-
dataset_object_identity: sample.datasetObjectIdentity,
|
|
18646
|
-
dataset_object_checksum: sample.datasetObjectChecksum,
|
|
18647
|
-
source_time_ms: sample.sourceTimeMs,
|
|
18648
|
-
target_time_ms: sample.targetTimeMs
|
|
18649
|
-
}),
|
|
18650
|
-
rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
|
|
18651
|
-
rawChecksum: sample.datasetObjectChecksum,
|
|
18652
|
-
redactedPayload: {
|
|
18653
|
-
dataset_object_identity: sample.datasetObjectIdentity,
|
|
18654
|
-
dataset_object_checksum: sample.datasetObjectChecksum
|
|
18655
|
-
},
|
|
18656
|
-
eventTimeMs: sample.sourceTimeMs,
|
|
18657
|
-
receivedTimeMs: sample.receivedTimeMs,
|
|
18658
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM
|
|
18659
|
-
};
|
|
18660
|
-
const canonical = buildCanonicalOrderBookRows({
|
|
18661
|
-
context: context2,
|
|
18662
|
-
rawCapture,
|
|
18663
|
-
depthLimit: request2.depth,
|
|
18664
|
-
constructionMode: request2.constructionMode,
|
|
18665
|
-
snapshot: {
|
|
18666
|
-
bids: sample.bids,
|
|
18667
|
-
asks: sample.asks,
|
|
18668
|
-
timestamp: sample.sourceTimeMs,
|
|
18669
|
-
receivedTimestamp: sample.receivedTimeMs,
|
|
18670
|
-
exchange: request2.scope.exchange,
|
|
18671
|
-
symbol: request2.scope.tradingPair,
|
|
18672
|
-
depthLimit: request2.depth,
|
|
18673
|
-
sequence: sample.sequence
|
|
18674
|
-
}
|
|
18675
|
-
});
|
|
18676
|
-
return [...canonical.levels, canonical.summary];
|
|
18447
|
+
rows: [
|
|
18448
|
+
archiveSelectionToArchiveRow(request2.wire, finalResolution.selection)
|
|
18449
|
+
]
|
|
18677
18450
|
});
|
|
18678
|
-
|
|
18679
|
-
|
|
18680
|
-
|
|
18681
|
-
|
|
18682
|
-
|
|
18683
|
-
canonicalSemanticDigest: semanticDigest(rows)
|
|
18684
|
-
};
|
|
18451
|
+
if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
|
|
18452
|
+
return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
18453
|
+
}
|
|
18454
|
+
} catch {
|
|
18455
|
+
return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
18685
18456
|
}
|
|
18457
|
+
return outcome(request2, "promoted", "promotion_qualified", {
|
|
18458
|
+
receipt,
|
|
18459
|
+
selection: finalResolution.selection
|
|
18460
|
+
});
|
|
18686
18461
|
}
|
|
18687
18462
|
// src/helpers/market-data-vendor-backfill/forwarder-client.ts
|
|
18688
18463
|
class ArchiveForwarderSubmissionError extends Error {
|
|
@@ -18825,12 +18600,15 @@ export {
|
|
|
18825
18600
|
REQUIRED_CLOCK_SCHEMA_ID,
|
|
18826
18601
|
QualifiedOrderBookArchiveReader,
|
|
18827
18602
|
PROMOTION_RECEIPT_SCHEMA_ID,
|
|
18603
|
+
LEGACY_RESOURCE_POLICY_ID,
|
|
18604
|
+
LEGACY_RESOURCE_POLICY,
|
|
18828
18605
|
FINAL_BACKFILL_STATUSES,
|
|
18829
18606
|
EFFECTIVE_ADAPTER_POLICY_PIN,
|
|
18830
18607
|
EFFECTIVE_ACQUISITION_POLICY_PIN,
|
|
18831
18608
|
CryptoHftDataError,
|
|
18832
18609
|
CryptoHftDataAdapter,
|
|
18833
18610
|
CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE,
|
|
18611
|
+
CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE,
|
|
18834
18612
|
CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE,
|
|
18835
18613
|
CRYPTOHFTDATA_API_URL,
|
|
18836
18614
|
CRYPTOHFTDATA_ADAPTER_VERSION,
|
|
@@ -18847,4 +18625,4 @@ export {
|
|
|
18847
18625
|
ACQUISITION_POLICY_ID
|
|
18848
18626
|
};
|
|
18849
18627
|
|
|
18850
|
-
//# debugId=
|
|
18628
|
+
//# debugId=352683EF086D61EB64756E2164756E21
|