@usherlabs/cex-broker 0.2.48 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cex-canonical-orderbook-export.js +30 -350
- package/dist/commands/cex-canonical-orderbook-export.js.map +7 -19
- package/dist/commands/cli.js +100 -35
- package/dist/commands/market-data-vendor-backfill.js +1463 -1742
- package/dist/commands/market-data-vendor-backfill.js.map +12 -24
- package/dist/helpers/logger.d.ts +4 -0
- package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +2 -1
- package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +29 -2
- package/dist/helpers/market-data-vendor-backfill/semantic-verification.d.ts +1 -0
- package/dist/helpers/otel.d.ts +1 -0
- package/dist/index.js +59 -31
- package/dist/index.js.map +5 -5
- package/dist/market-data-preparation/fixtures/conformance-v2.json +2 -2
- package/dist/market-data-preparation/policies/capability-policy-v1.json +28 -0
- package/dist/market-data-preparation/policies/capability-policy.json +3 -3
- package/dist/market-data-preparation.js +30 -351
- package/dist/market-data-preparation.js.map +9 -21
- package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +28 -0
- package/dist/market-data-vendor-backfill.js +1028 -1306
- package/dist/market-data-vendor-backfill.js.map +13 -25
- package/package.json +1 -1
|
@@ -1431,241 +1431,6 @@ var require_src = __commonJS((exports) => {
|
|
|
1431
1431
|
};
|
|
1432
1432
|
});
|
|
1433
1433
|
|
|
1434
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/types/LogRecord.js
|
|
1435
|
-
var require_LogRecord = __commonJS((exports) => {
|
|
1436
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1437
|
-
exports.SeverityNumber = undefined;
|
|
1438
|
-
var SeverityNumber;
|
|
1439
|
-
(function(SeverityNumber2) {
|
|
1440
|
-
SeverityNumber2[SeverityNumber2["UNSPECIFIED"] = 0] = "UNSPECIFIED";
|
|
1441
|
-
SeverityNumber2[SeverityNumber2["TRACE"] = 1] = "TRACE";
|
|
1442
|
-
SeverityNumber2[SeverityNumber2["TRACE2"] = 2] = "TRACE2";
|
|
1443
|
-
SeverityNumber2[SeverityNumber2["TRACE3"] = 3] = "TRACE3";
|
|
1444
|
-
SeverityNumber2[SeverityNumber2["TRACE4"] = 4] = "TRACE4";
|
|
1445
|
-
SeverityNumber2[SeverityNumber2["DEBUG"] = 5] = "DEBUG";
|
|
1446
|
-
SeverityNumber2[SeverityNumber2["DEBUG2"] = 6] = "DEBUG2";
|
|
1447
|
-
SeverityNumber2[SeverityNumber2["DEBUG3"] = 7] = "DEBUG3";
|
|
1448
|
-
SeverityNumber2[SeverityNumber2["DEBUG4"] = 8] = "DEBUG4";
|
|
1449
|
-
SeverityNumber2[SeverityNumber2["INFO"] = 9] = "INFO";
|
|
1450
|
-
SeverityNumber2[SeverityNumber2["INFO2"] = 10] = "INFO2";
|
|
1451
|
-
SeverityNumber2[SeverityNumber2["INFO3"] = 11] = "INFO3";
|
|
1452
|
-
SeverityNumber2[SeverityNumber2["INFO4"] = 12] = "INFO4";
|
|
1453
|
-
SeverityNumber2[SeverityNumber2["WARN"] = 13] = "WARN";
|
|
1454
|
-
SeverityNumber2[SeverityNumber2["WARN2"] = 14] = "WARN2";
|
|
1455
|
-
SeverityNumber2[SeverityNumber2["WARN3"] = 15] = "WARN3";
|
|
1456
|
-
SeverityNumber2[SeverityNumber2["WARN4"] = 16] = "WARN4";
|
|
1457
|
-
SeverityNumber2[SeverityNumber2["ERROR"] = 17] = "ERROR";
|
|
1458
|
-
SeverityNumber2[SeverityNumber2["ERROR2"] = 18] = "ERROR2";
|
|
1459
|
-
SeverityNumber2[SeverityNumber2["ERROR3"] = 19] = "ERROR3";
|
|
1460
|
-
SeverityNumber2[SeverityNumber2["ERROR4"] = 20] = "ERROR4";
|
|
1461
|
-
SeverityNumber2[SeverityNumber2["FATAL"] = 21] = "FATAL";
|
|
1462
|
-
SeverityNumber2[SeverityNumber2["FATAL2"] = 22] = "FATAL2";
|
|
1463
|
-
SeverityNumber2[SeverityNumber2["FATAL3"] = 23] = "FATAL3";
|
|
1464
|
-
SeverityNumber2[SeverityNumber2["FATAL4"] = 24] = "FATAL4";
|
|
1465
|
-
})(SeverityNumber = exports.SeverityNumber || (exports.SeverityNumber = {}));
|
|
1466
|
-
});
|
|
1467
|
-
|
|
1468
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLogger.js
|
|
1469
|
-
var require_NoopLogger = __commonJS((exports) => {
|
|
1470
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1471
|
-
exports.NOOP_LOGGER = exports.NoopLogger = undefined;
|
|
1472
|
-
|
|
1473
|
-
class NoopLogger {
|
|
1474
|
-
emit(_logRecord) {}
|
|
1475
|
-
}
|
|
1476
|
-
exports.NoopLogger = NoopLogger;
|
|
1477
|
-
exports.NOOP_LOGGER = new NoopLogger;
|
|
1478
|
-
});
|
|
1479
|
-
|
|
1480
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLoggerProvider.js
|
|
1481
|
-
var require_NoopLoggerProvider = __commonJS((exports) => {
|
|
1482
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1483
|
-
exports.NOOP_LOGGER_PROVIDER = exports.NoopLoggerProvider = undefined;
|
|
1484
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
1485
|
-
|
|
1486
|
-
class NoopLoggerProvider {
|
|
1487
|
-
getLogger(_name, _version, _options) {
|
|
1488
|
-
return new NoopLogger_1.NoopLogger;
|
|
1489
|
-
}
|
|
1490
|
-
}
|
|
1491
|
-
exports.NoopLoggerProvider = NoopLoggerProvider;
|
|
1492
|
-
exports.NOOP_LOGGER_PROVIDER = new NoopLoggerProvider;
|
|
1493
|
-
});
|
|
1494
|
-
|
|
1495
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLogger.js
|
|
1496
|
-
var require_ProxyLogger = __commonJS((exports) => {
|
|
1497
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1498
|
-
exports.ProxyLogger = undefined;
|
|
1499
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
1500
|
-
|
|
1501
|
-
class ProxyLogger {
|
|
1502
|
-
constructor(_provider, name, version, options) {
|
|
1503
|
-
this._provider = _provider;
|
|
1504
|
-
this.name = name;
|
|
1505
|
-
this.version = version;
|
|
1506
|
-
this.options = options;
|
|
1507
|
-
}
|
|
1508
|
-
emit(logRecord) {
|
|
1509
|
-
this._getLogger().emit(logRecord);
|
|
1510
|
-
}
|
|
1511
|
-
_getLogger() {
|
|
1512
|
-
if (this._delegate) {
|
|
1513
|
-
return this._delegate;
|
|
1514
|
-
}
|
|
1515
|
-
const logger = this._provider._getDelegateLogger(this.name, this.version, this.options);
|
|
1516
|
-
if (!logger) {
|
|
1517
|
-
return NoopLogger_1.NOOP_LOGGER;
|
|
1518
|
-
}
|
|
1519
|
-
this._delegate = logger;
|
|
1520
|
-
return this._delegate;
|
|
1521
|
-
}
|
|
1522
|
-
}
|
|
1523
|
-
exports.ProxyLogger = ProxyLogger;
|
|
1524
|
-
});
|
|
1525
|
-
|
|
1526
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLoggerProvider.js
|
|
1527
|
-
var require_ProxyLoggerProvider = __commonJS((exports) => {
|
|
1528
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1529
|
-
exports.ProxyLoggerProvider = undefined;
|
|
1530
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
1531
|
-
var ProxyLogger_1 = require_ProxyLogger();
|
|
1532
|
-
|
|
1533
|
-
class ProxyLoggerProvider {
|
|
1534
|
-
getLogger(name, version, options) {
|
|
1535
|
-
var _a;
|
|
1536
|
-
return (_a = this._getDelegateLogger(name, version, options)) !== null && _a !== undefined ? _a : new ProxyLogger_1.ProxyLogger(this, name, version, options);
|
|
1537
|
-
}
|
|
1538
|
-
_getDelegate() {
|
|
1539
|
-
var _a;
|
|
1540
|
-
return (_a = this._delegate) !== null && _a !== undefined ? _a : NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
|
|
1541
|
-
}
|
|
1542
|
-
_setDelegate(delegate) {
|
|
1543
|
-
this._delegate = delegate;
|
|
1544
|
-
}
|
|
1545
|
-
_getDelegateLogger(name, version, options) {
|
|
1546
|
-
var _a;
|
|
1547
|
-
return (_a = this._delegate) === null || _a === undefined ? undefined : _a.getLogger(name, version, options);
|
|
1548
|
-
}
|
|
1549
|
-
}
|
|
1550
|
-
exports.ProxyLoggerProvider = ProxyLoggerProvider;
|
|
1551
|
-
});
|
|
1552
|
-
|
|
1553
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/globalThis.js
|
|
1554
|
-
var require_globalThis2 = __commonJS((exports) => {
|
|
1555
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1556
|
-
exports._globalThis = undefined;
|
|
1557
|
-
exports._globalThis = typeof globalThis === "object" ? globalThis : global;
|
|
1558
|
-
});
|
|
1559
|
-
|
|
1560
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/index.js
|
|
1561
|
-
var require_node2 = __commonJS((exports) => {
|
|
1562
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1563
|
-
exports._globalThis = undefined;
|
|
1564
|
-
var globalThis_1 = require_globalThis2();
|
|
1565
|
-
Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
|
|
1566
|
-
return globalThis_1._globalThis;
|
|
1567
|
-
} });
|
|
1568
|
-
});
|
|
1569
|
-
|
|
1570
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/index.js
|
|
1571
|
-
var require_platform2 = __commonJS((exports) => {
|
|
1572
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1573
|
-
exports._globalThis = undefined;
|
|
1574
|
-
var node_1 = require_node2();
|
|
1575
|
-
Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
|
|
1576
|
-
return node_1._globalThis;
|
|
1577
|
-
} });
|
|
1578
|
-
});
|
|
1579
|
-
|
|
1580
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/internal/global-utils.js
|
|
1581
|
-
var require_global_utils2 = __commonJS((exports) => {
|
|
1582
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1583
|
-
exports.API_BACKWARDS_COMPATIBILITY_VERSION = exports.makeGetter = exports._global = exports.GLOBAL_LOGS_API_KEY = undefined;
|
|
1584
|
-
var platform_1 = require_platform2();
|
|
1585
|
-
exports.GLOBAL_LOGS_API_KEY = Symbol.for("io.opentelemetry.js.api.logs");
|
|
1586
|
-
exports._global = platform_1._globalThis;
|
|
1587
|
-
function makeGetter(requiredVersion, instance, fallback) {
|
|
1588
|
-
return (version) => version === requiredVersion ? instance : fallback;
|
|
1589
|
-
}
|
|
1590
|
-
exports.makeGetter = makeGetter;
|
|
1591
|
-
exports.API_BACKWARDS_COMPATIBILITY_VERSION = 1;
|
|
1592
|
-
});
|
|
1593
|
-
|
|
1594
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/api/logs.js
|
|
1595
|
-
var require_logs = __commonJS((exports) => {
|
|
1596
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1597
|
-
exports.LogsAPI = undefined;
|
|
1598
|
-
var global_utils_1 = require_global_utils2();
|
|
1599
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
1600
|
-
var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
|
|
1601
|
-
|
|
1602
|
-
class LogsAPI {
|
|
1603
|
-
constructor() {
|
|
1604
|
-
this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
1605
|
-
}
|
|
1606
|
-
static getInstance() {
|
|
1607
|
-
if (!this._instance) {
|
|
1608
|
-
this._instance = new LogsAPI;
|
|
1609
|
-
}
|
|
1610
|
-
return this._instance;
|
|
1611
|
-
}
|
|
1612
|
-
setGlobalLoggerProvider(provider) {
|
|
1613
|
-
if (global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) {
|
|
1614
|
-
return this.getLoggerProvider();
|
|
1615
|
-
}
|
|
1616
|
-
global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY] = (0, global_utils_1.makeGetter)(global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION, provider, NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER);
|
|
1617
|
-
this._proxyLoggerProvider._setDelegate(provider);
|
|
1618
|
-
return provider;
|
|
1619
|
-
}
|
|
1620
|
-
getLoggerProvider() {
|
|
1621
|
-
var _a, _b;
|
|
1622
|
-
return (_b = (_a = global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) === null || _a === undefined ? undefined : _a.call(global_utils_1._global, global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION)) !== null && _b !== undefined ? _b : this._proxyLoggerProvider;
|
|
1623
|
-
}
|
|
1624
|
-
getLogger(name, version, options) {
|
|
1625
|
-
return this.getLoggerProvider().getLogger(name, version, options);
|
|
1626
|
-
}
|
|
1627
|
-
disable() {
|
|
1628
|
-
delete global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY];
|
|
1629
|
-
this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
1630
|
-
}
|
|
1631
|
-
}
|
|
1632
|
-
exports.LogsAPI = LogsAPI;
|
|
1633
|
-
});
|
|
1634
|
-
|
|
1635
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/index.js
|
|
1636
|
-
var require_src2 = __commonJS((exports) => {
|
|
1637
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1638
|
-
exports.logs = exports.ProxyLoggerProvider = exports.ProxyLogger = exports.NoopLoggerProvider = exports.NOOP_LOGGER_PROVIDER = exports.NoopLogger = exports.NOOP_LOGGER = exports.SeverityNumber = undefined;
|
|
1639
|
-
var LogRecord_1 = require_LogRecord();
|
|
1640
|
-
Object.defineProperty(exports, "SeverityNumber", { enumerable: true, get: function() {
|
|
1641
|
-
return LogRecord_1.SeverityNumber;
|
|
1642
|
-
} });
|
|
1643
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
1644
|
-
Object.defineProperty(exports, "NOOP_LOGGER", { enumerable: true, get: function() {
|
|
1645
|
-
return NoopLogger_1.NOOP_LOGGER;
|
|
1646
|
-
} });
|
|
1647
|
-
Object.defineProperty(exports, "NoopLogger", { enumerable: true, get: function() {
|
|
1648
|
-
return NoopLogger_1.NoopLogger;
|
|
1649
|
-
} });
|
|
1650
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
1651
|
-
Object.defineProperty(exports, "NOOP_LOGGER_PROVIDER", { enumerable: true, get: function() {
|
|
1652
|
-
return NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
|
|
1653
|
-
} });
|
|
1654
|
-
Object.defineProperty(exports, "NoopLoggerProvider", { enumerable: true, get: function() {
|
|
1655
|
-
return NoopLoggerProvider_1.NoopLoggerProvider;
|
|
1656
|
-
} });
|
|
1657
|
-
var ProxyLogger_1 = require_ProxyLogger();
|
|
1658
|
-
Object.defineProperty(exports, "ProxyLogger", { enumerable: true, get: function() {
|
|
1659
|
-
return ProxyLogger_1.ProxyLogger;
|
|
1660
|
-
} });
|
|
1661
|
-
var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
|
|
1662
|
-
Object.defineProperty(exports, "ProxyLoggerProvider", { enumerable: true, get: function() {
|
|
1663
|
-
return ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
1664
|
-
} });
|
|
1665
|
-
var logs_1 = require_logs();
|
|
1666
|
-
exports.logs = logs_1.LogsAPI.getInstance();
|
|
1667
|
-
});
|
|
1668
|
-
|
|
1669
1434
|
// node_modules/ajv/dist/compile/codegen/code.js
|
|
1670
1435
|
var require_code = __commonJS((exports) => {
|
|
1671
1436
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -8697,102 +8462,6 @@ var BaseTransport = class {
|
|
|
8697
8462
|
return this.logger;
|
|
8698
8463
|
}
|
|
8699
8464
|
};
|
|
8700
|
-
var LoggerlessTransport = class {
|
|
8701
|
-
id;
|
|
8702
|
-
enabled;
|
|
8703
|
-
level;
|
|
8704
|
-
consoleDebug;
|
|
8705
|
-
constructor(config) {
|
|
8706
|
-
this.id = Date.now().toString() + Math.random().toString();
|
|
8707
|
-
this.enabled = config.enabled ?? true;
|
|
8708
|
-
this.consoleDebug = config.consoleDebug ?? false;
|
|
8709
|
-
this.level = config.level ?? "trace";
|
|
8710
|
-
}
|
|
8711
|
-
_sendToLogger(params) {
|
|
8712
|
-
if (!this.enabled)
|
|
8713
|
-
return;
|
|
8714
|
-
if (LogLevelPriority[params.logLevel] < LogLevelPriority[this.level])
|
|
8715
|
-
return;
|
|
8716
|
-
const messages = this.shipToLogger(params);
|
|
8717
|
-
if (this.consoleDebug)
|
|
8718
|
-
switch (params.logLevel) {
|
|
8719
|
-
case LogLevel.info:
|
|
8720
|
-
console.info(...messages);
|
|
8721
|
-
break;
|
|
8722
|
-
case LogLevel.warn:
|
|
8723
|
-
console.warn(...messages);
|
|
8724
|
-
break;
|
|
8725
|
-
case LogLevel.error:
|
|
8726
|
-
console.error(...messages);
|
|
8727
|
-
break;
|
|
8728
|
-
case LogLevel.trace:
|
|
8729
|
-
console.debug(...messages);
|
|
8730
|
-
break;
|
|
8731
|
-
case LogLevel.debug:
|
|
8732
|
-
console.debug(...messages);
|
|
8733
|
-
break;
|
|
8734
|
-
case LogLevel.fatal:
|
|
8735
|
-
console.debug(...messages);
|
|
8736
|
-
break;
|
|
8737
|
-
default:
|
|
8738
|
-
console.log(...messages);
|
|
8739
|
-
}
|
|
8740
|
-
}
|
|
8741
|
-
getLoggerInstance() {
|
|
8742
|
-
throw new Error("This transport does not have a logger instance");
|
|
8743
|
-
}
|
|
8744
|
-
};
|
|
8745
|
-
|
|
8746
|
-
// node_modules/@loglayer/transport-opentelemetry/dist/index.mjs
|
|
8747
|
-
var import_api_logs = __toESM(require_src2(), 1);
|
|
8748
|
-
var logLayerLevels = {
|
|
8749
|
-
fatal: import_api_logs.SeverityNumber.FATAL,
|
|
8750
|
-
error: import_api_logs.SeverityNumber.ERROR,
|
|
8751
|
-
warn: import_api_logs.SeverityNumber.WARN,
|
|
8752
|
-
info: import_api_logs.SeverityNumber.INFO,
|
|
8753
|
-
debug: import_api_logs.SeverityNumber.DEBUG,
|
|
8754
|
-
trace: import_api_logs.SeverityNumber.TRACE
|
|
8755
|
-
};
|
|
8756
|
-
function getSeverityNumber(level) {
|
|
8757
|
-
return logLayerLevels[level] ?? import_api_logs.SeverityNumber.TRACE;
|
|
8758
|
-
}
|
|
8759
|
-
function emitLogRecord(record, logger) {
|
|
8760
|
-
const { message, level, ...splat } = record;
|
|
8761
|
-
const attributes = {};
|
|
8762
|
-
for (const key in splat)
|
|
8763
|
-
if (Object.hasOwn(splat, key))
|
|
8764
|
-
attributes[key] = splat[key];
|
|
8765
|
-
const logRecord = {
|
|
8766
|
-
severityNumber: getSeverityNumber(level),
|
|
8767
|
-
severityText: level,
|
|
8768
|
-
body: message,
|
|
8769
|
-
attributes
|
|
8770
|
-
};
|
|
8771
|
-
logger.emit(logRecord);
|
|
8772
|
-
}
|
|
8773
|
-
var OpenTelemetryTransport = class extends LoggerlessTransport {
|
|
8774
|
-
_logger;
|
|
8775
|
-
onError;
|
|
8776
|
-
constructor(config = {}) {
|
|
8777
|
-
super(config);
|
|
8778
|
-
this._logger = import_api_logs.logs.getLogger("loglayer");
|
|
8779
|
-
this.onError = config.onError;
|
|
8780
|
-
}
|
|
8781
|
-
shipToLogger({ logLevel, messages, data, hasData }) {
|
|
8782
|
-
const assembled = {
|
|
8783
|
-
level: logLevel,
|
|
8784
|
-
message: messages.join(" "),
|
|
8785
|
-
...hasData ? data : {}
|
|
8786
|
-
};
|
|
8787
|
-
try {
|
|
8788
|
-
emitLogRecord(assembled, this._logger);
|
|
8789
|
-
} catch (error) {
|
|
8790
|
-
if (this.onError)
|
|
8791
|
-
this.onError(error);
|
|
8792
|
-
}
|
|
8793
|
-
return [assembled];
|
|
8794
|
-
}
|
|
8795
|
-
};
|
|
8796
8465
|
|
|
8797
8466
|
// node_modules/@loglayer/transport-tslog/dist/index.js
|
|
8798
8467
|
var TsLogTransport = class extends BaseTransport {
|
|
@@ -10964,13 +10633,7 @@ var tslogLogger = new Logger({
|
|
|
10964
10633
|
minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
|
|
10965
10634
|
});
|
|
10966
10635
|
var baseLogger = new LogLayer({
|
|
10967
|
-
transport: [
|
|
10968
|
-
new TsLogTransport({ id: "tslog", logger: tslogLogger }),
|
|
10969
|
-
new OpenTelemetryTransport({
|
|
10970
|
-
id: "otel",
|
|
10971
|
-
enabled: true
|
|
10972
|
-
})
|
|
10973
|
-
],
|
|
10636
|
+
transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
|
|
10974
10637
|
plugins: [openTelemetryPlugin()],
|
|
10975
10638
|
errorSerializer: serializeError
|
|
10976
10639
|
});
|
|
@@ -12089,12 +11752,13 @@ var result_schema_default = {
|
|
|
12089
11752
|
};
|
|
12090
11753
|
|
|
12091
11754
|
// src/helpers/market-data-vendor-backfill/manifests.ts
|
|
12092
|
-
var
|
|
11755
|
+
var LEGACY_CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
|
|
11756
|
+
var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v2";
|
|
12093
11757
|
var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
|
|
12094
11758
|
var ADAPTER_POLICY_ID = "cryptohftdata-orderbook-adapter/v1";
|
|
12095
11759
|
var ACQUISITION_POLICY_ID = "cryptohftdata-hourly-acquisition/v1";
|
|
12096
|
-
var
|
|
12097
|
-
policy_id:
|
|
11760
|
+
var legacyCapabilityPolicyContent = {
|
|
11761
|
+
policy_id: LEGACY_CAPABILITY_POLICY_ID,
|
|
12098
11762
|
provider: "cryptohftdata",
|
|
12099
11763
|
adapter_policy: {
|
|
12100
11764
|
policy_id: ADAPTER_POLICY_ID,
|
|
@@ -12120,6 +11784,18 @@ var capabilityPolicyContent = {
|
|
|
12120
11784
|
}
|
|
12121
11785
|
]
|
|
12122
11786
|
};
|
|
11787
|
+
var LEGACY_CAPABILITY_POLICY = Object.freeze({
|
|
11788
|
+
...legacyCapabilityPolicyContent,
|
|
11789
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent)
|
|
11790
|
+
});
|
|
11791
|
+
var capabilityPolicyContent = {
|
|
11792
|
+
...legacyCapabilityPolicyContent,
|
|
11793
|
+
policy_id: CAPABILITY_POLICY_ID,
|
|
11794
|
+
profiles: legacyCapabilityPolicyContent.profiles.map((profile) => ({
|
|
11795
|
+
...profile,
|
|
11796
|
+
source_policies: ["authoritative_window", "fill_gaps"]
|
|
11797
|
+
}))
|
|
11798
|
+
};
|
|
12123
11799
|
var CAPABILITY_POLICY = Object.freeze({
|
|
12124
11800
|
...capabilityPolicyContent,
|
|
12125
11801
|
policy_sha256: jcsSha256(capabilityPolicyContent)
|
|
@@ -12145,11 +11821,11 @@ var RESOURCE_POLICY = Object.freeze({
|
|
|
12145
11821
|
});
|
|
12146
11822
|
var EFFECTIVE_ADAPTER_POLICY_PIN = Object.freeze({
|
|
12147
11823
|
policy_id: ADAPTER_POLICY_ID,
|
|
12148
|
-
policy_sha256: jcsSha256(
|
|
11824
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent.adapter_policy)
|
|
12149
11825
|
});
|
|
12150
11826
|
var EFFECTIVE_ACQUISITION_POLICY_PIN = Object.freeze({
|
|
12151
11827
|
policy_id: ACQUISITION_POLICY_ID,
|
|
12152
|
-
policy_sha256: jcsSha256(
|
|
11828
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent.acquisition_policy)
|
|
12153
11829
|
});
|
|
12154
11830
|
var schemaArtifacts = [
|
|
12155
11831
|
["schemas/request.schema.json", request_schema_default],
|
|
@@ -12329,7 +12005,11 @@ var backfillRequestCodec = {
|
|
|
12329
12005
|
if (request.idempotency_key !== createBackfillIdempotencyKey(request)) {
|
|
12330
12006
|
throw new Error("idempotency_key does not match canonical business fields");
|
|
12331
12007
|
}
|
|
12332
|
-
|
|
12008
|
+
const capabilityPolicyMatches = [
|
|
12009
|
+
CAPABILITY_POLICY,
|
|
12010
|
+
LEGACY_CAPABILITY_POLICY
|
|
12011
|
+
].some((policy) => request.product_pins.capability_policy.policy_id === policy.policy_id && request.product_pins.capability_policy.policy_sha256 === policy.policy_sha256);
|
|
12012
|
+
if (!capabilityPolicyMatches || request.product_pins.resource_policy.policy_id !== RESOURCE_POLICY.policy_id || request.product_pins.resource_policy.policy_sha256 !== RESOURCE_POLICY.policy_sha256) {
|
|
12333
12013
|
throw new Error("request policy pins do not match the effective package policies");
|
|
12334
12014
|
}
|
|
12335
12015
|
if (request.required_clock.clock_id !== request.initial_selection.required_clock.clock_id || request.required_clock.clock_sha256 !== request.initial_selection.required_clock.clock_sha256 || request.required_clock.event_count !== request.initial_selection.required_clock.event_count) {
|
|
@@ -12763,12 +12443,13 @@ function assertStoredSelectionMatchesRequest(request, selection) {
|
|
|
12763
12443
|
}
|
|
12764
12444
|
}
|
|
12765
12445
|
function eligibleBundles(request, bundles) {
|
|
12766
|
-
|
|
12767
|
-
|
|
12768
|
-
|
|
12769
|
-
|
|
12770
|
-
return
|
|
12771
|
-
}
|
|
12446
|
+
const qualifiedVendorBundles = bundles.filter((bundle) => bundle.captureOrigin === "vendor_historical_backfill" && bundle.qualification?.state === "qualified");
|
|
12447
|
+
const productionBundles = bundles.filter((bundle) => bundle.captureOrigin === "production_capture");
|
|
12448
|
+
if (request.sourcePolicy === "authoritative_window") {
|
|
12449
|
+
const vendorSupport = chooseSupport(request, qualifiedVendorBundles);
|
|
12450
|
+
return vendorSupport.length === request.requiredClockTargetsMs.length ? qualifiedVendorBundles : productionBundles;
|
|
12451
|
+
}
|
|
12452
|
+
return [...productionBundles, ...qualifiedVendorBundles];
|
|
12772
12453
|
}
|
|
12773
12454
|
function originPriority(request, origin) {
|
|
12774
12455
|
if (request.sourcePolicy === "fill_gaps") {
|
|
@@ -12973,9 +12654,8 @@ function exactCandidateMatch(expected, actual) {
|
|
|
12973
12654
|
function requestedShapeMatches(request, rows) {
|
|
12974
12655
|
return rows.every(({ row }) => row.depth_limit === request.depth && row.construction_mode === request.constructionMode && row.schema_version === request.expectedProduct.canonicalSchemaVersion);
|
|
12975
12656
|
}
|
|
12976
|
-
function requiredClockCoverage(request, rows) {
|
|
12977
|
-
const summaryTimes = rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => {
|
|
12978
|
-
const value = row.source_time_ms;
|
|
12657
|
+
function requiredClockCoverage(request, rows, coverageSourceTimesMs) {
|
|
12658
|
+
const summaryTimes = (coverageSourceTimesMs ?? rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => row.source_time_ms)).map((value) => {
|
|
12979
12659
|
if (typeof value !== "number" && (typeof value !== "string" || !/^\d+$/.test(value))) {
|
|
12980
12660
|
return;
|
|
12981
12661
|
}
|
|
@@ -13048,7 +12728,7 @@ function verifySemanticPromotion(evidence) {
|
|
|
13048
12728
|
coverageVerified: false
|
|
13049
12729
|
};
|
|
13050
12730
|
}
|
|
13051
|
-
const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows);
|
|
12731
|
+
const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows, evidence.coverageSourceTimesMs);
|
|
13052
12732
|
if (!coverageVerified) {
|
|
13053
12733
|
return {
|
|
13054
12734
|
...base,
|
|
@@ -13423,7 +13103,13 @@ class QualifiedOrderBookArchiveReader {
|
|
|
13423
13103
|
suffixDigestBefore: baseline.verificationBaseline.suffixDigest,
|
|
13424
13104
|
suffixDigestAfter: timelineDigest(suffix),
|
|
13425
13105
|
seamVerified: this.seamIsOrdered(queriedRows),
|
|
13426
|
-
exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string")
|
|
13106
|
+
exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string"),
|
|
13107
|
+
...request.sourcePolicy === "fill_gaps" ? {
|
|
13108
|
+
coverageSourceTimesMs: [
|
|
13109
|
+
...baseline.selection.support_anchors.map((anchor) => Date.parse(anchor.source_time)),
|
|
13110
|
+
...summaries.map((row) => numberField(row, "source_time_ms"))
|
|
13111
|
+
]
|
|
13112
|
+
} : {}
|
|
13427
13113
|
});
|
|
13428
13114
|
return { ...semantic, captureBundleId };
|
|
13429
13115
|
}
|
|
@@ -13612,8 +13298,8 @@ var requestContent = {
|
|
|
13612
13298
|
},
|
|
13613
13299
|
product_pins: {
|
|
13614
13300
|
capability_policy: {
|
|
13615
|
-
policy_id:
|
|
13616
|
-
policy_sha256:
|
|
13301
|
+
policy_id: LEGACY_CAPABILITY_POLICY.policy_id,
|
|
13302
|
+
policy_sha256: LEGACY_CAPABILITY_POLICY.policy_sha256
|
|
13617
13303
|
},
|
|
13618
13304
|
resource_policy: {
|
|
13619
13305
|
policy_id: RESOURCE_POLICY.policy_id,
|
|
@@ -13766,422 +13452,14 @@ var CONFORMANCE_FIXTURES = Object.freeze({
|
|
|
13766
13452
|
promotion_identity_sha256: promotionReceipt.promotion_identity_sha256,
|
|
13767
13453
|
receipt_id: promotionReceipt.receipt_id,
|
|
13768
13454
|
result_sha256: result.result_sha256,
|
|
13769
|
-
capability_policy_sha256:
|
|
13455
|
+
capability_policy_sha256: LEGACY_CAPABILITY_POLICY.policy_sha256,
|
|
13770
13456
|
resource_policy_sha256: RESOURCE_POLICY.policy_sha256
|
|
13771
13457
|
},
|
|
13772
13458
|
jcs_edge_vector: jcsEdgeVector,
|
|
13773
13459
|
hashes: { jcs_edge_vector_sha256: jcsSha256(jcsEdgeVector) }
|
|
13774
13460
|
});
|
|
13775
|
-
// src/helpers/market-data-vendor-backfill/
|
|
13776
|
-
|
|
13777
|
-
var SHA256 = /^[0-9a-f]{64}$/;
|
|
13778
|
-
var REASON = /^[a-z][a-z0-9_]{0,127}$/;
|
|
13779
|
-
var EVENT_FIELDS = new Set([
|
|
13780
|
-
"qualification_event_id",
|
|
13781
|
-
"capture_bundle_id",
|
|
13782
|
-
"state",
|
|
13783
|
-
"receipt_id",
|
|
13784
|
-
"promotion_identity_sha256",
|
|
13785
|
-
"window",
|
|
13786
|
-
"event_at",
|
|
13787
|
-
"reason_code"
|
|
13788
|
-
]);
|
|
13789
|
-
function deterministicUuid(value) {
|
|
13790
|
-
const digest = jcsSha256(value).slice(0, 32).split("");
|
|
13791
|
-
digest[12] = "5";
|
|
13792
|
-
digest[16] = "8";
|
|
13793
|
-
const hex = digest.join("");
|
|
13794
|
-
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
13795
|
-
}
|
|
13796
|
-
function semanticEvent(event) {
|
|
13797
|
-
const { qualification_event_id: _eventId, ...semantic } = event;
|
|
13798
|
-
return semantic;
|
|
13799
|
-
}
|
|
13800
|
-
function assertTimestamp(value, field) {
|
|
13801
|
-
const parsed = Date.parse(value);
|
|
13802
|
-
if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
|
|
13803
|
-
throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
|
|
13804
|
-
}
|
|
13805
|
-
return parsed;
|
|
13806
|
-
}
|
|
13807
|
-
function parseQualificationEvent(value) {
|
|
13808
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
13809
|
-
throw new Error("qualification event must be an object");
|
|
13810
|
-
}
|
|
13811
|
-
const event = value;
|
|
13812
|
-
if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
|
|
13813
|
-
throw new Error("qualification event fields are invalid");
|
|
13814
|
-
}
|
|
13815
|
-
if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
|
|
13816
|
-
throw new Error("qualification window must be increasing");
|
|
13817
|
-
}
|
|
13818
|
-
assertTimestamp(event.event_at, "event_at");
|
|
13819
|
-
if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
|
|
13820
|
-
throw new Error("qualification_event_id does not match event content");
|
|
13821
|
-
}
|
|
13822
|
-
return event;
|
|
13823
|
-
}
|
|
13824
|
-
function finalizeQualificationEvent(event) {
|
|
13825
|
-
const semantic = semanticEvent(event);
|
|
13826
|
-
return parseQualificationEvent({
|
|
13827
|
-
...semantic,
|
|
13828
|
-
qualification_event_id: deterministicUuid(semantic)
|
|
13829
|
-
});
|
|
13830
|
-
}
|
|
13831
|
-
function qualificationEventToArchiveRow(eventInput) {
|
|
13832
|
-
const event = parseQualificationEvent(eventInput);
|
|
13833
|
-
return {
|
|
13834
|
-
table: "market_data.cex_order_book_capture_qualifications",
|
|
13835
|
-
row: {
|
|
13836
|
-
source: "external_backfill",
|
|
13837
|
-
capture_origin: "vendor_historical_backfill",
|
|
13838
|
-
source_mode: "vendor_historical_backfill_v1",
|
|
13839
|
-
deployment_id: "market-data-vendor-backfill",
|
|
13840
|
-
qualification_event_id: event.qualification_event_id,
|
|
13841
|
-
capture_bundle_id: event.capture_bundle_id,
|
|
13842
|
-
state: event.state,
|
|
13843
|
-
receipt_id: event.receipt_id,
|
|
13844
|
-
promotion_identity_sha256: event.promotion_identity_sha256,
|
|
13845
|
-
window_start_ms: Date.parse(event.window.start_at),
|
|
13846
|
-
window_end_ms: Date.parse(event.window.end_at),
|
|
13847
|
-
event_at_ms: Date.parse(event.event_at),
|
|
13848
|
-
reason_code: event.reason_code,
|
|
13849
|
-
event_json: jcsCanonicalize(event)
|
|
13850
|
-
}
|
|
13851
|
-
};
|
|
13852
|
-
}
|
|
13853
|
-
|
|
13854
|
-
// src/helpers/market-data-vendor-backfill/core.ts
|
|
13855
|
-
function createMarketDataVendorBackfillDependencies(dependencies) {
|
|
13856
|
-
for (const [name, value] of Object.entries({
|
|
13857
|
-
archive: dependencies.archive,
|
|
13858
|
-
providers: dependencies.providers,
|
|
13859
|
-
credentials: dependencies.credentials,
|
|
13860
|
-
forwarder: dependencies.forwarder,
|
|
13861
|
-
clock: dependencies.clock
|
|
13862
|
-
})) {
|
|
13863
|
-
if (!value || typeof value !== "object") {
|
|
13864
|
-
throw new TypeError(`Backfill dependency ${name} is required`);
|
|
13865
|
-
}
|
|
13866
|
-
}
|
|
13867
|
-
for (const [name, method] of Object.entries({
|
|
13868
|
-
"archive.resolveSelection": dependencies.archive.resolveSelection,
|
|
13869
|
-
"archive.verifyCandidate": dependencies.archive.verifyCandidate,
|
|
13870
|
-
"providers.capabilityFor": dependencies.providers.capabilityFor,
|
|
13871
|
-
"providers.acquire": dependencies.providers.acquire,
|
|
13872
|
-
"providers.normalize": dependencies.providers.normalize,
|
|
13873
|
-
"credentials.resolve": dependencies.credentials.resolve,
|
|
13874
|
-
"forwarder.preflight": dependencies.forwarder.preflight,
|
|
13875
|
-
"forwarder.submit": dependencies.forwarder.submit,
|
|
13876
|
-
"clock.nowMs": dependencies.clock.nowMs
|
|
13877
|
-
})) {
|
|
13878
|
-
if (typeof method !== "function") {
|
|
13879
|
-
throw new TypeError(`Backfill dependency ${name} is required`);
|
|
13880
|
-
}
|
|
13881
|
-
}
|
|
13882
|
-
return dependencies;
|
|
13883
|
-
}
|
|
13884
|
-
function outcome(request2, status, reasonCode, extra = {}) {
|
|
13885
|
-
return {
|
|
13886
|
-
status,
|
|
13887
|
-
reasonCode,
|
|
13888
|
-
...request2 ? {
|
|
13889
|
-
requestId: request2.requestId,
|
|
13890
|
-
idempotencyKey: request2.idempotencyKey,
|
|
13891
|
-
...request2.target ? { target: request2.target } : {}
|
|
13892
|
-
} : {},
|
|
13893
|
-
...extra
|
|
13894
|
-
};
|
|
13895
|
-
}
|
|
13896
|
-
function captureBundleId2(request2, capability, dataset) {
|
|
13897
|
-
return sha256Canonical({
|
|
13898
|
-
request_business_identity: request2.idempotencyKey,
|
|
13899
|
-
provider: capability.provider,
|
|
13900
|
-
provider_exchange_id: capability.providerExchangeId,
|
|
13901
|
-
resolved_symbol: capability.resolvedSymbol,
|
|
13902
|
-
adapter_version: capability.adapterVersion,
|
|
13903
|
-
objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
|
|
13904
|
-
identity,
|
|
13905
|
-
checksum,
|
|
13906
|
-
bytes,
|
|
13907
|
-
rows
|
|
13908
|
-
})),
|
|
13909
|
-
canonical_schema_version: request2.expectedProduct.canonicalSchemaVersion,
|
|
13910
|
-
checksum_algorithm: request2.expectedProduct.checksumAlgorithm
|
|
13911
|
-
});
|
|
13912
|
-
}
|
|
13913
|
-
function buildPromotionReceipt(request2, capability, normalized, verification, verificationTimeMs) {
|
|
13914
|
-
if (!request2.wire || !request2.initialSelection || !request2.expectedCanonicalSchema || !request2.coveragePolicy || !request2.productPins) {
|
|
13915
|
-
throw new Error("decoded final-v1 request context is missing");
|
|
13916
|
-
}
|
|
13917
|
-
return finalizePromotionReceipt({
|
|
13918
|
-
schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
|
|
13919
|
-
verified_at: new Date(verificationTimeMs).toISOString(),
|
|
13920
|
-
request_id: request2.requestId,
|
|
13921
|
-
idempotency_key: request2.idempotencyKey,
|
|
13922
|
-
source: EXTERNAL_BACKFILL_SOURCE,
|
|
13923
|
-
capture_origin: "vendor_historical_backfill",
|
|
13924
|
-
source_mode: "vendor_historical_backfill_v1",
|
|
13925
|
-
provider: capability.provider,
|
|
13926
|
-
adapter_version: capability.adapterVersion,
|
|
13927
|
-
effective_policies: {
|
|
13928
|
-
capability_policy: request2.productPins.capability_policy,
|
|
13929
|
-
resource_policy: request2.productPins.resource_policy,
|
|
13930
|
-
adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
|
|
13931
|
-
acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
|
|
13932
|
-
},
|
|
13933
|
-
capture_bundle_id: normalized.captureBundleId,
|
|
13934
|
-
scope: request2.wire.scope,
|
|
13935
|
-
window: request2.wire.window,
|
|
13936
|
-
depth: request2.depth,
|
|
13937
|
-
construction_mode: request2.constructionMode,
|
|
13938
|
-
canonical_schema: request2.expectedCanonicalSchema,
|
|
13939
|
-
coverage_policy: request2.coveragePolicy,
|
|
13940
|
-
selection_sha256: request2.initialSelection.selection_sha256,
|
|
13941
|
-
vendor_semantic_digest: normalized.vendorSemanticDigest,
|
|
13942
|
-
canonical_semantic_digest: verification.canonicalSemanticDigest,
|
|
13943
|
-
prefix_digest: verification.prefixDigest,
|
|
13944
|
-
suffix_digest: verification.suffixDigest,
|
|
13945
|
-
seam_verified: true,
|
|
13946
|
-
coverage_verified: true,
|
|
13947
|
-
dataset_objects: normalized.objects
|
|
13948
|
-
});
|
|
13949
|
-
}
|
|
13950
|
-
async function submitAll(dependencies, batches) {
|
|
13951
|
-
const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
|
|
13952
|
-
for (const batch of batches) {
|
|
13953
|
-
let accepted = false;
|
|
13954
|
-
for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
|
|
13955
|
-
try {
|
|
13956
|
-
const response = await dependencies.forwarder.submit(batch);
|
|
13957
|
-
accepted = response.ok && response.inserted === batch.rows.length;
|
|
13958
|
-
} catch {
|
|
13959
|
-
accepted = false;
|
|
13960
|
-
}
|
|
13961
|
-
if (accepted)
|
|
13962
|
-
break;
|
|
13963
|
-
if (attempt < maxAttempts)
|
|
13964
|
-
await dependencies.retry?.wait?.(attempt);
|
|
13965
|
-
}
|
|
13966
|
-
if (!accepted)
|
|
13967
|
-
return false;
|
|
13968
|
-
}
|
|
13969
|
-
return true;
|
|
13970
|
-
}
|
|
13971
|
-
function stableFailureReason(error, fallback) {
|
|
13972
|
-
if (error && typeof error === "object") {
|
|
13973
|
-
const reason = error.reason;
|
|
13974
|
-
if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
|
|
13975
|
-
return reason;
|
|
13976
|
-
}
|
|
13977
|
-
}
|
|
13978
|
-
return fallback;
|
|
13979
|
-
}
|
|
13980
|
-
function assertArchivePreflight(request2, resolution) {
|
|
13981
|
-
if (!request2.target || !request2.productionAuthorizationId) {
|
|
13982
|
-
throw new Error("request target or production authorization ID is missing");
|
|
13983
|
-
}
|
|
13984
|
-
const selection = archiveSelectionCodec.decode(resolution.selection);
|
|
13985
|
-
const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
|
|
13986
|
-
const receiptById = new Map;
|
|
13987
|
-
for (const receipt of receipts) {
|
|
13988
|
-
const existing = receiptById.get(receipt.receipt_id);
|
|
13989
|
-
if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
|
|
13990
|
-
throw new Error("stored receipt identity has conflicting content");
|
|
13991
|
-
}
|
|
13992
|
-
receiptById.set(receipt.receipt_id, receipt);
|
|
13993
|
-
}
|
|
13994
|
-
for (const bundle of selection.bundles) {
|
|
13995
|
-
if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
|
|
13996
|
-
throw new Error("vendor selection lacks its validated stored receipt");
|
|
13997
|
-
}
|
|
13998
|
-
}
|
|
13999
|
-
if (resolution.readerIdentity.environment !== request2.target.environment || resolution.readerIdentity.cluster !== request2.target.cluster) {
|
|
14000
|
-
throw new Error("archive reader cluster identity mismatch");
|
|
14001
|
-
}
|
|
14002
|
-
}
|
|
14003
|
-
function assertForwarderPreflight(request2, resolution, nowMs) {
|
|
14004
|
-
if (!request2.target || !request2.productionAuthorizationId) {
|
|
14005
|
-
throw new Error("request target or production authorization ID is missing");
|
|
14006
|
-
}
|
|
14007
|
-
if (resolution.forwarderIdentity.environment !== request2.target.environment || resolution.forwarderIdentity.cluster !== request2.target.cluster) {
|
|
14008
|
-
throw new Error("archive forwarder cluster identity mismatch");
|
|
14009
|
-
}
|
|
14010
|
-
const authorization = resolution.authorization;
|
|
14011
|
-
const expiresAtMs = Date.parse(authorization.expiresAt);
|
|
14012
|
-
if (authorization.authorizationId !== request2.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request2.target.environment || authorization.cluster !== request2.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
|
|
14013
|
-
throw new Error("production forwarder authorization is invalid");
|
|
14014
|
-
}
|
|
14015
|
-
}
|
|
14016
|
-
function resourcePolicyScopeExceeded(request2) {
|
|
14017
|
-
return request2.depth > RESOURCE_POLICY.request_bounds.max_depth || request2.window.endTimeMs - request2.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request2.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
|
|
14018
|
-
}
|
|
14019
|
-
function storedReceiptForSelection(resolution) {
|
|
14020
|
-
const receiptId = resolution.selection.receipt_ids[0];
|
|
14021
|
-
return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
|
|
14022
|
-
}
|
|
14023
|
-
async function runMarketDataVendorBackfill(input, dependencies) {
|
|
14024
|
-
let request2;
|
|
14025
|
-
try {
|
|
14026
|
-
const documents = input;
|
|
14027
|
-
request2 = decodeBackfillRunDocuments({
|
|
14028
|
-
request: documents.request,
|
|
14029
|
-
requiredClock: documents.requiredClock
|
|
14030
|
-
});
|
|
14031
|
-
} catch {
|
|
14032
|
-
return outcome(undefined, "request_invalid", "request_invalid");
|
|
14033
|
-
}
|
|
14034
|
-
let initialResolution;
|
|
14035
|
-
try {
|
|
14036
|
-
initialResolution = await dependencies.archive.resolveSelection(request2);
|
|
14037
|
-
assertArchivePreflight(request2, initialResolution);
|
|
14038
|
-
const forwarderPreflight = await dependencies.forwarder.preflight({
|
|
14039
|
-
authorizationId: request2.productionAuthorizationId,
|
|
14040
|
-
target: request2.target
|
|
14041
|
-
});
|
|
14042
|
-
assertForwarderPreflight(request2, forwarderPreflight, dependencies.clock.nowMs());
|
|
14043
|
-
} catch {
|
|
14044
|
-
return outcome(request2, "archive_preflight_failed", "archive_preflight_failed");
|
|
14045
|
-
}
|
|
14046
|
-
if (initialResolution.selection.coverage_class === "complete") {
|
|
14047
|
-
const receipt2 = storedReceiptForSelection(initialResolution);
|
|
14048
|
-
return outcome(request2, "already_covered", "qualified_coverage_complete", {
|
|
14049
|
-
selection: initialResolution.selection,
|
|
14050
|
-
...receipt2 ? { receipt: receipt2 } : {}
|
|
14051
|
-
});
|
|
14052
|
-
}
|
|
14053
|
-
if (resourcePolicyScopeExceeded(request2)) {
|
|
14054
|
-
return outcome(request2, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
|
|
14055
|
-
}
|
|
14056
|
-
let capability;
|
|
14057
|
-
try {
|
|
14058
|
-
capability = dependencies.providers.capabilityFor(request2);
|
|
14059
|
-
} catch {
|
|
14060
|
-
return outcome(request2, "capability_unsupported", "capability_probe_failed");
|
|
14061
|
-
}
|
|
14062
|
-
if (!capability) {
|
|
14063
|
-
return outcome(request2, "capability_unsupported", "scope_unsupported");
|
|
14064
|
-
}
|
|
14065
|
-
if (!request2.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
|
|
14066
|
-
return outcome(request2, "capability_unsupported", "adapter_version_unpinned");
|
|
14067
|
-
}
|
|
14068
|
-
let credential;
|
|
14069
|
-
try {
|
|
14070
|
-
credential = await dependencies.credentials.resolve(capability.provider);
|
|
14071
|
-
} catch {
|
|
14072
|
-
return outcome(request2, "credentials_missing", "credential_resolution_failed");
|
|
14073
|
-
}
|
|
14074
|
-
if (credential === undefined || credential === null) {
|
|
14075
|
-
return outcome(request2, "credentials_missing", "provider_credentials_missing");
|
|
14076
|
-
}
|
|
14077
|
-
let dataset;
|
|
14078
|
-
let normalized;
|
|
14079
|
-
try {
|
|
14080
|
-
dataset = await dependencies.providers.acquire(request2, capability, credential);
|
|
14081
|
-
const bundleId = captureBundleId2(request2, capability, dataset);
|
|
14082
|
-
normalized = await dependencies.providers.normalize(request2, capability, dataset, bundleId);
|
|
14083
|
-
if (normalized.captureBundleId !== bundleId) {
|
|
14084
|
-
return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
14085
|
-
reasonSubcode: "capture_identity_mismatch"
|
|
14086
|
-
});
|
|
14087
|
-
}
|
|
14088
|
-
} catch (error) {
|
|
14089
|
-
const reason = stableFailureReason(error, "provider_dataset_invalid");
|
|
14090
|
-
return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
14091
|
-
reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
|
|
14092
|
-
});
|
|
14093
|
-
}
|
|
14094
|
-
try {
|
|
14095
|
-
const candidateBatches = buildForwarderBatches({
|
|
14096
|
-
captureBundleId: normalized.captureBundleId,
|
|
14097
|
-
deploymentId: "market-data-vendor-backfill",
|
|
14098
|
-
rows: normalized.rows
|
|
14099
|
-
});
|
|
14100
|
-
if (!await submitAll(dependencies, candidateBatches)) {
|
|
14101
|
-
return outcome(request2, "archive_ingest_failed", "candidate_batch_rejected");
|
|
14102
|
-
}
|
|
14103
|
-
} catch {
|
|
14104
|
-
return outcome(request2, "archive_ingest_failed", "candidate_submission_failed");
|
|
14105
|
-
}
|
|
14106
|
-
let verification;
|
|
14107
|
-
try {
|
|
14108
|
-
verification = await dependencies.archive.verifyCandidate(request2, normalized, normalized.captureBundleId, initialResolution);
|
|
14109
|
-
} catch {
|
|
14110
|
-
return outcome(request2, "promotion_verification_failed", "candidate_query_failed");
|
|
14111
|
-
}
|
|
14112
|
-
if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
|
|
14113
|
-
return outcome(request2, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
|
|
14114
|
-
}
|
|
14115
|
-
let receipt;
|
|
14116
|
-
try {
|
|
14117
|
-
receipt = buildPromotionReceipt(request2, capability, normalized, verification, dependencies.clock.nowMs());
|
|
14118
|
-
const [batch] = buildForwarderBatches({
|
|
14119
|
-
captureBundleId: normalized.captureBundleId,
|
|
14120
|
-
deploymentId: "market-data-vendor-backfill",
|
|
14121
|
-
rows: [promotionReceiptToArchiveRow(receipt)]
|
|
14122
|
-
});
|
|
14123
|
-
if (!batch || !await submitAll(dependencies, [batch])) {
|
|
14124
|
-
return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
|
|
14125
|
-
}
|
|
14126
|
-
const qualification = finalizeQualificationEvent({
|
|
14127
|
-
capture_bundle_id: receipt.capture_bundle_id,
|
|
14128
|
-
state: "qualified",
|
|
14129
|
-
receipt_id: receipt.receipt_id,
|
|
14130
|
-
promotion_identity_sha256: receipt.promotion_identity_sha256,
|
|
14131
|
-
window: receipt.window,
|
|
14132
|
-
event_at: receipt.verified_at,
|
|
14133
|
-
reason_code: "promotion_verified"
|
|
14134
|
-
});
|
|
14135
|
-
const [qualificationBatch] = buildForwarderBatches({
|
|
14136
|
-
captureBundleId: normalized.captureBundleId,
|
|
14137
|
-
deploymentId: "market-data-vendor-backfill",
|
|
14138
|
-
rows: [qualificationEventToArchiveRow(qualification)]
|
|
14139
|
-
});
|
|
14140
|
-
if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
|
|
14141
|
-
return outcome(request2, "archive_ingest_failed", "qualification_commit_failed", { receipt });
|
|
14142
|
-
}
|
|
14143
|
-
} catch {
|
|
14144
|
-
return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
|
|
14145
|
-
}
|
|
14146
|
-
let finalResolution;
|
|
14147
|
-
try {
|
|
14148
|
-
finalResolution = await dependencies.archive.resolveSelection(request2);
|
|
14149
|
-
assertArchivePreflight(request2, finalResolution);
|
|
14150
|
-
} catch (error) {
|
|
14151
|
-
return outcome(request2, "promotion_verification_failed", "post_promotion_selection_failed", {
|
|
14152
|
-
receipt,
|
|
14153
|
-
reasonSubcode: stableFailureReason(error, "archive_selection_resolution_failed")
|
|
14154
|
-
});
|
|
14155
|
-
}
|
|
14156
|
-
if (finalResolution.selection.coverage_class !== "complete") {
|
|
14157
|
-
return outcome(request2, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
|
|
14158
|
-
}
|
|
14159
|
-
if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
|
|
14160
|
-
return outcome(request2, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
|
|
14161
|
-
}
|
|
14162
|
-
try {
|
|
14163
|
-
if (!request2.wire)
|
|
14164
|
-
throw new Error("decoded request wire is missing");
|
|
14165
|
-
const [selectionBatch] = buildForwarderBatches({
|
|
14166
|
-
captureBundleId: receipt.capture_bundle_id,
|
|
14167
|
-
deploymentId: "market-data-vendor-backfill",
|
|
14168
|
-
rows: [
|
|
14169
|
-
archiveSelectionToArchiveRow(request2.wire, finalResolution.selection)
|
|
14170
|
-
]
|
|
14171
|
-
});
|
|
14172
|
-
if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
|
|
14173
|
-
return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
14174
|
-
}
|
|
14175
|
-
} catch {
|
|
14176
|
-
return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
14177
|
-
}
|
|
14178
|
-
return outcome(request2, "promoted", "promotion_qualified", {
|
|
14179
|
-
receipt,
|
|
14180
|
-
selection: finalResolution.selection
|
|
14181
|
-
});
|
|
14182
|
-
}
|
|
14183
|
-
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
14184
|
-
import { createHash as createHash3 } from "node:crypto";
|
|
13461
|
+
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
13462
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
14185
13463
|
|
|
14186
13464
|
// node_modules/fzstd/esm/index.mjs
|
|
14187
13465
|
var ab = ArrayBuffer;
|
|
@@ -18090,599 +17368,1043 @@ function buildCanonicalOrderBookRows(input) {
|
|
|
18090
17368
|
const common = commonEvidenceFields(input.context, input.rawCapture, {
|
|
18091
17369
|
snapshotId,
|
|
18092
17370
|
sequence,
|
|
18093
|
-
depthLimit: input.depthLimit,
|
|
18094
|
-
eventTimeMs,
|
|
18095
|
-
receivedTimeMs
|
|
18096
|
-
});
|
|
18097
|
-
const levels = [
|
|
18098
|
-
["bid", bids],
|
|
18099
|
-
["ask", asks]
|
|
18100
|
-
].flatMap(([side, sideLevels]) => sideLevels.map(({ price, amount }, levelIndex) => {
|
|
18101
|
-
const row = checksumRow({
|
|
18102
|
-
...common,
|
|
18103
|
-
side,
|
|
18104
|
-
level_index: levelIndex,
|
|
18105
|
-
price,
|
|
18106
|
-
amount,
|
|
18107
|
-
notional: price * amount,
|
|
18108
|
-
mid_price: midPrice,
|
|
18109
|
-
spread_from_mid_bps: Math.abs((price - midPrice) / midPrice) * 1e4
|
|
18110
|
-
});
|
|
18111
|
-
return { table: "market_data.cex_order_book_levels", row };
|
|
18112
|
-
}));
|
|
18113
|
-
const bands = normalizedBands(input.measurementBandsBps);
|
|
18114
|
-
const bidDepth = bands.map((band) => {
|
|
18115
|
-
const minimumPrice = bestBid.price * (1 - band / 1e4);
|
|
18116
|
-
return bids.filter(({ price }) => price >= minimumPrice).reduce((sum, { amount }) => sum + amount, 0);
|
|
18117
|
-
});
|
|
18118
|
-
const askDepth = bands.map((band) => {
|
|
18119
|
-
const maximumPrice = bestAsk.price * (1 + band / 1e4);
|
|
18120
|
-
return asks.filter(({ price }) => price <= maximumPrice).reduce((sum, { amount }) => sum + amount, 0);
|
|
18121
|
-
});
|
|
18122
|
-
const summaryRow = checksumRow({
|
|
18123
|
-
...common,
|
|
18124
|
-
best_bid: bestBid.price,
|
|
18125
|
-
best_ask: bestAsk.price,
|
|
18126
|
-
best_bid_amount: bestBid.amount,
|
|
18127
|
-
best_ask_amount: bestAsk.amount,
|
|
18128
|
-
mid_price: midPrice,
|
|
18129
|
-
spread,
|
|
18130
|
-
spread_bps: spreadBps,
|
|
18131
|
-
staleness_ms: receivedTimeMs - eventTimeMs,
|
|
18132
|
-
bid_level_count: bids.length,
|
|
18133
|
-
ask_level_count: asks.length,
|
|
18134
|
-
measurement_bands_bps: bands,
|
|
18135
|
-
bid_depth_by_band: bidDepth,
|
|
18136
|
-
ask_depth_by_band: askDepth
|
|
18137
|
-
});
|
|
18138
|
-
return {
|
|
18139
|
-
snapshotId,
|
|
18140
|
-
levels,
|
|
18141
|
-
summary: {
|
|
18142
|
-
table: "market_data.cex_order_book_depth_summary",
|
|
18143
|
-
row: summaryRow
|
|
17371
|
+
depthLimit: input.depthLimit,
|
|
17372
|
+
eventTimeMs,
|
|
17373
|
+
receivedTimeMs
|
|
17374
|
+
});
|
|
17375
|
+
const levels = [
|
|
17376
|
+
["bid", bids],
|
|
17377
|
+
["ask", asks]
|
|
17378
|
+
].flatMap(([side, sideLevels]) => sideLevels.map(({ price, amount }, levelIndex) => {
|
|
17379
|
+
const row = checksumRow({
|
|
17380
|
+
...common,
|
|
17381
|
+
side,
|
|
17382
|
+
level_index: levelIndex,
|
|
17383
|
+
price,
|
|
17384
|
+
amount,
|
|
17385
|
+
notional: price * amount,
|
|
17386
|
+
mid_price: midPrice,
|
|
17387
|
+
spread_from_mid_bps: Math.abs((price - midPrice) / midPrice) * 1e4
|
|
17388
|
+
});
|
|
17389
|
+
return { table: "market_data.cex_order_book_levels", row };
|
|
17390
|
+
}));
|
|
17391
|
+
const bands = normalizedBands(input.measurementBandsBps);
|
|
17392
|
+
const bidDepth = bands.map((band) => {
|
|
17393
|
+
const minimumPrice = bestBid.price * (1 - band / 1e4);
|
|
17394
|
+
return bids.filter(({ price }) => price >= minimumPrice).reduce((sum, { amount }) => sum + amount, 0);
|
|
17395
|
+
});
|
|
17396
|
+
const askDepth = bands.map((band) => {
|
|
17397
|
+
const maximumPrice = bestAsk.price * (1 + band / 1e4);
|
|
17398
|
+
return asks.filter(({ price }) => price <= maximumPrice).reduce((sum, { amount }) => sum + amount, 0);
|
|
17399
|
+
});
|
|
17400
|
+
const summaryRow = checksumRow({
|
|
17401
|
+
...common,
|
|
17402
|
+
best_bid: bestBid.price,
|
|
17403
|
+
best_ask: bestAsk.price,
|
|
17404
|
+
best_bid_amount: bestBid.amount,
|
|
17405
|
+
best_ask_amount: bestAsk.amount,
|
|
17406
|
+
mid_price: midPrice,
|
|
17407
|
+
spread,
|
|
17408
|
+
spread_bps: spreadBps,
|
|
17409
|
+
staleness_ms: receivedTimeMs - eventTimeMs,
|
|
17410
|
+
bid_level_count: bids.length,
|
|
17411
|
+
ask_level_count: asks.length,
|
|
17412
|
+
measurement_bands_bps: bands,
|
|
17413
|
+
bid_depth_by_band: bidDepth,
|
|
17414
|
+
ask_depth_by_band: askDepth
|
|
17415
|
+
});
|
|
17416
|
+
return {
|
|
17417
|
+
snapshotId,
|
|
17418
|
+
levels,
|
|
17419
|
+
summary: {
|
|
17420
|
+
table: "market_data.cex_order_book_depth_summary",
|
|
17421
|
+
row: summaryRow
|
|
17422
|
+
}
|
|
17423
|
+
};
|
|
17424
|
+
}
|
|
17425
|
+
|
|
17426
|
+
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
17427
|
+
var CRYPTOHFTDATA_ADAPTER_VERSION = "cryptohftdata-orderbook/v2";
|
|
17428
|
+
var CRYPTOHFTDATA_API_URL = "https://api.cryptohftdata.com";
|
|
17429
|
+
var CRYPTOHFTDATA_HISTORY_START_MS = Date.UTC(2025, 5, 28);
|
|
17430
|
+
var HOUR_MS = 60 * 60 * 1000;
|
|
17431
|
+
var CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE = Object.freeze({
|
|
17432
|
+
profileId: "cryptohftdata/binance_spot/BTCUSDT/v1",
|
|
17433
|
+
exchange: "binance",
|
|
17434
|
+
tradingPair: "BTC-USDT",
|
|
17435
|
+
sourceSymbol: "BTCUSDT",
|
|
17436
|
+
marketType: "spot",
|
|
17437
|
+
providerExchangeId: "binance_spot",
|
|
17438
|
+
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
17439
|
+
maxDepth: 500,
|
|
17440
|
+
eventTimeUnit: "milliseconds",
|
|
17441
|
+
receivedTimeUnit: "nanoseconds",
|
|
17442
|
+
snapshotGrouping: "event_time_last_update_id_object",
|
|
17443
|
+
sequenceSemantics: "binance_u_U_pu",
|
|
17444
|
+
constructionModes: ["sampled_top_n_snapshot"],
|
|
17445
|
+
sourcePolicies: ["authoritative_window"]
|
|
17446
|
+
});
|
|
17447
|
+
var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
|
|
17448
|
+
profileId: "cryptohftdata/okx_spot/ARB-USDT/v1",
|
|
17449
|
+
exchange: "okx",
|
|
17450
|
+
tradingPair: "ARB-USDT",
|
|
17451
|
+
sourceSymbol: "ARB-USDT",
|
|
17452
|
+
marketType: "spot",
|
|
17453
|
+
providerExchangeId: "okx_spot",
|
|
17454
|
+
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
17455
|
+
maxDepth: 400,
|
|
17456
|
+
eventTimeUnit: "milliseconds",
|
|
17457
|
+
receivedTimeUnit: "nanoseconds",
|
|
17458
|
+
snapshotGrouping: "event_time_final_update_id_object",
|
|
17459
|
+
sequenceSemantics: "okx_seq_id_prev_seq_id",
|
|
17460
|
+
constructionModes: ["sampled_top_n_snapshot"],
|
|
17461
|
+
sourcePolicies: ["authoritative_window", "fill_gaps"]
|
|
17462
|
+
});
|
|
17463
|
+
|
|
17464
|
+
class CryptoHftDataError extends Error {
|
|
17465
|
+
reason;
|
|
17466
|
+
constructor(reason) {
|
|
17467
|
+
super(`CryptoHFTData backfill failed: ${reason}`);
|
|
17468
|
+
this.reason = reason;
|
|
17469
|
+
this.name = "CryptoHftDataError";
|
|
17470
|
+
}
|
|
17471
|
+
}
|
|
17472
|
+
function cryptoHftDataCapabilityFor(request2, profiles = []) {
|
|
17473
|
+
if (request2.providerPolicy.provider !== "cryptohftdata" || request2.scope.feed !== "ORDERBOOK" || request2.scope.exchange.trim().toLowerCase() === "mexc") {
|
|
17474
|
+
return;
|
|
17475
|
+
}
|
|
17476
|
+
const profile = profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.constructionModes.includes(request2.constructionMode) && candidate.sourcePolicies.includes(request2.sourcePolicy) && request2.window.startTimeMs >= candidate.historyStartMs && request2.depth <= candidate.maxDepth);
|
|
17477
|
+
if (!profile)
|
|
17478
|
+
return;
|
|
17479
|
+
return {
|
|
17480
|
+
provider: "cryptohftdata",
|
|
17481
|
+
adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
|
|
17482
|
+
providerExchangeId: profile.providerExchangeId,
|
|
17483
|
+
resolvedSymbol: profile.sourceSymbol
|
|
17484
|
+
};
|
|
17485
|
+
}
|
|
17486
|
+
function utcHourPath(hourMs) {
|
|
17487
|
+
const date = new Date(hourMs);
|
|
17488
|
+
return {
|
|
17489
|
+
date: `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String(date.getUTCDate()).padStart(2, "0")}`,
|
|
17490
|
+
hour: String(date.getUTCHours()).padStart(2, "0")
|
|
17491
|
+
};
|
|
17492
|
+
}
|
|
17493
|
+
function enumerateCryptoHftDataObjects(request2, providerExchangeId, resolvedSymbol) {
|
|
17494
|
+
if (request2.sourcePolicy === "fill_gaps") {
|
|
17495
|
+
const hours = new Set;
|
|
17496
|
+
for (const targetTimeMs of request2.requiredClockTargetsMs) {
|
|
17497
|
+
const firstHour2 = Math.floor(Math.max(0, targetTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
|
|
17498
|
+
const lastHour = Math.floor(targetTimeMs / HOUR_MS) * HOUR_MS;
|
|
17499
|
+
for (let hourMs = firstHour2;hourMs <= lastHour; hourMs += HOUR_MS) {
|
|
17500
|
+
hours.add(hourMs);
|
|
17501
|
+
}
|
|
17502
|
+
}
|
|
17503
|
+
return [...hours].sort((left, right) => left - right).map((hourMs) => {
|
|
17504
|
+
const utc = utcHourPath(hourMs);
|
|
17505
|
+
return `${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`;
|
|
17506
|
+
});
|
|
17507
|
+
}
|
|
17508
|
+
const firstHour = Math.floor((request2.window.startTimeMs - request2.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
|
|
17509
|
+
const objects = [];
|
|
17510
|
+
for (let hourMs = Math.max(0, firstHour);hourMs < request2.window.endTimeMs; hourMs += HOUR_MS) {
|
|
17511
|
+
const utc = utcHourPath(hourMs);
|
|
17512
|
+
objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
|
|
17513
|
+
}
|
|
17514
|
+
return objects;
|
|
17515
|
+
}
|
|
17516
|
+
function initialArchiveCoversTarget(request2, targetTimeMs) {
|
|
17517
|
+
return (request2.initialSelection?.support_anchors ?? []).some((anchor) => {
|
|
17518
|
+
const sourceTimeMs = Date.parse(anchor.source_time);
|
|
17519
|
+
return Number.isSafeInteger(sourceTimeMs) && sourceTimeMs <= targetTimeMs && targetTimeMs - sourceTimeMs <= request2.maxPriorAsOfLagMs;
|
|
17520
|
+
});
|
|
17521
|
+
}
|
|
17522
|
+
function providerAcquisitionRequest(request2) {
|
|
17523
|
+
if (request2.sourcePolicy !== "fill_gaps")
|
|
17524
|
+
return request2;
|
|
17525
|
+
const requiredClockTargetsMs = request2.requiredClockTargetsMs.filter((targetTimeMs) => !initialArchiveCoversTarget(request2, targetTimeMs));
|
|
17526
|
+
if (requiredClockTargetsMs.length === 0) {
|
|
17527
|
+
throw new CryptoHftDataError("fill_gaps_has_no_uncovered_clock_targets");
|
|
17528
|
+
}
|
|
17529
|
+
return { ...request2, requiredClockTargetsMs };
|
|
17530
|
+
}
|
|
17531
|
+
function unsignedString(value, field, required = false) {
|
|
17532
|
+
if (value === null || value === undefined) {
|
|
17533
|
+
if (required)
|
|
17534
|
+
throw new CryptoHftDataError(`schema_${field}_missing`);
|
|
17535
|
+
return;
|
|
17536
|
+
}
|
|
17537
|
+
const rendered = String(value);
|
|
17538
|
+
if (!/^\d+$/.test(rendered)) {
|
|
17539
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17540
|
+
}
|
|
17541
|
+
const parsed = BigInt(rendered);
|
|
17542
|
+
if (parsed > 18446744073709551615n) {
|
|
17543
|
+
throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
|
|
17544
|
+
}
|
|
17545
|
+
return parsed.toString(10);
|
|
17546
|
+
}
|
|
17547
|
+
function timestampMs2(value, field) {
|
|
17548
|
+
const rendered = String(value);
|
|
17549
|
+
if (!/^\d+$/.test(rendered)) {
|
|
17550
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17551
|
+
}
|
|
17552
|
+
const raw = BigInt(rendered);
|
|
17553
|
+
const milliseconds = field === "received_time" ? raw / 1000000n : raw;
|
|
17554
|
+
if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
17555
|
+
throw new CryptoHftDataError(`schema_${field}_unsafe`);
|
|
17556
|
+
}
|
|
17557
|
+
return Number(milliseconds);
|
|
17558
|
+
}
|
|
17559
|
+
function decimal(value, field, allowZero) {
|
|
17560
|
+
if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
|
|
17561
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17562
|
+
}
|
|
17563
|
+
const parsed = Number(value);
|
|
17564
|
+
if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
|
|
17565
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17566
|
+
}
|
|
17567
|
+
return parsed;
|
|
17568
|
+
}
|
|
17569
|
+
function validatedRow(request2, row) {
|
|
17570
|
+
if (row.symbol !== request2.scope.sourceSymbol) {
|
|
17571
|
+
throw new CryptoHftDataError("schema_symbol_mismatch");
|
|
17572
|
+
}
|
|
17573
|
+
if (row.event_type !== "snapshot" && row.event_type !== "update") {
|
|
17574
|
+
throw new CryptoHftDataError("schema_event_type_invalid");
|
|
17575
|
+
}
|
|
17576
|
+
if (row.side !== "bid" && row.side !== "ask") {
|
|
17577
|
+
throw new CryptoHftDataError("schema_side_invalid");
|
|
17578
|
+
}
|
|
17579
|
+
decimal(row.price, "price", false);
|
|
17580
|
+
decimal(row.quantity, "quantity", true);
|
|
17581
|
+
if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
|
|
17582
|
+
throw new CryptoHftDataError("schema_object_checksum_invalid");
|
|
17583
|
+
}
|
|
17584
|
+
const eventTimeMs = timestampMs2(row.event_time, "event_time");
|
|
17585
|
+
const receivedTimeMs = timestampMs2(row.received_time, "received_time");
|
|
17586
|
+
if (receivedTimeMs < eventTimeMs) {
|
|
17587
|
+
throw new CryptoHftDataError("received_time_precedes_event_time");
|
|
17588
|
+
}
|
|
17589
|
+
return { ...row, eventTimeMs, receivedTimeMs };
|
|
17590
|
+
}
|
|
17591
|
+
function groupKey(row, profile) {
|
|
17592
|
+
const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
|
|
17593
|
+
return [
|
|
17594
|
+
row.event_type,
|
|
17595
|
+
row.eventTimeMs,
|
|
17596
|
+
sequence,
|
|
17597
|
+
row.dataset_object_identity
|
|
17598
|
+
].join("\x00");
|
|
17599
|
+
}
|
|
17600
|
+
function absent(value) {
|
|
17601
|
+
return value === null || value === undefined;
|
|
17602
|
+
}
|
|
17603
|
+
function snapshotSequence(row, profile) {
|
|
17604
|
+
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
17605
|
+
if (String(row.last_update_id) !== "-1") {
|
|
17606
|
+
throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
|
|
18144
17607
|
}
|
|
18145
|
-
|
|
17608
|
+
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
|
|
17609
|
+
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
17610
|
+
}
|
|
17611
|
+
return unsignedString(row.final_update_id, "final_update_id", true);
|
|
17612
|
+
}
|
|
17613
|
+
return unsignedString(row.last_update_id, "last_update_id", true);
|
|
18146
17614
|
}
|
|
18147
|
-
|
|
18148
|
-
|
|
18149
|
-
|
|
18150
|
-
|
|
18151
|
-
|
|
18152
|
-
|
|
18153
|
-
|
|
18154
|
-
|
|
18155
|
-
|
|
18156
|
-
|
|
18157
|
-
|
|
18158
|
-
marketType: "spot",
|
|
18159
|
-
providerExchangeId: "binance_spot",
|
|
18160
|
-
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
18161
|
-
maxDepth: 500,
|
|
18162
|
-
eventTimeUnit: "milliseconds",
|
|
18163
|
-
receivedTimeUnit: "nanoseconds",
|
|
18164
|
-
snapshotGrouping: "event_time_last_update_id_object",
|
|
18165
|
-
sequenceSemantics: "binance_u_U_pu",
|
|
18166
|
-
constructionModes: ["sampled_top_n_snapshot"],
|
|
18167
|
-
sourcePolicies: ["authoritative_window"]
|
|
18168
|
-
});
|
|
18169
|
-
var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
|
|
18170
|
-
profileId: "cryptohftdata/okx_spot/ARB-USDT/v1",
|
|
18171
|
-
exchange: "okx",
|
|
18172
|
-
tradingPair: "ARB-USDT",
|
|
18173
|
-
sourceSymbol: "ARB-USDT",
|
|
18174
|
-
marketType: "spot",
|
|
18175
|
-
providerExchangeId: "okx_spot",
|
|
18176
|
-
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
18177
|
-
maxDepth: 400,
|
|
18178
|
-
eventTimeUnit: "milliseconds",
|
|
18179
|
-
receivedTimeUnit: "nanoseconds",
|
|
18180
|
-
snapshotGrouping: "event_time_final_update_id_object",
|
|
18181
|
-
sequenceSemantics: "okx_seq_id_prev_seq_id",
|
|
18182
|
-
constructionModes: ["sampled_top_n_snapshot"],
|
|
18183
|
-
sourcePolicies: ["authoritative_window"]
|
|
18184
|
-
});
|
|
18185
|
-
|
|
18186
|
-
class CryptoHftDataError extends Error {
|
|
18187
|
-
reason;
|
|
18188
|
-
constructor(reason) {
|
|
18189
|
-
super(`CryptoHFTData backfill failed: ${reason}`);
|
|
18190
|
-
this.reason = reason;
|
|
18191
|
-
this.name = "CryptoHftDataError";
|
|
17615
|
+
function sortedSide(levels, side) {
|
|
17616
|
+
return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
|
|
17617
|
+
}
|
|
17618
|
+
function applyRows(state, rows) {
|
|
17619
|
+
for (const row of rows) {
|
|
17620
|
+
const levels = row.side === "bid" ? state.bids : state.asks;
|
|
17621
|
+
const quantity = decimal(row.quantity, "quantity", true);
|
|
17622
|
+
if (quantity === 0)
|
|
17623
|
+
levels.delete(row.price);
|
|
17624
|
+
else
|
|
17625
|
+
levels.set(row.price, quantity);
|
|
18192
17626
|
}
|
|
18193
17627
|
}
|
|
18194
|
-
function
|
|
18195
|
-
|
|
18196
|
-
|
|
17628
|
+
function reconstructCryptoHftDataOrderBooks(request2, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
|
|
17629
|
+
const rows = inputRows.map((row, index) => ({
|
|
17630
|
+
...validatedRow(request2, row),
|
|
17631
|
+
originalIndex: index
|
|
17632
|
+
})).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
|
|
17633
|
+
const groups = [];
|
|
17634
|
+
for (const row of rows) {
|
|
17635
|
+
const current = groups.at(-1);
|
|
17636
|
+
if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
|
|
17637
|
+
groups.push([row]);
|
|
17638
|
+
} else {
|
|
17639
|
+
current.push(row);
|
|
17640
|
+
}
|
|
17641
|
+
}
|
|
17642
|
+
const earliestTargetTimeMs = Math.min(...request2.requiredClockTargetsMs);
|
|
17643
|
+
const latestTargetTimeMs = Math.max(...request2.requiredClockTargetsMs);
|
|
17644
|
+
const anchorIndex = groups.findIndex((group) => {
|
|
17645
|
+
const first = group[0];
|
|
17646
|
+
return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
|
|
17647
|
+
});
|
|
17648
|
+
if (anchorIndex < 0) {
|
|
17649
|
+
throw new CryptoHftDataError("update_before_snapshot");
|
|
17650
|
+
}
|
|
17651
|
+
let state;
|
|
17652
|
+
let previousFinalUpdateId;
|
|
17653
|
+
const states = [];
|
|
17654
|
+
for (const group of groups.slice(anchorIndex)) {
|
|
17655
|
+
const first = group[0];
|
|
17656
|
+
if (first.eventTimeMs > latestTargetTimeMs)
|
|
17657
|
+
break;
|
|
17658
|
+
if (first.event_type === "snapshot") {
|
|
17659
|
+
const sequence = snapshotSequence(first, profile);
|
|
17660
|
+
for (const row of group) {
|
|
17661
|
+
if (snapshotSequence(row, profile) !== sequence) {
|
|
17662
|
+
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
17663
|
+
}
|
|
17664
|
+
}
|
|
17665
|
+
if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
|
|
17666
|
+
throw new CryptoHftDataError("snapshot_sequence_regression");
|
|
17667
|
+
}
|
|
17668
|
+
state = {
|
|
17669
|
+
bids: new Map,
|
|
17670
|
+
asks: new Map,
|
|
17671
|
+
sequence,
|
|
17672
|
+
sourceTimeMs: first.eventTimeMs,
|
|
17673
|
+
receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
|
|
17674
|
+
datasetObjectIdentity: first.dataset_object_identity,
|
|
17675
|
+
datasetObjectChecksum: first.dataset_object_checksum
|
|
17676
|
+
};
|
|
17677
|
+
applyRows(state, group);
|
|
17678
|
+
previousFinalUpdateId = BigInt(sequence);
|
|
17679
|
+
} else {
|
|
17680
|
+
if (!state || previousFinalUpdateId === undefined) {
|
|
17681
|
+
throw new CryptoHftDataError("update_before_snapshot");
|
|
17682
|
+
}
|
|
17683
|
+
const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
|
|
17684
|
+
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
17685
|
+
if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
|
|
17686
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
17687
|
+
}
|
|
17688
|
+
const previous = unsignedString(first.last_update_id, "last_update_id", true);
|
|
17689
|
+
for (const row of group) {
|
|
17690
|
+
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
|
|
17691
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
17692
|
+
}
|
|
17693
|
+
}
|
|
17694
|
+
if (BigInt(previous) !== previousFinalUpdateId) {
|
|
17695
|
+
throw new CryptoHftDataError("update_chain_gap");
|
|
17696
|
+
}
|
|
17697
|
+
} else {
|
|
17698
|
+
const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
|
|
17699
|
+
const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
|
|
17700
|
+
for (const row of group) {
|
|
17701
|
+
if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
|
|
17702
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
17703
|
+
}
|
|
17704
|
+
}
|
|
17705
|
+
const expected = previousFinalUpdateId + 1n;
|
|
17706
|
+
if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
|
|
17707
|
+
throw new CryptoHftDataError("update_chain_gap");
|
|
17708
|
+
}
|
|
17709
|
+
}
|
|
17710
|
+
applyRows(state, group);
|
|
17711
|
+
state.sequence = finalUpdate.toString();
|
|
17712
|
+
state.sourceTimeMs = first.eventTimeMs;
|
|
17713
|
+
state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
|
|
17714
|
+
state.datasetObjectIdentity = first.dataset_object_identity;
|
|
17715
|
+
state.datasetObjectChecksum = first.dataset_object_checksum;
|
|
17716
|
+
previousFinalUpdateId = finalUpdate;
|
|
17717
|
+
}
|
|
17718
|
+
if (state) {
|
|
17719
|
+
states.push({
|
|
17720
|
+
...state,
|
|
17721
|
+
bids: new Map(state.bids),
|
|
17722
|
+
asks: new Map(state.asks)
|
|
17723
|
+
});
|
|
17724
|
+
}
|
|
17725
|
+
}
|
|
17726
|
+
const samples = [];
|
|
17727
|
+
for (const targetTimeMs of request2.requiredClockTargetsMs) {
|
|
17728
|
+
let prior;
|
|
17729
|
+
for (const candidate of states) {
|
|
17730
|
+
if (candidate.sourceTimeMs > targetTimeMs)
|
|
17731
|
+
break;
|
|
17732
|
+
prior = candidate;
|
|
17733
|
+
}
|
|
17734
|
+
if (!prior || targetTimeMs - prior.sourceTimeMs > request2.maxPriorAsOfLagMs) {
|
|
17735
|
+
throw new CryptoHftDataError("required_clock_coverage_insufficient");
|
|
17736
|
+
}
|
|
17737
|
+
const bids = sortedSide(prior.bids, "bid").slice(0, request2.depth);
|
|
17738
|
+
const asks = sortedSide(prior.asks, "ask").slice(0, request2.depth);
|
|
17739
|
+
if (bids.length === 0 || asks.length === 0) {
|
|
17740
|
+
throw new CryptoHftDataError("book_side_missing");
|
|
17741
|
+
}
|
|
17742
|
+
if (bids[0]?.[0] >= asks[0]?.[0]) {
|
|
17743
|
+
throw new CryptoHftDataError("book_crossed_or_locked");
|
|
17744
|
+
}
|
|
17745
|
+
samples.push({
|
|
17746
|
+
targetTimeMs,
|
|
17747
|
+
sourceTimeMs: prior.sourceTimeMs,
|
|
17748
|
+
receivedTimeMs: prior.receivedTimeMs,
|
|
17749
|
+
sequence: prior.sequence,
|
|
17750
|
+
bids,
|
|
17751
|
+
asks,
|
|
17752
|
+
datasetObjectIdentity: prior.datasetObjectIdentity,
|
|
17753
|
+
datasetObjectChecksum: prior.datasetObjectChecksum
|
|
17754
|
+
});
|
|
18197
17755
|
}
|
|
18198
|
-
|
|
18199
|
-
if (!profile)
|
|
18200
|
-
return;
|
|
18201
|
-
return {
|
|
18202
|
-
provider: "cryptohftdata",
|
|
18203
|
-
adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
|
|
18204
|
-
providerExchangeId: profile.providerExchangeId,
|
|
18205
|
-
resolvedSymbol: profile.sourceSymbol
|
|
18206
|
-
};
|
|
17756
|
+
return samples;
|
|
18207
17757
|
}
|
|
18208
|
-
function
|
|
18209
|
-
|
|
17758
|
+
function sha256Bytes(bytes) {
|
|
17759
|
+
return createHash3("sha256").update(bytes).digest("hex");
|
|
17760
|
+
}
|
|
17761
|
+
async function decodeCryptoHftParquetZstd(bytes) {
|
|
17762
|
+
const parquet = decompress(bytes);
|
|
17763
|
+
const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
|
|
17764
|
+
return parquetReadObjects({ file: buffer });
|
|
17765
|
+
}
|
|
17766
|
+
function parsedDatasetRow(value, object) {
|
|
18210
17767
|
return {
|
|
18211
|
-
|
|
18212
|
-
|
|
17768
|
+
received_time: value.received_time,
|
|
17769
|
+
event_time: value.event_time,
|
|
17770
|
+
transaction_time: value.transaction_time,
|
|
17771
|
+
symbol: value.symbol,
|
|
17772
|
+
event_type: value.event_type,
|
|
17773
|
+
first_update_id: value.first_update_id,
|
|
17774
|
+
final_update_id: value.final_update_id,
|
|
17775
|
+
prev_final_update_id: value.prev_final_update_id,
|
|
17776
|
+
last_update_id: value.last_update_id,
|
|
17777
|
+
side: value.side,
|
|
17778
|
+
price: value.price,
|
|
17779
|
+
quantity: value.quantity,
|
|
17780
|
+
order_count: value.order_count,
|
|
17781
|
+
dataset_object_identity: object.identity,
|
|
17782
|
+
dataset_object_checksum: object.checksum
|
|
18213
17783
|
};
|
|
18214
17784
|
}
|
|
18215
|
-
function
|
|
18216
|
-
|
|
18217
|
-
|
|
18218
|
-
|
|
18219
|
-
|
|
18220
|
-
|
|
17785
|
+
async function readBoundedObject(response, remainingBytes) {
|
|
17786
|
+
if (!response.body)
|
|
17787
|
+
return new Uint8Array;
|
|
17788
|
+
const reader = response.body.getReader();
|
|
17789
|
+
const chunks = [];
|
|
17790
|
+
let total = 0;
|
|
17791
|
+
while (true) {
|
|
17792
|
+
const { done, value } = await reader.read();
|
|
17793
|
+
if (done)
|
|
17794
|
+
break;
|
|
17795
|
+
total += value.byteLength;
|
|
17796
|
+
if (total > remainingBytes) {
|
|
17797
|
+
await reader.cancel();
|
|
17798
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
17799
|
+
}
|
|
17800
|
+
chunks.push(value);
|
|
18221
17801
|
}
|
|
18222
|
-
|
|
17802
|
+
const bytes = new Uint8Array(total);
|
|
17803
|
+
let offset = 0;
|
|
17804
|
+
for (const chunk of chunks) {
|
|
17805
|
+
bytes.set(chunk, offset);
|
|
17806
|
+
offset += chunk.byteLength;
|
|
17807
|
+
}
|
|
17808
|
+
return bytes;
|
|
18223
17809
|
}
|
|
18224
|
-
|
|
18225
|
-
|
|
18226
|
-
|
|
18227
|
-
|
|
18228
|
-
|
|
17810
|
+
|
|
17811
|
+
class CryptoHftDataAdapter {
|
|
17812
|
+
baseUrl;
|
|
17813
|
+
request;
|
|
17814
|
+
nowMs;
|
|
17815
|
+
decode;
|
|
17816
|
+
profiles;
|
|
17817
|
+
constructor(options = {}) {
|
|
17818
|
+
this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
|
|
17819
|
+
this.request = options.fetch ?? fetch;
|
|
17820
|
+
this.nowMs = options.nowMs ?? Date.now;
|
|
17821
|
+
this.decode = options.decode ?? decodeCryptoHftParquetZstd;
|
|
17822
|
+
this.profiles = options.profiles ?? [];
|
|
18229
17823
|
}
|
|
18230
|
-
|
|
18231
|
-
|
|
18232
|
-
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
17824
|
+
capabilityFor(request2) {
|
|
17825
|
+
return cryptoHftDataCapabilityFor(request2, this.profiles);
|
|
18233
17826
|
}
|
|
18234
|
-
|
|
18235
|
-
|
|
18236
|
-
|
|
17827
|
+
async discoverSymbols(providerExchangeId) {
|
|
17828
|
+
if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
|
|
17829
|
+
throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
|
|
17830
|
+
}
|
|
17831
|
+
const endpoint = new URL("/symbols", this.baseUrl);
|
|
17832
|
+
endpoint.searchParams.set("exchange", providerExchangeId);
|
|
17833
|
+
endpoint.searchParams.set("data_type", "orderbook");
|
|
17834
|
+
let response;
|
|
17835
|
+
try {
|
|
17836
|
+
response = await this.request(endpoint);
|
|
17837
|
+
} catch {
|
|
17838
|
+
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
17839
|
+
}
|
|
17840
|
+
if (!response.ok) {
|
|
17841
|
+
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
17842
|
+
}
|
|
17843
|
+
let body;
|
|
17844
|
+
try {
|
|
17845
|
+
body = await response.json();
|
|
17846
|
+
} catch {
|
|
17847
|
+
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
17848
|
+
}
|
|
17849
|
+
const record = body;
|
|
17850
|
+
if (!body || typeof body !== "object" || Array.isArray(body) || String(record.exchange).toLowerCase() !== providerExchangeId || String(record.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record.symbols) || !record.symbols.every((symbol) => typeof symbol === "string" && /^[A-Z0-9_-]+$/.test(symbol))) {
|
|
17851
|
+
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
17852
|
+
}
|
|
17853
|
+
return [...new Set(record.symbols)].sort();
|
|
18237
17854
|
}
|
|
18238
|
-
|
|
18239
|
-
|
|
18240
|
-
|
|
18241
|
-
|
|
18242
|
-
|
|
18243
|
-
|
|
17855
|
+
async acquire(request2, capability, credential) {
|
|
17856
|
+
const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
|
|
17857
|
+
if (typeof apiKey !== "string" || apiKey.length === 0) {
|
|
17858
|
+
throw new CryptoHftDataError("credentials_invalid");
|
|
17859
|
+
}
|
|
17860
|
+
const paths = enumerateCryptoHftDataObjects(request2, capability.providerExchangeId, capability.resolvedSymbol);
|
|
17861
|
+
if (paths.length > request2.budgets.maxFiles) {
|
|
17862
|
+
throw new CryptoHftDataError("budget_max_files_exceeded");
|
|
17863
|
+
}
|
|
17864
|
+
const started = this.nowMs();
|
|
17865
|
+
const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
|
|
17866
|
+
method: "POST",
|
|
17867
|
+
headers: { "content-type": "application/json", "X-API-Key": apiKey }
|
|
17868
|
+
});
|
|
17869
|
+
if (!tokenResponse.ok)
|
|
17870
|
+
throw new CryptoHftDataError("jwt_issuance_failed");
|
|
17871
|
+
const tokenBody = await tokenResponse.json();
|
|
17872
|
+
if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
|
|
17873
|
+
throw new CryptoHftDataError("jwt_response_invalid");
|
|
17874
|
+
}
|
|
17875
|
+
let totalBytes = 0;
|
|
17876
|
+
let totalRows = 0;
|
|
17877
|
+
const objects = [];
|
|
17878
|
+
const rows = [];
|
|
17879
|
+
for (const path of paths) {
|
|
17880
|
+
if (this.nowMs() - started > request2.budgets.maxDurationMs) {
|
|
17881
|
+
throw new CryptoHftDataError("budget_max_duration_exceeded");
|
|
17882
|
+
}
|
|
17883
|
+
const endpoint = new URL("/download", this.baseUrl);
|
|
17884
|
+
endpoint.searchParams.set("file", path);
|
|
17885
|
+
const response = await this.request(endpoint, {
|
|
17886
|
+
headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
|
|
17887
|
+
});
|
|
17888
|
+
if (!response.ok)
|
|
17889
|
+
throw new CryptoHftDataError("object_download_failed");
|
|
17890
|
+
const declaredBytes = Number(response.headers.get("content-length"));
|
|
17891
|
+
if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request2.budgets.maxBytes) {
|
|
17892
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
17893
|
+
}
|
|
17894
|
+
const bytes = await readBoundedObject(response, request2.budgets.maxBytes - totalBytes);
|
|
17895
|
+
totalBytes += bytes.byteLength;
|
|
17896
|
+
if (totalBytes > request2.budgets.maxBytes) {
|
|
17897
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
17898
|
+
}
|
|
17899
|
+
const decoded = await this.decode(bytes);
|
|
17900
|
+
totalRows += decoded.length;
|
|
17901
|
+
if (totalRows > request2.budgets.maxRows) {
|
|
17902
|
+
throw new CryptoHftDataError("budget_max_rows_exceeded");
|
|
17903
|
+
}
|
|
17904
|
+
const object = {
|
|
17905
|
+
identity: path,
|
|
17906
|
+
checksum: sha256Bytes(bytes),
|
|
17907
|
+
bytes: bytes.byteLength,
|
|
17908
|
+
rows: decoded.length
|
|
17909
|
+
};
|
|
17910
|
+
objects.push(object);
|
|
17911
|
+
for (const decodedRow of decoded) {
|
|
17912
|
+
const parsed = parsedDatasetRow(decodedRow, object);
|
|
17913
|
+
validatedRow(request2, parsed);
|
|
17914
|
+
rows.push(parsed);
|
|
17915
|
+
}
|
|
17916
|
+
}
|
|
17917
|
+
if (this.nowMs() - started > request2.budgets.maxDurationMs) {
|
|
17918
|
+
throw new CryptoHftDataError("budget_max_duration_exceeded");
|
|
17919
|
+
}
|
|
17920
|
+
return {
|
|
17921
|
+
objects,
|
|
17922
|
+
rows,
|
|
17923
|
+
vendorSemanticDigest: sha256Canonical(rows)
|
|
17924
|
+
};
|
|
18244
17925
|
}
|
|
18245
|
-
|
|
18246
|
-
|
|
18247
|
-
|
|
18248
|
-
|
|
17926
|
+
async normalize(request2, capability, dataset, captureBundleId2) {
|
|
17927
|
+
const profile = this.profiles.find((candidate) => candidate.exchange === request2.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request2.scope.tradingPair && candidate.sourceSymbol === request2.scope.sourceSymbol && candidate.marketType === request2.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
|
|
17928
|
+
if (!profile) {
|
|
17929
|
+
throw new CryptoHftDataError("profile_semantics_unavailable");
|
|
17930
|
+
}
|
|
17931
|
+
const samples = reconstructCryptoHftDataOrderBooks(request2, dataset.rows, profile);
|
|
17932
|
+
const context2 = {
|
|
17933
|
+
source: EXTERNAL_BACKFILL_SOURCE,
|
|
17934
|
+
deploymentId: "market-data-vendor-backfill",
|
|
17935
|
+
captureBundleId: captureBundleId2,
|
|
17936
|
+
exchange: request2.scope.exchange,
|
|
17937
|
+
symbol: request2.scope.tradingPair,
|
|
17938
|
+
tradingPair: request2.scope.tradingPair,
|
|
17939
|
+
sourceSymbol: request2.scope.sourceSymbol,
|
|
17940
|
+
assetType: request2.scope.marketType,
|
|
17941
|
+
feed: "ORDERBOOK",
|
|
17942
|
+
provider: "cryptohftdata",
|
|
17943
|
+
sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
|
|
17944
|
+
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
17945
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
17946
|
+
provenanceComplete: true
|
|
17947
|
+
};
|
|
17948
|
+
const rows = samples.flatMap((sample) => {
|
|
17949
|
+
const rawCapture = {
|
|
17950
|
+
rawCaptureId: sha256Canonical({
|
|
17951
|
+
capture_bundle_id: captureBundleId2,
|
|
17952
|
+
dataset_object_identity: sample.datasetObjectIdentity,
|
|
17953
|
+
dataset_object_checksum: sample.datasetObjectChecksum,
|
|
17954
|
+
source_time_ms: sample.sourceTimeMs,
|
|
17955
|
+
target_time_ms: sample.targetTimeMs
|
|
17956
|
+
}),
|
|
17957
|
+
rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
|
|
17958
|
+
rawChecksum: sample.datasetObjectChecksum,
|
|
17959
|
+
redactedPayload: {
|
|
17960
|
+
dataset_object_identity: sample.datasetObjectIdentity,
|
|
17961
|
+
dataset_object_checksum: sample.datasetObjectChecksum
|
|
17962
|
+
},
|
|
17963
|
+
eventTimeMs: sample.sourceTimeMs,
|
|
17964
|
+
receivedTimeMs: sample.receivedTimeMs,
|
|
17965
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM
|
|
17966
|
+
};
|
|
17967
|
+
const canonical = buildCanonicalOrderBookRows({
|
|
17968
|
+
context: context2,
|
|
17969
|
+
rawCapture,
|
|
17970
|
+
depthLimit: request2.depth,
|
|
17971
|
+
constructionMode: request2.constructionMode,
|
|
17972
|
+
snapshot: {
|
|
17973
|
+
bids: sample.bids,
|
|
17974
|
+
asks: sample.asks,
|
|
17975
|
+
timestamp: sample.sourceTimeMs,
|
|
17976
|
+
receivedTimestamp: sample.receivedTimeMs,
|
|
17977
|
+
exchange: request2.scope.exchange,
|
|
17978
|
+
symbol: request2.scope.tradingPair,
|
|
17979
|
+
depthLimit: request2.depth,
|
|
17980
|
+
sequence: sample.sequence
|
|
17981
|
+
}
|
|
17982
|
+
});
|
|
17983
|
+
return [...canonical.levels, canonical.summary];
|
|
17984
|
+
});
|
|
17985
|
+
return {
|
|
17986
|
+
captureBundleId: captureBundleId2,
|
|
17987
|
+
objects: dataset.objects,
|
|
17988
|
+
rows,
|
|
17989
|
+
vendorSemanticDigest: dataset.vendorSemanticDigest,
|
|
17990
|
+
canonicalSemanticDigest: semanticDigest(rows)
|
|
17991
|
+
};
|
|
18249
17992
|
}
|
|
18250
|
-
return Number(milliseconds);
|
|
18251
17993
|
}
|
|
18252
|
-
|
|
18253
|
-
|
|
18254
|
-
|
|
18255
|
-
|
|
18256
|
-
|
|
18257
|
-
|
|
18258
|
-
|
|
17994
|
+
|
|
17995
|
+
// src/helpers/market-data-vendor-backfill/qualification.ts
|
|
17996
|
+
var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
17997
|
+
var SHA256 = /^[0-9a-f]{64}$/;
|
|
17998
|
+
var REASON = /^[a-z][a-z0-9_]{0,127}$/;
|
|
17999
|
+
var EVENT_FIELDS = new Set([
|
|
18000
|
+
"qualification_event_id",
|
|
18001
|
+
"capture_bundle_id",
|
|
18002
|
+
"state",
|
|
18003
|
+
"receipt_id",
|
|
18004
|
+
"promotion_identity_sha256",
|
|
18005
|
+
"window",
|
|
18006
|
+
"event_at",
|
|
18007
|
+
"reason_code"
|
|
18008
|
+
]);
|
|
18009
|
+
function deterministicUuid(value) {
|
|
18010
|
+
const digest = jcsSha256(value).slice(0, 32).split("");
|
|
18011
|
+
digest[12] = "5";
|
|
18012
|
+
digest[16] = "8";
|
|
18013
|
+
const hex = digest.join("");
|
|
18014
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
18015
|
+
}
|
|
18016
|
+
function semanticEvent(event) {
|
|
18017
|
+
const { qualification_event_id: _eventId, ...semantic } = event;
|
|
18018
|
+
return semantic;
|
|
18019
|
+
}
|
|
18020
|
+
function assertTimestamp(value, field) {
|
|
18021
|
+
const parsed = Date.parse(value);
|
|
18022
|
+
if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
|
|
18023
|
+
throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
|
|
18259
18024
|
}
|
|
18260
18025
|
return parsed;
|
|
18261
18026
|
}
|
|
18262
|
-
function
|
|
18263
|
-
if (
|
|
18264
|
-
throw new
|
|
18265
|
-
}
|
|
18266
|
-
if (row.event_type !== "snapshot" && row.event_type !== "update") {
|
|
18267
|
-
throw new CryptoHftDataError("schema_event_type_invalid");
|
|
18027
|
+
function parseQualificationEvent(value) {
|
|
18028
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
18029
|
+
throw new Error("qualification event must be an object");
|
|
18268
18030
|
}
|
|
18269
|
-
|
|
18270
|
-
|
|
18031
|
+
const event = value;
|
|
18032
|
+
if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
|
|
18033
|
+
throw new Error("qualification event fields are invalid");
|
|
18271
18034
|
}
|
|
18272
|
-
|
|
18273
|
-
|
|
18274
|
-
if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
|
|
18275
|
-
throw new CryptoHftDataError("schema_object_checksum_invalid");
|
|
18035
|
+
if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
|
|
18036
|
+
throw new Error("qualification window must be increasing");
|
|
18276
18037
|
}
|
|
18277
|
-
|
|
18278
|
-
|
|
18279
|
-
|
|
18280
|
-
throw new CryptoHftDataError("received_time_precedes_event_time");
|
|
18038
|
+
assertTimestamp(event.event_at, "event_at");
|
|
18039
|
+
if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
|
|
18040
|
+
throw new Error("qualification_event_id does not match event content");
|
|
18281
18041
|
}
|
|
18282
|
-
return
|
|
18283
|
-
}
|
|
18284
|
-
function groupKey(row, profile) {
|
|
18285
|
-
const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
|
|
18286
|
-
return [
|
|
18287
|
-
row.event_type,
|
|
18288
|
-
row.eventTimeMs,
|
|
18289
|
-
sequence,
|
|
18290
|
-
row.dataset_object_identity
|
|
18291
|
-
].join("\x00");
|
|
18042
|
+
return event;
|
|
18292
18043
|
}
|
|
18293
|
-
function
|
|
18294
|
-
|
|
18044
|
+
function finalizeQualificationEvent(event) {
|
|
18045
|
+
const semantic = semanticEvent(event);
|
|
18046
|
+
return parseQualificationEvent({
|
|
18047
|
+
...semantic,
|
|
18048
|
+
qualification_event_id: deterministicUuid(semantic)
|
|
18049
|
+
});
|
|
18295
18050
|
}
|
|
18296
|
-
function
|
|
18297
|
-
|
|
18298
|
-
|
|
18299
|
-
|
|
18300
|
-
|
|
18301
|
-
|
|
18302
|
-
|
|
18051
|
+
function qualificationEventToArchiveRow(eventInput) {
|
|
18052
|
+
const event = parseQualificationEvent(eventInput);
|
|
18053
|
+
return {
|
|
18054
|
+
table: "market_data.cex_order_book_capture_qualifications",
|
|
18055
|
+
row: {
|
|
18056
|
+
source: "external_backfill",
|
|
18057
|
+
capture_origin: "vendor_historical_backfill",
|
|
18058
|
+
source_mode: "vendor_historical_backfill_v1",
|
|
18059
|
+
deployment_id: "market-data-vendor-backfill",
|
|
18060
|
+
qualification_event_id: event.qualification_event_id,
|
|
18061
|
+
capture_bundle_id: event.capture_bundle_id,
|
|
18062
|
+
state: event.state,
|
|
18063
|
+
receipt_id: event.receipt_id,
|
|
18064
|
+
promotion_identity_sha256: event.promotion_identity_sha256,
|
|
18065
|
+
window_start_ms: Date.parse(event.window.start_at),
|
|
18066
|
+
window_end_ms: Date.parse(event.window.end_at),
|
|
18067
|
+
event_at_ms: Date.parse(event.event_at),
|
|
18068
|
+
reason_code: event.reason_code,
|
|
18069
|
+
event_json: jcsCanonicalize(event)
|
|
18303
18070
|
}
|
|
18304
|
-
|
|
18305
|
-
}
|
|
18306
|
-
return unsignedString(row.last_update_id, "last_update_id", true);
|
|
18307
|
-
}
|
|
18308
|
-
function sortedSide(levels, side) {
|
|
18309
|
-
return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
|
|
18310
|
-
}
|
|
18311
|
-
function applyRows(state, rows) {
|
|
18312
|
-
for (const row of rows) {
|
|
18313
|
-
const levels = row.side === "bid" ? state.bids : state.asks;
|
|
18314
|
-
const quantity = decimal(row.quantity, "quantity", true);
|
|
18315
|
-
if (quantity === 0)
|
|
18316
|
-
levels.delete(row.price);
|
|
18317
|
-
else
|
|
18318
|
-
levels.set(row.price, quantity);
|
|
18319
|
-
}
|
|
18071
|
+
};
|
|
18320
18072
|
}
|
|
18321
|
-
|
|
18322
|
-
|
|
18323
|
-
|
|
18324
|
-
|
|
18325
|
-
|
|
18326
|
-
|
|
18327
|
-
|
|
18328
|
-
|
|
18329
|
-
|
|
18330
|
-
|
|
18331
|
-
|
|
18332
|
-
|
|
18073
|
+
|
|
18074
|
+
// src/helpers/market-data-vendor-backfill/core.ts
|
|
18075
|
+
function createMarketDataVendorBackfillDependencies(dependencies) {
|
|
18076
|
+
for (const [name, value] of Object.entries({
|
|
18077
|
+
archive: dependencies.archive,
|
|
18078
|
+
providers: dependencies.providers,
|
|
18079
|
+
credentials: dependencies.credentials,
|
|
18080
|
+
forwarder: dependencies.forwarder,
|
|
18081
|
+
clock: dependencies.clock
|
|
18082
|
+
})) {
|
|
18083
|
+
if (!value || typeof value !== "object") {
|
|
18084
|
+
throw new TypeError(`Backfill dependency ${name} is required`);
|
|
18333
18085
|
}
|
|
18334
18086
|
}
|
|
18335
|
-
const
|
|
18336
|
-
|
|
18337
|
-
|
|
18338
|
-
|
|
18339
|
-
|
|
18340
|
-
|
|
18341
|
-
|
|
18342
|
-
|
|
18343
|
-
|
|
18344
|
-
|
|
18345
|
-
|
|
18346
|
-
|
|
18347
|
-
|
|
18348
|
-
const first = group[0];
|
|
18349
|
-
if (first.eventTimeMs > latestTargetTimeMs)
|
|
18350
|
-
break;
|
|
18351
|
-
if (first.event_type === "snapshot") {
|
|
18352
|
-
const sequence = snapshotSequence(first, profile);
|
|
18353
|
-
for (const row of group) {
|
|
18354
|
-
if (snapshotSequence(row, profile) !== sequence) {
|
|
18355
|
-
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
18356
|
-
}
|
|
18357
|
-
}
|
|
18358
|
-
if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
|
|
18359
|
-
throw new CryptoHftDataError("snapshot_sequence_regression");
|
|
18360
|
-
}
|
|
18361
|
-
state = {
|
|
18362
|
-
bids: new Map,
|
|
18363
|
-
asks: new Map,
|
|
18364
|
-
sequence,
|
|
18365
|
-
sourceTimeMs: first.eventTimeMs,
|
|
18366
|
-
receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
|
|
18367
|
-
datasetObjectIdentity: first.dataset_object_identity,
|
|
18368
|
-
datasetObjectChecksum: first.dataset_object_checksum
|
|
18369
|
-
};
|
|
18370
|
-
applyRows(state, group);
|
|
18371
|
-
previousFinalUpdateId = BigInt(sequence);
|
|
18372
|
-
} else {
|
|
18373
|
-
if (!state || previousFinalUpdateId === undefined) {
|
|
18374
|
-
throw new CryptoHftDataError("update_before_snapshot");
|
|
18375
|
-
}
|
|
18376
|
-
const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
|
|
18377
|
-
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
18378
|
-
if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
|
|
18379
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
18380
|
-
}
|
|
18381
|
-
const previous = unsignedString(first.last_update_id, "last_update_id", true);
|
|
18382
|
-
for (const row of group) {
|
|
18383
|
-
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
|
|
18384
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
18385
|
-
}
|
|
18386
|
-
}
|
|
18387
|
-
if (BigInt(previous) !== previousFinalUpdateId) {
|
|
18388
|
-
throw new CryptoHftDataError("update_chain_gap");
|
|
18389
|
-
}
|
|
18390
|
-
} else {
|
|
18391
|
-
const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
|
|
18392
|
-
const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
|
|
18393
|
-
for (const row of group) {
|
|
18394
|
-
if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
|
|
18395
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
18396
|
-
}
|
|
18397
|
-
}
|
|
18398
|
-
const expected = previousFinalUpdateId + 1n;
|
|
18399
|
-
if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
|
|
18400
|
-
throw new CryptoHftDataError("update_chain_gap");
|
|
18401
|
-
}
|
|
18402
|
-
}
|
|
18403
|
-
applyRows(state, group);
|
|
18404
|
-
state.sequence = finalUpdate.toString();
|
|
18405
|
-
state.sourceTimeMs = first.eventTimeMs;
|
|
18406
|
-
state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
|
|
18407
|
-
state.datasetObjectIdentity = first.dataset_object_identity;
|
|
18408
|
-
state.datasetObjectChecksum = first.dataset_object_checksum;
|
|
18409
|
-
previousFinalUpdateId = finalUpdate;
|
|
18410
|
-
}
|
|
18411
|
-
if (state) {
|
|
18412
|
-
states.push({
|
|
18413
|
-
...state,
|
|
18414
|
-
bids: new Map(state.bids),
|
|
18415
|
-
asks: new Map(state.asks)
|
|
18416
|
-
});
|
|
18087
|
+
for (const [name, method] of Object.entries({
|
|
18088
|
+
"archive.resolveSelection": dependencies.archive.resolveSelection,
|
|
18089
|
+
"archive.verifyCandidate": dependencies.archive.verifyCandidate,
|
|
18090
|
+
"providers.capabilityFor": dependencies.providers.capabilityFor,
|
|
18091
|
+
"providers.acquire": dependencies.providers.acquire,
|
|
18092
|
+
"providers.normalize": dependencies.providers.normalize,
|
|
18093
|
+
"credentials.resolve": dependencies.credentials.resolve,
|
|
18094
|
+
"forwarder.preflight": dependencies.forwarder.preflight,
|
|
18095
|
+
"forwarder.submit": dependencies.forwarder.submit,
|
|
18096
|
+
"clock.nowMs": dependencies.clock.nowMs
|
|
18097
|
+
})) {
|
|
18098
|
+
if (typeof method !== "function") {
|
|
18099
|
+
throw new TypeError(`Backfill dependency ${name} is required`);
|
|
18417
18100
|
}
|
|
18418
18101
|
}
|
|
18419
|
-
|
|
18420
|
-
|
|
18421
|
-
|
|
18422
|
-
|
|
18423
|
-
|
|
18102
|
+
return dependencies;
|
|
18103
|
+
}
|
|
18104
|
+
function outcome(request2, status, reasonCode, extra = {}) {
|
|
18105
|
+
return {
|
|
18106
|
+
status,
|
|
18107
|
+
reasonCode,
|
|
18108
|
+
...request2 ? {
|
|
18109
|
+
requestId: request2.requestId,
|
|
18110
|
+
idempotencyKey: request2.idempotencyKey,
|
|
18111
|
+
...request2.target ? { target: request2.target } : {}
|
|
18112
|
+
} : {},
|
|
18113
|
+
...extra
|
|
18114
|
+
};
|
|
18115
|
+
}
|
|
18116
|
+
function captureBundleId2(request2, capability, dataset) {
|
|
18117
|
+
return sha256Canonical({
|
|
18118
|
+
request_business_identity: request2.idempotencyKey,
|
|
18119
|
+
provider: capability.provider,
|
|
18120
|
+
provider_exchange_id: capability.providerExchangeId,
|
|
18121
|
+
resolved_symbol: capability.resolvedSymbol,
|
|
18122
|
+
adapter_version: capability.adapterVersion,
|
|
18123
|
+
objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
|
|
18124
|
+
identity,
|
|
18125
|
+
checksum,
|
|
18126
|
+
bytes,
|
|
18127
|
+
rows
|
|
18128
|
+
})),
|
|
18129
|
+
canonical_schema_version: request2.expectedProduct.canonicalSchemaVersion,
|
|
18130
|
+
checksum_algorithm: request2.expectedProduct.checksumAlgorithm
|
|
18131
|
+
});
|
|
18132
|
+
}
|
|
18133
|
+
function buildPromotionReceipt(request2, capability, normalized, verification, verificationTimeMs) {
|
|
18134
|
+
if (!request2.wire || !request2.initialSelection || !request2.expectedCanonicalSchema || !request2.coveragePolicy || !request2.productPins) {
|
|
18135
|
+
throw new Error("decoded final-v1 request context is missing");
|
|
18136
|
+
}
|
|
18137
|
+
return finalizePromotionReceipt({
|
|
18138
|
+
schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
|
|
18139
|
+
verified_at: new Date(verificationTimeMs).toISOString(),
|
|
18140
|
+
request_id: request2.requestId,
|
|
18141
|
+
idempotency_key: request2.idempotencyKey,
|
|
18142
|
+
source: EXTERNAL_BACKFILL_SOURCE,
|
|
18143
|
+
capture_origin: "vendor_historical_backfill",
|
|
18144
|
+
source_mode: "vendor_historical_backfill_v1",
|
|
18145
|
+
provider: capability.provider,
|
|
18146
|
+
adapter_version: capability.adapterVersion,
|
|
18147
|
+
effective_policies: {
|
|
18148
|
+
capability_policy: request2.productPins.capability_policy,
|
|
18149
|
+
resource_policy: request2.productPins.resource_policy,
|
|
18150
|
+
adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
|
|
18151
|
+
acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
|
|
18152
|
+
},
|
|
18153
|
+
capture_bundle_id: normalized.captureBundleId,
|
|
18154
|
+
scope: request2.wire.scope,
|
|
18155
|
+
window: request2.wire.window,
|
|
18156
|
+
depth: request2.depth,
|
|
18157
|
+
construction_mode: request2.constructionMode,
|
|
18158
|
+
canonical_schema: request2.expectedCanonicalSchema,
|
|
18159
|
+
coverage_policy: request2.coveragePolicy,
|
|
18160
|
+
selection_sha256: request2.initialSelection.selection_sha256,
|
|
18161
|
+
vendor_semantic_digest: normalized.vendorSemanticDigest,
|
|
18162
|
+
canonical_semantic_digest: verification.canonicalSemanticDigest,
|
|
18163
|
+
prefix_digest: verification.prefixDigest,
|
|
18164
|
+
suffix_digest: verification.suffixDigest,
|
|
18165
|
+
seam_verified: true,
|
|
18166
|
+
coverage_verified: true,
|
|
18167
|
+
dataset_objects: normalized.objects
|
|
18168
|
+
});
|
|
18169
|
+
}
|
|
18170
|
+
async function submitAll(dependencies, batches) {
|
|
18171
|
+
const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
|
|
18172
|
+
for (const batch of batches) {
|
|
18173
|
+
let accepted = false;
|
|
18174
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
|
|
18175
|
+
try {
|
|
18176
|
+
const response = await dependencies.forwarder.submit(batch);
|
|
18177
|
+
accepted = response.ok && response.inserted === batch.rows.length;
|
|
18178
|
+
} catch {
|
|
18179
|
+
accepted = false;
|
|
18180
|
+
}
|
|
18181
|
+
if (accepted)
|
|
18424
18182
|
break;
|
|
18425
|
-
|
|
18183
|
+
if (attempt < maxAttempts)
|
|
18184
|
+
await dependencies.retry?.wait?.(attempt);
|
|
18426
18185
|
}
|
|
18427
|
-
if (!
|
|
18428
|
-
|
|
18186
|
+
if (!accepted)
|
|
18187
|
+
return false;
|
|
18188
|
+
}
|
|
18189
|
+
return true;
|
|
18190
|
+
}
|
|
18191
|
+
function stableFailureReason(error, fallback) {
|
|
18192
|
+
if (error && typeof error === "object") {
|
|
18193
|
+
const reason = error.reason;
|
|
18194
|
+
if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
|
|
18195
|
+
return reason;
|
|
18429
18196
|
}
|
|
18430
|
-
|
|
18431
|
-
|
|
18432
|
-
|
|
18433
|
-
|
|
18197
|
+
}
|
|
18198
|
+
return fallback;
|
|
18199
|
+
}
|
|
18200
|
+
function assertArchivePreflight(request2, resolution) {
|
|
18201
|
+
if (!request2.target || !request2.productionAuthorizationId) {
|
|
18202
|
+
throw new Error("request target or production authorization ID is missing");
|
|
18203
|
+
}
|
|
18204
|
+
const selection = archiveSelectionCodec.decode(resolution.selection);
|
|
18205
|
+
const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
|
|
18206
|
+
const receiptById = new Map;
|
|
18207
|
+
for (const receipt of receipts) {
|
|
18208
|
+
const existing = receiptById.get(receipt.receipt_id);
|
|
18209
|
+
if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
|
|
18210
|
+
throw new Error("stored receipt identity has conflicting content");
|
|
18434
18211
|
}
|
|
18435
|
-
|
|
18436
|
-
|
|
18212
|
+
receiptById.set(receipt.receipt_id, receipt);
|
|
18213
|
+
}
|
|
18214
|
+
for (const bundle of selection.bundles) {
|
|
18215
|
+
if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
|
|
18216
|
+
throw new Error("vendor selection lacks its validated stored receipt");
|
|
18437
18217
|
}
|
|
18438
|
-
samples.push({
|
|
18439
|
-
targetTimeMs,
|
|
18440
|
-
sourceTimeMs: prior.sourceTimeMs,
|
|
18441
|
-
receivedTimeMs: prior.receivedTimeMs,
|
|
18442
|
-
sequence: prior.sequence,
|
|
18443
|
-
bids,
|
|
18444
|
-
asks,
|
|
18445
|
-
datasetObjectIdentity: prior.datasetObjectIdentity,
|
|
18446
|
-
datasetObjectChecksum: prior.datasetObjectChecksum
|
|
18447
|
-
});
|
|
18448
18218
|
}
|
|
18449
|
-
|
|
18219
|
+
if (resolution.readerIdentity.environment !== request2.target.environment || resolution.readerIdentity.cluster !== request2.target.cluster) {
|
|
18220
|
+
throw new Error("archive reader cluster identity mismatch");
|
|
18221
|
+
}
|
|
18450
18222
|
}
|
|
18451
|
-
function
|
|
18452
|
-
|
|
18223
|
+
function assertForwarderPreflight(request2, resolution, nowMs) {
|
|
18224
|
+
if (!request2.target || !request2.productionAuthorizationId) {
|
|
18225
|
+
throw new Error("request target or production authorization ID is missing");
|
|
18226
|
+
}
|
|
18227
|
+
if (resolution.forwarderIdentity.environment !== request2.target.environment || resolution.forwarderIdentity.cluster !== request2.target.cluster) {
|
|
18228
|
+
throw new Error("archive forwarder cluster identity mismatch");
|
|
18229
|
+
}
|
|
18230
|
+
const authorization = resolution.authorization;
|
|
18231
|
+
const expiresAtMs = Date.parse(authorization.expiresAt);
|
|
18232
|
+
if (authorization.authorizationId !== request2.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request2.target.environment || authorization.cluster !== request2.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
|
|
18233
|
+
throw new Error("production forwarder authorization is invalid");
|
|
18234
|
+
}
|
|
18453
18235
|
}
|
|
18454
|
-
|
|
18455
|
-
|
|
18456
|
-
const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
|
|
18457
|
-
return parquetReadObjects({ file: buffer });
|
|
18236
|
+
function resourcePolicyScopeExceeded(request2) {
|
|
18237
|
+
return request2.depth > RESOURCE_POLICY.request_bounds.max_depth || request2.window.endTimeMs - request2.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request2.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
|
|
18458
18238
|
}
|
|
18459
|
-
function
|
|
18460
|
-
|
|
18461
|
-
|
|
18462
|
-
event_time: value.event_time,
|
|
18463
|
-
transaction_time: value.transaction_time,
|
|
18464
|
-
symbol: value.symbol,
|
|
18465
|
-
event_type: value.event_type,
|
|
18466
|
-
first_update_id: value.first_update_id,
|
|
18467
|
-
final_update_id: value.final_update_id,
|
|
18468
|
-
prev_final_update_id: value.prev_final_update_id,
|
|
18469
|
-
last_update_id: value.last_update_id,
|
|
18470
|
-
side: value.side,
|
|
18471
|
-
price: value.price,
|
|
18472
|
-
quantity: value.quantity,
|
|
18473
|
-
order_count: value.order_count,
|
|
18474
|
-
dataset_object_identity: object.identity,
|
|
18475
|
-
dataset_object_checksum: object.checksum
|
|
18476
|
-
};
|
|
18239
|
+
function storedReceiptForSelection(resolution) {
|
|
18240
|
+
const receiptId = resolution.selection.receipt_ids[0];
|
|
18241
|
+
return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
|
|
18477
18242
|
}
|
|
18478
|
-
async function
|
|
18479
|
-
|
|
18480
|
-
|
|
18481
|
-
|
|
18482
|
-
|
|
18483
|
-
|
|
18484
|
-
|
|
18485
|
-
|
|
18486
|
-
|
|
18487
|
-
|
|
18488
|
-
total += value.byteLength;
|
|
18489
|
-
if (total > remainingBytes) {
|
|
18490
|
-
await reader.cancel();
|
|
18491
|
-
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
18492
|
-
}
|
|
18493
|
-
chunks.push(value);
|
|
18243
|
+
async function runMarketDataVendorBackfill(input, dependencies) {
|
|
18244
|
+
let request2;
|
|
18245
|
+
try {
|
|
18246
|
+
const documents = input;
|
|
18247
|
+
request2 = decodeBackfillRunDocuments({
|
|
18248
|
+
request: documents.request,
|
|
18249
|
+
requiredClock: documents.requiredClock
|
|
18250
|
+
});
|
|
18251
|
+
} catch {
|
|
18252
|
+
return outcome(undefined, "request_invalid", "request_invalid");
|
|
18494
18253
|
}
|
|
18495
|
-
|
|
18496
|
-
|
|
18497
|
-
|
|
18498
|
-
|
|
18499
|
-
|
|
18254
|
+
let initialResolution;
|
|
18255
|
+
try {
|
|
18256
|
+
initialResolution = await dependencies.archive.resolveSelection(request2);
|
|
18257
|
+
assertArchivePreflight(request2, initialResolution);
|
|
18258
|
+
const forwarderPreflight = await dependencies.forwarder.preflight({
|
|
18259
|
+
authorizationId: request2.productionAuthorizationId,
|
|
18260
|
+
target: request2.target
|
|
18261
|
+
});
|
|
18262
|
+
assertForwarderPreflight(request2, forwarderPreflight, dependencies.clock.nowMs());
|
|
18263
|
+
} catch {
|
|
18264
|
+
return outcome(request2, "archive_preflight_failed", "archive_preflight_failed");
|
|
18500
18265
|
}
|
|
18501
|
-
|
|
18502
|
-
|
|
18503
|
-
|
|
18504
|
-
|
|
18505
|
-
|
|
18506
|
-
|
|
18507
|
-
|
|
18508
|
-
|
|
18509
|
-
|
|
18510
|
-
|
|
18511
|
-
|
|
18512
|
-
|
|
18513
|
-
|
|
18514
|
-
|
|
18515
|
-
|
|
18266
|
+
if (initialResolution.selection.coverage_class === "complete") {
|
|
18267
|
+
const receipt2 = storedReceiptForSelection(initialResolution);
|
|
18268
|
+
return outcome(request2, "already_covered", "qualified_coverage_complete", {
|
|
18269
|
+
selection: initialResolution.selection,
|
|
18270
|
+
...receipt2 ? { receipt: receipt2 } : {}
|
|
18271
|
+
});
|
|
18272
|
+
}
|
|
18273
|
+
if (resourcePolicyScopeExceeded(request2)) {
|
|
18274
|
+
return outcome(request2, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
|
|
18275
|
+
}
|
|
18276
|
+
if (request2.sourcePolicy === "fill_gaps" && (request2.productPins?.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request2.productPins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256)) {
|
|
18277
|
+
return outcome(request2, "capability_unsupported", "scope_unsupported", {
|
|
18278
|
+
reasonSubcode: "fill_gaps_requires_capability_policy_v2"
|
|
18279
|
+
});
|
|
18280
|
+
}
|
|
18281
|
+
let capability;
|
|
18282
|
+
try {
|
|
18283
|
+
capability = dependencies.providers.capabilityFor(request2);
|
|
18284
|
+
} catch {
|
|
18285
|
+
return outcome(request2, "capability_unsupported", "capability_probe_failed");
|
|
18286
|
+
}
|
|
18287
|
+
if (!capability) {
|
|
18288
|
+
return outcome(request2, "capability_unsupported", "scope_unsupported");
|
|
18289
|
+
}
|
|
18290
|
+
if (!request2.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
|
|
18291
|
+
return outcome(request2, "capability_unsupported", "adapter_version_unpinned");
|
|
18516
18292
|
}
|
|
18517
|
-
|
|
18518
|
-
|
|
18293
|
+
let credential;
|
|
18294
|
+
try {
|
|
18295
|
+
credential = await dependencies.credentials.resolve(capability.provider);
|
|
18296
|
+
} catch {
|
|
18297
|
+
return outcome(request2, "credentials_missing", "credential_resolution_failed");
|
|
18519
18298
|
}
|
|
18520
|
-
|
|
18521
|
-
|
|
18522
|
-
throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
|
|
18523
|
-
}
|
|
18524
|
-
const endpoint = new URL("/symbols", this.baseUrl);
|
|
18525
|
-
endpoint.searchParams.set("exchange", providerExchangeId);
|
|
18526
|
-
endpoint.searchParams.set("data_type", "orderbook");
|
|
18527
|
-
let response;
|
|
18528
|
-
try {
|
|
18529
|
-
response = await this.request(endpoint);
|
|
18530
|
-
} catch {
|
|
18531
|
-
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
18532
|
-
}
|
|
18533
|
-
if (!response.ok) {
|
|
18534
|
-
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
18535
|
-
}
|
|
18536
|
-
let body;
|
|
18537
|
-
try {
|
|
18538
|
-
body = await response.json();
|
|
18539
|
-
} catch {
|
|
18540
|
-
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
18541
|
-
}
|
|
18542
|
-
const record = body;
|
|
18543
|
-
if (!body || typeof body !== "object" || Array.isArray(body) || String(record.exchange).toLowerCase() !== providerExchangeId || String(record.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record.symbols) || !record.symbols.every((symbol) => typeof symbol === "string" && /^[A-Z0-9_-]+$/.test(symbol))) {
|
|
18544
|
-
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
18545
|
-
}
|
|
18546
|
-
return [...new Set(record.symbols)].sort();
|
|
18299
|
+
if (credential === undefined || credential === null) {
|
|
18300
|
+
return outcome(request2, "credentials_missing", "provider_credentials_missing");
|
|
18547
18301
|
}
|
|
18548
|
-
|
|
18549
|
-
|
|
18550
|
-
|
|
18551
|
-
|
|
18552
|
-
|
|
18553
|
-
const
|
|
18554
|
-
|
|
18555
|
-
|
|
18302
|
+
let dataset;
|
|
18303
|
+
let normalized;
|
|
18304
|
+
try {
|
|
18305
|
+
const acquisitionRequest = providerAcquisitionRequest(request2);
|
|
18306
|
+
dataset = await dependencies.providers.acquire(acquisitionRequest, capability, credential);
|
|
18307
|
+
const bundleId = captureBundleId2(request2, capability, dataset);
|
|
18308
|
+
normalized = await dependencies.providers.normalize(acquisitionRequest, capability, dataset, bundleId);
|
|
18309
|
+
if (normalized.captureBundleId !== bundleId) {
|
|
18310
|
+
return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
18311
|
+
reasonSubcode: "capture_identity_mismatch"
|
|
18312
|
+
});
|
|
18556
18313
|
}
|
|
18557
|
-
|
|
18558
|
-
const
|
|
18559
|
-
|
|
18560
|
-
|
|
18314
|
+
} catch (error) {
|
|
18315
|
+
const reason = stableFailureReason(error, "provider_dataset_invalid");
|
|
18316
|
+
return outcome(request2, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
18317
|
+
reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
|
|
18561
18318
|
});
|
|
18562
|
-
|
|
18563
|
-
|
|
18564
|
-
const
|
|
18565
|
-
|
|
18566
|
-
|
|
18319
|
+
}
|
|
18320
|
+
try {
|
|
18321
|
+
const candidateBatches = buildForwarderBatches({
|
|
18322
|
+
captureBundleId: normalized.captureBundleId,
|
|
18323
|
+
deploymentId: "market-data-vendor-backfill",
|
|
18324
|
+
rows: normalized.rows
|
|
18325
|
+
});
|
|
18326
|
+
if (!await submitAll(dependencies, candidateBatches)) {
|
|
18327
|
+
return outcome(request2, "archive_ingest_failed", "candidate_batch_rejected");
|
|
18567
18328
|
}
|
|
18568
|
-
|
|
18569
|
-
|
|
18570
|
-
|
|
18571
|
-
|
|
18572
|
-
|
|
18573
|
-
|
|
18574
|
-
|
|
18575
|
-
|
|
18576
|
-
|
|
18577
|
-
|
|
18578
|
-
|
|
18579
|
-
|
|
18580
|
-
|
|
18581
|
-
|
|
18582
|
-
|
|
18583
|
-
|
|
18584
|
-
|
|
18585
|
-
|
|
18586
|
-
|
|
18587
|
-
|
|
18588
|
-
|
|
18589
|
-
|
|
18590
|
-
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
18591
|
-
}
|
|
18592
|
-
const decoded = await this.decode(bytes);
|
|
18593
|
-
totalRows += decoded.length;
|
|
18594
|
-
if (totalRows > request2.budgets.maxRows) {
|
|
18595
|
-
throw new CryptoHftDataError("budget_max_rows_exceeded");
|
|
18596
|
-
}
|
|
18597
|
-
const object = {
|
|
18598
|
-
identity: path,
|
|
18599
|
-
checksum: sha256Bytes(bytes),
|
|
18600
|
-
bytes: bytes.byteLength,
|
|
18601
|
-
rows: decoded.length
|
|
18602
|
-
};
|
|
18603
|
-
objects.push(object);
|
|
18604
|
-
for (const decodedRow of decoded) {
|
|
18605
|
-
const parsed = parsedDatasetRow(decodedRow, object);
|
|
18606
|
-
validatedRow(request2, parsed);
|
|
18607
|
-
rows.push(parsed);
|
|
18608
|
-
}
|
|
18329
|
+
} catch {
|
|
18330
|
+
return outcome(request2, "archive_ingest_failed", "candidate_submission_failed");
|
|
18331
|
+
}
|
|
18332
|
+
let verification;
|
|
18333
|
+
try {
|
|
18334
|
+
verification = await dependencies.archive.verifyCandidate(request2, normalized, normalized.captureBundleId, initialResolution);
|
|
18335
|
+
} catch {
|
|
18336
|
+
return outcome(request2, "promotion_verification_failed", "candidate_query_failed");
|
|
18337
|
+
}
|
|
18338
|
+
if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
|
|
18339
|
+
return outcome(request2, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
|
|
18340
|
+
}
|
|
18341
|
+
let receipt;
|
|
18342
|
+
try {
|
|
18343
|
+
receipt = buildPromotionReceipt(request2, capability, normalized, verification, dependencies.clock.nowMs());
|
|
18344
|
+
const [batch] = buildForwarderBatches({
|
|
18345
|
+
captureBundleId: normalized.captureBundleId,
|
|
18346
|
+
deploymentId: "market-data-vendor-backfill",
|
|
18347
|
+
rows: [promotionReceiptToArchiveRow(receipt)]
|
|
18348
|
+
});
|
|
18349
|
+
if (!batch || !await submitAll(dependencies, [batch])) {
|
|
18350
|
+
return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
|
|
18609
18351
|
}
|
|
18610
|
-
|
|
18611
|
-
|
|
18352
|
+
const qualification = finalizeQualificationEvent({
|
|
18353
|
+
capture_bundle_id: receipt.capture_bundle_id,
|
|
18354
|
+
state: "qualified",
|
|
18355
|
+
receipt_id: receipt.receipt_id,
|
|
18356
|
+
promotion_identity_sha256: receipt.promotion_identity_sha256,
|
|
18357
|
+
window: receipt.window,
|
|
18358
|
+
event_at: receipt.verified_at,
|
|
18359
|
+
reason_code: "promotion_verified"
|
|
18360
|
+
});
|
|
18361
|
+
const [qualificationBatch] = buildForwarderBatches({
|
|
18362
|
+
captureBundleId: normalized.captureBundleId,
|
|
18363
|
+
deploymentId: "market-data-vendor-backfill",
|
|
18364
|
+
rows: [qualificationEventToArchiveRow(qualification)]
|
|
18365
|
+
});
|
|
18366
|
+
if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
|
|
18367
|
+
return outcome(request2, "archive_ingest_failed", "qualification_commit_failed", { receipt });
|
|
18612
18368
|
}
|
|
18613
|
-
|
|
18614
|
-
|
|
18615
|
-
rows,
|
|
18616
|
-
vendorSemanticDigest: sha256Canonical(rows)
|
|
18617
|
-
};
|
|
18369
|
+
} catch {
|
|
18370
|
+
return outcome(request2, "archive_ingest_failed", "promotion_commit_failed");
|
|
18618
18371
|
}
|
|
18619
|
-
|
|
18620
|
-
|
|
18621
|
-
|
|
18622
|
-
|
|
18623
|
-
|
|
18624
|
-
|
|
18625
|
-
|
|
18626
|
-
|
|
18372
|
+
let finalResolution;
|
|
18373
|
+
try {
|
|
18374
|
+
finalResolution = await dependencies.archive.resolveSelection(request2);
|
|
18375
|
+
assertArchivePreflight(request2, finalResolution);
|
|
18376
|
+
} catch (error) {
|
|
18377
|
+
return outcome(request2, "promotion_verification_failed", "post_promotion_selection_failed", {
|
|
18378
|
+
receipt,
|
|
18379
|
+
reasonSubcode: stableFailureReason(error, "archive_selection_resolution_failed")
|
|
18380
|
+
});
|
|
18381
|
+
}
|
|
18382
|
+
if (finalResolution.selection.coverage_class !== "complete") {
|
|
18383
|
+
return outcome(request2, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
|
|
18384
|
+
}
|
|
18385
|
+
if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
|
|
18386
|
+
return outcome(request2, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
|
|
18387
|
+
}
|
|
18388
|
+
try {
|
|
18389
|
+
if (!request2.wire)
|
|
18390
|
+
throw new Error("decoded request wire is missing");
|
|
18391
|
+
const [selectionBatch] = buildForwarderBatches({
|
|
18392
|
+
captureBundleId: receipt.capture_bundle_id,
|
|
18627
18393
|
deploymentId: "market-data-vendor-backfill",
|
|
18628
|
-
|
|
18629
|
-
|
|
18630
|
-
|
|
18631
|
-
tradingPair: request2.scope.tradingPair,
|
|
18632
|
-
sourceSymbol: request2.scope.sourceSymbol,
|
|
18633
|
-
assetType: request2.scope.marketType,
|
|
18634
|
-
feed: "ORDERBOOK",
|
|
18635
|
-
provider: "cryptohftdata",
|
|
18636
|
-
sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
|
|
18637
|
-
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
18638
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
18639
|
-
provenanceComplete: true
|
|
18640
|
-
};
|
|
18641
|
-
const rows = samples.flatMap((sample) => {
|
|
18642
|
-
const rawCapture = {
|
|
18643
|
-
rawCaptureId: sha256Canonical({
|
|
18644
|
-
capture_bundle_id: captureBundleId3,
|
|
18645
|
-
dataset_object_identity: sample.datasetObjectIdentity,
|
|
18646
|
-
dataset_object_checksum: sample.datasetObjectChecksum,
|
|
18647
|
-
source_time_ms: sample.sourceTimeMs,
|
|
18648
|
-
target_time_ms: sample.targetTimeMs
|
|
18649
|
-
}),
|
|
18650
|
-
rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
|
|
18651
|
-
rawChecksum: sample.datasetObjectChecksum,
|
|
18652
|
-
redactedPayload: {
|
|
18653
|
-
dataset_object_identity: sample.datasetObjectIdentity,
|
|
18654
|
-
dataset_object_checksum: sample.datasetObjectChecksum
|
|
18655
|
-
},
|
|
18656
|
-
eventTimeMs: sample.sourceTimeMs,
|
|
18657
|
-
receivedTimeMs: sample.receivedTimeMs,
|
|
18658
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM
|
|
18659
|
-
};
|
|
18660
|
-
const canonical = buildCanonicalOrderBookRows({
|
|
18661
|
-
context: context2,
|
|
18662
|
-
rawCapture,
|
|
18663
|
-
depthLimit: request2.depth,
|
|
18664
|
-
constructionMode: request2.constructionMode,
|
|
18665
|
-
snapshot: {
|
|
18666
|
-
bids: sample.bids,
|
|
18667
|
-
asks: sample.asks,
|
|
18668
|
-
timestamp: sample.sourceTimeMs,
|
|
18669
|
-
receivedTimestamp: sample.receivedTimeMs,
|
|
18670
|
-
exchange: request2.scope.exchange,
|
|
18671
|
-
symbol: request2.scope.tradingPair,
|
|
18672
|
-
depthLimit: request2.depth,
|
|
18673
|
-
sequence: sample.sequence
|
|
18674
|
-
}
|
|
18675
|
-
});
|
|
18676
|
-
return [...canonical.levels, canonical.summary];
|
|
18394
|
+
rows: [
|
|
18395
|
+
archiveSelectionToArchiveRow(request2.wire, finalResolution.selection)
|
|
18396
|
+
]
|
|
18677
18397
|
});
|
|
18678
|
-
|
|
18679
|
-
|
|
18680
|
-
|
|
18681
|
-
|
|
18682
|
-
|
|
18683
|
-
canonicalSemanticDigest: semanticDigest(rows)
|
|
18684
|
-
};
|
|
18398
|
+
if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
|
|
18399
|
+
return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
18400
|
+
}
|
|
18401
|
+
} catch {
|
|
18402
|
+
return outcome(request2, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
18685
18403
|
}
|
|
18404
|
+
return outcome(request2, "promoted", "promotion_qualified", {
|
|
18405
|
+
receipt,
|
|
18406
|
+
selection: finalResolution.selection
|
|
18407
|
+
});
|
|
18686
18408
|
}
|
|
18687
18409
|
// src/helpers/market-data-vendor-backfill/forwarder-client.ts
|
|
18688
18410
|
class ArchiveForwarderSubmissionError extends Error {
|
|
@@ -18847,4 +18569,4 @@ export {
|
|
|
18847
18569
|
ACQUISITION_POLICY_ID
|
|
18848
18570
|
};
|
|
18849
18571
|
|
|
18850
|
-
//# debugId=
|
|
18572
|
+
//# debugId=E83C0385A80F908D64756E2164756E21
|