@usherlabs/cex-broker 0.2.48 → 0.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cex-canonical-orderbook-export.js +72 -358
- package/dist/commands/cex-canonical-orderbook-export.js.map +7 -19
- package/dist/commands/cli.js +224 -67
- package/dist/commands/market-data-vendor-backfill.js +975 -1198
- package/dist/commands/market-data-vendor-backfill.js.map +13 -25
- package/dist/helpers/logger.d.ts +4 -0
- package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +3 -1
- package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +58 -4
- package/dist/helpers/market-data-vendor-backfill/semantic-verification.d.ts +1 -0
- package/dist/helpers/otel.d.ts +1 -0
- package/dist/helpers/trace-context.d.ts +8 -0
- package/dist/index.js +183 -63
- package/dist/index.js.map +9 -8
- package/dist/market-data-preparation/fixtures/conformance-v2.json +8 -8
- package/dist/market-data-preparation/policies/capability-policy-v1.json +28 -0
- package/dist/market-data-preparation/policies/capability-policy.json +14 -3
- package/dist/market-data-preparation/policies/resource-policy-v1.json +16 -0
- package/dist/market-data-preparation/policies/resource-policy.json +3 -3
- package/dist/market-data-preparation.js +70 -359
- package/dist/market-data-preparation.js.map +9 -21
- package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +39 -0
- package/dist/market-data-vendor-backfill/policies/resource-policy-v2.json +16 -0
- package/dist/market-data-vendor-backfill.d.ts +2 -2
- package/dist/market-data-vendor-backfill.js +1528 -1750
- package/dist/market-data-vendor-backfill.js.map +13 -25
- package/package.json +1 -1
|
@@ -8257,241 +8257,6 @@ var require_src = __commonJS((exports) => {
|
|
|
8257
8257
|
};
|
|
8258
8258
|
});
|
|
8259
8259
|
|
|
8260
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/types/LogRecord.js
|
|
8261
|
-
var require_LogRecord = __commonJS((exports) => {
|
|
8262
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8263
|
-
exports.SeverityNumber = undefined;
|
|
8264
|
-
var SeverityNumber;
|
|
8265
|
-
(function(SeverityNumber2) {
|
|
8266
|
-
SeverityNumber2[SeverityNumber2["UNSPECIFIED"] = 0] = "UNSPECIFIED";
|
|
8267
|
-
SeverityNumber2[SeverityNumber2["TRACE"] = 1] = "TRACE";
|
|
8268
|
-
SeverityNumber2[SeverityNumber2["TRACE2"] = 2] = "TRACE2";
|
|
8269
|
-
SeverityNumber2[SeverityNumber2["TRACE3"] = 3] = "TRACE3";
|
|
8270
|
-
SeverityNumber2[SeverityNumber2["TRACE4"] = 4] = "TRACE4";
|
|
8271
|
-
SeverityNumber2[SeverityNumber2["DEBUG"] = 5] = "DEBUG";
|
|
8272
|
-
SeverityNumber2[SeverityNumber2["DEBUG2"] = 6] = "DEBUG2";
|
|
8273
|
-
SeverityNumber2[SeverityNumber2["DEBUG3"] = 7] = "DEBUG3";
|
|
8274
|
-
SeverityNumber2[SeverityNumber2["DEBUG4"] = 8] = "DEBUG4";
|
|
8275
|
-
SeverityNumber2[SeverityNumber2["INFO"] = 9] = "INFO";
|
|
8276
|
-
SeverityNumber2[SeverityNumber2["INFO2"] = 10] = "INFO2";
|
|
8277
|
-
SeverityNumber2[SeverityNumber2["INFO3"] = 11] = "INFO3";
|
|
8278
|
-
SeverityNumber2[SeverityNumber2["INFO4"] = 12] = "INFO4";
|
|
8279
|
-
SeverityNumber2[SeverityNumber2["WARN"] = 13] = "WARN";
|
|
8280
|
-
SeverityNumber2[SeverityNumber2["WARN2"] = 14] = "WARN2";
|
|
8281
|
-
SeverityNumber2[SeverityNumber2["WARN3"] = 15] = "WARN3";
|
|
8282
|
-
SeverityNumber2[SeverityNumber2["WARN4"] = 16] = "WARN4";
|
|
8283
|
-
SeverityNumber2[SeverityNumber2["ERROR"] = 17] = "ERROR";
|
|
8284
|
-
SeverityNumber2[SeverityNumber2["ERROR2"] = 18] = "ERROR2";
|
|
8285
|
-
SeverityNumber2[SeverityNumber2["ERROR3"] = 19] = "ERROR3";
|
|
8286
|
-
SeverityNumber2[SeverityNumber2["ERROR4"] = 20] = "ERROR4";
|
|
8287
|
-
SeverityNumber2[SeverityNumber2["FATAL"] = 21] = "FATAL";
|
|
8288
|
-
SeverityNumber2[SeverityNumber2["FATAL2"] = 22] = "FATAL2";
|
|
8289
|
-
SeverityNumber2[SeverityNumber2["FATAL3"] = 23] = "FATAL3";
|
|
8290
|
-
SeverityNumber2[SeverityNumber2["FATAL4"] = 24] = "FATAL4";
|
|
8291
|
-
})(SeverityNumber = exports.SeverityNumber || (exports.SeverityNumber = {}));
|
|
8292
|
-
});
|
|
8293
|
-
|
|
8294
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLogger.js
|
|
8295
|
-
var require_NoopLogger = __commonJS((exports) => {
|
|
8296
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8297
|
-
exports.NOOP_LOGGER = exports.NoopLogger = undefined;
|
|
8298
|
-
|
|
8299
|
-
class NoopLogger {
|
|
8300
|
-
emit(_logRecord) {}
|
|
8301
|
-
}
|
|
8302
|
-
exports.NoopLogger = NoopLogger;
|
|
8303
|
-
exports.NOOP_LOGGER = new NoopLogger;
|
|
8304
|
-
});
|
|
8305
|
-
|
|
8306
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLoggerProvider.js
|
|
8307
|
-
var require_NoopLoggerProvider = __commonJS((exports) => {
|
|
8308
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8309
|
-
exports.NOOP_LOGGER_PROVIDER = exports.NoopLoggerProvider = undefined;
|
|
8310
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
8311
|
-
|
|
8312
|
-
class NoopLoggerProvider {
|
|
8313
|
-
getLogger(_name, _version, _options) {
|
|
8314
|
-
return new NoopLogger_1.NoopLogger;
|
|
8315
|
-
}
|
|
8316
|
-
}
|
|
8317
|
-
exports.NoopLoggerProvider = NoopLoggerProvider;
|
|
8318
|
-
exports.NOOP_LOGGER_PROVIDER = new NoopLoggerProvider;
|
|
8319
|
-
});
|
|
8320
|
-
|
|
8321
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLogger.js
|
|
8322
|
-
var require_ProxyLogger = __commonJS((exports) => {
|
|
8323
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8324
|
-
exports.ProxyLogger = undefined;
|
|
8325
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
8326
|
-
|
|
8327
|
-
class ProxyLogger {
|
|
8328
|
-
constructor(_provider, name, version2, options) {
|
|
8329
|
-
this._provider = _provider;
|
|
8330
|
-
this.name = name;
|
|
8331
|
-
this.version = version2;
|
|
8332
|
-
this.options = options;
|
|
8333
|
-
}
|
|
8334
|
-
emit(logRecord) {
|
|
8335
|
-
this._getLogger().emit(logRecord);
|
|
8336
|
-
}
|
|
8337
|
-
_getLogger() {
|
|
8338
|
-
if (this._delegate) {
|
|
8339
|
-
return this._delegate;
|
|
8340
|
-
}
|
|
8341
|
-
const logger = this._provider._getDelegateLogger(this.name, this.version, this.options);
|
|
8342
|
-
if (!logger) {
|
|
8343
|
-
return NoopLogger_1.NOOP_LOGGER;
|
|
8344
|
-
}
|
|
8345
|
-
this._delegate = logger;
|
|
8346
|
-
return this._delegate;
|
|
8347
|
-
}
|
|
8348
|
-
}
|
|
8349
|
-
exports.ProxyLogger = ProxyLogger;
|
|
8350
|
-
});
|
|
8351
|
-
|
|
8352
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLoggerProvider.js
|
|
8353
|
-
var require_ProxyLoggerProvider = __commonJS((exports) => {
|
|
8354
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8355
|
-
exports.ProxyLoggerProvider = undefined;
|
|
8356
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
8357
|
-
var ProxyLogger_1 = require_ProxyLogger();
|
|
8358
|
-
|
|
8359
|
-
class ProxyLoggerProvider {
|
|
8360
|
-
getLogger(name, version2, options) {
|
|
8361
|
-
var _a2;
|
|
8362
|
-
return (_a2 = this._getDelegateLogger(name, version2, options)) !== null && _a2 !== undefined ? _a2 : new ProxyLogger_1.ProxyLogger(this, name, version2, options);
|
|
8363
|
-
}
|
|
8364
|
-
_getDelegate() {
|
|
8365
|
-
var _a2;
|
|
8366
|
-
return (_a2 = this._delegate) !== null && _a2 !== undefined ? _a2 : NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
|
|
8367
|
-
}
|
|
8368
|
-
_setDelegate(delegate) {
|
|
8369
|
-
this._delegate = delegate;
|
|
8370
|
-
}
|
|
8371
|
-
_getDelegateLogger(name, version2, options) {
|
|
8372
|
-
var _a2;
|
|
8373
|
-
return (_a2 = this._delegate) === null || _a2 === undefined ? undefined : _a2.getLogger(name, version2, options);
|
|
8374
|
-
}
|
|
8375
|
-
}
|
|
8376
|
-
exports.ProxyLoggerProvider = ProxyLoggerProvider;
|
|
8377
|
-
});
|
|
8378
|
-
|
|
8379
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/globalThis.js
|
|
8380
|
-
var require_globalThis2 = __commonJS((exports) => {
|
|
8381
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8382
|
-
exports._globalThis = undefined;
|
|
8383
|
-
exports._globalThis = typeof globalThis === "object" ? globalThis : global;
|
|
8384
|
-
});
|
|
8385
|
-
|
|
8386
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/index.js
|
|
8387
|
-
var require_node2 = __commonJS((exports) => {
|
|
8388
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8389
|
-
exports._globalThis = undefined;
|
|
8390
|
-
var globalThis_1 = require_globalThis2();
|
|
8391
|
-
Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
|
|
8392
|
-
return globalThis_1._globalThis;
|
|
8393
|
-
} });
|
|
8394
|
-
});
|
|
8395
|
-
|
|
8396
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/index.js
|
|
8397
|
-
var require_platform2 = __commonJS((exports) => {
|
|
8398
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8399
|
-
exports._globalThis = undefined;
|
|
8400
|
-
var node_1 = require_node2();
|
|
8401
|
-
Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
|
|
8402
|
-
return node_1._globalThis;
|
|
8403
|
-
} });
|
|
8404
|
-
});
|
|
8405
|
-
|
|
8406
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/internal/global-utils.js
|
|
8407
|
-
var require_global_utils2 = __commonJS((exports) => {
|
|
8408
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8409
|
-
exports.API_BACKWARDS_COMPATIBILITY_VERSION = exports.makeGetter = exports._global = exports.GLOBAL_LOGS_API_KEY = undefined;
|
|
8410
|
-
var platform_1 = require_platform2();
|
|
8411
|
-
exports.GLOBAL_LOGS_API_KEY = Symbol.for("io.opentelemetry.js.api.logs");
|
|
8412
|
-
exports._global = platform_1._globalThis;
|
|
8413
|
-
function makeGetter(requiredVersion, instance, fallback) {
|
|
8414
|
-
return (version2) => version2 === requiredVersion ? instance : fallback;
|
|
8415
|
-
}
|
|
8416
|
-
exports.makeGetter = makeGetter;
|
|
8417
|
-
exports.API_BACKWARDS_COMPATIBILITY_VERSION = 1;
|
|
8418
|
-
});
|
|
8419
|
-
|
|
8420
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/api/logs.js
|
|
8421
|
-
var require_logs = __commonJS((exports) => {
|
|
8422
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8423
|
-
exports.LogsAPI = undefined;
|
|
8424
|
-
var global_utils_1 = require_global_utils2();
|
|
8425
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
8426
|
-
var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
|
|
8427
|
-
|
|
8428
|
-
class LogsAPI {
|
|
8429
|
-
constructor() {
|
|
8430
|
-
this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
8431
|
-
}
|
|
8432
|
-
static getInstance() {
|
|
8433
|
-
if (!this._instance) {
|
|
8434
|
-
this._instance = new LogsAPI;
|
|
8435
|
-
}
|
|
8436
|
-
return this._instance;
|
|
8437
|
-
}
|
|
8438
|
-
setGlobalLoggerProvider(provider) {
|
|
8439
|
-
if (global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) {
|
|
8440
|
-
return this.getLoggerProvider();
|
|
8441
|
-
}
|
|
8442
|
-
global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY] = (0, global_utils_1.makeGetter)(global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION, provider, NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER);
|
|
8443
|
-
this._proxyLoggerProvider._setDelegate(provider);
|
|
8444
|
-
return provider;
|
|
8445
|
-
}
|
|
8446
|
-
getLoggerProvider() {
|
|
8447
|
-
var _a2, _b;
|
|
8448
|
-
return (_b = (_a2 = global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) === null || _a2 === undefined ? undefined : _a2.call(global_utils_1._global, global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION)) !== null && _b !== undefined ? _b : this._proxyLoggerProvider;
|
|
8449
|
-
}
|
|
8450
|
-
getLogger(name, version2, options) {
|
|
8451
|
-
return this.getLoggerProvider().getLogger(name, version2, options);
|
|
8452
|
-
}
|
|
8453
|
-
disable() {
|
|
8454
|
-
delete global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY];
|
|
8455
|
-
this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
8456
|
-
}
|
|
8457
|
-
}
|
|
8458
|
-
exports.LogsAPI = LogsAPI;
|
|
8459
|
-
});
|
|
8460
|
-
|
|
8461
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/index.js
|
|
8462
|
-
var require_src2 = __commonJS((exports) => {
|
|
8463
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8464
|
-
exports.logs = exports.ProxyLoggerProvider = exports.ProxyLogger = exports.NoopLoggerProvider = exports.NOOP_LOGGER_PROVIDER = exports.NoopLogger = exports.NOOP_LOGGER = exports.SeverityNumber = undefined;
|
|
8465
|
-
var LogRecord_1 = require_LogRecord();
|
|
8466
|
-
Object.defineProperty(exports, "SeverityNumber", { enumerable: true, get: function() {
|
|
8467
|
-
return LogRecord_1.SeverityNumber;
|
|
8468
|
-
} });
|
|
8469
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
8470
|
-
Object.defineProperty(exports, "NOOP_LOGGER", { enumerable: true, get: function() {
|
|
8471
|
-
return NoopLogger_1.NOOP_LOGGER;
|
|
8472
|
-
} });
|
|
8473
|
-
Object.defineProperty(exports, "NoopLogger", { enumerable: true, get: function() {
|
|
8474
|
-
return NoopLogger_1.NoopLogger;
|
|
8475
|
-
} });
|
|
8476
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
8477
|
-
Object.defineProperty(exports, "NOOP_LOGGER_PROVIDER", { enumerable: true, get: function() {
|
|
8478
|
-
return NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
|
|
8479
|
-
} });
|
|
8480
|
-
Object.defineProperty(exports, "NoopLoggerProvider", { enumerable: true, get: function() {
|
|
8481
|
-
return NoopLoggerProvider_1.NoopLoggerProvider;
|
|
8482
|
-
} });
|
|
8483
|
-
var ProxyLogger_1 = require_ProxyLogger();
|
|
8484
|
-
Object.defineProperty(exports, "ProxyLogger", { enumerable: true, get: function() {
|
|
8485
|
-
return ProxyLogger_1.ProxyLogger;
|
|
8486
|
-
} });
|
|
8487
|
-
var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
|
|
8488
|
-
Object.defineProperty(exports, "ProxyLoggerProvider", { enumerable: true, get: function() {
|
|
8489
|
-
return ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
8490
|
-
} });
|
|
8491
|
-
var logs_1 = require_logs();
|
|
8492
|
-
exports.logs = logs_1.LogsAPI.getInstance();
|
|
8493
|
-
});
|
|
8494
|
-
|
|
8495
8260
|
// src/commands/market-data-vendor-backfill.ts
|
|
8496
8261
|
import { createHash as createHash5, randomUUID as randomUUID2 } from "node:crypto";
|
|
8497
8262
|
import path2 from "node:path";
|
|
@@ -22254,102 +22019,6 @@ var BaseTransport = class {
|
|
|
22254
22019
|
return this.logger;
|
|
22255
22020
|
}
|
|
22256
22021
|
};
|
|
22257
|
-
var LoggerlessTransport = class {
|
|
22258
|
-
id;
|
|
22259
|
-
enabled;
|
|
22260
|
-
level;
|
|
22261
|
-
consoleDebug;
|
|
22262
|
-
constructor(config2) {
|
|
22263
|
-
this.id = Date.now().toString() + Math.random().toString();
|
|
22264
|
-
this.enabled = config2.enabled ?? true;
|
|
22265
|
-
this.consoleDebug = config2.consoleDebug ?? false;
|
|
22266
|
-
this.level = config2.level ?? "trace";
|
|
22267
|
-
}
|
|
22268
|
-
_sendToLogger(params) {
|
|
22269
|
-
if (!this.enabled)
|
|
22270
|
-
return;
|
|
22271
|
-
if (LogLevelPriority[params.logLevel] < LogLevelPriority[this.level])
|
|
22272
|
-
return;
|
|
22273
|
-
const messages = this.shipToLogger(params);
|
|
22274
|
-
if (this.consoleDebug)
|
|
22275
|
-
switch (params.logLevel) {
|
|
22276
|
-
case LogLevel.info:
|
|
22277
|
-
console.info(...messages);
|
|
22278
|
-
break;
|
|
22279
|
-
case LogLevel.warn:
|
|
22280
|
-
console.warn(...messages);
|
|
22281
|
-
break;
|
|
22282
|
-
case LogLevel.error:
|
|
22283
|
-
console.error(...messages);
|
|
22284
|
-
break;
|
|
22285
|
-
case LogLevel.trace:
|
|
22286
|
-
console.debug(...messages);
|
|
22287
|
-
break;
|
|
22288
|
-
case LogLevel.debug:
|
|
22289
|
-
console.debug(...messages);
|
|
22290
|
-
break;
|
|
22291
|
-
case LogLevel.fatal:
|
|
22292
|
-
console.debug(...messages);
|
|
22293
|
-
break;
|
|
22294
|
-
default:
|
|
22295
|
-
console.log(...messages);
|
|
22296
|
-
}
|
|
22297
|
-
}
|
|
22298
|
-
getLoggerInstance() {
|
|
22299
|
-
throw new Error("This transport does not have a logger instance");
|
|
22300
|
-
}
|
|
22301
|
-
};
|
|
22302
|
-
|
|
22303
|
-
// node_modules/@loglayer/transport-opentelemetry/dist/index.mjs
|
|
22304
|
-
var import_api_logs = __toESM(require_src2(), 1);
|
|
22305
|
-
var logLayerLevels = {
|
|
22306
|
-
fatal: import_api_logs.SeverityNumber.FATAL,
|
|
22307
|
-
error: import_api_logs.SeverityNumber.ERROR,
|
|
22308
|
-
warn: import_api_logs.SeverityNumber.WARN,
|
|
22309
|
-
info: import_api_logs.SeverityNumber.INFO,
|
|
22310
|
-
debug: import_api_logs.SeverityNumber.DEBUG,
|
|
22311
|
-
trace: import_api_logs.SeverityNumber.TRACE
|
|
22312
|
-
};
|
|
22313
|
-
function getSeverityNumber(level) {
|
|
22314
|
-
return logLayerLevels[level] ?? import_api_logs.SeverityNumber.TRACE;
|
|
22315
|
-
}
|
|
22316
|
-
function emitLogRecord(record2, logger) {
|
|
22317
|
-
const { message, level, ...splat } = record2;
|
|
22318
|
-
const attributes = {};
|
|
22319
|
-
for (const key in splat)
|
|
22320
|
-
if (Object.hasOwn(splat, key))
|
|
22321
|
-
attributes[key] = splat[key];
|
|
22322
|
-
const logRecord = {
|
|
22323
|
-
severityNumber: getSeverityNumber(level),
|
|
22324
|
-
severityText: level,
|
|
22325
|
-
body: message,
|
|
22326
|
-
attributes
|
|
22327
|
-
};
|
|
22328
|
-
logger.emit(logRecord);
|
|
22329
|
-
}
|
|
22330
|
-
var OpenTelemetryTransport = class extends LoggerlessTransport {
|
|
22331
|
-
_logger;
|
|
22332
|
-
onError;
|
|
22333
|
-
constructor(config2 = {}) {
|
|
22334
|
-
super(config2);
|
|
22335
|
-
this._logger = import_api_logs.logs.getLogger("loglayer");
|
|
22336
|
-
this.onError = config2.onError;
|
|
22337
|
-
}
|
|
22338
|
-
shipToLogger({ logLevel, messages, data, hasData }) {
|
|
22339
|
-
const assembled = {
|
|
22340
|
-
level: logLevel,
|
|
22341
|
-
message: messages.join(" "),
|
|
22342
|
-
...hasData ? data : {}
|
|
22343
|
-
};
|
|
22344
|
-
try {
|
|
22345
|
-
emitLogRecord(assembled, this._logger);
|
|
22346
|
-
} catch (error48) {
|
|
22347
|
-
if (this.onError)
|
|
22348
|
-
this.onError(error48);
|
|
22349
|
-
}
|
|
22350
|
-
return [assembled];
|
|
22351
|
-
}
|
|
22352
|
-
};
|
|
22353
22022
|
|
|
22354
22023
|
// node_modules/@loglayer/transport-tslog/dist/index.js
|
|
22355
22024
|
var TsLogTransport = class extends BaseTransport {
|
|
@@ -24521,13 +24190,7 @@ var tslogLogger = new Logger({
|
|
|
24521
24190
|
minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
|
|
24522
24191
|
});
|
|
24523
24192
|
var baseLogger = new LogLayer({
|
|
24524
|
-
transport: [
|
|
24525
|
-
new TsLogTransport({ id: "tslog", logger: tslogLogger }),
|
|
24526
|
-
new OpenTelemetryTransport({
|
|
24527
|
-
id: "otel",
|
|
24528
|
-
enabled: true
|
|
24529
|
-
})
|
|
24530
|
-
],
|
|
24193
|
+
transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
|
|
24531
24194
|
plugins: [openTelemetryPlugin()],
|
|
24532
24195
|
errorSerializer: serializeError
|
|
24533
24196
|
});
|
|
@@ -25642,12 +25305,14 @@ var result_schema_default = {
|
|
|
25642
25305
|
};
|
|
25643
25306
|
|
|
25644
25307
|
// src/helpers/market-data-vendor-backfill/manifests.ts
|
|
25645
|
-
var
|
|
25646
|
-
var
|
|
25308
|
+
var LEGACY_CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
|
|
25309
|
+
var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v2";
|
|
25310
|
+
var LEGACY_RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
|
|
25311
|
+
var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v2";
|
|
25647
25312
|
var ADAPTER_POLICY_ID = "cryptohftdata-orderbook-adapter/v1";
|
|
25648
25313
|
var ACQUISITION_POLICY_ID = "cryptohftdata-hourly-acquisition/v1";
|
|
25649
|
-
var
|
|
25650
|
-
policy_id:
|
|
25314
|
+
var legacyCapabilityPolicyContent = {
|
|
25315
|
+
policy_id: LEGACY_CAPABILITY_POLICY_ID,
|
|
25651
25316
|
provider: "cryptohftdata",
|
|
25652
25317
|
adapter_policy: {
|
|
25653
25318
|
policy_id: ADAPTER_POLICY_ID,
|
|
@@ -25673,12 +25338,37 @@ var capabilityPolicyContent = {
|
|
|
25673
25338
|
}
|
|
25674
25339
|
]
|
|
25675
25340
|
};
|
|
25341
|
+
var LEGACY_CAPABILITY_POLICY = Object.freeze({
|
|
25342
|
+
...legacyCapabilityPolicyContent,
|
|
25343
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent)
|
|
25344
|
+
});
|
|
25345
|
+
var capabilityPolicyContent = {
|
|
25346
|
+
...legacyCapabilityPolicyContent,
|
|
25347
|
+
policy_id: CAPABILITY_POLICY_ID,
|
|
25348
|
+
profiles: [
|
|
25349
|
+
...legacyCapabilityPolicyContent.profiles.map((profile) => ({
|
|
25350
|
+
...profile,
|
|
25351
|
+
source_policies: ["authoritative_window", "fill_gaps"]
|
|
25352
|
+
})),
|
|
25353
|
+
{
|
|
25354
|
+
exchange: "okx",
|
|
25355
|
+
market_type: "spot",
|
|
25356
|
+
feed: "ORDERBOOK",
|
|
25357
|
+
canonical_trading_pair: "ARB-USDC",
|
|
25358
|
+
provider_exchange_id: "okx_spot",
|
|
25359
|
+
resolved_symbol: "ARB-USDC",
|
|
25360
|
+
construction_modes: ["sampled_top_n_snapshot"],
|
|
25361
|
+
source_policies: ["authoritative_window", "fill_gaps"],
|
|
25362
|
+
max_depth: 400
|
|
25363
|
+
}
|
|
25364
|
+
]
|
|
25365
|
+
};
|
|
25676
25366
|
var CAPABILITY_POLICY = Object.freeze({
|
|
25677
25367
|
...capabilityPolicyContent,
|
|
25678
25368
|
policy_sha256: jcsSha256(capabilityPolicyContent)
|
|
25679
25369
|
});
|
|
25680
|
-
var
|
|
25681
|
-
policy_id:
|
|
25370
|
+
var legacyResourcePolicyContent = {
|
|
25371
|
+
policy_id: LEGACY_RESOURCE_POLICY_ID,
|
|
25682
25372
|
limits: {
|
|
25683
25373
|
max_files: 1e4,
|
|
25684
25374
|
max_bytes: 100 * 1024 * 1024 * 1024,
|
|
@@ -25692,17 +25382,29 @@ var resourcePolicyContent = {
|
|
|
25692
25382
|
max_required_events: 1e5
|
|
25693
25383
|
}
|
|
25694
25384
|
};
|
|
25385
|
+
var LEGACY_RESOURCE_POLICY = Object.freeze({
|
|
25386
|
+
...legacyResourcePolicyContent,
|
|
25387
|
+
policy_sha256: jcsSha256(legacyResourcePolicyContent)
|
|
25388
|
+
});
|
|
25389
|
+
var resourcePolicyContent = {
|
|
25390
|
+
...legacyResourcePolicyContent,
|
|
25391
|
+
policy_id: RESOURCE_POLICY_ID,
|
|
25392
|
+
request_bounds: {
|
|
25393
|
+
...legacyResourcePolicyContent.request_bounds,
|
|
25394
|
+
max_window_ms: 31 * 24 * 60 * 60 * 1000
|
|
25395
|
+
}
|
|
25396
|
+
};
|
|
25695
25397
|
var RESOURCE_POLICY = Object.freeze({
|
|
25696
25398
|
...resourcePolicyContent,
|
|
25697
25399
|
policy_sha256: jcsSha256(resourcePolicyContent)
|
|
25698
25400
|
});
|
|
25699
25401
|
var EFFECTIVE_ADAPTER_POLICY_PIN = Object.freeze({
|
|
25700
25402
|
policy_id: ADAPTER_POLICY_ID,
|
|
25701
|
-
policy_sha256: jcsSha256(
|
|
25403
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent.adapter_policy)
|
|
25702
25404
|
});
|
|
25703
25405
|
var EFFECTIVE_ACQUISITION_POLICY_PIN = Object.freeze({
|
|
25704
25406
|
policy_id: ACQUISITION_POLICY_ID,
|
|
25705
|
-
policy_sha256: jcsSha256(
|
|
25407
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent.acquisition_policy)
|
|
25706
25408
|
});
|
|
25707
25409
|
var schemaArtifacts = [
|
|
25708
25410
|
["schemas/request.schema.json", request_schema_default],
|
|
@@ -25866,7 +25568,15 @@ var backfillRequestCodec = {
|
|
|
25866
25568
|
if (request.idempotency_key !== createBackfillIdempotencyKey(request)) {
|
|
25867
25569
|
throw new Error("idempotency_key does not match canonical business fields");
|
|
25868
25570
|
}
|
|
25869
|
-
|
|
25571
|
+
const capabilityPolicyMatches = [
|
|
25572
|
+
CAPABILITY_POLICY,
|
|
25573
|
+
LEGACY_CAPABILITY_POLICY
|
|
25574
|
+
].some((policy) => request.product_pins.capability_policy.policy_id === policy.policy_id && request.product_pins.capability_policy.policy_sha256 === policy.policy_sha256);
|
|
25575
|
+
const resourcePolicyMatches = [
|
|
25576
|
+
RESOURCE_POLICY,
|
|
25577
|
+
LEGACY_RESOURCE_POLICY
|
|
25578
|
+
].some((policy) => request.product_pins.resource_policy.policy_id === policy.policy_id && request.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
|
|
25579
|
+
if (!capabilityPolicyMatches || !resourcePolicyMatches) {
|
|
25870
25580
|
throw new Error("request policy pins do not match the effective package policies");
|
|
25871
25581
|
}
|
|
25872
25582
|
if (request.required_clock.clock_id !== request.initial_selection.required_clock.clock_id || request.required_clock.clock_sha256 !== request.initial_selection.required_clock.clock_sha256 || request.required_clock.event_count !== request.initial_selection.required_clock.event_count) {
|
|
@@ -25981,6 +25691,10 @@ function decodeBackfillRunDocuments(input) {
|
|
|
25981
25691
|
return targetTimeMs;
|
|
25982
25692
|
});
|
|
25983
25693
|
const capabilityProfile = CAPABILITY_POLICY.profiles.find((profile) => profile.exchange === wire.scope.exchange && profile.market_type === wire.scope.market_type && profile.feed === wire.scope.feed && profile.canonical_trading_pair === wire.scope.trading_pair);
|
|
25694
|
+
const resourcePolicy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((policy) => wire.product_pins.resource_policy.policy_id === policy.policy_id && wire.product_pins.resource_policy.policy_sha256 === policy.policy_sha256);
|
|
25695
|
+
if (!resourcePolicy) {
|
|
25696
|
+
throw new Error("request resource policy pin is unsupported");
|
|
25697
|
+
}
|
|
25984
25698
|
return {
|
|
25985
25699
|
schemaVersion: BACKFILL_REQUEST_SCHEMA_VERSION,
|
|
25986
25700
|
requestId: wire.request_id,
|
|
@@ -26005,11 +25719,11 @@ function decodeBackfillRunDocuments(input) {
|
|
|
26005
25719
|
maxPriorAsOfLagMs: wire.coverage_policy.max_asof_lag_ms,
|
|
26006
25720
|
sourcePolicy: wire.source_policy,
|
|
26007
25721
|
budgets: {
|
|
26008
|
-
maxFiles:
|
|
26009
|
-
maxBytes:
|
|
26010
|
-
maxRows:
|
|
26011
|
-
maxDurationMs:
|
|
26012
|
-
maxBoundaryLookbackMs: Math.min(
|
|
25722
|
+
maxFiles: resourcePolicy.limits.max_files,
|
|
25723
|
+
maxBytes: resourcePolicy.limits.max_bytes,
|
|
25724
|
+
maxRows: resourcePolicy.limits.max_rows,
|
|
25725
|
+
maxDurationMs: resourcePolicy.limits.max_duration_ms,
|
|
25726
|
+
maxBoundaryLookbackMs: Math.min(resourcePolicy.limits.max_boundary_lookback_ms, CAPABILITY_POLICY.acquisition_policy.initialization_lookback_ms)
|
|
26013
25727
|
},
|
|
26014
25728
|
expectedProduct: {
|
|
26015
25729
|
packageName: "@usherlabs/cex-broker",
|
|
@@ -27319,12 +27033,13 @@ function assertStoredSelectionMatchesRequest(request, selection) {
|
|
|
27319
27033
|
}
|
|
27320
27034
|
}
|
|
27321
27035
|
function eligibleBundles(request, bundles) {
|
|
27322
|
-
|
|
27323
|
-
|
|
27324
|
-
|
|
27325
|
-
|
|
27326
|
-
return
|
|
27327
|
-
}
|
|
27036
|
+
const qualifiedVendorBundles = bundles.filter((bundle) => bundle.captureOrigin === "vendor_historical_backfill" && bundle.qualification?.state === "qualified");
|
|
27037
|
+
const productionBundles = bundles.filter((bundle) => bundle.captureOrigin === "production_capture");
|
|
27038
|
+
if (request.sourcePolicy === "authoritative_window") {
|
|
27039
|
+
const vendorSupport = chooseSupport(request, qualifiedVendorBundles);
|
|
27040
|
+
return vendorSupport.length === request.requiredClockTargetsMs.length ? qualifiedVendorBundles : productionBundles;
|
|
27041
|
+
}
|
|
27042
|
+
return [...productionBundles, ...qualifiedVendorBundles];
|
|
27328
27043
|
}
|
|
27329
27044
|
function originPriority(request, origin) {
|
|
27330
27045
|
if (request.sourcePolicy === "fill_gaps") {
|
|
@@ -27512,9 +27227,8 @@ function exactCandidateMatch(expected, actual) {
|
|
|
27512
27227
|
function requestedShapeMatches(request, rows) {
|
|
27513
27228
|
return rows.every(({ row }) => row.depth_limit === request.depth && row.construction_mode === request.constructionMode && row.schema_version === request.expectedProduct.canonicalSchemaVersion);
|
|
27514
27229
|
}
|
|
27515
|
-
function requiredClockCoverage(request, rows) {
|
|
27516
|
-
const summaryTimes = rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => {
|
|
27517
|
-
const value = row.source_time_ms;
|
|
27230
|
+
function requiredClockCoverage(request, rows, coverageSourceTimesMs) {
|
|
27231
|
+
const summaryTimes = (coverageSourceTimesMs ?? rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => row.source_time_ms)).map((value) => {
|
|
27518
27232
|
if (typeof value !== "number" && (typeof value !== "string" || !/^\d+$/.test(value))) {
|
|
27519
27233
|
return;
|
|
27520
27234
|
}
|
|
@@ -27587,7 +27301,7 @@ function verifySemanticPromotion(evidence) {
|
|
|
27587
27301
|
coverageVerified: false
|
|
27588
27302
|
};
|
|
27589
27303
|
}
|
|
27590
|
-
const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows);
|
|
27304
|
+
const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows, evidence.coverageSourceTimesMs);
|
|
27591
27305
|
if (!coverageVerified) {
|
|
27592
27306
|
return {
|
|
27593
27307
|
...base,
|
|
@@ -27962,7 +27676,13 @@ class QualifiedOrderBookArchiveReader {
|
|
|
27962
27676
|
suffixDigestBefore: baseline.verificationBaseline.suffixDigest,
|
|
27963
27677
|
suffixDigestAfter: timelineDigest(suffix),
|
|
27964
27678
|
seamVerified: this.seamIsOrdered(queriedRows),
|
|
27965
|
-
exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string")
|
|
27679
|
+
exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string"),
|
|
27680
|
+
...request.sourcePolicy === "fill_gaps" ? {
|
|
27681
|
+
coverageSourceTimesMs: [
|
|
27682
|
+
...baseline.selection.support_anchors.map((anchor) => Date.parse(anchor.source_time)),
|
|
27683
|
+
...summaries.map((row) => numberField(row, "source_time_ms"))
|
|
27684
|
+
]
|
|
27685
|
+
} : {}
|
|
27966
27686
|
});
|
|
27967
27687
|
return { ...semantic, captureBundleId };
|
|
27968
27688
|
}
|
|
@@ -28078,388 +27798,8 @@ function buildForwarderBatches(input) {
|
|
|
28078
27798
|
return batches;
|
|
28079
27799
|
}
|
|
28080
27800
|
|
|
28081
|
-
// src/helpers/market-data-vendor-backfill/
|
|
28082
|
-
|
|
28083
|
-
var SHA256 = /^[0-9a-f]{64}$/;
|
|
28084
|
-
var REASON = /^[a-z][a-z0-9_]{0,127}$/;
|
|
28085
|
-
var EVENT_FIELDS = new Set([
|
|
28086
|
-
"qualification_event_id",
|
|
28087
|
-
"capture_bundle_id",
|
|
28088
|
-
"state",
|
|
28089
|
-
"receipt_id",
|
|
28090
|
-
"promotion_identity_sha256",
|
|
28091
|
-
"window",
|
|
28092
|
-
"event_at",
|
|
28093
|
-
"reason_code"
|
|
28094
|
-
]);
|
|
28095
|
-
function deterministicUuid(value) {
|
|
28096
|
-
const digest = jcsSha256(value).slice(0, 32).split("");
|
|
28097
|
-
digest[12] = "5";
|
|
28098
|
-
digest[16] = "8";
|
|
28099
|
-
const hex3 = digest.join("");
|
|
28100
|
-
return `${hex3.slice(0, 8)}-${hex3.slice(8, 12)}-${hex3.slice(12, 16)}-${hex3.slice(16, 20)}-${hex3.slice(20)}`;
|
|
28101
|
-
}
|
|
28102
|
-
function semanticEvent(event) {
|
|
28103
|
-
const { qualification_event_id: _eventId, ...semantic } = event;
|
|
28104
|
-
return semantic;
|
|
28105
|
-
}
|
|
28106
|
-
function assertTimestamp(value, field) {
|
|
28107
|
-
const parsed = Date.parse(value);
|
|
28108
|
-
if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
|
|
28109
|
-
throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
|
|
28110
|
-
}
|
|
28111
|
-
return parsed;
|
|
28112
|
-
}
|
|
28113
|
-
function parseQualificationEvent(value) {
|
|
28114
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
28115
|
-
throw new Error("qualification event must be an object");
|
|
28116
|
-
}
|
|
28117
|
-
const event = value;
|
|
28118
|
-
if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
|
|
28119
|
-
throw new Error("qualification event fields are invalid");
|
|
28120
|
-
}
|
|
28121
|
-
if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
|
|
28122
|
-
throw new Error("qualification window must be increasing");
|
|
28123
|
-
}
|
|
28124
|
-
assertTimestamp(event.event_at, "event_at");
|
|
28125
|
-
if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
|
|
28126
|
-
throw new Error("qualification_event_id does not match event content");
|
|
28127
|
-
}
|
|
28128
|
-
return event;
|
|
28129
|
-
}
|
|
28130
|
-
function finalizeQualificationEvent(event) {
|
|
28131
|
-
const semantic = semanticEvent(event);
|
|
28132
|
-
return parseQualificationEvent({
|
|
28133
|
-
...semantic,
|
|
28134
|
-
qualification_event_id: deterministicUuid(semantic)
|
|
28135
|
-
});
|
|
28136
|
-
}
|
|
28137
|
-
function qualificationEventToArchiveRow(eventInput) {
|
|
28138
|
-
const event = parseQualificationEvent(eventInput);
|
|
28139
|
-
return {
|
|
28140
|
-
table: "market_data.cex_order_book_capture_qualifications",
|
|
28141
|
-
row: {
|
|
28142
|
-
source: "external_backfill",
|
|
28143
|
-
capture_origin: "vendor_historical_backfill",
|
|
28144
|
-
source_mode: "vendor_historical_backfill_v1",
|
|
28145
|
-
deployment_id: "market-data-vendor-backfill",
|
|
28146
|
-
qualification_event_id: event.qualification_event_id,
|
|
28147
|
-
capture_bundle_id: event.capture_bundle_id,
|
|
28148
|
-
state: event.state,
|
|
28149
|
-
receipt_id: event.receipt_id,
|
|
28150
|
-
promotion_identity_sha256: event.promotion_identity_sha256,
|
|
28151
|
-
window_start_ms: Date.parse(event.window.start_at),
|
|
28152
|
-
window_end_ms: Date.parse(event.window.end_at),
|
|
28153
|
-
event_at_ms: Date.parse(event.event_at),
|
|
28154
|
-
reason_code: event.reason_code,
|
|
28155
|
-
event_json: jcsCanonicalize(event)
|
|
28156
|
-
}
|
|
28157
|
-
};
|
|
28158
|
-
}
|
|
28159
|
-
|
|
28160
|
-
// src/helpers/market-data-vendor-backfill/core.ts
|
|
28161
|
-
function outcome(request, status, reasonCode, extra = {}) {
|
|
28162
|
-
return {
|
|
28163
|
-
status,
|
|
28164
|
-
reasonCode,
|
|
28165
|
-
...request ? {
|
|
28166
|
-
requestId: request.requestId,
|
|
28167
|
-
idempotencyKey: request.idempotencyKey,
|
|
28168
|
-
...request.target ? { target: request.target } : {}
|
|
28169
|
-
} : {},
|
|
28170
|
-
...extra
|
|
28171
|
-
};
|
|
28172
|
-
}
|
|
28173
|
-
function captureBundleId(request, capability, dataset) {
|
|
28174
|
-
return sha256Canonical({
|
|
28175
|
-
request_business_identity: request.idempotencyKey,
|
|
28176
|
-
provider: capability.provider,
|
|
28177
|
-
provider_exchange_id: capability.providerExchangeId,
|
|
28178
|
-
resolved_symbol: capability.resolvedSymbol,
|
|
28179
|
-
adapter_version: capability.adapterVersion,
|
|
28180
|
-
objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
|
|
28181
|
-
identity,
|
|
28182
|
-
checksum,
|
|
28183
|
-
bytes,
|
|
28184
|
-
rows
|
|
28185
|
-
})),
|
|
28186
|
-
canonical_schema_version: request.expectedProduct.canonicalSchemaVersion,
|
|
28187
|
-
checksum_algorithm: request.expectedProduct.checksumAlgorithm
|
|
28188
|
-
});
|
|
28189
|
-
}
|
|
28190
|
-
function buildPromotionReceipt(request, capability, normalized, verification, verificationTimeMs) {
|
|
28191
|
-
if (!request.wire || !request.initialSelection || !request.expectedCanonicalSchema || !request.coveragePolicy || !request.productPins) {
|
|
28192
|
-
throw new Error("decoded final-v1 request context is missing");
|
|
28193
|
-
}
|
|
28194
|
-
return finalizePromotionReceipt({
|
|
28195
|
-
schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
|
|
28196
|
-
verified_at: new Date(verificationTimeMs).toISOString(),
|
|
28197
|
-
request_id: request.requestId,
|
|
28198
|
-
idempotency_key: request.idempotencyKey,
|
|
28199
|
-
source: EXTERNAL_BACKFILL_SOURCE,
|
|
28200
|
-
capture_origin: "vendor_historical_backfill",
|
|
28201
|
-
source_mode: "vendor_historical_backfill_v1",
|
|
28202
|
-
provider: capability.provider,
|
|
28203
|
-
adapter_version: capability.adapterVersion,
|
|
28204
|
-
effective_policies: {
|
|
28205
|
-
capability_policy: request.productPins.capability_policy,
|
|
28206
|
-
resource_policy: request.productPins.resource_policy,
|
|
28207
|
-
adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
|
|
28208
|
-
acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
|
|
28209
|
-
},
|
|
28210
|
-
capture_bundle_id: normalized.captureBundleId,
|
|
28211
|
-
scope: request.wire.scope,
|
|
28212
|
-
window: request.wire.window,
|
|
28213
|
-
depth: request.depth,
|
|
28214
|
-
construction_mode: request.constructionMode,
|
|
28215
|
-
canonical_schema: request.expectedCanonicalSchema,
|
|
28216
|
-
coverage_policy: request.coveragePolicy,
|
|
28217
|
-
selection_sha256: request.initialSelection.selection_sha256,
|
|
28218
|
-
vendor_semantic_digest: normalized.vendorSemanticDigest,
|
|
28219
|
-
canonical_semantic_digest: verification.canonicalSemanticDigest,
|
|
28220
|
-
prefix_digest: verification.prefixDigest,
|
|
28221
|
-
suffix_digest: verification.suffixDigest,
|
|
28222
|
-
seam_verified: true,
|
|
28223
|
-
coverage_verified: true,
|
|
28224
|
-
dataset_objects: normalized.objects
|
|
28225
|
-
});
|
|
28226
|
-
}
|
|
28227
|
-
async function submitAll(dependencies, batches) {
|
|
28228
|
-
const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
|
|
28229
|
-
for (const batch of batches) {
|
|
28230
|
-
let accepted = false;
|
|
28231
|
-
for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
|
|
28232
|
-
try {
|
|
28233
|
-
const response = await dependencies.forwarder.submit(batch);
|
|
28234
|
-
accepted = response.ok && response.inserted === batch.rows.length;
|
|
28235
|
-
} catch {
|
|
28236
|
-
accepted = false;
|
|
28237
|
-
}
|
|
28238
|
-
if (accepted)
|
|
28239
|
-
break;
|
|
28240
|
-
if (attempt < maxAttempts)
|
|
28241
|
-
await dependencies.retry?.wait?.(attempt);
|
|
28242
|
-
}
|
|
28243
|
-
if (!accepted)
|
|
28244
|
-
return false;
|
|
28245
|
-
}
|
|
28246
|
-
return true;
|
|
28247
|
-
}
|
|
28248
|
-
function stableFailureReason(error48, fallback) {
|
|
28249
|
-
if (error48 && typeof error48 === "object") {
|
|
28250
|
-
const reason = error48.reason;
|
|
28251
|
-
if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
|
|
28252
|
-
return reason;
|
|
28253
|
-
}
|
|
28254
|
-
}
|
|
28255
|
-
return fallback;
|
|
28256
|
-
}
|
|
28257
|
-
function assertArchivePreflight(request, resolution) {
|
|
28258
|
-
if (!request.target || !request.productionAuthorizationId) {
|
|
28259
|
-
throw new Error("request target or production authorization ID is missing");
|
|
28260
|
-
}
|
|
28261
|
-
const selection = archiveSelectionCodec.decode(resolution.selection);
|
|
28262
|
-
const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
|
|
28263
|
-
const receiptById = new Map;
|
|
28264
|
-
for (const receipt of receipts) {
|
|
28265
|
-
const existing = receiptById.get(receipt.receipt_id);
|
|
28266
|
-
if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
|
|
28267
|
-
throw new Error("stored receipt identity has conflicting content");
|
|
28268
|
-
}
|
|
28269
|
-
receiptById.set(receipt.receipt_id, receipt);
|
|
28270
|
-
}
|
|
28271
|
-
for (const bundle of selection.bundles) {
|
|
28272
|
-
if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
|
|
28273
|
-
throw new Error("vendor selection lacks its validated stored receipt");
|
|
28274
|
-
}
|
|
28275
|
-
}
|
|
28276
|
-
if (resolution.readerIdentity.environment !== request.target.environment || resolution.readerIdentity.cluster !== request.target.cluster) {
|
|
28277
|
-
throw new Error("archive reader cluster identity mismatch");
|
|
28278
|
-
}
|
|
28279
|
-
}
|
|
28280
|
-
function assertForwarderPreflight(request, resolution, nowMs) {
|
|
28281
|
-
if (!request.target || !request.productionAuthorizationId) {
|
|
28282
|
-
throw new Error("request target or production authorization ID is missing");
|
|
28283
|
-
}
|
|
28284
|
-
if (resolution.forwarderIdentity.environment !== request.target.environment || resolution.forwarderIdentity.cluster !== request.target.cluster) {
|
|
28285
|
-
throw new Error("archive forwarder cluster identity mismatch");
|
|
28286
|
-
}
|
|
28287
|
-
const authorization = resolution.authorization;
|
|
28288
|
-
const expiresAtMs = Date.parse(authorization.expiresAt);
|
|
28289
|
-
if (authorization.authorizationId !== request.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request.target.environment || authorization.cluster !== request.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
|
|
28290
|
-
throw new Error("production forwarder authorization is invalid");
|
|
28291
|
-
}
|
|
28292
|
-
}
|
|
28293
|
-
function resourcePolicyScopeExceeded(request) {
|
|
28294
|
-
return request.depth > RESOURCE_POLICY.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
|
|
28295
|
-
}
|
|
28296
|
-
function storedReceiptForSelection(resolution) {
|
|
28297
|
-
const receiptId = resolution.selection.receipt_ids[0];
|
|
28298
|
-
return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
|
|
28299
|
-
}
|
|
28300
|
-
async function runMarketDataVendorBackfill(input, dependencies) {
|
|
28301
|
-
let request;
|
|
28302
|
-
try {
|
|
28303
|
-
const documents = input;
|
|
28304
|
-
request = decodeBackfillRunDocuments({
|
|
28305
|
-
request: documents.request,
|
|
28306
|
-
requiredClock: documents.requiredClock
|
|
28307
|
-
});
|
|
28308
|
-
} catch {
|
|
28309
|
-
return outcome(undefined, "request_invalid", "request_invalid");
|
|
28310
|
-
}
|
|
28311
|
-
let initialResolution;
|
|
28312
|
-
try {
|
|
28313
|
-
initialResolution = await dependencies.archive.resolveSelection(request);
|
|
28314
|
-
assertArchivePreflight(request, initialResolution);
|
|
28315
|
-
const forwarderPreflight = await dependencies.forwarder.preflight({
|
|
28316
|
-
authorizationId: request.productionAuthorizationId,
|
|
28317
|
-
target: request.target
|
|
28318
|
-
});
|
|
28319
|
-
assertForwarderPreflight(request, forwarderPreflight, dependencies.clock.nowMs());
|
|
28320
|
-
} catch {
|
|
28321
|
-
return outcome(request, "archive_preflight_failed", "archive_preflight_failed");
|
|
28322
|
-
}
|
|
28323
|
-
if (initialResolution.selection.coverage_class === "complete") {
|
|
28324
|
-
const receipt2 = storedReceiptForSelection(initialResolution);
|
|
28325
|
-
return outcome(request, "already_covered", "qualified_coverage_complete", {
|
|
28326
|
-
selection: initialResolution.selection,
|
|
28327
|
-
...receipt2 ? { receipt: receipt2 } : {}
|
|
28328
|
-
});
|
|
28329
|
-
}
|
|
28330
|
-
if (resourcePolicyScopeExceeded(request)) {
|
|
28331
|
-
return outcome(request, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
|
|
28332
|
-
}
|
|
28333
|
-
let capability;
|
|
28334
|
-
try {
|
|
28335
|
-
capability = dependencies.providers.capabilityFor(request);
|
|
28336
|
-
} catch {
|
|
28337
|
-
return outcome(request, "capability_unsupported", "capability_probe_failed");
|
|
28338
|
-
}
|
|
28339
|
-
if (!capability) {
|
|
28340
|
-
return outcome(request, "capability_unsupported", "scope_unsupported");
|
|
28341
|
-
}
|
|
28342
|
-
if (!request.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
|
|
28343
|
-
return outcome(request, "capability_unsupported", "adapter_version_unpinned");
|
|
28344
|
-
}
|
|
28345
|
-
let credential;
|
|
28346
|
-
try {
|
|
28347
|
-
credential = await dependencies.credentials.resolve(capability.provider);
|
|
28348
|
-
} catch {
|
|
28349
|
-
return outcome(request, "credentials_missing", "credential_resolution_failed");
|
|
28350
|
-
}
|
|
28351
|
-
if (credential === undefined || credential === null) {
|
|
28352
|
-
return outcome(request, "credentials_missing", "provider_credentials_missing");
|
|
28353
|
-
}
|
|
28354
|
-
let dataset;
|
|
28355
|
-
let normalized;
|
|
28356
|
-
try {
|
|
28357
|
-
dataset = await dependencies.providers.acquire(request, capability, credential);
|
|
28358
|
-
const bundleId = captureBundleId(request, capability, dataset);
|
|
28359
|
-
normalized = await dependencies.providers.normalize(request, capability, dataset, bundleId);
|
|
28360
|
-
if (normalized.captureBundleId !== bundleId) {
|
|
28361
|
-
return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
28362
|
-
reasonSubcode: "capture_identity_mismatch"
|
|
28363
|
-
});
|
|
28364
|
-
}
|
|
28365
|
-
} catch (error48) {
|
|
28366
|
-
const reason = stableFailureReason(error48, "provider_dataset_invalid");
|
|
28367
|
-
return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
28368
|
-
reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
|
|
28369
|
-
});
|
|
28370
|
-
}
|
|
28371
|
-
try {
|
|
28372
|
-
const candidateBatches = buildForwarderBatches({
|
|
28373
|
-
captureBundleId: normalized.captureBundleId,
|
|
28374
|
-
deploymentId: "market-data-vendor-backfill",
|
|
28375
|
-
rows: normalized.rows
|
|
28376
|
-
});
|
|
28377
|
-
if (!await submitAll(dependencies, candidateBatches)) {
|
|
28378
|
-
return outcome(request, "archive_ingest_failed", "candidate_batch_rejected");
|
|
28379
|
-
}
|
|
28380
|
-
} catch {
|
|
28381
|
-
return outcome(request, "archive_ingest_failed", "candidate_submission_failed");
|
|
28382
|
-
}
|
|
28383
|
-
let verification;
|
|
28384
|
-
try {
|
|
28385
|
-
verification = await dependencies.archive.verifyCandidate(request, normalized, normalized.captureBundleId, initialResolution);
|
|
28386
|
-
} catch {
|
|
28387
|
-
return outcome(request, "promotion_verification_failed", "candidate_query_failed");
|
|
28388
|
-
}
|
|
28389
|
-
if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
|
|
28390
|
-
return outcome(request, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
|
|
28391
|
-
}
|
|
28392
|
-
let receipt;
|
|
28393
|
-
try {
|
|
28394
|
-
receipt = buildPromotionReceipt(request, capability, normalized, verification, dependencies.clock.nowMs());
|
|
28395
|
-
const [batch] = buildForwarderBatches({
|
|
28396
|
-
captureBundleId: normalized.captureBundleId,
|
|
28397
|
-
deploymentId: "market-data-vendor-backfill",
|
|
28398
|
-
rows: [promotionReceiptToArchiveRow(receipt)]
|
|
28399
|
-
});
|
|
28400
|
-
if (!batch || !await submitAll(dependencies, [batch])) {
|
|
28401
|
-
return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
|
|
28402
|
-
}
|
|
28403
|
-
const qualification = finalizeQualificationEvent({
|
|
28404
|
-
capture_bundle_id: receipt.capture_bundle_id,
|
|
28405
|
-
state: "qualified",
|
|
28406
|
-
receipt_id: receipt.receipt_id,
|
|
28407
|
-
promotion_identity_sha256: receipt.promotion_identity_sha256,
|
|
28408
|
-
window: receipt.window,
|
|
28409
|
-
event_at: receipt.verified_at,
|
|
28410
|
-
reason_code: "promotion_verified"
|
|
28411
|
-
});
|
|
28412
|
-
const [qualificationBatch] = buildForwarderBatches({
|
|
28413
|
-
captureBundleId: normalized.captureBundleId,
|
|
28414
|
-
deploymentId: "market-data-vendor-backfill",
|
|
28415
|
-
rows: [qualificationEventToArchiveRow(qualification)]
|
|
28416
|
-
});
|
|
28417
|
-
if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
|
|
28418
|
-
return outcome(request, "archive_ingest_failed", "qualification_commit_failed", { receipt });
|
|
28419
|
-
}
|
|
28420
|
-
} catch {
|
|
28421
|
-
return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
|
|
28422
|
-
}
|
|
28423
|
-
let finalResolution;
|
|
28424
|
-
try {
|
|
28425
|
-
finalResolution = await dependencies.archive.resolveSelection(request);
|
|
28426
|
-
assertArchivePreflight(request, finalResolution);
|
|
28427
|
-
} catch (error48) {
|
|
28428
|
-
return outcome(request, "promotion_verification_failed", "post_promotion_selection_failed", {
|
|
28429
|
-
receipt,
|
|
28430
|
-
reasonSubcode: stableFailureReason(error48, "archive_selection_resolution_failed")
|
|
28431
|
-
});
|
|
28432
|
-
}
|
|
28433
|
-
if (finalResolution.selection.coverage_class !== "complete") {
|
|
28434
|
-
return outcome(request, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
|
|
28435
|
-
}
|
|
28436
|
-
if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
|
|
28437
|
-
return outcome(request, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
|
|
28438
|
-
}
|
|
28439
|
-
try {
|
|
28440
|
-
if (!request.wire)
|
|
28441
|
-
throw new Error("decoded request wire is missing");
|
|
28442
|
-
const [selectionBatch] = buildForwarderBatches({
|
|
28443
|
-
captureBundleId: receipt.capture_bundle_id,
|
|
28444
|
-
deploymentId: "market-data-vendor-backfill",
|
|
28445
|
-
rows: [
|
|
28446
|
-
archiveSelectionToArchiveRow(request.wire, finalResolution.selection)
|
|
28447
|
-
]
|
|
28448
|
-
});
|
|
28449
|
-
if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
|
|
28450
|
-
return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
28451
|
-
}
|
|
28452
|
-
} catch {
|
|
28453
|
-
return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
28454
|
-
}
|
|
28455
|
-
return outcome(request, "promoted", "promotion_qualified", {
|
|
28456
|
-
receipt,
|
|
28457
|
-
selection: finalResolution.selection
|
|
28458
|
-
});
|
|
28459
|
-
}
|
|
28460
|
-
|
|
28461
|
-
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
28462
|
-
import { createHash as createHash4 } from "node:crypto";
|
|
27801
|
+
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
27802
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
28463
27803
|
|
|
28464
27804
|
// node_modules/fzstd/esm/index.mjs
|
|
28465
27805
|
var ab = ArrayBuffer;
|
|
@@ -32458,7 +31798,23 @@ var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
|
|
|
32458
31798
|
snapshotGrouping: "event_time_final_update_id_object",
|
|
32459
31799
|
sequenceSemantics: "okx_seq_id_prev_seq_id",
|
|
32460
31800
|
constructionModes: ["sampled_top_n_snapshot"],
|
|
32461
|
-
sourcePolicies: ["authoritative_window"]
|
|
31801
|
+
sourcePolicies: ["authoritative_window", "fill_gaps"]
|
|
31802
|
+
});
|
|
31803
|
+
var CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE = Object.freeze({
|
|
31804
|
+
profileId: "cryptohftdata/okx_spot/ARB-USDC/v1",
|
|
31805
|
+
exchange: "okx",
|
|
31806
|
+
tradingPair: "ARB-USDC",
|
|
31807
|
+
sourceSymbol: "ARB-USDC",
|
|
31808
|
+
marketType: "spot",
|
|
31809
|
+
providerExchangeId: "okx_spot",
|
|
31810
|
+
historyStartMs: CRYPTOHFTDATA_HISTORY_START_MS,
|
|
31811
|
+
maxDepth: 400,
|
|
31812
|
+
eventTimeUnit: "milliseconds",
|
|
31813
|
+
receivedTimeUnit: "nanoseconds",
|
|
31814
|
+
snapshotGrouping: "event_time_final_update_id_object",
|
|
31815
|
+
sequenceSemantics: "okx_seq_id_prev_seq_id",
|
|
31816
|
+
constructionModes: ["sampled_top_n_snapshot"],
|
|
31817
|
+
sourcePolicies: ["authoritative_window", "fill_gaps"]
|
|
32462
31818
|
});
|
|
32463
31819
|
|
|
32464
31820
|
class CryptoHftDataError extends Error {
|
|
@@ -32469,498 +31825,916 @@ class CryptoHftDataError extends Error {
|
|
|
32469
31825
|
this.name = "CryptoHftDataError";
|
|
32470
31826
|
}
|
|
32471
31827
|
}
|
|
32472
|
-
function cryptoHftDataCapabilityFor(request, profiles = []) {
|
|
32473
|
-
if (request.providerPolicy.provider !== "cryptohftdata" || request.scope.feed !== "ORDERBOOK" || request.scope.exchange.trim().toLowerCase() === "mexc") {
|
|
32474
|
-
return;
|
|
32475
|
-
}
|
|
32476
|
-
const profile = profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.constructionModes.includes(request.constructionMode) && candidate.sourcePolicies.includes(request.sourcePolicy) && request.window.startTimeMs >= candidate.historyStartMs && request.depth <= candidate.maxDepth);
|
|
32477
|
-
if (!profile)
|
|
32478
|
-
return;
|
|
32479
|
-
return {
|
|
32480
|
-
provider: "cryptohftdata",
|
|
32481
|
-
adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
|
|
32482
|
-
providerExchangeId: profile.providerExchangeId,
|
|
32483
|
-
resolvedSymbol: profile.sourceSymbol
|
|
32484
|
-
};
|
|
31828
|
+
function cryptoHftDataCapabilityFor(request, profiles = []) {
|
|
31829
|
+
if (request.providerPolicy.provider !== "cryptohftdata" || request.scope.feed !== "ORDERBOOK" || request.scope.exchange.trim().toLowerCase() === "mexc") {
|
|
31830
|
+
return;
|
|
31831
|
+
}
|
|
31832
|
+
const profile = profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.constructionModes.includes(request.constructionMode) && candidate.sourcePolicies.includes(request.sourcePolicy) && request.window.startTimeMs >= candidate.historyStartMs && request.depth <= candidate.maxDepth);
|
|
31833
|
+
if (!profile)
|
|
31834
|
+
return;
|
|
31835
|
+
return {
|
|
31836
|
+
provider: "cryptohftdata",
|
|
31837
|
+
adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
|
|
31838
|
+
providerExchangeId: profile.providerExchangeId,
|
|
31839
|
+
resolvedSymbol: profile.sourceSymbol
|
|
31840
|
+
};
|
|
31841
|
+
}
|
|
31842
|
+
function utcHourPath(hourMs) {
|
|
31843
|
+
const date5 = new Date(hourMs);
|
|
31844
|
+
return {
|
|
31845
|
+
date: `${date5.getUTCFullYear()}-${String(date5.getUTCMonth() + 1).padStart(2, "0")}-${String(date5.getUTCDate()).padStart(2, "0")}`,
|
|
31846
|
+
hour: String(date5.getUTCHours()).padStart(2, "0")
|
|
31847
|
+
};
|
|
31848
|
+
}
|
|
31849
|
+
function enumerateCryptoHftDataObjects(request, providerExchangeId, resolvedSymbol) {
|
|
31850
|
+
if (request.sourcePolicy === "fill_gaps") {
|
|
31851
|
+
const hours = new Set;
|
|
31852
|
+
for (const targetTimeMs of request.requiredClockTargetsMs) {
|
|
31853
|
+
const firstHour2 = Math.floor(Math.max(0, targetTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
|
|
31854
|
+
const lastHour = Math.floor(targetTimeMs / HOUR_MS) * HOUR_MS;
|
|
31855
|
+
for (let hourMs = firstHour2;hourMs <= lastHour; hourMs += HOUR_MS) {
|
|
31856
|
+
hours.add(hourMs);
|
|
31857
|
+
}
|
|
31858
|
+
}
|
|
31859
|
+
return [...hours].sort((left, right) => left - right).map((hourMs) => {
|
|
31860
|
+
const utc = utcHourPath(hourMs);
|
|
31861
|
+
return `${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`;
|
|
31862
|
+
});
|
|
31863
|
+
}
|
|
31864
|
+
const firstHour = Math.floor((request.window.startTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
|
|
31865
|
+
const objects = [];
|
|
31866
|
+
for (let hourMs = Math.max(0, firstHour);hourMs < request.window.endTimeMs; hourMs += HOUR_MS) {
|
|
31867
|
+
const utc = utcHourPath(hourMs);
|
|
31868
|
+
objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
|
|
31869
|
+
}
|
|
31870
|
+
return objects;
|
|
31871
|
+
}
|
|
31872
|
+
function initialArchiveCoversTarget(request, targetTimeMs) {
|
|
31873
|
+
return (request.initialSelection?.support_anchors ?? []).some((anchor) => {
|
|
31874
|
+
const sourceTimeMs = Date.parse(anchor.source_time);
|
|
31875
|
+
return Number.isSafeInteger(sourceTimeMs) && sourceTimeMs <= targetTimeMs && targetTimeMs - sourceTimeMs <= request.maxPriorAsOfLagMs;
|
|
31876
|
+
});
|
|
31877
|
+
}
|
|
31878
|
+
function providerAcquisitionRequest(request) {
|
|
31879
|
+
if (request.sourcePolicy !== "fill_gaps")
|
|
31880
|
+
return request;
|
|
31881
|
+
const requiredClockTargetsMs = request.requiredClockTargetsMs.filter((targetTimeMs) => !initialArchiveCoversTarget(request, targetTimeMs));
|
|
31882
|
+
if (requiredClockTargetsMs.length === 0) {
|
|
31883
|
+
throw new CryptoHftDataError("fill_gaps_has_no_uncovered_clock_targets");
|
|
31884
|
+
}
|
|
31885
|
+
return { ...request, requiredClockTargetsMs };
|
|
31886
|
+
}
|
|
31887
|
+
function unsignedString(value, field, required2 = false) {
|
|
31888
|
+
if (value === null || value === undefined) {
|
|
31889
|
+
if (required2)
|
|
31890
|
+
throw new CryptoHftDataError(`schema_${field}_missing`);
|
|
31891
|
+
return;
|
|
31892
|
+
}
|
|
31893
|
+
const rendered = String(value);
|
|
31894
|
+
if (!/^\d+$/.test(rendered)) {
|
|
31895
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
31896
|
+
}
|
|
31897
|
+
const parsed = BigInt(rendered);
|
|
31898
|
+
if (parsed > 18446744073709551615n) {
|
|
31899
|
+
throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
|
|
31900
|
+
}
|
|
31901
|
+
return parsed.toString(10);
|
|
31902
|
+
}
|
|
31903
|
+
function timestampMs2(value, field) {
|
|
31904
|
+
const rendered = String(value);
|
|
31905
|
+
if (!/^\d+$/.test(rendered)) {
|
|
31906
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
31907
|
+
}
|
|
31908
|
+
const raw = BigInt(rendered);
|
|
31909
|
+
const milliseconds = field === "received_time" ? raw / 1000000n : raw;
|
|
31910
|
+
if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
31911
|
+
throw new CryptoHftDataError(`schema_${field}_unsafe`);
|
|
31912
|
+
}
|
|
31913
|
+
return Number(milliseconds);
|
|
31914
|
+
}
|
|
31915
|
+
function decimal(value, field, allowZero) {
|
|
31916
|
+
if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
|
|
31917
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
31918
|
+
}
|
|
31919
|
+
const parsed = Number(value);
|
|
31920
|
+
if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
|
|
31921
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
31922
|
+
}
|
|
31923
|
+
return parsed;
|
|
31924
|
+
}
|
|
31925
|
+
function validatedRow(request, row) {
|
|
31926
|
+
if (row.symbol !== request.scope.sourceSymbol) {
|
|
31927
|
+
throw new CryptoHftDataError("schema_symbol_mismatch");
|
|
31928
|
+
}
|
|
31929
|
+
if (row.event_type !== "snapshot" && row.event_type !== "update") {
|
|
31930
|
+
throw new CryptoHftDataError("schema_event_type_invalid");
|
|
31931
|
+
}
|
|
31932
|
+
if (row.side !== "bid" && row.side !== "ask") {
|
|
31933
|
+
throw new CryptoHftDataError("schema_side_invalid");
|
|
31934
|
+
}
|
|
31935
|
+
decimal(row.price, "price", false);
|
|
31936
|
+
decimal(row.quantity, "quantity", true);
|
|
31937
|
+
if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
|
|
31938
|
+
throw new CryptoHftDataError("schema_object_checksum_invalid");
|
|
31939
|
+
}
|
|
31940
|
+
const eventTimeMs = timestampMs2(row.event_time, "event_time");
|
|
31941
|
+
const receivedTimeMs = timestampMs2(row.received_time, "received_time");
|
|
31942
|
+
if (receivedTimeMs < eventTimeMs) {
|
|
31943
|
+
throw new CryptoHftDataError("received_time_precedes_event_time");
|
|
31944
|
+
}
|
|
31945
|
+
return { ...row, eventTimeMs, receivedTimeMs };
|
|
31946
|
+
}
|
|
31947
|
+
function groupKey(row, profile) {
|
|
31948
|
+
const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
|
|
31949
|
+
return [
|
|
31950
|
+
row.event_type,
|
|
31951
|
+
row.eventTimeMs,
|
|
31952
|
+
sequence,
|
|
31953
|
+
row.dataset_object_identity
|
|
31954
|
+
].join("\x00");
|
|
31955
|
+
}
|
|
31956
|
+
function absent(value) {
|
|
31957
|
+
return value === null || value === undefined;
|
|
31958
|
+
}
|
|
31959
|
+
function snapshotSequence(row, profile) {
|
|
31960
|
+
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
31961
|
+
if (String(row.last_update_id) !== "-1") {
|
|
31962
|
+
throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
|
|
31963
|
+
}
|
|
31964
|
+
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
|
|
31965
|
+
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
31966
|
+
}
|
|
31967
|
+
return unsignedString(row.final_update_id, "final_update_id", true);
|
|
31968
|
+
}
|
|
31969
|
+
return unsignedString(row.last_update_id, "last_update_id", true);
|
|
31970
|
+
}
|
|
31971
|
+
function sortedSide(levels, side) {
|
|
31972
|
+
return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
|
|
31973
|
+
}
|
|
31974
|
+
function applyRows(state, rows) {
|
|
31975
|
+
for (const row of rows) {
|
|
31976
|
+
const levels = row.side === "bid" ? state.bids : state.asks;
|
|
31977
|
+
const quantity = decimal(row.quantity, "quantity", true);
|
|
31978
|
+
if (quantity === 0)
|
|
31979
|
+
levels.delete(row.price);
|
|
31980
|
+
else
|
|
31981
|
+
levels.set(row.price, quantity);
|
|
31982
|
+
}
|
|
31983
|
+
}
|
|
31984
|
+
function reconstructCryptoHftDataOrderBooks(request, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
|
|
31985
|
+
const rows = inputRows.map((row, index) => ({
|
|
31986
|
+
...validatedRow(request, row),
|
|
31987
|
+
originalIndex: index
|
|
31988
|
+
})).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
|
|
31989
|
+
const groups = [];
|
|
31990
|
+
for (const row of rows) {
|
|
31991
|
+
const current = groups.at(-1);
|
|
31992
|
+
if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
|
|
31993
|
+
groups.push([row]);
|
|
31994
|
+
} else {
|
|
31995
|
+
current.push(row);
|
|
31996
|
+
}
|
|
31997
|
+
}
|
|
31998
|
+
const earliestTargetTimeMs = Math.min(...request.requiredClockTargetsMs);
|
|
31999
|
+
const latestTargetTimeMs = Math.max(...request.requiredClockTargetsMs);
|
|
32000
|
+
const anchorIndex = groups.findIndex((group) => {
|
|
32001
|
+
const first = group[0];
|
|
32002
|
+
return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
|
|
32003
|
+
});
|
|
32004
|
+
if (anchorIndex < 0) {
|
|
32005
|
+
throw new CryptoHftDataError("update_before_snapshot");
|
|
32006
|
+
}
|
|
32007
|
+
let state;
|
|
32008
|
+
let previousFinalUpdateId;
|
|
32009
|
+
const states = [];
|
|
32010
|
+
for (const group of groups.slice(anchorIndex)) {
|
|
32011
|
+
const first = group[0];
|
|
32012
|
+
if (first.eventTimeMs > latestTargetTimeMs)
|
|
32013
|
+
break;
|
|
32014
|
+
if (first.event_type === "snapshot") {
|
|
32015
|
+
const sequence = snapshotSequence(first, profile);
|
|
32016
|
+
for (const row of group) {
|
|
32017
|
+
if (snapshotSequence(row, profile) !== sequence) {
|
|
32018
|
+
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
32019
|
+
}
|
|
32020
|
+
}
|
|
32021
|
+
if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
|
|
32022
|
+
throw new CryptoHftDataError("snapshot_sequence_regression");
|
|
32023
|
+
}
|
|
32024
|
+
state = {
|
|
32025
|
+
bids: new Map,
|
|
32026
|
+
asks: new Map,
|
|
32027
|
+
sequence,
|
|
32028
|
+
sourceTimeMs: first.eventTimeMs,
|
|
32029
|
+
receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
|
|
32030
|
+
datasetObjectIdentity: first.dataset_object_identity,
|
|
32031
|
+
datasetObjectChecksum: first.dataset_object_checksum
|
|
32032
|
+
};
|
|
32033
|
+
applyRows(state, group);
|
|
32034
|
+
previousFinalUpdateId = BigInt(sequence);
|
|
32035
|
+
} else {
|
|
32036
|
+
if (!state || previousFinalUpdateId === undefined) {
|
|
32037
|
+
throw new CryptoHftDataError("update_before_snapshot");
|
|
32038
|
+
}
|
|
32039
|
+
const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
|
|
32040
|
+
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
32041
|
+
if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
|
|
32042
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32043
|
+
}
|
|
32044
|
+
const previous = unsignedString(first.last_update_id, "last_update_id", true);
|
|
32045
|
+
for (const row of group) {
|
|
32046
|
+
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
|
|
32047
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32048
|
+
}
|
|
32049
|
+
}
|
|
32050
|
+
if (BigInt(previous) !== previousFinalUpdateId) {
|
|
32051
|
+
throw new CryptoHftDataError("update_chain_gap");
|
|
32052
|
+
}
|
|
32053
|
+
} else {
|
|
32054
|
+
const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
|
|
32055
|
+
const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
|
|
32056
|
+
for (const row of group) {
|
|
32057
|
+
if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
|
|
32058
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32059
|
+
}
|
|
32060
|
+
}
|
|
32061
|
+
const expected = previousFinalUpdateId + 1n;
|
|
32062
|
+
if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
|
|
32063
|
+
throw new CryptoHftDataError("update_chain_gap");
|
|
32064
|
+
}
|
|
32065
|
+
}
|
|
32066
|
+
applyRows(state, group);
|
|
32067
|
+
state.sequence = finalUpdate.toString();
|
|
32068
|
+
state.sourceTimeMs = first.eventTimeMs;
|
|
32069
|
+
state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
|
|
32070
|
+
state.datasetObjectIdentity = first.dataset_object_identity;
|
|
32071
|
+
state.datasetObjectChecksum = first.dataset_object_checksum;
|
|
32072
|
+
previousFinalUpdateId = finalUpdate;
|
|
32073
|
+
}
|
|
32074
|
+
if (state) {
|
|
32075
|
+
states.push({
|
|
32076
|
+
...state,
|
|
32077
|
+
bids: new Map(state.bids),
|
|
32078
|
+
asks: new Map(state.asks)
|
|
32079
|
+
});
|
|
32080
|
+
}
|
|
32081
|
+
}
|
|
32082
|
+
const samples = [];
|
|
32083
|
+
for (const targetTimeMs of request.requiredClockTargetsMs) {
|
|
32084
|
+
let prior;
|
|
32085
|
+
for (const candidate of states) {
|
|
32086
|
+
if (candidate.sourceTimeMs > targetTimeMs)
|
|
32087
|
+
break;
|
|
32088
|
+
prior = candidate;
|
|
32089
|
+
}
|
|
32090
|
+
if (!prior || targetTimeMs - prior.sourceTimeMs > request.maxPriorAsOfLagMs) {
|
|
32091
|
+
throw new CryptoHftDataError("required_clock_coverage_insufficient");
|
|
32092
|
+
}
|
|
32093
|
+
const bids = sortedSide(prior.bids, "bid").slice(0, request.depth);
|
|
32094
|
+
const asks = sortedSide(prior.asks, "ask").slice(0, request.depth);
|
|
32095
|
+
if (bids.length === 0 || asks.length === 0) {
|
|
32096
|
+
throw new CryptoHftDataError("book_side_missing");
|
|
32097
|
+
}
|
|
32098
|
+
if (bids[0]?.[0] >= asks[0]?.[0]) {
|
|
32099
|
+
throw new CryptoHftDataError("book_crossed_or_locked");
|
|
32100
|
+
}
|
|
32101
|
+
samples.push({
|
|
32102
|
+
targetTimeMs,
|
|
32103
|
+
sourceTimeMs: prior.sourceTimeMs,
|
|
32104
|
+
receivedTimeMs: prior.receivedTimeMs,
|
|
32105
|
+
sequence: prior.sequence,
|
|
32106
|
+
bids,
|
|
32107
|
+
asks,
|
|
32108
|
+
datasetObjectIdentity: prior.datasetObjectIdentity,
|
|
32109
|
+
datasetObjectChecksum: prior.datasetObjectChecksum
|
|
32110
|
+
});
|
|
32111
|
+
}
|
|
32112
|
+
return samples;
|
|
32113
|
+
}
|
|
32114
|
+
function sha256Bytes(bytes) {
|
|
32115
|
+
return createHash4("sha256").update(bytes).digest("hex");
|
|
32485
32116
|
}
|
|
32486
|
-
function
|
|
32487
|
-
const
|
|
32117
|
+
async function decodeCryptoHftParquetZstd(bytes) {
|
|
32118
|
+
const parquet = decompress(bytes);
|
|
32119
|
+
const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
|
|
32120
|
+
return parquetReadObjects({ file: buffer });
|
|
32121
|
+
}
|
|
32122
|
+
function parsedDatasetRow(value, object2) {
|
|
32488
32123
|
return {
|
|
32489
|
-
|
|
32490
|
-
|
|
32124
|
+
received_time: value.received_time,
|
|
32125
|
+
event_time: value.event_time,
|
|
32126
|
+
transaction_time: value.transaction_time,
|
|
32127
|
+
symbol: value.symbol,
|
|
32128
|
+
event_type: value.event_type,
|
|
32129
|
+
first_update_id: value.first_update_id,
|
|
32130
|
+
final_update_id: value.final_update_id,
|
|
32131
|
+
prev_final_update_id: value.prev_final_update_id,
|
|
32132
|
+
last_update_id: value.last_update_id,
|
|
32133
|
+
side: value.side,
|
|
32134
|
+
price: value.price,
|
|
32135
|
+
quantity: value.quantity,
|
|
32136
|
+
order_count: value.order_count,
|
|
32137
|
+
dataset_object_identity: object2.identity,
|
|
32138
|
+
dataset_object_checksum: object2.checksum
|
|
32491
32139
|
};
|
|
32492
32140
|
}
|
|
32493
|
-
function
|
|
32494
|
-
|
|
32495
|
-
|
|
32496
|
-
|
|
32497
|
-
|
|
32498
|
-
|
|
32141
|
+
async function readBoundedObject(response, remainingBytes) {
|
|
32142
|
+
if (!response.body)
|
|
32143
|
+
return new Uint8Array;
|
|
32144
|
+
const reader = response.body.getReader();
|
|
32145
|
+
const chunks = [];
|
|
32146
|
+
let total = 0;
|
|
32147
|
+
while (true) {
|
|
32148
|
+
const { done, value } = await reader.read();
|
|
32149
|
+
if (done)
|
|
32150
|
+
break;
|
|
32151
|
+
total += value.byteLength;
|
|
32152
|
+
if (total > remainingBytes) {
|
|
32153
|
+
await reader.cancel();
|
|
32154
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32155
|
+
}
|
|
32156
|
+
chunks.push(value);
|
|
32157
|
+
}
|
|
32158
|
+
const bytes = new Uint8Array(total);
|
|
32159
|
+
let offset = 0;
|
|
32160
|
+
for (const chunk of chunks) {
|
|
32161
|
+
bytes.set(chunk, offset);
|
|
32162
|
+
offset += chunk.byteLength;
|
|
32163
|
+
}
|
|
32164
|
+
return bytes;
|
|
32165
|
+
}
|
|
32166
|
+
|
|
32167
|
+
class CryptoHftDataAdapter {
|
|
32168
|
+
baseUrl;
|
|
32169
|
+
request;
|
|
32170
|
+
nowMs;
|
|
32171
|
+
decode;
|
|
32172
|
+
profiles;
|
|
32173
|
+
constructor(options = {}) {
|
|
32174
|
+
this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
|
|
32175
|
+
this.request = options.fetch ?? fetch;
|
|
32176
|
+
this.nowMs = options.nowMs ?? Date.now;
|
|
32177
|
+
this.decode = options.decode ?? decodeCryptoHftParquetZstd;
|
|
32178
|
+
this.profiles = options.profiles ?? [];
|
|
32179
|
+
}
|
|
32180
|
+
capabilityFor(request) {
|
|
32181
|
+
return cryptoHftDataCapabilityFor(request, this.profiles);
|
|
32182
|
+
}
|
|
32183
|
+
async discoverSymbols(providerExchangeId) {
|
|
32184
|
+
if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
|
|
32185
|
+
throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
|
|
32186
|
+
}
|
|
32187
|
+
const endpoint = new URL("/symbols", this.baseUrl);
|
|
32188
|
+
endpoint.searchParams.set("exchange", providerExchangeId);
|
|
32189
|
+
endpoint.searchParams.set("data_type", "orderbook");
|
|
32190
|
+
let response;
|
|
32191
|
+
try {
|
|
32192
|
+
response = await this.request(endpoint);
|
|
32193
|
+
} catch {
|
|
32194
|
+
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
32195
|
+
}
|
|
32196
|
+
if (!response.ok) {
|
|
32197
|
+
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
32198
|
+
}
|
|
32199
|
+
let body;
|
|
32200
|
+
try {
|
|
32201
|
+
body = await response.json();
|
|
32202
|
+
} catch {
|
|
32203
|
+
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
32204
|
+
}
|
|
32205
|
+
const record2 = body;
|
|
32206
|
+
if (!body || typeof body !== "object" || Array.isArray(body) || String(record2.exchange).toLowerCase() !== providerExchangeId || String(record2.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record2.symbols) || !record2.symbols.every((symbol2) => typeof symbol2 === "string" && /^[A-Z0-9_-]+$/.test(symbol2))) {
|
|
32207
|
+
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
32208
|
+
}
|
|
32209
|
+
return [...new Set(record2.symbols)].sort();
|
|
32210
|
+
}
|
|
32211
|
+
async acquire(request, capability, credential) {
|
|
32212
|
+
const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
|
|
32213
|
+
if (typeof apiKey !== "string" || apiKey.length === 0) {
|
|
32214
|
+
throw new CryptoHftDataError("credentials_invalid");
|
|
32215
|
+
}
|
|
32216
|
+
const paths = enumerateCryptoHftDataObjects(request, capability.providerExchangeId, capability.resolvedSymbol);
|
|
32217
|
+
if (paths.length > request.budgets.maxFiles) {
|
|
32218
|
+
throw new CryptoHftDataError("budget_max_files_exceeded");
|
|
32219
|
+
}
|
|
32220
|
+
const started = this.nowMs();
|
|
32221
|
+
const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
|
|
32222
|
+
method: "POST",
|
|
32223
|
+
headers: { "content-type": "application/json", "X-API-Key": apiKey }
|
|
32224
|
+
});
|
|
32225
|
+
if (!tokenResponse.ok)
|
|
32226
|
+
throw new CryptoHftDataError("jwt_issuance_failed");
|
|
32227
|
+
const tokenBody = await tokenResponse.json();
|
|
32228
|
+
if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
|
|
32229
|
+
throw new CryptoHftDataError("jwt_response_invalid");
|
|
32230
|
+
}
|
|
32231
|
+
let totalBytes = 0;
|
|
32232
|
+
let totalRows = 0;
|
|
32233
|
+
const objects = [];
|
|
32234
|
+
const rows = [];
|
|
32235
|
+
for (const path2 of paths) {
|
|
32236
|
+
if (this.nowMs() - started > request.budgets.maxDurationMs) {
|
|
32237
|
+
throw new CryptoHftDataError("budget_max_duration_exceeded");
|
|
32238
|
+
}
|
|
32239
|
+
const endpoint = new URL("/download", this.baseUrl);
|
|
32240
|
+
endpoint.searchParams.set("file", path2);
|
|
32241
|
+
const response = await this.request(endpoint, {
|
|
32242
|
+
headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
|
|
32243
|
+
});
|
|
32244
|
+
if (!response.ok)
|
|
32245
|
+
throw new CryptoHftDataError("object_download_failed");
|
|
32246
|
+
const declaredBytes = Number(response.headers.get("content-length"));
|
|
32247
|
+
if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request.budgets.maxBytes) {
|
|
32248
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32249
|
+
}
|
|
32250
|
+
const bytes = await readBoundedObject(response, request.budgets.maxBytes - totalBytes);
|
|
32251
|
+
totalBytes += bytes.byteLength;
|
|
32252
|
+
if (totalBytes > request.budgets.maxBytes) {
|
|
32253
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32254
|
+
}
|
|
32255
|
+
const decoded = await this.decode(bytes);
|
|
32256
|
+
totalRows += decoded.length;
|
|
32257
|
+
if (totalRows > request.budgets.maxRows) {
|
|
32258
|
+
throw new CryptoHftDataError("budget_max_rows_exceeded");
|
|
32259
|
+
}
|
|
32260
|
+
const object2 = {
|
|
32261
|
+
identity: path2,
|
|
32262
|
+
checksum: sha256Bytes(bytes),
|
|
32263
|
+
bytes: bytes.byteLength,
|
|
32264
|
+
rows: decoded.length
|
|
32265
|
+
};
|
|
32266
|
+
objects.push(object2);
|
|
32267
|
+
for (const decodedRow of decoded) {
|
|
32268
|
+
const parsed = parsedDatasetRow(decodedRow, object2);
|
|
32269
|
+
validatedRow(request, parsed);
|
|
32270
|
+
rows.push(parsed);
|
|
32271
|
+
}
|
|
32272
|
+
}
|
|
32273
|
+
if (this.nowMs() - started > request.budgets.maxDurationMs) {
|
|
32274
|
+
throw new CryptoHftDataError("budget_max_duration_exceeded");
|
|
32275
|
+
}
|
|
32276
|
+
return {
|
|
32277
|
+
objects,
|
|
32278
|
+
rows,
|
|
32279
|
+
vendorSemanticDigest: sha256Canonical(rows)
|
|
32280
|
+
};
|
|
32281
|
+
}
|
|
32282
|
+
async normalize(request, capability, dataset, captureBundleId) {
|
|
32283
|
+
const profile = this.profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
|
|
32284
|
+
if (!profile) {
|
|
32285
|
+
throw new CryptoHftDataError("profile_semantics_unavailable");
|
|
32286
|
+
}
|
|
32287
|
+
const samples = reconstructCryptoHftDataOrderBooks(request, dataset.rows, profile);
|
|
32288
|
+
const context2 = {
|
|
32289
|
+
source: EXTERNAL_BACKFILL_SOURCE,
|
|
32290
|
+
deploymentId: "market-data-vendor-backfill",
|
|
32291
|
+
captureBundleId,
|
|
32292
|
+
exchange: request.scope.exchange,
|
|
32293
|
+
symbol: request.scope.tradingPair,
|
|
32294
|
+
tradingPair: request.scope.tradingPair,
|
|
32295
|
+
sourceSymbol: request.scope.sourceSymbol,
|
|
32296
|
+
assetType: request.scope.marketType,
|
|
32297
|
+
feed: "ORDERBOOK",
|
|
32298
|
+
provider: "cryptohftdata",
|
|
32299
|
+
sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
|
|
32300
|
+
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
32301
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
32302
|
+
provenanceComplete: true
|
|
32303
|
+
};
|
|
32304
|
+
const rows = samples.flatMap((sample) => {
|
|
32305
|
+
const rawCapture = {
|
|
32306
|
+
rawCaptureId: sha256Canonical({
|
|
32307
|
+
capture_bundle_id: captureBundleId,
|
|
32308
|
+
dataset_object_identity: sample.datasetObjectIdentity,
|
|
32309
|
+
dataset_object_checksum: sample.datasetObjectChecksum,
|
|
32310
|
+
source_time_ms: sample.sourceTimeMs,
|
|
32311
|
+
target_time_ms: sample.targetTimeMs
|
|
32312
|
+
}),
|
|
32313
|
+
rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
|
|
32314
|
+
rawChecksum: sample.datasetObjectChecksum,
|
|
32315
|
+
redactedPayload: {
|
|
32316
|
+
dataset_object_identity: sample.datasetObjectIdentity,
|
|
32317
|
+
dataset_object_checksum: sample.datasetObjectChecksum
|
|
32318
|
+
},
|
|
32319
|
+
eventTimeMs: sample.sourceTimeMs,
|
|
32320
|
+
receivedTimeMs: sample.receivedTimeMs,
|
|
32321
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM
|
|
32322
|
+
};
|
|
32323
|
+
const canonical = buildCanonicalOrderBookRows({
|
|
32324
|
+
context: context2,
|
|
32325
|
+
rawCapture,
|
|
32326
|
+
depthLimit: request.depth,
|
|
32327
|
+
constructionMode: request.constructionMode,
|
|
32328
|
+
snapshot: {
|
|
32329
|
+
bids: sample.bids,
|
|
32330
|
+
asks: sample.asks,
|
|
32331
|
+
timestamp: sample.sourceTimeMs,
|
|
32332
|
+
receivedTimestamp: sample.receivedTimeMs,
|
|
32333
|
+
exchange: request.scope.exchange,
|
|
32334
|
+
symbol: request.scope.tradingPair,
|
|
32335
|
+
depthLimit: request.depth,
|
|
32336
|
+
sequence: sample.sequence
|
|
32337
|
+
}
|
|
32338
|
+
});
|
|
32339
|
+
return [...canonical.levels, canonical.summary];
|
|
32340
|
+
});
|
|
32341
|
+
return {
|
|
32342
|
+
captureBundleId,
|
|
32343
|
+
objects: dataset.objects,
|
|
32344
|
+
rows,
|
|
32345
|
+
vendorSemanticDigest: dataset.vendorSemanticDigest,
|
|
32346
|
+
canonicalSemanticDigest: semanticDigest(rows)
|
|
32347
|
+
};
|
|
32499
32348
|
}
|
|
32500
|
-
return objects;
|
|
32501
32349
|
}
|
|
32502
|
-
|
|
32503
|
-
|
|
32504
|
-
|
|
32505
|
-
|
|
32506
|
-
|
|
32507
|
-
|
|
32508
|
-
|
|
32509
|
-
|
|
32510
|
-
|
|
32511
|
-
|
|
32512
|
-
|
|
32513
|
-
|
|
32514
|
-
|
|
32515
|
-
|
|
32516
|
-
|
|
32350
|
+
|
|
32351
|
+
// src/helpers/market-data-vendor-backfill/qualification.ts
|
|
32352
|
+
var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
32353
|
+
var SHA256 = /^[0-9a-f]{64}$/;
|
|
32354
|
+
var REASON = /^[a-z][a-z0-9_]{0,127}$/;
|
|
32355
|
+
var EVENT_FIELDS = new Set([
|
|
32356
|
+
"qualification_event_id",
|
|
32357
|
+
"capture_bundle_id",
|
|
32358
|
+
"state",
|
|
32359
|
+
"receipt_id",
|
|
32360
|
+
"promotion_identity_sha256",
|
|
32361
|
+
"window",
|
|
32362
|
+
"event_at",
|
|
32363
|
+
"reason_code"
|
|
32364
|
+
]);
|
|
32365
|
+
function deterministicUuid(value) {
|
|
32366
|
+
const digest = jcsSha256(value).slice(0, 32).split("");
|
|
32367
|
+
digest[12] = "5";
|
|
32368
|
+
digest[16] = "8";
|
|
32369
|
+
const hex3 = digest.join("");
|
|
32370
|
+
return `${hex3.slice(0, 8)}-${hex3.slice(8, 12)}-${hex3.slice(12, 16)}-${hex3.slice(16, 20)}-${hex3.slice(20)}`;
|
|
32517
32371
|
}
|
|
32518
|
-
function
|
|
32519
|
-
const
|
|
32520
|
-
|
|
32521
|
-
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
32522
|
-
}
|
|
32523
|
-
const raw = BigInt(rendered);
|
|
32524
|
-
const milliseconds = field === "received_time" ? raw / 1000000n : raw;
|
|
32525
|
-
if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
32526
|
-
throw new CryptoHftDataError(`schema_${field}_unsafe`);
|
|
32527
|
-
}
|
|
32528
|
-
return Number(milliseconds);
|
|
32372
|
+
function semanticEvent(event) {
|
|
32373
|
+
const { qualification_event_id: _eventId, ...semantic } = event;
|
|
32374
|
+
return semantic;
|
|
32529
32375
|
}
|
|
32530
|
-
function
|
|
32531
|
-
|
|
32532
|
-
|
|
32533
|
-
|
|
32534
|
-
const parsed = Number(value);
|
|
32535
|
-
if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
|
|
32536
|
-
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
32376
|
+
function assertTimestamp(value, field) {
|
|
32377
|
+
const parsed = Date.parse(value);
|
|
32378
|
+
if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
|
|
32379
|
+
throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
|
|
32537
32380
|
}
|
|
32538
32381
|
return parsed;
|
|
32539
32382
|
}
|
|
32540
|
-
function
|
|
32541
|
-
if (
|
|
32542
|
-
throw new
|
|
32543
|
-
}
|
|
32544
|
-
if (row.event_type !== "snapshot" && row.event_type !== "update") {
|
|
32545
|
-
throw new CryptoHftDataError("schema_event_type_invalid");
|
|
32383
|
+
function parseQualificationEvent(value) {
|
|
32384
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
32385
|
+
throw new Error("qualification event must be an object");
|
|
32546
32386
|
}
|
|
32547
|
-
|
|
32548
|
-
|
|
32387
|
+
const event = value;
|
|
32388
|
+
if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
|
|
32389
|
+
throw new Error("qualification event fields are invalid");
|
|
32549
32390
|
}
|
|
32550
|
-
|
|
32551
|
-
|
|
32552
|
-
if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
|
|
32553
|
-
throw new CryptoHftDataError("schema_object_checksum_invalid");
|
|
32391
|
+
if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
|
|
32392
|
+
throw new Error("qualification window must be increasing");
|
|
32554
32393
|
}
|
|
32555
|
-
|
|
32556
|
-
|
|
32557
|
-
|
|
32558
|
-
throw new CryptoHftDataError("received_time_precedes_event_time");
|
|
32394
|
+
assertTimestamp(event.event_at, "event_at");
|
|
32395
|
+
if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
|
|
32396
|
+
throw new Error("qualification_event_id does not match event content");
|
|
32559
32397
|
}
|
|
32560
|
-
return
|
|
32561
|
-
}
|
|
32562
|
-
function groupKey(row, profile) {
|
|
32563
|
-
const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
|
|
32564
|
-
return [
|
|
32565
|
-
row.event_type,
|
|
32566
|
-
row.eventTimeMs,
|
|
32567
|
-
sequence,
|
|
32568
|
-
row.dataset_object_identity
|
|
32569
|
-
].join("\x00");
|
|
32398
|
+
return event;
|
|
32570
32399
|
}
|
|
32571
|
-
function
|
|
32572
|
-
|
|
32400
|
+
function finalizeQualificationEvent(event) {
|
|
32401
|
+
const semantic = semanticEvent(event);
|
|
32402
|
+
return parseQualificationEvent({
|
|
32403
|
+
...semantic,
|
|
32404
|
+
qualification_event_id: deterministicUuid(semantic)
|
|
32405
|
+
});
|
|
32573
32406
|
}
|
|
32574
|
-
function
|
|
32575
|
-
|
|
32576
|
-
|
|
32577
|
-
|
|
32578
|
-
|
|
32579
|
-
|
|
32580
|
-
|
|
32407
|
+
function qualificationEventToArchiveRow(eventInput) {
|
|
32408
|
+
const event = parseQualificationEvent(eventInput);
|
|
32409
|
+
return {
|
|
32410
|
+
table: "market_data.cex_order_book_capture_qualifications",
|
|
32411
|
+
row: {
|
|
32412
|
+
source: "external_backfill",
|
|
32413
|
+
capture_origin: "vendor_historical_backfill",
|
|
32414
|
+
source_mode: "vendor_historical_backfill_v1",
|
|
32415
|
+
deployment_id: "market-data-vendor-backfill",
|
|
32416
|
+
qualification_event_id: event.qualification_event_id,
|
|
32417
|
+
capture_bundle_id: event.capture_bundle_id,
|
|
32418
|
+
state: event.state,
|
|
32419
|
+
receipt_id: event.receipt_id,
|
|
32420
|
+
promotion_identity_sha256: event.promotion_identity_sha256,
|
|
32421
|
+
window_start_ms: Date.parse(event.window.start_at),
|
|
32422
|
+
window_end_ms: Date.parse(event.window.end_at),
|
|
32423
|
+
event_at_ms: Date.parse(event.event_at),
|
|
32424
|
+
reason_code: event.reason_code,
|
|
32425
|
+
event_json: jcsCanonicalize(event)
|
|
32581
32426
|
}
|
|
32582
|
-
|
|
32583
|
-
}
|
|
32584
|
-
return unsignedString(row.last_update_id, "last_update_id", true);
|
|
32585
|
-
}
|
|
32586
|
-
function sortedSide(levels, side) {
|
|
32587
|
-
return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
|
|
32588
|
-
}
|
|
32589
|
-
function applyRows(state, rows) {
|
|
32590
|
-
for (const row of rows) {
|
|
32591
|
-
const levels = row.side === "bid" ? state.bids : state.asks;
|
|
32592
|
-
const quantity = decimal(row.quantity, "quantity", true);
|
|
32593
|
-
if (quantity === 0)
|
|
32594
|
-
levels.delete(row.price);
|
|
32595
|
-
else
|
|
32596
|
-
levels.set(row.price, quantity);
|
|
32597
|
-
}
|
|
32427
|
+
};
|
|
32598
32428
|
}
|
|
32599
|
-
|
|
32600
|
-
|
|
32601
|
-
|
|
32602
|
-
|
|
32603
|
-
|
|
32604
|
-
|
|
32605
|
-
|
|
32606
|
-
|
|
32607
|
-
|
|
32608
|
-
|
|
32609
|
-
}
|
|
32610
|
-
|
|
32611
|
-
|
|
32612
|
-
|
|
32613
|
-
|
|
32614
|
-
|
|
32615
|
-
|
|
32616
|
-
|
|
32617
|
-
|
|
32618
|
-
|
|
32619
|
-
|
|
32620
|
-
|
|
32621
|
-
|
|
32622
|
-
|
|
32623
|
-
|
|
32624
|
-
|
|
32625
|
-
|
|
32626
|
-
|
|
32627
|
-
|
|
32628
|
-
|
|
32629
|
-
|
|
32630
|
-
|
|
32631
|
-
|
|
32632
|
-
|
|
32633
|
-
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
32634
|
-
}
|
|
32635
|
-
}
|
|
32636
|
-
if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
|
|
32637
|
-
throw new CryptoHftDataError("snapshot_sequence_regression");
|
|
32638
|
-
}
|
|
32639
|
-
state = {
|
|
32640
|
-
bids: new Map,
|
|
32641
|
-
asks: new Map,
|
|
32642
|
-
sequence,
|
|
32643
|
-
sourceTimeMs: first.eventTimeMs,
|
|
32644
|
-
receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
|
|
32645
|
-
datasetObjectIdentity: first.dataset_object_identity,
|
|
32646
|
-
datasetObjectChecksum: first.dataset_object_checksum
|
|
32647
|
-
};
|
|
32648
|
-
applyRows(state, group);
|
|
32649
|
-
previousFinalUpdateId = BigInt(sequence);
|
|
32650
|
-
} else {
|
|
32651
|
-
if (!state || previousFinalUpdateId === undefined) {
|
|
32652
|
-
throw new CryptoHftDataError("update_before_snapshot");
|
|
32653
|
-
}
|
|
32654
|
-
const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
|
|
32655
|
-
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
32656
|
-
if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
|
|
32657
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32658
|
-
}
|
|
32659
|
-
const previous = unsignedString(first.last_update_id, "last_update_id", true);
|
|
32660
|
-
for (const row of group) {
|
|
32661
|
-
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
|
|
32662
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32663
|
-
}
|
|
32664
|
-
}
|
|
32665
|
-
if (BigInt(previous) !== previousFinalUpdateId) {
|
|
32666
|
-
throw new CryptoHftDataError("update_chain_gap");
|
|
32667
|
-
}
|
|
32668
|
-
} else {
|
|
32669
|
-
const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
|
|
32670
|
-
const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
|
|
32671
|
-
for (const row of group) {
|
|
32672
|
-
if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
|
|
32673
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32674
|
-
}
|
|
32675
|
-
}
|
|
32676
|
-
const expected = previousFinalUpdateId + 1n;
|
|
32677
|
-
if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
|
|
32678
|
-
throw new CryptoHftDataError("update_chain_gap");
|
|
32679
|
-
}
|
|
32680
|
-
}
|
|
32681
|
-
applyRows(state, group);
|
|
32682
|
-
state.sequence = finalUpdate.toString();
|
|
32683
|
-
state.sourceTimeMs = first.eventTimeMs;
|
|
32684
|
-
state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
|
|
32685
|
-
state.datasetObjectIdentity = first.dataset_object_identity;
|
|
32686
|
-
state.datasetObjectChecksum = first.dataset_object_checksum;
|
|
32687
|
-
previousFinalUpdateId = finalUpdate;
|
|
32688
|
-
}
|
|
32689
|
-
if (state) {
|
|
32690
|
-
states.push({
|
|
32691
|
-
...state,
|
|
32692
|
-
bids: new Map(state.bids),
|
|
32693
|
-
asks: new Map(state.asks)
|
|
32694
|
-
});
|
|
32695
|
-
}
|
|
32429
|
+
|
|
32430
|
+
// src/helpers/market-data-vendor-backfill/core.ts
|
|
32431
|
+
function outcome(request, status, reasonCode, extra = {}) {
|
|
32432
|
+
return {
|
|
32433
|
+
status,
|
|
32434
|
+
reasonCode,
|
|
32435
|
+
...request ? {
|
|
32436
|
+
requestId: request.requestId,
|
|
32437
|
+
idempotencyKey: request.idempotencyKey,
|
|
32438
|
+
...request.target ? { target: request.target } : {}
|
|
32439
|
+
} : {},
|
|
32440
|
+
...extra
|
|
32441
|
+
};
|
|
32442
|
+
}
|
|
32443
|
+
function captureBundleId(request, capability, dataset) {
|
|
32444
|
+
return sha256Canonical({
|
|
32445
|
+
request_business_identity: request.idempotencyKey,
|
|
32446
|
+
provider: capability.provider,
|
|
32447
|
+
provider_exchange_id: capability.providerExchangeId,
|
|
32448
|
+
resolved_symbol: capability.resolvedSymbol,
|
|
32449
|
+
adapter_version: capability.adapterVersion,
|
|
32450
|
+
objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
|
|
32451
|
+
identity,
|
|
32452
|
+
checksum,
|
|
32453
|
+
bytes,
|
|
32454
|
+
rows
|
|
32455
|
+
})),
|
|
32456
|
+
canonical_schema_version: request.expectedProduct.canonicalSchemaVersion,
|
|
32457
|
+
checksum_algorithm: request.expectedProduct.checksumAlgorithm
|
|
32458
|
+
});
|
|
32459
|
+
}
|
|
32460
|
+
function buildPromotionReceipt(request, capability, normalized, verification, verificationTimeMs) {
|
|
32461
|
+
if (!request.wire || !request.initialSelection || !request.expectedCanonicalSchema || !request.coveragePolicy || !request.productPins) {
|
|
32462
|
+
throw new Error("decoded final-v1 request context is missing");
|
|
32696
32463
|
}
|
|
32697
|
-
|
|
32698
|
-
|
|
32699
|
-
|
|
32700
|
-
|
|
32701
|
-
|
|
32464
|
+
return finalizePromotionReceipt({
|
|
32465
|
+
schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
|
|
32466
|
+
verified_at: new Date(verificationTimeMs).toISOString(),
|
|
32467
|
+
request_id: request.requestId,
|
|
32468
|
+
idempotency_key: request.idempotencyKey,
|
|
32469
|
+
source: EXTERNAL_BACKFILL_SOURCE,
|
|
32470
|
+
capture_origin: "vendor_historical_backfill",
|
|
32471
|
+
source_mode: "vendor_historical_backfill_v1",
|
|
32472
|
+
provider: capability.provider,
|
|
32473
|
+
adapter_version: capability.adapterVersion,
|
|
32474
|
+
effective_policies: {
|
|
32475
|
+
capability_policy: request.productPins.capability_policy,
|
|
32476
|
+
resource_policy: request.productPins.resource_policy,
|
|
32477
|
+
adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
|
|
32478
|
+
acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
|
|
32479
|
+
},
|
|
32480
|
+
capture_bundle_id: normalized.captureBundleId,
|
|
32481
|
+
scope: request.wire.scope,
|
|
32482
|
+
window: request.wire.window,
|
|
32483
|
+
depth: request.depth,
|
|
32484
|
+
construction_mode: request.constructionMode,
|
|
32485
|
+
canonical_schema: request.expectedCanonicalSchema,
|
|
32486
|
+
coverage_policy: request.coveragePolicy,
|
|
32487
|
+
selection_sha256: request.initialSelection.selection_sha256,
|
|
32488
|
+
vendor_semantic_digest: normalized.vendorSemanticDigest,
|
|
32489
|
+
canonical_semantic_digest: verification.canonicalSemanticDigest,
|
|
32490
|
+
prefix_digest: verification.prefixDigest,
|
|
32491
|
+
suffix_digest: verification.suffixDigest,
|
|
32492
|
+
seam_verified: true,
|
|
32493
|
+
coverage_verified: true,
|
|
32494
|
+
dataset_objects: normalized.objects
|
|
32495
|
+
});
|
|
32496
|
+
}
|
|
32497
|
+
async function submitAll(dependencies, batches) {
|
|
32498
|
+
const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
|
|
32499
|
+
for (const batch of batches) {
|
|
32500
|
+
let accepted = false;
|
|
32501
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
|
|
32502
|
+
try {
|
|
32503
|
+
const response = await dependencies.forwarder.submit(batch);
|
|
32504
|
+
accepted = response.ok && response.inserted === batch.rows.length;
|
|
32505
|
+
} catch {
|
|
32506
|
+
accepted = false;
|
|
32507
|
+
}
|
|
32508
|
+
if (accepted)
|
|
32702
32509
|
break;
|
|
32703
|
-
|
|
32510
|
+
if (attempt < maxAttempts)
|
|
32511
|
+
await dependencies.retry?.wait?.(attempt);
|
|
32704
32512
|
}
|
|
32705
|
-
if (!
|
|
32706
|
-
|
|
32513
|
+
if (!accepted)
|
|
32514
|
+
return false;
|
|
32515
|
+
}
|
|
32516
|
+
return true;
|
|
32517
|
+
}
|
|
32518
|
+
function stableFailureReason(error48, fallback) {
|
|
32519
|
+
if (error48 && typeof error48 === "object") {
|
|
32520
|
+
const reason = error48.reason;
|
|
32521
|
+
if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
|
|
32522
|
+
return reason;
|
|
32707
32523
|
}
|
|
32708
|
-
|
|
32709
|
-
|
|
32710
|
-
|
|
32711
|
-
|
|
32524
|
+
}
|
|
32525
|
+
return fallback;
|
|
32526
|
+
}
|
|
32527
|
+
function assertArchivePreflight(request, resolution) {
|
|
32528
|
+
if (!request.target || !request.productionAuthorizationId) {
|
|
32529
|
+
throw new Error("request target or production authorization ID is missing");
|
|
32530
|
+
}
|
|
32531
|
+
const selection = archiveSelectionCodec.decode(resolution.selection);
|
|
32532
|
+
const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
|
|
32533
|
+
const receiptById = new Map;
|
|
32534
|
+
for (const receipt of receipts) {
|
|
32535
|
+
const existing = receiptById.get(receipt.receipt_id);
|
|
32536
|
+
if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
|
|
32537
|
+
throw new Error("stored receipt identity has conflicting content");
|
|
32712
32538
|
}
|
|
32713
|
-
|
|
32714
|
-
|
|
32539
|
+
receiptById.set(receipt.receipt_id, receipt);
|
|
32540
|
+
}
|
|
32541
|
+
for (const bundle of selection.bundles) {
|
|
32542
|
+
if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
|
|
32543
|
+
throw new Error("vendor selection lacks its validated stored receipt");
|
|
32715
32544
|
}
|
|
32716
|
-
samples.push({
|
|
32717
|
-
targetTimeMs,
|
|
32718
|
-
sourceTimeMs: prior.sourceTimeMs,
|
|
32719
|
-
receivedTimeMs: prior.receivedTimeMs,
|
|
32720
|
-
sequence: prior.sequence,
|
|
32721
|
-
bids,
|
|
32722
|
-
asks,
|
|
32723
|
-
datasetObjectIdentity: prior.datasetObjectIdentity,
|
|
32724
|
-
datasetObjectChecksum: prior.datasetObjectChecksum
|
|
32725
|
-
});
|
|
32726
32545
|
}
|
|
32727
|
-
|
|
32546
|
+
if (resolution.readerIdentity.environment !== request.target.environment || resolution.readerIdentity.cluster !== request.target.cluster) {
|
|
32547
|
+
throw new Error("archive reader cluster identity mismatch");
|
|
32548
|
+
}
|
|
32728
32549
|
}
|
|
32729
|
-
function
|
|
32730
|
-
|
|
32550
|
+
function assertForwarderPreflight(request, resolution, nowMs) {
|
|
32551
|
+
if (!request.target || !request.productionAuthorizationId) {
|
|
32552
|
+
throw new Error("request target or production authorization ID is missing");
|
|
32553
|
+
}
|
|
32554
|
+
if (resolution.forwarderIdentity.environment !== request.target.environment || resolution.forwarderIdentity.cluster !== request.target.cluster) {
|
|
32555
|
+
throw new Error("archive forwarder cluster identity mismatch");
|
|
32556
|
+
}
|
|
32557
|
+
const authorization = resolution.authorization;
|
|
32558
|
+
const expiresAtMs = Date.parse(authorization.expiresAt);
|
|
32559
|
+
if (authorization.authorizationId !== request.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request.target.environment || authorization.cluster !== request.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
|
|
32560
|
+
throw new Error("production forwarder authorization is invalid");
|
|
32561
|
+
}
|
|
32731
32562
|
}
|
|
32732
|
-
|
|
32733
|
-
const
|
|
32734
|
-
|
|
32735
|
-
|
|
32563
|
+
function resourcePolicyScopeExceeded(request) {
|
|
32564
|
+
const policy = [RESOURCE_POLICY, LEGACY_RESOURCE_POLICY].find((candidate) => request.productPins?.resource_policy.policy_id === candidate.policy_id && request.productPins.resource_policy.policy_sha256 === candidate.policy_sha256);
|
|
32565
|
+
if (!policy)
|
|
32566
|
+
return true;
|
|
32567
|
+
return request.depth > policy.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > policy.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > policy.request_bounds.max_required_events;
|
|
32736
32568
|
}
|
|
32737
|
-
function
|
|
32738
|
-
|
|
32739
|
-
|
|
32740
|
-
event_time: value.event_time,
|
|
32741
|
-
transaction_time: value.transaction_time,
|
|
32742
|
-
symbol: value.symbol,
|
|
32743
|
-
event_type: value.event_type,
|
|
32744
|
-
first_update_id: value.first_update_id,
|
|
32745
|
-
final_update_id: value.final_update_id,
|
|
32746
|
-
prev_final_update_id: value.prev_final_update_id,
|
|
32747
|
-
last_update_id: value.last_update_id,
|
|
32748
|
-
side: value.side,
|
|
32749
|
-
price: value.price,
|
|
32750
|
-
quantity: value.quantity,
|
|
32751
|
-
order_count: value.order_count,
|
|
32752
|
-
dataset_object_identity: object2.identity,
|
|
32753
|
-
dataset_object_checksum: object2.checksum
|
|
32754
|
-
};
|
|
32569
|
+
function storedReceiptForSelection(resolution) {
|
|
32570
|
+
const receiptId = resolution.selection.receipt_ids[0];
|
|
32571
|
+
return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
|
|
32755
32572
|
}
|
|
32756
|
-
async function
|
|
32757
|
-
|
|
32758
|
-
|
|
32759
|
-
|
|
32760
|
-
|
|
32761
|
-
|
|
32762
|
-
|
|
32763
|
-
|
|
32764
|
-
|
|
32765
|
-
|
|
32766
|
-
total += value.byteLength;
|
|
32767
|
-
if (total > remainingBytes) {
|
|
32768
|
-
await reader.cancel();
|
|
32769
|
-
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32770
|
-
}
|
|
32771
|
-
chunks.push(value);
|
|
32573
|
+
async function runMarketDataVendorBackfill(input, dependencies) {
|
|
32574
|
+
let request;
|
|
32575
|
+
try {
|
|
32576
|
+
const documents = input;
|
|
32577
|
+
request = decodeBackfillRunDocuments({
|
|
32578
|
+
request: documents.request,
|
|
32579
|
+
requiredClock: documents.requiredClock
|
|
32580
|
+
});
|
|
32581
|
+
} catch {
|
|
32582
|
+
return outcome(undefined, "request_invalid", "request_invalid");
|
|
32772
32583
|
}
|
|
32773
|
-
|
|
32774
|
-
|
|
32775
|
-
|
|
32776
|
-
|
|
32777
|
-
|
|
32584
|
+
let initialResolution;
|
|
32585
|
+
try {
|
|
32586
|
+
initialResolution = await dependencies.archive.resolveSelection(request);
|
|
32587
|
+
assertArchivePreflight(request, initialResolution);
|
|
32588
|
+
const forwarderPreflight = await dependencies.forwarder.preflight({
|
|
32589
|
+
authorizationId: request.productionAuthorizationId,
|
|
32590
|
+
target: request.target
|
|
32591
|
+
});
|
|
32592
|
+
assertForwarderPreflight(request, forwarderPreflight, dependencies.clock.nowMs());
|
|
32593
|
+
} catch {
|
|
32594
|
+
return outcome(request, "archive_preflight_failed", "archive_preflight_failed");
|
|
32778
32595
|
}
|
|
32779
|
-
|
|
32780
|
-
|
|
32781
|
-
|
|
32782
|
-
|
|
32783
|
-
|
|
32784
|
-
|
|
32785
|
-
|
|
32786
|
-
|
|
32787
|
-
|
|
32788
|
-
|
|
32789
|
-
|
|
32790
|
-
|
|
32791
|
-
|
|
32792
|
-
|
|
32793
|
-
|
|
32596
|
+
if (initialResolution.selection.coverage_class === "complete") {
|
|
32597
|
+
const receipt2 = storedReceiptForSelection(initialResolution);
|
|
32598
|
+
return outcome(request, "already_covered", "qualified_coverage_complete", {
|
|
32599
|
+
selection: initialResolution.selection,
|
|
32600
|
+
...receipt2 ? { receipt: receipt2 } : {}
|
|
32601
|
+
});
|
|
32602
|
+
}
|
|
32603
|
+
if (resourcePolicyScopeExceeded(request)) {
|
|
32604
|
+
return outcome(request, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
|
|
32605
|
+
}
|
|
32606
|
+
if (request.sourcePolicy === "fill_gaps" && (request.productPins?.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request.productPins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256)) {
|
|
32607
|
+
return outcome(request, "capability_unsupported", "scope_unsupported", {
|
|
32608
|
+
reasonSubcode: "fill_gaps_requires_capability_policy_v2"
|
|
32609
|
+
});
|
|
32610
|
+
}
|
|
32611
|
+
let capability;
|
|
32612
|
+
try {
|
|
32613
|
+
capability = dependencies.providers.capabilityFor(request);
|
|
32614
|
+
} catch {
|
|
32615
|
+
return outcome(request, "capability_unsupported", "capability_probe_failed");
|
|
32616
|
+
}
|
|
32617
|
+
if (!capability) {
|
|
32618
|
+
return outcome(request, "capability_unsupported", "scope_unsupported");
|
|
32619
|
+
}
|
|
32620
|
+
if (!request.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
|
|
32621
|
+
return outcome(request, "capability_unsupported", "adapter_version_unpinned");
|
|
32794
32622
|
}
|
|
32795
|
-
|
|
32796
|
-
|
|
32623
|
+
let credential;
|
|
32624
|
+
try {
|
|
32625
|
+
credential = await dependencies.credentials.resolve(capability.provider);
|
|
32626
|
+
} catch {
|
|
32627
|
+
return outcome(request, "credentials_missing", "credential_resolution_failed");
|
|
32797
32628
|
}
|
|
32798
|
-
|
|
32799
|
-
|
|
32800
|
-
throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
|
|
32801
|
-
}
|
|
32802
|
-
const endpoint = new URL("/symbols", this.baseUrl);
|
|
32803
|
-
endpoint.searchParams.set("exchange", providerExchangeId);
|
|
32804
|
-
endpoint.searchParams.set("data_type", "orderbook");
|
|
32805
|
-
let response;
|
|
32806
|
-
try {
|
|
32807
|
-
response = await this.request(endpoint);
|
|
32808
|
-
} catch {
|
|
32809
|
-
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
32810
|
-
}
|
|
32811
|
-
if (!response.ok) {
|
|
32812
|
-
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
32813
|
-
}
|
|
32814
|
-
let body;
|
|
32815
|
-
try {
|
|
32816
|
-
body = await response.json();
|
|
32817
|
-
} catch {
|
|
32818
|
-
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
32819
|
-
}
|
|
32820
|
-
const record2 = body;
|
|
32821
|
-
if (!body || typeof body !== "object" || Array.isArray(body) || String(record2.exchange).toLowerCase() !== providerExchangeId || String(record2.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record2.symbols) || !record2.symbols.every((symbol2) => typeof symbol2 === "string" && /^[A-Z0-9_-]+$/.test(symbol2))) {
|
|
32822
|
-
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
32823
|
-
}
|
|
32824
|
-
return [...new Set(record2.symbols)].sort();
|
|
32629
|
+
if (credential === undefined || credential === null) {
|
|
32630
|
+
return outcome(request, "credentials_missing", "provider_credentials_missing");
|
|
32825
32631
|
}
|
|
32826
|
-
|
|
32827
|
-
|
|
32828
|
-
|
|
32829
|
-
|
|
32830
|
-
|
|
32831
|
-
const
|
|
32832
|
-
|
|
32833
|
-
|
|
32632
|
+
let dataset;
|
|
32633
|
+
let normalized;
|
|
32634
|
+
try {
|
|
32635
|
+
const acquisitionRequest = providerAcquisitionRequest(request);
|
|
32636
|
+
dataset = await dependencies.providers.acquire(acquisitionRequest, capability, credential);
|
|
32637
|
+
const bundleId = captureBundleId(request, capability, dataset);
|
|
32638
|
+
normalized = await dependencies.providers.normalize(acquisitionRequest, capability, dataset, bundleId);
|
|
32639
|
+
if (normalized.captureBundleId !== bundleId) {
|
|
32640
|
+
return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
32641
|
+
reasonSubcode: "capture_identity_mismatch"
|
|
32642
|
+
});
|
|
32834
32643
|
}
|
|
32835
|
-
|
|
32836
|
-
const
|
|
32837
|
-
|
|
32838
|
-
|
|
32644
|
+
} catch (error48) {
|
|
32645
|
+
const reason = stableFailureReason(error48, "provider_dataset_invalid");
|
|
32646
|
+
return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
32647
|
+
reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
|
|
32839
32648
|
});
|
|
32840
|
-
|
|
32841
|
-
|
|
32842
|
-
const
|
|
32843
|
-
|
|
32844
|
-
|
|
32649
|
+
}
|
|
32650
|
+
try {
|
|
32651
|
+
const candidateBatches = buildForwarderBatches({
|
|
32652
|
+
captureBundleId: normalized.captureBundleId,
|
|
32653
|
+
deploymentId: "market-data-vendor-backfill",
|
|
32654
|
+
rows: normalized.rows
|
|
32655
|
+
});
|
|
32656
|
+
if (!await submitAll(dependencies, candidateBatches)) {
|
|
32657
|
+
return outcome(request, "archive_ingest_failed", "candidate_batch_rejected");
|
|
32845
32658
|
}
|
|
32846
|
-
|
|
32847
|
-
|
|
32848
|
-
|
|
32849
|
-
|
|
32850
|
-
|
|
32851
|
-
|
|
32852
|
-
|
|
32853
|
-
|
|
32854
|
-
|
|
32855
|
-
|
|
32856
|
-
|
|
32857
|
-
|
|
32858
|
-
|
|
32859
|
-
|
|
32860
|
-
|
|
32861
|
-
|
|
32862
|
-
|
|
32863
|
-
|
|
32864
|
-
|
|
32865
|
-
|
|
32866
|
-
|
|
32867
|
-
|
|
32868
|
-
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32869
|
-
}
|
|
32870
|
-
const decoded = await this.decode(bytes);
|
|
32871
|
-
totalRows += decoded.length;
|
|
32872
|
-
if (totalRows > request.budgets.maxRows) {
|
|
32873
|
-
throw new CryptoHftDataError("budget_max_rows_exceeded");
|
|
32874
|
-
}
|
|
32875
|
-
const object2 = {
|
|
32876
|
-
identity: path2,
|
|
32877
|
-
checksum: sha256Bytes(bytes),
|
|
32878
|
-
bytes: bytes.byteLength,
|
|
32879
|
-
rows: decoded.length
|
|
32880
|
-
};
|
|
32881
|
-
objects.push(object2);
|
|
32882
|
-
for (const decodedRow of decoded) {
|
|
32883
|
-
const parsed = parsedDatasetRow(decodedRow, object2);
|
|
32884
|
-
validatedRow(request, parsed);
|
|
32885
|
-
rows.push(parsed);
|
|
32886
|
-
}
|
|
32659
|
+
} catch {
|
|
32660
|
+
return outcome(request, "archive_ingest_failed", "candidate_submission_failed");
|
|
32661
|
+
}
|
|
32662
|
+
let verification;
|
|
32663
|
+
try {
|
|
32664
|
+
verification = await dependencies.archive.verifyCandidate(request, normalized, normalized.captureBundleId, initialResolution);
|
|
32665
|
+
} catch {
|
|
32666
|
+
return outcome(request, "promotion_verification_failed", "candidate_query_failed");
|
|
32667
|
+
}
|
|
32668
|
+
if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
|
|
32669
|
+
return outcome(request, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
|
|
32670
|
+
}
|
|
32671
|
+
let receipt;
|
|
32672
|
+
try {
|
|
32673
|
+
receipt = buildPromotionReceipt(request, capability, normalized, verification, dependencies.clock.nowMs());
|
|
32674
|
+
const [batch] = buildForwarderBatches({
|
|
32675
|
+
captureBundleId: normalized.captureBundleId,
|
|
32676
|
+
deploymentId: "market-data-vendor-backfill",
|
|
32677
|
+
rows: [promotionReceiptToArchiveRow(receipt)]
|
|
32678
|
+
});
|
|
32679
|
+
if (!batch || !await submitAll(dependencies, [batch])) {
|
|
32680
|
+
return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
|
|
32887
32681
|
}
|
|
32888
|
-
|
|
32889
|
-
|
|
32682
|
+
const qualification = finalizeQualificationEvent({
|
|
32683
|
+
capture_bundle_id: receipt.capture_bundle_id,
|
|
32684
|
+
state: "qualified",
|
|
32685
|
+
receipt_id: receipt.receipt_id,
|
|
32686
|
+
promotion_identity_sha256: receipt.promotion_identity_sha256,
|
|
32687
|
+
window: receipt.window,
|
|
32688
|
+
event_at: receipt.verified_at,
|
|
32689
|
+
reason_code: "promotion_verified"
|
|
32690
|
+
});
|
|
32691
|
+
const [qualificationBatch] = buildForwarderBatches({
|
|
32692
|
+
captureBundleId: normalized.captureBundleId,
|
|
32693
|
+
deploymentId: "market-data-vendor-backfill",
|
|
32694
|
+
rows: [qualificationEventToArchiveRow(qualification)]
|
|
32695
|
+
});
|
|
32696
|
+
if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
|
|
32697
|
+
return outcome(request, "archive_ingest_failed", "qualification_commit_failed", { receipt });
|
|
32890
32698
|
}
|
|
32891
|
-
|
|
32892
|
-
|
|
32893
|
-
rows,
|
|
32894
|
-
vendorSemanticDigest: sha256Canonical(rows)
|
|
32895
|
-
};
|
|
32699
|
+
} catch {
|
|
32700
|
+
return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
|
|
32896
32701
|
}
|
|
32897
|
-
|
|
32898
|
-
|
|
32899
|
-
|
|
32900
|
-
|
|
32901
|
-
|
|
32902
|
-
|
|
32903
|
-
|
|
32904
|
-
|
|
32702
|
+
let finalResolution;
|
|
32703
|
+
try {
|
|
32704
|
+
finalResolution = await dependencies.archive.resolveSelection(request);
|
|
32705
|
+
assertArchivePreflight(request, finalResolution);
|
|
32706
|
+
} catch (error48) {
|
|
32707
|
+
return outcome(request, "promotion_verification_failed", "post_promotion_selection_failed", {
|
|
32708
|
+
receipt,
|
|
32709
|
+
reasonSubcode: stableFailureReason(error48, "archive_selection_resolution_failed")
|
|
32710
|
+
});
|
|
32711
|
+
}
|
|
32712
|
+
if (finalResolution.selection.coverage_class !== "complete") {
|
|
32713
|
+
return outcome(request, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
|
|
32714
|
+
}
|
|
32715
|
+
if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
|
|
32716
|
+
return outcome(request, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
|
|
32717
|
+
}
|
|
32718
|
+
try {
|
|
32719
|
+
if (!request.wire)
|
|
32720
|
+
throw new Error("decoded request wire is missing");
|
|
32721
|
+
const [selectionBatch] = buildForwarderBatches({
|
|
32722
|
+
captureBundleId: receipt.capture_bundle_id,
|
|
32905
32723
|
deploymentId: "market-data-vendor-backfill",
|
|
32906
|
-
|
|
32907
|
-
|
|
32908
|
-
|
|
32909
|
-
tradingPair: request.scope.tradingPair,
|
|
32910
|
-
sourceSymbol: request.scope.sourceSymbol,
|
|
32911
|
-
assetType: request.scope.marketType,
|
|
32912
|
-
feed: "ORDERBOOK",
|
|
32913
|
-
provider: "cryptohftdata",
|
|
32914
|
-
sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
|
|
32915
|
-
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
32916
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
32917
|
-
provenanceComplete: true
|
|
32918
|
-
};
|
|
32919
|
-
const rows = samples.flatMap((sample) => {
|
|
32920
|
-
const rawCapture = {
|
|
32921
|
-
rawCaptureId: sha256Canonical({
|
|
32922
|
-
capture_bundle_id: captureBundleId2,
|
|
32923
|
-
dataset_object_identity: sample.datasetObjectIdentity,
|
|
32924
|
-
dataset_object_checksum: sample.datasetObjectChecksum,
|
|
32925
|
-
source_time_ms: sample.sourceTimeMs,
|
|
32926
|
-
target_time_ms: sample.targetTimeMs
|
|
32927
|
-
}),
|
|
32928
|
-
rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
|
|
32929
|
-
rawChecksum: sample.datasetObjectChecksum,
|
|
32930
|
-
redactedPayload: {
|
|
32931
|
-
dataset_object_identity: sample.datasetObjectIdentity,
|
|
32932
|
-
dataset_object_checksum: sample.datasetObjectChecksum
|
|
32933
|
-
},
|
|
32934
|
-
eventTimeMs: sample.sourceTimeMs,
|
|
32935
|
-
receivedTimeMs: sample.receivedTimeMs,
|
|
32936
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM
|
|
32937
|
-
};
|
|
32938
|
-
const canonical = buildCanonicalOrderBookRows({
|
|
32939
|
-
context: context2,
|
|
32940
|
-
rawCapture,
|
|
32941
|
-
depthLimit: request.depth,
|
|
32942
|
-
constructionMode: request.constructionMode,
|
|
32943
|
-
snapshot: {
|
|
32944
|
-
bids: sample.bids,
|
|
32945
|
-
asks: sample.asks,
|
|
32946
|
-
timestamp: sample.sourceTimeMs,
|
|
32947
|
-
receivedTimestamp: sample.receivedTimeMs,
|
|
32948
|
-
exchange: request.scope.exchange,
|
|
32949
|
-
symbol: request.scope.tradingPair,
|
|
32950
|
-
depthLimit: request.depth,
|
|
32951
|
-
sequence: sample.sequence
|
|
32952
|
-
}
|
|
32953
|
-
});
|
|
32954
|
-
return [...canonical.levels, canonical.summary];
|
|
32724
|
+
rows: [
|
|
32725
|
+
archiveSelectionToArchiveRow(request.wire, finalResolution.selection)
|
|
32726
|
+
]
|
|
32955
32727
|
});
|
|
32956
|
-
|
|
32957
|
-
|
|
32958
|
-
|
|
32959
|
-
|
|
32960
|
-
|
|
32961
|
-
canonicalSemanticDigest: semanticDigest(rows)
|
|
32962
|
-
};
|
|
32728
|
+
if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
|
|
32729
|
+
return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
32730
|
+
}
|
|
32731
|
+
} catch {
|
|
32732
|
+
return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
32963
32733
|
}
|
|
32734
|
+
return outcome(request, "promoted", "promotion_qualified", {
|
|
32735
|
+
receipt,
|
|
32736
|
+
selection: finalResolution.selection
|
|
32737
|
+
});
|
|
32964
32738
|
}
|
|
32965
32739
|
|
|
32966
32740
|
// src/helpers/market-data-vendor-backfill/forwarder-client.ts
|
|
@@ -33081,8 +32855,8 @@ function validateReleaseIdentity(release) {
|
|
|
33081
32855
|
return release;
|
|
33082
32856
|
}
|
|
33083
32857
|
function bakedReleaseIdentity() {
|
|
33084
|
-
const packageVersion = "0.2.
|
|
33085
|
-
const gitHead = "
|
|
32858
|
+
const packageVersion = "0.2.50";
|
|
32859
|
+
const gitHead = "41dbe7cc296da1c5a7964dca8f418800522f27e5";
|
|
33086
32860
|
return validateReleaseIdentity({ packageVersion, gitHead });
|
|
33087
32861
|
}
|
|
33088
32862
|
function readOptionalEnvironment(environment, name) {
|
|
@@ -33106,7 +32880,10 @@ function createBackfillDependenciesFromEnv(environment, sensitiveValues = new Se
|
|
|
33106
32880
|
password: clickhousePassword
|
|
33107
32881
|
})),
|
|
33108
32882
|
providers: new CryptoHftDataAdapter({
|
|
33109
|
-
profiles: [
|
|
32883
|
+
profiles: [
|
|
32884
|
+
CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE,
|
|
32885
|
+
CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE
|
|
32886
|
+
]
|
|
33110
32887
|
}),
|
|
33111
32888
|
forwarder: createArchiveForwarderClient({
|
|
33112
32889
|
url: forwarderUrl,
|
|
@@ -33313,4 +33090,4 @@ export {
|
|
|
33313
33090
|
createBackfillDependenciesFromEnv
|
|
33314
33091
|
};
|
|
33315
33092
|
|
|
33316
|
-
//# debugId=
|
|
33093
|
+
//# debugId=7BCBC113F3C0009864756E2164756E21
|