@usherlabs/cex-broker 0.2.47 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cex-canonical-orderbook-export.js +30 -350
- package/dist/commands/cex-canonical-orderbook-export.js.map +7 -19
- package/dist/commands/cli.js +100 -35
- package/dist/commands/market-data-vendor-backfill.js +1463 -1742
- package/dist/commands/market-data-vendor-backfill.js.map +12 -24
- package/dist/helpers/logger.d.ts +4 -0
- package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +2 -1
- package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +29 -2
- package/dist/helpers/market-data-vendor-backfill/semantic-verification.d.ts +1 -0
- package/dist/helpers/otel.d.ts +1 -0
- package/dist/index.js +59 -31
- package/dist/index.js.map +5 -5
- package/dist/market-data-preparation/fixtures/conformance-v2.json +2 -2
- package/dist/market-data-preparation/policies/capability-policy-v1.json +28 -0
- package/dist/market-data-preparation/policies/capability-policy.json +3 -3
- package/dist/market-data-preparation.js +30 -351
- package/dist/market-data-preparation.js.map +9 -21
- package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +28 -0
- package/dist/market-data-vendor-backfill.js +1028 -1306
- package/dist/market-data-vendor-backfill.js.map +13 -25
- package/package.json +1 -1
|
@@ -8257,241 +8257,6 @@ var require_src = __commonJS((exports) => {
|
|
|
8257
8257
|
};
|
|
8258
8258
|
});
|
|
8259
8259
|
|
|
8260
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/types/LogRecord.js
|
|
8261
|
-
var require_LogRecord = __commonJS((exports) => {
|
|
8262
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8263
|
-
exports.SeverityNumber = undefined;
|
|
8264
|
-
var SeverityNumber;
|
|
8265
|
-
(function(SeverityNumber2) {
|
|
8266
|
-
SeverityNumber2[SeverityNumber2["UNSPECIFIED"] = 0] = "UNSPECIFIED";
|
|
8267
|
-
SeverityNumber2[SeverityNumber2["TRACE"] = 1] = "TRACE";
|
|
8268
|
-
SeverityNumber2[SeverityNumber2["TRACE2"] = 2] = "TRACE2";
|
|
8269
|
-
SeverityNumber2[SeverityNumber2["TRACE3"] = 3] = "TRACE3";
|
|
8270
|
-
SeverityNumber2[SeverityNumber2["TRACE4"] = 4] = "TRACE4";
|
|
8271
|
-
SeverityNumber2[SeverityNumber2["DEBUG"] = 5] = "DEBUG";
|
|
8272
|
-
SeverityNumber2[SeverityNumber2["DEBUG2"] = 6] = "DEBUG2";
|
|
8273
|
-
SeverityNumber2[SeverityNumber2["DEBUG3"] = 7] = "DEBUG3";
|
|
8274
|
-
SeverityNumber2[SeverityNumber2["DEBUG4"] = 8] = "DEBUG4";
|
|
8275
|
-
SeverityNumber2[SeverityNumber2["INFO"] = 9] = "INFO";
|
|
8276
|
-
SeverityNumber2[SeverityNumber2["INFO2"] = 10] = "INFO2";
|
|
8277
|
-
SeverityNumber2[SeverityNumber2["INFO3"] = 11] = "INFO3";
|
|
8278
|
-
SeverityNumber2[SeverityNumber2["INFO4"] = 12] = "INFO4";
|
|
8279
|
-
SeverityNumber2[SeverityNumber2["WARN"] = 13] = "WARN";
|
|
8280
|
-
SeverityNumber2[SeverityNumber2["WARN2"] = 14] = "WARN2";
|
|
8281
|
-
SeverityNumber2[SeverityNumber2["WARN3"] = 15] = "WARN3";
|
|
8282
|
-
SeverityNumber2[SeverityNumber2["WARN4"] = 16] = "WARN4";
|
|
8283
|
-
SeverityNumber2[SeverityNumber2["ERROR"] = 17] = "ERROR";
|
|
8284
|
-
SeverityNumber2[SeverityNumber2["ERROR2"] = 18] = "ERROR2";
|
|
8285
|
-
SeverityNumber2[SeverityNumber2["ERROR3"] = 19] = "ERROR3";
|
|
8286
|
-
SeverityNumber2[SeverityNumber2["ERROR4"] = 20] = "ERROR4";
|
|
8287
|
-
SeverityNumber2[SeverityNumber2["FATAL"] = 21] = "FATAL";
|
|
8288
|
-
SeverityNumber2[SeverityNumber2["FATAL2"] = 22] = "FATAL2";
|
|
8289
|
-
SeverityNumber2[SeverityNumber2["FATAL3"] = 23] = "FATAL3";
|
|
8290
|
-
SeverityNumber2[SeverityNumber2["FATAL4"] = 24] = "FATAL4";
|
|
8291
|
-
})(SeverityNumber = exports.SeverityNumber || (exports.SeverityNumber = {}));
|
|
8292
|
-
});
|
|
8293
|
-
|
|
8294
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLogger.js
|
|
8295
|
-
var require_NoopLogger = __commonJS((exports) => {
|
|
8296
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8297
|
-
exports.NOOP_LOGGER = exports.NoopLogger = undefined;
|
|
8298
|
-
|
|
8299
|
-
class NoopLogger {
|
|
8300
|
-
emit(_logRecord) {}
|
|
8301
|
-
}
|
|
8302
|
-
exports.NoopLogger = NoopLogger;
|
|
8303
|
-
exports.NOOP_LOGGER = new NoopLogger;
|
|
8304
|
-
});
|
|
8305
|
-
|
|
8306
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/NoopLoggerProvider.js
|
|
8307
|
-
var require_NoopLoggerProvider = __commonJS((exports) => {
|
|
8308
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8309
|
-
exports.NOOP_LOGGER_PROVIDER = exports.NoopLoggerProvider = undefined;
|
|
8310
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
8311
|
-
|
|
8312
|
-
class NoopLoggerProvider {
|
|
8313
|
-
getLogger(_name, _version, _options) {
|
|
8314
|
-
return new NoopLogger_1.NoopLogger;
|
|
8315
|
-
}
|
|
8316
|
-
}
|
|
8317
|
-
exports.NoopLoggerProvider = NoopLoggerProvider;
|
|
8318
|
-
exports.NOOP_LOGGER_PROVIDER = new NoopLoggerProvider;
|
|
8319
|
-
});
|
|
8320
|
-
|
|
8321
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLogger.js
|
|
8322
|
-
var require_ProxyLogger = __commonJS((exports) => {
|
|
8323
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8324
|
-
exports.ProxyLogger = undefined;
|
|
8325
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
8326
|
-
|
|
8327
|
-
class ProxyLogger {
|
|
8328
|
-
constructor(_provider, name, version2, options) {
|
|
8329
|
-
this._provider = _provider;
|
|
8330
|
-
this.name = name;
|
|
8331
|
-
this.version = version2;
|
|
8332
|
-
this.options = options;
|
|
8333
|
-
}
|
|
8334
|
-
emit(logRecord) {
|
|
8335
|
-
this._getLogger().emit(logRecord);
|
|
8336
|
-
}
|
|
8337
|
-
_getLogger() {
|
|
8338
|
-
if (this._delegate) {
|
|
8339
|
-
return this._delegate;
|
|
8340
|
-
}
|
|
8341
|
-
const logger = this._provider._getDelegateLogger(this.name, this.version, this.options);
|
|
8342
|
-
if (!logger) {
|
|
8343
|
-
return NoopLogger_1.NOOP_LOGGER;
|
|
8344
|
-
}
|
|
8345
|
-
this._delegate = logger;
|
|
8346
|
-
return this._delegate;
|
|
8347
|
-
}
|
|
8348
|
-
}
|
|
8349
|
-
exports.ProxyLogger = ProxyLogger;
|
|
8350
|
-
});
|
|
8351
|
-
|
|
8352
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/ProxyLoggerProvider.js
|
|
8353
|
-
var require_ProxyLoggerProvider = __commonJS((exports) => {
|
|
8354
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8355
|
-
exports.ProxyLoggerProvider = undefined;
|
|
8356
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
8357
|
-
var ProxyLogger_1 = require_ProxyLogger();
|
|
8358
|
-
|
|
8359
|
-
class ProxyLoggerProvider {
|
|
8360
|
-
getLogger(name, version2, options) {
|
|
8361
|
-
var _a2;
|
|
8362
|
-
return (_a2 = this._getDelegateLogger(name, version2, options)) !== null && _a2 !== undefined ? _a2 : new ProxyLogger_1.ProxyLogger(this, name, version2, options);
|
|
8363
|
-
}
|
|
8364
|
-
_getDelegate() {
|
|
8365
|
-
var _a2;
|
|
8366
|
-
return (_a2 = this._delegate) !== null && _a2 !== undefined ? _a2 : NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
|
|
8367
|
-
}
|
|
8368
|
-
_setDelegate(delegate) {
|
|
8369
|
-
this._delegate = delegate;
|
|
8370
|
-
}
|
|
8371
|
-
_getDelegateLogger(name, version2, options) {
|
|
8372
|
-
var _a2;
|
|
8373
|
-
return (_a2 = this._delegate) === null || _a2 === undefined ? undefined : _a2.getLogger(name, version2, options);
|
|
8374
|
-
}
|
|
8375
|
-
}
|
|
8376
|
-
exports.ProxyLoggerProvider = ProxyLoggerProvider;
|
|
8377
|
-
});
|
|
8378
|
-
|
|
8379
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/globalThis.js
|
|
8380
|
-
var require_globalThis2 = __commonJS((exports) => {
|
|
8381
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8382
|
-
exports._globalThis = undefined;
|
|
8383
|
-
exports._globalThis = typeof globalThis === "object" ? globalThis : global;
|
|
8384
|
-
});
|
|
8385
|
-
|
|
8386
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/node/index.js
|
|
8387
|
-
var require_node2 = __commonJS((exports) => {
|
|
8388
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8389
|
-
exports._globalThis = undefined;
|
|
8390
|
-
var globalThis_1 = require_globalThis2();
|
|
8391
|
-
Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
|
|
8392
|
-
return globalThis_1._globalThis;
|
|
8393
|
-
} });
|
|
8394
|
-
});
|
|
8395
|
-
|
|
8396
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/platform/index.js
|
|
8397
|
-
var require_platform2 = __commonJS((exports) => {
|
|
8398
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8399
|
-
exports._globalThis = undefined;
|
|
8400
|
-
var node_1 = require_node2();
|
|
8401
|
-
Object.defineProperty(exports, "_globalThis", { enumerable: true, get: function() {
|
|
8402
|
-
return node_1._globalThis;
|
|
8403
|
-
} });
|
|
8404
|
-
});
|
|
8405
|
-
|
|
8406
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/internal/global-utils.js
|
|
8407
|
-
var require_global_utils2 = __commonJS((exports) => {
|
|
8408
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8409
|
-
exports.API_BACKWARDS_COMPATIBILITY_VERSION = exports.makeGetter = exports._global = exports.GLOBAL_LOGS_API_KEY = undefined;
|
|
8410
|
-
var platform_1 = require_platform2();
|
|
8411
|
-
exports.GLOBAL_LOGS_API_KEY = Symbol.for("io.opentelemetry.js.api.logs");
|
|
8412
|
-
exports._global = platform_1._globalThis;
|
|
8413
|
-
function makeGetter(requiredVersion, instance, fallback) {
|
|
8414
|
-
return (version2) => version2 === requiredVersion ? instance : fallback;
|
|
8415
|
-
}
|
|
8416
|
-
exports.makeGetter = makeGetter;
|
|
8417
|
-
exports.API_BACKWARDS_COMPATIBILITY_VERSION = 1;
|
|
8418
|
-
});
|
|
8419
|
-
|
|
8420
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/api/logs.js
|
|
8421
|
-
var require_logs = __commonJS((exports) => {
|
|
8422
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8423
|
-
exports.LogsAPI = undefined;
|
|
8424
|
-
var global_utils_1 = require_global_utils2();
|
|
8425
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
8426
|
-
var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
|
|
8427
|
-
|
|
8428
|
-
class LogsAPI {
|
|
8429
|
-
constructor() {
|
|
8430
|
-
this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
8431
|
-
}
|
|
8432
|
-
static getInstance() {
|
|
8433
|
-
if (!this._instance) {
|
|
8434
|
-
this._instance = new LogsAPI;
|
|
8435
|
-
}
|
|
8436
|
-
return this._instance;
|
|
8437
|
-
}
|
|
8438
|
-
setGlobalLoggerProvider(provider) {
|
|
8439
|
-
if (global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) {
|
|
8440
|
-
return this.getLoggerProvider();
|
|
8441
|
-
}
|
|
8442
|
-
global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY] = (0, global_utils_1.makeGetter)(global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION, provider, NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER);
|
|
8443
|
-
this._proxyLoggerProvider._setDelegate(provider);
|
|
8444
|
-
return provider;
|
|
8445
|
-
}
|
|
8446
|
-
getLoggerProvider() {
|
|
8447
|
-
var _a2, _b;
|
|
8448
|
-
return (_b = (_a2 = global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY]) === null || _a2 === undefined ? undefined : _a2.call(global_utils_1._global, global_utils_1.API_BACKWARDS_COMPATIBILITY_VERSION)) !== null && _b !== undefined ? _b : this._proxyLoggerProvider;
|
|
8449
|
-
}
|
|
8450
|
-
getLogger(name, version2, options) {
|
|
8451
|
-
return this.getLoggerProvider().getLogger(name, version2, options);
|
|
8452
|
-
}
|
|
8453
|
-
disable() {
|
|
8454
|
-
delete global_utils_1._global[global_utils_1.GLOBAL_LOGS_API_KEY];
|
|
8455
|
-
this._proxyLoggerProvider = new ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
8456
|
-
}
|
|
8457
|
-
}
|
|
8458
|
-
exports.LogsAPI = LogsAPI;
|
|
8459
|
-
});
|
|
8460
|
-
|
|
8461
|
-
// node_modules/@loglayer/transport-opentelemetry/node_modules/@opentelemetry/api-logs/build/src/index.js
|
|
8462
|
-
var require_src2 = __commonJS((exports) => {
|
|
8463
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8464
|
-
exports.logs = exports.ProxyLoggerProvider = exports.ProxyLogger = exports.NoopLoggerProvider = exports.NOOP_LOGGER_PROVIDER = exports.NoopLogger = exports.NOOP_LOGGER = exports.SeverityNumber = undefined;
|
|
8465
|
-
var LogRecord_1 = require_LogRecord();
|
|
8466
|
-
Object.defineProperty(exports, "SeverityNumber", { enumerable: true, get: function() {
|
|
8467
|
-
return LogRecord_1.SeverityNumber;
|
|
8468
|
-
} });
|
|
8469
|
-
var NoopLogger_1 = require_NoopLogger();
|
|
8470
|
-
Object.defineProperty(exports, "NOOP_LOGGER", { enumerable: true, get: function() {
|
|
8471
|
-
return NoopLogger_1.NOOP_LOGGER;
|
|
8472
|
-
} });
|
|
8473
|
-
Object.defineProperty(exports, "NoopLogger", { enumerable: true, get: function() {
|
|
8474
|
-
return NoopLogger_1.NoopLogger;
|
|
8475
|
-
} });
|
|
8476
|
-
var NoopLoggerProvider_1 = require_NoopLoggerProvider();
|
|
8477
|
-
Object.defineProperty(exports, "NOOP_LOGGER_PROVIDER", { enumerable: true, get: function() {
|
|
8478
|
-
return NoopLoggerProvider_1.NOOP_LOGGER_PROVIDER;
|
|
8479
|
-
} });
|
|
8480
|
-
Object.defineProperty(exports, "NoopLoggerProvider", { enumerable: true, get: function() {
|
|
8481
|
-
return NoopLoggerProvider_1.NoopLoggerProvider;
|
|
8482
|
-
} });
|
|
8483
|
-
var ProxyLogger_1 = require_ProxyLogger();
|
|
8484
|
-
Object.defineProperty(exports, "ProxyLogger", { enumerable: true, get: function() {
|
|
8485
|
-
return ProxyLogger_1.ProxyLogger;
|
|
8486
|
-
} });
|
|
8487
|
-
var ProxyLoggerProvider_1 = require_ProxyLoggerProvider();
|
|
8488
|
-
Object.defineProperty(exports, "ProxyLoggerProvider", { enumerable: true, get: function() {
|
|
8489
|
-
return ProxyLoggerProvider_1.ProxyLoggerProvider;
|
|
8490
|
-
} });
|
|
8491
|
-
var logs_1 = require_logs();
|
|
8492
|
-
exports.logs = logs_1.LogsAPI.getInstance();
|
|
8493
|
-
});
|
|
8494
|
-
|
|
8495
8260
|
// src/commands/market-data-vendor-backfill.ts
|
|
8496
8261
|
import { createHash as createHash5, randomUUID as randomUUID2 } from "node:crypto";
|
|
8497
8262
|
import path2 from "node:path";
|
|
@@ -22254,102 +22019,6 @@ var BaseTransport = class {
|
|
|
22254
22019
|
return this.logger;
|
|
22255
22020
|
}
|
|
22256
22021
|
};
|
|
22257
|
-
var LoggerlessTransport = class {
|
|
22258
|
-
id;
|
|
22259
|
-
enabled;
|
|
22260
|
-
level;
|
|
22261
|
-
consoleDebug;
|
|
22262
|
-
constructor(config2) {
|
|
22263
|
-
this.id = Date.now().toString() + Math.random().toString();
|
|
22264
|
-
this.enabled = config2.enabled ?? true;
|
|
22265
|
-
this.consoleDebug = config2.consoleDebug ?? false;
|
|
22266
|
-
this.level = config2.level ?? "trace";
|
|
22267
|
-
}
|
|
22268
|
-
_sendToLogger(params) {
|
|
22269
|
-
if (!this.enabled)
|
|
22270
|
-
return;
|
|
22271
|
-
if (LogLevelPriority[params.logLevel] < LogLevelPriority[this.level])
|
|
22272
|
-
return;
|
|
22273
|
-
const messages = this.shipToLogger(params);
|
|
22274
|
-
if (this.consoleDebug)
|
|
22275
|
-
switch (params.logLevel) {
|
|
22276
|
-
case LogLevel.info:
|
|
22277
|
-
console.info(...messages);
|
|
22278
|
-
break;
|
|
22279
|
-
case LogLevel.warn:
|
|
22280
|
-
console.warn(...messages);
|
|
22281
|
-
break;
|
|
22282
|
-
case LogLevel.error:
|
|
22283
|
-
console.error(...messages);
|
|
22284
|
-
break;
|
|
22285
|
-
case LogLevel.trace:
|
|
22286
|
-
console.debug(...messages);
|
|
22287
|
-
break;
|
|
22288
|
-
case LogLevel.debug:
|
|
22289
|
-
console.debug(...messages);
|
|
22290
|
-
break;
|
|
22291
|
-
case LogLevel.fatal:
|
|
22292
|
-
console.debug(...messages);
|
|
22293
|
-
break;
|
|
22294
|
-
default:
|
|
22295
|
-
console.log(...messages);
|
|
22296
|
-
}
|
|
22297
|
-
}
|
|
22298
|
-
getLoggerInstance() {
|
|
22299
|
-
throw new Error("This transport does not have a logger instance");
|
|
22300
|
-
}
|
|
22301
|
-
};
|
|
22302
|
-
|
|
22303
|
-
// node_modules/@loglayer/transport-opentelemetry/dist/index.mjs
|
|
22304
|
-
var import_api_logs = __toESM(require_src2(), 1);
|
|
22305
|
-
var logLayerLevels = {
|
|
22306
|
-
fatal: import_api_logs.SeverityNumber.FATAL,
|
|
22307
|
-
error: import_api_logs.SeverityNumber.ERROR,
|
|
22308
|
-
warn: import_api_logs.SeverityNumber.WARN,
|
|
22309
|
-
info: import_api_logs.SeverityNumber.INFO,
|
|
22310
|
-
debug: import_api_logs.SeverityNumber.DEBUG,
|
|
22311
|
-
trace: import_api_logs.SeverityNumber.TRACE
|
|
22312
|
-
};
|
|
22313
|
-
function getSeverityNumber(level) {
|
|
22314
|
-
return logLayerLevels[level] ?? import_api_logs.SeverityNumber.TRACE;
|
|
22315
|
-
}
|
|
22316
|
-
function emitLogRecord(record2, logger) {
|
|
22317
|
-
const { message, level, ...splat } = record2;
|
|
22318
|
-
const attributes = {};
|
|
22319
|
-
for (const key in splat)
|
|
22320
|
-
if (Object.hasOwn(splat, key))
|
|
22321
|
-
attributes[key] = splat[key];
|
|
22322
|
-
const logRecord = {
|
|
22323
|
-
severityNumber: getSeverityNumber(level),
|
|
22324
|
-
severityText: level,
|
|
22325
|
-
body: message,
|
|
22326
|
-
attributes
|
|
22327
|
-
};
|
|
22328
|
-
logger.emit(logRecord);
|
|
22329
|
-
}
|
|
22330
|
-
var OpenTelemetryTransport = class extends LoggerlessTransport {
|
|
22331
|
-
_logger;
|
|
22332
|
-
onError;
|
|
22333
|
-
constructor(config2 = {}) {
|
|
22334
|
-
super(config2);
|
|
22335
|
-
this._logger = import_api_logs.logs.getLogger("loglayer");
|
|
22336
|
-
this.onError = config2.onError;
|
|
22337
|
-
}
|
|
22338
|
-
shipToLogger({ logLevel, messages, data, hasData }) {
|
|
22339
|
-
const assembled = {
|
|
22340
|
-
level: logLevel,
|
|
22341
|
-
message: messages.join(" "),
|
|
22342
|
-
...hasData ? data : {}
|
|
22343
|
-
};
|
|
22344
|
-
try {
|
|
22345
|
-
emitLogRecord(assembled, this._logger);
|
|
22346
|
-
} catch (error48) {
|
|
22347
|
-
if (this.onError)
|
|
22348
|
-
this.onError(error48);
|
|
22349
|
-
}
|
|
22350
|
-
return [assembled];
|
|
22351
|
-
}
|
|
22352
|
-
};
|
|
22353
22022
|
|
|
22354
22023
|
// node_modules/@loglayer/transport-tslog/dist/index.js
|
|
22355
22024
|
var TsLogTransport = class extends BaseTransport {
|
|
@@ -24521,13 +24190,7 @@ var tslogLogger = new Logger({
|
|
|
24521
24190
|
minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
|
|
24522
24191
|
});
|
|
24523
24192
|
var baseLogger = new LogLayer({
|
|
24524
|
-
transport: [
|
|
24525
|
-
new TsLogTransport({ id: "tslog", logger: tslogLogger }),
|
|
24526
|
-
new OpenTelemetryTransport({
|
|
24527
|
-
id: "otel",
|
|
24528
|
-
enabled: true
|
|
24529
|
-
})
|
|
24530
|
-
],
|
|
24193
|
+
transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
|
|
24531
24194
|
plugins: [openTelemetryPlugin()],
|
|
24532
24195
|
errorSerializer: serializeError
|
|
24533
24196
|
});
|
|
@@ -25642,12 +25305,13 @@ var result_schema_default = {
|
|
|
25642
25305
|
};
|
|
25643
25306
|
|
|
25644
25307
|
// src/helpers/market-data-vendor-backfill/manifests.ts
|
|
25645
|
-
var
|
|
25308
|
+
var LEGACY_CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v1";
|
|
25309
|
+
var CAPABILITY_POLICY_ID = "market-data-vendor-backfill-capabilities/v2";
|
|
25646
25310
|
var RESOURCE_POLICY_ID = "market-data-vendor-backfill-resources/v1";
|
|
25647
25311
|
var ADAPTER_POLICY_ID = "cryptohftdata-orderbook-adapter/v1";
|
|
25648
25312
|
var ACQUISITION_POLICY_ID = "cryptohftdata-hourly-acquisition/v1";
|
|
25649
|
-
var
|
|
25650
|
-
policy_id:
|
|
25313
|
+
var legacyCapabilityPolicyContent = {
|
|
25314
|
+
policy_id: LEGACY_CAPABILITY_POLICY_ID,
|
|
25651
25315
|
provider: "cryptohftdata",
|
|
25652
25316
|
adapter_policy: {
|
|
25653
25317
|
policy_id: ADAPTER_POLICY_ID,
|
|
@@ -25673,6 +25337,18 @@ var capabilityPolicyContent = {
|
|
|
25673
25337
|
}
|
|
25674
25338
|
]
|
|
25675
25339
|
};
|
|
25340
|
+
var LEGACY_CAPABILITY_POLICY = Object.freeze({
|
|
25341
|
+
...legacyCapabilityPolicyContent,
|
|
25342
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent)
|
|
25343
|
+
});
|
|
25344
|
+
var capabilityPolicyContent = {
|
|
25345
|
+
...legacyCapabilityPolicyContent,
|
|
25346
|
+
policy_id: CAPABILITY_POLICY_ID,
|
|
25347
|
+
profiles: legacyCapabilityPolicyContent.profiles.map((profile) => ({
|
|
25348
|
+
...profile,
|
|
25349
|
+
source_policies: ["authoritative_window", "fill_gaps"]
|
|
25350
|
+
}))
|
|
25351
|
+
};
|
|
25676
25352
|
var CAPABILITY_POLICY = Object.freeze({
|
|
25677
25353
|
...capabilityPolicyContent,
|
|
25678
25354
|
policy_sha256: jcsSha256(capabilityPolicyContent)
|
|
@@ -25698,11 +25374,11 @@ var RESOURCE_POLICY = Object.freeze({
|
|
|
25698
25374
|
});
|
|
25699
25375
|
var EFFECTIVE_ADAPTER_POLICY_PIN = Object.freeze({
|
|
25700
25376
|
policy_id: ADAPTER_POLICY_ID,
|
|
25701
|
-
policy_sha256: jcsSha256(
|
|
25377
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent.adapter_policy)
|
|
25702
25378
|
});
|
|
25703
25379
|
var EFFECTIVE_ACQUISITION_POLICY_PIN = Object.freeze({
|
|
25704
25380
|
policy_id: ACQUISITION_POLICY_ID,
|
|
25705
|
-
policy_sha256: jcsSha256(
|
|
25381
|
+
policy_sha256: jcsSha256(legacyCapabilityPolicyContent.acquisition_policy)
|
|
25706
25382
|
});
|
|
25707
25383
|
var schemaArtifacts = [
|
|
25708
25384
|
["schemas/request.schema.json", request_schema_default],
|
|
@@ -25866,7 +25542,11 @@ var backfillRequestCodec = {
|
|
|
25866
25542
|
if (request.idempotency_key !== createBackfillIdempotencyKey(request)) {
|
|
25867
25543
|
throw new Error("idempotency_key does not match canonical business fields");
|
|
25868
25544
|
}
|
|
25869
|
-
|
|
25545
|
+
const capabilityPolicyMatches = [
|
|
25546
|
+
CAPABILITY_POLICY,
|
|
25547
|
+
LEGACY_CAPABILITY_POLICY
|
|
25548
|
+
].some((policy) => request.product_pins.capability_policy.policy_id === policy.policy_id && request.product_pins.capability_policy.policy_sha256 === policy.policy_sha256);
|
|
25549
|
+
if (!capabilityPolicyMatches || request.product_pins.resource_policy.policy_id !== RESOURCE_POLICY.policy_id || request.product_pins.resource_policy.policy_sha256 !== RESOURCE_POLICY.policy_sha256) {
|
|
25870
25550
|
throw new Error("request policy pins do not match the effective package policies");
|
|
25871
25551
|
}
|
|
25872
25552
|
if (request.required_clock.clock_id !== request.initial_selection.required_clock.clock_id || request.required_clock.clock_sha256 !== request.initial_selection.required_clock.clock_sha256 || request.required_clock.event_count !== request.initial_selection.required_clock.event_count) {
|
|
@@ -27319,12 +26999,13 @@ function assertStoredSelectionMatchesRequest(request, selection) {
|
|
|
27319
26999
|
}
|
|
27320
27000
|
}
|
|
27321
27001
|
function eligibleBundles(request, bundles) {
|
|
27322
|
-
|
|
27323
|
-
|
|
27324
|
-
|
|
27325
|
-
|
|
27326
|
-
return
|
|
27327
|
-
}
|
|
27002
|
+
const qualifiedVendorBundles = bundles.filter((bundle) => bundle.captureOrigin === "vendor_historical_backfill" && bundle.qualification?.state === "qualified");
|
|
27003
|
+
const productionBundles = bundles.filter((bundle) => bundle.captureOrigin === "production_capture");
|
|
27004
|
+
if (request.sourcePolicy === "authoritative_window") {
|
|
27005
|
+
const vendorSupport = chooseSupport(request, qualifiedVendorBundles);
|
|
27006
|
+
return vendorSupport.length === request.requiredClockTargetsMs.length ? qualifiedVendorBundles : productionBundles;
|
|
27007
|
+
}
|
|
27008
|
+
return [...productionBundles, ...qualifiedVendorBundles];
|
|
27328
27009
|
}
|
|
27329
27010
|
function originPriority(request, origin) {
|
|
27330
27011
|
if (request.sourcePolicy === "fill_gaps") {
|
|
@@ -27512,9 +27193,8 @@ function exactCandidateMatch(expected, actual) {
|
|
|
27512
27193
|
function requestedShapeMatches(request, rows) {
|
|
27513
27194
|
return rows.every(({ row }) => row.depth_limit === request.depth && row.construction_mode === request.constructionMode && row.schema_version === request.expectedProduct.canonicalSchemaVersion);
|
|
27514
27195
|
}
|
|
27515
|
-
function requiredClockCoverage(request, rows) {
|
|
27516
|
-
const summaryTimes = rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => {
|
|
27517
|
-
const value = row.source_time_ms;
|
|
27196
|
+
function requiredClockCoverage(request, rows, coverageSourceTimesMs) {
|
|
27197
|
+
const summaryTimes = (coverageSourceTimesMs ?? rows.filter(({ table }) => table === "market_data.cex_order_book_depth_summary").map(({ row }) => row.source_time_ms)).map((value) => {
|
|
27518
27198
|
if (typeof value !== "number" && (typeof value !== "string" || !/^\d+$/.test(value))) {
|
|
27519
27199
|
return;
|
|
27520
27200
|
}
|
|
@@ -27587,7 +27267,7 @@ function verifySemanticPromotion(evidence) {
|
|
|
27587
27267
|
coverageVerified: false
|
|
27588
27268
|
};
|
|
27589
27269
|
}
|
|
27590
|
-
const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows);
|
|
27270
|
+
const coverageVerified = requiredClockCoverage(evidence.request, evidence.candidateRows, evidence.coverageSourceTimesMs);
|
|
27591
27271
|
if (!coverageVerified) {
|
|
27592
27272
|
return {
|
|
27593
27273
|
...base,
|
|
@@ -27962,7 +27642,13 @@ class QualifiedOrderBookArchiveReader {
|
|
|
27962
27642
|
suffixDigestBefore: baseline.verificationBaseline.suffixDigest,
|
|
27963
27643
|
suffixDigestAfter: timelineDigest(suffix),
|
|
27964
27644
|
seamVerified: this.seamIsOrdered(queriedRows),
|
|
27965
|
-
exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string")
|
|
27645
|
+
exporterCompatible: queriedRows.every(({ row }) => typeof row.capture_bundle_id === "string" && typeof row.normalized_row_checksum === "string"),
|
|
27646
|
+
...request.sourcePolicy === "fill_gaps" ? {
|
|
27647
|
+
coverageSourceTimesMs: [
|
|
27648
|
+
...baseline.selection.support_anchors.map((anchor) => Date.parse(anchor.source_time)),
|
|
27649
|
+
...summaries.map((row) => numberField(row, "source_time_ms"))
|
|
27650
|
+
]
|
|
27651
|
+
} : {}
|
|
27966
27652
|
});
|
|
27967
27653
|
return { ...semantic, captureBundleId };
|
|
27968
27654
|
}
|
|
@@ -28078,972 +27764,592 @@ function buildForwarderBatches(input) {
|
|
|
28078
27764
|
return batches;
|
|
28079
27765
|
}
|
|
28080
27766
|
|
|
28081
|
-
// src/helpers/market-data-vendor-backfill/
|
|
28082
|
-
|
|
28083
|
-
var SHA256 = /^[0-9a-f]{64}$/;
|
|
28084
|
-
var REASON = /^[a-z][a-z0-9_]{0,127}$/;
|
|
28085
|
-
var EVENT_FIELDS = new Set([
|
|
28086
|
-
"qualification_event_id",
|
|
28087
|
-
"capture_bundle_id",
|
|
28088
|
-
"state",
|
|
28089
|
-
"receipt_id",
|
|
28090
|
-
"promotion_identity_sha256",
|
|
28091
|
-
"window",
|
|
28092
|
-
"event_at",
|
|
28093
|
-
"reason_code"
|
|
28094
|
-
]);
|
|
28095
|
-
function deterministicUuid(value) {
|
|
28096
|
-
const digest = jcsSha256(value).slice(0, 32).split("");
|
|
28097
|
-
digest[12] = "5";
|
|
28098
|
-
digest[16] = "8";
|
|
28099
|
-
const hex3 = digest.join("");
|
|
28100
|
-
return `${hex3.slice(0, 8)}-${hex3.slice(8, 12)}-${hex3.slice(12, 16)}-${hex3.slice(16, 20)}-${hex3.slice(20)}`;
|
|
28101
|
-
}
|
|
28102
|
-
function semanticEvent(event) {
|
|
28103
|
-
const { qualification_event_id: _eventId, ...semantic } = event;
|
|
28104
|
-
return semantic;
|
|
28105
|
-
}
|
|
28106
|
-
function assertTimestamp(value, field) {
|
|
28107
|
-
const parsed = Date.parse(value);
|
|
28108
|
-
if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
|
|
28109
|
-
throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
|
|
28110
|
-
}
|
|
28111
|
-
return parsed;
|
|
28112
|
-
}
|
|
28113
|
-
function parseQualificationEvent(value) {
|
|
28114
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
28115
|
-
throw new Error("qualification event must be an object");
|
|
28116
|
-
}
|
|
28117
|
-
const event = value;
|
|
28118
|
-
if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
|
|
28119
|
-
throw new Error("qualification event fields are invalid");
|
|
28120
|
-
}
|
|
28121
|
-
if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
|
|
28122
|
-
throw new Error("qualification window must be increasing");
|
|
28123
|
-
}
|
|
28124
|
-
assertTimestamp(event.event_at, "event_at");
|
|
28125
|
-
if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
|
|
28126
|
-
throw new Error("qualification_event_id does not match event content");
|
|
28127
|
-
}
|
|
28128
|
-
return event;
|
|
28129
|
-
}
|
|
28130
|
-
function finalizeQualificationEvent(event) {
|
|
28131
|
-
const semantic = semanticEvent(event);
|
|
28132
|
-
return parseQualificationEvent({
|
|
28133
|
-
...semantic,
|
|
28134
|
-
qualification_event_id: deterministicUuid(semantic)
|
|
28135
|
-
});
|
|
28136
|
-
}
|
|
28137
|
-
function qualificationEventToArchiveRow(eventInput) {
|
|
28138
|
-
const event = parseQualificationEvent(eventInput);
|
|
28139
|
-
return {
|
|
28140
|
-
table: "market_data.cex_order_book_capture_qualifications",
|
|
28141
|
-
row: {
|
|
28142
|
-
source: "external_backfill",
|
|
28143
|
-
capture_origin: "vendor_historical_backfill",
|
|
28144
|
-
source_mode: "vendor_historical_backfill_v1",
|
|
28145
|
-
deployment_id: "market-data-vendor-backfill",
|
|
28146
|
-
qualification_event_id: event.qualification_event_id,
|
|
28147
|
-
capture_bundle_id: event.capture_bundle_id,
|
|
28148
|
-
state: event.state,
|
|
28149
|
-
receipt_id: event.receipt_id,
|
|
28150
|
-
promotion_identity_sha256: event.promotion_identity_sha256,
|
|
28151
|
-
window_start_ms: Date.parse(event.window.start_at),
|
|
28152
|
-
window_end_ms: Date.parse(event.window.end_at),
|
|
28153
|
-
event_at_ms: Date.parse(event.event_at),
|
|
28154
|
-
reason_code: event.reason_code,
|
|
28155
|
-
event_json: jcsCanonicalize(event)
|
|
28156
|
-
}
|
|
28157
|
-
};
|
|
28158
|
-
}
|
|
27767
|
+
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
27768
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
28159
27769
|
|
|
28160
|
-
//
|
|
28161
|
-
|
|
28162
|
-
|
|
28163
|
-
|
|
28164
|
-
|
|
28165
|
-
|
|
28166
|
-
|
|
28167
|
-
|
|
28168
|
-
|
|
28169
|
-
|
|
28170
|
-
|
|
28171
|
-
|
|
28172
|
-
|
|
28173
|
-
|
|
28174
|
-
|
|
28175
|
-
|
|
28176
|
-
|
|
28177
|
-
|
|
28178
|
-
|
|
28179
|
-
|
|
28180
|
-
|
|
28181
|
-
|
|
28182
|
-
|
|
28183
|
-
|
|
28184
|
-
|
|
28185
|
-
|
|
28186
|
-
|
|
28187
|
-
|
|
28188
|
-
|
|
28189
|
-
|
|
28190
|
-
|
|
28191
|
-
if (
|
|
28192
|
-
|
|
27770
|
+
// node_modules/fzstd/esm/index.mjs
|
|
27771
|
+
var ab = ArrayBuffer;
|
|
27772
|
+
var u8 = Uint8Array;
|
|
27773
|
+
var u16 = Uint16Array;
|
|
27774
|
+
var i16 = Int16Array;
|
|
27775
|
+
var i32 = Int32Array;
|
|
27776
|
+
var slc = function(v, s, e) {
|
|
27777
|
+
if (u8.prototype.slice)
|
|
27778
|
+
return u8.prototype.slice.call(v, s, e);
|
|
27779
|
+
if (s == null || s < 0)
|
|
27780
|
+
s = 0;
|
|
27781
|
+
if (e == null || e > v.length)
|
|
27782
|
+
e = v.length;
|
|
27783
|
+
var n = new u8(e - s);
|
|
27784
|
+
n.set(v.subarray(s, e));
|
|
27785
|
+
return n;
|
|
27786
|
+
};
|
|
27787
|
+
var fill = function(v, n, s, e) {
|
|
27788
|
+
if (u8.prototype.fill)
|
|
27789
|
+
return u8.prototype.fill.call(v, n, s, e);
|
|
27790
|
+
if (s == null || s < 0)
|
|
27791
|
+
s = 0;
|
|
27792
|
+
if (e == null || e > v.length)
|
|
27793
|
+
e = v.length;
|
|
27794
|
+
for (;s < e; ++s)
|
|
27795
|
+
v[s] = n;
|
|
27796
|
+
return v;
|
|
27797
|
+
};
|
|
27798
|
+
var cpw = function(v, t, s, e) {
|
|
27799
|
+
if (u8.prototype.copyWithin)
|
|
27800
|
+
return u8.prototype.copyWithin.call(v, t, s, e);
|
|
27801
|
+
if (s == null || s < 0)
|
|
27802
|
+
s = 0;
|
|
27803
|
+
if (e == null || e > v.length)
|
|
27804
|
+
e = v.length;
|
|
27805
|
+
while (s < e) {
|
|
27806
|
+
v[t++] = v[s++];
|
|
28193
27807
|
}
|
|
28194
|
-
|
|
28195
|
-
|
|
28196
|
-
|
|
28197
|
-
|
|
28198
|
-
|
|
28199
|
-
|
|
28200
|
-
|
|
28201
|
-
|
|
28202
|
-
|
|
28203
|
-
|
|
28204
|
-
|
|
28205
|
-
|
|
28206
|
-
|
|
28207
|
-
|
|
28208
|
-
|
|
28209
|
-
|
|
28210
|
-
|
|
28211
|
-
|
|
28212
|
-
|
|
28213
|
-
|
|
28214
|
-
|
|
28215
|
-
|
|
28216
|
-
|
|
28217
|
-
|
|
28218
|
-
|
|
28219
|
-
|
|
28220
|
-
|
|
28221
|
-
|
|
28222
|
-
|
|
28223
|
-
|
|
28224
|
-
|
|
28225
|
-
|
|
28226
|
-
|
|
28227
|
-
|
|
28228
|
-
|
|
28229
|
-
|
|
28230
|
-
|
|
28231
|
-
|
|
28232
|
-
|
|
28233
|
-
|
|
28234
|
-
|
|
28235
|
-
|
|
28236
|
-
|
|
28237
|
-
|
|
28238
|
-
if (accepted)
|
|
28239
|
-
break;
|
|
28240
|
-
if (attempt < maxAttempts)
|
|
28241
|
-
await dependencies.retry?.wait?.(attempt);
|
|
27808
|
+
};
|
|
27809
|
+
var ec = [
|
|
27810
|
+
"invalid zstd data",
|
|
27811
|
+
"window size too large (>2046MB)",
|
|
27812
|
+
"invalid block type",
|
|
27813
|
+
"FSE accuracy too high",
|
|
27814
|
+
"match distance too far back",
|
|
27815
|
+
"unexpected EOF"
|
|
27816
|
+
];
|
|
27817
|
+
var err = function(ind, msg, nt) {
|
|
27818
|
+
var e = new Error(msg || ec[ind]);
|
|
27819
|
+
e.code = ind;
|
|
27820
|
+
if (Error.captureStackTrace)
|
|
27821
|
+
Error.captureStackTrace(e, err);
|
|
27822
|
+
if (!nt)
|
|
27823
|
+
throw e;
|
|
27824
|
+
return e;
|
|
27825
|
+
};
|
|
27826
|
+
var rb = function(d, b, n) {
|
|
27827
|
+
var i = 0, o = 0;
|
|
27828
|
+
for (;i < n; ++i)
|
|
27829
|
+
o |= d[b++] << (i << 3);
|
|
27830
|
+
return o;
|
|
27831
|
+
};
|
|
27832
|
+
var b4 = function(d, b) {
|
|
27833
|
+
return (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0;
|
|
27834
|
+
};
|
|
27835
|
+
var rzfh = function(dat, w) {
|
|
27836
|
+
var n3 = dat[0] | dat[1] << 8 | dat[2] << 16;
|
|
27837
|
+
if (n3 == 3126568 && dat[3] == 253) {
|
|
27838
|
+
var flg = dat[4];
|
|
27839
|
+
var ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6;
|
|
27840
|
+
if (flg & 8)
|
|
27841
|
+
err(0);
|
|
27842
|
+
var bt = 6 - ss;
|
|
27843
|
+
var db = df == 3 ? 4 : df;
|
|
27844
|
+
var di = rb(dat, bt, db);
|
|
27845
|
+
bt += db;
|
|
27846
|
+
var fsb = fcf ? 1 << fcf : ss;
|
|
27847
|
+
var fss = rb(dat, bt, fsb) + (fcf == 1 && 256);
|
|
27848
|
+
var ws = fss;
|
|
27849
|
+
if (!ss) {
|
|
27850
|
+
var wb = 1 << 10 + (dat[5] >> 3);
|
|
27851
|
+
ws = wb + (wb >> 3) * (dat[5] & 7);
|
|
28242
27852
|
}
|
|
28243
|
-
if (
|
|
28244
|
-
|
|
27853
|
+
if (ws > 2145386496)
|
|
27854
|
+
err(1);
|
|
27855
|
+
var buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
|
|
27856
|
+
buf[0] = 1, buf[4] = 4, buf[8] = 8;
|
|
27857
|
+
return {
|
|
27858
|
+
b: bt + fsb,
|
|
27859
|
+
y: 0,
|
|
27860
|
+
l: 0,
|
|
27861
|
+
d: di,
|
|
27862
|
+
w: w && w != 1 ? w : buf.subarray(12),
|
|
27863
|
+
e: ws,
|
|
27864
|
+
o: new i32(buf.buffer, 0, 3),
|
|
27865
|
+
u: fss,
|
|
27866
|
+
c: cc,
|
|
27867
|
+
m: Math.min(131072, ws)
|
|
27868
|
+
};
|
|
27869
|
+
} else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
|
|
27870
|
+
return b4(dat, 4) + 8;
|
|
28245
27871
|
}
|
|
28246
|
-
|
|
28247
|
-
}
|
|
28248
|
-
function
|
|
28249
|
-
|
|
28250
|
-
|
|
28251
|
-
|
|
28252
|
-
|
|
27872
|
+
err(0);
|
|
27873
|
+
};
|
|
27874
|
+
var msb = function(val) {
|
|
27875
|
+
var bits = 0;
|
|
27876
|
+
for (;1 << bits <= val; ++bits)
|
|
27877
|
+
;
|
|
27878
|
+
return bits - 1;
|
|
27879
|
+
};
|
|
27880
|
+
var rfse = function(dat, bt, mal) {
|
|
27881
|
+
var tpos = (bt << 3) + 4;
|
|
27882
|
+
var al = (dat[bt] & 15) + 5;
|
|
27883
|
+
if (al > mal)
|
|
27884
|
+
err(3);
|
|
27885
|
+
var sz = 1 << al;
|
|
27886
|
+
var probs = sz, sym = -1, re = -1, i = -1, ht = sz;
|
|
27887
|
+
var buf = new ab(512 + (sz << 2));
|
|
27888
|
+
var freq = new i16(buf, 0, 256);
|
|
27889
|
+
var dstate = new u16(buf, 0, 256);
|
|
27890
|
+
var nstate = new u16(buf, 512, sz);
|
|
27891
|
+
var bb1 = 512 + (sz << 1);
|
|
27892
|
+
var syms = new u8(buf, bb1, sz);
|
|
27893
|
+
var nbits = new u8(buf, bb1 + sz);
|
|
27894
|
+
while (sym < 255 && probs > 0) {
|
|
27895
|
+
var bits = msb(probs + 1);
|
|
27896
|
+
var cbt = tpos >> 3;
|
|
27897
|
+
var msk = (1 << bits + 1) - 1;
|
|
27898
|
+
var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
|
|
27899
|
+
var msk1fb = (1 << bits) - 1;
|
|
27900
|
+
var msv = msk - probs - 1;
|
|
27901
|
+
var sval = val & msk1fb;
|
|
27902
|
+
if (sval < msv)
|
|
27903
|
+
tpos += bits, val = sval;
|
|
27904
|
+
else {
|
|
27905
|
+
tpos += bits + 1;
|
|
27906
|
+
if (val > msk1fb)
|
|
27907
|
+
val -= msv;
|
|
28253
27908
|
}
|
|
28254
|
-
|
|
28255
|
-
|
|
28256
|
-
|
|
28257
|
-
|
|
28258
|
-
|
|
28259
|
-
|
|
28260
|
-
|
|
28261
|
-
|
|
28262
|
-
|
|
28263
|
-
|
|
28264
|
-
|
|
28265
|
-
|
|
28266
|
-
|
|
28267
|
-
throw new Error("stored receipt identity has conflicting content");
|
|
27909
|
+
freq[++sym] = --val;
|
|
27910
|
+
if (val == -1) {
|
|
27911
|
+
probs += val;
|
|
27912
|
+
syms[--ht] = sym;
|
|
27913
|
+
} else
|
|
27914
|
+
probs -= val;
|
|
27915
|
+
if (!val) {
|
|
27916
|
+
do {
|
|
27917
|
+
var rbt = tpos >> 3;
|
|
27918
|
+
re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
|
|
27919
|
+
tpos += 2;
|
|
27920
|
+
sym += re;
|
|
27921
|
+
} while (re == 3);
|
|
28268
27922
|
}
|
|
28269
|
-
receiptById.set(receipt.receipt_id, receipt);
|
|
28270
27923
|
}
|
|
28271
|
-
|
|
28272
|
-
|
|
28273
|
-
|
|
27924
|
+
if (sym > 255 || probs)
|
|
27925
|
+
err(0);
|
|
27926
|
+
var sympos = 0;
|
|
27927
|
+
var sstep = (sz >> 1) + (sz >> 3) + 3;
|
|
27928
|
+
var smask = sz - 1;
|
|
27929
|
+
for (var s = 0;s <= sym; ++s) {
|
|
27930
|
+
var sf = freq[s];
|
|
27931
|
+
if (sf < 1) {
|
|
27932
|
+
dstate[s] = -sf;
|
|
27933
|
+
continue;
|
|
27934
|
+
}
|
|
27935
|
+
for (i = 0;i < sf; ++i) {
|
|
27936
|
+
syms[sympos] = s;
|
|
27937
|
+
do {
|
|
27938
|
+
sympos = sympos + sstep & smask;
|
|
27939
|
+
} while (sympos >= ht);
|
|
28274
27940
|
}
|
|
28275
27941
|
}
|
|
28276
|
-
if (
|
|
28277
|
-
|
|
28278
|
-
|
|
28279
|
-
|
|
28280
|
-
|
|
28281
|
-
|
|
28282
|
-
throw new Error("request target or production authorization ID is missing");
|
|
28283
|
-
}
|
|
28284
|
-
if (resolution.forwarderIdentity.environment !== request.target.environment || resolution.forwarderIdentity.cluster !== request.target.cluster) {
|
|
28285
|
-
throw new Error("archive forwarder cluster identity mismatch");
|
|
28286
|
-
}
|
|
28287
|
-
const authorization = resolution.authorization;
|
|
28288
|
-
const expiresAtMs = Date.parse(authorization.expiresAt);
|
|
28289
|
-
if (authorization.authorizationId !== request.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request.target.environment || authorization.cluster !== request.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
|
|
28290
|
-
throw new Error("production forwarder authorization is invalid");
|
|
28291
|
-
}
|
|
28292
|
-
}
|
|
28293
|
-
function resourcePolicyScopeExceeded(request) {
|
|
28294
|
-
return request.depth > RESOURCE_POLICY.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
|
|
28295
|
-
}
|
|
28296
|
-
function storedReceiptForSelection(resolution) {
|
|
28297
|
-
const receiptId = resolution.selection.receipt_ids[0];
|
|
28298
|
-
return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
|
|
28299
|
-
}
|
|
28300
|
-
async function runMarketDataVendorBackfill(input, dependencies) {
|
|
28301
|
-
let request;
|
|
28302
|
-
try {
|
|
28303
|
-
const documents = input;
|
|
28304
|
-
request = decodeBackfillRunDocuments({
|
|
28305
|
-
request: documents.request,
|
|
28306
|
-
requiredClock: documents.requiredClock
|
|
28307
|
-
});
|
|
28308
|
-
} catch {
|
|
28309
|
-
return outcome(undefined, "request_invalid", "request_invalid");
|
|
28310
|
-
}
|
|
28311
|
-
let initialResolution;
|
|
28312
|
-
try {
|
|
28313
|
-
initialResolution = await dependencies.archive.resolveSelection(request);
|
|
28314
|
-
assertArchivePreflight(request, initialResolution);
|
|
28315
|
-
const forwarderPreflight = await dependencies.forwarder.preflight({
|
|
28316
|
-
authorizationId: request.productionAuthorizationId,
|
|
28317
|
-
target: request.target
|
|
28318
|
-
});
|
|
28319
|
-
assertForwarderPreflight(request, forwarderPreflight, dependencies.clock.nowMs());
|
|
28320
|
-
} catch {
|
|
28321
|
-
return outcome(request, "archive_preflight_failed", "archive_preflight_failed");
|
|
28322
|
-
}
|
|
28323
|
-
if (initialResolution.selection.coverage_class === "complete") {
|
|
28324
|
-
const receipt2 = storedReceiptForSelection(initialResolution);
|
|
28325
|
-
return outcome(request, "already_covered", "qualified_coverage_complete", {
|
|
28326
|
-
selection: initialResolution.selection,
|
|
28327
|
-
...receipt2 ? { receipt: receipt2 } : {}
|
|
28328
|
-
});
|
|
28329
|
-
}
|
|
28330
|
-
if (resourcePolicyScopeExceeded(request)) {
|
|
28331
|
-
return outcome(request, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
|
|
28332
|
-
}
|
|
28333
|
-
let capability;
|
|
28334
|
-
try {
|
|
28335
|
-
capability = dependencies.providers.capabilityFor(request);
|
|
28336
|
-
} catch {
|
|
28337
|
-
return outcome(request, "capability_unsupported", "capability_probe_failed");
|
|
28338
|
-
}
|
|
28339
|
-
if (!capability) {
|
|
28340
|
-
return outcome(request, "capability_unsupported", "scope_unsupported");
|
|
28341
|
-
}
|
|
28342
|
-
if (!request.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
|
|
28343
|
-
return outcome(request, "capability_unsupported", "adapter_version_unpinned");
|
|
28344
|
-
}
|
|
28345
|
-
let credential;
|
|
28346
|
-
try {
|
|
28347
|
-
credential = await dependencies.credentials.resolve(capability.provider);
|
|
28348
|
-
} catch {
|
|
28349
|
-
return outcome(request, "credentials_missing", "credential_resolution_failed");
|
|
28350
|
-
}
|
|
28351
|
-
if (credential === undefined || credential === null) {
|
|
28352
|
-
return outcome(request, "credentials_missing", "provider_credentials_missing");
|
|
28353
|
-
}
|
|
28354
|
-
let dataset;
|
|
28355
|
-
let normalized;
|
|
28356
|
-
try {
|
|
28357
|
-
dataset = await dependencies.providers.acquire(request, capability, credential);
|
|
28358
|
-
const bundleId = captureBundleId(request, capability, dataset);
|
|
28359
|
-
normalized = await dependencies.providers.normalize(request, capability, dataset, bundleId);
|
|
28360
|
-
if (normalized.captureBundleId !== bundleId) {
|
|
28361
|
-
return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
28362
|
-
reasonSubcode: "capture_identity_mismatch"
|
|
28363
|
-
});
|
|
28364
|
-
}
|
|
28365
|
-
} catch (error48) {
|
|
28366
|
-
const reason = stableFailureReason(error48, "provider_dataset_invalid");
|
|
28367
|
-
return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
28368
|
-
reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
|
|
28369
|
-
});
|
|
28370
|
-
}
|
|
28371
|
-
try {
|
|
28372
|
-
const candidateBatches = buildForwarderBatches({
|
|
28373
|
-
captureBundleId: normalized.captureBundleId,
|
|
28374
|
-
deploymentId: "market-data-vendor-backfill",
|
|
28375
|
-
rows: normalized.rows
|
|
28376
|
-
});
|
|
28377
|
-
if (!await submitAll(dependencies, candidateBatches)) {
|
|
28378
|
-
return outcome(request, "archive_ingest_failed", "candidate_batch_rejected");
|
|
28379
|
-
}
|
|
28380
|
-
} catch {
|
|
28381
|
-
return outcome(request, "archive_ingest_failed", "candidate_submission_failed");
|
|
28382
|
-
}
|
|
28383
|
-
let verification;
|
|
28384
|
-
try {
|
|
28385
|
-
verification = await dependencies.archive.verifyCandidate(request, normalized, normalized.captureBundleId, initialResolution);
|
|
28386
|
-
} catch {
|
|
28387
|
-
return outcome(request, "promotion_verification_failed", "candidate_query_failed");
|
|
28388
|
-
}
|
|
28389
|
-
if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
|
|
28390
|
-
return outcome(request, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
|
|
27942
|
+
if (sympos)
|
|
27943
|
+
err(0);
|
|
27944
|
+
for (i = 0;i < sz; ++i) {
|
|
27945
|
+
var ns = dstate[syms[i]]++;
|
|
27946
|
+
var nb = nbits[i] = al - msb(ns);
|
|
27947
|
+
nstate[i] = (ns << nb) - sz;
|
|
28391
27948
|
}
|
|
28392
|
-
|
|
28393
|
-
|
|
28394
|
-
|
|
28395
|
-
|
|
28396
|
-
|
|
28397
|
-
|
|
28398
|
-
|
|
28399
|
-
|
|
28400
|
-
|
|
28401
|
-
|
|
27949
|
+
return [tpos + 7 >> 3, {
|
|
27950
|
+
b: al,
|
|
27951
|
+
s: syms,
|
|
27952
|
+
n: nbits,
|
|
27953
|
+
t: nstate
|
|
27954
|
+
}];
|
|
27955
|
+
};
|
|
27956
|
+
var rhu = function(dat, bt) {
|
|
27957
|
+
var i = 0, wc = -1;
|
|
27958
|
+
var buf = new u8(292), hb = dat[bt];
|
|
27959
|
+
var hw = buf.subarray(0, 256);
|
|
27960
|
+
var rc = buf.subarray(256, 268);
|
|
27961
|
+
var ri = new u16(buf.buffer, 268);
|
|
27962
|
+
if (hb < 128) {
|
|
27963
|
+
var _a2 = rfse(dat, bt + 1, 6), ebt = _a2[0], fdt = _a2[1];
|
|
27964
|
+
bt += hb;
|
|
27965
|
+
var epos = ebt << 3;
|
|
27966
|
+
var lb = dat[bt];
|
|
27967
|
+
if (!lb)
|
|
27968
|
+
err(0);
|
|
27969
|
+
var st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
|
|
27970
|
+
var fpos = (++bt << 3) - 8 + msb(lb);
|
|
27971
|
+
for (;; ) {
|
|
27972
|
+
fpos -= btr1;
|
|
27973
|
+
if (fpos < epos)
|
|
27974
|
+
break;
|
|
27975
|
+
var cbt = fpos >> 3;
|
|
27976
|
+
st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
|
|
27977
|
+
hw[++wc] = fdt.s[st1];
|
|
27978
|
+
fpos -= btr2;
|
|
27979
|
+
if (fpos < epos)
|
|
27980
|
+
break;
|
|
27981
|
+
cbt = fpos >> 3;
|
|
27982
|
+
st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
|
|
27983
|
+
hw[++wc] = fdt.s[st2];
|
|
27984
|
+
btr1 = fdt.n[st1];
|
|
27985
|
+
st1 = fdt.t[st1];
|
|
27986
|
+
btr2 = fdt.n[st2];
|
|
27987
|
+
st2 = fdt.t[st2];
|
|
28402
27988
|
}
|
|
28403
|
-
|
|
28404
|
-
|
|
28405
|
-
|
|
28406
|
-
|
|
28407
|
-
|
|
28408
|
-
|
|
28409
|
-
|
|
28410
|
-
|
|
28411
|
-
});
|
|
28412
|
-
const [qualificationBatch] = buildForwarderBatches({
|
|
28413
|
-
captureBundleId: normalized.captureBundleId,
|
|
28414
|
-
deploymentId: "market-data-vendor-backfill",
|
|
28415
|
-
rows: [qualificationEventToArchiveRow(qualification)]
|
|
28416
|
-
});
|
|
28417
|
-
if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
|
|
28418
|
-
return outcome(request, "archive_ingest_failed", "qualification_commit_failed", { receipt });
|
|
27989
|
+
if (++wc > 255)
|
|
27990
|
+
err(0);
|
|
27991
|
+
} else {
|
|
27992
|
+
wc = hb - 127;
|
|
27993
|
+
for (;i < wc; i += 2) {
|
|
27994
|
+
var byte = dat[++bt];
|
|
27995
|
+
hw[i] = byte >> 4;
|
|
27996
|
+
hw[i + 1] = byte & 15;
|
|
28419
27997
|
}
|
|
28420
|
-
|
|
28421
|
-
return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
|
|
27998
|
+
++bt;
|
|
28422
27999
|
}
|
|
28423
|
-
|
|
28424
|
-
|
|
28425
|
-
|
|
28426
|
-
|
|
28427
|
-
|
|
28428
|
-
|
|
28429
|
-
receipt,
|
|
28430
|
-
reasonSubcode: stableFailureReason(error48, "archive_selection_resolution_failed")
|
|
28431
|
-
});
|
|
28000
|
+
var wes = 0;
|
|
28001
|
+
for (i = 0;i < wc; ++i) {
|
|
28002
|
+
var wt = hw[i];
|
|
28003
|
+
if (wt > 11)
|
|
28004
|
+
err(0);
|
|
28005
|
+
wes += wt && 1 << wt - 1;
|
|
28432
28006
|
}
|
|
28433
|
-
|
|
28434
|
-
|
|
28007
|
+
var mb = msb(wes) + 1;
|
|
28008
|
+
var ts = 1 << mb;
|
|
28009
|
+
var rem = ts - wes;
|
|
28010
|
+
if (rem & rem - 1)
|
|
28011
|
+
err(0);
|
|
28012
|
+
hw[wc++] = msb(rem) + 1;
|
|
28013
|
+
for (i = 0;i < wc; ++i) {
|
|
28014
|
+
var wt = hw[i];
|
|
28015
|
+
++rc[hw[i] = wt && mb + 1 - wt];
|
|
28435
28016
|
}
|
|
28436
|
-
|
|
28437
|
-
|
|
28017
|
+
var hbuf = new u8(ts << 1);
|
|
28018
|
+
var syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
|
|
28019
|
+
ri[mb] = 0;
|
|
28020
|
+
for (i = mb;i > 0; --i) {
|
|
28021
|
+
var pv = ri[i];
|
|
28022
|
+
fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
|
|
28438
28023
|
}
|
|
28439
|
-
|
|
28440
|
-
|
|
28441
|
-
|
|
28442
|
-
|
|
28443
|
-
|
|
28444
|
-
|
|
28445
|
-
|
|
28446
|
-
archiveSelectionToArchiveRow(request.wire, finalResolution.selection)
|
|
28447
|
-
]
|
|
28448
|
-
});
|
|
28449
|
-
if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
|
|
28450
|
-
return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
28024
|
+
if (ri[0] != ts)
|
|
28025
|
+
err(0);
|
|
28026
|
+
for (i = 0;i < wc; ++i) {
|
|
28027
|
+
var bits = hw[i];
|
|
28028
|
+
if (bits) {
|
|
28029
|
+
var code = ri[bits];
|
|
28030
|
+
fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
|
|
28451
28031
|
}
|
|
28452
|
-
} catch {
|
|
28453
|
-
return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
28454
28032
|
}
|
|
28455
|
-
return
|
|
28456
|
-
|
|
28457
|
-
|
|
28458
|
-
|
|
28459
|
-
}
|
|
28460
|
-
|
|
28461
|
-
// src/helpers/market-data-vendor-backfill/cryptohftdata.ts
|
|
28462
|
-
import { createHash as createHash4 } from "node:crypto";
|
|
28463
|
-
|
|
28464
|
-
// node_modules/fzstd/esm/index.mjs
|
|
28465
|
-
var ab = ArrayBuffer;
|
|
28466
|
-
var u8 = Uint8Array;
|
|
28467
|
-
var u16 = Uint16Array;
|
|
28468
|
-
var i16 = Int16Array;
|
|
28469
|
-
var i32 = Int32Array;
|
|
28470
|
-
var slc = function(v, s, e) {
|
|
28471
|
-
if (u8.prototype.slice)
|
|
28472
|
-
return u8.prototype.slice.call(v, s, e);
|
|
28473
|
-
if (s == null || s < 0)
|
|
28474
|
-
s = 0;
|
|
28475
|
-
if (e == null || e > v.length)
|
|
28476
|
-
e = v.length;
|
|
28477
|
-
var n = new u8(e - s);
|
|
28478
|
-
n.set(v.subarray(s, e));
|
|
28479
|
-
return n;
|
|
28480
|
-
};
|
|
28481
|
-
var fill = function(v, n, s, e) {
|
|
28482
|
-
if (u8.prototype.fill)
|
|
28483
|
-
return u8.prototype.fill.call(v, n, s, e);
|
|
28484
|
-
if (s == null || s < 0)
|
|
28485
|
-
s = 0;
|
|
28486
|
-
if (e == null || e > v.length)
|
|
28487
|
-
e = v.length;
|
|
28488
|
-
for (;s < e; ++s)
|
|
28489
|
-
v[s] = n;
|
|
28490
|
-
return v;
|
|
28033
|
+
return [bt, {
|
|
28034
|
+
n: nb,
|
|
28035
|
+
b: mb,
|
|
28036
|
+
s: syms
|
|
28037
|
+
}];
|
|
28491
28038
|
};
|
|
28492
|
-
var
|
|
28493
|
-
|
|
28494
|
-
|
|
28495
|
-
|
|
28496
|
-
|
|
28497
|
-
|
|
28498
|
-
|
|
28499
|
-
|
|
28500
|
-
|
|
28039
|
+
var dllt = rfse(/* @__PURE__ */ new u8([
|
|
28040
|
+
81,
|
|
28041
|
+
16,
|
|
28042
|
+
99,
|
|
28043
|
+
140,
|
|
28044
|
+
49,
|
|
28045
|
+
198,
|
|
28046
|
+
24,
|
|
28047
|
+
99,
|
|
28048
|
+
12,
|
|
28049
|
+
33,
|
|
28050
|
+
196,
|
|
28051
|
+
24,
|
|
28052
|
+
99,
|
|
28053
|
+
102,
|
|
28054
|
+
102,
|
|
28055
|
+
134,
|
|
28056
|
+
70,
|
|
28057
|
+
146,
|
|
28058
|
+
4
|
|
28059
|
+
]), 0, 6)[1];
|
|
28060
|
+
var dmlt = rfse(/* @__PURE__ */ new u8([
|
|
28061
|
+
33,
|
|
28062
|
+
20,
|
|
28063
|
+
196,
|
|
28064
|
+
24,
|
|
28065
|
+
99,
|
|
28066
|
+
140,
|
|
28067
|
+
33,
|
|
28068
|
+
132,
|
|
28069
|
+
16,
|
|
28070
|
+
66,
|
|
28071
|
+
8,
|
|
28072
|
+
33,
|
|
28073
|
+
132,
|
|
28074
|
+
16,
|
|
28075
|
+
66,
|
|
28076
|
+
8,
|
|
28077
|
+
33,
|
|
28078
|
+
68,
|
|
28079
|
+
68,
|
|
28080
|
+
68,
|
|
28081
|
+
68,
|
|
28082
|
+
68,
|
|
28083
|
+
68,
|
|
28084
|
+
68,
|
|
28085
|
+
68,
|
|
28086
|
+
36,
|
|
28087
|
+
9
|
|
28088
|
+
]), 0, 6)[1];
|
|
28089
|
+
var doct = rfse(/* @__PURE__ */ new u8([
|
|
28090
|
+
32,
|
|
28091
|
+
132,
|
|
28092
|
+
16,
|
|
28093
|
+
66,
|
|
28094
|
+
102,
|
|
28095
|
+
70,
|
|
28096
|
+
68,
|
|
28097
|
+
68,
|
|
28098
|
+
68,
|
|
28099
|
+
68,
|
|
28100
|
+
36,
|
|
28101
|
+
73,
|
|
28102
|
+
2
|
|
28103
|
+
]), 0, 5)[1];
|
|
28104
|
+
var b2bl = function(b, s) {
|
|
28105
|
+
var len = b.length, bl = new i32(len);
|
|
28106
|
+
for (var i = 0;i < len; ++i) {
|
|
28107
|
+
bl[i] = s;
|
|
28108
|
+
s += 1 << b[i];
|
|
28501
28109
|
}
|
|
28110
|
+
return bl;
|
|
28502
28111
|
};
|
|
28503
|
-
var
|
|
28504
|
-
|
|
28505
|
-
|
|
28506
|
-
|
|
28507
|
-
|
|
28508
|
-
|
|
28509
|
-
|
|
28510
|
-
|
|
28511
|
-
|
|
28512
|
-
|
|
28513
|
-
|
|
28514
|
-
|
|
28515
|
-
|
|
28516
|
-
|
|
28517
|
-
|
|
28518
|
-
|
|
28519
|
-
|
|
28520
|
-
|
|
28521
|
-
|
|
28522
|
-
|
|
28523
|
-
|
|
28524
|
-
|
|
28112
|
+
var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
28113
|
+
0,
|
|
28114
|
+
0,
|
|
28115
|
+
0,
|
|
28116
|
+
0,
|
|
28117
|
+
16843009,
|
|
28118
|
+
50528770,
|
|
28119
|
+
134678020,
|
|
28120
|
+
202050057,
|
|
28121
|
+
269422093
|
|
28122
|
+
])).buffer, 0, 36);
|
|
28123
|
+
var llbl = /* @__PURE__ */ b2bl(llb, 0);
|
|
28124
|
+
var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
28125
|
+
0,
|
|
28126
|
+
0,
|
|
28127
|
+
0,
|
|
28128
|
+
0,
|
|
28129
|
+
0,
|
|
28130
|
+
0,
|
|
28131
|
+
0,
|
|
28132
|
+
0,
|
|
28133
|
+
16843009,
|
|
28134
|
+
50528770,
|
|
28135
|
+
117769220,
|
|
28136
|
+
185207048,
|
|
28137
|
+
252579084,
|
|
28138
|
+
16
|
|
28139
|
+
])).buffer, 0, 53);
|
|
28140
|
+
var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
|
|
28141
|
+
var dhu = function(dat, out, hu) {
|
|
28142
|
+
var len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
|
|
28143
|
+
if (!lb)
|
|
28144
|
+
err(0);
|
|
28145
|
+
var st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
|
|
28146
|
+
for (;pos > eb && i < ss; ) {
|
|
28147
|
+
var cbt = pos >> 3;
|
|
28148
|
+
var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
|
|
28149
|
+
st = (st << btr | val) & msk;
|
|
28150
|
+
out[++i] = hu.s[st];
|
|
28151
|
+
pos -= btr = hu.n[st];
|
|
28152
|
+
}
|
|
28153
|
+
if (pos != eb || i + 1 != ss)
|
|
28154
|
+
err(0);
|
|
28525
28155
|
};
|
|
28526
|
-
var
|
|
28527
|
-
|
|
28156
|
+
var dhu4 = function(dat, out, hu) {
|
|
28157
|
+
var bt = 6;
|
|
28158
|
+
var ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
|
|
28159
|
+
dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
|
|
28160
|
+
dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
|
|
28161
|
+
dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
|
|
28162
|
+
dhu(dat.subarray(bt), out.subarray(sz3), hu);
|
|
28528
28163
|
};
|
|
28529
|
-
var
|
|
28530
|
-
var
|
|
28531
|
-
|
|
28532
|
-
|
|
28533
|
-
|
|
28534
|
-
|
|
28535
|
-
|
|
28536
|
-
|
|
28537
|
-
|
|
28538
|
-
|
|
28539
|
-
bt
|
|
28540
|
-
|
|
28541
|
-
|
|
28542
|
-
|
|
28543
|
-
if (!ss) {
|
|
28544
|
-
var wb = 1 << 10 + (dat[5] >> 3);
|
|
28545
|
-
ws = wb + (wb >> 3) * (dat[5] & 7);
|
|
28164
|
+
var rzb = function(dat, st, out) {
|
|
28165
|
+
var _a2;
|
|
28166
|
+
var bt = st.b;
|
|
28167
|
+
var b0 = dat[bt], btype = b0 >> 1 & 3;
|
|
28168
|
+
st.l = b0 & 1;
|
|
28169
|
+
var sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
|
|
28170
|
+
var ebt = (bt += 3) + sz;
|
|
28171
|
+
if (btype == 1) {
|
|
28172
|
+
if (bt >= dat.length)
|
|
28173
|
+
return;
|
|
28174
|
+
st.b = bt + 1;
|
|
28175
|
+
if (out) {
|
|
28176
|
+
fill(out, dat[bt], st.y, st.y += sz);
|
|
28177
|
+
return out;
|
|
28546
28178
|
}
|
|
28547
|
-
|
|
28548
|
-
err(1);
|
|
28549
|
-
var buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
|
|
28550
|
-
buf[0] = 1, buf[4] = 4, buf[8] = 8;
|
|
28551
|
-
return {
|
|
28552
|
-
b: bt + fsb,
|
|
28553
|
-
y: 0,
|
|
28554
|
-
l: 0,
|
|
28555
|
-
d: di,
|
|
28556
|
-
w: w && w != 1 ? w : buf.subarray(12),
|
|
28557
|
-
e: ws,
|
|
28558
|
-
o: new i32(buf.buffer, 0, 3),
|
|
28559
|
-
u: fss,
|
|
28560
|
-
c: cc,
|
|
28561
|
-
m: Math.min(131072, ws)
|
|
28562
|
-
};
|
|
28563
|
-
} else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
|
|
28564
|
-
return b4(dat, 4) + 8;
|
|
28179
|
+
return fill(new u8(sz), dat[bt]);
|
|
28565
28180
|
}
|
|
28566
|
-
|
|
28567
|
-
|
|
28568
|
-
|
|
28569
|
-
|
|
28570
|
-
|
|
28571
|
-
|
|
28572
|
-
|
|
28573
|
-
|
|
28574
|
-
var rfse = function(dat, bt, mal) {
|
|
28575
|
-
var tpos = (bt << 3) + 4;
|
|
28576
|
-
var al = (dat[bt] & 15) + 5;
|
|
28577
|
-
if (al > mal)
|
|
28578
|
-
err(3);
|
|
28579
|
-
var sz = 1 << al;
|
|
28580
|
-
var probs = sz, sym = -1, re = -1, i = -1, ht = sz;
|
|
28581
|
-
var buf = new ab(512 + (sz << 2));
|
|
28582
|
-
var freq = new i16(buf, 0, 256);
|
|
28583
|
-
var dstate = new u16(buf, 0, 256);
|
|
28584
|
-
var nstate = new u16(buf, 512, sz);
|
|
28585
|
-
var bb1 = 512 + (sz << 1);
|
|
28586
|
-
var syms = new u8(buf, bb1, sz);
|
|
28587
|
-
var nbits = new u8(buf, bb1 + sz);
|
|
28588
|
-
while (sym < 255 && probs > 0) {
|
|
28589
|
-
var bits = msb(probs + 1);
|
|
28590
|
-
var cbt = tpos >> 3;
|
|
28591
|
-
var msk = (1 << bits + 1) - 1;
|
|
28592
|
-
var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
|
|
28593
|
-
var msk1fb = (1 << bits) - 1;
|
|
28594
|
-
var msv = msk - probs - 1;
|
|
28595
|
-
var sval = val & msk1fb;
|
|
28596
|
-
if (sval < msv)
|
|
28597
|
-
tpos += bits, val = sval;
|
|
28598
|
-
else {
|
|
28599
|
-
tpos += bits + 1;
|
|
28600
|
-
if (val > msk1fb)
|
|
28601
|
-
val -= msv;
|
|
28602
|
-
}
|
|
28603
|
-
freq[++sym] = --val;
|
|
28604
|
-
if (val == -1) {
|
|
28605
|
-
probs += val;
|
|
28606
|
-
syms[--ht] = sym;
|
|
28607
|
-
} else
|
|
28608
|
-
probs -= val;
|
|
28609
|
-
if (!val) {
|
|
28610
|
-
do {
|
|
28611
|
-
var rbt = tpos >> 3;
|
|
28612
|
-
re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
|
|
28613
|
-
tpos += 2;
|
|
28614
|
-
sym += re;
|
|
28615
|
-
} while (re == 3);
|
|
28181
|
+
if (ebt > dat.length)
|
|
28182
|
+
return;
|
|
28183
|
+
if (btype == 0) {
|
|
28184
|
+
st.b = ebt;
|
|
28185
|
+
if (out) {
|
|
28186
|
+
out.set(dat.subarray(bt, ebt), st.y);
|
|
28187
|
+
st.y += sz;
|
|
28188
|
+
return out;
|
|
28616
28189
|
}
|
|
28190
|
+
return slc(dat, bt, ebt);
|
|
28617
28191
|
}
|
|
28618
|
-
if (
|
|
28619
|
-
|
|
28620
|
-
|
|
28621
|
-
|
|
28622
|
-
|
|
28623
|
-
|
|
28624
|
-
|
|
28625
|
-
|
|
28626
|
-
|
|
28627
|
-
|
|
28192
|
+
if (btype == 2) {
|
|
28193
|
+
var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
|
|
28194
|
+
var lss = b3 >> 4, lcs = 0, s4 = 0;
|
|
28195
|
+
if (lbt < 2) {
|
|
28196
|
+
if (sf & 1)
|
|
28197
|
+
lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
|
|
28198
|
+
else
|
|
28199
|
+
lss = b3 >> 3;
|
|
28200
|
+
} else {
|
|
28201
|
+
s4 = sf;
|
|
28202
|
+
if (sf < 2)
|
|
28203
|
+
lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
|
|
28204
|
+
else if (sf == 2)
|
|
28205
|
+
lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
|
|
28206
|
+
else
|
|
28207
|
+
lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
|
|
28628
28208
|
}
|
|
28629
|
-
|
|
28630
|
-
|
|
28631
|
-
|
|
28632
|
-
|
|
28633
|
-
|
|
28209
|
+
++bt;
|
|
28210
|
+
var buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
|
|
28211
|
+
var spl = buf.length - lss;
|
|
28212
|
+
if (lbt == 0)
|
|
28213
|
+
buf.set(dat.subarray(bt, bt += lss), spl);
|
|
28214
|
+
else if (lbt == 1)
|
|
28215
|
+
fill(buf, dat[bt++], spl);
|
|
28216
|
+
else {
|
|
28217
|
+
var hu = st.h;
|
|
28218
|
+
if (lbt == 2) {
|
|
28219
|
+
var hud = rhu(dat, bt);
|
|
28220
|
+
lcs += bt - (bt = hud[0]);
|
|
28221
|
+
st.h = hu = hud[1];
|
|
28222
|
+
} else if (!hu)
|
|
28223
|
+
err(0);
|
|
28224
|
+
(s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
|
|
28634
28225
|
}
|
|
28226
|
+
var ns = dat[bt++];
|
|
28227
|
+
if (ns) {
|
|
28228
|
+
if (ns == 255)
|
|
28229
|
+
ns = (dat[bt++] | dat[bt++] << 8) + 32512;
|
|
28230
|
+
else if (ns > 127)
|
|
28231
|
+
ns = ns - 128 << 8 | dat[bt++];
|
|
28232
|
+
var scm = dat[bt++];
|
|
28233
|
+
if (scm & 3)
|
|
28234
|
+
err(0);
|
|
28235
|
+
var dts = [dmlt, doct, dllt];
|
|
28236
|
+
for (var i = 2;i > -1; --i) {
|
|
28237
|
+
var md = scm >> (i << 1) + 2 & 3;
|
|
28238
|
+
if (md == 1) {
|
|
28239
|
+
var rbuf = new u8([0, 0, dat[bt++]]);
|
|
28240
|
+
dts[i] = {
|
|
28241
|
+
s: rbuf.subarray(2, 3),
|
|
28242
|
+
n: rbuf.subarray(0, 1),
|
|
28243
|
+
t: new u16(rbuf.buffer, 0, 1),
|
|
28244
|
+
b: 0
|
|
28245
|
+
};
|
|
28246
|
+
} else if (md == 2) {
|
|
28247
|
+
_a2 = rfse(dat, bt, 9 - (i & 1)), bt = _a2[0], dts[i] = _a2[1];
|
|
28248
|
+
} else if (md == 3) {
|
|
28249
|
+
if (!st.t)
|
|
28250
|
+
err(0);
|
|
28251
|
+
dts[i] = st.t[i];
|
|
28252
|
+
}
|
|
28253
|
+
}
|
|
28254
|
+
var _b = st.t = dts, mlt = _b[0], oct = _b[1], llt = _b[2];
|
|
28255
|
+
var lb = dat[ebt - 1];
|
|
28256
|
+
if (!lb)
|
|
28257
|
+
err(0);
|
|
28258
|
+
var spos = (ebt << 3) - 8 + msb(lb) - llt.b, cbt = spos >> 3, oubt = 0;
|
|
28259
|
+
var lst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << llt.b) - 1;
|
|
28260
|
+
cbt = (spos -= oct.b) >> 3;
|
|
28261
|
+
var ost = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << oct.b) - 1;
|
|
28262
|
+
cbt = (spos -= mlt.b) >> 3;
|
|
28263
|
+
var mst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mlt.b) - 1;
|
|
28264
|
+
for (++ns;--ns; ) {
|
|
28265
|
+
var llc = llt.s[lst];
|
|
28266
|
+
var lbtr = llt.n[lst];
|
|
28267
|
+
var mlc = mlt.s[mst];
|
|
28268
|
+
var mbtr = mlt.n[mst];
|
|
28269
|
+
var ofc = oct.s[ost];
|
|
28270
|
+
var obtr = oct.n[ost];
|
|
28271
|
+
cbt = (spos -= ofc) >> 3;
|
|
28272
|
+
var ofp = 1 << ofc;
|
|
28273
|
+
var off = ofp + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16 | dat[cbt + 3] << 24) >>> (spos & 7) & ofp - 1);
|
|
28274
|
+
cbt = (spos -= mlb[mlc]) >> 3;
|
|
28275
|
+
var ml = mlbl[mlc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << mlb[mlc]) - 1);
|
|
28276
|
+
cbt = (spos -= llb[llc]) >> 3;
|
|
28277
|
+
var ll = llbl[llc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << llb[llc]) - 1);
|
|
28278
|
+
cbt = (spos -= lbtr) >> 3;
|
|
28279
|
+
lst = llt.t[lst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << lbtr) - 1);
|
|
28280
|
+
cbt = (spos -= mbtr) >> 3;
|
|
28281
|
+
mst = mlt.t[mst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mbtr) - 1);
|
|
28282
|
+
cbt = (spos -= obtr) >> 3;
|
|
28283
|
+
ost = oct.t[ost] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << obtr) - 1);
|
|
28284
|
+
if (off > 3) {
|
|
28285
|
+
st.o[2] = st.o[1];
|
|
28286
|
+
st.o[1] = st.o[0];
|
|
28287
|
+
st.o[0] = off -= 3;
|
|
28288
|
+
} else {
|
|
28289
|
+
var idx = off - (ll != 0);
|
|
28290
|
+
if (idx) {
|
|
28291
|
+
off = idx == 3 ? st.o[0] - 1 : st.o[idx];
|
|
28292
|
+
if (idx > 1)
|
|
28293
|
+
st.o[2] = st.o[1];
|
|
28294
|
+
st.o[1] = st.o[0];
|
|
28295
|
+
st.o[0] = off;
|
|
28296
|
+
} else
|
|
28297
|
+
off = st.o[0];
|
|
28298
|
+
}
|
|
28299
|
+
for (var i = 0;i < ll; ++i) {
|
|
28300
|
+
buf[oubt + i] = buf[spl + i];
|
|
28301
|
+
}
|
|
28302
|
+
oubt += ll, spl += ll;
|
|
28303
|
+
var stin = oubt - off;
|
|
28304
|
+
if (stin < 0) {
|
|
28305
|
+
var len = -stin;
|
|
28306
|
+
var bs = st.e + stin;
|
|
28307
|
+
if (len > ml)
|
|
28308
|
+
len = ml;
|
|
28309
|
+
for (var i = 0;i < len; ++i) {
|
|
28310
|
+
buf[oubt + i] = st.w[bs + i];
|
|
28311
|
+
}
|
|
28312
|
+
oubt += len, ml -= len, stin = 0;
|
|
28313
|
+
}
|
|
28314
|
+
for (var i = 0;i < ml; ++i) {
|
|
28315
|
+
buf[oubt + i] = buf[stin + i];
|
|
28316
|
+
}
|
|
28317
|
+
oubt += ml;
|
|
28318
|
+
}
|
|
28319
|
+
if (oubt != spl) {
|
|
28320
|
+
while (spl < buf.length) {
|
|
28321
|
+
buf[oubt++] = buf[spl++];
|
|
28322
|
+
}
|
|
28323
|
+
} else
|
|
28324
|
+
oubt = buf.length;
|
|
28325
|
+
if (out)
|
|
28326
|
+
st.y += oubt;
|
|
28327
|
+
else
|
|
28328
|
+
buf = slc(buf, 0, oubt);
|
|
28329
|
+
} else if (out) {
|
|
28330
|
+
st.y += lss;
|
|
28331
|
+
if (spl) {
|
|
28332
|
+
for (var i = 0;i < lss; ++i) {
|
|
28333
|
+
buf[i] = buf[spl + i];
|
|
28334
|
+
}
|
|
28335
|
+
}
|
|
28336
|
+
} else if (spl)
|
|
28337
|
+
buf = slc(buf, spl);
|
|
28338
|
+
st.b = ebt;
|
|
28339
|
+
return buf;
|
|
28635
28340
|
}
|
|
28636
|
-
|
|
28637
|
-
|
|
28638
|
-
|
|
28639
|
-
|
|
28640
|
-
|
|
28641
|
-
|
|
28341
|
+
err(2);
|
|
28342
|
+
};
|
|
28343
|
+
var cct = function(bufs, ol) {
|
|
28344
|
+
if (bufs.length == 1)
|
|
28345
|
+
return bufs[0];
|
|
28346
|
+
var buf = new u8(ol);
|
|
28347
|
+
for (var i = 0, b = 0;i < bufs.length; ++i) {
|
|
28348
|
+
var chk = bufs[i];
|
|
28349
|
+
buf.set(chk, b);
|
|
28350
|
+
b += chk.length;
|
|
28642
28351
|
}
|
|
28643
|
-
return
|
|
28644
|
-
b: al,
|
|
28645
|
-
s: syms,
|
|
28646
|
-
n: nbits,
|
|
28647
|
-
t: nstate
|
|
28648
|
-
}];
|
|
28649
|
-
};
|
|
28650
|
-
var rhu = function(dat, bt) {
|
|
28651
|
-
var i = 0, wc = -1;
|
|
28652
|
-
var buf = new u8(292), hb = dat[bt];
|
|
28653
|
-
var hw = buf.subarray(0, 256);
|
|
28654
|
-
var rc = buf.subarray(256, 268);
|
|
28655
|
-
var ri = new u16(buf.buffer, 268);
|
|
28656
|
-
if (hb < 128) {
|
|
28657
|
-
var _a2 = rfse(dat, bt + 1, 6), ebt = _a2[0], fdt = _a2[1];
|
|
28658
|
-
bt += hb;
|
|
28659
|
-
var epos = ebt << 3;
|
|
28660
|
-
var lb = dat[bt];
|
|
28661
|
-
if (!lb)
|
|
28662
|
-
err(0);
|
|
28663
|
-
var st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
|
|
28664
|
-
var fpos = (++bt << 3) - 8 + msb(lb);
|
|
28665
|
-
for (;; ) {
|
|
28666
|
-
fpos -= btr1;
|
|
28667
|
-
if (fpos < epos)
|
|
28668
|
-
break;
|
|
28669
|
-
var cbt = fpos >> 3;
|
|
28670
|
-
st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
|
|
28671
|
-
hw[++wc] = fdt.s[st1];
|
|
28672
|
-
fpos -= btr2;
|
|
28673
|
-
if (fpos < epos)
|
|
28674
|
-
break;
|
|
28675
|
-
cbt = fpos >> 3;
|
|
28676
|
-
st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
|
|
28677
|
-
hw[++wc] = fdt.s[st2];
|
|
28678
|
-
btr1 = fdt.n[st1];
|
|
28679
|
-
st1 = fdt.t[st1];
|
|
28680
|
-
btr2 = fdt.n[st2];
|
|
28681
|
-
st2 = fdt.t[st2];
|
|
28682
|
-
}
|
|
28683
|
-
if (++wc > 255)
|
|
28684
|
-
err(0);
|
|
28685
|
-
} else {
|
|
28686
|
-
wc = hb - 127;
|
|
28687
|
-
for (;i < wc; i += 2) {
|
|
28688
|
-
var byte = dat[++bt];
|
|
28689
|
-
hw[i] = byte >> 4;
|
|
28690
|
-
hw[i + 1] = byte & 15;
|
|
28691
|
-
}
|
|
28692
|
-
++bt;
|
|
28693
|
-
}
|
|
28694
|
-
var wes = 0;
|
|
28695
|
-
for (i = 0;i < wc; ++i) {
|
|
28696
|
-
var wt = hw[i];
|
|
28697
|
-
if (wt > 11)
|
|
28698
|
-
err(0);
|
|
28699
|
-
wes += wt && 1 << wt - 1;
|
|
28700
|
-
}
|
|
28701
|
-
var mb = msb(wes) + 1;
|
|
28702
|
-
var ts = 1 << mb;
|
|
28703
|
-
var rem = ts - wes;
|
|
28704
|
-
if (rem & rem - 1)
|
|
28705
|
-
err(0);
|
|
28706
|
-
hw[wc++] = msb(rem) + 1;
|
|
28707
|
-
for (i = 0;i < wc; ++i) {
|
|
28708
|
-
var wt = hw[i];
|
|
28709
|
-
++rc[hw[i] = wt && mb + 1 - wt];
|
|
28710
|
-
}
|
|
28711
|
-
var hbuf = new u8(ts << 1);
|
|
28712
|
-
var syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
|
|
28713
|
-
ri[mb] = 0;
|
|
28714
|
-
for (i = mb;i > 0; --i) {
|
|
28715
|
-
var pv = ri[i];
|
|
28716
|
-
fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
|
|
28717
|
-
}
|
|
28718
|
-
if (ri[0] != ts)
|
|
28719
|
-
err(0);
|
|
28720
|
-
for (i = 0;i < wc; ++i) {
|
|
28721
|
-
var bits = hw[i];
|
|
28722
|
-
if (bits) {
|
|
28723
|
-
var code = ri[bits];
|
|
28724
|
-
fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
|
|
28725
|
-
}
|
|
28726
|
-
}
|
|
28727
|
-
return [bt, {
|
|
28728
|
-
n: nb,
|
|
28729
|
-
b: mb,
|
|
28730
|
-
s: syms
|
|
28731
|
-
}];
|
|
28732
|
-
};
|
|
28733
|
-
var dllt = rfse(/* @__PURE__ */ new u8([
|
|
28734
|
-
81,
|
|
28735
|
-
16,
|
|
28736
|
-
99,
|
|
28737
|
-
140,
|
|
28738
|
-
49,
|
|
28739
|
-
198,
|
|
28740
|
-
24,
|
|
28741
|
-
99,
|
|
28742
|
-
12,
|
|
28743
|
-
33,
|
|
28744
|
-
196,
|
|
28745
|
-
24,
|
|
28746
|
-
99,
|
|
28747
|
-
102,
|
|
28748
|
-
102,
|
|
28749
|
-
134,
|
|
28750
|
-
70,
|
|
28751
|
-
146,
|
|
28752
|
-
4
|
|
28753
|
-
]), 0, 6)[1];
|
|
28754
|
-
var dmlt = rfse(/* @__PURE__ */ new u8([
|
|
28755
|
-
33,
|
|
28756
|
-
20,
|
|
28757
|
-
196,
|
|
28758
|
-
24,
|
|
28759
|
-
99,
|
|
28760
|
-
140,
|
|
28761
|
-
33,
|
|
28762
|
-
132,
|
|
28763
|
-
16,
|
|
28764
|
-
66,
|
|
28765
|
-
8,
|
|
28766
|
-
33,
|
|
28767
|
-
132,
|
|
28768
|
-
16,
|
|
28769
|
-
66,
|
|
28770
|
-
8,
|
|
28771
|
-
33,
|
|
28772
|
-
68,
|
|
28773
|
-
68,
|
|
28774
|
-
68,
|
|
28775
|
-
68,
|
|
28776
|
-
68,
|
|
28777
|
-
68,
|
|
28778
|
-
68,
|
|
28779
|
-
68,
|
|
28780
|
-
36,
|
|
28781
|
-
9
|
|
28782
|
-
]), 0, 6)[1];
|
|
28783
|
-
var doct = rfse(/* @__PURE__ */ new u8([
|
|
28784
|
-
32,
|
|
28785
|
-
132,
|
|
28786
|
-
16,
|
|
28787
|
-
66,
|
|
28788
|
-
102,
|
|
28789
|
-
70,
|
|
28790
|
-
68,
|
|
28791
|
-
68,
|
|
28792
|
-
68,
|
|
28793
|
-
68,
|
|
28794
|
-
36,
|
|
28795
|
-
73,
|
|
28796
|
-
2
|
|
28797
|
-
]), 0, 5)[1];
|
|
28798
|
-
var b2bl = function(b, s) {
|
|
28799
|
-
var len = b.length, bl = new i32(len);
|
|
28800
|
-
for (var i = 0;i < len; ++i) {
|
|
28801
|
-
bl[i] = s;
|
|
28802
|
-
s += 1 << b[i];
|
|
28803
|
-
}
|
|
28804
|
-
return bl;
|
|
28805
|
-
};
|
|
28806
|
-
var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
28807
|
-
0,
|
|
28808
|
-
0,
|
|
28809
|
-
0,
|
|
28810
|
-
0,
|
|
28811
|
-
16843009,
|
|
28812
|
-
50528770,
|
|
28813
|
-
134678020,
|
|
28814
|
-
202050057,
|
|
28815
|
-
269422093
|
|
28816
|
-
])).buffer, 0, 36);
|
|
28817
|
-
var llbl = /* @__PURE__ */ b2bl(llb, 0);
|
|
28818
|
-
var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
28819
|
-
0,
|
|
28820
|
-
0,
|
|
28821
|
-
0,
|
|
28822
|
-
0,
|
|
28823
|
-
0,
|
|
28824
|
-
0,
|
|
28825
|
-
0,
|
|
28826
|
-
0,
|
|
28827
|
-
16843009,
|
|
28828
|
-
50528770,
|
|
28829
|
-
117769220,
|
|
28830
|
-
185207048,
|
|
28831
|
-
252579084,
|
|
28832
|
-
16
|
|
28833
|
-
])).buffer, 0, 53);
|
|
28834
|
-
var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
|
|
28835
|
-
var dhu = function(dat, out, hu) {
|
|
28836
|
-
var len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
|
|
28837
|
-
if (!lb)
|
|
28838
|
-
err(0);
|
|
28839
|
-
var st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
|
|
28840
|
-
for (;pos > eb && i < ss; ) {
|
|
28841
|
-
var cbt = pos >> 3;
|
|
28842
|
-
var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
|
|
28843
|
-
st = (st << btr | val) & msk;
|
|
28844
|
-
out[++i] = hu.s[st];
|
|
28845
|
-
pos -= btr = hu.n[st];
|
|
28846
|
-
}
|
|
28847
|
-
if (pos != eb || i + 1 != ss)
|
|
28848
|
-
err(0);
|
|
28849
|
-
};
|
|
28850
|
-
var dhu4 = function(dat, out, hu) {
|
|
28851
|
-
var bt = 6;
|
|
28852
|
-
var ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
|
|
28853
|
-
dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
|
|
28854
|
-
dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
|
|
28855
|
-
dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
|
|
28856
|
-
dhu(dat.subarray(bt), out.subarray(sz3), hu);
|
|
28857
|
-
};
|
|
28858
|
-
var rzb = function(dat, st, out) {
|
|
28859
|
-
var _a2;
|
|
28860
|
-
var bt = st.b;
|
|
28861
|
-
var b0 = dat[bt], btype = b0 >> 1 & 3;
|
|
28862
|
-
st.l = b0 & 1;
|
|
28863
|
-
var sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
|
|
28864
|
-
var ebt = (bt += 3) + sz;
|
|
28865
|
-
if (btype == 1) {
|
|
28866
|
-
if (bt >= dat.length)
|
|
28867
|
-
return;
|
|
28868
|
-
st.b = bt + 1;
|
|
28869
|
-
if (out) {
|
|
28870
|
-
fill(out, dat[bt], st.y, st.y += sz);
|
|
28871
|
-
return out;
|
|
28872
|
-
}
|
|
28873
|
-
return fill(new u8(sz), dat[bt]);
|
|
28874
|
-
}
|
|
28875
|
-
if (ebt > dat.length)
|
|
28876
|
-
return;
|
|
28877
|
-
if (btype == 0) {
|
|
28878
|
-
st.b = ebt;
|
|
28879
|
-
if (out) {
|
|
28880
|
-
out.set(dat.subarray(bt, ebt), st.y);
|
|
28881
|
-
st.y += sz;
|
|
28882
|
-
return out;
|
|
28883
|
-
}
|
|
28884
|
-
return slc(dat, bt, ebt);
|
|
28885
|
-
}
|
|
28886
|
-
if (btype == 2) {
|
|
28887
|
-
var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
|
|
28888
|
-
var lss = b3 >> 4, lcs = 0, s4 = 0;
|
|
28889
|
-
if (lbt < 2) {
|
|
28890
|
-
if (sf & 1)
|
|
28891
|
-
lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
|
|
28892
|
-
else
|
|
28893
|
-
lss = b3 >> 3;
|
|
28894
|
-
} else {
|
|
28895
|
-
s4 = sf;
|
|
28896
|
-
if (sf < 2)
|
|
28897
|
-
lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
|
|
28898
|
-
else if (sf == 2)
|
|
28899
|
-
lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
|
|
28900
|
-
else
|
|
28901
|
-
lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
|
|
28902
|
-
}
|
|
28903
|
-
++bt;
|
|
28904
|
-
var buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
|
|
28905
|
-
var spl = buf.length - lss;
|
|
28906
|
-
if (lbt == 0)
|
|
28907
|
-
buf.set(dat.subarray(bt, bt += lss), spl);
|
|
28908
|
-
else if (lbt == 1)
|
|
28909
|
-
fill(buf, dat[bt++], spl);
|
|
28910
|
-
else {
|
|
28911
|
-
var hu = st.h;
|
|
28912
|
-
if (lbt == 2) {
|
|
28913
|
-
var hud = rhu(dat, bt);
|
|
28914
|
-
lcs += bt - (bt = hud[0]);
|
|
28915
|
-
st.h = hu = hud[1];
|
|
28916
|
-
} else if (!hu)
|
|
28917
|
-
err(0);
|
|
28918
|
-
(s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
|
|
28919
|
-
}
|
|
28920
|
-
var ns = dat[bt++];
|
|
28921
|
-
if (ns) {
|
|
28922
|
-
if (ns == 255)
|
|
28923
|
-
ns = (dat[bt++] | dat[bt++] << 8) + 32512;
|
|
28924
|
-
else if (ns > 127)
|
|
28925
|
-
ns = ns - 128 << 8 | dat[bt++];
|
|
28926
|
-
var scm = dat[bt++];
|
|
28927
|
-
if (scm & 3)
|
|
28928
|
-
err(0);
|
|
28929
|
-
var dts = [dmlt, doct, dllt];
|
|
28930
|
-
for (var i = 2;i > -1; --i) {
|
|
28931
|
-
var md = scm >> (i << 1) + 2 & 3;
|
|
28932
|
-
if (md == 1) {
|
|
28933
|
-
var rbuf = new u8([0, 0, dat[bt++]]);
|
|
28934
|
-
dts[i] = {
|
|
28935
|
-
s: rbuf.subarray(2, 3),
|
|
28936
|
-
n: rbuf.subarray(0, 1),
|
|
28937
|
-
t: new u16(rbuf.buffer, 0, 1),
|
|
28938
|
-
b: 0
|
|
28939
|
-
};
|
|
28940
|
-
} else if (md == 2) {
|
|
28941
|
-
_a2 = rfse(dat, bt, 9 - (i & 1)), bt = _a2[0], dts[i] = _a2[1];
|
|
28942
|
-
} else if (md == 3) {
|
|
28943
|
-
if (!st.t)
|
|
28944
|
-
err(0);
|
|
28945
|
-
dts[i] = st.t[i];
|
|
28946
|
-
}
|
|
28947
|
-
}
|
|
28948
|
-
var _b = st.t = dts, mlt = _b[0], oct = _b[1], llt = _b[2];
|
|
28949
|
-
var lb = dat[ebt - 1];
|
|
28950
|
-
if (!lb)
|
|
28951
|
-
err(0);
|
|
28952
|
-
var spos = (ebt << 3) - 8 + msb(lb) - llt.b, cbt = spos >> 3, oubt = 0;
|
|
28953
|
-
var lst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << llt.b) - 1;
|
|
28954
|
-
cbt = (spos -= oct.b) >> 3;
|
|
28955
|
-
var ost = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << oct.b) - 1;
|
|
28956
|
-
cbt = (spos -= mlt.b) >> 3;
|
|
28957
|
-
var mst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mlt.b) - 1;
|
|
28958
|
-
for (++ns;--ns; ) {
|
|
28959
|
-
var llc = llt.s[lst];
|
|
28960
|
-
var lbtr = llt.n[lst];
|
|
28961
|
-
var mlc = mlt.s[mst];
|
|
28962
|
-
var mbtr = mlt.n[mst];
|
|
28963
|
-
var ofc = oct.s[ost];
|
|
28964
|
-
var obtr = oct.n[ost];
|
|
28965
|
-
cbt = (spos -= ofc) >> 3;
|
|
28966
|
-
var ofp = 1 << ofc;
|
|
28967
|
-
var off = ofp + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16 | dat[cbt + 3] << 24) >>> (spos & 7) & ofp - 1);
|
|
28968
|
-
cbt = (spos -= mlb[mlc]) >> 3;
|
|
28969
|
-
var ml = mlbl[mlc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << mlb[mlc]) - 1);
|
|
28970
|
-
cbt = (spos -= llb[llc]) >> 3;
|
|
28971
|
-
var ll = llbl[llc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << llb[llc]) - 1);
|
|
28972
|
-
cbt = (spos -= lbtr) >> 3;
|
|
28973
|
-
lst = llt.t[lst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << lbtr) - 1);
|
|
28974
|
-
cbt = (spos -= mbtr) >> 3;
|
|
28975
|
-
mst = mlt.t[mst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mbtr) - 1);
|
|
28976
|
-
cbt = (spos -= obtr) >> 3;
|
|
28977
|
-
ost = oct.t[ost] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << obtr) - 1);
|
|
28978
|
-
if (off > 3) {
|
|
28979
|
-
st.o[2] = st.o[1];
|
|
28980
|
-
st.o[1] = st.o[0];
|
|
28981
|
-
st.o[0] = off -= 3;
|
|
28982
|
-
} else {
|
|
28983
|
-
var idx = off - (ll != 0);
|
|
28984
|
-
if (idx) {
|
|
28985
|
-
off = idx == 3 ? st.o[0] - 1 : st.o[idx];
|
|
28986
|
-
if (idx > 1)
|
|
28987
|
-
st.o[2] = st.o[1];
|
|
28988
|
-
st.o[1] = st.o[0];
|
|
28989
|
-
st.o[0] = off;
|
|
28990
|
-
} else
|
|
28991
|
-
off = st.o[0];
|
|
28992
|
-
}
|
|
28993
|
-
for (var i = 0;i < ll; ++i) {
|
|
28994
|
-
buf[oubt + i] = buf[spl + i];
|
|
28995
|
-
}
|
|
28996
|
-
oubt += ll, spl += ll;
|
|
28997
|
-
var stin = oubt - off;
|
|
28998
|
-
if (stin < 0) {
|
|
28999
|
-
var len = -stin;
|
|
29000
|
-
var bs = st.e + stin;
|
|
29001
|
-
if (len > ml)
|
|
29002
|
-
len = ml;
|
|
29003
|
-
for (var i = 0;i < len; ++i) {
|
|
29004
|
-
buf[oubt + i] = st.w[bs + i];
|
|
29005
|
-
}
|
|
29006
|
-
oubt += len, ml -= len, stin = 0;
|
|
29007
|
-
}
|
|
29008
|
-
for (var i = 0;i < ml; ++i) {
|
|
29009
|
-
buf[oubt + i] = buf[stin + i];
|
|
29010
|
-
}
|
|
29011
|
-
oubt += ml;
|
|
29012
|
-
}
|
|
29013
|
-
if (oubt != spl) {
|
|
29014
|
-
while (spl < buf.length) {
|
|
29015
|
-
buf[oubt++] = buf[spl++];
|
|
29016
|
-
}
|
|
29017
|
-
} else
|
|
29018
|
-
oubt = buf.length;
|
|
29019
|
-
if (out)
|
|
29020
|
-
st.y += oubt;
|
|
29021
|
-
else
|
|
29022
|
-
buf = slc(buf, 0, oubt);
|
|
29023
|
-
} else if (out) {
|
|
29024
|
-
st.y += lss;
|
|
29025
|
-
if (spl) {
|
|
29026
|
-
for (var i = 0;i < lss; ++i) {
|
|
29027
|
-
buf[i] = buf[spl + i];
|
|
29028
|
-
}
|
|
29029
|
-
}
|
|
29030
|
-
} else if (spl)
|
|
29031
|
-
buf = slc(buf, spl);
|
|
29032
|
-
st.b = ebt;
|
|
29033
|
-
return buf;
|
|
29034
|
-
}
|
|
29035
|
-
err(2);
|
|
29036
|
-
};
|
|
29037
|
-
var cct = function(bufs, ol) {
|
|
29038
|
-
if (bufs.length == 1)
|
|
29039
|
-
return bufs[0];
|
|
29040
|
-
var buf = new u8(ol);
|
|
29041
|
-
for (var i = 0, b = 0;i < bufs.length; ++i) {
|
|
29042
|
-
var chk = bufs[i];
|
|
29043
|
-
buf.set(chk, b);
|
|
29044
|
-
b += chk.length;
|
|
29045
|
-
}
|
|
29046
|
-
return buf;
|
|
28352
|
+
return buf;
|
|
29047
28353
|
};
|
|
29048
28354
|
function decompress(dat, buf) {
|
|
29049
28355
|
var bufs = [], nb = +!buf;
|
|
@@ -32458,7 +31764,7 @@ var CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE = Object.freeze({
|
|
|
32458
31764
|
snapshotGrouping: "event_time_final_update_id_object",
|
|
32459
31765
|
sequenceSemantics: "okx_seq_id_prev_seq_id",
|
|
32460
31766
|
constructionModes: ["sampled_top_n_snapshot"],
|
|
32461
|
-
sourcePolicies: ["authoritative_window"]
|
|
31767
|
+
sourcePolicies: ["authoritative_window", "fill_gaps"]
|
|
32462
31768
|
});
|
|
32463
31769
|
|
|
32464
31770
|
class CryptoHftDataError extends Error {
|
|
@@ -32469,498 +31775,913 @@ class CryptoHftDataError extends Error {
|
|
|
32469
31775
|
this.name = "CryptoHftDataError";
|
|
32470
31776
|
}
|
|
32471
31777
|
}
|
|
32472
|
-
function cryptoHftDataCapabilityFor(request, profiles = []) {
|
|
32473
|
-
if (request.providerPolicy.provider !== "cryptohftdata" || request.scope.feed !== "ORDERBOOK" || request.scope.exchange.trim().toLowerCase() === "mexc") {
|
|
32474
|
-
return;
|
|
32475
|
-
}
|
|
32476
|
-
const profile = profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.constructionModes.includes(request.constructionMode) && candidate.sourcePolicies.includes(request.sourcePolicy) && request.window.startTimeMs >= candidate.historyStartMs && request.depth <= candidate.maxDepth);
|
|
32477
|
-
if (!profile)
|
|
32478
|
-
return;
|
|
32479
|
-
return {
|
|
32480
|
-
provider: "cryptohftdata",
|
|
32481
|
-
adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
|
|
32482
|
-
providerExchangeId: profile.providerExchangeId,
|
|
32483
|
-
resolvedSymbol: profile.sourceSymbol
|
|
32484
|
-
};
|
|
31778
|
+
function cryptoHftDataCapabilityFor(request, profiles = []) {
|
|
31779
|
+
if (request.providerPolicy.provider !== "cryptohftdata" || request.scope.feed !== "ORDERBOOK" || request.scope.exchange.trim().toLowerCase() === "mexc") {
|
|
31780
|
+
return;
|
|
31781
|
+
}
|
|
31782
|
+
const profile = profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.constructionModes.includes(request.constructionMode) && candidate.sourcePolicies.includes(request.sourcePolicy) && request.window.startTimeMs >= candidate.historyStartMs && request.depth <= candidate.maxDepth);
|
|
31783
|
+
if (!profile)
|
|
31784
|
+
return;
|
|
31785
|
+
return {
|
|
31786
|
+
provider: "cryptohftdata",
|
|
31787
|
+
adapterVersion: CRYPTOHFTDATA_ADAPTER_VERSION,
|
|
31788
|
+
providerExchangeId: profile.providerExchangeId,
|
|
31789
|
+
resolvedSymbol: profile.sourceSymbol
|
|
31790
|
+
};
|
|
31791
|
+
}
|
|
31792
|
+
function utcHourPath(hourMs) {
|
|
31793
|
+
const date5 = new Date(hourMs);
|
|
31794
|
+
return {
|
|
31795
|
+
date: `${date5.getUTCFullYear()}-${String(date5.getUTCMonth() + 1).padStart(2, "0")}-${String(date5.getUTCDate()).padStart(2, "0")}`,
|
|
31796
|
+
hour: String(date5.getUTCHours()).padStart(2, "0")
|
|
31797
|
+
};
|
|
31798
|
+
}
|
|
31799
|
+
function enumerateCryptoHftDataObjects(request, providerExchangeId, resolvedSymbol) {
|
|
31800
|
+
if (request.sourcePolicy === "fill_gaps") {
|
|
31801
|
+
const hours = new Set;
|
|
31802
|
+
for (const targetTimeMs of request.requiredClockTargetsMs) {
|
|
31803
|
+
const firstHour2 = Math.floor(Math.max(0, targetTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
|
|
31804
|
+
const lastHour = Math.floor(targetTimeMs / HOUR_MS) * HOUR_MS;
|
|
31805
|
+
for (let hourMs = firstHour2;hourMs <= lastHour; hourMs += HOUR_MS) {
|
|
31806
|
+
hours.add(hourMs);
|
|
31807
|
+
}
|
|
31808
|
+
}
|
|
31809
|
+
return [...hours].sort((left, right) => left - right).map((hourMs) => {
|
|
31810
|
+
const utc = utcHourPath(hourMs);
|
|
31811
|
+
return `${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`;
|
|
31812
|
+
});
|
|
31813
|
+
}
|
|
31814
|
+
const firstHour = Math.floor((request.window.startTimeMs - request.budgets.maxBoundaryLookbackMs) / HOUR_MS) * HOUR_MS;
|
|
31815
|
+
const objects = [];
|
|
31816
|
+
for (let hourMs = Math.max(0, firstHour);hourMs < request.window.endTimeMs; hourMs += HOUR_MS) {
|
|
31817
|
+
const utc = utcHourPath(hourMs);
|
|
31818
|
+
objects.push(`${providerExchangeId}/${utc.date}/${utc.hour}/${resolvedSymbol}_orderbook.parquet.zst`);
|
|
31819
|
+
}
|
|
31820
|
+
return objects;
|
|
31821
|
+
}
|
|
31822
|
+
function initialArchiveCoversTarget(request, targetTimeMs) {
|
|
31823
|
+
return (request.initialSelection?.support_anchors ?? []).some((anchor) => {
|
|
31824
|
+
const sourceTimeMs = Date.parse(anchor.source_time);
|
|
31825
|
+
return Number.isSafeInteger(sourceTimeMs) && sourceTimeMs <= targetTimeMs && targetTimeMs - sourceTimeMs <= request.maxPriorAsOfLagMs;
|
|
31826
|
+
});
|
|
31827
|
+
}
|
|
31828
|
+
function providerAcquisitionRequest(request) {
|
|
31829
|
+
if (request.sourcePolicy !== "fill_gaps")
|
|
31830
|
+
return request;
|
|
31831
|
+
const requiredClockTargetsMs = request.requiredClockTargetsMs.filter((targetTimeMs) => !initialArchiveCoversTarget(request, targetTimeMs));
|
|
31832
|
+
if (requiredClockTargetsMs.length === 0) {
|
|
31833
|
+
throw new CryptoHftDataError("fill_gaps_has_no_uncovered_clock_targets");
|
|
31834
|
+
}
|
|
31835
|
+
return { ...request, requiredClockTargetsMs };
|
|
31836
|
+
}
|
|
31837
|
+
function unsignedString(value, field, required2 = false) {
|
|
31838
|
+
if (value === null || value === undefined) {
|
|
31839
|
+
if (required2)
|
|
31840
|
+
throw new CryptoHftDataError(`schema_${field}_missing`);
|
|
31841
|
+
return;
|
|
31842
|
+
}
|
|
31843
|
+
const rendered = String(value);
|
|
31844
|
+
if (!/^\d+$/.test(rendered)) {
|
|
31845
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
31846
|
+
}
|
|
31847
|
+
const parsed = BigInt(rendered);
|
|
31848
|
+
if (parsed > 18446744073709551615n) {
|
|
31849
|
+
throw new CryptoHftDataError(`schema_${field}_exceeds_uint64`);
|
|
31850
|
+
}
|
|
31851
|
+
return parsed.toString(10);
|
|
31852
|
+
}
|
|
31853
|
+
function timestampMs2(value, field) {
|
|
31854
|
+
const rendered = String(value);
|
|
31855
|
+
if (!/^\d+$/.test(rendered)) {
|
|
31856
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
31857
|
+
}
|
|
31858
|
+
const raw = BigInt(rendered);
|
|
31859
|
+
const milliseconds = field === "received_time" ? raw / 1000000n : raw;
|
|
31860
|
+
if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
31861
|
+
throw new CryptoHftDataError(`schema_${field}_unsafe`);
|
|
31862
|
+
}
|
|
31863
|
+
return Number(milliseconds);
|
|
31864
|
+
}
|
|
31865
|
+
function decimal(value, field, allowZero) {
|
|
31866
|
+
if (!/^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) {
|
|
31867
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
31868
|
+
}
|
|
31869
|
+
const parsed = Number(value);
|
|
31870
|
+
if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
|
|
31871
|
+
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
31872
|
+
}
|
|
31873
|
+
return parsed;
|
|
31874
|
+
}
|
|
31875
|
+
function validatedRow(request, row) {
|
|
31876
|
+
if (row.symbol !== request.scope.sourceSymbol) {
|
|
31877
|
+
throw new CryptoHftDataError("schema_symbol_mismatch");
|
|
31878
|
+
}
|
|
31879
|
+
if (row.event_type !== "snapshot" && row.event_type !== "update") {
|
|
31880
|
+
throw new CryptoHftDataError("schema_event_type_invalid");
|
|
31881
|
+
}
|
|
31882
|
+
if (row.side !== "bid" && row.side !== "ask") {
|
|
31883
|
+
throw new CryptoHftDataError("schema_side_invalid");
|
|
31884
|
+
}
|
|
31885
|
+
decimal(row.price, "price", false);
|
|
31886
|
+
decimal(row.quantity, "quantity", true);
|
|
31887
|
+
if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
|
|
31888
|
+
throw new CryptoHftDataError("schema_object_checksum_invalid");
|
|
31889
|
+
}
|
|
31890
|
+
const eventTimeMs = timestampMs2(row.event_time, "event_time");
|
|
31891
|
+
const receivedTimeMs = timestampMs2(row.received_time, "received_time");
|
|
31892
|
+
if (receivedTimeMs < eventTimeMs) {
|
|
31893
|
+
throw new CryptoHftDataError("received_time_precedes_event_time");
|
|
31894
|
+
}
|
|
31895
|
+
return { ...row, eventTimeMs, receivedTimeMs };
|
|
31896
|
+
}
|
|
31897
|
+
function groupKey(row, profile) {
|
|
31898
|
+
const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
|
|
31899
|
+
return [
|
|
31900
|
+
row.event_type,
|
|
31901
|
+
row.eventTimeMs,
|
|
31902
|
+
sequence,
|
|
31903
|
+
row.dataset_object_identity
|
|
31904
|
+
].join("\x00");
|
|
31905
|
+
}
|
|
31906
|
+
function absent(value) {
|
|
31907
|
+
return value === null || value === undefined;
|
|
31908
|
+
}
|
|
31909
|
+
function snapshotSequence(row, profile) {
|
|
31910
|
+
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
31911
|
+
if (String(row.last_update_id) !== "-1") {
|
|
31912
|
+
throw new CryptoHftDataError("schema_last_update_id_snapshot_sentinel");
|
|
31913
|
+
}
|
|
31914
|
+
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id)) {
|
|
31915
|
+
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
31916
|
+
}
|
|
31917
|
+
return unsignedString(row.final_update_id, "final_update_id", true);
|
|
31918
|
+
}
|
|
31919
|
+
return unsignedString(row.last_update_id, "last_update_id", true);
|
|
31920
|
+
}
|
|
31921
|
+
function sortedSide(levels, side) {
|
|
31922
|
+
return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
|
|
31923
|
+
}
|
|
31924
|
+
function applyRows(state, rows) {
|
|
31925
|
+
for (const row of rows) {
|
|
31926
|
+
const levels = row.side === "bid" ? state.bids : state.asks;
|
|
31927
|
+
const quantity = decimal(row.quantity, "quantity", true);
|
|
31928
|
+
if (quantity === 0)
|
|
31929
|
+
levels.delete(row.price);
|
|
31930
|
+
else
|
|
31931
|
+
levels.set(row.price, quantity);
|
|
31932
|
+
}
|
|
31933
|
+
}
|
|
31934
|
+
function reconstructCryptoHftDataOrderBooks(request, inputRows, profile = CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE) {
|
|
31935
|
+
const rows = inputRows.map((row, index) => ({
|
|
31936
|
+
...validatedRow(request, row),
|
|
31937
|
+
originalIndex: index
|
|
31938
|
+
})).sort((left, right) => left.eventTimeMs - right.eventTimeMs || left.originalIndex - right.originalIndex);
|
|
31939
|
+
const groups = [];
|
|
31940
|
+
for (const row of rows) {
|
|
31941
|
+
const current = groups.at(-1);
|
|
31942
|
+
if (!current || groupKey(current[0], profile) !== groupKey(row, profile)) {
|
|
31943
|
+
groups.push([row]);
|
|
31944
|
+
} else {
|
|
31945
|
+
current.push(row);
|
|
31946
|
+
}
|
|
31947
|
+
}
|
|
31948
|
+
const earliestTargetTimeMs = Math.min(...request.requiredClockTargetsMs);
|
|
31949
|
+
const latestTargetTimeMs = Math.max(...request.requiredClockTargetsMs);
|
|
31950
|
+
const anchorIndex = groups.findIndex((group) => {
|
|
31951
|
+
const first = group[0];
|
|
31952
|
+
return first.event_type === "snapshot" && first.eventTimeMs <= earliestTargetTimeMs;
|
|
31953
|
+
});
|
|
31954
|
+
if (anchorIndex < 0) {
|
|
31955
|
+
throw new CryptoHftDataError("update_before_snapshot");
|
|
31956
|
+
}
|
|
31957
|
+
let state;
|
|
31958
|
+
let previousFinalUpdateId;
|
|
31959
|
+
const states = [];
|
|
31960
|
+
for (const group of groups.slice(anchorIndex)) {
|
|
31961
|
+
const first = group[0];
|
|
31962
|
+
if (first.eventTimeMs > latestTargetTimeMs)
|
|
31963
|
+
break;
|
|
31964
|
+
if (first.event_type === "snapshot") {
|
|
31965
|
+
const sequence = snapshotSequence(first, profile);
|
|
31966
|
+
for (const row of group) {
|
|
31967
|
+
if (snapshotSequence(row, profile) !== sequence) {
|
|
31968
|
+
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
31969
|
+
}
|
|
31970
|
+
}
|
|
31971
|
+
if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
|
|
31972
|
+
throw new CryptoHftDataError("snapshot_sequence_regression");
|
|
31973
|
+
}
|
|
31974
|
+
state = {
|
|
31975
|
+
bids: new Map,
|
|
31976
|
+
asks: new Map,
|
|
31977
|
+
sequence,
|
|
31978
|
+
sourceTimeMs: first.eventTimeMs,
|
|
31979
|
+
receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
|
|
31980
|
+
datasetObjectIdentity: first.dataset_object_identity,
|
|
31981
|
+
datasetObjectChecksum: first.dataset_object_checksum
|
|
31982
|
+
};
|
|
31983
|
+
applyRows(state, group);
|
|
31984
|
+
previousFinalUpdateId = BigInt(sequence);
|
|
31985
|
+
} else {
|
|
31986
|
+
if (!state || previousFinalUpdateId === undefined) {
|
|
31987
|
+
throw new CryptoHftDataError("update_before_snapshot");
|
|
31988
|
+
}
|
|
31989
|
+
const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
|
|
31990
|
+
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
31991
|
+
if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
|
|
31992
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
31993
|
+
}
|
|
31994
|
+
const previous = unsignedString(first.last_update_id, "last_update_id", true);
|
|
31995
|
+
for (const row of group) {
|
|
31996
|
+
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
|
|
31997
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
31998
|
+
}
|
|
31999
|
+
}
|
|
32000
|
+
if (BigInt(previous) !== previousFinalUpdateId) {
|
|
32001
|
+
throw new CryptoHftDataError("update_chain_gap");
|
|
32002
|
+
}
|
|
32003
|
+
} else {
|
|
32004
|
+
const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
|
|
32005
|
+
const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
|
|
32006
|
+
for (const row of group) {
|
|
32007
|
+
if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
|
|
32008
|
+
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32009
|
+
}
|
|
32010
|
+
}
|
|
32011
|
+
const expected = previousFinalUpdateId + 1n;
|
|
32012
|
+
if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
|
|
32013
|
+
throw new CryptoHftDataError("update_chain_gap");
|
|
32014
|
+
}
|
|
32015
|
+
}
|
|
32016
|
+
applyRows(state, group);
|
|
32017
|
+
state.sequence = finalUpdate.toString();
|
|
32018
|
+
state.sourceTimeMs = first.eventTimeMs;
|
|
32019
|
+
state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
|
|
32020
|
+
state.datasetObjectIdentity = first.dataset_object_identity;
|
|
32021
|
+
state.datasetObjectChecksum = first.dataset_object_checksum;
|
|
32022
|
+
previousFinalUpdateId = finalUpdate;
|
|
32023
|
+
}
|
|
32024
|
+
if (state) {
|
|
32025
|
+
states.push({
|
|
32026
|
+
...state,
|
|
32027
|
+
bids: new Map(state.bids),
|
|
32028
|
+
asks: new Map(state.asks)
|
|
32029
|
+
});
|
|
32030
|
+
}
|
|
32031
|
+
}
|
|
32032
|
+
const samples = [];
|
|
32033
|
+
for (const targetTimeMs of request.requiredClockTargetsMs) {
|
|
32034
|
+
let prior;
|
|
32035
|
+
for (const candidate of states) {
|
|
32036
|
+
if (candidate.sourceTimeMs > targetTimeMs)
|
|
32037
|
+
break;
|
|
32038
|
+
prior = candidate;
|
|
32039
|
+
}
|
|
32040
|
+
if (!prior || targetTimeMs - prior.sourceTimeMs > request.maxPriorAsOfLagMs) {
|
|
32041
|
+
throw new CryptoHftDataError("required_clock_coverage_insufficient");
|
|
32042
|
+
}
|
|
32043
|
+
const bids = sortedSide(prior.bids, "bid").slice(0, request.depth);
|
|
32044
|
+
const asks = sortedSide(prior.asks, "ask").slice(0, request.depth);
|
|
32045
|
+
if (bids.length === 0 || asks.length === 0) {
|
|
32046
|
+
throw new CryptoHftDataError("book_side_missing");
|
|
32047
|
+
}
|
|
32048
|
+
if (bids[0]?.[0] >= asks[0]?.[0]) {
|
|
32049
|
+
throw new CryptoHftDataError("book_crossed_or_locked");
|
|
32050
|
+
}
|
|
32051
|
+
samples.push({
|
|
32052
|
+
targetTimeMs,
|
|
32053
|
+
sourceTimeMs: prior.sourceTimeMs,
|
|
32054
|
+
receivedTimeMs: prior.receivedTimeMs,
|
|
32055
|
+
sequence: prior.sequence,
|
|
32056
|
+
bids,
|
|
32057
|
+
asks,
|
|
32058
|
+
datasetObjectIdentity: prior.datasetObjectIdentity,
|
|
32059
|
+
datasetObjectChecksum: prior.datasetObjectChecksum
|
|
32060
|
+
});
|
|
32061
|
+
}
|
|
32062
|
+
return samples;
|
|
32063
|
+
}
|
|
32064
|
+
function sha256Bytes(bytes) {
|
|
32065
|
+
return createHash4("sha256").update(bytes).digest("hex");
|
|
32485
32066
|
}
|
|
32486
|
-
function
|
|
32487
|
-
const
|
|
32067
|
+
async function decodeCryptoHftParquetZstd(bytes) {
|
|
32068
|
+
const parquet = decompress(bytes);
|
|
32069
|
+
const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
|
|
32070
|
+
return parquetReadObjects({ file: buffer });
|
|
32071
|
+
}
|
|
32072
|
+
function parsedDatasetRow(value, object2) {
|
|
32488
32073
|
return {
|
|
32489
|
-
|
|
32490
|
-
|
|
32074
|
+
received_time: value.received_time,
|
|
32075
|
+
event_time: value.event_time,
|
|
32076
|
+
transaction_time: value.transaction_time,
|
|
32077
|
+
symbol: value.symbol,
|
|
32078
|
+
event_type: value.event_type,
|
|
32079
|
+
first_update_id: value.first_update_id,
|
|
32080
|
+
final_update_id: value.final_update_id,
|
|
32081
|
+
prev_final_update_id: value.prev_final_update_id,
|
|
32082
|
+
last_update_id: value.last_update_id,
|
|
32083
|
+
side: value.side,
|
|
32084
|
+
price: value.price,
|
|
32085
|
+
quantity: value.quantity,
|
|
32086
|
+
order_count: value.order_count,
|
|
32087
|
+
dataset_object_identity: object2.identity,
|
|
32088
|
+
dataset_object_checksum: object2.checksum
|
|
32491
32089
|
};
|
|
32492
32090
|
}
|
|
32493
|
-
function
|
|
32494
|
-
|
|
32495
|
-
|
|
32496
|
-
|
|
32497
|
-
|
|
32498
|
-
|
|
32091
|
+
async function readBoundedObject(response, remainingBytes) {
|
|
32092
|
+
if (!response.body)
|
|
32093
|
+
return new Uint8Array;
|
|
32094
|
+
const reader = response.body.getReader();
|
|
32095
|
+
const chunks = [];
|
|
32096
|
+
let total = 0;
|
|
32097
|
+
while (true) {
|
|
32098
|
+
const { done, value } = await reader.read();
|
|
32099
|
+
if (done)
|
|
32100
|
+
break;
|
|
32101
|
+
total += value.byteLength;
|
|
32102
|
+
if (total > remainingBytes) {
|
|
32103
|
+
await reader.cancel();
|
|
32104
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32105
|
+
}
|
|
32106
|
+
chunks.push(value);
|
|
32107
|
+
}
|
|
32108
|
+
const bytes = new Uint8Array(total);
|
|
32109
|
+
let offset = 0;
|
|
32110
|
+
for (const chunk of chunks) {
|
|
32111
|
+
bytes.set(chunk, offset);
|
|
32112
|
+
offset += chunk.byteLength;
|
|
32113
|
+
}
|
|
32114
|
+
return bytes;
|
|
32115
|
+
}
|
|
32116
|
+
|
|
32117
|
+
class CryptoHftDataAdapter {
|
|
32118
|
+
baseUrl;
|
|
32119
|
+
request;
|
|
32120
|
+
nowMs;
|
|
32121
|
+
decode;
|
|
32122
|
+
profiles;
|
|
32123
|
+
constructor(options = {}) {
|
|
32124
|
+
this.baseUrl = options.baseUrl ?? CRYPTOHFTDATA_API_URL;
|
|
32125
|
+
this.request = options.fetch ?? fetch;
|
|
32126
|
+
this.nowMs = options.nowMs ?? Date.now;
|
|
32127
|
+
this.decode = options.decode ?? decodeCryptoHftParquetZstd;
|
|
32128
|
+
this.profiles = options.profiles ?? [];
|
|
32129
|
+
}
|
|
32130
|
+
capabilityFor(request) {
|
|
32131
|
+
return cryptoHftDataCapabilityFor(request, this.profiles);
|
|
32132
|
+
}
|
|
32133
|
+
async discoverSymbols(providerExchangeId) {
|
|
32134
|
+
if (!/^[a-z0-9_]+$/.test(providerExchangeId)) {
|
|
32135
|
+
throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
|
|
32136
|
+
}
|
|
32137
|
+
const endpoint = new URL("/symbols", this.baseUrl);
|
|
32138
|
+
endpoint.searchParams.set("exchange", providerExchangeId);
|
|
32139
|
+
endpoint.searchParams.set("data_type", "orderbook");
|
|
32140
|
+
let response;
|
|
32141
|
+
try {
|
|
32142
|
+
response = await this.request(endpoint);
|
|
32143
|
+
} catch {
|
|
32144
|
+
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
32145
|
+
}
|
|
32146
|
+
if (!response.ok) {
|
|
32147
|
+
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
32148
|
+
}
|
|
32149
|
+
let body;
|
|
32150
|
+
try {
|
|
32151
|
+
body = await response.json();
|
|
32152
|
+
} catch {
|
|
32153
|
+
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
32154
|
+
}
|
|
32155
|
+
const record2 = body;
|
|
32156
|
+
if (!body || typeof body !== "object" || Array.isArray(body) || String(record2.exchange).toLowerCase() !== providerExchangeId || String(record2.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record2.symbols) || !record2.symbols.every((symbol2) => typeof symbol2 === "string" && /^[A-Z0-9_-]+$/.test(symbol2))) {
|
|
32157
|
+
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
32158
|
+
}
|
|
32159
|
+
return [...new Set(record2.symbols)].sort();
|
|
32160
|
+
}
|
|
32161
|
+
async acquire(request, capability, credential) {
|
|
32162
|
+
const apiKey = credential && typeof credential === "object" ? credential.apiKey : undefined;
|
|
32163
|
+
if (typeof apiKey !== "string" || apiKey.length === 0) {
|
|
32164
|
+
throw new CryptoHftDataError("credentials_invalid");
|
|
32165
|
+
}
|
|
32166
|
+
const paths = enumerateCryptoHftDataObjects(request, capability.providerExchangeId, capability.resolvedSymbol);
|
|
32167
|
+
if (paths.length > request.budgets.maxFiles) {
|
|
32168
|
+
throw new CryptoHftDataError("budget_max_files_exceeded");
|
|
32169
|
+
}
|
|
32170
|
+
const started = this.nowMs();
|
|
32171
|
+
const tokenResponse = await this.request(`${this.baseUrl}/jwt-token`, {
|
|
32172
|
+
method: "POST",
|
|
32173
|
+
headers: { "content-type": "application/json", "X-API-Key": apiKey }
|
|
32174
|
+
});
|
|
32175
|
+
if (!tokenResponse.ok)
|
|
32176
|
+
throw new CryptoHftDataError("jwt_issuance_failed");
|
|
32177
|
+
const tokenBody = await tokenResponse.json();
|
|
32178
|
+
if (typeof tokenBody.jwt_token !== "string" || !tokenBody.jwt_token) {
|
|
32179
|
+
throw new CryptoHftDataError("jwt_response_invalid");
|
|
32180
|
+
}
|
|
32181
|
+
let totalBytes = 0;
|
|
32182
|
+
let totalRows = 0;
|
|
32183
|
+
const objects = [];
|
|
32184
|
+
const rows = [];
|
|
32185
|
+
for (const path2 of paths) {
|
|
32186
|
+
if (this.nowMs() - started > request.budgets.maxDurationMs) {
|
|
32187
|
+
throw new CryptoHftDataError("budget_max_duration_exceeded");
|
|
32188
|
+
}
|
|
32189
|
+
const endpoint = new URL("/download", this.baseUrl);
|
|
32190
|
+
endpoint.searchParams.set("file", path2);
|
|
32191
|
+
const response = await this.request(endpoint, {
|
|
32192
|
+
headers: { Authorization: `Bearer ${tokenBody.jwt_token}` }
|
|
32193
|
+
});
|
|
32194
|
+
if (!response.ok)
|
|
32195
|
+
throw new CryptoHftDataError("object_download_failed");
|
|
32196
|
+
const declaredBytes = Number(response.headers.get("content-length"));
|
|
32197
|
+
if (Number.isFinite(declaredBytes) && totalBytes + declaredBytes > request.budgets.maxBytes) {
|
|
32198
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32199
|
+
}
|
|
32200
|
+
const bytes = await readBoundedObject(response, request.budgets.maxBytes - totalBytes);
|
|
32201
|
+
totalBytes += bytes.byteLength;
|
|
32202
|
+
if (totalBytes > request.budgets.maxBytes) {
|
|
32203
|
+
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32204
|
+
}
|
|
32205
|
+
const decoded = await this.decode(bytes);
|
|
32206
|
+
totalRows += decoded.length;
|
|
32207
|
+
if (totalRows > request.budgets.maxRows) {
|
|
32208
|
+
throw new CryptoHftDataError("budget_max_rows_exceeded");
|
|
32209
|
+
}
|
|
32210
|
+
const object2 = {
|
|
32211
|
+
identity: path2,
|
|
32212
|
+
checksum: sha256Bytes(bytes),
|
|
32213
|
+
bytes: bytes.byteLength,
|
|
32214
|
+
rows: decoded.length
|
|
32215
|
+
};
|
|
32216
|
+
objects.push(object2);
|
|
32217
|
+
for (const decodedRow of decoded) {
|
|
32218
|
+
const parsed = parsedDatasetRow(decodedRow, object2);
|
|
32219
|
+
validatedRow(request, parsed);
|
|
32220
|
+
rows.push(parsed);
|
|
32221
|
+
}
|
|
32222
|
+
}
|
|
32223
|
+
if (this.nowMs() - started > request.budgets.maxDurationMs) {
|
|
32224
|
+
throw new CryptoHftDataError("budget_max_duration_exceeded");
|
|
32225
|
+
}
|
|
32226
|
+
return {
|
|
32227
|
+
objects,
|
|
32228
|
+
rows,
|
|
32229
|
+
vendorSemanticDigest: sha256Canonical(rows)
|
|
32230
|
+
};
|
|
32231
|
+
}
|
|
32232
|
+
async normalize(request, capability, dataset, captureBundleId) {
|
|
32233
|
+
const profile = this.profiles.find((candidate) => candidate.exchange === request.scope.exchange.trim().toLowerCase() && candidate.tradingPair === request.scope.tradingPair && candidate.sourceSymbol === request.scope.sourceSymbol && candidate.marketType === request.scope.marketType && candidate.providerExchangeId === capability.providerExchangeId);
|
|
32234
|
+
if (!profile) {
|
|
32235
|
+
throw new CryptoHftDataError("profile_semantics_unavailable");
|
|
32236
|
+
}
|
|
32237
|
+
const samples = reconstructCryptoHftDataOrderBooks(request, dataset.rows, profile);
|
|
32238
|
+
const context2 = {
|
|
32239
|
+
source: EXTERNAL_BACKFILL_SOURCE,
|
|
32240
|
+
deploymentId: "market-data-vendor-backfill",
|
|
32241
|
+
captureBundleId,
|
|
32242
|
+
exchange: request.scope.exchange,
|
|
32243
|
+
symbol: request.scope.tradingPair,
|
|
32244
|
+
tradingPair: request.scope.tradingPair,
|
|
32245
|
+
sourceSymbol: request.scope.sourceSymbol,
|
|
32246
|
+
assetType: request.scope.marketType,
|
|
32247
|
+
feed: "ORDERBOOK",
|
|
32248
|
+
provider: "cryptohftdata",
|
|
32249
|
+
sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
|
|
32250
|
+
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
32251
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
32252
|
+
provenanceComplete: true
|
|
32253
|
+
};
|
|
32254
|
+
const rows = samples.flatMap((sample) => {
|
|
32255
|
+
const rawCapture = {
|
|
32256
|
+
rawCaptureId: sha256Canonical({
|
|
32257
|
+
capture_bundle_id: captureBundleId,
|
|
32258
|
+
dataset_object_identity: sample.datasetObjectIdentity,
|
|
32259
|
+
dataset_object_checksum: sample.datasetObjectChecksum,
|
|
32260
|
+
source_time_ms: sample.sourceTimeMs,
|
|
32261
|
+
target_time_ms: sample.targetTimeMs
|
|
32262
|
+
}),
|
|
32263
|
+
rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
|
|
32264
|
+
rawChecksum: sample.datasetObjectChecksum,
|
|
32265
|
+
redactedPayload: {
|
|
32266
|
+
dataset_object_identity: sample.datasetObjectIdentity,
|
|
32267
|
+
dataset_object_checksum: sample.datasetObjectChecksum
|
|
32268
|
+
},
|
|
32269
|
+
eventTimeMs: sample.sourceTimeMs,
|
|
32270
|
+
receivedTimeMs: sample.receivedTimeMs,
|
|
32271
|
+
checksumAlgorithm: CHECKSUM_ALGORITHM
|
|
32272
|
+
};
|
|
32273
|
+
const canonical = buildCanonicalOrderBookRows({
|
|
32274
|
+
context: context2,
|
|
32275
|
+
rawCapture,
|
|
32276
|
+
depthLimit: request.depth,
|
|
32277
|
+
constructionMode: request.constructionMode,
|
|
32278
|
+
snapshot: {
|
|
32279
|
+
bids: sample.bids,
|
|
32280
|
+
asks: sample.asks,
|
|
32281
|
+
timestamp: sample.sourceTimeMs,
|
|
32282
|
+
receivedTimestamp: sample.receivedTimeMs,
|
|
32283
|
+
exchange: request.scope.exchange,
|
|
32284
|
+
symbol: request.scope.tradingPair,
|
|
32285
|
+
depthLimit: request.depth,
|
|
32286
|
+
sequence: sample.sequence
|
|
32287
|
+
}
|
|
32288
|
+
});
|
|
32289
|
+
return [...canonical.levels, canonical.summary];
|
|
32290
|
+
});
|
|
32291
|
+
return {
|
|
32292
|
+
captureBundleId,
|
|
32293
|
+
objects: dataset.objects,
|
|
32294
|
+
rows,
|
|
32295
|
+
vendorSemanticDigest: dataset.vendorSemanticDigest,
|
|
32296
|
+
canonicalSemanticDigest: semanticDigest(rows)
|
|
32297
|
+
};
|
|
32499
32298
|
}
|
|
32500
|
-
return objects;
|
|
32501
32299
|
}
|
|
32502
|
-
|
|
32503
|
-
|
|
32504
|
-
|
|
32505
|
-
|
|
32506
|
-
|
|
32507
|
-
|
|
32508
|
-
|
|
32509
|
-
|
|
32510
|
-
|
|
32511
|
-
|
|
32512
|
-
|
|
32513
|
-
|
|
32514
|
-
|
|
32515
|
-
|
|
32516
|
-
|
|
32300
|
+
|
|
32301
|
+
// src/helpers/market-data-vendor-backfill/qualification.ts
|
|
32302
|
+
var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
32303
|
+
var SHA256 = /^[0-9a-f]{64}$/;
|
|
32304
|
+
var REASON = /^[a-z][a-z0-9_]{0,127}$/;
|
|
32305
|
+
var EVENT_FIELDS = new Set([
|
|
32306
|
+
"qualification_event_id",
|
|
32307
|
+
"capture_bundle_id",
|
|
32308
|
+
"state",
|
|
32309
|
+
"receipt_id",
|
|
32310
|
+
"promotion_identity_sha256",
|
|
32311
|
+
"window",
|
|
32312
|
+
"event_at",
|
|
32313
|
+
"reason_code"
|
|
32314
|
+
]);
|
|
32315
|
+
function deterministicUuid(value) {
|
|
32316
|
+
const digest = jcsSha256(value).slice(0, 32).split("");
|
|
32317
|
+
digest[12] = "5";
|
|
32318
|
+
digest[16] = "8";
|
|
32319
|
+
const hex3 = digest.join("");
|
|
32320
|
+
return `${hex3.slice(0, 8)}-${hex3.slice(8, 12)}-${hex3.slice(12, 16)}-${hex3.slice(16, 20)}-${hex3.slice(20)}`;
|
|
32517
32321
|
}
|
|
32518
|
-
function
|
|
32519
|
-
const
|
|
32520
|
-
|
|
32521
|
-
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
32522
|
-
}
|
|
32523
|
-
const raw = BigInt(rendered);
|
|
32524
|
-
const milliseconds = field === "received_time" ? raw / 1000000n : raw;
|
|
32525
|
-
if (milliseconds > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
32526
|
-
throw new CryptoHftDataError(`schema_${field}_unsafe`);
|
|
32527
|
-
}
|
|
32528
|
-
return Number(milliseconds);
|
|
32322
|
+
function semanticEvent(event) {
|
|
32323
|
+
const { qualification_event_id: _eventId, ...semantic } = event;
|
|
32324
|
+
return semantic;
|
|
32529
32325
|
}
|
|
32530
|
-
function
|
|
32531
|
-
|
|
32532
|
-
|
|
32533
|
-
|
|
32534
|
-
const parsed = Number(value);
|
|
32535
|
-
if (!Number.isFinite(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) {
|
|
32536
|
-
throw new CryptoHftDataError(`schema_${field}_invalid`);
|
|
32326
|
+
function assertTimestamp(value, field) {
|
|
32327
|
+
const parsed = Date.parse(value);
|
|
32328
|
+
if (!Number.isSafeInteger(parsed) || new Date(parsed).toISOString() !== value) {
|
|
32329
|
+
throw new Error(`${field} must be a fixed UTC RFC3339 timestamp`);
|
|
32537
32330
|
}
|
|
32538
32331
|
return parsed;
|
|
32539
32332
|
}
|
|
32540
|
-
function
|
|
32541
|
-
if (
|
|
32542
|
-
throw new
|
|
32543
|
-
}
|
|
32544
|
-
if (row.event_type !== "snapshot" && row.event_type !== "update") {
|
|
32545
|
-
throw new CryptoHftDataError("schema_event_type_invalid");
|
|
32333
|
+
function parseQualificationEvent(value) {
|
|
32334
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
32335
|
+
throw new Error("qualification event must be an object");
|
|
32546
32336
|
}
|
|
32547
|
-
|
|
32548
|
-
|
|
32337
|
+
const event = value;
|
|
32338
|
+
if (Object.keys(event).length !== EVENT_FIELDS.size || !Object.keys(event).every((field) => EVENT_FIELDS.has(field)) || !UUID.test(event.qualification_event_id) || !SHA256.test(event.capture_bundle_id) || !["qualified", "quarantined", "revoked"].includes(event.state) || !SHA256.test(event.receipt_id) || !SHA256.test(event.promotion_identity_sha256) || !event.window || typeof event.window !== "object" || Object.keys(event.window).length !== 2 || !REASON.test(event.reason_code)) {
|
|
32339
|
+
throw new Error("qualification event fields are invalid");
|
|
32549
32340
|
}
|
|
32550
|
-
|
|
32551
|
-
|
|
32552
|
-
if (!/^[a-f0-9]{64}$/.test(row.dataset_object_checksum)) {
|
|
32553
|
-
throw new CryptoHftDataError("schema_object_checksum_invalid");
|
|
32341
|
+
if (assertTimestamp(event.window.end_at, "window.end_at") <= assertTimestamp(event.window.start_at, "window.start_at")) {
|
|
32342
|
+
throw new Error("qualification window must be increasing");
|
|
32554
32343
|
}
|
|
32555
|
-
|
|
32556
|
-
|
|
32557
|
-
|
|
32558
|
-
throw new CryptoHftDataError("received_time_precedes_event_time");
|
|
32344
|
+
assertTimestamp(event.event_at, "event_at");
|
|
32345
|
+
if (deterministicUuid(semanticEvent(event)) !== event.qualification_event_id) {
|
|
32346
|
+
throw new Error("qualification_event_id does not match event content");
|
|
32559
32347
|
}
|
|
32560
|
-
return
|
|
32561
|
-
}
|
|
32562
|
-
function groupKey(row, profile) {
|
|
32563
|
-
const sequence = row.event_type === "snapshot" ? profile.snapshotGrouping === "event_time_final_update_id_object" ? unsignedString(row.final_update_id, "final_update_id", true) : unsignedString(row.last_update_id, "last_update_id", true) : unsignedString(row.final_update_id, "final_update_id", true);
|
|
32564
|
-
return [
|
|
32565
|
-
row.event_type,
|
|
32566
|
-
row.eventTimeMs,
|
|
32567
|
-
sequence,
|
|
32568
|
-
row.dataset_object_identity
|
|
32569
|
-
].join("\x00");
|
|
32348
|
+
return event;
|
|
32570
32349
|
}
|
|
32571
|
-
function
|
|
32572
|
-
|
|
32350
|
+
function finalizeQualificationEvent(event) {
|
|
32351
|
+
const semantic = semanticEvent(event);
|
|
32352
|
+
return parseQualificationEvent({
|
|
32353
|
+
...semantic,
|
|
32354
|
+
qualification_event_id: deterministicUuid(semantic)
|
|
32355
|
+
});
|
|
32573
32356
|
}
|
|
32574
|
-
function
|
|
32575
|
-
|
|
32576
|
-
|
|
32577
|
-
|
|
32578
|
-
|
|
32579
|
-
|
|
32580
|
-
|
|
32357
|
+
function qualificationEventToArchiveRow(eventInput) {
|
|
32358
|
+
const event = parseQualificationEvent(eventInput);
|
|
32359
|
+
return {
|
|
32360
|
+
table: "market_data.cex_order_book_capture_qualifications",
|
|
32361
|
+
row: {
|
|
32362
|
+
source: "external_backfill",
|
|
32363
|
+
capture_origin: "vendor_historical_backfill",
|
|
32364
|
+
source_mode: "vendor_historical_backfill_v1",
|
|
32365
|
+
deployment_id: "market-data-vendor-backfill",
|
|
32366
|
+
qualification_event_id: event.qualification_event_id,
|
|
32367
|
+
capture_bundle_id: event.capture_bundle_id,
|
|
32368
|
+
state: event.state,
|
|
32369
|
+
receipt_id: event.receipt_id,
|
|
32370
|
+
promotion_identity_sha256: event.promotion_identity_sha256,
|
|
32371
|
+
window_start_ms: Date.parse(event.window.start_at),
|
|
32372
|
+
window_end_ms: Date.parse(event.window.end_at),
|
|
32373
|
+
event_at_ms: Date.parse(event.event_at),
|
|
32374
|
+
reason_code: event.reason_code,
|
|
32375
|
+
event_json: jcsCanonicalize(event)
|
|
32581
32376
|
}
|
|
32582
|
-
|
|
32583
|
-
}
|
|
32584
|
-
return unsignedString(row.last_update_id, "last_update_id", true);
|
|
32585
|
-
}
|
|
32586
|
-
function sortedSide(levels, side) {
|
|
32587
|
-
return [...levels.entries()].map(([price, quantity]) => [Number(price), quantity]).sort((left, right) => side === "bid" ? right[0] - left[0] : left[0] - right[0]);
|
|
32588
|
-
}
|
|
32589
|
-
function applyRows(state, rows) {
|
|
32590
|
-
for (const row of rows) {
|
|
32591
|
-
const levels = row.side === "bid" ? state.bids : state.asks;
|
|
32592
|
-
const quantity = decimal(row.quantity, "quantity", true);
|
|
32593
|
-
if (quantity === 0)
|
|
32594
|
-
levels.delete(row.price);
|
|
32595
|
-
else
|
|
32596
|
-
levels.set(row.price, quantity);
|
|
32597
|
-
}
|
|
32377
|
+
};
|
|
32598
32378
|
}
|
|
32599
|
-
|
|
32600
|
-
|
|
32601
|
-
|
|
32602
|
-
|
|
32603
|
-
|
|
32604
|
-
|
|
32605
|
-
|
|
32606
|
-
|
|
32607
|
-
|
|
32608
|
-
|
|
32609
|
-
}
|
|
32610
|
-
|
|
32611
|
-
|
|
32612
|
-
|
|
32613
|
-
|
|
32614
|
-
|
|
32615
|
-
|
|
32616
|
-
|
|
32617
|
-
|
|
32618
|
-
|
|
32619
|
-
|
|
32620
|
-
|
|
32621
|
-
|
|
32622
|
-
|
|
32623
|
-
|
|
32624
|
-
|
|
32625
|
-
|
|
32626
|
-
|
|
32627
|
-
|
|
32628
|
-
|
|
32629
|
-
|
|
32630
|
-
|
|
32631
|
-
|
|
32632
|
-
|
|
32633
|
-
throw new CryptoHftDataError("ambiguous_snapshot_group");
|
|
32634
|
-
}
|
|
32635
|
-
}
|
|
32636
|
-
if (profile.sequenceSemantics === "binance_u_U_pu" && previousFinalUpdateId !== undefined && BigInt(sequence) < previousFinalUpdateId) {
|
|
32637
|
-
throw new CryptoHftDataError("snapshot_sequence_regression");
|
|
32638
|
-
}
|
|
32639
|
-
state = {
|
|
32640
|
-
bids: new Map,
|
|
32641
|
-
asks: new Map,
|
|
32642
|
-
sequence,
|
|
32643
|
-
sourceTimeMs: first.eventTimeMs,
|
|
32644
|
-
receivedTimeMs: Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs)),
|
|
32645
|
-
datasetObjectIdentity: first.dataset_object_identity,
|
|
32646
|
-
datasetObjectChecksum: first.dataset_object_checksum
|
|
32647
|
-
};
|
|
32648
|
-
applyRows(state, group);
|
|
32649
|
-
previousFinalUpdateId = BigInt(sequence);
|
|
32650
|
-
} else {
|
|
32651
|
-
if (!state || previousFinalUpdateId === undefined) {
|
|
32652
|
-
throw new CryptoHftDataError("update_before_snapshot");
|
|
32653
|
-
}
|
|
32654
|
-
const finalUpdate = BigInt(unsignedString(first.final_update_id, "final_update_id", true));
|
|
32655
|
-
if (profile.sequenceSemantics === "okx_seq_id_prev_seq_id") {
|
|
32656
|
-
if (!absent(first.first_update_id) || !absent(first.prev_final_update_id)) {
|
|
32657
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32658
|
-
}
|
|
32659
|
-
const previous = unsignedString(first.last_update_id, "last_update_id", true);
|
|
32660
|
-
for (const row of group) {
|
|
32661
|
-
if (!absent(row.first_update_id) || !absent(row.prev_final_update_id) || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.last_update_id, "last_update_id", true) !== previous) {
|
|
32662
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32663
|
-
}
|
|
32664
|
-
}
|
|
32665
|
-
if (BigInt(previous) !== previousFinalUpdateId) {
|
|
32666
|
-
throw new CryptoHftDataError("update_chain_gap");
|
|
32667
|
-
}
|
|
32668
|
-
} else {
|
|
32669
|
-
const firstUpdate = BigInt(unsignedString(first.first_update_id, "first_update_id", true));
|
|
32670
|
-
const previous = unsignedString(first.prev_final_update_id, "prev_final_update_id");
|
|
32671
|
-
for (const row of group) {
|
|
32672
|
-
if (unsignedString(row.first_update_id, "first_update_id", true) !== firstUpdate.toString() || unsignedString(row.final_update_id, "final_update_id", true) !== finalUpdate.toString() || unsignedString(row.prev_final_update_id, "prev_final_update_id") !== previous) {
|
|
32673
|
-
throw new CryptoHftDataError("ambiguous_update_group");
|
|
32674
|
-
}
|
|
32675
|
-
}
|
|
32676
|
-
const expected = previousFinalUpdateId + 1n;
|
|
32677
|
-
if (firstUpdate > expected || finalUpdate < expected || previous !== undefined && BigInt(previous) !== previousFinalUpdateId) {
|
|
32678
|
-
throw new CryptoHftDataError("update_chain_gap");
|
|
32679
|
-
}
|
|
32680
|
-
}
|
|
32681
|
-
applyRows(state, group);
|
|
32682
|
-
state.sequence = finalUpdate.toString();
|
|
32683
|
-
state.sourceTimeMs = first.eventTimeMs;
|
|
32684
|
-
state.receivedTimeMs = Math.max(...group.map(({ receivedTimeMs }) => receivedTimeMs));
|
|
32685
|
-
state.datasetObjectIdentity = first.dataset_object_identity;
|
|
32686
|
-
state.datasetObjectChecksum = first.dataset_object_checksum;
|
|
32687
|
-
previousFinalUpdateId = finalUpdate;
|
|
32688
|
-
}
|
|
32689
|
-
if (state) {
|
|
32690
|
-
states.push({
|
|
32691
|
-
...state,
|
|
32692
|
-
bids: new Map(state.bids),
|
|
32693
|
-
asks: new Map(state.asks)
|
|
32694
|
-
});
|
|
32695
|
-
}
|
|
32379
|
+
|
|
32380
|
+
// src/helpers/market-data-vendor-backfill/core.ts
|
|
32381
|
+
function outcome(request, status, reasonCode, extra = {}) {
|
|
32382
|
+
return {
|
|
32383
|
+
status,
|
|
32384
|
+
reasonCode,
|
|
32385
|
+
...request ? {
|
|
32386
|
+
requestId: request.requestId,
|
|
32387
|
+
idempotencyKey: request.idempotencyKey,
|
|
32388
|
+
...request.target ? { target: request.target } : {}
|
|
32389
|
+
} : {},
|
|
32390
|
+
...extra
|
|
32391
|
+
};
|
|
32392
|
+
}
|
|
32393
|
+
function captureBundleId(request, capability, dataset) {
|
|
32394
|
+
return sha256Canonical({
|
|
32395
|
+
request_business_identity: request.idempotencyKey,
|
|
32396
|
+
provider: capability.provider,
|
|
32397
|
+
provider_exchange_id: capability.providerExchangeId,
|
|
32398
|
+
resolved_symbol: capability.resolvedSymbol,
|
|
32399
|
+
adapter_version: capability.adapterVersion,
|
|
32400
|
+
objects: dataset.objects.map(({ identity, checksum, bytes, rows }) => ({
|
|
32401
|
+
identity,
|
|
32402
|
+
checksum,
|
|
32403
|
+
bytes,
|
|
32404
|
+
rows
|
|
32405
|
+
})),
|
|
32406
|
+
canonical_schema_version: request.expectedProduct.canonicalSchemaVersion,
|
|
32407
|
+
checksum_algorithm: request.expectedProduct.checksumAlgorithm
|
|
32408
|
+
});
|
|
32409
|
+
}
|
|
32410
|
+
function buildPromotionReceipt(request, capability, normalized, verification, verificationTimeMs) {
|
|
32411
|
+
if (!request.wire || !request.initialSelection || !request.expectedCanonicalSchema || !request.coveragePolicy || !request.productPins) {
|
|
32412
|
+
throw new Error("decoded final-v1 request context is missing");
|
|
32696
32413
|
}
|
|
32697
|
-
|
|
32698
|
-
|
|
32699
|
-
|
|
32700
|
-
|
|
32701
|
-
|
|
32414
|
+
return finalizePromotionReceipt({
|
|
32415
|
+
schema_id: PROMOTION_RECEIPT_SCHEMA_ID,
|
|
32416
|
+
verified_at: new Date(verificationTimeMs).toISOString(),
|
|
32417
|
+
request_id: request.requestId,
|
|
32418
|
+
idempotency_key: request.idempotencyKey,
|
|
32419
|
+
source: EXTERNAL_BACKFILL_SOURCE,
|
|
32420
|
+
capture_origin: "vendor_historical_backfill",
|
|
32421
|
+
source_mode: "vendor_historical_backfill_v1",
|
|
32422
|
+
provider: capability.provider,
|
|
32423
|
+
adapter_version: capability.adapterVersion,
|
|
32424
|
+
effective_policies: {
|
|
32425
|
+
capability_policy: request.productPins.capability_policy,
|
|
32426
|
+
resource_policy: request.productPins.resource_policy,
|
|
32427
|
+
adapter_policy: EFFECTIVE_ADAPTER_POLICY_PIN,
|
|
32428
|
+
acquisition_policy: EFFECTIVE_ACQUISITION_POLICY_PIN
|
|
32429
|
+
},
|
|
32430
|
+
capture_bundle_id: normalized.captureBundleId,
|
|
32431
|
+
scope: request.wire.scope,
|
|
32432
|
+
window: request.wire.window,
|
|
32433
|
+
depth: request.depth,
|
|
32434
|
+
construction_mode: request.constructionMode,
|
|
32435
|
+
canonical_schema: request.expectedCanonicalSchema,
|
|
32436
|
+
coverage_policy: request.coveragePolicy,
|
|
32437
|
+
selection_sha256: request.initialSelection.selection_sha256,
|
|
32438
|
+
vendor_semantic_digest: normalized.vendorSemanticDigest,
|
|
32439
|
+
canonical_semantic_digest: verification.canonicalSemanticDigest,
|
|
32440
|
+
prefix_digest: verification.prefixDigest,
|
|
32441
|
+
suffix_digest: verification.suffixDigest,
|
|
32442
|
+
seam_verified: true,
|
|
32443
|
+
coverage_verified: true,
|
|
32444
|
+
dataset_objects: normalized.objects
|
|
32445
|
+
});
|
|
32446
|
+
}
|
|
32447
|
+
async function submitAll(dependencies, batches) {
|
|
32448
|
+
const maxAttempts = Math.max(1, Math.min(10, dependencies.retry?.maxAttempts ?? 3));
|
|
32449
|
+
for (const batch of batches) {
|
|
32450
|
+
let accepted = false;
|
|
32451
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt += 1) {
|
|
32452
|
+
try {
|
|
32453
|
+
const response = await dependencies.forwarder.submit(batch);
|
|
32454
|
+
accepted = response.ok && response.inserted === batch.rows.length;
|
|
32455
|
+
} catch {
|
|
32456
|
+
accepted = false;
|
|
32457
|
+
}
|
|
32458
|
+
if (accepted)
|
|
32702
32459
|
break;
|
|
32703
|
-
|
|
32460
|
+
if (attempt < maxAttempts)
|
|
32461
|
+
await dependencies.retry?.wait?.(attempt);
|
|
32704
32462
|
}
|
|
32705
|
-
if (!
|
|
32706
|
-
|
|
32463
|
+
if (!accepted)
|
|
32464
|
+
return false;
|
|
32465
|
+
}
|
|
32466
|
+
return true;
|
|
32467
|
+
}
|
|
32468
|
+
function stableFailureReason(error48, fallback) {
|
|
32469
|
+
if (error48 && typeof error48 === "object") {
|
|
32470
|
+
const reason = error48.reason;
|
|
32471
|
+
if (typeof reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(reason)) {
|
|
32472
|
+
return reason;
|
|
32707
32473
|
}
|
|
32708
|
-
|
|
32709
|
-
|
|
32710
|
-
|
|
32711
|
-
|
|
32474
|
+
}
|
|
32475
|
+
return fallback;
|
|
32476
|
+
}
|
|
32477
|
+
function assertArchivePreflight(request, resolution) {
|
|
32478
|
+
if (!request.target || !request.productionAuthorizationId) {
|
|
32479
|
+
throw new Error("request target or production authorization ID is missing");
|
|
32480
|
+
}
|
|
32481
|
+
const selection = archiveSelectionCodec.decode(resolution.selection);
|
|
32482
|
+
const receipts = resolution.receipts.map((receipt) => promotionReceiptCodec.decode(receipt));
|
|
32483
|
+
const receiptById = new Map;
|
|
32484
|
+
for (const receipt of receipts) {
|
|
32485
|
+
const existing = receiptById.get(receipt.receipt_id);
|
|
32486
|
+
if (existing && jcsCanonicalize(existing) !== jcsCanonicalize(receipt)) {
|
|
32487
|
+
throw new Error("stored receipt identity has conflicting content");
|
|
32712
32488
|
}
|
|
32713
|
-
|
|
32714
|
-
|
|
32489
|
+
receiptById.set(receipt.receipt_id, receipt);
|
|
32490
|
+
}
|
|
32491
|
+
for (const bundle of selection.bundles) {
|
|
32492
|
+
if (bundle.capture_origin === "vendor_historical_backfill" && (!bundle.qualification || !receiptById.has(bundle.qualification.receipt_id))) {
|
|
32493
|
+
throw new Error("vendor selection lacks its validated stored receipt");
|
|
32715
32494
|
}
|
|
32716
|
-
samples.push({
|
|
32717
|
-
targetTimeMs,
|
|
32718
|
-
sourceTimeMs: prior.sourceTimeMs,
|
|
32719
|
-
receivedTimeMs: prior.receivedTimeMs,
|
|
32720
|
-
sequence: prior.sequence,
|
|
32721
|
-
bids,
|
|
32722
|
-
asks,
|
|
32723
|
-
datasetObjectIdentity: prior.datasetObjectIdentity,
|
|
32724
|
-
datasetObjectChecksum: prior.datasetObjectChecksum
|
|
32725
|
-
});
|
|
32726
32495
|
}
|
|
32727
|
-
|
|
32496
|
+
if (resolution.readerIdentity.environment !== request.target.environment || resolution.readerIdentity.cluster !== request.target.cluster) {
|
|
32497
|
+
throw new Error("archive reader cluster identity mismatch");
|
|
32498
|
+
}
|
|
32728
32499
|
}
|
|
32729
|
-
function
|
|
32730
|
-
|
|
32500
|
+
function assertForwarderPreflight(request, resolution, nowMs) {
|
|
32501
|
+
if (!request.target || !request.productionAuthorizationId) {
|
|
32502
|
+
throw new Error("request target or production authorization ID is missing");
|
|
32503
|
+
}
|
|
32504
|
+
if (resolution.forwarderIdentity.environment !== request.target.environment || resolution.forwarderIdentity.cluster !== request.target.cluster) {
|
|
32505
|
+
throw new Error("archive forwarder cluster identity mismatch");
|
|
32506
|
+
}
|
|
32507
|
+
const authorization = resolution.authorization;
|
|
32508
|
+
const expiresAtMs = Date.parse(authorization.expiresAt);
|
|
32509
|
+
if (authorization.authorizationId !== request.productionAuthorizationId || authorization.scope !== "production" || authorization.environment !== request.target.environment || authorization.cluster !== request.target.cluster || authorization.credentialValidated !== true || !Number.isSafeInteger(expiresAtMs) || new Date(expiresAtMs).toISOString() !== authorization.expiresAt || expiresAtMs <= nowMs) {
|
|
32510
|
+
throw new Error("production forwarder authorization is invalid");
|
|
32511
|
+
}
|
|
32731
32512
|
}
|
|
32732
|
-
|
|
32733
|
-
|
|
32734
|
-
const buffer = parquet.buffer.slice(parquet.byteOffset, parquet.byteOffset + parquet.byteLength);
|
|
32735
|
-
return parquetReadObjects({ file: buffer });
|
|
32513
|
+
function resourcePolicyScopeExceeded(request) {
|
|
32514
|
+
return request.depth > RESOURCE_POLICY.request_bounds.max_depth || request.window.endTimeMs - request.window.startTimeMs > RESOURCE_POLICY.request_bounds.max_window_ms || request.requiredClockTargetsMs.length > RESOURCE_POLICY.request_bounds.max_required_events;
|
|
32736
32515
|
}
|
|
32737
|
-
function
|
|
32738
|
-
|
|
32739
|
-
|
|
32740
|
-
event_time: value.event_time,
|
|
32741
|
-
transaction_time: value.transaction_time,
|
|
32742
|
-
symbol: value.symbol,
|
|
32743
|
-
event_type: value.event_type,
|
|
32744
|
-
first_update_id: value.first_update_id,
|
|
32745
|
-
final_update_id: value.final_update_id,
|
|
32746
|
-
prev_final_update_id: value.prev_final_update_id,
|
|
32747
|
-
last_update_id: value.last_update_id,
|
|
32748
|
-
side: value.side,
|
|
32749
|
-
price: value.price,
|
|
32750
|
-
quantity: value.quantity,
|
|
32751
|
-
order_count: value.order_count,
|
|
32752
|
-
dataset_object_identity: object2.identity,
|
|
32753
|
-
dataset_object_checksum: object2.checksum
|
|
32754
|
-
};
|
|
32516
|
+
function storedReceiptForSelection(resolution) {
|
|
32517
|
+
const receiptId = resolution.selection.receipt_ids[0];
|
|
32518
|
+
return receiptId ? resolution.receipts.find((receipt) => receipt.receipt_id === receiptId) : undefined;
|
|
32755
32519
|
}
|
|
32756
|
-
async function
|
|
32757
|
-
|
|
32758
|
-
|
|
32759
|
-
|
|
32760
|
-
|
|
32761
|
-
|
|
32762
|
-
|
|
32763
|
-
|
|
32764
|
-
|
|
32765
|
-
|
|
32766
|
-
total += value.byteLength;
|
|
32767
|
-
if (total > remainingBytes) {
|
|
32768
|
-
await reader.cancel();
|
|
32769
|
-
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32770
|
-
}
|
|
32771
|
-
chunks.push(value);
|
|
32520
|
+
async function runMarketDataVendorBackfill(input, dependencies) {
|
|
32521
|
+
let request;
|
|
32522
|
+
try {
|
|
32523
|
+
const documents = input;
|
|
32524
|
+
request = decodeBackfillRunDocuments({
|
|
32525
|
+
request: documents.request,
|
|
32526
|
+
requiredClock: documents.requiredClock
|
|
32527
|
+
});
|
|
32528
|
+
} catch {
|
|
32529
|
+
return outcome(undefined, "request_invalid", "request_invalid");
|
|
32772
32530
|
}
|
|
32773
|
-
|
|
32774
|
-
|
|
32775
|
-
|
|
32776
|
-
|
|
32777
|
-
|
|
32531
|
+
let initialResolution;
|
|
32532
|
+
try {
|
|
32533
|
+
initialResolution = await dependencies.archive.resolveSelection(request);
|
|
32534
|
+
assertArchivePreflight(request, initialResolution);
|
|
32535
|
+
const forwarderPreflight = await dependencies.forwarder.preflight({
|
|
32536
|
+
authorizationId: request.productionAuthorizationId,
|
|
32537
|
+
target: request.target
|
|
32538
|
+
});
|
|
32539
|
+
assertForwarderPreflight(request, forwarderPreflight, dependencies.clock.nowMs());
|
|
32540
|
+
} catch {
|
|
32541
|
+
return outcome(request, "archive_preflight_failed", "archive_preflight_failed");
|
|
32778
32542
|
}
|
|
32779
|
-
|
|
32780
|
-
|
|
32781
|
-
|
|
32782
|
-
|
|
32783
|
-
|
|
32784
|
-
|
|
32785
|
-
|
|
32786
|
-
|
|
32787
|
-
|
|
32788
|
-
|
|
32789
|
-
|
|
32790
|
-
|
|
32791
|
-
|
|
32792
|
-
|
|
32793
|
-
|
|
32543
|
+
if (initialResolution.selection.coverage_class === "complete") {
|
|
32544
|
+
const receipt2 = storedReceiptForSelection(initialResolution);
|
|
32545
|
+
return outcome(request, "already_covered", "qualified_coverage_complete", {
|
|
32546
|
+
selection: initialResolution.selection,
|
|
32547
|
+
...receipt2 ? { receipt: receipt2 } : {}
|
|
32548
|
+
});
|
|
32549
|
+
}
|
|
32550
|
+
if (resourcePolicyScopeExceeded(request)) {
|
|
32551
|
+
return outcome(request, "capability_unsupported", "capability_unsupported", { reasonSubcode: "resource_policy_scope_exceeded" });
|
|
32552
|
+
}
|
|
32553
|
+
if (request.sourcePolicy === "fill_gaps" && (request.productPins?.capability_policy.policy_id !== CAPABILITY_POLICY.policy_id || request.productPins.capability_policy.policy_sha256 !== CAPABILITY_POLICY.policy_sha256)) {
|
|
32554
|
+
return outcome(request, "capability_unsupported", "scope_unsupported", {
|
|
32555
|
+
reasonSubcode: "fill_gaps_requires_capability_policy_v2"
|
|
32556
|
+
});
|
|
32557
|
+
}
|
|
32558
|
+
let capability;
|
|
32559
|
+
try {
|
|
32560
|
+
capability = dependencies.providers.capabilityFor(request);
|
|
32561
|
+
} catch {
|
|
32562
|
+
return outcome(request, "capability_unsupported", "capability_probe_failed");
|
|
32563
|
+
}
|
|
32564
|
+
if (!capability) {
|
|
32565
|
+
return outcome(request, "capability_unsupported", "scope_unsupported");
|
|
32566
|
+
}
|
|
32567
|
+
if (!request.providerPolicy.allowedAdapterVersions.includes(capability.adapterVersion)) {
|
|
32568
|
+
return outcome(request, "capability_unsupported", "adapter_version_unpinned");
|
|
32794
32569
|
}
|
|
32795
|
-
|
|
32796
|
-
|
|
32570
|
+
let credential;
|
|
32571
|
+
try {
|
|
32572
|
+
credential = await dependencies.credentials.resolve(capability.provider);
|
|
32573
|
+
} catch {
|
|
32574
|
+
return outcome(request, "credentials_missing", "credential_resolution_failed");
|
|
32797
32575
|
}
|
|
32798
|
-
|
|
32799
|
-
|
|
32800
|
-
throw new CryptoHftDataError("symbol_discovery_exchange_invalid");
|
|
32801
|
-
}
|
|
32802
|
-
const endpoint = new URL("/symbols", this.baseUrl);
|
|
32803
|
-
endpoint.searchParams.set("exchange", providerExchangeId);
|
|
32804
|
-
endpoint.searchParams.set("data_type", "orderbook");
|
|
32805
|
-
let response;
|
|
32806
|
-
try {
|
|
32807
|
-
response = await this.request(endpoint);
|
|
32808
|
-
} catch {
|
|
32809
|
-
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
32810
|
-
}
|
|
32811
|
-
if (!response.ok) {
|
|
32812
|
-
throw new CryptoHftDataError("symbol_discovery_failed");
|
|
32813
|
-
}
|
|
32814
|
-
let body;
|
|
32815
|
-
try {
|
|
32816
|
-
body = await response.json();
|
|
32817
|
-
} catch {
|
|
32818
|
-
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
32819
|
-
}
|
|
32820
|
-
const record2 = body;
|
|
32821
|
-
if (!body || typeof body !== "object" || Array.isArray(body) || String(record2.exchange).toLowerCase() !== providerExchangeId || String(record2.data_type).toLowerCase() !== "orderbook" || !Array.isArray(record2.symbols) || !record2.symbols.every((symbol2) => typeof symbol2 === "string" && /^[A-Z0-9_-]+$/.test(symbol2))) {
|
|
32822
|
-
throw new CryptoHftDataError("symbol_discovery_response_invalid");
|
|
32823
|
-
}
|
|
32824
|
-
return [...new Set(record2.symbols)].sort();
|
|
32576
|
+
if (credential === undefined || credential === null) {
|
|
32577
|
+
return outcome(request, "credentials_missing", "provider_credentials_missing");
|
|
32825
32578
|
}
|
|
32826
|
-
|
|
32827
|
-
|
|
32828
|
-
|
|
32829
|
-
|
|
32830
|
-
|
|
32831
|
-
const
|
|
32832
|
-
|
|
32833
|
-
|
|
32579
|
+
let dataset;
|
|
32580
|
+
let normalized;
|
|
32581
|
+
try {
|
|
32582
|
+
const acquisitionRequest = providerAcquisitionRequest(request);
|
|
32583
|
+
dataset = await dependencies.providers.acquire(acquisitionRequest, capability, credential);
|
|
32584
|
+
const bundleId = captureBundleId(request, capability, dataset);
|
|
32585
|
+
normalized = await dependencies.providers.normalize(acquisitionRequest, capability, dataset, bundleId);
|
|
32586
|
+
if (normalized.captureBundleId !== bundleId) {
|
|
32587
|
+
return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
32588
|
+
reasonSubcode: "capture_identity_mismatch"
|
|
32589
|
+
});
|
|
32834
32590
|
}
|
|
32835
|
-
|
|
32836
|
-
const
|
|
32837
|
-
|
|
32838
|
-
|
|
32591
|
+
} catch (error48) {
|
|
32592
|
+
const reason = stableFailureReason(error48, "provider_dataset_invalid");
|
|
32593
|
+
return outcome(request, "vendor_fetch_failed", "vendor_fetch_failed", {
|
|
32594
|
+
reasonSubcode: reason.startsWith("budget_") ? "resource_limit_exceeded" : reason
|
|
32839
32595
|
});
|
|
32840
|
-
|
|
32841
|
-
|
|
32842
|
-
const
|
|
32843
|
-
|
|
32844
|
-
|
|
32596
|
+
}
|
|
32597
|
+
try {
|
|
32598
|
+
const candidateBatches = buildForwarderBatches({
|
|
32599
|
+
captureBundleId: normalized.captureBundleId,
|
|
32600
|
+
deploymentId: "market-data-vendor-backfill",
|
|
32601
|
+
rows: normalized.rows
|
|
32602
|
+
});
|
|
32603
|
+
if (!await submitAll(dependencies, candidateBatches)) {
|
|
32604
|
+
return outcome(request, "archive_ingest_failed", "candidate_batch_rejected");
|
|
32845
32605
|
}
|
|
32846
|
-
|
|
32847
|
-
|
|
32848
|
-
|
|
32849
|
-
|
|
32850
|
-
|
|
32851
|
-
|
|
32852
|
-
|
|
32853
|
-
|
|
32854
|
-
|
|
32855
|
-
|
|
32856
|
-
|
|
32857
|
-
|
|
32858
|
-
|
|
32859
|
-
|
|
32860
|
-
|
|
32861
|
-
|
|
32862
|
-
|
|
32863
|
-
|
|
32864
|
-
|
|
32865
|
-
|
|
32866
|
-
|
|
32867
|
-
|
|
32868
|
-
throw new CryptoHftDataError("budget_max_bytes_exceeded");
|
|
32869
|
-
}
|
|
32870
|
-
const decoded = await this.decode(bytes);
|
|
32871
|
-
totalRows += decoded.length;
|
|
32872
|
-
if (totalRows > request.budgets.maxRows) {
|
|
32873
|
-
throw new CryptoHftDataError("budget_max_rows_exceeded");
|
|
32874
|
-
}
|
|
32875
|
-
const object2 = {
|
|
32876
|
-
identity: path2,
|
|
32877
|
-
checksum: sha256Bytes(bytes),
|
|
32878
|
-
bytes: bytes.byteLength,
|
|
32879
|
-
rows: decoded.length
|
|
32880
|
-
};
|
|
32881
|
-
objects.push(object2);
|
|
32882
|
-
for (const decodedRow of decoded) {
|
|
32883
|
-
const parsed = parsedDatasetRow(decodedRow, object2);
|
|
32884
|
-
validatedRow(request, parsed);
|
|
32885
|
-
rows.push(parsed);
|
|
32886
|
-
}
|
|
32606
|
+
} catch {
|
|
32607
|
+
return outcome(request, "archive_ingest_failed", "candidate_submission_failed");
|
|
32608
|
+
}
|
|
32609
|
+
let verification;
|
|
32610
|
+
try {
|
|
32611
|
+
verification = await dependencies.archive.verifyCandidate(request, normalized, normalized.captureBundleId, initialResolution);
|
|
32612
|
+
} catch {
|
|
32613
|
+
return outcome(request, "promotion_verification_failed", "candidate_query_failed");
|
|
32614
|
+
}
|
|
32615
|
+
if (!verification.passed || verification.captureBundleId !== normalized.captureBundleId || verification.canonicalSemanticDigest !== normalized.canonicalSemanticDigest || !verification.seamVerified || !verification.coverageVerified) {
|
|
32616
|
+
return outcome(request, "promotion_verification_failed", verification.reasonCode ?? "semantic_verification_failed");
|
|
32617
|
+
}
|
|
32618
|
+
let receipt;
|
|
32619
|
+
try {
|
|
32620
|
+
receipt = buildPromotionReceipt(request, capability, normalized, verification, dependencies.clock.nowMs());
|
|
32621
|
+
const [batch] = buildForwarderBatches({
|
|
32622
|
+
captureBundleId: normalized.captureBundleId,
|
|
32623
|
+
deploymentId: "market-data-vendor-backfill",
|
|
32624
|
+
rows: [promotionReceiptToArchiveRow(receipt)]
|
|
32625
|
+
});
|
|
32626
|
+
if (!batch || !await submitAll(dependencies, [batch])) {
|
|
32627
|
+
return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
|
|
32887
32628
|
}
|
|
32888
|
-
|
|
32889
|
-
|
|
32629
|
+
const qualification = finalizeQualificationEvent({
|
|
32630
|
+
capture_bundle_id: receipt.capture_bundle_id,
|
|
32631
|
+
state: "qualified",
|
|
32632
|
+
receipt_id: receipt.receipt_id,
|
|
32633
|
+
promotion_identity_sha256: receipt.promotion_identity_sha256,
|
|
32634
|
+
window: receipt.window,
|
|
32635
|
+
event_at: receipt.verified_at,
|
|
32636
|
+
reason_code: "promotion_verified"
|
|
32637
|
+
});
|
|
32638
|
+
const [qualificationBatch] = buildForwarderBatches({
|
|
32639
|
+
captureBundleId: normalized.captureBundleId,
|
|
32640
|
+
deploymentId: "market-data-vendor-backfill",
|
|
32641
|
+
rows: [qualificationEventToArchiveRow(qualification)]
|
|
32642
|
+
});
|
|
32643
|
+
if (!qualificationBatch || !await submitAll(dependencies, [qualificationBatch])) {
|
|
32644
|
+
return outcome(request, "archive_ingest_failed", "qualification_commit_failed", { receipt });
|
|
32890
32645
|
}
|
|
32891
|
-
|
|
32892
|
-
|
|
32893
|
-
rows,
|
|
32894
|
-
vendorSemanticDigest: sha256Canonical(rows)
|
|
32895
|
-
};
|
|
32646
|
+
} catch {
|
|
32647
|
+
return outcome(request, "archive_ingest_failed", "promotion_commit_failed");
|
|
32896
32648
|
}
|
|
32897
|
-
|
|
32898
|
-
|
|
32899
|
-
|
|
32900
|
-
|
|
32901
|
-
|
|
32902
|
-
|
|
32903
|
-
|
|
32904
|
-
|
|
32649
|
+
let finalResolution;
|
|
32650
|
+
try {
|
|
32651
|
+
finalResolution = await dependencies.archive.resolveSelection(request);
|
|
32652
|
+
assertArchivePreflight(request, finalResolution);
|
|
32653
|
+
} catch (error48) {
|
|
32654
|
+
return outcome(request, "promotion_verification_failed", "post_promotion_selection_failed", {
|
|
32655
|
+
receipt,
|
|
32656
|
+
reasonSubcode: stableFailureReason(error48, "archive_selection_resolution_failed")
|
|
32657
|
+
});
|
|
32658
|
+
}
|
|
32659
|
+
if (finalResolution.selection.coverage_class !== "complete") {
|
|
32660
|
+
return outcome(request, "promotion_verification_failed", "qualified_coverage_incomplete", { receipt, selection: finalResolution.selection });
|
|
32661
|
+
}
|
|
32662
|
+
if (finalResolution.selection.bundles.some((bundle) => bundle.capture_bundle_id === receipt.capture_bundle_id && bundle.capture_origin === "vendor_historical_backfill" && bundle.qualification?.receipt_id === receipt.receipt_id) === false) {
|
|
32663
|
+
return outcome(request, "promotion_verification_failed", "promoted_receipt_not_selected", { receipt, selection: finalResolution.selection });
|
|
32664
|
+
}
|
|
32665
|
+
try {
|
|
32666
|
+
if (!request.wire)
|
|
32667
|
+
throw new Error("decoded request wire is missing");
|
|
32668
|
+
const [selectionBatch] = buildForwarderBatches({
|
|
32669
|
+
captureBundleId: receipt.capture_bundle_id,
|
|
32905
32670
|
deploymentId: "market-data-vendor-backfill",
|
|
32906
|
-
|
|
32907
|
-
|
|
32908
|
-
|
|
32909
|
-
tradingPair: request.scope.tradingPair,
|
|
32910
|
-
sourceSymbol: request.scope.sourceSymbol,
|
|
32911
|
-
assetType: request.scope.marketType,
|
|
32912
|
-
feed: "ORDERBOOK",
|
|
32913
|
-
provider: "cryptohftdata",
|
|
32914
|
-
sourceMode: HISTORICAL_VENDOR_SOURCE_MODE,
|
|
32915
|
-
schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION,
|
|
32916
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM,
|
|
32917
|
-
provenanceComplete: true
|
|
32918
|
-
};
|
|
32919
|
-
const rows = samples.flatMap((sample) => {
|
|
32920
|
-
const rawCapture = {
|
|
32921
|
-
rawCaptureId: sha256Canonical({
|
|
32922
|
-
capture_bundle_id: captureBundleId2,
|
|
32923
|
-
dataset_object_identity: sample.datasetObjectIdentity,
|
|
32924
|
-
dataset_object_checksum: sample.datasetObjectChecksum,
|
|
32925
|
-
source_time_ms: sample.sourceTimeMs,
|
|
32926
|
-
target_time_ms: sample.targetTimeMs
|
|
32927
|
-
}),
|
|
32928
|
-
rawCaptureScope: VENDOR_DATASET_RAW_CAPTURE_SCOPE,
|
|
32929
|
-
rawChecksum: sample.datasetObjectChecksum,
|
|
32930
|
-
redactedPayload: {
|
|
32931
|
-
dataset_object_identity: sample.datasetObjectIdentity,
|
|
32932
|
-
dataset_object_checksum: sample.datasetObjectChecksum
|
|
32933
|
-
},
|
|
32934
|
-
eventTimeMs: sample.sourceTimeMs,
|
|
32935
|
-
receivedTimeMs: sample.receivedTimeMs,
|
|
32936
|
-
checksumAlgorithm: CHECKSUM_ALGORITHM
|
|
32937
|
-
};
|
|
32938
|
-
const canonical = buildCanonicalOrderBookRows({
|
|
32939
|
-
context: context2,
|
|
32940
|
-
rawCapture,
|
|
32941
|
-
depthLimit: request.depth,
|
|
32942
|
-
constructionMode: request.constructionMode,
|
|
32943
|
-
snapshot: {
|
|
32944
|
-
bids: sample.bids,
|
|
32945
|
-
asks: sample.asks,
|
|
32946
|
-
timestamp: sample.sourceTimeMs,
|
|
32947
|
-
receivedTimestamp: sample.receivedTimeMs,
|
|
32948
|
-
exchange: request.scope.exchange,
|
|
32949
|
-
symbol: request.scope.tradingPair,
|
|
32950
|
-
depthLimit: request.depth,
|
|
32951
|
-
sequence: sample.sequence
|
|
32952
|
-
}
|
|
32953
|
-
});
|
|
32954
|
-
return [...canonical.levels, canonical.summary];
|
|
32671
|
+
rows: [
|
|
32672
|
+
archiveSelectionToArchiveRow(request.wire, finalResolution.selection)
|
|
32673
|
+
]
|
|
32955
32674
|
});
|
|
32956
|
-
|
|
32957
|
-
|
|
32958
|
-
|
|
32959
|
-
|
|
32960
|
-
|
|
32961
|
-
canonicalSemanticDigest: semanticDigest(rows)
|
|
32962
|
-
};
|
|
32675
|
+
if (!selectionBatch || !await submitAll(dependencies, [selectionBatch])) {
|
|
32676
|
+
return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
32677
|
+
}
|
|
32678
|
+
} catch {
|
|
32679
|
+
return outcome(request, "archive_ingest_failed", "selection_persistence_failed", { receipt, selection: finalResolution.selection });
|
|
32963
32680
|
}
|
|
32681
|
+
return outcome(request, "promoted", "promotion_qualified", {
|
|
32682
|
+
receipt,
|
|
32683
|
+
selection: finalResolution.selection
|
|
32684
|
+
});
|
|
32964
32685
|
}
|
|
32965
32686
|
|
|
32966
32687
|
// src/helpers/market-data-vendor-backfill/forwarder-client.ts
|
|
@@ -33081,8 +32802,8 @@ function validateReleaseIdentity(release) {
|
|
|
33081
32802
|
return release;
|
|
33082
32803
|
}
|
|
33083
32804
|
function bakedReleaseIdentity() {
|
|
33084
|
-
const packageVersion = "0.2.
|
|
33085
|
-
const gitHead = "
|
|
32805
|
+
const packageVersion = "0.2.49";
|
|
32806
|
+
const gitHead = "c1ffe071e7696c5b17c8f058da01d61cb27ec8a4";
|
|
33086
32807
|
return validateReleaseIdentity({ packageVersion, gitHead });
|
|
33087
32808
|
}
|
|
33088
32809
|
function readOptionalEnvironment(environment, name) {
|
|
@@ -33313,4 +33034,4 @@ export {
|
|
|
33313
33034
|
createBackfillDependenciesFromEnv
|
|
33314
33035
|
};
|
|
33315
33036
|
|
|
33316
|
-
//# debugId=
|
|
33037
|
+
//# debugId=793AA74075FC03DE64756E2164756E21
|