@usherlabs/cex-broker 0.2.47 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cex-canonical-orderbook-export.js +30 -350
- package/dist/commands/cex-canonical-orderbook-export.js.map +7 -19
- package/dist/commands/cli.js +100 -35
- package/dist/commands/market-data-vendor-backfill.js +1463 -1742
- package/dist/commands/market-data-vendor-backfill.js.map +12 -24
- package/dist/helpers/logger.d.ts +4 -0
- package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +2 -1
- package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +29 -2
- package/dist/helpers/market-data-vendor-backfill/semantic-verification.d.ts +1 -0
- package/dist/helpers/otel.d.ts +1 -0
- package/dist/index.js +59 -31
- package/dist/index.js.map +5 -5
- package/dist/market-data-preparation/fixtures/conformance-v2.json +2 -2
- package/dist/market-data-preparation/policies/capability-policy-v1.json +28 -0
- package/dist/market-data-preparation/policies/capability-policy.json +3 -3
- package/dist/market-data-preparation.js +30 -351
- package/dist/market-data-preparation.js.map +9 -21
- package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +28 -0
- package/dist/market-data-vendor-backfill.js +1028 -1306
- package/dist/market-data-vendor-backfill.js.map +13 -25
- package/package.json +1 -1
package/dist/commands/cli.js
CHANGED
|
@@ -313534,16 +313534,20 @@ var tslogLogger = new Logger({
|
|
|
313534
313534
|
minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
|
|
313535
313535
|
});
|
|
313536
313536
|
var baseLogger = new LogLayer({
|
|
313537
|
-
transport: [
|
|
313538
|
-
new TsLogTransport({ id: "tslog", logger: tslogLogger }),
|
|
313539
|
-
new OpenTelemetryTransport({
|
|
313540
|
-
id: "otel",
|
|
313541
|
-
enabled: true
|
|
313542
|
-
})
|
|
313543
|
-
],
|
|
313537
|
+
transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
|
|
313544
313538
|
plugins: [openTelemetryPlugin()],
|
|
313545
313539
|
errorSerializer: serializeError
|
|
313546
313540
|
});
|
|
313541
|
+
var OTEL_TRANSPORT_ID = "otel";
|
|
313542
|
+
function attachOtelLogTransport() {
|
|
313543
|
+
baseLogger.addTransport(new OpenTelemetryTransport({
|
|
313544
|
+
id: OTEL_TRANSPORT_ID,
|
|
313545
|
+
enabled: true
|
|
313546
|
+
}));
|
|
313547
|
+
}
|
|
313548
|
+
function detachOtelLogTransport() {
|
|
313549
|
+
baseLogger.removeTransport(OTEL_TRANSPORT_ID);
|
|
313550
|
+
}
|
|
313547
313551
|
if (process.env.LOG_LEVEL !== "debug") {
|
|
313548
313552
|
baseLogger.setLevel("info");
|
|
313549
313553
|
}
|
|
@@ -317970,11 +317974,14 @@ var import_sdk_metrics = __toESM(require_src12(), 1);
|
|
|
317970
317974
|
var DEFAULT_SERVICE = "cex-broker";
|
|
317971
317975
|
var DEFAULT_OTLP_PORT = 4318;
|
|
317972
317976
|
var EXPORT_INTERVAL_MS = 5000;
|
|
317977
|
+
var EXPORT_TIMEOUT_MS = 2000;
|
|
317978
|
+
var SHUTDOWN_TIMEOUT_MS = 3000;
|
|
317973
317979
|
|
|
317974
317980
|
class BaseOtelSignal {
|
|
317975
317981
|
signal;
|
|
317976
317982
|
provider = null;
|
|
317977
317983
|
isEnabled = false;
|
|
317984
|
+
closePromise = null;
|
|
317978
317985
|
serviceName;
|
|
317979
317986
|
constructor(config, signal) {
|
|
317980
317987
|
this.signal = signal;
|
|
@@ -318006,19 +318013,35 @@ class BaseOtelSignal {
|
|
|
318006
318013
|
isOtelEnabled() {
|
|
318007
318014
|
return this.isEnabled && this.provider !== null;
|
|
318008
318015
|
}
|
|
318009
|
-
|
|
318010
|
-
if (
|
|
318011
|
-
return;
|
|
318012
|
-
|
|
318013
|
-
try {
|
|
318014
|
-
await this.shutdownProvider(this.provider);
|
|
318015
|
-
log.info(`OTel ${this.signal} provider shut down`);
|
|
318016
|
-
} catch (error) {
|
|
318017
|
-
log.error(`Error shutting down OTel ${this.signal} provider:`, error);
|
|
318018
|
-
}
|
|
318016
|
+
close() {
|
|
318017
|
+
if (this.closePromise)
|
|
318018
|
+
return this.closePromise;
|
|
318019
|
+
const provider = this.provider;
|
|
318019
318020
|
this.provider = null;
|
|
318020
318021
|
this.isEnabled = false;
|
|
318021
318022
|
this.onProviderClosed();
|
|
318023
|
+
if (!provider) {
|
|
318024
|
+
this.closePromise = Promise.resolve();
|
|
318025
|
+
return this.closePromise;
|
|
318026
|
+
}
|
|
318027
|
+
const shutdown = Promise.resolve().then(() => this.shutdownProvider(provider)).then(() => {
|
|
318028
|
+
log.info(`OTel ${this.signal} provider shut down`);
|
|
318029
|
+
}, (error) => {
|
|
318030
|
+
log.error(`Error shutting down OTel ${this.signal} provider:`, error);
|
|
318031
|
+
});
|
|
318032
|
+
let deadlineTimer;
|
|
318033
|
+
const deadline = new Promise((resolve) => {
|
|
318034
|
+
deadlineTimer = setTimeout(() => {
|
|
318035
|
+
deadlineTimer = undefined;
|
|
318036
|
+
log.warn(`OTel ${this.signal} provider shutdown exceeded ${SHUTDOWN_TIMEOUT_MS}ms; continuing shutdown`);
|
|
318037
|
+
resolve();
|
|
318038
|
+
}, SHUTDOWN_TIMEOUT_MS);
|
|
318039
|
+
});
|
|
318040
|
+
this.closePromise = Promise.race([shutdown, deadline]).finally(() => {
|
|
318041
|
+
if (deadlineTimer !== undefined)
|
|
318042
|
+
clearTimeout(deadlineTimer);
|
|
318043
|
+
});
|
|
318044
|
+
return this.closePromise;
|
|
318022
318045
|
}
|
|
318023
318046
|
}
|
|
318024
318047
|
function toAttributes(labels, service) {
|
|
@@ -318041,11 +318064,13 @@ class OtelMetrics extends BaseOtelSignal {
|
|
|
318041
318064
|
}
|
|
318042
318065
|
createProvider(endpoint, serviceName, appendSignalPath) {
|
|
318043
318066
|
const exporter = new import_exporter_metrics_otlp_http.OTLPMetricExporter({
|
|
318044
|
-
url: appendOtlpPath(endpoint, "metrics", appendSignalPath)
|
|
318067
|
+
url: appendOtlpPath(endpoint, "metrics", appendSignalPath),
|
|
318068
|
+
timeoutMillis: EXPORT_TIMEOUT_MS
|
|
318045
318069
|
});
|
|
318046
318070
|
const reader = new import_sdk_metrics.PeriodicExportingMetricReader({
|
|
318047
318071
|
exporter,
|
|
318048
|
-
exportIntervalMillis: EXPORT_INTERVAL_MS
|
|
318072
|
+
exportIntervalMillis: EXPORT_INTERVAL_MS,
|
|
318073
|
+
exportTimeoutMillis: EXPORT_TIMEOUT_MS
|
|
318049
318074
|
});
|
|
318050
318075
|
const resource = import_resources.resourceFromAttributes({
|
|
318051
318076
|
"service.name": serviceName
|
|
@@ -318166,15 +318191,23 @@ class OtelMetrics extends BaseOtelSignal {
|
|
|
318166
318191
|
}
|
|
318167
318192
|
|
|
318168
318193
|
class OtelLogs extends BaseOtelSignal {
|
|
318169
|
-
logger
|
|
318194
|
+
logger;
|
|
318170
318195
|
constructor(config) {
|
|
318171
318196
|
super(config, "logs");
|
|
318197
|
+
const provider = this.getProvider();
|
|
318198
|
+
this.logger = provider?.getLogger("cex-broker-logs", "1.0.0") ?? null;
|
|
318199
|
+
if (provider) {
|
|
318200
|
+
attachOtelLogTransport();
|
|
318201
|
+
}
|
|
318172
318202
|
}
|
|
318173
318203
|
createProvider(endpoint, serviceName, appendSignalPath) {
|
|
318174
318204
|
const exporter = new import_exporter_logs_otlp_http.OTLPLogExporter({
|
|
318175
|
-
url: appendOtlpPath(endpoint, "logs", appendSignalPath)
|
|
318205
|
+
url: appendOtlpPath(endpoint, "logs", appendSignalPath),
|
|
318206
|
+
timeoutMillis: EXPORT_TIMEOUT_MS
|
|
318207
|
+
});
|
|
318208
|
+
const processor = new import_sdk_logs.BatchLogRecordProcessor(exporter, {
|
|
318209
|
+
exportTimeoutMillis: EXPORT_TIMEOUT_MS
|
|
318176
318210
|
});
|
|
318177
|
-
const processor = new import_sdk_logs.BatchLogRecordProcessor(exporter);
|
|
318178
318211
|
const resource = import_resources.resourceFromAttributes({
|
|
318179
318212
|
"service.name": serviceName
|
|
318180
318213
|
});
|
|
@@ -318185,12 +318218,12 @@ class OtelLogs extends BaseOtelSignal {
|
|
|
318185
318218
|
}
|
|
318186
318219
|
onProviderCreated(provider) {
|
|
318187
318220
|
import_api_logs3.logs.setGlobalLoggerProvider(provider);
|
|
318188
|
-
this.logger = provider.getLogger("cex-broker-logs", "1.0.0");
|
|
318189
318221
|
}
|
|
318190
318222
|
shutdownProvider(provider) {
|
|
318191
|
-
return provider.
|
|
318223
|
+
return provider.shutdown();
|
|
318192
318224
|
}
|
|
318193
318225
|
onProviderClosed() {
|
|
318226
|
+
detachOtelLogTransport();
|
|
318194
318227
|
this.logger = null;
|
|
318195
318228
|
}
|
|
318196
318229
|
emit(record) {
|
|
@@ -337587,12 +337620,7 @@ class CEXBroker {
|
|
|
337587
337620
|
if (this.brokerArchiver) {
|
|
337588
337621
|
await this.brokerArchiver.close();
|
|
337589
337622
|
}
|
|
337590
|
-
|
|
337591
|
-
await this.otelMetrics.close();
|
|
337592
|
-
}
|
|
337593
|
-
if (this.otelLogs) {
|
|
337594
|
-
await this.otelLogs.close();
|
|
337595
|
-
}
|
|
337623
|
+
await Promise.all([this.otelMetrics?.close(), this.otelLogs?.close()]);
|
|
337596
337624
|
}
|
|
337597
337625
|
async run() {
|
|
337598
337626
|
const marketArchiveState = resolveMarketCaptureArchiveState({
|
|
@@ -337717,8 +337745,25 @@ async function startBrokerCommand(policyPath, port, whitelistIps, verityProverUr
|
|
|
337717
337745
|
});
|
|
337718
337746
|
broker.loadEnvConfig();
|
|
337719
337747
|
await broker.run();
|
|
337748
|
+
return broker;
|
|
337720
337749
|
}
|
|
337721
337750
|
|
|
337751
|
+
// src/helpers/logger.ts
|
|
337752
|
+
var tslogLogger2 = new Logger({
|
|
337753
|
+
type: "pretty",
|
|
337754
|
+
stylePrettyLogs: true,
|
|
337755
|
+
minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
|
|
337756
|
+
});
|
|
337757
|
+
var baseLogger2 = new LogLayer({
|
|
337758
|
+
transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger2 })],
|
|
337759
|
+
plugins: [openTelemetryPlugin()],
|
|
337760
|
+
errorSerializer: serializeError
|
|
337761
|
+
});
|
|
337762
|
+
if (process.env.LOG_LEVEL !== "debug") {
|
|
337763
|
+
baseLogger2.setLevel("info");
|
|
337764
|
+
}
|
|
337765
|
+
var log2 = baseLogger2;
|
|
337766
|
+
|
|
337722
337767
|
// src/cli.ts
|
|
337723
337768
|
var program2 = new Command;
|
|
337724
337769
|
program2.name("cex-broker").description("CLI to start the CEXBroker service").requiredOption("-p, --policy <path>", "Policy JSON file").option("--port <number>", "Port number (default: 8086)", "8086").option("-w, --whitelist <addresses...>", "IPv4 address whitelist (space-separated list)").option("--whitelistAll", "Allow all IPv4 addresses (development mode)").option("--verityProverUrl <url>", "Verity Prover Url").action(async (options) => {
|
|
@@ -337728,15 +337773,35 @@ program2.name("cex-broker").description("CLI to start the CEXBroker service").re
|
|
|
337728
337773
|
const isValidIPv4 = (ip) => /^(\d{1,3}\.){3}\d{1,3}$/.test(ip) && ip.split(".").every((part) => Number(part) >= 0 && Number(part) <= 255);
|
|
337729
337774
|
for (const ip of whitelist) {
|
|
337730
337775
|
if (!isValidIPv4(ip)) {
|
|
337731
|
-
|
|
337732
|
-
process.exit(1);
|
|
337776
|
+
throw new Error(`Invalid IPv4 address: ${ip}`);
|
|
337733
337777
|
}
|
|
337734
337778
|
}
|
|
337735
337779
|
}
|
|
337736
|
-
await startBrokerCommand(options.policy, parseInt(options.port, 10), whitelist, options.verityProverUrl);
|
|
337780
|
+
const broker = await startBrokerCommand(options.policy, parseInt(options.port, 10), whitelist, options.verityProverUrl);
|
|
337781
|
+
let shutdownPromise;
|
|
337782
|
+
const onSignal = (signal) => {
|
|
337783
|
+
if (shutdownPromise)
|
|
337784
|
+
return;
|
|
337785
|
+
log2.withMetadata({ signal }).info("CEXBroker graceful shutdown requested");
|
|
337786
|
+
shutdownPromise = (async () => {
|
|
337787
|
+
try {
|
|
337788
|
+
await broker.stop();
|
|
337789
|
+
log2.withMetadata({ signal }).info("CEXBroker graceful shutdown complete");
|
|
337790
|
+
process.exitCode = 0;
|
|
337791
|
+
} catch (error48) {
|
|
337792
|
+
log2.withMetadata({ signal, error: error48 }).error("CEXBroker graceful shutdown failed");
|
|
337793
|
+
process.exitCode = 1;
|
|
337794
|
+
} finally {
|
|
337795
|
+
process.off("SIGTERM", onSignal);
|
|
337796
|
+
process.off("SIGINT", onSignal);
|
|
337797
|
+
}
|
|
337798
|
+
})();
|
|
337799
|
+
};
|
|
337800
|
+
process.on("SIGTERM", onSignal);
|
|
337801
|
+
process.on("SIGINT", onSignal);
|
|
337737
337802
|
} catch (err2) {
|
|
337738
337803
|
console.error("\u274C Failed to start broker:", err2);
|
|
337739
|
-
process.
|
|
337804
|
+
process.exitCode = 1;
|
|
337740
337805
|
}
|
|
337741
337806
|
});
|
|
337742
|
-
program2.
|
|
337807
|
+
await program2.parseAsync(process.argv);
|