@usherlabs/cex-broker 0.2.48 → 0.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cex-canonical-orderbook-export.js +72 -358
- package/dist/commands/cex-canonical-orderbook-export.js.map +7 -19
- package/dist/commands/cli.js +224 -67
- package/dist/commands/market-data-vendor-backfill.js +975 -1198
- package/dist/commands/market-data-vendor-backfill.js.map +13 -25
- package/dist/helpers/logger.d.ts +4 -0
- package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +3 -1
- package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +58 -4
- package/dist/helpers/market-data-vendor-backfill/semantic-verification.d.ts +1 -0
- package/dist/helpers/otel.d.ts +1 -0
- package/dist/helpers/trace-context.d.ts +8 -0
- package/dist/index.js +183 -63
- package/dist/index.js.map +9 -8
- package/dist/market-data-preparation/fixtures/conformance-v2.json +8 -8
- package/dist/market-data-preparation/policies/capability-policy-v1.json +28 -0
- package/dist/market-data-preparation/policies/capability-policy.json +14 -3
- package/dist/market-data-preparation/policies/resource-policy-v1.json +16 -0
- package/dist/market-data-preparation/policies/resource-policy.json +3 -3
- package/dist/market-data-preparation.js +70 -359
- package/dist/market-data-preparation.js.map +9 -21
- package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +39 -0
- package/dist/market-data-vendor-backfill/policies/resource-policy-v2.json +16 -0
- package/dist/market-data-vendor-backfill.d.ts +2 -2
- package/dist/market-data-vendor-backfill.js +1528 -1750
- package/dist/market-data-vendor-backfill.js.map +13 -25
- package/package.json +1 -1
package/dist/helpers/logger.d.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { LogLayer } from "loglayer";
|
|
2
|
+
export declare function attachOtelLogTransport(): void;
|
|
3
|
+
export declare function detachOtelLogTransport(): void;
|
|
1
4
|
declare const log: {
|
|
2
5
|
trace: (...args: unknown[]) => void;
|
|
3
6
|
debug: (...args: unknown[]) => void;
|
|
@@ -5,5 +8,6 @@ declare const log: {
|
|
|
5
8
|
warn: (...args: unknown[]) => void;
|
|
6
9
|
error: (...args: unknown[]) => void;
|
|
7
10
|
fatal: (...args: unknown[]) => void;
|
|
11
|
+
withMetadata: LogLayer["withMetadata"];
|
|
8
12
|
};
|
|
9
13
|
export { log };
|
|
@@ -16,10 +16,11 @@ export type CryptoHftDataCapabilityProfile = Readonly<{
|
|
|
16
16
|
snapshotGrouping: "event_time_last_update_id_object" | "event_time_final_update_id_object";
|
|
17
17
|
sequenceSemantics: "binance_u_U_pu" | "okx_seq_id_prev_seq_id";
|
|
18
18
|
constructionModes: readonly ["sampled_top_n_snapshot"];
|
|
19
|
-
sourcePolicies: readonly
|
|
19
|
+
sourcePolicies: readonly ("authoritative_window" | "fill_gaps")[];
|
|
20
20
|
}>;
|
|
21
21
|
export declare const CRYPTOHFTDATA_BINANCE_SPOT_BTCUSDT_PROFILE: CryptoHftDataCapabilityProfile;
|
|
22
22
|
export declare const CRYPTOHFTDATA_OKX_SPOT_ARBUSDT_PROFILE: CryptoHftDataCapabilityProfile;
|
|
23
|
+
export declare const CRYPTOHFTDATA_OKX_SPOT_ARBUSDC_PROFILE: CryptoHftDataCapabilityProfile;
|
|
23
24
|
export declare class CryptoHftDataError extends Error {
|
|
24
25
|
readonly reason: string;
|
|
25
26
|
constructor(reason: string);
|
|
@@ -53,6 +54,7 @@ export type ReconstructedCryptoHftBook = {
|
|
|
53
54
|
};
|
|
54
55
|
export declare function cryptoHftDataCapabilityFor(request: MarketDataVendorBackfillRequest, profiles?: readonly CryptoHftDataCapabilityProfile[]): ProviderCapability | undefined;
|
|
55
56
|
export declare function enumerateCryptoHftDataObjects(request: MarketDataVendorBackfillRequest, providerExchangeId: string, resolvedSymbol: string): string[];
|
|
57
|
+
export declare function providerAcquisitionRequest(request: MarketDataVendorBackfillRequest): MarketDataVendorBackfillRequest;
|
|
56
58
|
export declare function reconstructCryptoHftDataOrderBooks(request: MarketDataVendorBackfillRequest, inputRows: readonly CryptoHftDataOrderBookRow[], profile?: CryptoHftDataCapabilityProfile): ReconstructedCryptoHftBook[];
|
|
57
59
|
export declare function decodeCryptoHftParquetZstd(bytes: Uint8Array): Promise<Record<string, unknown>[]>;
|
|
58
60
|
type CryptoHftDataAdapterOptions = {
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
export declare const
|
|
2
|
-
export declare const
|
|
1
|
+
export declare const LEGACY_CAPABILITY_POLICY_ID: "market-data-vendor-backfill-capabilities/v1";
|
|
2
|
+
export declare const CAPABILITY_POLICY_ID: "market-data-vendor-backfill-capabilities/v2";
|
|
3
|
+
export declare const LEGACY_RESOURCE_POLICY_ID: "market-data-vendor-backfill-resources/v1";
|
|
4
|
+
export declare const RESOURCE_POLICY_ID: "market-data-vendor-backfill-resources/v2";
|
|
3
5
|
export declare const ADAPTER_POLICY_ID: "cryptohftdata-orderbook-adapter/v1";
|
|
4
6
|
export declare const ACQUISITION_POLICY_ID: "cryptohftdata-hourly-acquisition/v1";
|
|
5
|
-
export declare const
|
|
7
|
+
export declare const LEGACY_CAPABILITY_POLICY: Readonly<{
|
|
6
8
|
policy_sha256: string;
|
|
7
9
|
policy_id: "market-data-vendor-backfill-capabilities/v1";
|
|
8
10
|
provider: "cryptohftdata";
|
|
@@ -28,7 +30,43 @@ export declare const CAPABILITY_POLICY: Readonly<{
|
|
|
28
30
|
readonly max_depth: 400;
|
|
29
31
|
}];
|
|
30
32
|
}>;
|
|
31
|
-
export declare const
|
|
33
|
+
export declare const CAPABILITY_POLICY: Readonly<{
|
|
34
|
+
policy_sha256: string;
|
|
35
|
+
policy_id: "market-data-vendor-backfill-capabilities/v2";
|
|
36
|
+
profiles: readonly [...{
|
|
37
|
+
source_policies: readonly ["authoritative_window", "fill_gaps"];
|
|
38
|
+
exchange: "okx";
|
|
39
|
+
market_type: "spot";
|
|
40
|
+
feed: "ORDERBOOK";
|
|
41
|
+
canonical_trading_pair: "ARB-USDT";
|
|
42
|
+
provider_exchange_id: "okx_spot";
|
|
43
|
+
resolved_symbol: "ARB-USDT";
|
|
44
|
+
construction_modes: readonly ["sampled_top_n_snapshot"];
|
|
45
|
+
max_depth: 400;
|
|
46
|
+
}[], {
|
|
47
|
+
readonly exchange: "okx";
|
|
48
|
+
readonly market_type: "spot";
|
|
49
|
+
readonly feed: "ORDERBOOK";
|
|
50
|
+
readonly canonical_trading_pair: "ARB-USDC";
|
|
51
|
+
readonly provider_exchange_id: "okx_spot";
|
|
52
|
+
readonly resolved_symbol: "ARB-USDC";
|
|
53
|
+
readonly construction_modes: readonly ["sampled_top_n_snapshot"];
|
|
54
|
+
readonly source_policies: readonly ["authoritative_window", "fill_gaps"];
|
|
55
|
+
readonly max_depth: 400;
|
|
56
|
+
}];
|
|
57
|
+
provider: "cryptohftdata";
|
|
58
|
+
adapter_policy: {
|
|
59
|
+
readonly policy_id: "cryptohftdata-orderbook-adapter/v1";
|
|
60
|
+
readonly adapter_version: "cryptohftdata-orderbook/v2";
|
|
61
|
+
};
|
|
62
|
+
acquisition_policy: {
|
|
63
|
+
readonly policy_id: "cryptohftdata-hourly-acquisition/v1";
|
|
64
|
+
readonly object_granularity: "utc_hour";
|
|
65
|
+
readonly authentication: "bearer";
|
|
66
|
+
readonly initialization_lookback_ms: 0;
|
|
67
|
+
};
|
|
68
|
+
}>;
|
|
69
|
+
export declare const LEGACY_RESOURCE_POLICY: Readonly<{
|
|
32
70
|
policy_sha256: string;
|
|
33
71
|
policy_id: "market-data-vendor-backfill-resources/v1";
|
|
34
72
|
limits: {
|
|
@@ -44,6 +82,22 @@ export declare const RESOURCE_POLICY: Readonly<{
|
|
|
44
82
|
readonly max_required_events: 100000;
|
|
45
83
|
};
|
|
46
84
|
}>;
|
|
85
|
+
export declare const RESOURCE_POLICY: Readonly<{
|
|
86
|
+
policy_sha256: string;
|
|
87
|
+
policy_id: "market-data-vendor-backfill-resources/v2";
|
|
88
|
+
request_bounds: {
|
|
89
|
+
readonly max_window_ms: number;
|
|
90
|
+
readonly max_depth: 500;
|
|
91
|
+
readonly max_required_events: 100000;
|
|
92
|
+
};
|
|
93
|
+
limits: {
|
|
94
|
+
readonly max_files: 10000;
|
|
95
|
+
readonly max_bytes: number;
|
|
96
|
+
readonly max_rows: 1000000000;
|
|
97
|
+
readonly max_duration_ms: number;
|
|
98
|
+
readonly max_boundary_lookback_ms: number;
|
|
99
|
+
};
|
|
100
|
+
}>;
|
|
47
101
|
export declare const EFFECTIVE_ADAPTER_POLICY_PIN: Readonly<{
|
|
48
102
|
policy_id: "cryptohftdata-orderbook-adapter/v1";
|
|
49
103
|
policy_sha256: string;
|
package/dist/helpers/otel.d.ts
CHANGED
|
@@ -37,6 +37,7 @@ declare abstract class BaseOtelSignal<TProvider> {
|
|
|
37
37
|
private readonly signal;
|
|
38
38
|
private provider;
|
|
39
39
|
private isEnabled;
|
|
40
|
+
private closePromise;
|
|
40
41
|
private readonly serviceName;
|
|
41
42
|
protected constructor(config: OtelConfig | undefined, signal: "metrics" | "logs");
|
|
42
43
|
protected abstract createProvider(endpoint: string, serviceName: string, appendSignalPath: boolean): TProvider;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Metadata } from "@grpc/grpc-js";
|
|
2
|
+
/** gRPC metadata key for lightweight operational correlation with callers. */
|
|
3
|
+
export declare const TRACE_METADATA_KEY = "x-trace-id";
|
|
4
|
+
/**
|
|
5
|
+
* Returns a validated caller correlation ID without mutating inbound metadata.
|
|
6
|
+
* Invalid values are deliberately discarded before they can reach telemetry.
|
|
7
|
+
*/
|
|
8
|
+
export declare function extractTraceId(metadata: Metadata): string | undefined;
|
package/dist/index.js
CHANGED
|
@@ -289048,16 +289048,20 @@ var tslogLogger = new Logger({
|
|
|
289048
289048
|
minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
|
|
289049
289049
|
});
|
|
289050
289050
|
var baseLogger = new LogLayer({
|
|
289051
|
-
transport: [
|
|
289052
|
-
new TsLogTransport({ id: "tslog", logger: tslogLogger }),
|
|
289053
|
-
new OpenTelemetryTransport({
|
|
289054
|
-
id: "otel",
|
|
289055
|
-
enabled: true
|
|
289056
|
-
})
|
|
289057
|
-
],
|
|
289051
|
+
transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
|
|
289058
289052
|
plugins: [openTelemetryPlugin()],
|
|
289059
289053
|
errorSerializer: serializeError
|
|
289060
289054
|
});
|
|
289055
|
+
var OTEL_TRANSPORT_ID = "otel";
|
|
289056
|
+
function attachOtelLogTransport() {
|
|
289057
|
+
baseLogger.addTransport(new OpenTelemetryTransport({
|
|
289058
|
+
id: OTEL_TRANSPORT_ID,
|
|
289059
|
+
enabled: true
|
|
289060
|
+
}));
|
|
289061
|
+
}
|
|
289062
|
+
function detachOtelLogTransport() {
|
|
289063
|
+
baseLogger.removeTransport(OTEL_TRANSPORT_ID);
|
|
289064
|
+
}
|
|
289061
289065
|
if (process.env.LOG_LEVEL !== "debug") {
|
|
289062
289066
|
baseLogger.setLevel("info");
|
|
289063
289067
|
}
|
|
@@ -293484,11 +293488,14 @@ var import_sdk_metrics = __toESM(require_src9(), 1);
|
|
|
293484
293488
|
var DEFAULT_SERVICE = "cex-broker";
|
|
293485
293489
|
var DEFAULT_OTLP_PORT = 4318;
|
|
293486
293490
|
var EXPORT_INTERVAL_MS = 5000;
|
|
293491
|
+
var EXPORT_TIMEOUT_MS = 2000;
|
|
293492
|
+
var SHUTDOWN_TIMEOUT_MS = 3000;
|
|
293487
293493
|
|
|
293488
293494
|
class BaseOtelSignal {
|
|
293489
293495
|
signal;
|
|
293490
293496
|
provider = null;
|
|
293491
293497
|
isEnabled = false;
|
|
293498
|
+
closePromise = null;
|
|
293492
293499
|
serviceName;
|
|
293493
293500
|
constructor(config, signal) {
|
|
293494
293501
|
this.signal = signal;
|
|
@@ -293520,19 +293527,35 @@ class BaseOtelSignal {
|
|
|
293520
293527
|
isOtelEnabled() {
|
|
293521
293528
|
return this.isEnabled && this.provider !== null;
|
|
293522
293529
|
}
|
|
293523
|
-
|
|
293524
|
-
if (
|
|
293525
|
-
return;
|
|
293526
|
-
|
|
293527
|
-
try {
|
|
293528
|
-
await this.shutdownProvider(this.provider);
|
|
293529
|
-
log.info(`OTel ${this.signal} provider shut down`);
|
|
293530
|
-
} catch (error) {
|
|
293531
|
-
log.error(`Error shutting down OTel ${this.signal} provider:`, error);
|
|
293532
|
-
}
|
|
293530
|
+
close() {
|
|
293531
|
+
if (this.closePromise)
|
|
293532
|
+
return this.closePromise;
|
|
293533
|
+
const provider = this.provider;
|
|
293533
293534
|
this.provider = null;
|
|
293534
293535
|
this.isEnabled = false;
|
|
293535
293536
|
this.onProviderClosed();
|
|
293537
|
+
if (!provider) {
|
|
293538
|
+
this.closePromise = Promise.resolve();
|
|
293539
|
+
return this.closePromise;
|
|
293540
|
+
}
|
|
293541
|
+
const shutdown = Promise.resolve().then(() => this.shutdownProvider(provider)).then(() => {
|
|
293542
|
+
log.info(`OTel ${this.signal} provider shut down`);
|
|
293543
|
+
}, (error) => {
|
|
293544
|
+
log.error(`Error shutting down OTel ${this.signal} provider:`, error);
|
|
293545
|
+
});
|
|
293546
|
+
let deadlineTimer;
|
|
293547
|
+
const deadline = new Promise((resolve) => {
|
|
293548
|
+
deadlineTimer = setTimeout(() => {
|
|
293549
|
+
deadlineTimer = undefined;
|
|
293550
|
+
log.warn(`OTel ${this.signal} provider shutdown exceeded ${SHUTDOWN_TIMEOUT_MS}ms; continuing shutdown`);
|
|
293551
|
+
resolve();
|
|
293552
|
+
}, SHUTDOWN_TIMEOUT_MS);
|
|
293553
|
+
});
|
|
293554
|
+
this.closePromise = Promise.race([shutdown, deadline]).finally(() => {
|
|
293555
|
+
if (deadlineTimer !== undefined)
|
|
293556
|
+
clearTimeout(deadlineTimer);
|
|
293557
|
+
});
|
|
293558
|
+
return this.closePromise;
|
|
293536
293559
|
}
|
|
293537
293560
|
}
|
|
293538
293561
|
function toAttributes(labels, service) {
|
|
@@ -293555,11 +293578,13 @@ class OtelMetrics extends BaseOtelSignal {
|
|
|
293555
293578
|
}
|
|
293556
293579
|
createProvider(endpoint, serviceName, appendSignalPath) {
|
|
293557
293580
|
const exporter = new import_exporter_metrics_otlp_http.OTLPMetricExporter({
|
|
293558
|
-
url: appendOtlpPath(endpoint, "metrics", appendSignalPath)
|
|
293581
|
+
url: appendOtlpPath(endpoint, "metrics", appendSignalPath),
|
|
293582
|
+
timeoutMillis: EXPORT_TIMEOUT_MS
|
|
293559
293583
|
});
|
|
293560
293584
|
const reader = new import_sdk_metrics.PeriodicExportingMetricReader({
|
|
293561
293585
|
exporter,
|
|
293562
|
-
exportIntervalMillis: EXPORT_INTERVAL_MS
|
|
293586
|
+
exportIntervalMillis: EXPORT_INTERVAL_MS,
|
|
293587
|
+
exportTimeoutMillis: EXPORT_TIMEOUT_MS
|
|
293563
293588
|
});
|
|
293564
293589
|
const resource = import_resources.resourceFromAttributes({
|
|
293565
293590
|
"service.name": serviceName
|
|
@@ -293680,15 +293705,23 @@ class OtelMetrics extends BaseOtelSignal {
|
|
|
293680
293705
|
}
|
|
293681
293706
|
|
|
293682
293707
|
class OtelLogs extends BaseOtelSignal {
|
|
293683
|
-
logger
|
|
293708
|
+
logger;
|
|
293684
293709
|
constructor(config) {
|
|
293685
293710
|
super(config, "logs");
|
|
293711
|
+
const provider = this.getProvider();
|
|
293712
|
+
this.logger = provider?.getLogger("cex-broker-logs", "1.0.0") ?? null;
|
|
293713
|
+
if (provider) {
|
|
293714
|
+
attachOtelLogTransport();
|
|
293715
|
+
}
|
|
293686
293716
|
}
|
|
293687
293717
|
createProvider(endpoint, serviceName, appendSignalPath) {
|
|
293688
293718
|
const exporter = new import_exporter_logs_otlp_http.OTLPLogExporter({
|
|
293689
|
-
url: appendOtlpPath(endpoint, "logs", appendSignalPath)
|
|
293719
|
+
url: appendOtlpPath(endpoint, "logs", appendSignalPath),
|
|
293720
|
+
timeoutMillis: EXPORT_TIMEOUT_MS
|
|
293721
|
+
});
|
|
293722
|
+
const processor = new import_sdk_logs.BatchLogRecordProcessor(exporter, {
|
|
293723
|
+
exportTimeoutMillis: EXPORT_TIMEOUT_MS
|
|
293690
293724
|
});
|
|
293691
|
-
const processor = new import_sdk_logs.BatchLogRecordProcessor(exporter);
|
|
293692
293725
|
const resource = import_resources.resourceFromAttributes({
|
|
293693
293726
|
"service.name": serviceName
|
|
293694
293727
|
});
|
|
@@ -293699,12 +293732,12 @@ class OtelLogs extends BaseOtelSignal {
|
|
|
293699
293732
|
}
|
|
293700
293733
|
onProviderCreated(provider) {
|
|
293701
293734
|
import_api_logs3.logs.setGlobalLoggerProvider(provider);
|
|
293702
|
-
this.logger = provider.getLogger("cex-broker-logs", "1.0.0");
|
|
293703
293735
|
}
|
|
293704
293736
|
shutdownProvider(provider) {
|
|
293705
|
-
return provider.
|
|
293737
|
+
return provider.shutdown();
|
|
293706
293738
|
}
|
|
293707
293739
|
onProviderClosed() {
|
|
293740
|
+
detachOtelLogTransport();
|
|
293708
293741
|
this.logger = null;
|
|
293709
293742
|
}
|
|
293710
293743
|
emit(record) {
|
|
@@ -310635,6 +310668,24 @@ function selectBrokerAccountForCex(normalizedCex, brokers, metadata) {
|
|
|
310635
310668
|
return selectBrokerAccount(brokers[normalizedCex], metadata) ?? undefined;
|
|
310636
310669
|
}
|
|
310637
310670
|
|
|
310671
|
+
// src/helpers/trace-context.ts
|
|
310672
|
+
var TRACE_METADATA_KEY = "x-trace-id";
|
|
310673
|
+
var MAX_TRACE_METADATA_LENGTH = 128;
|
|
310674
|
+
var OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
|
|
310675
|
+
var UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
310676
|
+
var ZERO_OTEL_TRACE_ID = "00000000000000000000000000000000";
|
|
310677
|
+
function extractTraceId(metadata) {
|
|
310678
|
+
const raw = metadata.get(TRACE_METADATA_KEY)[0];
|
|
310679
|
+
if (typeof raw !== "string" || raw.length > MAX_TRACE_METADATA_LENGTH) {
|
|
310680
|
+
return;
|
|
310681
|
+
}
|
|
310682
|
+
const traceId = raw.trim();
|
|
310683
|
+
if (OTEL_TRACE_ID_PATTERN.test(traceId) && traceId !== ZERO_OTEL_TRACE_ID || UUID_V4_PATTERN.test(traceId)) {
|
|
310684
|
+
return traceId;
|
|
310685
|
+
}
|
|
310686
|
+
return;
|
|
310687
|
+
}
|
|
310688
|
+
|
|
310638
310689
|
// src/handlers/execute-action/order-book-call.ts
|
|
310639
310690
|
import * as grpc4 from "@grpc/grpc-js";
|
|
310640
310691
|
async function handleOrderBookCall(ctx) {
|
|
@@ -311938,6 +311989,15 @@ function isPublicMarketDataAction(action, payload) {
|
|
|
311938
311989
|
return false;
|
|
311939
311990
|
return isOrderBookCallMethod(payload?.method ?? payload?.functionName);
|
|
311940
311991
|
}
|
|
311992
|
+
function grpcStatusName(error48) {
|
|
311993
|
+
if (!error48) {
|
|
311994
|
+
return "OK";
|
|
311995
|
+
}
|
|
311996
|
+
if (typeof error48.code !== "number") {
|
|
311997
|
+
return "UNKNOWN";
|
|
311998
|
+
}
|
|
311999
|
+
return grpc12.status[error48.code] ?? "UNKNOWN";
|
|
312000
|
+
}
|
|
311941
312001
|
function createExecuteActionHandler(deps) {
|
|
311942
312002
|
const {
|
|
311943
312003
|
policy,
|
|
@@ -311954,12 +312014,28 @@ function createExecuteActionHandler(deps) {
|
|
|
311954
312014
|
const startTime = Date.now();
|
|
311955
312015
|
const { action: rawAction, cex: cex3, symbol: symbol2 } = call.request;
|
|
311956
312016
|
const action = resolveAction(rawAction);
|
|
312017
|
+
const actionName = getActionName(action);
|
|
312018
|
+
const operationalCex = cex3?.trim().toLowerCase() || "unknown";
|
|
312019
|
+
const traceId = extractTraceId(call.metadata);
|
|
312020
|
+
const traceFields = traceId === undefined ? {} : { trace_id: traceId };
|
|
311957
312021
|
let actionCompleted = false;
|
|
311958
312022
|
const wrappedCallback = (error48, value) => {
|
|
311959
312023
|
if (!actionCompleted) {
|
|
311960
312024
|
actionCompleted = true;
|
|
311961
312025
|
const latency = Date.now() - startTime;
|
|
311962
|
-
const
|
|
312026
|
+
const terminalFields = {
|
|
312027
|
+
action: actionName,
|
|
312028
|
+
cex: operationalCex,
|
|
312029
|
+
latency_ms: latency,
|
|
312030
|
+
outcome: error48 ? "error" : "success",
|
|
312031
|
+
grpc_status: grpcStatusName(error48),
|
|
312032
|
+
...traceFields
|
|
312033
|
+
};
|
|
312034
|
+
if (error48) {
|
|
312035
|
+
log.withMetadata(terminalFields).error("ExecuteAction failed");
|
|
312036
|
+
} else {
|
|
312037
|
+
log.withMetadata(terminalFields).info("ExecuteAction completed");
|
|
312038
|
+
}
|
|
311963
312039
|
otelMetrics?.recordHistogram("execute_action_duration_ms", latency, {
|
|
311964
312040
|
action: actionName,
|
|
311965
312041
|
cex: cex3 || "unknown"
|
|
@@ -311980,9 +312056,13 @@ function createExecuteActionHandler(deps) {
|
|
|
311980
312056
|
callback(error48, value);
|
|
311981
312057
|
};
|
|
311982
312058
|
try {
|
|
311983
|
-
log.
|
|
312059
|
+
log.withMetadata({
|
|
312060
|
+
action: actionName,
|
|
312061
|
+
cex: operationalCex,
|
|
312062
|
+
...traceFields
|
|
312063
|
+
}).info("ExecuteAction started");
|
|
311984
312064
|
otelMetrics?.recordCounter("execute_action_requests_total", 1, {
|
|
311985
|
-
action:
|
|
312065
|
+
action: actionName,
|
|
311986
312066
|
cex: cex3 || "unknown"
|
|
311987
312067
|
});
|
|
311988
312068
|
if (!authenticateRequest(call, whitelistIps)) {
|
|
@@ -312220,6 +312300,9 @@ async function writeSubscribeError(call, isClosed, frame) {
|
|
|
312220
312300
|
call.end();
|
|
312221
312301
|
}
|
|
312222
312302
|
}
|
|
312303
|
+
function grpcStatusName2(status14) {
|
|
312304
|
+
return typeof status14 === "number" ? grpc13.status[status14] ?? "UNKNOWN" : "UNKNOWN";
|
|
312305
|
+
}
|
|
312223
312306
|
function getBinanceEventMarketId(event) {
|
|
312224
312307
|
const value = event.s;
|
|
312225
312308
|
return typeof value === "string" ? value : null;
|
|
@@ -312352,6 +312435,24 @@ function createSubscribeHandler(deps) {
|
|
|
312352
312435
|
});
|
|
312353
312436
|
return async (call) => {
|
|
312354
312437
|
const subscribeStartTime = Date.now();
|
|
312438
|
+
const request = call.request;
|
|
312439
|
+
const { cex: cex3, symbol: symbol2, type: type2 } = request;
|
|
312440
|
+
const metadata = call.metadata;
|
|
312441
|
+
const traceId = extractTraceId(metadata);
|
|
312442
|
+
const traceFields = traceId === undefined ? {} : { trace_id: traceId };
|
|
312443
|
+
const operationalCex = cex3?.trim().toLowerCase() || "unknown";
|
|
312444
|
+
const operationalSymbol = symbol2?.trim() || "unknown";
|
|
312445
|
+
const subscriptionType2 = resolveSubscriptionType(type2);
|
|
312446
|
+
const subscriptionTypeName = getSubscriptionTypeName(subscriptionType2);
|
|
312447
|
+
const operationalFields = {
|
|
312448
|
+
cex: operationalCex,
|
|
312449
|
+
symbol: operationalSymbol,
|
|
312450
|
+
subscription_type: subscriptionTypeName,
|
|
312451
|
+
...traceFields
|
|
312452
|
+
};
|
|
312453
|
+
let terminalLogged = false;
|
|
312454
|
+
let terminalOutcome = "completed";
|
|
312455
|
+
let terminalGrpcStatus = "OK";
|
|
312355
312456
|
let streamClosed = false;
|
|
312356
312457
|
let ownedBroker = null;
|
|
312357
312458
|
let ownedBrokerClosePromise;
|
|
@@ -312374,12 +312475,41 @@ function createSubscribeHandler(deps) {
|
|
|
312374
312475
|
const closeOwnedBrokerOnCallEnd = () => {
|
|
312375
312476
|
closeOwnedBroker();
|
|
312376
312477
|
};
|
|
312377
|
-
|
|
312478
|
+
const logTerminalOutcome = (outcome) => {
|
|
312479
|
+
if (terminalLogged) {
|
|
312480
|
+
return;
|
|
312481
|
+
}
|
|
312482
|
+
terminalLogged = true;
|
|
312483
|
+
const fields = {
|
|
312484
|
+
...operationalFields,
|
|
312485
|
+
duration_ms: Date.now() - subscribeStartTime,
|
|
312486
|
+
outcome,
|
|
312487
|
+
grpc_status: outcome === "cancelled" ? "CANCELLED" : terminalGrpcStatus
|
|
312488
|
+
};
|
|
312489
|
+
if (outcome === "error") {
|
|
312490
|
+
log.withMetadata(fields).error("Subscribe failed");
|
|
312491
|
+
} else if (outcome === "cancelled") {
|
|
312492
|
+
log.withMetadata(fields).info("Subscribe cancelled");
|
|
312493
|
+
} else {
|
|
312494
|
+
log.withMetadata(fields).info("Subscribe ended");
|
|
312495
|
+
}
|
|
312496
|
+
};
|
|
312497
|
+
const writeTerminalError = async (frame, status14 = grpc13.status.UNKNOWN) => {
|
|
312498
|
+
terminalOutcome = "error";
|
|
312499
|
+
terminalGrpcStatus = grpcStatusName2(status14);
|
|
312500
|
+
await writeSubscribeError(call, isStreamClosed, frame);
|
|
312501
|
+
};
|
|
312502
|
+
log.withMetadata(operationalFields).info("Subscribe started");
|
|
312503
|
+
call.once("cancelled", () => {
|
|
312504
|
+
terminalGrpcStatus = "CANCELLED";
|
|
312505
|
+
markStreamClosed();
|
|
312506
|
+
logTerminalOutcome("cancelled");
|
|
312507
|
+
});
|
|
312378
312508
|
call.once("cancelled", closeOwnedBrokerOnCallEnd);
|
|
312379
312509
|
call.once("error", closeOwnedBrokerOnCallEnd);
|
|
312380
312510
|
call.once("end", () => {
|
|
312381
312511
|
markStreamClosed();
|
|
312382
|
-
|
|
312512
|
+
logTerminalOutcome(terminalOutcome);
|
|
312383
312513
|
const duration3 = Date.now() - subscribeStartTime;
|
|
312384
312514
|
otelMetrics?.recordHistogram("subscribe_duration_ms", duration3, {
|
|
312385
312515
|
cex: call.request?.cex || "unknown",
|
|
@@ -312388,12 +312518,16 @@ function createSubscribeHandler(deps) {
|
|
|
312388
312518
|
});
|
|
312389
312519
|
call.once("error", (error48) => {
|
|
312390
312520
|
markStreamClosed();
|
|
312391
|
-
|
|
312521
|
+
terminalOutcome = "error";
|
|
312522
|
+
terminalGrpcStatus = grpcStatusName2(typeof error48 === "object" && error48 !== null && "code" in error48 ? error48.code : undefined);
|
|
312523
|
+
logTerminalOutcome(call.cancelled ? "cancelled" : "error");
|
|
312392
312524
|
otelMetrics?.recordCounter("subscribe_errors_total", 1, {
|
|
312393
312525
|
error_type: error48 instanceof Error ? error48.message : "unknown"
|
|
312394
312526
|
});
|
|
312395
312527
|
});
|
|
312396
312528
|
if (!authenticateRequest(call, whitelistIps)) {
|
|
312529
|
+
terminalOutcome = "error";
|
|
312530
|
+
terminalGrpcStatus = "PERMISSION_DENIED";
|
|
312397
312531
|
otelMetrics?.recordCounter("subscribe_errors_total", 1, {
|
|
312398
312532
|
error_type: "permission_denied"
|
|
312399
312533
|
});
|
|
@@ -312404,32 +312538,22 @@ function createSubscribeHandler(deps) {
|
|
|
312404
312538
|
call.destroy(new Error("Access denied: Unauthorized IP"));
|
|
312405
312539
|
return;
|
|
312406
312540
|
}
|
|
312407
|
-
const metadata = call.metadata;
|
|
312408
|
-
let subscriptionType2 = SubscriptionType.ORDERBOOK;
|
|
312409
312541
|
try {
|
|
312410
|
-
const
|
|
312411
|
-
const { cex: cex3, symbol: symbol2, type: type2, options } = request;
|
|
312412
|
-
subscriptionType2 = resolveSubscriptionType(type2);
|
|
312413
|
-
log.info(`Request - Subscribe:`, {
|
|
312414
|
-
cex: request.cex,
|
|
312415
|
-
symbol: request.symbol,
|
|
312416
|
-
type: subscriptionType2
|
|
312417
|
-
});
|
|
312418
|
-
const subscriptionTypeName = getSubscriptionTypeName(subscriptionType2);
|
|
312542
|
+
const { options } = request;
|
|
312419
312543
|
otelMetrics?.recordCounter("subscribe_requests_total", 1, {
|
|
312420
312544
|
cex: cex3 || "unknown",
|
|
312421
312545
|
symbol: symbol2 || "unknown",
|
|
312422
312546
|
type: subscriptionTypeName
|
|
312423
312547
|
});
|
|
312424
312548
|
if (!cex3 || !symbol2) {
|
|
312425
|
-
await
|
|
312549
|
+
await writeTerminalError({
|
|
312426
312550
|
data: JSON.stringify({
|
|
312427
312551
|
error: "cex, symbol, and type are required"
|
|
312428
312552
|
}),
|
|
312429
312553
|
timestamp: Date.now(),
|
|
312430
312554
|
symbol: symbol2 || "",
|
|
312431
312555
|
type: subscriptionType2
|
|
312432
|
-
});
|
|
312556
|
+
}, grpc13.status.INVALID_ARGUMENT);
|
|
312433
312557
|
return;
|
|
312434
312558
|
}
|
|
312435
312559
|
if (isPublicMarketDataSubscription(subscriptionType2)) {
|
|
@@ -312458,7 +312582,7 @@ function createSubscribeHandler(deps) {
|
|
|
312458
312582
|
} catch (error48) {
|
|
312459
312583
|
const message = getErrorMessage(error48);
|
|
312460
312584
|
if (!isStreamClosed()) {
|
|
312461
|
-
await
|
|
312585
|
+
await writeTerminalError({
|
|
312462
312586
|
data: JSON.stringify({ error: message }),
|
|
312463
312587
|
timestamp: Date.now(),
|
|
312464
312588
|
symbol: symbol2,
|
|
@@ -312479,14 +312603,14 @@ function createSubscribeHandler(deps) {
|
|
|
312479
312603
|
const selectedBroker = selectedBrokerAccount?.exchange ?? createBroker(normalizedCex, metadata);
|
|
312480
312604
|
const broker = selectedBroker ?? createPublicBroker(normalizedCex);
|
|
312481
312605
|
if (!broker) {
|
|
312482
|
-
await
|
|
312606
|
+
await writeTerminalError({
|
|
312483
312607
|
data: JSON.stringify({
|
|
312484
312608
|
error: "Exchange not registered and no API metadata found"
|
|
312485
312609
|
}),
|
|
312486
312610
|
timestamp: Date.now(),
|
|
312487
312611
|
symbol: symbol2,
|
|
312488
312612
|
type: subscriptionType2
|
|
312489
|
-
});
|
|
312613
|
+
}, grpc13.status.NOT_FOUND);
|
|
312490
312614
|
return;
|
|
312491
312615
|
}
|
|
312492
312616
|
if (!selectedBrokerAccount) {
|
|
@@ -312514,28 +312638,28 @@ function createSubscribeHandler(deps) {
|
|
|
312514
312638
|
if (isBinanceSpotAccountSubscription(normalizedCex, subscriptionType2, options?.marketType)) {
|
|
312515
312639
|
const accountBroker = selectedBrokerAccount?.exchange ?? selectedBroker;
|
|
312516
312640
|
if (!accountBroker) {
|
|
312517
|
-
await
|
|
312641
|
+
await writeTerminalError({
|
|
312518
312642
|
data: JSON.stringify({
|
|
312519
312643
|
error: "Binance account subscriptions require API credentials"
|
|
312520
312644
|
}),
|
|
312521
312645
|
timestamp: Date.now(),
|
|
312522
312646
|
symbol: resolvedSymbol,
|
|
312523
312647
|
type: subscriptionType2
|
|
312524
|
-
});
|
|
312648
|
+
}, grpc13.status.FAILED_PRECONDITION);
|
|
312525
312649
|
return;
|
|
312526
312650
|
}
|
|
312527
312651
|
const marketId = subscriptionType2 === SubscriptionType.ORDERS ? await getBinanceMarketId(accountBroker, resolvedSymbol) : undefined;
|
|
312528
312652
|
let userDataSource;
|
|
312529
312653
|
if (selectedBrokerAccount) {
|
|
312530
312654
|
if (!userDataStreamSupervisor) {
|
|
312531
|
-
await
|
|
312655
|
+
await writeTerminalError({
|
|
312532
312656
|
data: JSON.stringify({
|
|
312533
312657
|
error: "Configured account user-data supervisor is unavailable"
|
|
312534
312658
|
}),
|
|
312535
312659
|
timestamp: Date.now(),
|
|
312536
312660
|
symbol: resolvedSymbol,
|
|
312537
312661
|
type: subscriptionType2
|
|
312538
|
-
});
|
|
312662
|
+
}, grpc13.status.FAILED_PRECONDITION);
|
|
312539
312663
|
return;
|
|
312540
312664
|
}
|
|
312541
312665
|
userDataSource = userDataStreamSupervisor.subscribe({
|
|
@@ -312558,7 +312682,7 @@ function createSubscribeHandler(deps) {
|
|
|
312558
312682
|
} catch (error48) {
|
|
312559
312683
|
const message = getErrorMessage(error48);
|
|
312560
312684
|
log.error(`Error fetching balance for ${cex3}:`, error48);
|
|
312561
|
-
await
|
|
312685
|
+
await writeTerminalError({
|
|
312562
312686
|
data: JSON.stringify({
|
|
312563
312687
|
error: `Failed to fetch balance: ${message}`
|
|
312564
312688
|
}),
|
|
@@ -312577,7 +312701,7 @@ function createSubscribeHandler(deps) {
|
|
|
312577
312701
|
} catch (error48) {
|
|
312578
312702
|
log.error(`Error fetching orders for ${resolvedSymbol} on ${cex3}:`, error48);
|
|
312579
312703
|
const message = getErrorMessage(error48);
|
|
312580
|
-
await
|
|
312704
|
+
await writeTerminalError({
|
|
312581
312705
|
data: JSON.stringify({
|
|
312582
312706
|
error: `Failed to fetch orders: ${message}`
|
|
312583
312707
|
}),
|
|
@@ -312588,26 +312712,27 @@ function createSubscribeHandler(deps) {
|
|
|
312588
312712
|
}
|
|
312589
312713
|
break;
|
|
312590
312714
|
default:
|
|
312591
|
-
await
|
|
312715
|
+
await writeTerminalError({
|
|
312592
312716
|
data: JSON.stringify({ error: "Invalid subscription type" }),
|
|
312593
312717
|
timestamp: Date.now(),
|
|
312594
312718
|
symbol: symbol2,
|
|
312595
312719
|
type: subscriptionType2
|
|
312596
|
-
});
|
|
312720
|
+
}, grpc13.status.INVALID_ARGUMENT);
|
|
312597
312721
|
}
|
|
312598
312722
|
} catch (error48) {
|
|
312599
312723
|
log.error("Error in Subscribe stream:", error48);
|
|
312600
312724
|
const message = getErrorMessage(error48);
|
|
312601
|
-
await
|
|
312725
|
+
await writeTerminalError({
|
|
312602
312726
|
data: JSON.stringify({ error: `Internal server error: ${message}` }),
|
|
312603
312727
|
timestamp: Date.now(),
|
|
312604
312728
|
symbol: "",
|
|
312605
312729
|
type: subscriptionType2
|
|
312606
|
-
});
|
|
312730
|
+
}, grpc13.status.INTERNAL);
|
|
312607
312731
|
} finally {
|
|
312608
312732
|
call.off("cancelled", closeOwnedBrokerOnCallEnd);
|
|
312609
312733
|
call.off("error", closeOwnedBrokerOnCallEnd);
|
|
312610
312734
|
await closeOwnedBroker();
|
|
312735
|
+
logTerminalOutcome(call.cancelled ? "cancelled" : terminalOutcome);
|
|
312611
312736
|
}
|
|
312612
312737
|
};
|
|
312613
312738
|
}
|
|
@@ -313101,12 +313226,7 @@ class CEXBroker {
|
|
|
313101
313226
|
if (this.brokerArchiver) {
|
|
313102
313227
|
await this.brokerArchiver.close();
|
|
313103
313228
|
}
|
|
313104
|
-
|
|
313105
|
-
await this.otelMetrics.close();
|
|
313106
|
-
}
|
|
313107
|
-
if (this.otelLogs) {
|
|
313108
|
-
await this.otelLogs.close();
|
|
313109
|
-
}
|
|
313229
|
+
await Promise.all([this.otelMetrics?.close(), this.otelLogs?.close()]);
|
|
313110
313230
|
}
|
|
313111
313231
|
async run() {
|
|
313112
313232
|
const marketArchiveState = resolveMarketCaptureArchiveState({
|
|
@@ -313224,4 +313344,4 @@ export {
|
|
|
313224
313344
|
CEXBroker as default
|
|
313225
313345
|
};
|
|
313226
313346
|
|
|
313227
|
-
//# debugId=
|
|
313347
|
+
//# debugId=B0B080620F355A6764756E2164756E21
|