@blaxel/telemetry 0.2.36-preview.80 → 0.2.36-preview.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/.tsbuildinfo +1 -0
- package/dist/esm/.tsbuildinfo +1 -0
- package/dist/esm/auth_refresh_exporters.js +193 -0
- package/dist/esm/index.js +10 -0
- package/dist/esm/json_logger.js +151 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/telemetry.js +274 -0
- package/dist/esm/telemetry_provider.js +58 -0
- package/package.json +17 -26
- /package/dist/{auth_refresh_exporters.js → cjs/auth_refresh_exporters.js} +0 -0
- /package/dist/{index.js → cjs/index.js} +0 -0
- /package/dist/{json_logger.js → cjs/json_logger.js} +0 -0
- /package/dist/{telemetry.js → cjs/telemetry.js} +0 -0
- /package/dist/{telemetry_provider.js → cjs/telemetry_provider.js} +0 -0
- /package/dist/{auth_refresh_exporters.d.ts → cjs/types/auth_refresh_exporters.d.ts} +0 -0
- /package/dist/{index.d.ts → cjs/types/index.d.ts} +0 -0
- /package/dist/{json_logger.d.ts → cjs/types/json_logger.d.ts} +0 -0
- /package/dist/{telemetry.d.ts → cjs/types/telemetry.d.ts} +0 -0
- /package/dist/{telemetry_provider.d.ts → cjs/types/telemetry_provider.d.ts} +0 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { authenticate, logger, settings } from "@blaxel/core";
|
|
2
|
+
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
|
|
3
|
+
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
|
4
|
+
/**
|
|
5
|
+
* SpanExporter that refreshes authentication before each export.
|
|
6
|
+
* This is necessary for long-running containers where tokens may expire.
|
|
7
|
+
*/
|
|
8
|
+
export class AuthRefreshingSpanExporter {
|
|
9
|
+
createExporter;
|
|
10
|
+
constructor(createExporter) {
|
|
11
|
+
this.createExporter = createExporter;
|
|
12
|
+
logger.debug("[AuthRefreshingSpanExporter] Initialized");
|
|
13
|
+
}
|
|
14
|
+
currentExporter = null;
|
|
15
|
+
export(spans, resultCallback) {
|
|
16
|
+
logger.debug(`[AuthRefreshingSpanExporter] Exporting ${spans.length} spans`);
|
|
17
|
+
// Execute async operations but return void as required by interface
|
|
18
|
+
this.doExport(spans, resultCallback).catch(error => {
|
|
19
|
+
logger.error("[AuthRefreshingSpanExporter] Fatal error in export:", error);
|
|
20
|
+
resultCallback({ code: 1, error: error });
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
async doExport(spans, resultCallback) {
|
|
24
|
+
try {
|
|
25
|
+
logger.debug("[AuthRefreshingSpanExporter] Starting authentication refresh");
|
|
26
|
+
const startTime = Date.now();
|
|
27
|
+
// Always refresh auth before export
|
|
28
|
+
await authenticate();
|
|
29
|
+
const authTime = Date.now() - startTime;
|
|
30
|
+
logger.debug(`[AuthRefreshingSpanExporter] Authentication completed in ${authTime}ms`);
|
|
31
|
+
// Log current auth status
|
|
32
|
+
if (settings.authorization) {
|
|
33
|
+
logger.debug("[AuthRefreshingSpanExporter] Authorization token is present");
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
logger.warn("[AuthRefreshingSpanExporter] No authorization token after authentication!");
|
|
37
|
+
}
|
|
38
|
+
logger.debug("[AuthRefreshingSpanExporter] Creating new exporter");
|
|
39
|
+
// Create new exporter with fresh headers
|
|
40
|
+
this.currentExporter = this.createExporter();
|
|
41
|
+
logger.debug("[AuthRefreshingSpanExporter] New exporter created with fresh auth headers");
|
|
42
|
+
// Export using the fresh exporter
|
|
43
|
+
if (this.currentExporter && this.currentExporter.export) {
|
|
44
|
+
logger.debug("[AuthRefreshingSpanExporter] Calling export on fresh exporter");
|
|
45
|
+
this.currentExporter.export(spans, resultCallback);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
const error = new Error('Exporter not initialized');
|
|
49
|
+
logger.error("[AuthRefreshingSpanExporter] Exporter not properly initialized", error);
|
|
50
|
+
resultCallback({ code: 1, error });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
logger.error("[AuthRefreshingSpanExporter] Error during authentication or export:", error);
|
|
55
|
+
logger.error("[AuthRefreshingSpanExporter] Error details:", {
|
|
56
|
+
message: error.message,
|
|
57
|
+
stack: error.stack
|
|
58
|
+
});
|
|
59
|
+
resultCallback({ code: 1, error: error });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
async shutdown() {
|
|
63
|
+
logger.debug("[AuthRefreshingSpanExporter] Shutting down");
|
|
64
|
+
if (this.currentExporter) {
|
|
65
|
+
return this.currentExporter.shutdown();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
async forceFlush() {
|
|
69
|
+
logger.debug("[AuthRefreshingSpanExporter] Force flushing");
|
|
70
|
+
if (this.currentExporter && this.currentExporter.forceFlush) {
|
|
71
|
+
return this.currentExporter.forceFlush();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* MetricExporter that refreshes authentication before each export.
|
|
77
|
+
* This is necessary for long-running containers where tokens may expire.
|
|
78
|
+
*/
|
|
79
|
+
export class AuthRefreshingMetricExporter {
|
|
80
|
+
createExporter;
|
|
81
|
+
constructor(createExporter) {
|
|
82
|
+
this.createExporter = createExporter;
|
|
83
|
+
logger.debug("[AuthRefreshingMetricExporter] Initialized");
|
|
84
|
+
}
|
|
85
|
+
currentExporter = null;
|
|
86
|
+
export(metrics, resultCallback) {
|
|
87
|
+
logger.debug("[AuthRefreshingMetricExporter] Exporting metrics");
|
|
88
|
+
// Execute async operations but return void
|
|
89
|
+
this.doExport(metrics, resultCallback).catch(error => {
|
|
90
|
+
logger.error("[AuthRefreshingMetricExporter] Fatal error in export:", error);
|
|
91
|
+
resultCallback({ code: 1, error: error });
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
async doExport(metrics, resultCallback) {
|
|
95
|
+
try {
|
|
96
|
+
logger.debug("[AuthRefreshingMetricExporter] Starting authentication refresh");
|
|
97
|
+
const startTime = Date.now();
|
|
98
|
+
// Always refresh auth before export
|
|
99
|
+
await authenticate();
|
|
100
|
+
const authTime = Date.now() - startTime;
|
|
101
|
+
logger.debug(`[AuthRefreshingMetricExporter] Authentication completed in ${authTime}ms`);
|
|
102
|
+
// Log current auth status
|
|
103
|
+
if (settings.authorization) {
|
|
104
|
+
logger.debug("[AuthRefreshingMetricExporter] Authorization token is present");
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
logger.warn("[AuthRefreshingMetricExporter] No authorization token after authentication!");
|
|
108
|
+
}
|
|
109
|
+
logger.debug("[AuthRefreshingMetricExporter] Creating new exporter");
|
|
110
|
+
// Create new exporter with fresh headers
|
|
111
|
+
this.currentExporter = this.createExporter();
|
|
112
|
+
logger.debug("[AuthRefreshingMetricExporter] New exporter created with fresh auth headers");
|
|
113
|
+
// Export using the fresh exporter
|
|
114
|
+
if (this.currentExporter && this.currentExporter.export) {
|
|
115
|
+
logger.debug("[AuthRefreshingMetricExporter] Calling export on fresh exporter");
|
|
116
|
+
this.currentExporter.export(metrics, resultCallback);
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
const error = new Error('Exporter not initialized');
|
|
120
|
+
logger.error("[AuthRefreshingMetricExporter] Exporter not properly initialized", error);
|
|
121
|
+
resultCallback({ code: 1, error });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
logger.error("[AuthRefreshingMetricExporter] Error during authentication or export:", error);
|
|
126
|
+
logger.error("[AuthRefreshingMetricExporter] Error details:", {
|
|
127
|
+
message: error.message,
|
|
128
|
+
stack: error.stack
|
|
129
|
+
});
|
|
130
|
+
resultCallback({ code: 1, error: error });
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
async shutdown() {
|
|
134
|
+
logger.debug("[AuthRefreshingMetricExporter] Shutting down");
|
|
135
|
+
if (this.currentExporter) {
|
|
136
|
+
return this.currentExporter.shutdown();
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
async forceFlush() {
|
|
140
|
+
logger.debug("[AuthRefreshingMetricExporter] Force flushing");
|
|
141
|
+
if (this.currentExporter && this.currentExporter.forceFlush) {
|
|
142
|
+
return this.currentExporter.forceFlush();
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Creates an OTLP Trace Exporter with the current auth headers
|
|
148
|
+
*/
|
|
149
|
+
export function createTraceExporter() {
|
|
150
|
+
const headers = {};
|
|
151
|
+
if (settings.authorization) {
|
|
152
|
+
headers["x-blaxel-authorization"] = settings.authorization;
|
|
153
|
+
logger.debug("[createTraceExporter] Added authorization header");
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
logger.warn("[createTraceExporter] No authorization available");
|
|
157
|
+
}
|
|
158
|
+
if (settings.workspace) {
|
|
159
|
+
headers["x-blaxel-workspace"] = settings.workspace;
|
|
160
|
+
logger.debug(`[createTraceExporter] Added workspace header: ${settings.workspace}`);
|
|
161
|
+
}
|
|
162
|
+
else {
|
|
163
|
+
logger.warn("[createTraceExporter] No workspace available");
|
|
164
|
+
}
|
|
165
|
+
logger.debug("[createTraceExporter] Creating OTLPTraceExporter with headers:", Object.keys(headers));
|
|
166
|
+
return new OTLPTraceExporter({
|
|
167
|
+
headers,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Creates an OTLP Metric Exporter with the current auth headers
|
|
172
|
+
*/
|
|
173
|
+
export function createMetricExporter() {
|
|
174
|
+
const headers = {};
|
|
175
|
+
if (settings.authorization) {
|
|
176
|
+
headers["x-blaxel-authorization"] = settings.authorization;
|
|
177
|
+
logger.debug("[createMetricExporter] Added authorization header");
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
logger.warn("[createMetricExporter] No authorization available");
|
|
181
|
+
}
|
|
182
|
+
if (settings.workspace) {
|
|
183
|
+
headers["x-blaxel-workspace"] = settings.workspace;
|
|
184
|
+
logger.debug(`[createMetricExporter] Added workspace header: ${settings.workspace}`);
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
logger.warn("[createMetricExporter] No workspace available");
|
|
188
|
+
}
|
|
189
|
+
logger.debug("[createMetricExporter] Creating OTLPMetricExporter with headers:", Object.keys(headers));
|
|
190
|
+
return new OTLPMetricExporter({
|
|
191
|
+
headers,
|
|
192
|
+
});
|
|
193
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { settings } from "@blaxel/core";
|
|
2
|
+
import { setJsonLogger } from "./json_logger.js";
|
|
3
|
+
import { blaxelTelemetry } from "./telemetry.js";
|
|
4
|
+
blaxelTelemetry.initialize();
|
|
5
|
+
if (settings.loggerType === "json") {
|
|
6
|
+
setJsonLogger();
|
|
7
|
+
}
|
|
8
|
+
export { AuthRefreshingMetricExporter, AuthRefreshingSpanExporter, createMetricExporter, createTraceExporter } from "./auth_refresh_exporters.js";
|
|
9
|
+
export { setJsonLogger } from "./json_logger.js";
|
|
10
|
+
export { blaxelTelemetry };
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/* eslint-disable no-console */
|
|
2
|
+
import { env, stringify } from '@blaxel/core';
|
|
3
|
+
import { trace } from '@opentelemetry/api';
|
|
4
|
+
export function setJsonLogger() {
|
|
5
|
+
console.debug = (message, ...args) => {
|
|
6
|
+
const msg = formatLogMessage("DEBUG", message, args);
|
|
7
|
+
originalLogger.log(msg);
|
|
8
|
+
};
|
|
9
|
+
console.log = (message, ...args) => {
|
|
10
|
+
const msg = formatLogMessage("INFO", message, args);
|
|
11
|
+
originalLogger.log(msg);
|
|
12
|
+
};
|
|
13
|
+
console.info = (message, ...args) => {
|
|
14
|
+
const msg = formatLogMessage("INFO", message, args);
|
|
15
|
+
originalLogger.log(msg);
|
|
16
|
+
};
|
|
17
|
+
console.warn = (message, ...args) => {
|
|
18
|
+
const msg = formatLogMessage("WARN", message, args);
|
|
19
|
+
originalLogger.log(msg);
|
|
20
|
+
};
|
|
21
|
+
console.error = (message, ...args) => {
|
|
22
|
+
const msg = formatLogMessage("ERROR", message, args);
|
|
23
|
+
originalLogger.log(msg);
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export const originalLogger = {
|
|
27
|
+
info: console.info,
|
|
28
|
+
error: console.error,
|
|
29
|
+
warn: console.warn,
|
|
30
|
+
debug: console.debug,
|
|
31
|
+
log: console.log,
|
|
32
|
+
};
|
|
33
|
+
const traceIdName = env.BL_LOGGER_TRACE_ID || 'trace_id';
|
|
34
|
+
const spanIdName = env.BL_LOGGER_SPAN_ID || 'span_id';
|
|
35
|
+
const labelsName = env.BL_LOGGER_LABELS || 'labels';
|
|
36
|
+
const traceIdPrefix = env.BL_LOGGER_TRACE_ID_PREFIX || '';
|
|
37
|
+
const spanIdPrefix = env.BL_LOGGER_SPAN_ID_PREFIX || '';
|
|
38
|
+
const taskIndex = env.BL_TASK_KEY || 'TASK_INDEX';
|
|
39
|
+
const taskPrefix = env.BL_TASK_PREFIX || '';
|
|
40
|
+
const executionKey = env.BL_EXECUTION_KEY || 'BL_EXECUTION_ID';
|
|
41
|
+
const executionPrefix = env.BL_EXECUTION_PREFIX || '';
|
|
42
|
+
// Validate environment variables to prevent issues
|
|
43
|
+
function validateEnvVar(value, defaultValue, varName) {
|
|
44
|
+
if (!value || value.trim() === '') {
|
|
45
|
+
originalLogger.warn(`Warning: ${varName} environment variable is empty, using default: ${defaultValue}`);
|
|
46
|
+
return defaultValue;
|
|
47
|
+
}
|
|
48
|
+
return value;
|
|
49
|
+
}
|
|
50
|
+
const validatedLabelsName = validateEnvVar(labelsName, 'labels', 'BL_LOGGER_LABELS');
|
|
51
|
+
// Enhanced error serialization to capture all properties
|
|
52
|
+
function serializeError(error) {
|
|
53
|
+
const serialized = {
|
|
54
|
+
message: error.message,
|
|
55
|
+
name: error.name,
|
|
56
|
+
stack: error.stack
|
|
57
|
+
};
|
|
58
|
+
// Capture any additional properties on the error object
|
|
59
|
+
for (const key of Object.keys(error)) {
|
|
60
|
+
if (!(key in serialized)) {
|
|
61
|
+
try {
|
|
62
|
+
const value = error[key];
|
|
63
|
+
// Avoid circular references by limiting depth
|
|
64
|
+
serialized[key] = typeof value === 'object' ? stringify(value, 2) : value;
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
serialized[key] = '[Unserializable]';
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return serialized;
|
|
72
|
+
}
|
|
73
|
+
// Enhanced stringify function with better error handling
|
|
74
|
+
function enhancedStringify(obj, maxDepth = 2) {
|
|
75
|
+
if (obj instanceof Error) {
|
|
76
|
+
return JSON.stringify(serializeError(obj));
|
|
77
|
+
}
|
|
78
|
+
// Handle circular references by using a simple set to track seen objects
|
|
79
|
+
const seen = new WeakSet();
|
|
80
|
+
const stringifyWithCircularCheck = (value, depth = 0) => {
|
|
81
|
+
if (value === null || value === undefined)
|
|
82
|
+
return value;
|
|
83
|
+
if (typeof value !== 'object')
|
|
84
|
+
return value;
|
|
85
|
+
if (seen.has(value)) {
|
|
86
|
+
return '[Circular Reference]';
|
|
87
|
+
}
|
|
88
|
+
if (depth >= maxDepth) {
|
|
89
|
+
return Array.isArray(value) ? '[Array]' : '[Object]';
|
|
90
|
+
}
|
|
91
|
+
seen.add(value);
|
|
92
|
+
if (Array.isArray(value)) {
|
|
93
|
+
return value.map(item => stringifyWithCircularCheck(item, depth + 1));
|
|
94
|
+
}
|
|
95
|
+
const result = {};
|
|
96
|
+
for (const [key, val] of Object.entries(value)) {
|
|
97
|
+
result[key] = stringifyWithCircularCheck(val, depth + 1);
|
|
98
|
+
}
|
|
99
|
+
return result;
|
|
100
|
+
};
|
|
101
|
+
try {
|
|
102
|
+
const processed = stringifyWithCircularCheck(obj);
|
|
103
|
+
return JSON.stringify(processed);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return stringify(obj, maxDepth);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
// Format a log message with appropriate color and prefix
|
|
110
|
+
function formatLogMessage(severity, message, args) {
|
|
111
|
+
const messageStr = typeof message === "string" ? message : enhancedStringify(message, 2);
|
|
112
|
+
const argsStr = args.map(arg => typeof arg === "string" ? arg : enhancedStringify(arg, 2)).join(" ");
|
|
113
|
+
const msg = `${messageStr}${argsStr ? " " + argsStr : ""}`;
|
|
114
|
+
const logEntry = {
|
|
115
|
+
message: msg,
|
|
116
|
+
severity
|
|
117
|
+
};
|
|
118
|
+
logEntry[validatedLabelsName] = {};
|
|
119
|
+
const currentSpan = trace.getActiveSpan();
|
|
120
|
+
if (currentSpan) {
|
|
121
|
+
const { traceId, spanId } = currentSpan.spanContext();
|
|
122
|
+
logEntry[traceIdName] = `${traceIdPrefix}${traceId}`;
|
|
123
|
+
logEntry[spanIdName] = `${spanIdPrefix}${spanId}`;
|
|
124
|
+
}
|
|
125
|
+
const taskId = env[taskIndex] || null;
|
|
126
|
+
if (taskId) {
|
|
127
|
+
logEntry[validatedLabelsName]['blaxel-task'] = `${taskPrefix}${taskId}`;
|
|
128
|
+
}
|
|
129
|
+
const executionId = env[executionKey] || null;
|
|
130
|
+
if (executionId) {
|
|
131
|
+
logEntry[validatedLabelsName]['blaxel-execution'] = `${executionPrefix}${executionId.split('-').pop()}`;
|
|
132
|
+
}
|
|
133
|
+
try {
|
|
134
|
+
return JSON.stringify(logEntry);
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
// Fallback for serialization errors
|
|
138
|
+
const fallbackEntry = {
|
|
139
|
+
message: `JSON serialization failed: ${msg}`,
|
|
140
|
+
severity,
|
|
141
|
+
error: error instanceof Error ? error.message : String(error)
|
|
142
|
+
};
|
|
143
|
+
try {
|
|
144
|
+
return JSON.stringify(fallbackEntry);
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
// Last resort fallback
|
|
148
|
+
return `{"message":"${severity}: ${msg.replace(/"/g, '\\"')}","severity":"${severity}","error":"Failed to serialize log entry"}`;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"type":"module"}
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import { authenticate, env, logger, settings, telemetryRegistry } from "@blaxel/core";
|
|
2
|
+
import { metrics, trace } from "@opentelemetry/api";
|
|
3
|
+
import { registerInstrumentations } from "@opentelemetry/instrumentation";
|
|
4
|
+
import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
|
|
5
|
+
import { envDetector } from "@opentelemetry/resources";
|
|
6
|
+
import { MeterProvider, PeriodicExportingMetricReader, } from "@opentelemetry/sdk-metrics";
|
|
7
|
+
import { AlwaysOnSampler, BatchSpanProcessor, NodeTracerProvider, } from "@opentelemetry/sdk-trace-node";
|
|
8
|
+
import { AuthRefreshingMetricExporter, AuthRefreshingSpanExporter, createMetricExporter, createTraceExporter } from "./auth_refresh_exporters.js";
|
|
9
|
+
import { OtelTelemetryProvider } from "./telemetry_provider.js";
|
|
10
|
+
export class BlaxelResource {
|
|
11
|
+
attributes;
|
|
12
|
+
constructor(attributes) {
|
|
13
|
+
this.attributes = attributes;
|
|
14
|
+
}
|
|
15
|
+
merge(other) {
|
|
16
|
+
if (other?.attributes) {
|
|
17
|
+
for (const [key, value] of Object.entries(other.attributes)) {
|
|
18
|
+
if (typeof value === "string") {
|
|
19
|
+
this.attributes[key] = value;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return this;
|
|
24
|
+
}
|
|
25
|
+
getRawAttributes() {
|
|
26
|
+
return Object.entries(this.attributes).map(([key, value]) => [key, value]);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export class DefaultAttributesSpanProcessor {
|
|
30
|
+
defaultAttributes;
|
|
31
|
+
constructor(defaultAttributes) {
|
|
32
|
+
this.defaultAttributes = defaultAttributes;
|
|
33
|
+
}
|
|
34
|
+
onStart(span) {
|
|
35
|
+
Object.entries(this.defaultAttributes).forEach(([key, value]) => {
|
|
36
|
+
span.setAttribute(key, value);
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
onEnd() { }
|
|
40
|
+
shutdown() {
|
|
41
|
+
return Promise.resolve();
|
|
42
|
+
}
|
|
43
|
+
forceFlush() {
|
|
44
|
+
return Promise.resolve();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
class HasBeenProcessedSpanProcessor extends BatchSpanProcessor {
|
|
48
|
+
constructor(exporter, config) {
|
|
49
|
+
super(exporter, config);
|
|
50
|
+
}
|
|
51
|
+
onEnd(span) {
|
|
52
|
+
super.onEnd(span);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
class TelemetryManager {
|
|
56
|
+
nodeTracerProvider;
|
|
57
|
+
meterProvider;
|
|
58
|
+
initialized;
|
|
59
|
+
configured;
|
|
60
|
+
shuttingDown;
|
|
61
|
+
constructor() {
|
|
62
|
+
this.nodeTracerProvider = null;
|
|
63
|
+
this.meterProvider = null;
|
|
64
|
+
this.initialized = false;
|
|
65
|
+
this.configured = false;
|
|
66
|
+
this.shuttingDown = false;
|
|
67
|
+
}
|
|
68
|
+
// This method need to stay sync to avoid non booted instrumentations
|
|
69
|
+
initialize() {
|
|
70
|
+
if (!this.enabled || this.initialized) {
|
|
71
|
+
logger.debug(`[TelemetryManager] Initialize skipped - enabled: ${this.enabled}, initialized: ${this.initialized}`);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
logger.debug("[TelemetryManager] Starting telemetry initialization");
|
|
75
|
+
this.instrumentApp();
|
|
76
|
+
this.setupSignalHandler();
|
|
77
|
+
this.initialized = true;
|
|
78
|
+
logger.debug("[TelemetryManager] Telemetry initialized, setting configuration async");
|
|
79
|
+
this.setConfiguration().catch((error) => {
|
|
80
|
+
logger.error("[TelemetryManager] Error setting configuration:", error);
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
async setConfiguration() {
|
|
84
|
+
if (!this.enabled || this.configured) {
|
|
85
|
+
logger.debug(`[TelemetryManager] SetConfiguration skipped - enabled: ${this.enabled}, configured: ${this.configured}`);
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
logger.debug("[TelemetryManager] Starting authentication for telemetry configuration");
|
|
89
|
+
await authenticate();
|
|
90
|
+
logger.debug("[TelemetryManager] Authentication completed, setting up exporters");
|
|
91
|
+
this.setExporters();
|
|
92
|
+
logger.debug("[TelemetryManager] Telemetry configuration complete");
|
|
93
|
+
this.configured = true;
|
|
94
|
+
}
|
|
95
|
+
get tracer() {
|
|
96
|
+
return trace.getTracer("blaxel");
|
|
97
|
+
}
|
|
98
|
+
get enabled() {
|
|
99
|
+
return env.BL_ENABLE_OPENTELEMETRY === "true";
|
|
100
|
+
}
|
|
101
|
+
get authHeaders() {
|
|
102
|
+
const headers = {};
|
|
103
|
+
if (settings.authorization) {
|
|
104
|
+
headers["x-blaxel-authorization"] = settings.authorization;
|
|
105
|
+
}
|
|
106
|
+
if (settings.workspace) {
|
|
107
|
+
headers["x-blaxel-workspace"] = settings.workspace;
|
|
108
|
+
}
|
|
109
|
+
return headers;
|
|
110
|
+
}
|
|
111
|
+
async sleep(ms) {
|
|
112
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
113
|
+
}
|
|
114
|
+
async flush() {
|
|
115
|
+
if (this.nodeTracerProvider) {
|
|
116
|
+
await this.nodeTracerProvider.shutdown();
|
|
117
|
+
}
|
|
118
|
+
if (this.meterProvider) {
|
|
119
|
+
await this.meterProvider.shutdown();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
setupSignalHandler() {
|
|
123
|
+
const forward = (sig) => {
|
|
124
|
+
if (this.shuttingDown)
|
|
125
|
+
return;
|
|
126
|
+
this.shuttingDown = true;
|
|
127
|
+
this.shutdownApp()
|
|
128
|
+
.catch((error) => {
|
|
129
|
+
logger.debug("Fatal error during shutdown:", error);
|
|
130
|
+
})
|
|
131
|
+
.finally(() => {
|
|
132
|
+
try {
|
|
133
|
+
// Re-send the original signal to let the default handler terminate the process with the correct code
|
|
134
|
+
process.kill(process.pid, sig);
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
process.exit(1);
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
};
|
|
141
|
+
process.once("SIGINT", () => forward("SIGINT"));
|
|
142
|
+
process.once("SIGTERM", () => forward("SIGTERM"));
|
|
143
|
+
process.once("uncaughtException", (err) => {
|
|
144
|
+
logger.error("Uncaught exception:", err);
|
|
145
|
+
this.shutdownApp().catch((error) => {
|
|
146
|
+
logger.debug("Fatal error during shutdown:", error);
|
|
147
|
+
}).finally(() => {
|
|
148
|
+
// Re-throw on next tick so Node's default handler sets exit code and prints stack
|
|
149
|
+
setImmediate(() => {
|
|
150
|
+
throw (err instanceof Error ? err : new Error(String(err)));
|
|
151
|
+
});
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Get resource attributes for OpenTelemetry.
|
|
157
|
+
*/
|
|
158
|
+
get resourceAttributes() {
|
|
159
|
+
const resource = envDetector.detect();
|
|
160
|
+
const attributes = resource.attributes || {};
|
|
161
|
+
if (settings.name) {
|
|
162
|
+
attributes["service.name"] = settings.name;
|
|
163
|
+
attributes["workload.id"] = settings.name;
|
|
164
|
+
}
|
|
165
|
+
if (settings.workspace) {
|
|
166
|
+
attributes["workspace"] = settings.workspace;
|
|
167
|
+
}
|
|
168
|
+
if (settings.type) {
|
|
169
|
+
attributes["workload.type"] = settings.type + "s";
|
|
170
|
+
}
|
|
171
|
+
// Only keep string values
|
|
172
|
+
const stringAttrs = {};
|
|
173
|
+
for (const [k, v] of Object.entries(attributes)) {
|
|
174
|
+
if (typeof v === "string")
|
|
175
|
+
stringAttrs[k] = v;
|
|
176
|
+
}
|
|
177
|
+
return stringAttrs;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Initialize and return the OTLP Metric Exporter.
|
|
181
|
+
*/
|
|
182
|
+
getMetricExporter() {
|
|
183
|
+
return createMetricExporter();
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Initialize and return the OTLP Trace Exporter.
|
|
187
|
+
*/
|
|
188
|
+
getTraceExporter() {
|
|
189
|
+
return createTraceExporter();
|
|
190
|
+
}
|
|
191
|
+
instrumentApp() {
|
|
192
|
+
telemetryRegistry.registerProvider(new OtelTelemetryProvider());
|
|
193
|
+
const httpInstrumentation = new HttpInstrumentation({
|
|
194
|
+
requireParentforOutgoingSpans: true,
|
|
195
|
+
});
|
|
196
|
+
registerInstrumentations({
|
|
197
|
+
instrumentations: [httpInstrumentation],
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
setExporters() {
|
|
201
|
+
const resource = new BlaxelResource(this.resourceAttributes);
|
|
202
|
+
logger.debug("[TelemetryManager] Setting up exporters with authentication refresh");
|
|
203
|
+
// Configure batch processor options with 1-second delay
|
|
204
|
+
const batchProcessorOptions = {
|
|
205
|
+
scheduledDelayMillis: 1000, // Export every 1 second
|
|
206
|
+
exportTimeoutMillis: 5000, // Timeout for export
|
|
207
|
+
maxExportBatchSize: 512, // Max batch size
|
|
208
|
+
maxQueueSize: 2048 // Max queue size
|
|
209
|
+
};
|
|
210
|
+
logger.debug("[TelemetryManager] Batch processor options:", batchProcessorOptions);
|
|
211
|
+
// Create auth-refreshing trace exporter
|
|
212
|
+
const traceExporter = new AuthRefreshingSpanExporter(() => this.getTraceExporter());
|
|
213
|
+
logger.debug("[TelemetryManager] Created AuthRefreshingSpanExporter");
|
|
214
|
+
this.nodeTracerProvider = new NodeTracerProvider({
|
|
215
|
+
resource,
|
|
216
|
+
sampler: new AlwaysOnSampler(),
|
|
217
|
+
spanProcessors: [
|
|
218
|
+
new DefaultAttributesSpanProcessor({
|
|
219
|
+
"workload.id": settings.name || "",
|
|
220
|
+
"workload.type": settings.type ? settings.type + "s" : "",
|
|
221
|
+
workspace: settings.workspace || "",
|
|
222
|
+
}),
|
|
223
|
+
new BatchSpanProcessor(traceExporter, batchProcessorOptions),
|
|
224
|
+
new HasBeenProcessedSpanProcessor(traceExporter, batchProcessorOptions),
|
|
225
|
+
],
|
|
226
|
+
});
|
|
227
|
+
this.nodeTracerProvider.register();
|
|
228
|
+
logger.debug("[TelemetryManager] Trace provider registered");
|
|
229
|
+
// Create auth-refreshing metric exporter
|
|
230
|
+
const metricExporter = new AuthRefreshingMetricExporter(() => this.getMetricExporter());
|
|
231
|
+
logger.debug("[TelemetryManager] Created AuthRefreshingMetricExporter");
|
|
232
|
+
this.meterProvider = new MeterProvider({
|
|
233
|
+
resource,
|
|
234
|
+
readers: [
|
|
235
|
+
new PeriodicExportingMetricReader({
|
|
236
|
+
exporter: metricExporter,
|
|
237
|
+
exportIntervalMillis: 1000, // Changed from 60000 to 1000 (1 second)
|
|
238
|
+
}),
|
|
239
|
+
],
|
|
240
|
+
});
|
|
241
|
+
metrics.setGlobalMeterProvider(this.meterProvider);
|
|
242
|
+
logger.debug("[TelemetryManager] Metric provider configured with 1-second export interval");
|
|
243
|
+
}
|
|
244
|
+
async shutdownApp() {
|
|
245
|
+
try {
|
|
246
|
+
const maxSleepTime = 5000;
|
|
247
|
+
const startTime = Date.now();
|
|
248
|
+
while (!this.configured && Date.now() - startTime < maxSleepTime) {
|
|
249
|
+
await this.sleep(100);
|
|
250
|
+
}
|
|
251
|
+
const shutdownPromises = [];
|
|
252
|
+
if (this.nodeTracerProvider) {
|
|
253
|
+
shutdownPromises.push(this.nodeTracerProvider
|
|
254
|
+
.shutdown()
|
|
255
|
+
.catch((error) => logger.debug("Error shutting down tracer provider:", error)));
|
|
256
|
+
}
|
|
257
|
+
if (this.meterProvider) {
|
|
258
|
+
shutdownPromises.push(this.meterProvider
|
|
259
|
+
.shutdown()
|
|
260
|
+
.catch((error) => logger.debug("Error shutting down meter provider:", error)));
|
|
261
|
+
}
|
|
262
|
+
// Wait for all providers to shutdown with a timeout
|
|
263
|
+
await Promise.race([
|
|
264
|
+
Promise.all(shutdownPromises),
|
|
265
|
+
new Promise((resolve) => setTimeout(resolve, 5000)), // 5 second timeout
|
|
266
|
+
]);
|
|
267
|
+
logger.debug("Instrumentation shutdown complete");
|
|
268
|
+
}
|
|
269
|
+
catch (error) {
|
|
270
|
+
logger.error("Error during shutdown:", error);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
export const blaxelTelemetry = new TelemetryManager();
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { context as otelContext, ROOT_CONTEXT, SpanStatusCode, trace } from "@opentelemetry/api";
|
|
2
|
+
import { blaxelTelemetry } from "./telemetry.js";
|
|
3
|
+
class OtelSpan {
|
|
4
|
+
span;
|
|
5
|
+
closed = false;
|
|
6
|
+
constructor(span) {
|
|
7
|
+
this.span = span;
|
|
8
|
+
}
|
|
9
|
+
setAttribute(key, value) {
|
|
10
|
+
this.span.setAttribute(key, value);
|
|
11
|
+
}
|
|
12
|
+
setAttributes(attributes) {
|
|
13
|
+
Object.entries(attributes).forEach(([k, v]) => this.span.setAttribute(k, v));
|
|
14
|
+
}
|
|
15
|
+
recordException(error) {
|
|
16
|
+
this.span.recordException(error);
|
|
17
|
+
this.span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
|
|
18
|
+
}
|
|
19
|
+
setStatus(status, message) {
|
|
20
|
+
this.span.setStatus({
|
|
21
|
+
code: status === 'ok' ? SpanStatusCode.OK : SpanStatusCode.ERROR,
|
|
22
|
+
message,
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
end() {
|
|
26
|
+
this.closed = true;
|
|
27
|
+
this.span.end();
|
|
28
|
+
}
|
|
29
|
+
getContext() {
|
|
30
|
+
return this.span.spanContext();
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
export class OtelTelemetryProvider {
|
|
34
|
+
spans = [];
|
|
35
|
+
retrieveActiveSpanContext() {
|
|
36
|
+
for (let i = this.spans.length - 1; i >= 0; i--) {
|
|
37
|
+
const span = this.spans[i];
|
|
38
|
+
if (!span.closed) {
|
|
39
|
+
return trace.setSpanContext(ROOT_CONTEXT, span.getContext());
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return otelContext.active();
|
|
43
|
+
}
|
|
44
|
+
startSpan(name, options) {
|
|
45
|
+
const tracer = trace.getTracer("blaxel");
|
|
46
|
+
const otelOptions = {
|
|
47
|
+
attributes: options?.attributes,
|
|
48
|
+
root: options?.isRoot,
|
|
49
|
+
};
|
|
50
|
+
const ctx = this.retrieveActiveSpanContext();
|
|
51
|
+
const span = new OtelSpan(tracer.startSpan(name, otelOptions, ctx));
|
|
52
|
+
this.spans.push(span);
|
|
53
|
+
return span;
|
|
54
|
+
}
|
|
55
|
+
async flush() {
|
|
56
|
+
await blaxelTelemetry.flush();
|
|
57
|
+
}
|
|
58
|
+
}
|