@blaxel/telemetry 0.2.17-dev.119 → 0.2.17-dev.120

package/dist/telemetry.d.ts

@@ -37,8 +37,10 @@ declare class TelemetryManager {
     setConfiguration(): Promise<void>;
     get tracer(): import("@opentelemetry/api").Tracer;
     get enabled(): boolean;
+    get isLambdaEnvironment(): boolean;
     get authHeaders(): Record<string, string>;
     sleep(ms: number): Promise<unknown>;
+    retryContextExtraction(headers: Record<string, string | string[]>, span: Span, traceparentParts: string[]): Promise<void>;
     flush(): Promise<void>;
     getLogger(): Promise<Logger>;
     setupSignalHandler(): void;
@@ -58,8 +60,7 @@ declare class TelemetryManager {
      * Initialize and return the OTLP Log Exporter.
      */
     getLogExporter(): OTLPLogExporter;
-    setupTracerProvider(): void;
-    instrumentApp(): void;
+    instrumentAppAsync(): Promise<void>;
     setExporters(): void;
     shutdownApp(): Promise<void>;
 }

package/dist/telemetry.js

@@ -13,9 +13,6 @@ const resources_1 = require("@opentelemetry/resources");
 const sdk_logs_1 = require("@opentelemetry/sdk-logs");
 const sdk_metrics_1 = require("@opentelemetry/sdk-metrics");
 const sdk_trace_node_1 = require("@opentelemetry/sdk-trace-node");
-// W3CTraceContextPropagator import - trying different paths
-// import { W3CTraceContextPropagator } from "@opentelemetry/core";
-// import { W3CTraceContextPropagator } from "@opentelemetry/sdk-trace-base";
 const telemetry_provider_1 = require("./telemetry_provider");
 class BlaxelResource {
     attributes;
@@ -81,7 +78,9 @@ class TelemetryManager {
         if (!this.enabled || this.initialized) {
             return;
         }
-        this.instrumentApp();
+        this.instrumentAppAsync().catch((error) => {
+            core_1.logger.error("Error during async instrumentation setup:", error);
+        });
         this.setupSignalHandler();
         this.initialized = true;
         this.setConfiguration().catch((error) => {
@@ -104,6 +103,9 @@ class TelemetryManager {
     get enabled() {
         return core_1.env.BL_ENABLE_OPENTELEMETRY === "true";
     }
+    get isLambdaEnvironment() {
+        return !!(process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.LAMBDA_TASK_ROOT);
+    }
     get authHeaders() {
         const headers = {};
         if (core_1.settings.authorization) {
@@ -117,6 +119,64 @@ class TelemetryManager {
     async sleep(ms) {
         return new Promise((resolve) => setTimeout(resolve, ms));
     }
+    async retryContextExtraction(headers, span, traceparentParts) {
+        // More aggressive retries in Lambda environment
+        const maxRetries = this.isLambdaEnvironment ? 5 : 3;
+        const retryDelay = this.isLambdaEnvironment ? 100 : 50; // Longer delays for Lambda cold starts
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+            try {
+                core_1.logger.debug(`Context extraction attempt ${attempt}/${maxRetries}`);
+                // Check if propagation is ready
+                const propagationFields = api_1.propagation.fields();
+                core_1.logger.debug("Current propagation fields:", propagationFields);
+                if (propagationFields.length === 0 && attempt < maxRetries) {
+                    core_1.logger.debug(`Propagation not ready on attempt ${attempt}, waiting ${retryDelay * attempt}ms...`);
+                    await this.sleep(retryDelay * attempt);
+                    continue;
+                }
+                // Try to extract trace context
+                const extractedContext = api_1.propagation.extract(api_1.context.active(), headers);
+                const extractedSpan = api_1.trace.getSpan(extractedContext);
+                if (extractedSpan) {
+                    const extractedSpanContext = extractedSpan.spanContext();
+                    core_1.logger.debug("Manual context extraction succeeded on attempt " + attempt + ":", JSON.stringify({
+                        traceId: extractedSpanContext.traceId,
+                        spanId: extractedSpanContext.spanId,
+                        traceFlags: extractedSpanContext.traceFlags,
+                    }));
+                    return; // Success!
+                }
+                if (attempt < maxRetries) {
+                    core_1.logger.debug(`Context extraction failed on attempt ${attempt}, retrying...`);
+                    await this.sleep(retryDelay * attempt);
+                }
+            }
+            catch (error) {
+                core_1.logger.debug(`Error on context extraction attempt ${attempt}:`, error);
+                if (attempt < maxRetries) {
+                    await this.sleep(retryDelay * attempt);
+                }
+            }
+        }
+        // All retries failed - add fallback attributes
+        core_1.logger.debug("All context extraction attempts failed, adding fallback attributes");
+        if (traceparentParts.length === 4 &&
+            traceparentParts[1] !== "00000000000000000000000000000000") {
+            try {
+                span.setAttributes({
+                    "trace.parent.trace_id": traceparentParts[1],
+                    "trace.parent.span_id": traceparentParts[2],
+                    "trace.parent.trace_flags": traceparentParts[3],
+                    "trace.extraction.failed": "true",
+                    "trace.extraction.reason": "lambda_cold_start",
+                });
+                core_1.logger.debug("Added fallback parent trace info as span attributes");
+            }
+            catch (error) {
+                core_1.logger.debug("Error setting fallback trace attributes:", error);
+            }
+        }
+    }
     async flush() {
         if (this.nodeTracerProvider) {
             await this.nodeTracerProvider.shutdown();
@@ -194,23 +254,24 @@ class TelemetryManager {
             headers: this.authHeaders,
         });
     }
-    setupTracerProvider() {
-        if (this.nodeTracerProvider) {
-            return; // Already set up
+    async instrumentAppAsync() {
+        core_1.logger.debug("Available propagation fields before setup:", api_1.propagation.fields());
+        if (this.isLambdaEnvironment) {
+            core_1.logger.debug("Lambda environment detected - using enhanced cold start handling");
         }
-        const resource = new BlaxelResource(this.resourceAttributes);
-        this.nodeTracerProvider = new sdk_trace_node_1.NodeTracerProvider({
-            resource,
+        // Note: W3C propagator should be auto-registered by NodeTracerProvider
+        core_1.logger.debug("Will check propagation after tracer registration...");
+        core_1.telemetryRegistry.registerProvider(new telemetry_provider_1.OtelTelemetryProvider());
+        // Set up a basic tracer provider early to enable propagation
+        const earlyResource = new BlaxelResource(this.resourceAttributes);
+        const earlyTracerProvider = new sdk_trace_node_1.NodeTracerProvider({
+            resource: earlyResource,
             sampler: new sdk_trace_node_1.AlwaysOnSampler(),
         });
-        this.nodeTracerProvider.register();
-        core_1.logger.debug("NodeTracerProvider registered early - propagation fields:", api_1.propagation.fields());
-    }
-    instrumentApp() {
-        core_1.logger.debug("Available propagation fields before setup:", api_1.propagation.fields());
-        // Set up tracer provider FIRST before instrumentation
-        this.setupTracerProvider();
-        core_1.telemetryRegistry.registerProvider(new telemetry_provider_1.OtelTelemetryProvider());
+        earlyTracerProvider.register();
+        // Small delay to ensure propagation is ready (especially important for Lambda cold starts)
+        await this.sleep(10);
+        core_1.logger.debug("Early tracer provider registered, checking propagation:", api_1.propagation.fields());
         const httpInstrumentation = new instrumentation_http_1.HttpInstrumentation({
             requireParentforOutgoingSpans: true,
             requireParentforIncomingSpans: false, // Allow root spans for incoming requests
@@ -254,21 +315,8 @@ class TelemetryManager {
                                     core_1.logger.debug("Traceparent contains invalid IDs");
                                 }
                             }
-                            // Extract trace context manually to see what should be extracted
-                            const extractedContext = api_1.propagation.extract(api_1.context.active(), headers);
-                            const extractedSpan = api_1.trace.getSpan(extractedContext);
-                            if (extractedSpan) {
-                                const extractedSpanContext = extractedSpan.spanContext();
-                                core_1.logger.debug("Manual context extraction result:", JSON.stringify({
-                                    traceId: extractedSpanContext.traceId,
-                                    spanId: extractedSpanContext.spanId,
-                                    traceFlags: extractedSpanContext.traceFlags,
-                                }));
-                            }
-                            else {
-                                core_1.logger.debug("Manual context extraction failed - no span found");
-                                core_1.logger.debug("Available propagation fields:", api_1.propagation.fields());
-                            }
+                            // Extract trace context with retry for Lambda cold starts
+                            void this.retryContextExtraction(headers, span, parts);
                         }
                         catch (error) {
                             core_1.logger.debug("Manual context extraction error:", error);
@@ -307,23 +355,22 @@ class TelemetryManager {
         this.loggerProvider.addLogRecordProcessor(new sdk_logs_1.BatchLogRecordProcessor(logExporter));
         api_logs_1.logs.setGlobalLoggerProvider(this.loggerProvider);
         const traceExporter = this.getTraceExporter();
-        if (!this.nodeTracerProvider) {
-            // Create tracer provider if not already created
-            this.nodeTracerProvider = new sdk_trace_node_1.NodeTracerProvider({
-                resource,
-                sampler: new sdk_trace_node_1.AlwaysOnSampler(),
-                spanProcessors: [
-                    new DefaultAttributesSpanProcessor({
-                        "workload.id": core_1.settings.name || "",
-                        "workload.type": core_1.settings.type ? core_1.settings.type + "s" : "",
-                        workspace: core_1.settings.workspace || "",
-                    }),
-                    new sdk_trace_node_1.BatchSpanProcessor(traceExporter),
-                    new HasBeenProcessedSpanProcessor(traceExporter),
-                ],
-            });
-            this.nodeTracerProvider.register();
-        }
+        // Create a new tracer provider with full configuration
+        this.nodeTracerProvider = new sdk_trace_node_1.NodeTracerProvider({
+            resource,
+            sampler: new sdk_trace_node_1.AlwaysOnSampler(),
+            spanProcessors: [
+                new DefaultAttributesSpanProcessor({
+                    "workload.id": core_1.settings.name || "",
+                    "workload.type": core_1.settings.type ? core_1.settings.type + "s" : "",
+                    workspace: core_1.settings.workspace || "",
+                }),
+                new sdk_trace_node_1.BatchSpanProcessor(traceExporter),
+                new HasBeenProcessedSpanProcessor(traceExporter),
+            ],
+        });
+        // Replace the early tracer provider with the fully configured one
+        this.nodeTracerProvider.register();
         // Ensure W3C trace context propagation is working
         core_1.logger.debug("Propagation fields after tracer registration:", api_1.propagation.fields());
         const metricExporter = this.getMetricExporter();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blaxel/telemetry",
-  "version": "0.2.17-dev.119",
+  "version": "0.2.17-dev.120",
   "description": "Blaxel SDK for TypeScript",
   "license": "MIT",
   "author": "Blaxel, INC (https://blaxel.ai)",
@@ -71,7 +71,7 @@
     "@opentelemetry/sdk-trace-base": "^2.0.0",
     "@opentelemetry/sdk-trace-node": "^2.0.0",
     "ai": "^4.3.13",
-    "@blaxel/core": "0.2.17-dev.119"
+    "@blaxel/core": "0.2.17-dev.120"
   },
   "devDependencies": {
     "@eslint/js": "^9.26.0",