@blaxel/core 0.3.12-preview.254 → 0.3.12-preview.256

package/dist/esm/common/h2fetch.js

@@ -1,6 +1,7 @@
 import http2 from "http2";
 import { settings } from "./settings.js";
 import { refH2SessionForActiveRequest } from "./h2ref.js";
+import { recordH2Fallback } from "./h2stats.js";
 // Statuses the Response constructor refuses to pair with a body. Handing it one
 // throws, so a 204 (e.g. from a DELETE) would blow up instead of resolving.
 const NULL_BODY_STATUSES = new Set([101, 103, 204, 205, 304]);
@@ -158,6 +159,7 @@ export async function withUploadSlot(domain, fn) {
 export function createH2Fetch(session) {
     return (input) => {
         if (session.closed || session.destroyed) {
+            recordH2Fallback(new URL(input.url).hostname, "session-unusable");
             return globalThis.fetch(input);
         }
         return _h2Request(session, input);
@@ -223,6 +225,7 @@ async function h2GatewayAttempt(pool, domain, send, fallback) {
             let h2RequestCreated = false;
             try {
                 return await send(session, {
+                    domain,
                     onH2RequestCreated: () => {
                         h2RequestCreated = true;
                     },
@@ -241,6 +244,7 @@ async function h2GatewayAttempt(pool, domain, send, fallback) {
         // No usable session: free the slot before falling back over a different
         // connection (no stream opens on the shared session).
         rel();
+        recordH2Fallback(domain, "no-session");
         return await fallback();
     }
     catch (err) {
@@ -280,6 +284,7 @@ function h2RequestDirectInternal(session, url, init, options) {
         // Pre-flight fallback (session unusable): no stream opens on the shared
         // session, so free any held slot before going over globalThis.fetch.
         options?.releaseSlot?.();
+        recordH2Fallback(options?.domain ?? new URL(url).hostname, "session-unusable");
         return globalThis.fetch(url, init);
     }
     const parsed = new URL(url);
@@ -322,6 +327,7 @@ function h2RequestDirectInternal(session, url, init, options) {
             // nothing has been sent on the wire yet). No stream opens on the shared
             // session, so free any held slot before falling back.
             options?.releaseSlot?.();
+            recordH2Fallback(options?.domain ?? parsed.hostname, "unsupported-body");
             return globalThis.fetch(url, init);
         }
         if (!h2Headers["content-length"]) {
@@ -401,6 +407,7 @@ function _h2Send(session, h2Headers, body, signal, fallbackUrl, fallbackInit, op
             // H2 frames were sent. No stream opened on the shared session, so free
             // the slot before retrying over globalThis.fetch.
             releaseSlot();
+            recordH2Fallback(options?.domain ?? new URL(fallbackUrl).hostname, session.closed || session.destroyed ? "session-unusable" : "request-rejected");
             globalThis.fetch(fallbackUrl, fallbackInit).then(resolve, reject);
             return;
         }

package/dist/esm/common/h2pool.js

@@ -1,3 +1,4 @@
+import { recordH2EstablishFailure } from "./h2stats.js";
 const DEFAULT_MAX_IDLE_MS = 5_000;
 const DEFAULT_PING_TIMEOUT_MS = 500;
 /**
@@ -39,6 +40,22 @@ export class H2Pool {
         this.attachEvictionListeners(domain, session);
         return session;
     }
+    startEstablish(domain) {
+        const pending = this.establish(domain)
+            .then((session) => {
+            this.cache(domain, session);
+            return session;
+        })
+            .catch((error) => {
+            recordH2EstablishFailure(domain, error);
+            return null;
+        })
+            .finally(() => {
+            this.inflight.delete(domain);
+        });
+        this.inflight.set(domain, pending);
+        return pending;
+    }
     attachEvictionListeners(domain, session) {
         const evict = () => {
             // Only evict if this specific session is still the cached one.
@@ -157,16 +174,7 @@ export class H2Pool {
             return;
         if (this.inflight.has(domain))
             return;
-        const p = this.establish(domain)
-            .then((session) => {
-            this.cache(domain, session);
-            return session;
-        })
-            .catch(() => null)
-            .finally(() => {
-            this.inflight.delete(domain);
-        });
-        this.inflight.set(domain, p);
+        void this.startEstablish(domain);
     }
     /**
      * Synchronous cache check. Returns a live cached session or `null`.
@@ -214,17 +222,7 @@ export class H2Pool {
         const freshCached = this.tryGet(domain);
         if (freshCached)
             return freshCached;
-        const p = this.establish(domain)
-            .then((session) => {
-            this.cache(domain, session);
-            return session;
-        })
-            .catch(() => null)
-            .finally(() => {
-            this.inflight.delete(domain);
-        });
-        this.inflight.set(domain, p);
-        return p;
+        return this.startEstablish(domain);
     }
     /** Close all sessions (for cleanup). */
     closeAll() {

package/dist/esm/common/h2stats.js

@@ -0,0 +1,112 @@
+import { logger } from "./logger.js";
+import { reportH2TransportDegradation } from "./sentry.js";
+const emptyReasons = () => ({
+    "no-session": 0,
+    "request-rejected": 0,
+    "session-unusable": 0,
+    "unsupported-body": 0,
+});
+const MAX_TRACKED_DOMAINS = 100;
+const H2_DEBUG_STATS_SYMBOL = Symbol.for("blaxel.h2stats");
+const h2DebugStatsEnabled = typeof process !== "undefined" && process.env?.BL_H2_DEBUG_STATS === "1";
+function publishDebugSnapshot(snapshot) {
+    if (!h2DebugStatsEnabled)
+        return;
+    try {
+        globalThis[H2_DEBUG_STATS_SYMBOL] = snapshot;
+    }
+    catch {
+        // Debug diagnostics must never change transport behavior.
+    }
+}
+const emptyDomainStats = () => ({
+    establishFailures: 0,
+    fetchFallbacks: 0,
+    fallbacksByReason: emptyReasons(),
+});
+class H2TransportStatsStore {
+    totals = emptyDomainStats();
+    byDomain = new Map();
+    snapshot() {
+        return {
+            ...this.clone(this.totals),
+            byDomain: Object.fromEntries([...this.byDomain].map(([domain, stats]) => [domain, this.clone(stats)])),
+        };
+    }
+    reset() {
+        this.totals = emptyDomainStats();
+        this.byDomain.clear();
+        publishDebugSnapshot(this.snapshot());
+    }
+    /** @internal */
+    recordEstablishFailure(domain, error) {
+        this.totals.establishFailures++;
+        this.forDomain(domain).establishFailures++;
+        try {
+            const message = error instanceof Error ? error.message : String(error);
+            logger.debug(`H2 session establishment failed for ${domain}: ${message}`);
+        }
+        catch {
+            // Diagnostics must never change transport behavior.
+        }
+        reportH2TransportDegradation(domain, "establish-failure");
+        publishDebugSnapshot(this.snapshot());
+    }
+    /** @internal */
+    recordFallback(domain, reason) {
+        this.totals.fetchFallbacks++;
+        this.totals.fallbacksByReason[reason]++;
+        const stats = this.forDomain(domain);
+        stats.fetchFallbacks++;
+        stats.fallbacksByReason[reason]++;
+        try {
+            logger.debug(`H2 transport falling back to fetch for ${domain}: ${reason}`);
+        }
+        catch {
+            // Diagnostics must never change transport behavior.
+        }
+        reportH2TransportDegradation(domain, reason);
+        publishDebugSnapshot(this.snapshot());
+    }
+    forDomain(domain) {
+        let stats = this.byDomain.get(domain);
+        if (stats) {
+            this.byDomain.delete(domain);
+        }
+        else {
+            if (this.byDomain.size >= MAX_TRACKED_DOMAINS) {
+                const oldest = this.byDomain.keys().next().value;
+                if (oldest !== undefined)
+                    this.byDomain.delete(oldest);
+            }
+            stats = emptyDomainStats();
+        }
+        this.byDomain.set(domain, stats);
+        return stats;
+    }
+    clone(stats) {
+        return {
+            establishFailures: stats.establishFailures,
+            fetchFallbacks: stats.fetchFallbacks,
+            fallbacksByReason: { ...stats.fallbacksByReason },
+        };
+    }
+}
+const store = new H2TransportStatsStore();
+publishDebugSnapshot(store.snapshot());
+/** @internal */
+export function snapshotH2TransportStats() {
+    return store.snapshot();
+}
+/** @internal */
+export function resetH2TransportStats() {
+    store.reset();
+}
+/** @internal */
+export function recordH2EstablishFailure(domain, error) {
+    store.recordEstablishFailure(domain, error);
+}
+/** @internal */
+export function recordH2Fallback(domain, reason) {
+    store.recordFallback(domain, reason);
+}

package/dist/esm/common/sentry.js

@@ -1,3 +1,4 @@
+import { crypto } from "./node.js";
 import { settings } from "./settings.js";
 const PACKAGE_LAYOUT_MARKERS = [
     "/src/common/sentry.ts",
@@ -24,7 +25,11 @@ const SAFE_ERROR_NAMES = new Set([
     "AggregateError",
 ]);
 const MAX_IN_FLIGHT_EVENTS = 20;
+const MAX_ACTIVE_H2_ROLLUPS = 20;
+const MAX_CACHED_DOMAIN_TAGS = 100;
+const MAX_H2_EVENTS_PER_PROCESS = 100;
 const DELIVERY_TIMEOUT_MS = 500;
+const H2_ROLLUP_WINDOW_MS = 60_000;
 const SAFE_FILENAME_SEGMENT = /^[A-Za-z0-9._-]+$/;
 let sentryInitialized = false;
 let handlersRegistered = false;
@@ -32,6 +37,9 @@ let sentryConfig = null;
 let flushPromise = null;
 const capturedExceptions = new WeakSet();
 const inFlightDeliveries = new Set();
+const domainTagCache = new Map();
+const h2DegradationRollups = new Map();
+let h2EventsEmitted = 0;
 /**
  * Normalize stack filenames without resolving or exposing host filesystem data.
  */
@@ -204,6 +212,91 @@ function errorToSentryEvent(error, ownedFrames) {
         },
     };
 }
+function runtimeTag() {
+    const host = globalThis;
+    if (host.process?.versions?.bun)
+        return `bun/${host.process.versions.bun}`;
+    if (host.Deno?.version?.deno)
+        return `deno/${host.Deno.version.deno}`;
+    if (host.process?.versions?.node)
+        return `node/${host.process.versions.node}`;
+    return "browser";
+}
+function normalizeDomain(domain) {
+    const normalized = domain.trim().toLowerCase().replace(/\.$/, "");
+    return normalized.length > 0 ? normalized : null;
+}
+function blaxelEdgeDomainTag(domain) {
+    const edgeSuffixes = [
+        { environment: "prod", suffix: ".bl.run" },
+        { environment: "dev", suffix: ".runv2.blaxel.dev" },
+    ];
+    for (const { environment, suffix } of edgeSuffixes) {
+        if (!domain.endsWith(suffix))
+            continue;
+        const region = domain.slice(0, -suffix.length).split(".").at(-1);
+        if (region && /^[a-z0-9-]+$/.test(region)) {
+            return `blaxel-edge:${environment}:${region}`;
+        }
+        return null;
+    }
+    return null;
+}
+function uncachedDomainTag(domain) {
+    const stableBlaxelDomains = {
+        "api.blaxel.ai": "blaxel-api:prod",
+        "api.blaxel.dev": "blaxel-api:dev",
+        "run.blaxel.ai": "blaxel-run:prod",
+        "run.blaxel.dev": "blaxel-run:dev",
+    };
+    const stableTag = stableBlaxelDomains[domain];
+    if (stableTag)
+        return stableTag;
+    const edgeTag = blaxelEdgeDomainTag(domain);
+    if (edgeTag)
+        return edgeTag;
+    if (!crypto)
+        return null;
+    try {
+        return `sha256:${crypto.createHash("sha256").update(domain).digest("hex")}`;
+    }
+    catch {
+        return null;
+    }
+}
+function domainToTag(domain) {
+    if (domainTagCache.has(domain))
+        return domainTagCache.get(domain) ?? null;
+    const domainTag = uncachedDomainTag(domain);
+    if (domainTagCache.size >= MAX_CACHED_DOMAIN_TAGS) {
+        const oldestDomain = domainTagCache.keys().next().value;
+        if (oldestDomain !== undefined)
+            domainTagCache.delete(oldestDomain);
+    }
+    domainTagCache.set(domain, domainTag);
+    return domainTag;
+}
+function h2DegradationToSentryEvent(reason, domainTag, count) {
+    return {
+        event_id: generateEventId(),
+        timestamp: Date.now() / 1000,
+        platform: "javascript",
+        level: "warning",
+        environment: settings.env,
+        release: `sdk-typescript@${settings.version}`,
+        message: `h2 transport degradation: ${reason}`,
+        fingerprint: ["h2-degradation", reason, domainTag],
+        tags: {
+            "blaxel.version": settings.version,
+            "blaxel.commit": settings.commit,
+            "blaxel.error_source": "h2-transport-degradation",
+            "blaxel.runtime": runtimeTag(),
+            reason,
+            domainTag,
+        },
+        extra: { count },
+    };
+}
 async function sendToSentry(event) {
     if (!sentryConfig)
         return;
@@ -245,12 +338,71 @@ async function sendToSentry(event) {
 }
 function scheduleDelivery(event) {
     if (inFlightDeliveries.size >= MAX_IN_FLIGHT_EVENTS)
-        return;
+        return false;
     // Contain rejections from setup that occurs before sendToSentry's internal
     // fetch guard (for example, a host-provided AbortController implementation).
     const delivery = sendToSentry(event).catch(() => undefined);
     inFlightDeliveries.add(delivery);
     void delivery.then(() => inFlightDeliveries.delete(delivery));
+    return true;
+}
+function emitH2DegradationRollup(key) {
+    const rollup = h2DegradationRollups.get(key);
+    if (!rollup)
+        return;
+    h2DegradationRollups.delete(key);
+    clearTimeout(rollup.timer);
+    if (h2EventsEmitted >= MAX_H2_EVENTS_PER_PROCESS)
+        return;
+    if (scheduleDelivery(h2DegradationToSentryEvent(rollup.reason, rollup.domainTag, rollup.count))) {
+        h2EventsEmitted++;
+    }
+}
+function flushH2DegradationRollups() {
+    for (const key of [...h2DegradationRollups.keys()]) {
+        emitH2DegradationRollup(key);
+    }
+}
+/** @internal */
+export function reportH2TransportDegradation(domain, reason) {
+    if (!sentryInitialized || !sentryConfig)
+        return;
+    try {
+        if (h2EventsEmitted >= MAX_H2_EVENTS_PER_PROCESS)
+            return;
+        const normalizedDomain = normalizeDomain(domain);
+        if (!normalizedDomain)
+            return;
+        const domainTag = domainToTag(normalizedDomain);
+        if (!domainTag)
+            return;
+        const key = `${reason}\0${domainTag}`;
+        const existing = h2DegradationRollups.get(key);
+        if (existing) {
+            existing.count++;
+            return;
+        }
+        if (h2DegradationRollups.size >= MAX_ACTIVE_H2_ROLLUPS)
+            return;
+        const timer = setTimeout(() => {
+            try {
+                emitH2DegradationRollup(key);
+            }
+            catch {
+                // Telemetry must never change transport behavior.
+            }
+        }, H2_ROLLUP_WINDOW_MS);
+        timer.unref?.();
+        h2DegradationRollups.set(key, {
+            count: 1,
+            domainTag,
+            reason,
+            timer,
+        });
+    }
+    catch {
+        // Telemetry must never change transport behavior.
+    }
 }
 function captureException(error) {
     if (!sentryInitialized || !sentryConfig || capturedExceptions.has(error)) {
@@ -318,7 +470,15 @@ export function initSentry() {
  * Events are sent exactly once; flush never re-enqueues them.
  */
 export async function flushSentry(timeout = DELIVERY_TIMEOUT_MS) {
-    if (!sentryInitialized || inFlightDeliveries.size === 0)
+    if (!sentryInitialized)
+        return;
+    try {
+        flushH2DegradationRollups();
+    }
+    catch {
+        // Flushing telemetry must never affect application shutdown.
+    }
+    if (inFlightDeliveries.size === 0)
         return;
     if (flushPromise) {
         await flushPromise;