catalyst-core 1.1.0 → 1.2.0-canary.1

package/dist/otel.js

@@ -1,27 +1,448 @@
-import { metrics } from "@opentelemetry/api"
 import { NodeSDK } from "@opentelemetry/sdk-node"
 import { resourceFromAttributes } from "@opentelemetry/resources"
-import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node"
+import {
+    metrics,
+    trace,
+    TraceFlags,
+    SpanStatusCode,
+    context,
+    createContextKey,
+    isSpanContextValid,
+} from "@opentelemetry/api"
+import {
+    BatchSpanProcessor,
+    TraceIdRatioBasedSampler,
+    ParentBasedSampler,
+    SamplingDecision,
+} from "@opentelemetry/sdk-trace-node"
+import { setGlobalErrorHandler } from "@opentelemetry/core"
 import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"
 import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-grpc"
 import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"
-import { TraceIdRatioBasedSampler, ParentBasedSampler } from "@opentelemetry/sdk-trace-node"
 import { OTLPTraceExporter as OTLPTraceExporterHTTP } from "@opentelemetry/exporter-trace-otlp-http"
 import { OTLPMetricExporter as OTLPMetricExporterHTTP } from "@opentelemetry/exporter-metrics-otlp-http"
-import { trace, context, createContextKey } from "@opentelemetry/api"
+import fs from "fs"
+import os from "os"
 
 export const IS_BOT_KEY = createContextKey("catalyst.is_bot")
 
 import semanticConventions from "@opentelemetry/semantic-conventions"
 const { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION, ATTR_DEPLOYMENT_ENVIRONMENT } = semanticConventions
 
-// The app-wide logger only exists once the app calls configureLogger() from
-// catalyst-core/logger, which nothing in core does for it. Resolve it per call,
-// so a logger configured after this module loads is still picked up, and fall
-// back to console so init() never throws a ReferenceError on a bare `logger`.
-const getLogger = () => global.logger ?? console
+const formatMB = (bytes) => (bytes / 1024 / 1024).toFixed(1)
+
+function getLogger() {
+    const l =
+        (typeof logger !== "undefined" && logger) ||
+        (typeof global !== "undefined" && global.logger) ||
+        console
+    return {
+        info: (msg, ...meta) => (l.info ? l.info(msg, ...meta) : console.log(msg, ...meta)),
+        warn: (msg, ...meta) =>
+            l.warn ? l.warn(msg, ...meta) : l.info ? l.info(msg, ...meta) : console.warn(msg, ...meta),
+        error: (msg, ...meta) => (l.error ? l.error(msg, ...meta) : console.error(msg, ...meta)),
+        debug: (msg, ...meta) => (l.debug ? l.debug(msg, ...meta) : console.debug(msg, ...meta)),
+    }
+}
+
+// undefined if the file is absent, or (for a limit) unbounded.
+function readCgroupBytesFile(path) {
+    try {
+        const raw = fs.readFileSync(path, "utf8").trim()
+        const bytes = raw === "max" ? NaN : Number(raw)
+        return Number.isFinite(bytes) && bytes > 0 && bytes < Number.MAX_SAFE_INTEGER ? bytes : undefined
+    } catch {
+        return undefined
+    }
+}
+
+// os.totalmem() reports the node's memory, not the pod's limit — read the
+// cgroup directly instead. Cached: fixed for the process's lifetime.
+let memoryLimitBytes
+function getMemoryLimitBytes() {
+    return (memoryLimitBytes ??=
+        readCgroupBytesFile("/sys/fs/cgroup/memory.max") ??
+        readCgroupBytesFile("/sys/fs/cgroup/memory/memory.limit_in_bytes") ??
+        os.totalmem())
+}
+
+// Whole-cgroup usage, not just this process's RSS — differs if anything else
+// shares the container. Changes constantly, so never cached (unlike the limit).
+function getCgroupCurrentBytes() {
+    return (
+        readCgroupBytesFile("/sys/fs/cgroup/memory.current") ??
+        readCgroupBytesFile("/sys/fs/cgroup/memory/memory.usage_in_bytes")
+    )
+}
+
+// 85% of the memory limit — leaves headroom before an OOM kill.
+function highMemoryRssMB() {
+    return Math.round((getMemoryLimitBytes() * 0.85) / 1024 / 1024)
+}
+
+// 0.4 = 50% of the limit as heap budget (common container guidance) × 80% warning point.
+function highMemoryHeapUsedMB() {
+    return Math.round((getMemoryLimitBytes() * 0.4) / 1024 / 1024)
+}
+
+let lastHighMemoryAlertTime = 0
+function checkHighMemoryAlert(mem = process.memoryUsage(), contextStr = "", thresholds = {}) {
+    // Overrides are mainly for testing (e.g. forcing a low staging threshold).
+    // Validated so a bad value falls back instead of misfiring every check.
+    const isValidThreshold = (v) => typeof v === "number" && v > 0
+    const rssLimitMB = isValidThreshold(thresholds.rssMB) ? thresholds.rssMB : highMemoryRssMB()
+    const heapLimitMB = isValidThreshold(thresholds.heapUsedMB)
+        ? thresholds.heapUsedMB
+        : highMemoryHeapUsedMB()
+
+    const now = Date.now()
+    // Throttle alert to at most once per 60 seconds
+    if (now - lastHighMemoryAlertTime < 60000) return
+
+    const rssMB = mem.rss / 1024 / 1024
+    const heapUsedMB = mem.heapUsed / 1024 / 1024
+
+    if (rssMB > rssLimitMB || heapUsedMB > heapLimitMB) {
+        lastHighMemoryAlertTime = now
+        const ratio = (mem.rss / (mem.heapUsed || 1)).toFixed(2)
+        getLogger().warn(
+            `⚠️ [OTEL High Memory Warning] RSS=${formatMB(mem.rss)}MB, HeapUsed=${formatMB(mem.heapUsed)}MB, HeapTotal=${formatMB(mem.heapTotal)}MB, External=${formatMB(mem.external)}MB (RSS/Heap ratio: ${ratio}) ${contextStr ? `| Context: ${contextStr}` : ""}`
+        )
+    }
+}
+
+// Bit-range sampling decision helper
+function getDecisionForBits(traceId, start, end, rate) {
+    if (rate >= 1.0) return true
+    if (rate <= 0.0) return false
+    if (!traceId || traceId.length < end) return false
+    const hexPart = traceId.substring(start, end)
+    const value = parseInt(hexPart, 16)
+    const maxValue = Math.pow(16, end - start) - 1
+    return value / maxValue < rate
+}
+
+// Custom Status Aware Sampler
+class StatusAwareSampler {
+    constructor(samplingRate) {
+        this.samplingRate = samplingRate
+    }
+
+    shouldSample(context, traceId) {
+        const parentSpanContext = trace.getSpanContext(context)
+
+        if (parentSpanContext && isSpanContextValid(parentSpanContext)) {
+            if (parentSpanContext.traceFlags & TraceFlags.SAMPLED) {
+                return { decision: SamplingDecision.RECORD_AND_SAMPLED }
+            } else {
+                return { decision: SamplingDecision.RECORD }
+            }
+        }
+
+        // Head sampling using characters 0-12
+        const isHeadSampled = getDecisionForBits(traceId, 0, 12, this.samplingRate)
+        if (isHeadSampled) {
+            return { decision: SamplingDecision.RECORD_AND_SAMPLED }
+        } else {
+            return { decision: SamplingDecision.RECORD }
+        }
+    }
+
+    toString() {
+        return `StatusAwareSampler{samplingRate=${this.samplingRate}}`
+    }
+}
+
+// Custom Span Processor for Status Aware Error Sampling
+class PromotingSpanProcessor {
+    constructor(exporter, config) {
+        this.exporter = exporter
+        this.config = config
+        this.buffer = new Map()
+        this.promotedTraces = new Map()
+
+        // Delegating BatchSpanProcessor for head-sampled spans
+        this.batchProcessor = new BatchSpanProcessor(exporter, config.batchProcessorConfig)
+
+        this.cleanupInterval = setInterval(() => {
+            this._cleanupBuffers()
+        }, 30000)
+
+        if (this.cleanupInterval.unref) {
+            this.cleanupInterval.unref()
+        }
+    }
+
+    onStart(span, parentContext) {
+        const isSampled = (span.spanContext().traceFlags & TraceFlags.SAMPLED) !== 0
+        if (isSampled) {
+            this.batchProcessor.onStart(span, parentContext)
+        }
+    }
+
+    onEnd(span) {
+        const traceId = span.spanContext().traceId
+        const isSampled = (span.spanContext().traceFlags & TraceFlags.SAMPLED) !== 0
+
+        if (this.promotedTraces.has(traceId)) {
+            const promotionRate = this.promotedTraces.get(traceId).promotionRate
+            this._exportSpanImmediately(span, promotionRate)
+            return
+        }
+
+        if (isSampled) {
+            this.batchProcessor.onEnd(span)
+            return
+        }
+
+        // Buffer the RECORD_ONLY span
+        let record = this.buffer.get(traceId)
+        if (!record) {
+            record = {
+                spans: [],
+                rootSpan: null,
+                hasChildError: false,
+                timestamp: Date.now(),
+            }
+            this.buffer.set(traceId, record)
+        }
+
+        const isRoot = !span.parentSpanId
+        if (isRoot) {
+            record.rootSpan = span
+        } else {
+            record.spans.push(span)
+        }
+
+        if (!isRoot && span.status && span.status.code === SpanStatusCode.ERROR) {
+            record.hasChildError = true
+        }
+
+        if (isRoot) {
+            this._evaluateAndProcessPromotion(traceId, record)
+        }
+    }
+
+    _evaluateAndProcessPromotion(traceId, record) {
+        const rootSpan = record.rootSpan
+        if (!rootSpan) return
+
+        const isBot = rootSpan.attributes["http.response.is_bot"] === true
+        if (isBot && !this.config.promoteBotTraffic) {
+            this.buffer.delete(traceId)
+            return
+        }
+
+        const statusCode = Number(
+            rootSpan.attributes["http.status_code"] || rootSpan.attributes["http.response.status_code"] || 0
+        )
+
+        const is5xx = statusCode >= 500 && statusCode < 600
+        const is4xx = statusCode >= 400 && statusCode < 500
+        const isRootError = rootSpan.status && rootSpan.status.code === SpanStatusCode.ERROR && !is4xx
+        const isSkipped = this.config.skipPromotionCodes.includes(statusCode)
+
+        let shouldPromote = false
+        let promotionRate = this.config.samplingRate
+
+        if ((is5xx || isRootError) && !isSkipped) {
+            shouldPromote = getDecisionForBits(traceId, 12, 24, this.config.rate5xx)
+            promotionRate = this.config.rate5xx
+        } else if (is4xx && !isSkipped) {
+            shouldPromote = getDecisionForBits(traceId, 12, 24, this.config.rate4xx)
+            promotionRate = this.config.rate4xx
+        } else if (
+            this.config.promoteHandledErrors &&
+            record.hasChildError &&
+            statusCode >= 200 &&
+            statusCode < 300 &&
+            !isSkipped
+        ) {
+            shouldPromote = getDecisionForBits(traceId, 12, 24, this.config.rate5xx)
+            promotionRate = this.config.rate5xx
+        }
+
+        if (shouldPromote) {
+            this.promotedTraces.set(traceId, {
+                timestamp: Date.now(),
+                promotionRate,
+            })
+
+            let spansToExport = []
+            if (this.config.exportFullTraceOnError) {
+                spansToExport = [rootSpan, ...record.spans]
+            } else {
+                spansToExport = [rootSpan]
+                for (const s of record.spans) {
+                    if (s.status && s.status.code === SpanStatusCode.ERROR) {
+                        spansToExport.push(s)
+                    }
+                }
+            }
+
+            const finalSampleRate = this.config.reportActualPromotionRate
+                ? promotionRate
+                : this.config.samplingRate
+            for (const s of spansToExport) {
+                // Required: batchProcessor.onEnd() drops spans without SAMPLED.
+                s.spanContext().traceFlags |= TraceFlags.SAMPLED
+                s.attributes["promoted"] = true
+                s.attributes["sample_rate"] = finalSampleRate
+            }
+
+            this._export(spansToExport)
+        }
+
+        this.buffer.delete(traceId)
+    }
+
+    _exportSpanImmediately(span, promotionRate) {
+        const finalSampleRate = this.config.reportActualPromotionRate
+            ? promotionRate
+            : this.config.samplingRate
+        // Required: batchProcessor.onEnd() drops spans without SAMPLED.
+        span.spanContext().traceFlags |= TraceFlags.SAMPLED
+        span.attributes["promoted"] = true
+        span.attributes["sample_rate"] = finalSampleRate
+
+        this._export([span])
+    }
+
+    // Routes promoted spans through the same BatchSpanProcessor as normal
+    // traffic, for its queueing/backpressure. Trade-off: they export on the
+    // next batch flush, not instantly. Skipping onStart() is safe — it's a no-op.
+    _export(spans) {
+        for (const s of spans) {
+            this.batchProcessor.onEnd(s)
+        }
+    }
+
+    _cleanupBuffers() {
+        const now = Date.now()
+        // Trade-off: short TTL bounds memory but drops buffered child spans (and
+        // hasChildError) for traces whose root outlives it — clock starts at the
+        // first span seen for that trace, not request start. The root itself is
+        // still evaluated fresh when it ends, so a late error can still promote,
+        // just without children. Revisit if this proves too aggressive.
+        const BUFFER_TTL = 60 * 1000 // 60 seconds TTL for unresolved buffer
+        const PROMOTED_TTL = 60 * 1000 // 60 seconds TTL for promoted trace cache
+        const mem = process.memoryUsage()
+
+        let expiredBuffer = 0
+        for (const [traceId, record] of this.buffer.entries()) {
+            if (now - record.timestamp > BUFFER_TTL) {
+                this.buffer.delete(traceId)
+                expiredBuffer++
+            } else {
+                break
+            }
+        }
+
+        if (expiredBuffer > 0) {
+            getLogger().info(
+                `🧹 [OTEL Cleanup] PromotingSpanProcessor: evicted ${expiredBuffer} expired unresolved trace(s) from buffer (remaining size=${this.buffer.size})`
+            )
+        }
+
+        let bufferOverflowEvicted = 0
+        if (this.buffer.size > 1024) {
+            const toDelete = this.buffer.size - 1024
+            for (const traceId of this.buffer.keys()) {
+                this.buffer.delete(traceId)
+                bufferOverflowEvicted++
+                if (bufferOverflowEvicted >= toDelete) break
+            }
+            // Buffer filling faster than roots resolve — traces dropped pre-promotion.
+            getLogger().warn(
+                `⚠️ [OTEL Buffer Overflow] PromotingSpanProcessor: buffer overflow, dropped ${bufferOverflowEvicted} unresolved trace(s) (buffer size=${this.buffer.size}, rss=${formatMB(mem.rss)}MB, heapUsed=${formatMB(mem.heapUsed)}MB)`
+            )
+        }
+
+        let expiredPromoted = 0
+        for (const [traceId, data] of this.promotedTraces.entries()) {
+            if (now - data.timestamp > PROMOTED_TTL) {
+                this.promotedTraces.delete(traceId)
+                expiredPromoted++
+            } else {
+                break
+            }
+        }
 
+        if (expiredPromoted > 0) {
+            getLogger().info(
+                `🧹 [OTEL Cleanup] PromotingSpanProcessor: evicted ${expiredPromoted} expired trace(s) from promotedTraces (remaining size=${this.promotedTraces.size})`
+            )
+        }
+
+        let promotedOverflowEvicted = 0
+        if (this.promotedTraces.size > 1024) {
+            const toDelete = this.promotedTraces.size - 1024
+            for (const traceId of this.promotedTraces.keys()) {
+                this.promotedTraces.delete(traceId)
+                promotedOverflowEvicted++
+                if (promotedOverflowEvicted >= toDelete) break
+            }
+            getLogger().warn(
+                `⚠️ [OTEL Buffer Overflow] PromotingSpanProcessor: promotedTraces overflow, dropped ${promotedOverflowEvicted} cache entry(s) (promotedTraces size=${this.promotedTraces.size}, rss=${formatMB(mem.rss)}MB, heapUsed=${formatMB(mem.heapUsed)}MB)`
+            )
+        }
+    }
+
+    forceFlush() {
+        return this.batchProcessor.forceFlush()
+    }
+
+    shutdown() {
+        if (this.cleanupInterval) {
+            clearInterval(this.cleanupInterval)
+        }
+        const mem = process.memoryUsage()
+        getLogger().info(
+            `📡 [OTEL Shutdown] PromotingSpanProcessor: shutting down, discarding ${this.buffer.size} unresolved trace(s) and ${this.promotedTraces.size} promoted-trace record(s) | Memory at exit: RSS=${formatMB(mem.rss)}MB, HeapUsed=${formatMB(mem.heapUsed)}MB`
+        )
+        this.buffer.clear()
+        this.promotedTraces.clear()
+        return this.batchProcessor.shutdown()
+    }
+}
+
+/**
+ * Initializes the OpenTelemetry Node SDK — traces, optional metrics, and (when
+ * errorSampling.ENABLED is set) status-aware error sampling. No-op unless
+ * OTEL_ENABLE is set (see server/expressServer.js and server/renderer/handler.jsx
+ * for the same guard on the call sites that import this module).
+ *
+ * @param {object} [config]
+ * @param {string} [config.serviceName="catalyst-server"]
+ * @param {string} [config.serviceVersion="1.0.0"]
+ * @param {string} [config.environment="development"]
+ * @param {string} [config.traceUrl="http://localhost:4317"] - OTLP trace collector endpoint
+ * @param {string} [config.metricUrl="http://localhost:4317"] - OTLP metric collector endpoint
+ * @param {string} [config.traceProtocol="grpc"] - "grpc" or "http"
+ * @param {string} [config.metricProtocol] - "grpc" or "http"; metrics stay disabled if omitted
+ * @param {object} [config.traceHeaders]
+ * @param {object} [config.metricHeaders]
+ * @param {object} [config.batchProcessorConfig] - passed to BatchSpanProcessor (maxQueueSize, scheduledDelayMillis, ...)
+ * @param {number} [config.exportIntervalMillis=10000]
+ * @param {number} [config.diagnosticsIntervalMillis=60000] - memory heartbeat interval, ms; requires errorSampling.ENABLED
+ * @param {boolean} [config.enableMemoryDiagnostics=true] - memory heartbeat + high-memory warnings; requires errorSampling.ENABLED
+ * @param {number} [config.highMemoryRssMB] - overrides the auto-detected RSS warning threshold; for testing; requires errorSampling.ENABLED
+ * @param {number} [config.highMemoryHeapUsedMB] - overrides the auto-detected heapUsed warning threshold; for testing; requires errorSampling.ENABLED
+ * @param {Array} [config.instrumentations] - defaults to getNodeAutoInstrumentations()
+ * @param {number} [config.samplingRate=1.0] - head-sampling rate in [0, 1] for non-error traffic
+ * @param {*} [config.grpcCredentials]
+ * @param {object} [config.errorSampling] - status-aware error sampling, off by default
+ * @param {boolean} [config.errorSampling.ENABLED=false] - turns on StatusAwareSampler + PromotingSpanProcessor
+ * @param {number} [config.errorSampling.RATE_5XX=1.0] - promotion rate for 5xx roots (and non-4xx roots with OTEL ERROR status)
+ * @param {number} [config.errorSampling.RATE_4XX=samplingRate] - promotion rate for 4xx roots
+ * @param {boolean} [config.errorSampling.EXPORT_FULL_TRACE_ON_ERROR=false] - export every buffered span on promotion, not just root + error spans
+ * @param {boolean} [config.errorSampling.PROMOTE_HANDLED_ERRORS=false] - promote 2xx roots (at RATE_5XX) if any child span recorded an error
+ * @param {boolean} [config.errorSampling.REPORT_ACTUAL_PROMOTION_RATE=false] - report the real promotion rate in the exported sample_rate attribute instead of samplingRate
+ * @param {number[]} [config.errorSampling.SKIP_PROMOTION_CODES=[408,504,524,598,599]] - status codes excluded from promotion regardless of 4xx/5xx
+ * @param {boolean} [config.errorSampling.PROMOTE_BOT_TRAFFIC=false] - allow bot-attributed traces (http.response.is_bot) to be promoted
+ * @returns {{sdk: NodeSDK|null, meter: object|null}}
+ */
 function init(config = {}) {
     // OpenTelemetry is opt-in — bail out unless explicitly enabled. Mirrors the
     // OTEL_ENABLE guard in server/expressServer.js and server/renderer/handler.jsx
@@ -31,6 +452,9 @@ function init(config = {}) {
         return { sdk: null, meter: null }
     }
 
+    // Routes export failures to the app logger instead of OTEL's silent default.
+    setGlobalErrorHandler((err) => getLogger().error("❌ OpenTelemetry export error:", err))
+
     const {
         serviceName = "catalyst-server",
         serviceVersion = "1.0.0",
@@ -43,9 +467,14 @@ function init(config = {}) {
         metricHeaders = {},
         batchProcessorConfig = {},
         exportIntervalMillis = 10000,
+        diagnosticsIntervalMillis = 60000,
+        enableMemoryDiagnostics = true,
+        highMemoryRssMB,
+        highMemoryHeapUsedMB,
         instrumentations,
         samplingRate = 1.0,
         grpcCredentials,
+        errorSampling = {},
     } = config
 
     try {
@@ -67,9 +496,36 @@ function init(config = {}) {
             })
         }
 
-        const sampler = new ParentBasedSampler({
-            root: new TraceIdRatioBasedSampler(samplingRate),
-        })
+        let sampler
+        let spanProcessor
+
+        const isErrorSamplingEnabled = errorSampling && errorSampling.ENABLED === true
+
+        if (isErrorSamplingEnabled) {
+            getLogger().info("⚙️ OpenTelemetry initializing with Status Aware Error Sampling")
+            sampler = new StatusAwareSampler(samplingRate)
+
+            const errorSamplingConfig = {
+                samplingRate,
+                rate5xx: typeof errorSampling.RATE_5XX === "number" ? errorSampling.RATE_5XX : 1.0,
+                rate4xx: typeof errorSampling.RATE_4XX === "number" ? errorSampling.RATE_4XX : samplingRate,
+                exportFullTraceOnError: errorSampling.EXPORT_FULL_TRACE_ON_ERROR === true,
+                promoteHandledErrors: errorSampling.PROMOTE_HANDLED_ERRORS === true,
+                reportActualPromotionRate: errorSampling.REPORT_ACTUAL_PROMOTION_RATE === true,
+                skipPromotionCodes: Array.isArray(errorSampling.SKIP_PROMOTION_CODES)
+                    ? errorSampling.SKIP_PROMOTION_CODES
+                    : [408, 504, 524, 598, 599],
+                promoteBotTraffic: errorSampling.PROMOTE_BOT_TRAFFIC === true,
+                batchProcessorConfig,
+            }
+
+            spanProcessor = new PromotingSpanProcessor(otlpTraceExporter, errorSamplingConfig)
+        } else {
+            sampler = new ParentBasedSampler({
+                root: new TraceIdRatioBasedSampler(samplingRate),
+            })
+            spanProcessor = new BatchSpanProcessor(otlpTraceExporter, batchProcessorConfig)
+        }
 
         const sdkConfig = {
             resource: resourceFromAttributes({
@@ -77,7 +533,7 @@ function init(config = {}) {
                 [ATTR_SERVICE_VERSION]: serviceVersion,
                 [ATTR_DEPLOYMENT_ENVIRONMENT]: environment,
             }),
-            spanProcessor: new BatchSpanProcessor(otlpTraceExporter, batchProcessorConfig),
+            spanProcessor,
             instrumentations: instrumentations ?? [getNodeAutoInstrumentations()],
             sampler,
         }
@@ -92,6 +548,32 @@ function init(config = {}) {
         sdk.start()
         getLogger().info("✅ OpenTelemetry started successfully")
 
+        // Only under status-aware sampling: that's the only path with unbounded
+        // buffer growth to watch. Plain BatchSpanProcessor's queue is already bounded.
+        let heartbeatInterval = null
+        if (isErrorSamplingEnabled && enableMemoryDiagnostics) {
+            const memoryThresholds = { rssMB: highMemoryRssMB, heapUsedMB: highMemoryHeapUsedMB }
+            heartbeatInterval = setInterval(() => {
+                const mem = process.memoryUsage()
+                const ratio = (mem.rss / (mem.heapUsed || 1)).toFixed(2)
+                const bufferInfo = ` | Buffers: active=${spanProcessor.buffer.size}, promotedCache=${spanProcessor.promotedTraces.size}`
+                // Not ratioed against memory.request — no cgroup file exposes it.
+                const cgroupCurrentBytes = getCgroupCurrentBytes()
+                const cgroupInfo =
+                    cgroupCurrentBytes !== undefined
+                        ? ` | CgroupCurrent=${formatMB(cgroupCurrentBytes)}MB`
+                        : ""
+                getLogger().info(
+                    `📊 [OTEL Heartbeat] Memory: RSS=${formatMB(mem.rss)}MB, HeapUsed=${formatMB(mem.heapUsed)}MB, HeapTotal=${formatMB(mem.heapTotal)}MB, External=${formatMB(mem.external)}MB (RSS/Heap: ${ratio})${cgroupInfo}${bufferInfo} | Uptime: ${process.uptime().toFixed(0)}s`
+                )
+                checkHighMemoryAlert(mem, "heartbeat", memoryThresholds)
+            }, diagnosticsIntervalMillis)
+
+            if (heartbeatInterval.unref) {
+                heartbeatInterval.unref()
+            }
+        }
+
         // Initialize custom metrics only if metrics are enabled
         let meter = null
         if (metricProtocol) {
@@ -99,7 +581,14 @@ function init(config = {}) {
         }
 
         const gracefulShutdown = (signal) => {
-            getLogger().info(`📡 Received ${signal}, shutting down OpenTelemetry gracefully...`)
+            if (heartbeatInterval) {
+                clearInterval(heartbeatInterval)
+            }
+            const mem = process.memoryUsage()
+            const uptime = process.uptime().toFixed(1)
+            getLogger().info(
+                `📡 [OTEL Shutdown] Received ${signal}, shutting down OpenTelemetry gracefully... [Uptime: ${uptime}s, Memory at exit: RSS=${formatMB(mem.rss)}MB, HeapUsed=${formatMB(mem.heapUsed)}MB, HeapTotal=${formatMB(mem.heapTotal)}MB, External=${formatMB(mem.external)}MB]`
+            )
             sdk.shutdown()
                 .then(() => getLogger().info("✅ OpenTelemetry shutdown completed"))
                 .catch((error) => {

package/dist/scripts/build.js

@@ -361,31 +361,39 @@ async function build() {
     // Lets `serve` refuse a build that no longer matches config/config.json.
     writeBuildInfo(process.env.PWD, configJSON, argumentsObject)
 
-    const manifest = new Spinner().start("Generating offline manifest")
-    try {
-        await runBuildStep(
-            ["./dist/scripts/generateOfflineManifest.js"],
-            { ...spawnBase, env: baseEnv },
-            "Generating the offline manifest"
-        )
-    } catch (error) {
-        manifest.stop("fail", "Generating offline manifest failed")
-        // Without this the manifest step fell through to the top-level handler,
-        // which prints a raw stack trace. The step's output is captured, so
-        // show its tail -- the exit code alone says nothing about why -- and
-        // all of it under --verbose/--debug.
-        process.stderr.write(
-            diagnostic({
-                message: `Generating the offline manifest failed: ${error.message}`,
-                scope: "offline manifest",
-            }) + (outputMode === "default" ? fencedTail(error.output) : "")
+    // Same rule as isOfflineSnapshotsEnabled in vite.config.js.
+    const offlineSnapshotsDisabled = [false, "false"].includes(configJSON.OFFLINE_SNAPSHOTS_ENABLE)
+    if (offlineSnapshotsDisabled) {
+        console.log(
+            `${GUTTER}${t.dim(glyph.info)} ${t.dim("Skipped offline manifest (OFFLINE_SNAPSHOTS_ENABLE is false)")}`
         )
-        if (outputMode !== "default") {
-            process.stderr.write(verboseOutput({ output: error.output, mode: outputMode, error }))
+    } else {
+        const manifest = new Spinner().start("Generating offline manifest")
+        try {
+            await runBuildStep(
+                ["./dist/scripts/generateOfflineManifest.js"],
+                { ...spawnBase, env: baseEnv },
+                "Generating the offline manifest"
+            )
+        } catch (error) {
+            manifest.stop("fail", "Generating offline manifest failed")
+            // Without this the manifest step fell through to the top-level handler,
+            // which prints a raw stack trace. The step's output is captured, so
+            // show its tail -- the exit code alone says nothing about why -- and
+            // all of it under --verbose/--debug.
+            process.stderr.write(
+                diagnostic({
+                    message: `Generating the offline manifest failed: ${error.message}`,
+                    scope: "offline manifest",
+                }) + (outputMode === "default" ? fencedTail(error.output) : "")
+            )
+            if (outputMode !== "default") {
+                process.stderr.write(verboseOutput({ output: error.output, mode: outputMode, error }))
+            }
+            process.exit(exitCodeFrom(error))
         }
-        process.exit(exitCodeFrom(error))
+        manifest.stop("done", "Generated offline manifest")
     }
-    manifest.stop("done", "Generated offline manifest")
 
     printAssetTable(buildOutputPath, configJSON.BUILD_OUTPUT_PATH || "build")
 

package/dist/server/renderer/document/Body.jsx

@@ -27,18 +27,19 @@ export function Body(props) {
     fetcherData = {},
     children,
     safeArea = DEFAULT_SAFE_AREA,
-    nativeWebView = false
+    nativeWebView = false,
+    nonce
   } = props;
   return <body>
-            <script
+            <script nonce={nonce}
     /* eslint-disable-next-line risxss/catch-potential-xss-react */ dangerouslySetInnerHTML={{
       __html: `window.__SAFE_AREA_INITIAL__ = ${JSON.stringify(safeArea)}; window.__CATALYST_NATIVE_WEBVIEW__ = ${nativeWebView ? "true" : "false"}`
     }} />
             {process.env.NODE_ENV === "development" &&
     // Base pinned to Vite's resolved base by server/expressServer.js.
-    <script type="module" src={`${process.env.APP_MOUNT_PATH || ""}/client/index.js`}></script>}
+    <script type="module" nonce={nonce} src={`${process.env.APP_MOUNT_PATH || ""}/client/index.js`}></script>}
             {jsx}
-            <script
+            <script nonce={nonce}
     /* eslint-disable */ dangerouslySetInnerHTML={{
       __html: `
                     window.__INITIAL_STATE__ = ${serialize(initialState)}

package/dist/server/renderer/document/Head.jsx

@@ -16,12 +16,13 @@ export function Head(props) {
     metaTags,
     isBot,
     publicAssetPath,
+    nonce,
     children
   } = props;
   return <head>
             <meta charSet="utf-8" />
             <meta name="viewport" content="width=device-width, initial-scale=1" />
-            {process.env.NODE_ENV === "development" && <FastRefresh />}
+            {process.env.NODE_ENV === "development" && <FastRefresh nonce={nonce} />}
 
             {publicAssetPath && <link rel="preconnect" href={publicAssetPath} />}
             {publicAssetPath && <link rel="dns-prefetch" href={publicAssetPath} />}