@autter/runtime-node 1.0.0 → 1.2.0

package/dist/index.js

@@ -198,8 +198,19 @@ function createBrowserRelayHandler(opts) {
 }
 
 // src/server.ts
-import { trace, SpanStatusCode } from "@opentelemetry/api";
-import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import { ServerResponse } from "http";
+import {
+  context,
+  trace,
+  SpanKind,
+  SpanStatusCode,
+  TraceFlags
+} from "@opentelemetry/api";
+import { getRPCMetadata, RPCType } from "@opentelemetry/core";
+import {
+  AggregationTemporalityPreference,
+  OTLPMetricExporter
+} from "@opentelemetry/exporter-metrics-otlp-http";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
 import { Resource } from "@opentelemetry/resources";
@@ -210,13 +221,748 @@ import {
   ParentBasedSampler,
   TraceIdRatioBasedSampler,
   AlwaysOnSampler,
-  BasicTracerProvider
+  BasicTracerProvider,
+  SamplingDecision
 } from "@opentelemetry/sdk-trace-base";
 import {
   ATTR_SERVICE_NAME,
   ATTR_SERVICE_VERSION
 } from "@opentelemetry/semantic-conventions";
+
+// src/lifecycle.ts
+import { ExportResultCode } from "@opentelemetry/core";
+var debugEnabled = process.env.AUTTER_DEBUG === "1" || process.env.AUTTER_DEBUG === "true";
+function setDebugMode(enabled) {
+  debugEnabled = enabled;
+}
+function isDebugEnabled() {
+  return debugEnabled;
+}
+function debugLog(...args) {
+  if (debugEnabled) {
+    console.error("[autter]", ...args);
+  }
+}
+var TelemetryStats = class {
+  /** Captures queued via our APIs and not yet seen leaving an exporter. */
+  pendingCaptures = 0;
+  markCaptured(count = 1) {
+    this.pendingCaptures += count;
+  }
+  /** An exporter batch left the process successfully. */
+  onExported(spanCount) {
+    this.pendingCaptures = Math.max(0, this.pendingCaptures - spanCount);
+    if (debugEnabled && spanCount > 0) {
+      debugLog(`exported ${spanCount} span${spanCount === 1 ? "" : "s"}`);
+    }
+  }
+  markAllFlushed() {
+    this.pendingCaptures = 0;
+  }
+};
+var telemetryStats = new TelemetryStats();
+var CountingExporter = class {
+  constructor(inner) {
+    this.inner = inner;
+  }
+  inner;
+  export(spans, resultCallback) {
+    this.inner.export(spans, (result) => {
+      if (result.code === ExportResultCode.SUCCESS) {
+        telemetryStats.onExported(spans.length);
+      }
+      resultCallback(result);
+    });
+  }
+  shutdown() {
+    return this.inner.shutdown();
+  }
+  forceFlush() {
+    return this.inner.forceFlush?.() ?? Promise.resolve();
+  }
+};
+var flushTargets = /* @__PURE__ */ new Map();
+function registerFlushTarget(key, target) {
+  flushTargets.set(key, target);
+}
+function unregisterFlushTargets() {
+  flushTargets.clear();
+}
+function activeFlushTargets() {
+  return [...flushTargets.values()];
+}
+var DEFAULT_SIGNALS = ["SIGINT", "SIGTERM"];
+var SIGNAL_EXIT_CODES = {
+  SIGINT: 130,
+  SIGTERM: 143
+};
+function installAutterAutoFlush(options = {}) {
+  const logger = options.logger ?? console;
+  const log = options.log !== false;
+  const warnOnUnflushedExit = options.warnOnUnflushedExit !== false;
+  const timeoutMs = Math.max(100, options.timeoutMs ?? 3e3);
+  const signals = options.signals ?? DEFAULT_SIGNALS;
+  let disposed = false;
+  let inFlight = null;
+  const doFlush = async () => {
+    const targets = options.targets ?? activeFlushTargets();
+    if (targets.length === 0) return true;
+    let timer;
+    const timedOut = new Promise((resolve) => {
+      timer = setTimeout(() => resolve(false), timeoutMs);
+    });
+    const drained = Promise.allSettled(
+      targets.map(
+        (target) => Promise.resolve().then(() => target.forceFlush())
+      )
+    ).then(
+      (results) => results.every((result) => result.status === "fulfilled")
+    );
+    let ok;
+    try {
+      ok = await Promise.race([drained, timedOut]);
+    } finally {
+      clearTimeout(timer);
+    }
+    if (ok) {
+      telemetryStats.markAllFlushed();
+      if (log) {
+        logger.log("[autter] telemetry flushed");
+      }
+    } else {
+      logger.error(
+        `[autter] flush did not finish within ${timeoutMs}ms \u2014 some telemetry may be lost`
+      );
+    }
+    return ok;
+  };
+  const flush = (reason = "exit") => {
+    if (disposed) return Promise.resolve(false);
+    if (!inFlight) {
+      inFlight = doFlush().finally(() => {
+        inFlight = null;
+      });
+    }
+    if (isDebugEnabled()) debugLog(`auto-flush requested on ${reason}`);
+    return inFlight;
+  };
+  const signalCounts = /* @__PURE__ */ new Map();
+  const onSignal = (signal) => {
+    const count = (signalCounts.get(signal) ?? 0) + 1;
+    signalCounts.set(signal, count);
+    const exitCode = SIGNAL_EXIT_CODES[signal] ?? 1;
+    if (count > 1) {
+      process.exit(exitCode);
+    }
+    const otherHandlers = process.listenerCount(signal) - 1;
+    if (otherHandlers > 0) {
+      void flush(signal);
+      return;
+    }
+    void flush(signal).finally(() => {
+      process.exit(exitCode);
+    });
+  };
+  const MAX_BEFORE_EXIT_FLUSHES = 10;
+  let beforeExitFlushes = 0;
+  const onBeforeExit = () => {
+    if (disposed) return;
+    const targets = options.targets ?? activeFlushTargets();
+    const worthIt = beforeExitFlushes === 0 ? targets.length > 0 : telemetryStats.pendingCaptures > 0;
+    if (!worthIt || beforeExitFlushes >= MAX_BEFORE_EXIT_FLUSHES) return;
+    beforeExitFlushes++;
+    const keepAlive = setTimeout(() => {
+    }, timeoutMs);
+    void flush("beforeExit").finally(() => clearTimeout(keepAlive));
+  };
+  const onExit = () => {
+    if (!warnOnUnflushedExit || disposed) return;
+    const pending = telemetryStats.pendingCaptures;
+    if (pending > 0) {
+      logger.error(
+        `[autter] process exiting with ~${pending} captured item(s) never confirmed exported \u2014 await autter.shutdown() on graceful shutdown`
+      );
+    }
+  };
+  const signalHandlers = /* @__PURE__ */ new Map();
+  process.on("beforeExit", onBeforeExit);
+  for (const signal of signals) {
+    const handler = () => onSignal(signal);
+    signalHandlers.set(signal, handler);
+    process.on(signal, handler);
+  }
+  process.on("exit", onExit);
+  return {
+    flush,
+    dispose() {
+      disposed = true;
+      process.removeListener("beforeExit", onBeforeExit);
+      process.removeListener("exit", onExit);
+      for (const [signal, handler] of signalHandlers) {
+        process.removeListener(signal, handler);
+      }
+      signalHandlers.clear();
+    }
+  };
+}
+
+// src/redact.ts
+var SENSITIVE_KEY_PATTERNS = [
+  /e-?mail/,
+  /pass(word|wd|phrase)|^pass$/,
+  /token/,
+  /secret/,
+  /credential/,
+  /(api|access|secret|private|consumer|client|signing|encryption)-?[_.]?key/,
+  /^-?x?-?authorization$|^auth(-|_|$)|bearer/,
+  /cookie/,
+  /phone|msisdn/,
+  /\bssn\b|social[-_ ]?security/,
+  /cvv|cvc|card([-_. ]?(number|num|no))?$/,
+  /credit[-_.]?card/,
+  /connection[-_.]?string/,
+  /recovery[-_.]?code|\botp\b|magic[-_.]?link/
+];
+var PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g;
+var EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
+var JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g;
+var OPENAI_KEY_RE = /\bsk-[A-Za-z0-9]{20,}\b/g;
+var GITHUB_TOKEN_RE = /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g;
+var AWS_KEY_RE = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g;
+var SLACK_TOKEN_RE = /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g;
+var BEARER_RE = /\bbearer\s+[A-Za-z0-9._~+/=-]{10,}/gi;
+var URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi;
+function toCaseInsensitive(pattern) {
+  return typeof pattern === "string" ? new RegExp(pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i") : new RegExp(pattern.source, pattern.flags.includes("i") ? pattern.flags : `${pattern.flags}i`);
+}
+function compile(options) {
+  const keyPatterns = [...SENSITIVE_KEY_PATTERNS];
+  const valuePatterns = [
+    JWT_RE,
+    OPENAI_KEY_RE,
+    GITHUB_TOKEN_RE,
+    AWS_KEY_RE,
+    SLACK_TOKEN_RE,
+    BEARER_RE
+  ];
+  if (options?.additionalKeyPatterns) {
+    for (const p of options.additionalKeyPatterns) {
+      keyPatterns.push(toCaseInsensitive(p));
+    }
+  }
+  if (options?.additionalValuePatterns) {
+    for (const p of options.additionalValuePatterns) {
+      if (!valuePatterns.some((existing) => existing.source === (typeof p === "string" ? p : p.source))) {
+        valuePatterns.push(typeof p === "string" ? new RegExp(p, "gi") : p);
+      }
+    }
+  }
+  return {
+    keyPatterns,
+    valuePatterns,
+    urlCredentials: URL_CREDENTIALS_RE,
+    privateKeyBlock: PRIVATE_KEY_BLOCK_RE,
+    mask: options?.mask ?? "[redacted]",
+    scrubEmailValues: options?.scrubEmailValues !== false
+  };
+}
+function redactString(value, r) {
+  let out = value.replace(r.privateKeyBlock, r.mask);
+  for (const re of r.valuePatterns) {
+    out = out.replace(re, r.mask);
+  }
+  out = out.replace(r.urlCredentials, "$1" + r.mask + "@");
+  if (r.scrubEmailValues) {
+    out = out.replace(EMAIL_VALUE_RE, r.mask);
+  }
+  return out;
+}
+var MAX_REDACTION_DEPTH = 64;
+var MAX_REDACTION_WORK = 1e4;
+var MAX_COLLECTION_ENTRIES = 1e3;
+function canTraverse(depth, state) {
+  if (depth > MAX_REDACTION_DEPTH || state.remainingWork <= 0) return false;
+  state.remainingWork -= 1;
+  return true;
+}
+function redactValue(value, r, state, depth) {
+  if (typeof value === "string") return redactString(value, r);
+  let isArray = false;
+  try {
+    isArray = Array.isArray(value);
+  } catch {
+    return r.mask;
+  }
+  if (isArray) {
+    return redactArray(value, r, state, depth);
+  }
+  if (typeof value === "object" && value !== null) {
+    return redactObject(value, r, state, depth);
+  }
+  return value;
+}
+function redactArray(value, r, state, depth) {
+  const existing = state.ancestors.get(value);
+  if (existing) return r.mask;
+  if (!canTraverse(depth, state)) return r.mask;
+  const out = [];
+  state.ancestors.set(value, out);
+  let length;
+  try {
+    length = value.length;
+  } catch {
+    state.ancestors.delete(value);
+    return r.mask;
+  }
+  const limit = Math.min(length, MAX_COLLECTION_ENTRIES);
+  for (let i = 0; i < limit; i += 1) {
+    let item;
+    try {
+      item = value[i];
+    } catch {
+      out.push(r.mask);
+      continue;
+    }
+    out.push(redactValue(item, r, state, depth + 1));
+  }
+  if (length > limit) {
+    out.push(r.mask);
+  }
+  state.ancestors.delete(value);
+  return out;
+}
+function redactObject(value, r, state, depth) {
+  const existing = state.ancestors.get(value);
+  if (existing) return r.mask;
+  if (!canTraverse(depth, state)) return r.mask;
+  const out = {};
+  state.ancestors.set(value, out);
+  let count = 0;
+  let truncated = false;
+  try {
+    for (const key in value) {
+      if (!Object.prototype.propertyIsEnumerable.call(
+        value,
+        key
+      )) {
+        continue;
+      }
+      if (count >= MAX_COLLECTION_ENTRIES) {
+        truncated = true;
+        break;
+      }
+      let nestedValue;
+      try {
+        nestedValue = value[key];
+      } catch {
+        out[key] = r.mask;
+        count += 1;
+        continue;
+      }
+      count += 1;
+      out[key] = isSensitiveKey(key, nestedValue, r) ? r.mask : redactValue(
+        nestedValue,
+        r,
+        state,
+        depth + 1
+      );
+    }
+  } catch {
+    truncated = true;
+  }
+  if (truncated) {
+    out.__redaction_truncated__ = r.mask;
+  }
+  state.ancestors.delete(value);
+  return out;
+}
+var USAGE_TOKEN_KEYS = /* @__PURE__ */ new Set([
+  "input_tokens",
+  "output_tokens",
+  "prompt_tokens",
+  "completion_tokens",
+  "total_tokens",
+  "token_count",
+  "gen_ai.usage.input_tokens",
+  "gen_ai.usage.output_tokens",
+  "gen_ai.usage.prompt_tokens",
+  "gen_ai.usage.completion_tokens",
+  "gen_ai.usage.total_tokens",
+  "gen_ai.usage.token_count"
+]);
+function isSensitiveKey(key, value, r) {
+  const lowered = key.toLowerCase();
+  if (USAGE_TOKEN_KEYS.has(lowered)) {
+    return !(typeof value === "number" && Number.isFinite(value) && value >= 0);
+  }
+  return r.keyPatterns.some((re) => re.test(lowered));
+}
+function redactAttributes(attributes, options) {
+  const r = compile(options);
+  return redactWith(attributes, r);
+}
+function redactWith(attributes, r) {
+  const out = {};
+  if (!attributes) return out;
+  const state = {
+    ancestors: /* @__PURE__ */ new WeakMap(),
+    remainingWork: MAX_REDACTION_WORK
+  };
+  let count = 0;
+  let truncated = false;
+  try {
+    for (const key in attributes) {
+      if (!Object.prototype.propertyIsEnumerable.call(
+        attributes,
+        key
+      )) {
+        continue;
+      }
+      if (count >= MAX_COLLECTION_ENTRIES || state.remainingWork <= 0) {
+        truncated = true;
+        break;
+      }
+      let value;
+      try {
+        value = attributes[key];
+      } catch {
+        out[key] = r.mask;
+        count += 1;
+        state.remainingWork -= 1;
+        continue;
+      }
+      count += 1;
+      state.remainingWork -= 1;
+      if (value === void 0) continue;
+      out[key] = isSensitiveKey(key, value, r) ? r.mask : redactValue(value, r, state, 0);
+    }
+  } catch {
+    truncated = true;
+  }
+  if (truncated) {
+    out.__redaction_truncated__ = r.mask;
+  }
+  return out;
+}
+function makeRedactor(options) {
+  if (options === false) {
+    return (attributes) => ({ ...attributes ?? {} });
+  }
+  const r = compile(options === true ? void 0 : options);
+  return (attributes) => redactWith(attributes, r);
+}
+
+// src/server.ts
+var LLM_MARKER_ATTRIBUTES = [
+  "gen_ai.operation.name",
+  "gen_ai.system",
+  "gen_ai.provider.name",
+  "gen_ai.request.model",
+  "autter.llm.provider"
+];
+function isLlmSpan(name, attributes) {
+  if (name.startsWith("ai.") || name.startsWith("gen_ai.")) return true;
+  return LLM_MARKER_ATTRIBUTES.some((key) => attributes[key] !== void 0);
+}
+var LlmAwareSampler = class {
+  constructor(delegate) {
+    this.delegate = delegate;
+  }
+  delegate;
+  shouldSample(ctx, traceId, spanName, spanKind, attributes, links) {
+    if (isLlmSpan(spanName, attributes)) {
+      return { decision: SamplingDecision.RECORD_AND_SAMPLED };
+    }
+    return this.delegate.shouldSample(
+      ctx,
+      traceId,
+      spanName,
+      spanKind,
+      attributes,
+      links
+    );
+  }
+  toString() {
+    return `LlmAware(${this.delegate.toString()})`;
+  }
+};
+var RecordUnsampledSampler = class {
+  constructor(delegate) {
+    this.delegate = delegate;
+  }
+  delegate;
+  shouldSample(ctx, traceId, spanName, spanKind, attributes, links) {
+    const result = this.delegate.shouldSample(
+      ctx,
+      traceId,
+      spanName,
+      spanKind,
+      attributes,
+      links
+    );
+    if (result.decision === SamplingDecision.NOT_RECORD) {
+      return { ...result, decision: SamplingDecision.RECORD };
+    }
+    return result;
+  }
+  toString() {
+    return `RecordUnsampled(${this.delegate.toString()})`;
+  }
+};
+var RETENTION_MAX_SPANS_PER_TRACE = 256;
+var RETENTION_MAX_BUFFERED_SPANS = 5e3;
+var RETENTION_SWEEP_INTERVAL_MS = 1e4;
+var RETENTION_TRACE_TTL_MS = 3e4;
+function spanIndicatesError(span) {
+  if (span.status.code === SpanStatusCode.ERROR) return true;
+  return span.events.some((event) => event.name === "exception");
+}
+var TAIL_RETAINED_ATTR = "autter.tail_retained";
+function withSampledFlag(span) {
+  const spanContext = {
+    ...span.spanContext(),
+    traceFlags: span.spanContext().traceFlags | TraceFlags.SAMPLED
+  };
+  return {
+    name: span.name,
+    kind: span.kind,
+    spanContext: () => spanContext,
+    parentSpanId: span.parentSpanId,
+    startTime: span.startTime,
+    endTime: span.endTime,
+    status: span.status,
+    attributes: { ...span.attributes, [TAIL_RETAINED_ATTR]: true },
+    links: span.links,
+    events: span.events,
+    duration: span.duration,
+    ended: span.ended,
+    resource: span.resource,
+    instrumentationLibrary: span.instrumentationLibrary,
+    droppedAttributesCount: span.droppedAttributesCount,
+    droppedEventsCount: span.droppedEventsCount,
+    droppedLinksCount: span.droppedLinksCount
+  };
+}
+var ErrorTraceRetentionProcessor = class {
+  constructor(inner) {
+    this.inner = inner;
+    this.sweeper = setInterval(() => this.sweep(), RETENTION_SWEEP_INTERVAL_MS);
+    this.sweeper.unref?.();
+  }
+  inner;
+  traces = /* @__PURE__ */ new Map();
+  bufferedSpans = 0;
+  sweeper;
+  onStart(_span, _parentContext) {
+  }
+  onEnd(span) {
+    const ctx = span.spanContext();
+    if ((ctx.traceFlags & TraceFlags.SAMPLED) !== 0) return;
+    const entry = this.entryFor(ctx.traceId);
+    if (entry.retained) {
+      this.forward(span);
+    } else if (spanIndicatesError(span)) {
+      entry.retained = true;
+      this.flush(entry);
+      this.forward(span);
+    } else if (entry.spans.length < RETENTION_MAX_SPANS_PER_TRACE) {
+      entry.spans.push(span);
+      this.bufferedSpans++;
+      this.evictIfOverBudget();
+    }
+    if (span.parentSpanId === void 0 && !entry.retained) {
+      this.drop(ctx.traceId);
+    }
+  }
+  /** Promote the active trace, if any — called on captured errors. */
+  retainActiveTrace() {
+    const active2 = trace.getActiveSpan()?.spanContext();
+    if (!active2 || (active2.traceFlags & TraceFlags.SAMPLED) !== 0) return;
+    const entry = this.entryFor(active2.traceId);
+    if (entry.retained) return;
+    entry.retained = true;
+    this.flush(entry);
+  }
+  forceFlush() {
+    return this.inner.forceFlush();
+  }
+  shutdown() {
+    clearInterval(this.sweeper);
+    this.traces.clear();
+    this.bufferedSpans = 0;
+    return this.inner.shutdown();
+  }
+  entryFor(traceId) {
+    let entry = this.traces.get(traceId);
+    if (!entry) {
+      entry = { spans: [], retained: false, firstSeenAtMs: Date.now() };
+      this.traces.set(traceId, entry);
+    }
+    return entry;
+  }
+  forward(span) {
+    this.inner.onEnd(withSampledFlag(span));
+  }
+  flush(entry) {
+    for (const span of entry.spans) this.forward(span);
+    this.bufferedSpans -= entry.spans.length;
+    entry.spans = [];
+  }
+  drop(traceId) {
+    const entry = this.traces.get(traceId);
+    if (!entry) return;
+    this.bufferedSpans -= entry.spans.length;
+    this.traces.delete(traceId);
+  }
+  evictIfOverBudget() {
+    if (this.bufferedSpans <= RETENTION_MAX_BUFFERED_SPANS) return;
+    for (const [traceId, entry] of this.traces) {
+      if (entry.spans.length === 0) continue;
+      this.drop(traceId);
+      if (this.bufferedSpans <= RETENTION_MAX_BUFFERED_SPANS) return;
+    }
+  }
+  sweep() {
+    const cutoff = Date.now() - RETENTION_TRACE_TTL_MS;
+    for (const [traceId, entry] of this.traces) {
+      if (entry.firstSeenAtMs <= cutoff) this.drop(traceId);
+    }
+  }
+};
+function llmBaseAttributes(info) {
+  return {
+    "gen_ai.operation.name": info.operation ?? "chat",
+    // Both keys: gen_ai.system is the pre-1.37 semconv name and what
+    // most tooling still matches on.
+    "gen_ai.provider.name": info.provider,
+    "gen_ai.system": info.provider,
+    "gen_ai.request.model": info.model,
+    ...info.userId ? { "autter.user_id": info.userId } : {},
+    ...info.sessionId ? { "autter.session_id": info.sessionId } : {},
+    // Caller-supplied extras go through redaction like every other
+    // custom attribute family (prompts can contain emails/keys).
+    ...info.attributes ? activeRedactor(info.attributes) : {}
+  };
+}
+function usageAttributes(usage) {
+  const attrs = {};
+  if (usage.inputTokens !== void 0) {
+    attrs["gen_ai.usage.input_tokens"] = Math.max(0, Math.round(usage.inputTokens));
+  }
+  if (usage.outputTokens !== void 0) {
+    attrs["gen_ai.usage.output_tokens"] = Math.max(
+      0,
+      Math.round(usage.outputTokens)
+    );
+  }
+  if (usage.costUsd !== void 0) {
+    attrs["autter.llm.cost_usd"] = usage.costUsd;
+  }
+  return attrs;
+}
+async function runLlmSpan(tracer, info, fn) {
+  const span = tracer.startSpan(`${info.operation ?? "chat"} ${info.model}`, {
+    kind: SpanKind.CLIENT,
+    attributes: llmBaseAttributes(info)
+  });
+  const setResponseModel = (model) => span.setAttribute("gen_ai.response.model", model);
+  const handle = {
+    setUsage: (usage) => span.setAttributes(usageAttributes(usage)),
+    setCost: (usd) => span.setAttribute("autter.llm.cost_usd", usd),
+    setResponseModel,
+    setModel: setResponseModel,
+    setAttributes: (attributes) => span.setAttributes(attributes)
+  };
+  try {
+    const result = await context.with(
+      trace.setSpan(context.active(), span),
+      () => fn(handle)
+    );
+    span.setStatus({ code: SpanStatusCode.OK });
+    return result;
+  } catch (err) {
+    if (err instanceof Error) {
+      span.recordException(err);
+      span.setAttribute("error.type", err.name);
+    }
+    span.setStatus({
+      code: SpanStatusCode.ERROR,
+      message: err instanceof Error ? err.message : String(err)
+    });
+    throw err;
+  } finally {
+    span.end();
+  }
+}
+function recordLlmCall(tracer, call) {
+  const durationMs = Math.max(0, call.durationMs ?? 0);
+  const endTime = /* @__PURE__ */ new Date();
+  const startTime = new Date(endTime.getTime() - durationMs);
+  const span = tracer.startSpan(
+    `${call.operation ?? "chat"} ${call.model}`,
+    {
+      kind: SpanKind.CLIENT,
+      startTime,
+      attributes: { ...llmBaseAttributes(call), ...usageAttributes(call) }
+    }
+  );
+  if (call.error !== void 0 && call.error !== null) {
+    if (call.error instanceof Error) {
+      span.recordException(call.error);
+      span.setAttribute("error.type", call.error.name);
+    }
+    span.setStatus({
+      code: SpanStatusCode.ERROR,
+      message: call.error instanceof Error ? call.error.message : String(call.error)
+    });
+  } else {
+    span.setStatus({ code: SpanStatusCode.OK });
+  }
+  span.end(endTime);
+}
+async function runWithSpan(tracer, name, fn, attributes) {
+  const span = tracer.startSpan(name, { attributes });
+  try {
+    const result = await context.with(
+      trace.setSpan(context.active(), span),
+      fn
+    );
+    span.setStatus({ code: SpanStatusCode.OK });
+    return result;
+  } catch (err) {
+    if (err instanceof Error) span.recordException(err);
+    span.setStatus({
+      code: SpanStatusCode.ERROR,
+      message: err instanceof Error ? err.message : String(err)
+    });
+    throw err;
+  } finally {
+    span.end();
+  }
+}
+function expressRouteOf(req) {
+  const props = req;
+  const path = props?.route?.path;
+  if (typeof path !== "string" || path === "") return null;
+  const base = typeof props?.baseUrl === "string" ? props.baseUrl : "";
+  const route = base + path;
+  return route.startsWith("/") ? route : null;
+}
+function captureExpressRoute(_span, response) {
+  if (!(response instanceof ServerResponse)) return;
+  const rpcMetadata = getRPCMetadata(context.active());
+  if (rpcMetadata?.type !== RPCType.HTTP) return;
+  const setRoute = () => {
+    if (rpcMetadata.route) return;
+    const route = expressRouteOf(response.req);
+    if (route) rpcMetadata.route = route;
+  };
+  response.prependListener("finish", setRoute);
+  response.prependListener("close", setRoute);
+}
 var active = null;
+var activeAlwaysOnProvider = null;
+var activeRedactor = makeRedactor(true);
 function initAutterServer(options) {
   if (active) return active;
   const endpoint = (options.endpoint ?? "https://otlp.autter.dev").replace(
@@ -225,65 +971,112 @@ function initAutterServer(options) {
   );
   const headers = { authorization: `Bearer ${options.apiKey}` };
   const environment = options.environment ?? process.env.NODE_ENV ?? "production";
+  if (isDebugEnabled() || options.debug === true) setDebugMode(true);
+  debugLog(`initialising service=${options.service} endpoint=${endpoint}`);
+  activeRedactor = makeRedactor(options.redactAttributes ?? true);
   const resource = new Resource({
     [ATTR_SERVICE_NAME]: options.service,
     ...options.release ? { [ATTR_SERVICE_VERSION]: options.release } : {},
-    "deployment.environment": environment
+    "deployment.environment": environment,
+    // Tells the ingester request metrics arrive on the metrics pipe, so
+    // it must not also fold our server spans into usage rollups (that
+    // would double-count every sampled request).
+    "autter.metrics_wired": true
+  });
+  const headSampler = new ParentBasedSampler({
+    root: new TraceIdRatioBasedSampler(options.traceSampleRate ?? 0.01)
+  });
+  const retainOnError = options.retainTracesOnError !== false;
+  let sampler = retainOnError ? new RecordUnsampledSampler(headSampler) : headSampler;
+  if (options.llmTracing !== false) sampler = new LlmAwareSampler(sampler);
+  const errorTraceBuffer = retainOnError ? new ErrorTraceRetentionProcessor(
+    new BatchSpanProcessor(
+      new CountingExporter(
+        new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers })
+      ),
+      { scheduledDelayMillis: 2e3 }
+    )
+  ) : null;
+  const mainSpanProcessor = new BatchSpanProcessor(
+    new CountingExporter(
+      new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers })
+    )
+  );
+  const metricReader = new PeriodicExportingMetricReader({
+    exporter: new OTLPMetricExporter({
+      url: `${endpoint}/v1/metrics`,
+      headers,
+      // Deltas, not lifetime totals: the ingester SUMs data points
+      // into runtime_metrics_1m, and the default (cumulative)
+      // temporality would re-count every past request on each
+      // 60 s export.
+      temporalityPreference: AggregationTemporalityPreference.DELTA
+    }),
+    exportIntervalMillis: options.metricIntervalMs ?? 6e4
   });
   const sdk = new NodeSDK({
     resource,
-    sampler: new ParentBasedSampler({
-      root: new TraceIdRatioBasedSampler(options.traceSampleRate ?? 0.01)
-    }),
-    traceExporter: new OTLPTraceExporter({
-      url: `${endpoint}/v1/traces`,
-      headers
-    }),
-    metricReader: new PeriodicExportingMetricReader({
-      exporter: new OTLPMetricExporter({
-        url: `${endpoint}/v1/metrics`,
-        headers
-      }),
-      exportIntervalMillis: options.metricIntervalMs ?? 6e4
-    }),
+    sampler,
+    spanProcessors: [
+      mainSpanProcessor,
+      ...errorTraceBuffer ? [errorTraceBuffer] : []
+    ],
+    metricReader,
     instrumentations: [
-      new HttpInstrumentation(),
+      new HttpInstrumentation({ responseHook: captureExpressRoute }),
       ...options.instrumentations ?? []
     ]
   });
   sdk.start();
-  const errorProvider = new BasicTracerProvider({
+  const alwaysOnProvider = new BasicTracerProvider({
     resource,
     sampler: new AlwaysOnSampler(),
     spanProcessors: [
       new BatchSpanProcessor(
-        new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }),
+        new CountingExporter(
+          new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers })
+        ),
         { scheduledDelayMillis: 2e3 }
       )
     ]
   });
-  const errorTracer = errorProvider.getTracer("autter-errors");
+  const errorTracer = alwaysOnProvider.getTracer("autter-errors");
+  const processTracer = alwaysOnProvider.getTracer("autter-processes");
+  const llmTracer = alwaysOnProvider.getTracer("autter-llm");
   function captureException2(error, attributes) {
+    errorTraceBuffer?.retainActiveTrace();
+    telemetryStats.markCaptured();
     const isError = error instanceof Error;
     const message = isError ? error.message : String(error);
     const span = errorTracer.startSpan(isError ? error.name : "Error", {
-      attributes: { "autter.severity": "error", ...attributes }
+      attributes: {
+        "autter.severity": "error",
+        // Redaction is applied here, not at export time: PII must not
+        // leave the process even if an exporter misbehaves.
+        ...activeRedactor(attributes)
+      }
     });
-    if (isError) {
+    if (isError && error.stack) {
       span.recordException(error);
     } else {
+      const stack = new Error().stack?.split("\n").filter((line, i) => i === 0 || !line.includes("captureException")).join("\n");
       span.addEvent("exception", {
-        "exception.type": "Error",
-        "exception.message": message
+        "exception.type": isError ? error.name : "Error",
+        "exception.message": message,
+        ...stack ? { "exception.stacktrace": stack } : {}
       });
     }
     span.setStatus({ code: SpanStatusCode.ERROR, message });
     span.end();
   }
   function captureMessage2(message, severity = "warning", attributes) {
+    if (severity === "error" || severity === "fatal") {
+      errorTraceBuffer?.retainActiveTrace();
+    }
+    telemetryStats.markCaptured();
     const stack = new Error().stack?.split("\n").filter((line, i) => i === 0 || !line.includes("captureMessage")).join("\n");
     const span = errorTracer.startSpan("Message", {
-      attributes: { "autter.severity": severity, ...attributes }
+      attributes: { "autter.severity": severity, ...activeRedactor(attributes) }
     });
     span.addEvent("exception", {
       "exception.type": "Message",
@@ -297,19 +1090,54 @@ function initAutterServer(options) {
   if (options.captureGlobalErrors !== false) {
     process.on("uncaughtExceptionMonitor", (err) => {
       captureException2(err, { "autter.unhandled": true });
-      void errorProvider.forceFlush().catch(() => {
+      void flushTarget.forceFlush();
+    });
+    process.on("unhandledRejection", (reason) => {
+      captureException2(reason, {
+        "autter.unhandled": true,
+        ...reason instanceof Error ? {} : { "autter.severity": "warning" }
       });
     });
   }
+  const flushTarget = {
+    forceFlush: async () => {
+      const results = await Promise.allSettled([
+        Promise.resolve().then(() => alwaysOnProvider.forceFlush()),
+        Promise.resolve().then(() => mainSpanProcessor.forceFlush()),
+        ...errorTraceBuffer ? [Promise.resolve().then(() => errorTraceBuffer.forceFlush())] : [],
+        Promise.resolve().then(() => metricReader.forceFlush())
+      ]);
+      const failed = results.find(
+        (result) => result.status === "rejected"
+      );
+      if (failed) {
+        throw failed.reason;
+      }
+    }
+  };
+  registerFlushTarget("active-server", flushTarget);
+  let autoFlushHandle = null;
+  if (options.autoFlush !== false) {
+    autoFlushHandle = installAutterAutoFlush();
+  }
   const server = {
     captureException: captureException2,
     captureMessage: captureMessage2,
+    withProcessSpan: (name, fn, attributes) => runWithSpan(processTracer, name, fn, activeRedactor(attributes)),
+    withLlmCall: (info, fn) => runLlmSpan(llmTracer, info, fn),
+    trackLlmCall: (call) => recordLlmCall(llmTracer, call),
     shutdown: async () => {
       active = null;
-      await Promise.allSettled([errorProvider.shutdown(), sdk.shutdown()]);
+      activeAlwaysOnProvider = null;
+      autoFlushHandle?.dispose();
+      autoFlushHandle = null;
+      unregisterFlushTargets();
+      telemetryStats.markAllFlushed();
+      await Promise.allSettled([alwaysOnProvider.shutdown(), sdk.shutdown()]);
     }
   };
   active = server;
+  activeAlwaysOnProvider = alwaysOnProvider;
   return server;
 }
 function captureException(error, attributes) {
@@ -325,6 +1153,21 @@ function captureException(error, attributes) {
   });
   span.end();
 }
+function withProcessSpan(name, fn, attributes) {
+  if (active) return active.withProcessSpan(name, fn, attributes);
+  return runWithSpan(trace.getTracer("autter-processes"), name, fn, attributes);
+}
+function withLlmCall(info, fn) {
+  if (active) return active.withLlmCall(info, fn);
+  return runLlmSpan(trace.getTracer("autter-llm"), info, fn);
+}
+function trackLlmCall(call) {
+  if (active) {
+    active.trackLlmCall(call);
+    return;
+  }
+  recordLlmCall(trace.getTracer("autter-llm"), call);
+}
 function captureMessage(message, severity = "warning", attributes) {
   if (active) {
     active.captureMessage(message, severity, attributes);
@@ -341,11 +1184,271 @@ function captureMessage(message, severity = "warning", attributes) {
   span.setStatus({ code: SpanStatusCode.ERROR, message });
   span.end();
 }
+function makeSafeCapture(options) {
+  const clean = (attributes) => attributes ? { ...redactAttributes(attributes, options) } : void 0;
+  return {
+    captureException: (error, attributes) => captureException(error, clean(attributes)),
+    captureMessage: (message, severity, attributes) => captureMessage(message, severity ?? "warning", clean(attributes))
+  };
+}
+function alwaysOnTracer(name) {
+  return activeAlwaysOnProvider ? activeAlwaysOnProvider.getTracer(name) : trace.getTracer(name);
+}
+function autterLlmTracer() {
+  return alwaysOnTracer("autter-llm");
+}
+async function emitLlmSelftestTrace() {
+  const provider = activeAlwaysOnProvider;
+  if (!provider) {
+    throw new Error(
+      "emitLlmSelftestTrace() requires initAutterServer() to have been called"
+    );
+  }
+  let traceId = "";
+  await withLlmCall(
+    {
+      provider: "autter-selftest",
+      model: "autter-selftest",
+      attributes: { "autter.selftest": true }
+    },
+    (llm) => {
+      traceId = trace.getActiveSpan()?.spanContext().traceId ?? "";
+      llm.setUsage({ inputTokens: 1, outputTokens: 1 });
+      llm.setCost(0);
+    }
+  );
+  await provider.forceFlush();
+  return { traceId };
+}
+
+// src/llm-instrument.ts
+import {
+  SpanKind as SpanKind2,
+  SpanStatusCode as SpanStatusCode2
+} from "@opentelemetry/api";
+var CALL_METHODS = /* @__PURE__ */ new Set([
+  "create",
+  // openai chat/completions/embeddings/responses, anthropic messages
+  "stream",
+  // anthropic messages.stream
+  "generateContent",
+  "generateContentStream",
+  // google genai
+  "embedContent",
+  "batchEmbedContents",
+  "parse"
+  // openai beta structured outputs
+]);
+function detectProvider(client) {
+  const name = client.constructor?.name?.toLowerCase() ?? "";
+  for (const known of ["openai", "anthropic", "mistral", "groq", "cohere"]) {
+    if (name.includes(known)) return known;
+  }
+  if (name.includes("google") || name.includes("genai")) return "google";
+  const baseUrl = String(
+    client.baseURL ?? client.baseUrl ?? ""
+  );
+  const host = baseUrl.match(/https?:\/\/([^/]+)/)?.[1] ?? "";
+  if (host.includes("openai")) return "openai";
+  if (host.includes("anthropic")) return "anthropic";
+  if (host.includes("googleapis")) return "google";
+  if (host.includes("mistral")) return "mistral";
+  if (host.includes("groq")) return "groq";
+  return "unknown";
+}
+function extractUsage(result) {
+  const r = result;
+  const usage = r?.usage ?? r?.message?.usage;
+  const meta = r?.usageMetadata;
+  const input = usage?.prompt_tokens ?? usage?.input_tokens ?? meta?.promptTokenCount ?? void 0;
+  const output = usage?.completion_tokens ?? usage?.output_tokens ?? meta?.candidatesTokenCount ?? void 0;
+  const responseModel = r?.model ?? r?.modelVersion;
+  return {
+    inputTokens: typeof input === "number" ? input : void 0,
+    outputTokens: typeof output === "number" ? output : void 0,
+    responseModel: typeof responseModel === "string" ? responseModel : void 0
+  };
+}
+function applyUsage(span, usage) {
+  if (usage.inputTokens !== void 0) {
+    span.setAttribute("gen_ai.usage.input_tokens", usage.inputTokens);
+  }
+  if (usage.outputTokens !== void 0) {
+    span.setAttribute("gen_ai.usage.output_tokens", usage.outputTokens);
+  }
+  if (usage.responseModel) {
+    span.setAttribute("gen_ai.response.model", usage.responseModel);
+  }
+}
+function endWithError(span, err) {
+  if (err instanceof Error) span.recordException(err);
+  span.setStatus({
+    code: SpanStatusCode2.ERROR,
+    message: err instanceof Error ? err.message : String(err)
+  });
+  span.end();
+}
+function wrapStream(stream, span) {
+  let ended = false;
+  const seen = {
+    inputTokens: void 0,
+    outputTokens: void 0,
+    responseModel: void 0
+  };
+  const finish = (err) => {
+    if (ended) return;
+    ended = true;
+    applyUsage(span, seen);
+    span.setAttribute("autter.llm.streamed", true);
+    if (err !== void 0) {
+      endWithError(span, err);
+    } else {
+      span.setStatus({ code: SpanStatusCode2.OK });
+      span.end();
+    }
+  };
+  return new Proxy(stream, {
+    get(target, prop, receiver) {
+      if (prop === Symbol.asyncIterator) {
+        return () => {
+          const inner = target[Symbol.asyncIterator]();
+          return {
+            async next() {
+              try {
+                const step = await inner.next();
+                if (!step.done) {
+                  const usage = extractUsage(step.value);
+                  if (usage.inputTokens !== void 0) seen.inputTokens = usage.inputTokens;
+                  if (usage.outputTokens !== void 0) seen.outputTokens = usage.outputTokens;
+                  if (usage.responseModel) seen.responseModel = usage.responseModel;
+                } else {
+                  finish();
+                }
+                return step;
+              } catch (err) {
+                finish(err);
+                throw err;
+              }
+            },
+            async return(value2) {
+              finish();
+              return inner.return ? inner.return(value2) : { done: true, value: void 0 };
+            },
+            async throw(err) {
+              finish(err);
+              if (inner.throw) return inner.throw(err);
+              throw err;
+            }
+          };
+        };
+      }
+      const value = Reflect.get(target, prop, receiver);
+      return typeof value === "function" ? value.bind(target) : value;
+    }
+  });
+}
+function instrumentLlmClient(client, options) {
+  const provider = (options?.provider ?? detectProvider(client)).toLowerCase();
+  const proxied = /* @__PURE__ */ new WeakMap();
+  function resolveUserId() {
+    return typeof options?.userId === "function" ? options.userId() : options?.userId;
+  }
+  function traceCall(fn, self, callPath, args) {
+    const request = args[0];
+    const operation = /embed/i.test(callPath) ? "embeddings" : "chat";
+    const userId = resolveUserId();
+    const span = autterLlmTracer().startSpan(`${operation} ${request.model}`, {
+      kind: SpanKind2.CLIENT,
+      attributes: {
+        "gen_ai.operation.name": operation,
+        "gen_ai.provider.name": provider,
+        "gen_ai.system": provider,
+        "gen_ai.request.model": request.model,
+        ...userId ? { "autter.user_id": userId } : {},
+        ...options?.attributes
+      }
+    });
+    let result;
+    try {
+      result = fn.apply(self, args);
+    } catch (err) {
+      endWithError(span, err);
+      throw err;
+    }
+    if (!(result instanceof Promise)) {
+      if (result && typeof result === "object" && Symbol.asyncIterator in result) {
+        return wrapStream(result, span);
+      }
+      applyUsage(span, extractUsage(result));
+      span.setStatus({ code: SpanStatusCode2.OK });
+      span.end();
+      return result;
+    }
+    return result.then(
+      (resolved) => {
+        if (resolved && typeof resolved === "object" && Symbol.asyncIterator in resolved) {
+          return wrapStream(resolved, span);
+        }
+        applyUsage(span, extractUsage(resolved));
+        span.setStatus({ code: SpanStatusCode2.OK });
+        span.end();
+        return resolved;
+      },
+      (err) => {
+        endWithError(span, err);
+        throw err;
+      }
+    );
+  }
+  function deepProxy(target, path = "") {
+    const cached = proxied.get(target);
+    if (cached) return cached;
+    const proxy = new Proxy(target, {
+      get(t, prop, receiver) {
+        const value = Reflect.get(t, prop, receiver);
+        if (typeof prop === "symbol") return value;
+        const propPath = path ? `${path}.${prop}` : prop;
+        if (typeof value === "function") {
+          if (CALL_METHODS.has(prop)) {
+            return (...args) => {
+              const first = args[0];
+              if (first && typeof first === "object" && typeof first.model === "string") {
+                return traceCall(
+                  value,
+                  t,
+                  propPath,
+                  args
+                );
+              }
+              return value.apply(t, args);
+            };
+          }
+          return value.bind(t);
+        }
+        if (value !== null && typeof value === "object" && !(value instanceof Promise)) {
+          return deepProxy(value, propPath);
+        }
+        return value;
+      }
+    });
+    proxied.set(target, proxy);
+    return proxy;
+  }
+  return deepProxy(client);
+}
 export {
   captureException,
   captureMessage,
   createBrowserRelayFetchHandler,
   createBrowserRelayHandler,
+  emitLlmSelftestTrace,
   initAutterServer,
-  sanitizeBrowserPayload
+  installAutterAutoFlush,
+  instrumentLlmClient,
+  makeSafeCapture,
+  redactAttributes,
+  sanitizeBrowserPayload,
+  trackLlmCall,
+  withLlmCall,
+  withProcessSpan
 };