@opengeni/observability 0.8.29 → 0.8.30

package/dist/index.js

@@ -1,9 +1,280 @@
 // src/index.ts
-import { createHash } from "crypto";
+import { createHash as createHash3 } from "crypto";
 import { collectDefaultMetrics, Counter, Gauge, Histogram, Registry } from "prom-client";
 import { SandboxBackend } from "@opengeni/contracts";
+
+// src/trace-context.ts
+import { AsyncLocalStorage } from "async_hooks";
+import { createHash } from "crypto";
+var storage = new AsyncLocalStorage();
+function ownData(value, key) {
+  if (typeof value !== "object" || value === null) return void 0;
+  try {
+    const descriptor = Object.getOwnPropertyDescriptor(value, key);
+    return descriptor && "value" in descriptor ? descriptor.value : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function validTraceContext(value) {
+  const traceId = ownData(value, "traceId");
+  const spanId = ownData(value, "spanId");
+  if (typeof traceId !== "string" || typeof spanId !== "string") return void 0;
+  if (!/^[0-9a-f]{32}$/.test(traceId) || /^0+$/.test(traceId)) return void 0;
+  if (!/^[0-9a-f]{16}$/.test(spanId) || /^0+$/.test(spanId)) return void 0;
+  return { traceId, spanId };
+}
+function currentTraceContext() {
+  return validTraceContext(storage.getStore());
+}
+function withTraceContext(context, run) {
+  const valid = validTraceContext(context);
+  const addLink = ownData(context, "addLink");
+  return storage.run(
+    valid ? {
+      ...valid,
+      addLink: typeof addLink === "function" ? addLink : void 0
+    } : void 0,
+    run
+  );
+}
+function admissionTraceContext(eventId) {
+  if (typeof eventId !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(eventId))
+    return void 0;
+  const digest = createHash("sha256").update("opengeni:accepted-event-trace:v1\0").update(eventId.toLowerCase()).digest("hex");
+  return validTraceContext({ traceId: digest.slice(0, 32), spanId: digest.slice(32, 48) });
+}
+function linkCurrentSpanToAdmission(eventId) {
+  const link = admissionTraceContext(eventId);
+  if (link) storage.getStore()?.addLink?.(link);
+}
+function parseTraceparent(value) {
+  if (typeof value !== "string") return void 0;
+  const match = /^00-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/.exec(value ?? "");
+  return match ? validTraceContext({ traceId: match[1], spanId: match[2] }) : void 0;
+}
+function traceparent(context) {
+  const valid = validTraceContext(context);
+  return valid ? `00-${valid.traceId}-${valid.spanId}-01` : void 0;
+}
+
+// src/export-queue.ts
+var ExportQueue = class {
+  constructor(observer, capacity = 256) {
+    this.observer = observer;
+    this.capacity = capacity;
+  }
+  queue = [];
+  running;
+  observe(outcome) {
+    try {
+      this.observer(outcome);
+    } catch {
+    }
+  }
+  enqueue(send) {
+    if (this.queue.length >= this.capacity) {
+      this.observe("dropped");
+      return;
+    }
+    this.queue.push(send);
+    this.start();
+  }
+  start() {
+    if (!this.running) {
+      this.running = Promise.resolve().then(() => this.drain()).finally(() => {
+        this.running = void 0;
+        if (this.queue.length) this.start();
+      });
+    }
+  }
+  async drain() {
+    while (this.queue.length) {
+      const send = this.queue.shift();
+      for (let attempt = 0; attempt < 3; attempt++) {
+        try {
+          await send();
+          this.observe("exported");
+          break;
+        } catch {
+          if (attempt === 2) this.observe("failed");
+          else {
+            this.observe("retried");
+            await new Promise((resolve) => setTimeout(resolve, 25 * 2 ** attempt));
+          }
+        }
+      }
+    }
+  }
+  async flush(timeoutMs = 1e3) {
+    if (!this.running) return;
+    let timer;
+    try {
+      await Promise.race([
+        this.running,
+        new Promise((resolve) => {
+          timer = setTimeout(
+            resolve,
+            Number.isFinite(timeoutMs) ? Math.max(1, Math.min(timeoutMs, 5e3)) : 1e3
+          );
+        })
+      ]);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+};
+
+// src/failure-diagnostic.ts
+import { createHash as createHash2, randomUUID } from "crypto";
+import { SessionEventType } from "@opengeni/contracts";
+var CODES = [
+  "api_startup_failed",
+  "api_unhandled_rejection",
+  "api_uncaught_exception",
+  "db_deadlock",
+  "db_serialization_failure",
+  "db_failure",
+  "retained_process_fenced",
+  "retained_process_proof_failed"
+];
+var STAGES = [
+  "startup",
+  "running",
+  "session_events.append_generic",
+  "session_events.append_for_turn_attempt",
+  "preclaim",
+  "session_attempts.claim",
+  "failure_settlement",
+  "sandbox_retained_processes.proof"
+];
+var RETRIES = ["not_retryable", "exhausted", "unknown"];
+var CONSTRAINTS = /* @__PURE__ */ new Set([
+  "session_events_workspace_account_fk",
+  "session_events_turn_association_check",
+  "session_events_payload_bytes_check",
+  "session_events_type_bytes_check",
+  "session_events_duplicate_classification_check",
+  "sandbox_retained_processes_reconcile_proof_check",
+  "sandbox_retained_processes_reconcile_claim_check",
+  "sandbox_retained_processes_settlement_check"
+]);
+var ERROR_NAMES = /* @__PURE__ */ new Set([
+  "Error",
+  "TypeError",
+  "RangeError",
+  "AggregateError",
+  "PostgresError",
+  "DrizzleQueryError",
+  "SessionEventPersistenceError",
+  "SandboxWorkspaceMutationFencedError"
+]);
+var DIAGNOSTIC_SOURCE_FILES = [
+  "apps/api/src/index.ts",
+  "apps/api/src/app.ts",
+  "apps/api/src/fatal-process-boundary.ts",
+  "apps/api/dist/process/index.js",
+  "apps/worker/src/activities/agent-turn/run.ts",
+  "apps/worker/src/activities/agent-turn/claim.ts",
+  "apps/worker/src/activities/agent-turn/failure-settlement.ts",
+  "apps/worker/src/activities/sandbox-lease.ts",
+  "apps/worker/dist/process/index.js",
+  "packages/db/src/index.ts",
+  "packages/db/src/persistence-errors.ts",
+  "packages/storage/src/index.ts",
+  "packages/observability/src/index.ts"
+];
+var DIAGNOSTIC_POSTGRES_FUNCTIONS = ["admit_session_attempt_personal_resources"];
+function reviewedSource(location) {
+  return DIAGNOSTIC_SOURCE_FILES.find((file) => location === file || location.endsWith(`/${file}`));
+}
+function postgresContext(error) {
+  if (errorKind(error) !== "PostgresError") return [];
+  const where = own(error, "where");
+  if (typeof where !== "string") return [];
+  return where.slice(0, 8192).split("\n").slice(0, 16).flatMap((value) => {
+    const match = /^PL\/pgSQL function (?:public\.)?([a-z_]+)\([^\n]*\) line (\d{1,7}) at /.exec(
+      value
+    );
+    if (!match || !DIAGNOSTIC_POSTGRES_FUNCTIONS.some((name) => name === match[1])) return [];
+    return [{ functionName: match[1], line: Number(match[2]) }];
+  }).slice(0, 8);
+}
+function own(value, key) {
+  try {
+    if (!value || typeof value !== "object") return void 0;
+    const descriptor = Object.getOwnPropertyDescriptor(value, key);
+    return descriptor && "value" in descriptor ? descriptor.value : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function errorKind(error) {
+  const name = own(error, "name");
+  if (typeof name === "string" && ERROR_NAMES.has(name)) return name;
+  try {
+    const prototype = Object.getPrototypeOf(error);
+    if (prototype === TypeError.prototype) return "TypeError";
+    if (prototype === RangeError.prototype) return "RangeError";
+    if (prototype === AggregateError.prototype) return "AggregateError";
+  } catch {
+  }
+  return "Error";
+}
+function failureDiagnostic(input, revision) {
+  const causes = [];
+  const pgContext = [];
+  const seen = /* @__PURE__ */ new Set();
+  let error = input.error;
+  while (error && typeof error === "object" && !seen.has(error) && causes.length < 4) {
+    seen.add(error);
+    pgContext.push(...postgresContext(error).slice(0, 8 - pgContext.length));
+    const stack = own(error, "stack");
+    const frames = typeof stack === "string" ? stack.slice(0, 16384).split("\n").slice(1, 33).flatMap((frame) => {
+      const match = /(?:\(|\s)([^\s()]+):(\d{1,7}):(\d{1,7})\)?$/.exec(frame);
+      if (!match) return [];
+      return [
+        {
+          locationHash: createHash2("sha256").update(match[1]).digest("hex"),
+          ...reviewedSource(match[1]) ? { source: reviewedSource(match[1]) } : {},
+          line: Number(match[2]),
+          column: Number(match[3])
+        }
+      ];
+    }) : [];
+    causes.push({
+      kind: errorKind(error),
+      frames
+    });
+    error = own(error, "cause");
+  }
+  const uuid = (value) => typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value) ? value : void 0;
+  return {
+    schema: "opengeni.failure-diagnostic.v1",
+    diagnosticId: randomUUID(),
+    code: CODES.includes(input.code) ? input.code : "db_failure",
+    stage: STAGES.includes(input.stage) ? input.stage : "failure_settlement",
+    retryDecision: input.retryDecision && RETRIES.includes(input.retryDecision) ? input.retryDecision : "unknown",
+    attemptId: uuid(input.attemptId),
+    sessionId: uuid(input.sessionId),
+    processId: uuid(input.processId),
+    turnId: uuid(input.turnId),
+    attempts: Number.isSafeInteger(input.attempts) && input.attempts >= 0 && input.attempts <= 1e6 ? input.attempts : void 0,
+    sqlState: typeof input.sqlState === "string" && /^[0-9A-Z]{5}$/.test(input.sqlState) ? input.sqlState : void 0,
+    constraint: input.constraint && CONSTRAINTS.has(input.constraint) ? input.constraint : void 0,
+    deploymentRevision: revision && /^[0-9a-f]{40}$/.test(revision) ? revision : void 0,
+    causes,
+    postgresContext: pgContext,
+    eventTypes: (input.eventTypes ?? []).slice(0, 32).filter((type) => SessionEventType.safeParse(type).success)
+  };
+}
+
+// src/index.ts
 function sandboxLeaseTelemetryKey(workspaceId, sandboxGroupId) {
-  return `slk_${createHash("sha256").update("opengeni:sandbox-lease-telemetry:v1\0").update(workspaceId).update("\0").update(sandboxGroupId).digest("hex").slice(0, 32)}`;
+  return `slk_${createHash3("sha256").update("opengeni:sandbox-lease-telemetry:v1\0").update(workspaceId).update("\0").update(sandboxGroupId).digest("hex").slice(0, 32)}`;
+}
+function turnExecutionTelemetryKey(workspaceId, sessionId, attemptId) {
+  return `turn_${createHash3("sha256").update("opengeni:turn-execution-telemetry:v1\0").update(workspaceId).update("\0").update(sessionId).update("\0").update(attemptId).digest("hex").slice(0, 32)}`;
 }
 var OPENGENI_OBSERVABILITY_DISTRIBUTION = {
   monitoringNamespaceLabel: "opengeni.ai/monitoring",
@@ -51,26 +322,6 @@ var SANDBOX_OPERATION_NAMES = /* @__PURE__ */ new Set([
   "serializeSessionState"
 ]);
 var TENANCY_COMPATIBILITY_LANES = [
-  /**
-   * A provider connection carrying `authority_scope = 'legacy_user'` was
-   * resolved for accepted use: a personal row with no common authority or
-   * grant, admitted through `legacy_user_compatibility` provenance instead of
-   * an explicit delegation. Migration 0256 backfilled the existing rows, but
-   * its `bind_connection_authority` trigger still classifies a NEW personal
-   * connection this way whenever the inserting subject has no active
-   * organization membership, so this counter is expected to stay above zero
-   * until that classification changes. That is the finding, not a defect.
-   */
-  "connection_legacy_user",
-  /**
-   * A workspace-scope connection ref carried no connection id, so the
-   * accepted-use authority (migration 0279) could not identify the exact row
-   * and the request fell back to the unprivileged pre-snapshot resolution.
-   * This lane writes no `connection_use_audit_facts` row and leaves no durable
-   * trace of any kind, so a use counter is the only possible evidence it was
-   * taken at all.
-   */
-  "connection_pre_snapshot_ref",
   /**
    * A NEW persistable `/workspace` mutation was refused because the writer
    * behind it has no recorded authority (`authority_unattributed`): either a
@@ -376,6 +627,17 @@ var PUBLIC_TELEMETRY_ERROR_ORIGINS = /* @__PURE__ */ new Set([
   "worker",
   "worker-lifecycle"
 ]);
+var MATERIALIZATION_FAILURE_REASONS = /* @__PURE__ */ new Set([
+  "path_not_visible",
+  "command_failed",
+  "command_pending",
+  "invalid_response",
+  "command_error"
+]);
+var MATERIALIZATION_FAILURE_METRIC = {
+  name: "opengeni_sandbox_materialization_verification_failures_total",
+  help: "Sandbox destination visibility-check failures by bounded reason; exact command evidence is retained in the session failure event."
+};
 function createObservability(settings, options) {
   return new Observability(settings, options);
 }
@@ -409,14 +671,54 @@ var Observability = class {
       });
       this.registerSandboxRolloutConfig();
       this.registerTenancyCompatibilityLanes();
+      if (["worker", "worker-control", "worker-turn", "api"].includes(options.component)) {
+        for (const backend of [...SANDBOX_OPERATION_BACKENDS, "unknown"]) {
+          for (const reason of [...MATERIALIZATION_FAILURE_REASONS, "unknown"]) {
+            this.incrementCounter({
+              ...MATERIALIZATION_FAILURE_METRIC,
+              labels: { backend, reason },
+              amount: 0
+            });
+          }
+        }
+      }
+      if (options.component === "worker" || options.component === "worker-control") {
+        for (const outcome of WORKFLOW_WAKE_OUTCOMES) {
+          this.incrementCounter({ ...WORKFLOW_WAKE_METRIC, labels: { outcome }, amount: 0 });
+        }
+        for (const reason of [...WORKFLOW_WAKE_ADMISSION_BLOCKERS, "unknown"]) {
+          this.incrementCounter({ ...WORKFLOW_WAKE_BLOCKER_METRIC, labels: { reason }, amount: 0 });
+        }
+      }
     }
   }
+  spanBatch = [];
+  spanBatchScheduled = false;
+  diagnosticQueue = new ExportQueue((outcome) => {
+    this.incrementCounter({
+      name: "opengeni_diagnostic_exports_total",
+      help: "Protected diagnostic export outcomes.",
+      labels: { outcome }
+    });
+  });
   registry = new Registry();
   counters = /* @__PURE__ */ new Map();
   gauges = /* @__PURE__ */ new Map();
   histograms = /* @__PURE__ */ new Map();
   registrations = /* @__PURE__ */ new Map();
-  pendingExports = /* @__PURE__ */ new Set();
+  exportQueue = new ExportQueue((outcome) => {
+    this.incrementCounter({
+      name: "opengeni_telemetry_exports_total",
+      help: "Bounded telemetry export outcomes.",
+      labels: { outcome }
+    });
+    if (outcome === "failed")
+      this.warn("OTLP span export failed", {
+        errorClass: "TelemetryExportError",
+        errorCode: "otlp_export_failed",
+        origin: "observability"
+      });
+  }, 8);
   now;
   exporter;
   resourceAttributes;
@@ -479,6 +781,7 @@ var Observability = class {
       }
       return;
     }
+    const context = currentTraceContext();
     const record = {
       timestamp: new Date(this.now()).toISOString(),
       level,
@@ -486,7 +789,8 @@ var Observability = class {
       service: this.settings.serviceName,
       environment: this.settings.environment,
       component: this.options.component,
-      ...cleanAttributes(publicAttributes)
+      ...cleanAttributes(publicAttributes),
+      ...context ? { traceId: context.traceId, spanId: context.spanId } : {}
     };
     const serialized = JSON.stringify(record);
     if (level === "warn") {
@@ -497,14 +801,25 @@ var Observability = class {
       console.log(serialized);
     }
   }
-  startSpan(name, attributes = {}) {
-    const traceId = randomHex(16);
+  startSpan(name, attributes = {}, options = {}) {
+    const parent = options.parent === null ? void 0 : validTraceContext(options.parent ?? currentTraceContext());
+    const links = (options.links ?? []).slice(0, 8).flatMap((link) => {
+      const valid = validTraceContext(link);
+      return valid ? [valid] : [];
+    });
+    const traceId = parent?.traceId ?? randomHex(16);
     const spanId = randomHex(8);
-    const startMs = this.now();
+    const now = this.now();
+    const startMs = Number.isFinite(options.startTimeMs) ? Math.min(now, Math.max(0, options.startTimeMs)) : now;
     let ended = false;
     return {
       traceId,
       spanId,
+      addLink: (context) => {
+        const valid = validTraceContext(context);
+        if (!ended && valid && links.length < 8 && !links.some((link) => link.traceId === valid.traceId && link.spanId === valid.spanId))
+          links.push(valid);
+      },
       end: (input = {}) => {
         if (ended) {
           return;
@@ -515,7 +830,9 @@ var Observability = class {
         this.exportSpan({
           traceId,
           spanId,
-          name,
+          ...parent ? { parentSpanId: parent.spanId } : {},
+          links,
+          name: boundedOtlpString(name),
           startMs,
           endMs: this.now(),
           attributes: {
@@ -621,11 +938,68 @@ var Observability = class {
     }
     return await this.registry.metrics();
   }
-  /** Wait for every OTLP export accepted before or during this drain. */
-  async flush() {
-    while (this.pendingExports.size > 0) {
-      await Promise.allSettled([...this.pendingExports]);
+  /** Emit a content-free anchor after successful, non-replayed admission. */
+  recordAdmissionTrace(eventId) {
+    const identity = admissionTraceContext(eventId);
+    if (!identity) return;
+    const current = currentTraceContext();
+    const now = this.now();
+    this.exportSpan({
+      ...identity,
+      name: "api.turn.admitted",
+      startMs: now,
+      endMs: now,
+      attributes: {},
+      links: current ? [current] : []
+    });
+  }
+  /** Emit a closed protected record independently of application persistence. */
+  recordFailureDiagnostic(input) {
+    const record = failureDiagnostic(input, this.settings.deploymentRevision);
+    const endpoint = this.settings.observabilityDiagnosticsEndpoint;
+    if (!endpoint) {
+      this.incrementCounter({
+        name: "opengeni_diagnostic_exports_total",
+        help: "Protected diagnostic export outcomes.",
+        labels: { outcome: "disabled" }
+      });
+      return record.diagnosticId;
     }
+    const context = currentTraceContext();
+    const body = {
+      resourceLogs: [
+        {
+          resource: { attributes: otlpAttributes(this.resourceAttributes) },
+          scopeLogs: [
+            {
+              scope: { name: "@opengeni/observability/diagnostics", version: "1" },
+              logRecords: [
+                {
+                  timeUnixNano: millisToNanos(this.now()),
+                  severityNumber: 17,
+                  severityText: "ERROR",
+                  ...context ? { traceId: context.traceId, spanId: context.spanId } : {},
+                  body: { stringValue: JSON.stringify(record) }
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    };
+    this.diagnosticQueue.enqueue(
+      () => this.exporter(
+        `${endpoint.replace(/\/$/, "")}/v1/logs`,
+        body,
+        parseHeaders(this.settings.observabilityDiagnosticsHeaders ?? "")
+      )
+    );
+    return record.diagnosticId;
+  }
+  /** Best-effort bounded drain, including when a custom transport never settles. */
+  async flush(timeoutMs = 1e3) {
+    this.submitSpanBatch();
+    await Promise.all([this.exportQueue.flush(timeoutMs), this.diagnosticQueue.flush(timeoutMs)]);
   }
   counter(name, help, labelNames) {
     const existing = this.counters.get(name);
@@ -677,7 +1051,6 @@ var Observability = class {
     if (!this.settings.observabilityOtlpEndpoint) {
       return;
     }
-    const endpoint = `${this.settings.observabilityOtlpEndpoint.replace(/\/$/, "")}/v1/traces`;
     const body = {
       resourceSpans: [
         {
@@ -694,6 +1067,8 @@ var Observability = class {
                 {
                   traceId: span.traceId,
                   spanId: span.spanId,
+                  ...span.parentSpanId ? { parentSpanId: span.parentSpanId } : {},
+                  links: span.links,
                   name: span.name,
                   kind: 1,
                   startTimeUnixNano: millisToNanos(span.startMs),
@@ -707,19 +1082,28 @@ var Observability = class {
         }
       ]
     };
-    let pendingExport;
-    pendingExport = Promise.resolve().then(
-      () => this.exporter(endpoint, body, parseHeaders(this.settings.observabilityOtlpHeaders))
-    ).catch(() => {
-      this.warn("OTLP span export failed", {
-        errorClass: "TelemetryExportError",
-        errorCode: "otlp_export_failed",
-        origin: "observability"
+    this.spanBatch.push(body.resourceSpans[0]);
+    if (!this.spanBatchScheduled) {
+      this.spanBatchScheduled = true;
+      queueMicrotask(() => {
+        this.spanBatchScheduled = false;
+        this.submitSpanBatch();
       });
-    }).finally(() => {
-      this.pendingExports.delete(pendingExport);
-    });
-    this.pendingExports.add(pendingExport);
+    }
+    if (this.spanBatch.length >= 32) this.submitSpanBatch();
+  }
+  submitSpanBatch() {
+    if (!this.spanBatch.length || !this.settings.observabilityOtlpEndpoint) return;
+    const resourceSpans = this.spanBatch;
+    this.spanBatch = [];
+    const endpoint = `${this.settings.observabilityOtlpEndpoint.replace(/\/$/, "")}/v1/traces`;
+    this.exportQueue.enqueue(
+      () => this.exporter(
+        endpoint,
+        { resourceSpans },
+        parseHeaders(this.settings.observabilityOtlpHeaders)
+      )
+    );
   }
 };
 function sandboxOperationMetricObserver(observability) {
@@ -738,6 +1122,13 @@ function sandboxOperationMetricObserver(observability) {
         labels: { backend, op },
         value: Math.max(0, observation.durationMs) / 1e3
       });
+      if (op === "materializeEntry" && observation.outcome === "failed" && observation.materializationFailureReason) {
+        const reason = MATERIALIZATION_FAILURE_REASONS.has(observation.materializationFailureReason) ? observation.materializationFailureReason : "unknown";
+        observability.incrementCounter({
+          ...MATERIALIZATION_FAILURE_METRIC,
+          labels: { backend, reason }
+        });
+      }
     } catch {
       try {
         observability.incrementCounter({
@@ -750,6 +1141,32 @@ function sandboxOperationMetricObserver(observability) {
     }
   };
 }
+function sandboxCaptureWaitMetricObserver(observability) {
+  return (observation) => {
+    const stage = observation.captureWaitStage;
+    if (stage !== "admission" && stage !== "provider") return;
+    if (!Number.isFinite(observation.durationMs) || observation.durationMs < 0) return;
+    const backend = SANDBOX_OPERATION_BACKENDS.has(observation.backend) ? observation.backend : "unknown";
+    const op = SANDBOX_OPERATION_NAMES.has(observation.op) ? observation.op : "unknown";
+    const outcome = observation.outcome === "ok" ? "completed" : "failed";
+    try {
+      observability.observeHistogram({
+        name: "opengeni_sandbox_capture_wait_duration_seconds",
+        help: "Workspace capture gate wait duration across every routed operation, not only startup.",
+        labels: { backend, op, stage, outcome },
+        value: observation.durationMs / 1e3
+      });
+      observability.startSpan(
+        `sandbox.capture_wait.${stage}`,
+        { backend, outcome },
+        {
+          startTimeMs: Date.now() - observation.durationMs
+        }
+      ).end({ ...outcome === "failed" ? { error: true } : {} });
+    } catch {
+    }
+  };
+}
 function interactionOperationMetricObserver(observability) {
   if (!observability) return () => void 0;
   return (observation) => {
@@ -858,6 +1275,70 @@ function workspaceInsightsMetricObserver(observability) {
 function boundedMetricEnum(allowed, value) {
   return allowed.has(value) ? value : "unknown";
 }
+var WORKFLOW_WAKE_ADMISSION_BLOCKERS = /* @__PURE__ */ new Set([
+  "pending_agent_steer",
+  "pending_prompt_turn",
+  "pending_quiescence",
+  "pending_machine_input",
+  "pending_input_wait"
+]);
+var WORKFLOW_WAKE_OUTCOMES = [
+  "signal_accepted",
+  "acknowledged",
+  "pending_admission",
+  "unconfirmed",
+  "failed"
+];
+var WORKFLOW_WAKE_METRIC = {
+  name: "opengeni_session_workflow_wake_observations_total",
+  help: "Workflow wake dispatch observations; signal acceptance overlaps with admission outcomes, and retries are counted again."
+};
+var WORKFLOW_WAKE_BLOCKER_METRIC = {
+  name: "opengeni_session_workflow_wake_admission_blockers_total",
+  help: "Workflow wake dispatch attempts awaiting durable admission, by bounded blocker; repeated revisions count again."
+};
+function recordWorkflowWakeReconciliation(observability, result) {
+  const count = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
+  try {
+    for (const [outcome, amount] of [
+      ["signal_accepted", result.signaled],
+      ["acknowledged", result.delivered],
+      ["pending_admission", result.pendingAdmission],
+      ["unconfirmed", result.unconfirmed],
+      ["failed", result.failed]
+    ]) {
+      observability.incrementCounter({
+        ...WORKFLOW_WAKE_METRIC,
+        labels: { outcome },
+        amount: count(amount)
+      });
+      if (count(amount) > 0) {
+        observability.info("Session workflow wake dispatch observed", {
+          surface: "session_workflow_wake",
+          outcome,
+          attempts: count(amount)
+        });
+      }
+    }
+    for (const [blocker, amount] of Object.entries(result.pendingAdmissionBlockers)) {
+      if (count(amount) === 0) continue;
+      const reason = boundedMetricEnum(WORKFLOW_WAKE_ADMISSION_BLOCKERS, blocker);
+      observability.incrementCounter({
+        ...WORKFLOW_WAKE_BLOCKER_METRIC,
+        labels: { reason },
+        amount: count(amount)
+      });
+      observability.info("Session workflow wake admission pending", {
+        surface: "session_workflow_wake",
+        outcome: "pending_admission",
+        reason,
+        attempts: count(amount)
+      });
+    }
+  } catch {
+    recordObserverFailure(observability, "session_workflow_wake");
+  }
+}
 function boundedMetricDuration(durationMs) {
   return Number.isFinite(durationMs) ? Math.max(0, durationMs) / 1e3 : 0;
 }
@@ -874,7 +1355,7 @@ function recordObserverFailure(observability, observer) {
 function logStartupDependencyRetry(observability, event) {
   const dependency = PUBLIC_STARTUP_DEPENDENCIES.has(event.label) ? event.label : "Dependency";
   observability.warn(
-    `Startup dependency ${dependency} connection failed; retrying (${event.attempt}/${event.attempts} in ${event.delayMs}ms)`,
+    `Startup dependency ${dependency} check failed; retrying (${event.attempt}/${event.attempts} in ${event.delayMs}ms)`,
     {
       dependency,
       attempt: event.attempt,
@@ -905,6 +1386,7 @@ function projectPublicTelemetryAttributes(attributes) {
       ...projectStartupDependencyAttributes(attributes),
       ...projectPublicChannelADiagnosticAttributes(attributes),
       ...projectApiFatalDiagnosticAttributes(attributes),
+      ...projectSnapshotDiagnosticAttributes(attributes),
       ...projectPublicDiagnosticAttributes(attributes)
     };
   }
@@ -958,6 +1440,42 @@ function projectPublicChannelADiagnosticAttributes(attributes) {
     ...typeof sandboxLeaseKey === "string" && PUBLIC_TELEMETRY_OPAQUE_ATTRIBUTE_PATTERNS.get("sandboxLeaseKey")?.test(sandboxLeaseKey) ? { sandboxLeaseKey } : {}
   };
 }
+function projectSnapshotDiagnosticAttributes(attributes) {
+  if (attributes.errorClass !== "SnapshotOperationError") return {};
+  const projected = {};
+  const causeName = attributes.causeName;
+  if (typeof causeName === "string" && ["SnapshotTimeoutError", "SandboxProviderCaptureTimeoutError", "SandboxProviderError"].includes(
+    causeName
+  ))
+    projected.causeName = causeName;
+  const name = attributes.providerErrorName;
+  if (typeof name === "string" && [
+    "ClientError",
+    "TimeoutError",
+    "ConnectionError",
+    "AuthError",
+    "NotFoundError",
+    "InvalidError",
+    "RemoteError",
+    "AbortError"
+  ].includes(name))
+    projected.providerErrorName = name;
+  const grpc = attributes.providerGrpcCode;
+  if (name === "ClientError" && typeof grpc === "number" && Number.isInteger(grpc) && grpc >= 0 && grpc <= 16)
+    projected.providerGrpcCode = grpc;
+  const http = attributes.providerHttpStatus;
+  if (typeof http === "number" && Number.isInteger(http) && http >= 100 && http <= 599)
+    projected.providerHttpStatus = http;
+  if (typeof attributes.providerRetryable === "boolean")
+    projected.providerRetryable = attributes.providerRetryable;
+  const leaseKey = attributes.sandboxLeaseKey;
+  if (typeof leaseKey === "string" && /^slk_[0-9a-f]{32}$/.test(leaseKey))
+    projected.sandboxLeaseKey = leaseKey;
+  const epoch = attributes.leaseEpoch;
+  if (typeof epoch === "number" && Number.isSafeInteger(epoch) && epoch >= 0)
+    projected.leaseEpoch = epoch;
+  return projected;
+}
 function projectPublicDiagnosticAttributes(attributes) {
   const errorClass = attributes.errorClass;
   const errorCode = attributes.errorCode;
@@ -1043,7 +1561,8 @@ async function defaultExporter(url, body, headers) {
       "content-type": "application/json",
       ...headers
     },
-    body: JSON.stringify(body)
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(1e3)
   });
   if (!response.ok) {
     throw new Error(`OTLP endpoint returned HTTP ${response.status}`);
@@ -1053,15 +1572,24 @@ export {
   OPENGENI_OBSERVABILITY_DISTRIBUTION,
   Observability,
   TENANCY_COMPATIBILITY_LANES,
+  admissionTraceContext,
   createObservability,
+  currentTraceContext,
   interactionAuthMetricObserver,
   interactionInterventionMetricObserver,
   interactionOperationMetricObserver,
+  linkCurrentSpanToAdmission,
   logStartupDependencyRetry,
   parseHeaders,
+  parseTraceparent,
   recordTenancyCompatibilityLaneUse,
+  recordWorkflowWakeReconciliation,
+  sandboxCaptureWaitMetricObserver,
   sandboxLeaseTelemetryKey,
   sandboxOperationMetricObserver,
+  traceparent,
+  turnExecutionTelemetryKey,
+  withTraceContext,
   workspaceInsightsMetricObserver
 };
 //# sourceMappingURL=index.js.map