@opengeni/observability 0.8.29 → 0.8.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,6 +1,24 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { collectDefaultMetrics, Counter, Gauge, Histogram, Registry } from "prom-client";
3
3
  import { SandboxBackend } from "@opengeni/contracts";
4
+ import {
5
+ currentTraceContext,
6
+ validTraceContext,
7
+ admissionTraceContext,
8
+ type TraceContext,
9
+ } from "./trace-context";
10
+ import { ExportQueue } from "./export-queue";
11
+ import { failureDiagnostic, type FailureDiagnosticInput } from "./failure-diagnostic";
12
+ export type { FailureDiagnosticInput } from "./failure-diagnostic";
13
+ export {
14
+ currentTraceContext,
15
+ withTraceContext,
16
+ parseTraceparent,
17
+ traceparent,
18
+ admissionTraceContext,
19
+ linkCurrentSpanToAdmission,
20
+ type TraceContext,
21
+ } from "./trace-context";
4
22
 
5
23
  export type AttributeValue = string | number | boolean | null | undefined;
6
24
  export type Attributes = Record<string, AttributeValue>;
@@ -13,6 +31,9 @@ export type ObservabilitySettings = {
13
31
  observabilityMetricsEnabled: boolean;
14
32
  observabilityOtlpEndpoint?: string | undefined;
15
33
  observabilityOtlpHeaders: string;
34
+ /** Explicit restricted OTLP logs destination; never falls back to the public exporter. */
35
+ observabilityDiagnosticsEndpoint?: string | undefined;
36
+ observabilityDiagnosticsHeaders?: string | undefined;
16
37
  sandboxOwnershipEnabled?: boolean | undefined;
17
38
  sandboxLazyProvisionEnabled?: boolean | undefined;
18
39
  rigVerificationLeaseOwnershipEnabled?: boolean | undefined;
@@ -27,6 +48,7 @@ export type ObservabilityOptions = {
27
48
  export type Span = {
28
49
  traceId: string;
29
50
  spanId: string;
51
+ addLink?: (context: TraceContext) => void;
30
52
  end: (input?: { attributes?: Attributes; error?: unknown }) => void;
31
53
  };
32
54
 
@@ -49,6 +71,25 @@ export function sandboxLeaseTelemetryKey(workspaceId: string, sandboxGroupId: st
49
71
  .slice(0, 32)}`;
50
72
  }
51
73
 
74
+ /** Content-free lookup key for a worker execution attempt, never a metric label.
75
+ * Operators derive it from authorized session/attempt records to find a trace
76
+ * without publishing raw tenancy or conversation identifiers to telemetry. */
77
+ export function turnExecutionTelemetryKey(
78
+ workspaceId: string,
79
+ sessionId: string,
80
+ attemptId: string,
81
+ ): string {
82
+ return `turn_${createHash("sha256")
83
+ .update("opengeni:turn-execution-telemetry:v1\0")
84
+ .update(workspaceId)
85
+ .update("\0")
86
+ .update(sessionId)
87
+ .update("\0")
88
+ .update(attemptId)
89
+ .digest("hex")
90
+ .slice(0, 32)}`;
91
+ }
92
+
52
93
  /**
53
94
  * Stable selectors shared by OpenGeni's runtime metrics and optional
54
95
  * Prometheus/Grafana distribution. Operators can use these values for custom
@@ -111,26 +152,6 @@ const SANDBOX_OPERATION_NAMES = new Set([
111
152
  * credential, name, or content, may ever become a label here.
112
153
  */
113
154
  export const TENANCY_COMPATIBILITY_LANES = [
114
- /**
115
- * A provider connection carrying `authority_scope = 'legacy_user'` was
116
- * resolved for accepted use: a personal row with no common authority or
117
- * grant, admitted through `legacy_user_compatibility` provenance instead of
118
- * an explicit delegation. Migration 0256 backfilled the existing rows, but
119
- * its `bind_connection_authority` trigger still classifies a NEW personal
120
- * connection this way whenever the inserting subject has no active
121
- * organization membership, so this counter is expected to stay above zero
122
- * until that classification changes. That is the finding, not a defect.
123
- */
124
- "connection_legacy_user",
125
- /**
126
- * A workspace-scope connection ref carried no connection id, so the
127
- * accepted-use authority (migration 0279) could not identify the exact row
128
- * and the request fell back to the unprivileged pre-snapshot resolution.
129
- * This lane writes no `connection_use_audit_facts` row and leaves no durable
130
- * trace of any kind, so a use counter is the only possible evidence it was
131
- * taken at all.
132
- */
133
- "connection_pre_snapshot_ref",
134
155
  /**
135
156
  * A NEW persistable `/workspace` mutation was refused because the writer
136
157
  * behind it has no recorded authority (`authority_unattributed`): either a
@@ -481,8 +502,21 @@ export type SandboxOperationMetricObservation = {
481
502
  op: string;
482
503
  outcome: "ok" | "not_found" | "failed";
483
504
  durationMs: number;
505
+ materializationFailureReason?: string;
484
506
  };
485
507
 
508
+ const MATERIALIZATION_FAILURE_REASONS = new Set([
509
+ "path_not_visible",
510
+ "command_failed",
511
+ "command_pending",
512
+ "invalid_response",
513
+ "command_error",
514
+ ]);
515
+ const MATERIALIZATION_FAILURE_METRIC = {
516
+ name: "opengeni_sandbox_materialization_verification_failures_total",
517
+ help: "Sandbox destination visibility-check failures by bounded reason; exact command evidence is retained in the session failure event.",
518
+ } as const;
519
+
486
520
  export function createObservability(
487
521
  settings: ObservabilitySettings,
488
522
  options: ObservabilityOptions,
@@ -496,12 +530,33 @@ type MetricRegistration = {
496
530
  };
497
531
 
498
532
  export class Observability {
533
+ private spanBatch: unknown[] = [];
534
+ private spanBatchScheduled = false;
535
+ private readonly diagnosticQueue = new ExportQueue((outcome) => {
536
+ this.incrementCounter({
537
+ name: "opengeni_diagnostic_exports_total",
538
+ help: "Protected diagnostic export outcomes.",
539
+ labels: { outcome },
540
+ });
541
+ });
499
542
  private readonly registry = new Registry();
500
543
  private readonly counters = new Map<string, Counter<string>>();
501
544
  private readonly gauges = new Map<string, Gauge<string>>();
502
545
  private readonly histograms = new Map<string, Histogram<string>>();
503
546
  private readonly registrations = new Map<string, MetricRegistration>();
504
- private readonly pendingExports = new Set<Promise<void>>();
547
+ private readonly exportQueue = new ExportQueue((outcome) => {
548
+ this.incrementCounter({
549
+ name: "opengeni_telemetry_exports_total",
550
+ help: "Bounded telemetry export outcomes.",
551
+ labels: { outcome },
552
+ });
553
+ if (outcome === "failed")
554
+ this.warn("OTLP span export failed", {
555
+ errorClass: "TelemetryExportError",
556
+ errorCode: "otlp_export_failed",
557
+ origin: "observability",
558
+ });
559
+ }, 8);
505
560
  private readonly now: () => number;
506
561
  private readonly exporter: (
507
562
  url: string,
@@ -540,6 +595,27 @@ export class Observability {
540
595
  });
541
596
  this.registerSandboxRolloutConfig();
542
597
  this.registerTenancyCompatibilityLanes();
598
+ if (["worker", "worker-control", "worker-turn", "api"].includes(options.component)) {
599
+ // Publish the finite series at startup so a first failure has a zero
600
+ // baseline and healthy zeroes differ from a missing instrumentation path.
601
+ for (const backend of [...SANDBOX_OPERATION_BACKENDS, "unknown"]) {
602
+ for (const reason of [...MATERIALIZATION_FAILURE_REASONS, "unknown"]) {
603
+ this.incrementCounter({
604
+ ...MATERIALIZATION_FAILURE_METRIC,
605
+ labels: { backend, reason },
606
+ amount: 0,
607
+ });
608
+ }
609
+ }
610
+ }
611
+ if (options.component === "worker" || options.component === "worker-control") {
612
+ for (const outcome of WORKFLOW_WAKE_OUTCOMES) {
613
+ this.incrementCounter({ ...WORKFLOW_WAKE_METRIC, labels: { outcome }, amount: 0 });
614
+ }
615
+ for (const reason of [...WORKFLOW_WAKE_ADMISSION_BLOCKERS, "unknown"]) {
616
+ this.incrementCounter({ ...WORKFLOW_WAKE_BLOCKER_METRIC, labels: { reason }, amount: 0 });
617
+ }
618
+ }
543
619
  }
544
620
  }
545
621
 
@@ -617,6 +693,7 @@ export class Observability {
617
693
  }
618
694
  return;
619
695
  }
696
+ const context = currentTraceContext();
620
697
  const record = {
621
698
  timestamp: new Date(this.now()).toISOString(),
622
699
  level,
@@ -625,6 +702,7 @@ export class Observability {
625
702
  environment: this.settings.environment,
626
703
  component: this.options.component,
627
704
  ...cleanAttributes(publicAttributes),
705
+ ...(context ? { traceId: context.traceId, spanId: context.spanId } : {}),
628
706
  };
629
707
  const serialized = JSON.stringify(record);
630
708
  if (level === "warn") {
@@ -636,14 +714,39 @@ export class Observability {
636
714
  }
637
715
  }
638
716
 
639
- startSpan(name: string, attributes: Attributes = {}): Span {
640
- const traceId = randomHex(16);
717
+ startSpan(
718
+ name: string,
719
+ attributes: Attributes = {},
720
+ options: { parent?: TraceContext | null; links?: TraceContext[]; startTimeMs?: number } = {},
721
+ ): Span {
722
+ const parent =
723
+ options.parent === null
724
+ ? undefined
725
+ : validTraceContext(options.parent ?? currentTraceContext());
726
+ const links = (options.links ?? []).slice(0, 8).flatMap((link) => {
727
+ const valid = validTraceContext(link);
728
+ return valid ? [valid] : [];
729
+ });
730
+ const traceId = parent?.traceId ?? randomHex(16);
641
731
  const spanId = randomHex(8);
642
- const startMs = this.now();
732
+ const now = this.now();
733
+ const startMs = Number.isFinite(options.startTimeMs)
734
+ ? Math.min(now, Math.max(0, options.startTimeMs!))
735
+ : now;
643
736
  let ended = false;
644
737
  return {
645
738
  traceId,
646
739
  spanId,
740
+ addLink: (context) => {
741
+ const valid = validTraceContext(context);
742
+ if (
743
+ !ended &&
744
+ valid &&
745
+ links.length < 8 &&
746
+ !links.some((link) => link.traceId === valid.traceId && link.spanId === valid.spanId)
747
+ )
748
+ links.push(valid);
749
+ },
647
750
  end: (input = {}) => {
648
751
  if (ended) {
649
752
  return;
@@ -657,7 +760,9 @@ export class Observability {
657
760
  this.exportSpan({
658
761
  traceId,
659
762
  spanId,
660
- name,
763
+ ...(parent ? { parentSpanId: parent.spanId } : {}),
764
+ links,
765
+ name: boundedOtlpString(name),
661
766
  startMs,
662
767
  endMs: this.now(),
663
768
  attributes: {
@@ -792,11 +897,70 @@ export class Observability {
792
897
  return await this.registry.metrics();
793
898
  }
794
899
 
795
- /** Wait for every OTLP export accepted before or during this drain. */
796
- async flush(): Promise<void> {
797
- while (this.pendingExports.size > 0) {
798
- await Promise.allSettled([...this.pendingExports]);
900
+ /** Emit a content-free anchor after successful, non-replayed admission. */
901
+ recordAdmissionTrace(eventId: string): void {
902
+ const identity = admissionTraceContext(eventId);
903
+ if (!identity) return;
904
+ const current = currentTraceContext();
905
+ const now = this.now();
906
+ this.exportSpan({
907
+ ...identity,
908
+ name: "api.turn.admitted",
909
+ startMs: now,
910
+ endMs: now,
911
+ attributes: {},
912
+ links: current ? [current] : [],
913
+ });
914
+ }
915
+
916
+ /** Emit a closed protected record independently of application persistence. */
917
+ recordFailureDiagnostic(input: FailureDiagnosticInput): string {
918
+ const record = failureDiagnostic(input, this.settings.deploymentRevision);
919
+ const endpoint = this.settings.observabilityDiagnosticsEndpoint;
920
+ if (!endpoint) {
921
+ this.incrementCounter({
922
+ name: "opengeni_diagnostic_exports_total",
923
+ help: "Protected diagnostic export outcomes.",
924
+ labels: { outcome: "disabled" },
925
+ });
926
+ return record.diagnosticId;
799
927
  }
928
+ const context = currentTraceContext();
929
+ const body = {
930
+ resourceLogs: [
931
+ {
932
+ resource: { attributes: otlpAttributes(this.resourceAttributes) },
933
+ scopeLogs: [
934
+ {
935
+ scope: { name: "@opengeni/observability/diagnostics", version: "1" },
936
+ logRecords: [
937
+ {
938
+ timeUnixNano: millisToNanos(this.now()),
939
+ severityNumber: 17,
940
+ severityText: "ERROR",
941
+ ...(context ? { traceId: context.traceId, spanId: context.spanId } : {}),
942
+ body: { stringValue: JSON.stringify(record) },
943
+ },
944
+ ],
945
+ },
946
+ ],
947
+ },
948
+ ],
949
+ };
950
+ this.diagnosticQueue.enqueue(() =>
951
+ this.exporter(
952
+ `${endpoint.replace(/\/$/, "")}/v1/logs`,
953
+ body,
954
+ parseHeaders(this.settings.observabilityDiagnosticsHeaders ?? ""),
955
+ ),
956
+ );
957
+ return record.diagnosticId;
958
+ }
959
+
960
+ /** Best-effort bounded drain, including when a custom transport never settles. */
961
+ async flush(timeoutMs = 1_000): Promise<void> {
962
+ this.submitSpanBatch();
963
+ await Promise.all([this.exportQueue.flush(timeoutMs), this.diagnosticQueue.flush(timeoutMs)]);
800
964
  }
801
965
 
802
966
  private counter(name: string, help: string, labelNames: string[]): Counter<string> {
@@ -868,6 +1032,8 @@ export class Observability {
868
1032
  private exportSpan(span: {
869
1033
  traceId: string;
870
1034
  spanId: string;
1035
+ parentSpanId?: string;
1036
+ links: TraceContext[];
871
1037
  name: string;
872
1038
  startMs: number;
873
1039
  endMs: number;
@@ -877,7 +1043,6 @@ export class Observability {
877
1043
  if (!this.settings.observabilityOtlpEndpoint) {
878
1044
  return;
879
1045
  }
880
- const endpoint = `${this.settings.observabilityOtlpEndpoint.replace(/\/$/, "")}/v1/traces`;
881
1046
  const body = {
882
1047
  resourceSpans: [
883
1048
  {
@@ -894,6 +1059,8 @@ export class Observability {
894
1059
  {
895
1060
  traceId: span.traceId,
896
1061
  spanId: span.spanId,
1062
+ ...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}),
1063
+ links: span.links,
897
1064
  name: span.name,
898
1065
  kind: 1,
899
1066
  startTimeUnixNano: millisToNanos(span.startMs),
@@ -907,22 +1074,29 @@ export class Observability {
907
1074
  },
908
1075
  ],
909
1076
  };
910
- let pendingExport: Promise<void>;
911
- pendingExport = Promise.resolve()
912
- .then(() =>
913
- this.exporter(endpoint, body, parseHeaders(this.settings.observabilityOtlpHeaders)),
914
- )
915
- .catch(() => {
916
- this.warn("OTLP span export failed", {
917
- errorClass: "TelemetryExportError",
918
- errorCode: "otlp_export_failed",
919
- origin: "observability",
920
- });
921
- })
922
- .finally(() => {
923
- this.pendingExports.delete(pendingExport);
1077
+ this.spanBatch.push(body.resourceSpans[0]);
1078
+ if (!this.spanBatchScheduled) {
1079
+ this.spanBatchScheduled = true;
1080
+ queueMicrotask(() => {
1081
+ this.spanBatchScheduled = false;
1082
+ this.submitSpanBatch();
924
1083
  });
925
- this.pendingExports.add(pendingExport);
1084
+ }
1085
+ if (this.spanBatch.length >= 32) this.submitSpanBatch();
1086
+ }
1087
+
1088
+ private submitSpanBatch(): void {
1089
+ if (!this.spanBatch.length || !this.settings.observabilityOtlpEndpoint) return;
1090
+ const resourceSpans = this.spanBatch;
1091
+ this.spanBatch = [];
1092
+ const endpoint = `${this.settings.observabilityOtlpEndpoint.replace(/\/$/, "")}/v1/traces`;
1093
+ this.exportQueue.enqueue(() =>
1094
+ this.exporter(
1095
+ endpoint,
1096
+ { resourceSpans },
1097
+ parseHeaders(this.settings.observabilityOtlpHeaders),
1098
+ ),
1099
+ );
926
1100
  }
927
1101
  }
928
1102
 
@@ -957,6 +1131,19 @@ export function sandboxOperationMetricObserver(
957
1131
  labels: { backend, op },
958
1132
  value: Math.max(0, observation.durationMs) / 1_000,
959
1133
  });
1134
+ if (
1135
+ op === "materializeEntry" &&
1136
+ observation.outcome === "failed" &&
1137
+ observation.materializationFailureReason
1138
+ ) {
1139
+ const reason = MATERIALIZATION_FAILURE_REASONS.has(observation.materializationFailureReason)
1140
+ ? observation.materializationFailureReason
1141
+ : "unknown";
1142
+ observability.incrementCounter({
1143
+ ...MATERIALIZATION_FAILURE_METRIC,
1144
+ labels: { backend, reason },
1145
+ });
1146
+ }
960
1147
  } catch {
961
1148
  try {
962
1149
  observability.incrementCounter({
@@ -972,6 +1159,45 @@ export function sandboxOperationMetricObserver(
972
1159
  };
973
1160
  }
974
1161
 
1162
+ /** Capture waits remain separate from physical provider-call accounting. */
1163
+ export function sandboxCaptureWaitMetricObserver(observability: Observability) {
1164
+ return (observation: {
1165
+ backend: string;
1166
+ op: string;
1167
+ outcome: "ok" | "failed";
1168
+ durationMs: number;
1169
+ captureWaitStage: "admission" | "provider";
1170
+ }): void => {
1171
+ const stage = observation.captureWaitStage;
1172
+ if (stage !== "admission" && stage !== "provider") return;
1173
+ if (!Number.isFinite(observation.durationMs) || observation.durationMs < 0) return;
1174
+ const backend = SANDBOX_OPERATION_BACKENDS.has(observation.backend)
1175
+ ? observation.backend
1176
+ : "unknown";
1177
+ const op = SANDBOX_OPERATION_NAMES.has(observation.op) ? observation.op : "unknown";
1178
+ const outcome = observation.outcome === "ok" ? "completed" : "failed";
1179
+ try {
1180
+ observability.observeHistogram({
1181
+ name: "opengeni_sandbox_capture_wait_duration_seconds",
1182
+ help: "Workspace capture gate wait duration across every routed operation, not only startup.",
1183
+ labels: { backend, op, stage, outcome },
1184
+ value: observation.durationMs / 1_000,
1185
+ });
1186
+ observability
1187
+ .startSpan(
1188
+ `sandbox.capture_wait.${stage}`,
1189
+ { backend, outcome },
1190
+ {
1191
+ startTimeMs: Date.now() - observation.durationMs,
1192
+ },
1193
+ )
1194
+ .end({ ...(outcome === "failed" ? { error: true } : {}) });
1195
+ } catch {
1196
+ // Diagnostics never alter capture admission or physical settlement.
1197
+ }
1198
+ };
1199
+ }
1200
+
975
1201
  export type InteractionOperationMetricObservation = {
976
1202
  resource: string;
977
1203
  operation: string;
@@ -1131,6 +1357,87 @@ function boundedMetricEnum(allowed: ReadonlySet<string>, value: string): string
1131
1357
  return allowed.has(value) ? value : "unknown";
1132
1358
  }
1133
1359
 
1360
+ const WORKFLOW_WAKE_ADMISSION_BLOCKERS = new Set([
1361
+ "pending_agent_steer",
1362
+ "pending_prompt_turn",
1363
+ "pending_quiescence",
1364
+ "pending_machine_input",
1365
+ "pending_input_wait",
1366
+ ]);
1367
+ const WORKFLOW_WAKE_OUTCOMES = [
1368
+ "signal_accepted",
1369
+ "acknowledged",
1370
+ "pending_admission",
1371
+ "unconfirmed",
1372
+ "failed",
1373
+ ] as const;
1374
+ const WORKFLOW_WAKE_METRIC = {
1375
+ name: "opengeni_session_workflow_wake_observations_total",
1376
+ help: "Workflow wake dispatch observations; signal acceptance overlaps with admission outcomes, and retries are counted again.",
1377
+ };
1378
+ const WORKFLOW_WAKE_BLOCKER_METRIC = {
1379
+ name: "opengeni_session_workflow_wake_admission_blockers_total",
1380
+ help: "Workflow wake dispatch attempts awaiting durable admission, by bounded blocker; repeated revisions count again.",
1381
+ };
1382
+
1383
+ /** Dispatcher observations count attempts, including repeated observations of
1384
+ * the same revision. Transport acceptance is never an admission receipt. */
1385
+ export function recordWorkflowWakeReconciliation(
1386
+ observability: Observability,
1387
+ result: {
1388
+ signaled: number;
1389
+ delivered: number;
1390
+ pendingAdmission: number;
1391
+ unconfirmed: number;
1392
+ failed: number;
1393
+ pendingAdmissionBlockers: Record<string, number | undefined>;
1394
+ },
1395
+ ): void {
1396
+ const count = (value: number | undefined) =>
1397
+ typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
1398
+ try {
1399
+ for (const [outcome, amount] of [
1400
+ ["signal_accepted", result.signaled],
1401
+ ["acknowledged", result.delivered],
1402
+ ["pending_admission", result.pendingAdmission],
1403
+ ["unconfirmed", result.unconfirmed],
1404
+ ["failed", result.failed],
1405
+ ] as const) {
1406
+ observability.incrementCounter({
1407
+ ...WORKFLOW_WAKE_METRIC,
1408
+ labels: { outcome },
1409
+ amount: count(amount),
1410
+ });
1411
+ if (count(amount) > 0) {
1412
+ // Use the reviewed public telemetry vocabulary. Arbitrary aggregate
1413
+ // property names are intentionally omitted by the external sink.
1414
+ observability.info("Session workflow wake dispatch observed", {
1415
+ surface: "session_workflow_wake",
1416
+ outcome,
1417
+ attempts: count(amount),
1418
+ });
1419
+ }
1420
+ }
1421
+ for (const [blocker, amount] of Object.entries(result.pendingAdmissionBlockers)) {
1422
+ if (count(amount) === 0) continue;
1423
+ const reason = boundedMetricEnum(WORKFLOW_WAKE_ADMISSION_BLOCKERS, blocker);
1424
+ observability.incrementCounter({
1425
+ ...WORKFLOW_WAKE_BLOCKER_METRIC,
1426
+ labels: { reason },
1427
+ amount: count(amount),
1428
+ });
1429
+ observability.info("Session workflow wake admission pending", {
1430
+ surface: "session_workflow_wake",
1431
+ outcome: "pending_admission",
1432
+ reason,
1433
+ attempts: count(amount),
1434
+ });
1435
+ }
1436
+ } catch {
1437
+ recordObserverFailure(observability, "session_workflow_wake");
1438
+ }
1439
+ }
1440
+
1134
1441
  function boundedMetricDuration(durationMs: number): number {
1135
1442
  return Number.isFinite(durationMs) ? Math.max(0, durationMs) / 1_000 : 0;
1136
1443
  }
@@ -1162,7 +1469,7 @@ export function logStartupDependencyRetry(
1162
1469
  ): void {
1163
1470
  const dependency = PUBLIC_STARTUP_DEPENDENCIES.has(event.label) ? event.label : "Dependency";
1164
1471
  observability.warn(
1165
- `Startup dependency ${dependency} connection failed; retrying (${event.attempt}/${event.attempts} in ${event.delayMs}ms)`,
1472
+ `Startup dependency ${dependency} check failed; retrying (${event.attempt}/${event.attempts} in ${event.delayMs}ms)`,
1166
1473
  {
1167
1474
  dependency,
1168
1475
  attempt: event.attempt,
@@ -1200,6 +1507,7 @@ function projectPublicTelemetryAttributes(attributes: Attributes): Attributes {
1200
1507
  ...projectStartupDependencyAttributes(attributes),
1201
1508
  ...projectPublicChannelADiagnosticAttributes(attributes),
1202
1509
  ...projectApiFatalDiagnosticAttributes(attributes),
1510
+ ...projectSnapshotDiagnosticAttributes(attributes),
1203
1511
  ...projectPublicDiagnosticAttributes(attributes),
1204
1512
  };
1205
1513
  }
@@ -1273,6 +1581,55 @@ function projectPublicChannelADiagnosticAttributes(attributes: Attributes): Attr
1273
1581
  };
1274
1582
  }
1275
1583
 
1584
+ function projectSnapshotDiagnosticAttributes(attributes: Attributes): Attributes {
1585
+ if (attributes.errorClass !== "SnapshotOperationError") return {};
1586
+ const projected: Attributes = {};
1587
+ const causeName = attributes.causeName;
1588
+ if (
1589
+ typeof causeName === "string" &&
1590
+ ["SnapshotTimeoutError", "SandboxProviderCaptureTimeoutError", "SandboxProviderError"].includes(
1591
+ causeName,
1592
+ )
1593
+ )
1594
+ projected.causeName = causeName;
1595
+ const name = attributes.providerErrorName;
1596
+ if (
1597
+ typeof name === "string" &&
1598
+ [
1599
+ "ClientError",
1600
+ "TimeoutError",
1601
+ "ConnectionError",
1602
+ "AuthError",
1603
+ "NotFoundError",
1604
+ "InvalidError",
1605
+ "RemoteError",
1606
+ "AbortError",
1607
+ ].includes(name)
1608
+ )
1609
+ projected.providerErrorName = name;
1610
+ const grpc = attributes.providerGrpcCode;
1611
+ if (
1612
+ name === "ClientError" &&
1613
+ typeof grpc === "number" &&
1614
+ Number.isInteger(grpc) &&
1615
+ grpc >= 0 &&
1616
+ grpc <= 16
1617
+ )
1618
+ projected.providerGrpcCode = grpc;
1619
+ const http = attributes.providerHttpStatus;
1620
+ if (typeof http === "number" && Number.isInteger(http) && http >= 100 && http <= 599)
1621
+ projected.providerHttpStatus = http;
1622
+ if (typeof attributes.providerRetryable === "boolean")
1623
+ projected.providerRetryable = attributes.providerRetryable;
1624
+ const leaseKey = attributes.sandboxLeaseKey;
1625
+ if (typeof leaseKey === "string" && /^slk_[0-9a-f]{32}$/.test(leaseKey))
1626
+ projected.sandboxLeaseKey = leaseKey;
1627
+ const epoch = attributes.leaseEpoch;
1628
+ if (typeof epoch === "number" && Number.isSafeInteger(epoch) && epoch >= 0)
1629
+ projected.leaseEpoch = epoch;
1630
+ return projected;
1631
+ }
1632
+
1276
1633
  function projectPublicDiagnosticAttributes(attributes: Attributes): Attributes {
1277
1634
  const errorClass = attributes.errorClass;
1278
1635
  const errorCode = attributes.errorCode;
@@ -1409,6 +1766,7 @@ async function defaultExporter(
1409
1766
  ...headers,
1410
1767
  },
1411
1768
  body: JSON.stringify(body),
1769
+ signal: AbortSignal.timeout(1_000),
1412
1770
  });
1413
1771
  if (!response.ok) {
1414
1772
  throw new Error(`OTLP endpoint returned HTTP ${response.status}`);
@@ -0,0 +1,78 @@
1
+ import { AsyncLocalStorage } from "node:async_hooks";
2
+ import { createHash } from "node:crypto";
3
+
4
+ /** W3C-compatible identity only: never propagate baggage or user attributes. */
5
+ export type TraceContext = { traceId: string; spanId: string };
6
+ type ActiveContext = TraceContext & { addLink?: ((link: TraceContext) => void) | undefined };
7
+ const storage = new AsyncLocalStorage<ActiveContext | undefined>();
8
+
9
+ function ownData(value: unknown, key: string): unknown {
10
+ if (typeof value !== "object" || value === null) return undefined;
11
+ try {
12
+ const descriptor = Object.getOwnPropertyDescriptor(value, key);
13
+ return descriptor && "value" in descriptor ? descriptor.value : undefined;
14
+ } catch {
15
+ return undefined;
16
+ }
17
+ }
18
+
19
+ export function validTraceContext(value: unknown): TraceContext | undefined {
20
+ // Snapshot primitive own data once. Never invoke getters, coercion or toJSON.
21
+ const traceId = ownData(value, "traceId");
22
+ const spanId = ownData(value, "spanId");
23
+ if (typeof traceId !== "string" || typeof spanId !== "string") return undefined;
24
+ if (!/^[0-9a-f]{32}$/.test(traceId) || /^0+$/.test(traceId)) return undefined;
25
+ if (!/^[0-9a-f]{16}$/.test(spanId) || /^0+$/.test(spanId)) return undefined;
26
+ return { traceId, spanId };
27
+ }
28
+
29
+ export function currentTraceContext(): TraceContext | undefined {
30
+ return validTraceContext(storage.getStore());
31
+ }
32
+
33
+ /** Scoped run, never enterWith: interleaved requests cannot inherit each other's spans. */
34
+ export function withTraceContext<T>(context: ActiveContext | undefined, run: () => T): T {
35
+ const valid = validTraceContext(context);
36
+ const addLink = ownData(context, "addLink");
37
+ return storage.run(
38
+ valid
39
+ ? {
40
+ ...valid,
41
+ addLink:
42
+ typeof addLink === "function" ? (addLink as ActiveContext["addLink"]) : undefined,
43
+ }
44
+ : undefined,
45
+ run,
46
+ );
47
+ }
48
+
49
+ /** Stable identity of an actually emitted admission anchor, not session ancestry. */
50
+ export function admissionTraceContext(eventId: string): TraceContext | undefined {
51
+ if (
52
+ typeof eventId !== "string" ||
53
+ !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(eventId)
54
+ )
55
+ return undefined;
56
+ const digest = createHash("sha256")
57
+ .update("opengeni:accepted-event-trace:v1\0")
58
+ .update(eventId.toLowerCase())
59
+ .digest("hex");
60
+ return validTraceContext({ traceId: digest.slice(0, 32), spanId: digest.slice(32, 48) });
61
+ }
62
+
63
+ export function linkCurrentSpanToAdmission(eventId: string): void {
64
+ const link = admissionTraceContext(eventId);
65
+ if (link) storage.getStore()?.addLink?.(link);
66
+ }
67
+
68
+ /** Remote context must be admitted by the caller's trust boundary before use. */
69
+ export function parseTraceparent(value: string | undefined): TraceContext | undefined {
70
+ if (typeof value !== "string") return undefined;
71
+ const match = /^00-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/.exec(value ?? "");
72
+ return match ? validTraceContext({ traceId: match[1]!, spanId: match[2]! }) : undefined;
73
+ }
74
+
75
+ export function traceparent(context: TraceContext): string | undefined {
76
+ const valid = validTraceContext(context);
77
+ return valid ? `00-${valid.traceId}-${valid.spanId}-01` : undefined;
78
+ }