@opengeni/observability 0.8.29 → 0.8.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/export-queue.d.ts +13 -0
- package/dist/failure-diagnostic.d.ts +50 -0
- package/dist/index.d.ts +47 -24
- package/dist/index.js +575 -47
- package/dist/index.js.map +1 -1
- package/dist/trace-context.d.ts +19 -0
- package/package.json +2 -2
- package/src/export-queue.ts +74 -0
- package/src/failure-diagnostic.ts +203 -0
- package/src/index.ts +404 -46
- package/src/trace-context.ts +78 -0
package/src/index.ts
CHANGED
|
@@ -1,6 +1,24 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { collectDefaultMetrics, Counter, Gauge, Histogram, Registry } from "prom-client";
|
|
3
3
|
import { SandboxBackend } from "@opengeni/contracts";
|
|
4
|
+
import {
|
|
5
|
+
currentTraceContext,
|
|
6
|
+
validTraceContext,
|
|
7
|
+
admissionTraceContext,
|
|
8
|
+
type TraceContext,
|
|
9
|
+
} from "./trace-context";
|
|
10
|
+
import { ExportQueue } from "./export-queue";
|
|
11
|
+
import { failureDiagnostic, type FailureDiagnosticInput } from "./failure-diagnostic";
|
|
12
|
+
export type { FailureDiagnosticInput } from "./failure-diagnostic";
|
|
13
|
+
export {
|
|
14
|
+
currentTraceContext,
|
|
15
|
+
withTraceContext,
|
|
16
|
+
parseTraceparent,
|
|
17
|
+
traceparent,
|
|
18
|
+
admissionTraceContext,
|
|
19
|
+
linkCurrentSpanToAdmission,
|
|
20
|
+
type TraceContext,
|
|
21
|
+
} from "./trace-context";
|
|
4
22
|
|
|
5
23
|
export type AttributeValue = string | number | boolean | null | undefined;
|
|
6
24
|
export type Attributes = Record<string, AttributeValue>;
|
|
@@ -13,6 +31,9 @@ export type ObservabilitySettings = {
|
|
|
13
31
|
observabilityMetricsEnabled: boolean;
|
|
14
32
|
observabilityOtlpEndpoint?: string | undefined;
|
|
15
33
|
observabilityOtlpHeaders: string;
|
|
34
|
+
/** Explicit restricted OTLP logs destination; never falls back to the public exporter. */
|
|
35
|
+
observabilityDiagnosticsEndpoint?: string | undefined;
|
|
36
|
+
observabilityDiagnosticsHeaders?: string | undefined;
|
|
16
37
|
sandboxOwnershipEnabled?: boolean | undefined;
|
|
17
38
|
sandboxLazyProvisionEnabled?: boolean | undefined;
|
|
18
39
|
rigVerificationLeaseOwnershipEnabled?: boolean | undefined;
|
|
@@ -27,6 +48,7 @@ export type ObservabilityOptions = {
|
|
|
27
48
|
export type Span = {
|
|
28
49
|
traceId: string;
|
|
29
50
|
spanId: string;
|
|
51
|
+
addLink?: (context: TraceContext) => void;
|
|
30
52
|
end: (input?: { attributes?: Attributes; error?: unknown }) => void;
|
|
31
53
|
};
|
|
32
54
|
|
|
@@ -49,6 +71,25 @@ export function sandboxLeaseTelemetryKey(workspaceId: string, sandboxGroupId: st
|
|
|
49
71
|
.slice(0, 32)}`;
|
|
50
72
|
}
|
|
51
73
|
|
|
74
|
+
/** Content-free lookup key for a worker execution attempt, never a metric label.
|
|
75
|
+
* Operators derive it from authorized session/attempt records to find a trace
|
|
76
|
+
* without publishing raw tenancy or conversation identifiers to telemetry. */
|
|
77
|
+
export function turnExecutionTelemetryKey(
|
|
78
|
+
workspaceId: string,
|
|
79
|
+
sessionId: string,
|
|
80
|
+
attemptId: string,
|
|
81
|
+
): string {
|
|
82
|
+
return `turn_${createHash("sha256")
|
|
83
|
+
.update("opengeni:turn-execution-telemetry:v1\0")
|
|
84
|
+
.update(workspaceId)
|
|
85
|
+
.update("\0")
|
|
86
|
+
.update(sessionId)
|
|
87
|
+
.update("\0")
|
|
88
|
+
.update(attemptId)
|
|
89
|
+
.digest("hex")
|
|
90
|
+
.slice(0, 32)}`;
|
|
91
|
+
}
|
|
92
|
+
|
|
52
93
|
/**
|
|
53
94
|
* Stable selectors shared by OpenGeni's runtime metrics and optional
|
|
54
95
|
* Prometheus/Grafana distribution. Operators can use these values for custom
|
|
@@ -111,26 +152,6 @@ const SANDBOX_OPERATION_NAMES = new Set([
|
|
|
111
152
|
* credential, name, or content, may ever become a label here.
|
|
112
153
|
*/
|
|
113
154
|
export const TENANCY_COMPATIBILITY_LANES = [
|
|
114
|
-
/**
|
|
115
|
-
* A provider connection carrying `authority_scope = 'legacy_user'` was
|
|
116
|
-
* resolved for accepted use: a personal row with no common authority or
|
|
117
|
-
* grant, admitted through `legacy_user_compatibility` provenance instead of
|
|
118
|
-
* an explicit delegation. Migration 0256 backfilled the existing rows, but
|
|
119
|
-
* its `bind_connection_authority` trigger still classifies a NEW personal
|
|
120
|
-
* connection this way whenever the inserting subject has no active
|
|
121
|
-
* organization membership, so this counter is expected to stay above zero
|
|
122
|
-
* until that classification changes. That is the finding, not a defect.
|
|
123
|
-
*/
|
|
124
|
-
"connection_legacy_user",
|
|
125
|
-
/**
|
|
126
|
-
* A workspace-scope connection ref carried no connection id, so the
|
|
127
|
-
* accepted-use authority (migration 0279) could not identify the exact row
|
|
128
|
-
* and the request fell back to the unprivileged pre-snapshot resolution.
|
|
129
|
-
* This lane writes no `connection_use_audit_facts` row and leaves no durable
|
|
130
|
-
* trace of any kind, so a use counter is the only possible evidence it was
|
|
131
|
-
* taken at all.
|
|
132
|
-
*/
|
|
133
|
-
"connection_pre_snapshot_ref",
|
|
134
155
|
/**
|
|
135
156
|
* A NEW persistable `/workspace` mutation was refused because the writer
|
|
136
157
|
* behind it has no recorded authority (`authority_unattributed`): either a
|
|
@@ -481,8 +502,21 @@ export type SandboxOperationMetricObservation = {
|
|
|
481
502
|
op: string;
|
|
482
503
|
outcome: "ok" | "not_found" | "failed";
|
|
483
504
|
durationMs: number;
|
|
505
|
+
materializationFailureReason?: string;
|
|
484
506
|
};
|
|
485
507
|
|
|
508
|
+
const MATERIALIZATION_FAILURE_REASONS = new Set([
|
|
509
|
+
"path_not_visible",
|
|
510
|
+
"command_failed",
|
|
511
|
+
"command_pending",
|
|
512
|
+
"invalid_response",
|
|
513
|
+
"command_error",
|
|
514
|
+
]);
|
|
515
|
+
const MATERIALIZATION_FAILURE_METRIC = {
|
|
516
|
+
name: "opengeni_sandbox_materialization_verification_failures_total",
|
|
517
|
+
help: "Sandbox destination visibility-check failures by bounded reason; exact command evidence is retained in the session failure event.",
|
|
518
|
+
} as const;
|
|
519
|
+
|
|
486
520
|
export function createObservability(
|
|
487
521
|
settings: ObservabilitySettings,
|
|
488
522
|
options: ObservabilityOptions,
|
|
@@ -496,12 +530,33 @@ type MetricRegistration = {
|
|
|
496
530
|
};
|
|
497
531
|
|
|
498
532
|
export class Observability {
|
|
533
|
+
private spanBatch: unknown[] = [];
|
|
534
|
+
private spanBatchScheduled = false;
|
|
535
|
+
private readonly diagnosticQueue = new ExportQueue((outcome) => {
|
|
536
|
+
this.incrementCounter({
|
|
537
|
+
name: "opengeni_diagnostic_exports_total",
|
|
538
|
+
help: "Protected diagnostic export outcomes.",
|
|
539
|
+
labels: { outcome },
|
|
540
|
+
});
|
|
541
|
+
});
|
|
499
542
|
private readonly registry = new Registry();
|
|
500
543
|
private readonly counters = new Map<string, Counter<string>>();
|
|
501
544
|
private readonly gauges = new Map<string, Gauge<string>>();
|
|
502
545
|
private readonly histograms = new Map<string, Histogram<string>>();
|
|
503
546
|
private readonly registrations = new Map<string, MetricRegistration>();
|
|
504
|
-
private readonly
|
|
547
|
+
private readonly exportQueue = new ExportQueue((outcome) => {
|
|
548
|
+
this.incrementCounter({
|
|
549
|
+
name: "opengeni_telemetry_exports_total",
|
|
550
|
+
help: "Bounded telemetry export outcomes.",
|
|
551
|
+
labels: { outcome },
|
|
552
|
+
});
|
|
553
|
+
if (outcome === "failed")
|
|
554
|
+
this.warn("OTLP span export failed", {
|
|
555
|
+
errorClass: "TelemetryExportError",
|
|
556
|
+
errorCode: "otlp_export_failed",
|
|
557
|
+
origin: "observability",
|
|
558
|
+
});
|
|
559
|
+
}, 8);
|
|
505
560
|
private readonly now: () => number;
|
|
506
561
|
private readonly exporter: (
|
|
507
562
|
url: string,
|
|
@@ -540,6 +595,27 @@ export class Observability {
|
|
|
540
595
|
});
|
|
541
596
|
this.registerSandboxRolloutConfig();
|
|
542
597
|
this.registerTenancyCompatibilityLanes();
|
|
598
|
+
if (["worker", "worker-control", "worker-turn", "api"].includes(options.component)) {
|
|
599
|
+
// Publish the finite series at startup so a first failure has a zero
|
|
600
|
+
// baseline and healthy zeroes differ from a missing instrumentation path.
|
|
601
|
+
for (const backend of [...SANDBOX_OPERATION_BACKENDS, "unknown"]) {
|
|
602
|
+
for (const reason of [...MATERIALIZATION_FAILURE_REASONS, "unknown"]) {
|
|
603
|
+
this.incrementCounter({
|
|
604
|
+
...MATERIALIZATION_FAILURE_METRIC,
|
|
605
|
+
labels: { backend, reason },
|
|
606
|
+
amount: 0,
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
if (options.component === "worker" || options.component === "worker-control") {
|
|
612
|
+
for (const outcome of WORKFLOW_WAKE_OUTCOMES) {
|
|
613
|
+
this.incrementCounter({ ...WORKFLOW_WAKE_METRIC, labels: { outcome }, amount: 0 });
|
|
614
|
+
}
|
|
615
|
+
for (const reason of [...WORKFLOW_WAKE_ADMISSION_BLOCKERS, "unknown"]) {
|
|
616
|
+
this.incrementCounter({ ...WORKFLOW_WAKE_BLOCKER_METRIC, labels: { reason }, amount: 0 });
|
|
617
|
+
}
|
|
618
|
+
}
|
|
543
619
|
}
|
|
544
620
|
}
|
|
545
621
|
|
|
@@ -617,6 +693,7 @@ export class Observability {
|
|
|
617
693
|
}
|
|
618
694
|
return;
|
|
619
695
|
}
|
|
696
|
+
const context = currentTraceContext();
|
|
620
697
|
const record = {
|
|
621
698
|
timestamp: new Date(this.now()).toISOString(),
|
|
622
699
|
level,
|
|
@@ -625,6 +702,7 @@ export class Observability {
|
|
|
625
702
|
environment: this.settings.environment,
|
|
626
703
|
component: this.options.component,
|
|
627
704
|
...cleanAttributes(publicAttributes),
|
|
705
|
+
...(context ? { traceId: context.traceId, spanId: context.spanId } : {}),
|
|
628
706
|
};
|
|
629
707
|
const serialized = JSON.stringify(record);
|
|
630
708
|
if (level === "warn") {
|
|
@@ -636,14 +714,39 @@ export class Observability {
|
|
|
636
714
|
}
|
|
637
715
|
}
|
|
638
716
|
|
|
639
|
-
startSpan(
|
|
640
|
-
|
|
717
|
+
startSpan(
|
|
718
|
+
name: string,
|
|
719
|
+
attributes: Attributes = {},
|
|
720
|
+
options: { parent?: TraceContext | null; links?: TraceContext[]; startTimeMs?: number } = {},
|
|
721
|
+
): Span {
|
|
722
|
+
const parent =
|
|
723
|
+
options.parent === null
|
|
724
|
+
? undefined
|
|
725
|
+
: validTraceContext(options.parent ?? currentTraceContext());
|
|
726
|
+
const links = (options.links ?? []).slice(0, 8).flatMap((link) => {
|
|
727
|
+
const valid = validTraceContext(link);
|
|
728
|
+
return valid ? [valid] : [];
|
|
729
|
+
});
|
|
730
|
+
const traceId = parent?.traceId ?? randomHex(16);
|
|
641
731
|
const spanId = randomHex(8);
|
|
642
|
-
const
|
|
732
|
+
const now = this.now();
|
|
733
|
+
const startMs = Number.isFinite(options.startTimeMs)
|
|
734
|
+
? Math.min(now, Math.max(0, options.startTimeMs!))
|
|
735
|
+
: now;
|
|
643
736
|
let ended = false;
|
|
644
737
|
return {
|
|
645
738
|
traceId,
|
|
646
739
|
spanId,
|
|
740
|
+
addLink: (context) => {
|
|
741
|
+
const valid = validTraceContext(context);
|
|
742
|
+
if (
|
|
743
|
+
!ended &&
|
|
744
|
+
valid &&
|
|
745
|
+
links.length < 8 &&
|
|
746
|
+
!links.some((link) => link.traceId === valid.traceId && link.spanId === valid.spanId)
|
|
747
|
+
)
|
|
748
|
+
links.push(valid);
|
|
749
|
+
},
|
|
647
750
|
end: (input = {}) => {
|
|
648
751
|
if (ended) {
|
|
649
752
|
return;
|
|
@@ -657,7 +760,9 @@ export class Observability {
|
|
|
657
760
|
this.exportSpan({
|
|
658
761
|
traceId,
|
|
659
762
|
spanId,
|
|
660
|
-
|
|
763
|
+
...(parent ? { parentSpanId: parent.spanId } : {}),
|
|
764
|
+
links,
|
|
765
|
+
name: boundedOtlpString(name),
|
|
661
766
|
startMs,
|
|
662
767
|
endMs: this.now(),
|
|
663
768
|
attributes: {
|
|
@@ -792,11 +897,70 @@ export class Observability {
|
|
|
792
897
|
return await this.registry.metrics();
|
|
793
898
|
}
|
|
794
899
|
|
|
795
|
-
/**
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
900
|
+
/** Emit a content-free anchor after successful, non-replayed admission. */
|
|
901
|
+
recordAdmissionTrace(eventId: string): void {
|
|
902
|
+
const identity = admissionTraceContext(eventId);
|
|
903
|
+
if (!identity) return;
|
|
904
|
+
const current = currentTraceContext();
|
|
905
|
+
const now = this.now();
|
|
906
|
+
this.exportSpan({
|
|
907
|
+
...identity,
|
|
908
|
+
name: "api.turn.admitted",
|
|
909
|
+
startMs: now,
|
|
910
|
+
endMs: now,
|
|
911
|
+
attributes: {},
|
|
912
|
+
links: current ? [current] : [],
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
/** Emit a closed protected record independently of application persistence. */
|
|
917
|
+
recordFailureDiagnostic(input: FailureDiagnosticInput): string {
|
|
918
|
+
const record = failureDiagnostic(input, this.settings.deploymentRevision);
|
|
919
|
+
const endpoint = this.settings.observabilityDiagnosticsEndpoint;
|
|
920
|
+
if (!endpoint) {
|
|
921
|
+
this.incrementCounter({
|
|
922
|
+
name: "opengeni_diagnostic_exports_total",
|
|
923
|
+
help: "Protected diagnostic export outcomes.",
|
|
924
|
+
labels: { outcome: "disabled" },
|
|
925
|
+
});
|
|
926
|
+
return record.diagnosticId;
|
|
799
927
|
}
|
|
928
|
+
const context = currentTraceContext();
|
|
929
|
+
const body = {
|
|
930
|
+
resourceLogs: [
|
|
931
|
+
{
|
|
932
|
+
resource: { attributes: otlpAttributes(this.resourceAttributes) },
|
|
933
|
+
scopeLogs: [
|
|
934
|
+
{
|
|
935
|
+
scope: { name: "@opengeni/observability/diagnostics", version: "1" },
|
|
936
|
+
logRecords: [
|
|
937
|
+
{
|
|
938
|
+
timeUnixNano: millisToNanos(this.now()),
|
|
939
|
+
severityNumber: 17,
|
|
940
|
+
severityText: "ERROR",
|
|
941
|
+
...(context ? { traceId: context.traceId, spanId: context.spanId } : {}),
|
|
942
|
+
body: { stringValue: JSON.stringify(record) },
|
|
943
|
+
},
|
|
944
|
+
],
|
|
945
|
+
},
|
|
946
|
+
],
|
|
947
|
+
},
|
|
948
|
+
],
|
|
949
|
+
};
|
|
950
|
+
this.diagnosticQueue.enqueue(() =>
|
|
951
|
+
this.exporter(
|
|
952
|
+
`${endpoint.replace(/\/$/, "")}/v1/logs`,
|
|
953
|
+
body,
|
|
954
|
+
parseHeaders(this.settings.observabilityDiagnosticsHeaders ?? ""),
|
|
955
|
+
),
|
|
956
|
+
);
|
|
957
|
+
return record.diagnosticId;
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
/** Best-effort bounded drain, including when a custom transport never settles. */
|
|
961
|
+
async flush(timeoutMs = 1_000): Promise<void> {
|
|
962
|
+
this.submitSpanBatch();
|
|
963
|
+
await Promise.all([this.exportQueue.flush(timeoutMs), this.diagnosticQueue.flush(timeoutMs)]);
|
|
800
964
|
}
|
|
801
965
|
|
|
802
966
|
private counter(name: string, help: string, labelNames: string[]): Counter<string> {
|
|
@@ -868,6 +1032,8 @@ export class Observability {
|
|
|
868
1032
|
private exportSpan(span: {
|
|
869
1033
|
traceId: string;
|
|
870
1034
|
spanId: string;
|
|
1035
|
+
parentSpanId?: string;
|
|
1036
|
+
links: TraceContext[];
|
|
871
1037
|
name: string;
|
|
872
1038
|
startMs: number;
|
|
873
1039
|
endMs: number;
|
|
@@ -877,7 +1043,6 @@ export class Observability {
|
|
|
877
1043
|
if (!this.settings.observabilityOtlpEndpoint) {
|
|
878
1044
|
return;
|
|
879
1045
|
}
|
|
880
|
-
const endpoint = `${this.settings.observabilityOtlpEndpoint.replace(/\/$/, "")}/v1/traces`;
|
|
881
1046
|
const body = {
|
|
882
1047
|
resourceSpans: [
|
|
883
1048
|
{
|
|
@@ -894,6 +1059,8 @@ export class Observability {
|
|
|
894
1059
|
{
|
|
895
1060
|
traceId: span.traceId,
|
|
896
1061
|
spanId: span.spanId,
|
|
1062
|
+
...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}),
|
|
1063
|
+
links: span.links,
|
|
897
1064
|
name: span.name,
|
|
898
1065
|
kind: 1,
|
|
899
1066
|
startTimeUnixNano: millisToNanos(span.startMs),
|
|
@@ -907,22 +1074,29 @@ export class Observability {
|
|
|
907
1074
|
},
|
|
908
1075
|
],
|
|
909
1076
|
};
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
.
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
this.warn("OTLP span export failed", {
|
|
917
|
-
errorClass: "TelemetryExportError",
|
|
918
|
-
errorCode: "otlp_export_failed",
|
|
919
|
-
origin: "observability",
|
|
920
|
-
});
|
|
921
|
-
})
|
|
922
|
-
.finally(() => {
|
|
923
|
-
this.pendingExports.delete(pendingExport);
|
|
1077
|
+
this.spanBatch.push(body.resourceSpans[0]);
|
|
1078
|
+
if (!this.spanBatchScheduled) {
|
|
1079
|
+
this.spanBatchScheduled = true;
|
|
1080
|
+
queueMicrotask(() => {
|
|
1081
|
+
this.spanBatchScheduled = false;
|
|
1082
|
+
this.submitSpanBatch();
|
|
924
1083
|
});
|
|
925
|
-
|
|
1084
|
+
}
|
|
1085
|
+
if (this.spanBatch.length >= 32) this.submitSpanBatch();
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
private submitSpanBatch(): void {
|
|
1089
|
+
if (!this.spanBatch.length || !this.settings.observabilityOtlpEndpoint) return;
|
|
1090
|
+
const resourceSpans = this.spanBatch;
|
|
1091
|
+
this.spanBatch = [];
|
|
1092
|
+
const endpoint = `${this.settings.observabilityOtlpEndpoint.replace(/\/$/, "")}/v1/traces`;
|
|
1093
|
+
this.exportQueue.enqueue(() =>
|
|
1094
|
+
this.exporter(
|
|
1095
|
+
endpoint,
|
|
1096
|
+
{ resourceSpans },
|
|
1097
|
+
parseHeaders(this.settings.observabilityOtlpHeaders),
|
|
1098
|
+
),
|
|
1099
|
+
);
|
|
926
1100
|
}
|
|
927
1101
|
}
|
|
928
1102
|
|
|
@@ -957,6 +1131,19 @@ export function sandboxOperationMetricObserver(
|
|
|
957
1131
|
labels: { backend, op },
|
|
958
1132
|
value: Math.max(0, observation.durationMs) / 1_000,
|
|
959
1133
|
});
|
|
1134
|
+
if (
|
|
1135
|
+
op === "materializeEntry" &&
|
|
1136
|
+
observation.outcome === "failed" &&
|
|
1137
|
+
observation.materializationFailureReason
|
|
1138
|
+
) {
|
|
1139
|
+
const reason = MATERIALIZATION_FAILURE_REASONS.has(observation.materializationFailureReason)
|
|
1140
|
+
? observation.materializationFailureReason
|
|
1141
|
+
: "unknown";
|
|
1142
|
+
observability.incrementCounter({
|
|
1143
|
+
...MATERIALIZATION_FAILURE_METRIC,
|
|
1144
|
+
labels: { backend, reason },
|
|
1145
|
+
});
|
|
1146
|
+
}
|
|
960
1147
|
} catch {
|
|
961
1148
|
try {
|
|
962
1149
|
observability.incrementCounter({
|
|
@@ -972,6 +1159,45 @@ export function sandboxOperationMetricObserver(
|
|
|
972
1159
|
};
|
|
973
1160
|
}
|
|
974
1161
|
|
|
1162
|
+
/** Capture waits remain separate from physical provider-call accounting. */
|
|
1163
|
+
export function sandboxCaptureWaitMetricObserver(observability: Observability) {
|
|
1164
|
+
return (observation: {
|
|
1165
|
+
backend: string;
|
|
1166
|
+
op: string;
|
|
1167
|
+
outcome: "ok" | "failed";
|
|
1168
|
+
durationMs: number;
|
|
1169
|
+
captureWaitStage: "admission" | "provider";
|
|
1170
|
+
}): void => {
|
|
1171
|
+
const stage = observation.captureWaitStage;
|
|
1172
|
+
if (stage !== "admission" && stage !== "provider") return;
|
|
1173
|
+
if (!Number.isFinite(observation.durationMs) || observation.durationMs < 0) return;
|
|
1174
|
+
const backend = SANDBOX_OPERATION_BACKENDS.has(observation.backend)
|
|
1175
|
+
? observation.backend
|
|
1176
|
+
: "unknown";
|
|
1177
|
+
const op = SANDBOX_OPERATION_NAMES.has(observation.op) ? observation.op : "unknown";
|
|
1178
|
+
const outcome = observation.outcome === "ok" ? "completed" : "failed";
|
|
1179
|
+
try {
|
|
1180
|
+
observability.observeHistogram({
|
|
1181
|
+
name: "opengeni_sandbox_capture_wait_duration_seconds",
|
|
1182
|
+
help: "Workspace capture gate wait duration across every routed operation, not only startup.",
|
|
1183
|
+
labels: { backend, op, stage, outcome },
|
|
1184
|
+
value: observation.durationMs / 1_000,
|
|
1185
|
+
});
|
|
1186
|
+
observability
|
|
1187
|
+
.startSpan(
|
|
1188
|
+
`sandbox.capture_wait.${stage}`,
|
|
1189
|
+
{ backend, outcome },
|
|
1190
|
+
{
|
|
1191
|
+
startTimeMs: Date.now() - observation.durationMs,
|
|
1192
|
+
},
|
|
1193
|
+
)
|
|
1194
|
+
.end({ ...(outcome === "failed" ? { error: true } : {}) });
|
|
1195
|
+
} catch {
|
|
1196
|
+
// Diagnostics never alter capture admission or physical settlement.
|
|
1197
|
+
}
|
|
1198
|
+
};
|
|
1199
|
+
}
|
|
1200
|
+
|
|
975
1201
|
export type InteractionOperationMetricObservation = {
|
|
976
1202
|
resource: string;
|
|
977
1203
|
operation: string;
|
|
@@ -1131,6 +1357,87 @@ function boundedMetricEnum(allowed: ReadonlySet<string>, value: string): string
|
|
|
1131
1357
|
return allowed.has(value) ? value : "unknown";
|
|
1132
1358
|
}
|
|
1133
1359
|
|
|
1360
|
+
const WORKFLOW_WAKE_ADMISSION_BLOCKERS = new Set([
|
|
1361
|
+
"pending_agent_steer",
|
|
1362
|
+
"pending_prompt_turn",
|
|
1363
|
+
"pending_quiescence",
|
|
1364
|
+
"pending_machine_input",
|
|
1365
|
+
"pending_input_wait",
|
|
1366
|
+
]);
|
|
1367
|
+
const WORKFLOW_WAKE_OUTCOMES = [
|
|
1368
|
+
"signal_accepted",
|
|
1369
|
+
"acknowledged",
|
|
1370
|
+
"pending_admission",
|
|
1371
|
+
"unconfirmed",
|
|
1372
|
+
"failed",
|
|
1373
|
+
] as const;
|
|
1374
|
+
const WORKFLOW_WAKE_METRIC = {
|
|
1375
|
+
name: "opengeni_session_workflow_wake_observations_total",
|
|
1376
|
+
help: "Workflow wake dispatch observations; signal acceptance overlaps with admission outcomes, and retries are counted again.",
|
|
1377
|
+
};
|
|
1378
|
+
const WORKFLOW_WAKE_BLOCKER_METRIC = {
|
|
1379
|
+
name: "opengeni_session_workflow_wake_admission_blockers_total",
|
|
1380
|
+
help: "Workflow wake dispatch attempts awaiting durable admission, by bounded blocker; repeated revisions count again.",
|
|
1381
|
+
};
|
|
1382
|
+
|
|
1383
|
+
/** Dispatcher observations count attempts, including repeated observations of
|
|
1384
|
+
* the same revision. Transport acceptance is never an admission receipt. */
|
|
1385
|
+
export function recordWorkflowWakeReconciliation(
|
|
1386
|
+
observability: Observability,
|
|
1387
|
+
result: {
|
|
1388
|
+
signaled: number;
|
|
1389
|
+
delivered: number;
|
|
1390
|
+
pendingAdmission: number;
|
|
1391
|
+
unconfirmed: number;
|
|
1392
|
+
failed: number;
|
|
1393
|
+
pendingAdmissionBlockers: Record<string, number | undefined>;
|
|
1394
|
+
},
|
|
1395
|
+
): void {
|
|
1396
|
+
const count = (value: number | undefined) =>
|
|
1397
|
+
typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
1398
|
+
try {
|
|
1399
|
+
for (const [outcome, amount] of [
|
|
1400
|
+
["signal_accepted", result.signaled],
|
|
1401
|
+
["acknowledged", result.delivered],
|
|
1402
|
+
["pending_admission", result.pendingAdmission],
|
|
1403
|
+
["unconfirmed", result.unconfirmed],
|
|
1404
|
+
["failed", result.failed],
|
|
1405
|
+
] as const) {
|
|
1406
|
+
observability.incrementCounter({
|
|
1407
|
+
...WORKFLOW_WAKE_METRIC,
|
|
1408
|
+
labels: { outcome },
|
|
1409
|
+
amount: count(amount),
|
|
1410
|
+
});
|
|
1411
|
+
if (count(amount) > 0) {
|
|
1412
|
+
// Use the reviewed public telemetry vocabulary. Arbitrary aggregate
|
|
1413
|
+
// property names are intentionally omitted by the external sink.
|
|
1414
|
+
observability.info("Session workflow wake dispatch observed", {
|
|
1415
|
+
surface: "session_workflow_wake",
|
|
1416
|
+
outcome,
|
|
1417
|
+
attempts: count(amount),
|
|
1418
|
+
});
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
for (const [blocker, amount] of Object.entries(result.pendingAdmissionBlockers)) {
|
|
1422
|
+
if (count(amount) === 0) continue;
|
|
1423
|
+
const reason = boundedMetricEnum(WORKFLOW_WAKE_ADMISSION_BLOCKERS, blocker);
|
|
1424
|
+
observability.incrementCounter({
|
|
1425
|
+
...WORKFLOW_WAKE_BLOCKER_METRIC,
|
|
1426
|
+
labels: { reason },
|
|
1427
|
+
amount: count(amount),
|
|
1428
|
+
});
|
|
1429
|
+
observability.info("Session workflow wake admission pending", {
|
|
1430
|
+
surface: "session_workflow_wake",
|
|
1431
|
+
outcome: "pending_admission",
|
|
1432
|
+
reason,
|
|
1433
|
+
attempts: count(amount),
|
|
1434
|
+
});
|
|
1435
|
+
}
|
|
1436
|
+
} catch {
|
|
1437
|
+
recordObserverFailure(observability, "session_workflow_wake");
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1134
1441
|
function boundedMetricDuration(durationMs: number): number {
|
|
1135
1442
|
return Number.isFinite(durationMs) ? Math.max(0, durationMs) / 1_000 : 0;
|
|
1136
1443
|
}
|
|
@@ -1162,7 +1469,7 @@ export function logStartupDependencyRetry(
|
|
|
1162
1469
|
): void {
|
|
1163
1470
|
const dependency = PUBLIC_STARTUP_DEPENDENCIES.has(event.label) ? event.label : "Dependency";
|
|
1164
1471
|
observability.warn(
|
|
1165
|
-
`Startup dependency ${dependency}
|
|
1472
|
+
`Startup dependency ${dependency} check failed; retrying (${event.attempt}/${event.attempts} in ${event.delayMs}ms)`,
|
|
1166
1473
|
{
|
|
1167
1474
|
dependency,
|
|
1168
1475
|
attempt: event.attempt,
|
|
@@ -1200,6 +1507,7 @@ function projectPublicTelemetryAttributes(attributes: Attributes): Attributes {
|
|
|
1200
1507
|
...projectStartupDependencyAttributes(attributes),
|
|
1201
1508
|
...projectPublicChannelADiagnosticAttributes(attributes),
|
|
1202
1509
|
...projectApiFatalDiagnosticAttributes(attributes),
|
|
1510
|
+
...projectSnapshotDiagnosticAttributes(attributes),
|
|
1203
1511
|
...projectPublicDiagnosticAttributes(attributes),
|
|
1204
1512
|
};
|
|
1205
1513
|
}
|
|
@@ -1273,6 +1581,55 @@ function projectPublicChannelADiagnosticAttributes(attributes: Attributes): Attr
|
|
|
1273
1581
|
};
|
|
1274
1582
|
}
|
|
1275
1583
|
|
|
1584
|
+
function projectSnapshotDiagnosticAttributes(attributes: Attributes): Attributes {
|
|
1585
|
+
if (attributes.errorClass !== "SnapshotOperationError") return {};
|
|
1586
|
+
const projected: Attributes = {};
|
|
1587
|
+
const causeName = attributes.causeName;
|
|
1588
|
+
if (
|
|
1589
|
+
typeof causeName === "string" &&
|
|
1590
|
+
["SnapshotTimeoutError", "SandboxProviderCaptureTimeoutError", "SandboxProviderError"].includes(
|
|
1591
|
+
causeName,
|
|
1592
|
+
)
|
|
1593
|
+
)
|
|
1594
|
+
projected.causeName = causeName;
|
|
1595
|
+
const name = attributes.providerErrorName;
|
|
1596
|
+
if (
|
|
1597
|
+
typeof name === "string" &&
|
|
1598
|
+
[
|
|
1599
|
+
"ClientError",
|
|
1600
|
+
"TimeoutError",
|
|
1601
|
+
"ConnectionError",
|
|
1602
|
+
"AuthError",
|
|
1603
|
+
"NotFoundError",
|
|
1604
|
+
"InvalidError",
|
|
1605
|
+
"RemoteError",
|
|
1606
|
+
"AbortError",
|
|
1607
|
+
].includes(name)
|
|
1608
|
+
)
|
|
1609
|
+
projected.providerErrorName = name;
|
|
1610
|
+
const grpc = attributes.providerGrpcCode;
|
|
1611
|
+
if (
|
|
1612
|
+
name === "ClientError" &&
|
|
1613
|
+
typeof grpc === "number" &&
|
|
1614
|
+
Number.isInteger(grpc) &&
|
|
1615
|
+
grpc >= 0 &&
|
|
1616
|
+
grpc <= 16
|
|
1617
|
+
)
|
|
1618
|
+
projected.providerGrpcCode = grpc;
|
|
1619
|
+
const http = attributes.providerHttpStatus;
|
|
1620
|
+
if (typeof http === "number" && Number.isInteger(http) && http >= 100 && http <= 599)
|
|
1621
|
+
projected.providerHttpStatus = http;
|
|
1622
|
+
if (typeof attributes.providerRetryable === "boolean")
|
|
1623
|
+
projected.providerRetryable = attributes.providerRetryable;
|
|
1624
|
+
const leaseKey = attributes.sandboxLeaseKey;
|
|
1625
|
+
if (typeof leaseKey === "string" && /^slk_[0-9a-f]{32}$/.test(leaseKey))
|
|
1626
|
+
projected.sandboxLeaseKey = leaseKey;
|
|
1627
|
+
const epoch = attributes.leaseEpoch;
|
|
1628
|
+
if (typeof epoch === "number" && Number.isSafeInteger(epoch) && epoch >= 0)
|
|
1629
|
+
projected.leaseEpoch = epoch;
|
|
1630
|
+
return projected;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1276
1633
|
function projectPublicDiagnosticAttributes(attributes: Attributes): Attributes {
|
|
1277
1634
|
const errorClass = attributes.errorClass;
|
|
1278
1635
|
const errorCode = attributes.errorCode;
|
|
@@ -1409,6 +1766,7 @@ async function defaultExporter(
|
|
|
1409
1766
|
...headers,
|
|
1410
1767
|
},
|
|
1411
1768
|
body: JSON.stringify(body),
|
|
1769
|
+
signal: AbortSignal.timeout(1_000),
|
|
1412
1770
|
});
|
|
1413
1771
|
if (!response.ok) {
|
|
1414
1772
|
throw new Error(`OTLP endpoint returned HTTP ${response.status}`);
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
|
|
4
|
+
/** W3C-compatible identity only: never propagate baggage or user attributes. */
|
|
5
|
+
export type TraceContext = { traceId: string; spanId: string };
|
|
6
|
+
type ActiveContext = TraceContext & { addLink?: ((link: TraceContext) => void) | undefined };
|
|
7
|
+
const storage = new AsyncLocalStorage<ActiveContext | undefined>();
|
|
8
|
+
|
|
9
|
+
function ownData(value: unknown, key: string): unknown {
|
|
10
|
+
if (typeof value !== "object" || value === null) return undefined;
|
|
11
|
+
try {
|
|
12
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
13
|
+
return descriptor && "value" in descriptor ? descriptor.value : undefined;
|
|
14
|
+
} catch {
|
|
15
|
+
return undefined;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function validTraceContext(value: unknown): TraceContext | undefined {
|
|
20
|
+
// Snapshot primitive own data once. Never invoke getters, coercion or toJSON.
|
|
21
|
+
const traceId = ownData(value, "traceId");
|
|
22
|
+
const spanId = ownData(value, "spanId");
|
|
23
|
+
if (typeof traceId !== "string" || typeof spanId !== "string") return undefined;
|
|
24
|
+
if (!/^[0-9a-f]{32}$/.test(traceId) || /^0+$/.test(traceId)) return undefined;
|
|
25
|
+
if (!/^[0-9a-f]{16}$/.test(spanId) || /^0+$/.test(spanId)) return undefined;
|
|
26
|
+
return { traceId, spanId };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function currentTraceContext(): TraceContext | undefined {
|
|
30
|
+
return validTraceContext(storage.getStore());
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Scoped run, never enterWith: interleaved requests cannot inherit each other's spans. */
|
|
34
|
+
export function withTraceContext<T>(context: ActiveContext | undefined, run: () => T): T {
|
|
35
|
+
const valid = validTraceContext(context);
|
|
36
|
+
const addLink = ownData(context, "addLink");
|
|
37
|
+
return storage.run(
|
|
38
|
+
valid
|
|
39
|
+
? {
|
|
40
|
+
...valid,
|
|
41
|
+
addLink:
|
|
42
|
+
typeof addLink === "function" ? (addLink as ActiveContext["addLink"]) : undefined,
|
|
43
|
+
}
|
|
44
|
+
: undefined,
|
|
45
|
+
run,
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Stable identity of an actually emitted admission anchor, not session ancestry. */
|
|
50
|
+
export function admissionTraceContext(eventId: string): TraceContext | undefined {
|
|
51
|
+
if (
|
|
52
|
+
typeof eventId !== "string" ||
|
|
53
|
+
!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(eventId)
|
|
54
|
+
)
|
|
55
|
+
return undefined;
|
|
56
|
+
const digest = createHash("sha256")
|
|
57
|
+
.update("opengeni:accepted-event-trace:v1\0")
|
|
58
|
+
.update(eventId.toLowerCase())
|
|
59
|
+
.digest("hex");
|
|
60
|
+
return validTraceContext({ traceId: digest.slice(0, 32), spanId: digest.slice(32, 48) });
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function linkCurrentSpanToAdmission(eventId: string): void {
|
|
64
|
+
const link = admissionTraceContext(eventId);
|
|
65
|
+
if (link) storage.getStore()?.addLink?.(link);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Remote context must be admitted by the caller's trust boundary before use. */
|
|
69
|
+
export function parseTraceparent(value: string | undefined): TraceContext | undefined {
|
|
70
|
+
if (typeof value !== "string") return undefined;
|
|
71
|
+
const match = /^00-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/.exec(value ?? "");
|
|
72
|
+
return match ? validTraceContext({ traceId: match[1]!, spanId: match[2]! }) : undefined;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function traceparent(context: TraceContext): string | undefined {
|
|
76
|
+
const valid = validTraceContext(context);
|
|
77
|
+
return valid ? `00-${valid.traceId}-${valid.spanId}-01` : undefined;
|
|
78
|
+
}
|