@opengeni/observability 0.8.29 → 0.8.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/export-queue.d.ts +13 -0
- package/dist/failure-diagnostic.d.ts +50 -0
- package/dist/index.d.ts +47 -24
- package/dist/index.js +575 -47
- package/dist/index.js.map +1 -1
- package/dist/trace-context.d.ts +19 -0
- package/package.json +2 -2
- package/src/export-queue.ts +74 -0
- package/src/failure-diagnostic.ts +203 -0
- package/src/index.ts +404 -46
- package/src/trace-context.ts +78 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** One bounded lane per observer. A hung custom transport holds one slot, not an unbounded retry fanout. */
|
|
2
|
+
export declare class ExportQueue {
|
|
3
|
+
private readonly observer;
|
|
4
|
+
private readonly capacity;
|
|
5
|
+
private readonly queue;
|
|
6
|
+
private running;
|
|
7
|
+
constructor(observer: (outcome: "exported" | "retried" | "failed" | "dropped") => void, capacity?: number);
|
|
8
|
+
private observe;
|
|
9
|
+
enqueue(send: () => Promise<void>): void;
|
|
10
|
+
private start;
|
|
11
|
+
private drain;
|
|
12
|
+
flush(timeoutMs?: number): Promise<void>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
declare const CODES: readonly ["api_startup_failed", "api_unhandled_rejection", "api_uncaught_exception", "db_deadlock", "db_serialization_failure", "db_failure", "retained_process_fenced", "retained_process_proof_failed"];
|
|
2
|
+
declare const STAGES: readonly ["startup", "running", "session_events.append_generic", "session_events.append_for_turn_attempt", "preclaim", "session_attempts.claim", "failure_settlement", "sandbox_retained_processes.proof"];
|
|
3
|
+
declare const RETRIES: readonly ["not_retryable", "exhausted", "unknown"];
|
|
4
|
+
export declare const DIAGNOSTIC_SOURCE_FILES: readonly ["apps/api/src/index.ts", "apps/api/src/app.ts", "apps/api/src/fatal-process-boundary.ts", "apps/api/dist/process/index.js", "apps/worker/src/activities/agent-turn/run.ts", "apps/worker/src/activities/agent-turn/claim.ts", "apps/worker/src/activities/agent-turn/failure-settlement.ts", "apps/worker/src/activities/sandbox-lease.ts", "apps/worker/dist/process/index.js", "packages/db/src/index.ts", "packages/db/src/persistence-errors.ts", "packages/storage/src/index.ts", "packages/observability/src/index.ts"];
|
|
5
|
+
export declare const DIAGNOSTIC_POSTGRES_FUNCTIONS: readonly ["admit_session_attempt_personal_resources"];
|
|
6
|
+
export type FailureDiagnosticInput = {
|
|
7
|
+
code: (typeof CODES)[number];
|
|
8
|
+
stage: (typeof STAGES)[number];
|
|
9
|
+
retryDecision?: (typeof RETRIES)[number];
|
|
10
|
+
error: unknown;
|
|
11
|
+
attemptId?: string;
|
|
12
|
+
sessionId?: string;
|
|
13
|
+
processId?: string;
|
|
14
|
+
turnId?: string;
|
|
15
|
+
attempts?: number;
|
|
16
|
+
sqlState?: string | null;
|
|
17
|
+
constraint?: string;
|
|
18
|
+
eventTypes?: readonly string[];
|
|
19
|
+
};
|
|
20
|
+
/** No arbitrary property reads, Error.toJSON, messages, SQL, parameters or function names. */
|
|
21
|
+
export declare function failureDiagnostic(input: FailureDiagnosticInput, revision?: string): {
|
|
22
|
+
schema: string;
|
|
23
|
+
diagnosticId: `${string}-${string}-${string}-${string}-${string}`;
|
|
24
|
+
code: "api_startup_failed" | "api_uncaught_exception" | "api_unhandled_rejection" | "db_deadlock" | "db_failure" | "db_serialization_failure" | "retained_process_fenced" | "retained_process_proof_failed";
|
|
25
|
+
stage: "failure_settlement" | "preclaim" | "running" | "sandbox_retained_processes.proof" | "session_attempts.claim" | "session_events.append_for_turn_attempt" | "session_events.append_generic" | "startup";
|
|
26
|
+
retryDecision: "exhausted" | "not_retryable" | "unknown";
|
|
27
|
+
attemptId: string | undefined;
|
|
28
|
+
sessionId: string | undefined;
|
|
29
|
+
processId: string | undefined;
|
|
30
|
+
turnId: string | undefined;
|
|
31
|
+
attempts: number | undefined;
|
|
32
|
+
sqlState: string | undefined;
|
|
33
|
+
constraint: string | undefined;
|
|
34
|
+
deploymentRevision: string | undefined;
|
|
35
|
+
causes: {
|
|
36
|
+
kind: string;
|
|
37
|
+
frames: Array<{
|
|
38
|
+
locationHash: string;
|
|
39
|
+
source?: string;
|
|
40
|
+
line: number;
|
|
41
|
+
column: number;
|
|
42
|
+
}>;
|
|
43
|
+
}[];
|
|
44
|
+
postgresContext: {
|
|
45
|
+
functionName: string;
|
|
46
|
+
line: number;
|
|
47
|
+
}[];
|
|
48
|
+
eventTypes: string[];
|
|
49
|
+
};
|
|
50
|
+
export {};
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import { type TraceContext } from "./trace-context.js";
|
|
2
|
+
import { type FailureDiagnosticInput } from "./failure-diagnostic.js";
|
|
3
|
+
export type { FailureDiagnosticInput } from "./failure-diagnostic.js";
|
|
4
|
+
export { currentTraceContext, withTraceContext, parseTraceparent, traceparent, admissionTraceContext, linkCurrentSpanToAdmission, type TraceContext, } from "./trace-context.js";
|
|
1
5
|
export type AttributeValue = string | number | boolean | null | undefined;
|
|
2
6
|
export type Attributes = Record<string, AttributeValue>;
|
|
3
7
|
export type ObservabilitySettings = {
|
|
@@ -8,6 +12,9 @@ export type ObservabilitySettings = {
|
|
|
8
12
|
observabilityMetricsEnabled: boolean;
|
|
9
13
|
observabilityOtlpEndpoint?: string | undefined;
|
|
10
14
|
observabilityOtlpHeaders: string;
|
|
15
|
+
/** Explicit restricted OTLP logs destination; never falls back to the public exporter. */
|
|
16
|
+
observabilityDiagnosticsEndpoint?: string | undefined;
|
|
17
|
+
observabilityDiagnosticsHeaders?: string | undefined;
|
|
11
18
|
sandboxOwnershipEnabled?: boolean | undefined;
|
|
12
19
|
sandboxLazyProvisionEnabled?: boolean | undefined;
|
|
13
20
|
rigVerificationLeaseOwnershipEnabled?: boolean | undefined;
|
|
@@ -20,6 +27,7 @@ export type ObservabilityOptions = {
|
|
|
20
27
|
export type Span = {
|
|
21
28
|
traceId: string;
|
|
22
29
|
spanId: string;
|
|
30
|
+
addLink?: (context: TraceContext) => void;
|
|
23
31
|
end: (input?: {
|
|
24
32
|
attributes?: Attributes;
|
|
25
33
|
error?: unknown;
|
|
@@ -34,6 +42,10 @@ export type MetricLabels = Record<string, AttributeValue>;
|
|
|
34
42
|
* digest.
|
|
35
43
|
*/
|
|
36
44
|
export declare function sandboxLeaseTelemetryKey(workspaceId: string, sandboxGroupId: string): string;
|
|
45
|
+
/** Content-free lookup key for a worker execution attempt, never a metric label.
|
|
46
|
+
* Operators derive it from authorized session/attempt records to find a trace
|
|
47
|
+
* without publishing raw tenancy or conversation identifiers to telemetry. */
|
|
48
|
+
export declare function turnExecutionTelemetryKey(workspaceId: string, sessionId: string, attemptId: string): string;
|
|
37
49
|
/**
|
|
38
50
|
* Stable selectors shared by OpenGeni's runtime metrics and optional
|
|
39
51
|
* Prometheus/Grafana distribution. Operators can use these values for custom
|
|
@@ -68,26 +80,6 @@ export declare const OPENGENI_OBSERVABILITY_DISTRIBUTION: {
|
|
|
68
80
|
* credential, name, or content, may ever become a label here.
|
|
69
81
|
*/
|
|
70
82
|
export declare const TENANCY_COMPATIBILITY_LANES: readonly [
|
|
71
|
-
/**
|
|
72
|
-
* A provider connection carrying `authority_scope = 'legacy_user'` was
|
|
73
|
-
* resolved for accepted use: a personal row with no common authority or
|
|
74
|
-
* grant, admitted through `legacy_user_compatibility` provenance instead of
|
|
75
|
-
* an explicit delegation. Migration 0256 backfilled the existing rows, but
|
|
76
|
-
* its `bind_connection_authority` trigger still classifies a NEW personal
|
|
77
|
-
* connection this way whenever the inserting subject has no active
|
|
78
|
-
* organization membership, so this counter is expected to stay above zero
|
|
79
|
-
* until that classification changes. That is the finding, not a defect.
|
|
80
|
-
*/
|
|
81
|
-
"connection_legacy_user",
|
|
82
|
-
/**
|
|
83
|
-
* A workspace-scope connection ref carried no connection id, so the
|
|
84
|
-
* accepted-use authority (migration 0279) could not identify the exact row
|
|
85
|
-
* and the request fell back to the unprivileged pre-snapshot resolution.
|
|
86
|
-
* This lane writes no `connection_use_audit_facts` row and leaves no durable
|
|
87
|
-
* trace of any kind, so a use counter is the only possible evidence it was
|
|
88
|
-
* taken at all.
|
|
89
|
-
*/
|
|
90
|
-
"connection_pre_snapshot_ref",
|
|
91
83
|
/**
|
|
92
84
|
* A NEW persistable `/workspace` mutation was refused because the writer
|
|
93
85
|
* behind it has no recorded authority (`authority_unattributed`): either a
|
|
@@ -111,17 +103,21 @@ export type SandboxOperationMetricObservation = {
|
|
|
111
103
|
op: string;
|
|
112
104
|
outcome: "ok" | "not_found" | "failed";
|
|
113
105
|
durationMs: number;
|
|
106
|
+
materializationFailureReason?: string;
|
|
114
107
|
};
|
|
115
108
|
export declare function createObservability(settings: ObservabilitySettings, options: ObservabilityOptions): Observability;
|
|
116
109
|
export declare class Observability {
|
|
117
110
|
private readonly settings;
|
|
118
111
|
private readonly options;
|
|
112
|
+
private spanBatch;
|
|
113
|
+
private spanBatchScheduled;
|
|
114
|
+
private readonly diagnosticQueue;
|
|
119
115
|
private readonly registry;
|
|
120
116
|
private readonly counters;
|
|
121
117
|
private readonly gauges;
|
|
122
118
|
private readonly histograms;
|
|
123
119
|
private readonly registrations;
|
|
124
|
-
private readonly
|
|
120
|
+
private readonly exportQueue;
|
|
125
121
|
private readonly now;
|
|
126
122
|
private readonly exporter;
|
|
127
123
|
private readonly resourceAttributes;
|
|
@@ -139,7 +135,11 @@ export declare class Observability {
|
|
|
139
135
|
warn(message: string, attributes?: Attributes): void;
|
|
140
136
|
error(message: string, attributes?: Attributes): void;
|
|
141
137
|
log(level: "debug" | "info" | "warn" | "error", message: string, attributes?: Attributes): void;
|
|
142
|
-
startSpan(name: string, attributes?: Attributes
|
|
138
|
+
startSpan(name: string, attributes?: Attributes, options?: {
|
|
139
|
+
parent?: TraceContext | null;
|
|
140
|
+
links?: TraceContext[];
|
|
141
|
+
startTimeMs?: number;
|
|
142
|
+
}): Span;
|
|
143
143
|
recordHttpRequest(input: {
|
|
144
144
|
method: string;
|
|
145
145
|
route: string;
|
|
@@ -177,14 +177,19 @@ export declare class Observability {
|
|
|
177
177
|
buckets?: number[];
|
|
178
178
|
}): void;
|
|
179
179
|
prometheusMetrics(): Promise<string>;
|
|
180
|
-
/**
|
|
181
|
-
|
|
180
|
+
/** Emit a content-free anchor after successful, non-replayed admission. */
|
|
181
|
+
recordAdmissionTrace(eventId: string): void;
|
|
182
|
+
/** Emit a closed protected record independently of application persistence. */
|
|
183
|
+
recordFailureDiagnostic(input: FailureDiagnosticInput): string;
|
|
184
|
+
/** Best-effort bounded drain, including when a custom transport never settles. */
|
|
185
|
+
flush(timeoutMs?: number): Promise<void>;
|
|
182
186
|
private counter;
|
|
183
187
|
private gauge;
|
|
184
188
|
private histogram;
|
|
185
189
|
private register;
|
|
186
190
|
private assertRegistration;
|
|
187
191
|
private exportSpan;
|
|
192
|
+
private submitSpanBatch;
|
|
188
193
|
}
|
|
189
194
|
/**
|
|
190
195
|
* Convert routed sandbox-provider observations into one bounded Prometheus
|
|
@@ -198,6 +203,14 @@ export declare class Observability {
|
|
|
198
203
|
* then swallowed at this telemetry boundary.
|
|
199
204
|
*/
|
|
200
205
|
export declare function sandboxOperationMetricObserver(observability: Observability): (observation: SandboxOperationMetricObservation) => void;
|
|
206
|
+
/** Capture waits remain separate from physical provider-call accounting. */
|
|
207
|
+
export declare function sandboxCaptureWaitMetricObserver(observability: Observability): (observation: {
|
|
208
|
+
backend: string;
|
|
209
|
+
op: string;
|
|
210
|
+
outcome: "ok" | "failed";
|
|
211
|
+
durationMs: number;
|
|
212
|
+
captureWaitStage: "admission" | "provider";
|
|
213
|
+
}) => void;
|
|
201
214
|
export type InteractionOperationMetricObservation = {
|
|
202
215
|
resource: string;
|
|
203
216
|
operation: string;
|
|
@@ -236,6 +249,16 @@ export type WorkspaceInsightsMetricObservation = {
|
|
|
236
249
|
* tenant ids and caller-supplied provider/model values out of telemetry.
|
|
237
250
|
*/
|
|
238
251
|
export declare function workspaceInsightsMetricObserver(observability: Observability | null | undefined): (observation: WorkspaceInsightsMetricObservation) => void;
|
|
252
|
+
/** Dispatcher observations count attempts, including repeated observations of
|
|
253
|
+
* the same revision. Transport acceptance is never an admission receipt. */
|
|
254
|
+
export declare function recordWorkflowWakeReconciliation(observability: Observability, result: {
|
|
255
|
+
signaled: number;
|
|
256
|
+
delivered: number;
|
|
257
|
+
pendingAdmission: number;
|
|
258
|
+
unconfirmed: number;
|
|
259
|
+
failed: number;
|
|
260
|
+
pendingAdmissionBlockers: Record<string, number | undefined>;
|
|
261
|
+
}): void;
|
|
239
262
|
export type StartupDependencyRetryEvent = {
|
|
240
263
|
label: string;
|
|
241
264
|
attempt: number;
|