@senad-d/observme 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +4 -0
- package/CHANGELOG.md +14 -0
- package/README.md +15 -4
- package/dashboards/observme-agent-node-graphs.json +5 -5
- package/dashboards/observme-agents.json +169 -4
- package/dashboards/observme-alerts.yaml +16 -3
- package/dashboards/observme-overview.json +6 -6
- package/dashboards/observme-trace-journey.json +4 -4
- package/docs/agent-subagent-observability-requirements.md +15 -7
- package/docs/compatibility-matrix.md +10 -2
- package/docs/configuration.md +5 -0
- package/docs/extension-integration.md +14 -2
- package/docs/reference/04-telemetry-semantic-conventions.md +38 -1
- package/docs/reference/05-otel-pipeline-and-collector.md +23 -10
- package/docs/reference/09-dashboards-alerts-slos.md +132 -3
- package/docs/reference/10-testing-release-operations.md +44 -14
- package/docs/reference/11-deployment-runbooks.md +61 -5
- package/docs/reference/12-configuration-reference.md +18 -1
- package/docs/review-validation.md +17 -0
- package/examples/README.md +7 -2
- package/examples/collector.yaml +8 -8
- package/examples/observme.yaml +1 -0
- package/package.json +3 -1
- package/src/config/bootstrap-project-config.ts +1 -0
- package/src/config/defaults.ts +1 -0
- package/src/config/load-config.ts +11 -0
- package/src/config/schema.ts +9 -0
- package/src/config/validate.ts +20 -1
- package/src/integration.ts +46 -4
- package/src/pi/active-agent-lease.ts +101 -0
- package/src/pi/event-handlers/lifecycle.ts +120 -46
- package/src/pi/handler-runtime.ts +21 -3
- package/src/pi/handler-types.ts +10 -2
- package/src/pi/integration-api.ts +184 -13
- package/src/semconv/metrics.ts +6 -0
package/src/pi/handler-types.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Counter, Histogram, Meter, Span, Tracer, UpDownCounter } from "@opentelemetry/api";
|
|
1
|
+
import type { Counter, Histogram, Meter, ObservableGauge, Span, Tracer, UpDownCounter } from "@opentelemetry/api";
|
|
2
2
|
import type { Logger } from "@opentelemetry/api-logs";
|
|
3
3
|
import type { EnsureProjectConfig as BootstrapEnsureProjectConfig } from "../config/bootstrap-project-config.ts";
|
|
4
4
|
import type {
|
|
@@ -13,6 +13,7 @@ import type { ObservMeOtelSdkController } from "../otel/sdk.ts";
|
|
|
13
13
|
import type { ObservMeTraceSdk } from "../otel/traces.ts";
|
|
14
14
|
import type { BoundedMap } from "../util/bounded-map.ts";
|
|
15
15
|
import type { AgentLineageContext } from "./agent-lineage.ts";
|
|
16
|
+
import type { ActiveAgentLeaseController } from "./active-agent-lease.ts";
|
|
16
17
|
import type { AgentTreeTracker } from "./agent-tree-tracker.ts";
|
|
17
18
|
import type {
|
|
18
19
|
AgentWaitJoinState,
|
|
@@ -22,7 +23,10 @@ import type {
|
|
|
22
23
|
|
|
23
24
|
export type AttributePrimitive = boolean | number | string | string[];
|
|
24
25
|
export type AttributeMap = Record<string, AttributePrimitive>;
|
|
25
|
-
export type TelemetryMeter = Pick<
|
|
26
|
+
export type TelemetryMeter = Pick<
|
|
27
|
+
Meter,
|
|
28
|
+
"createCounter" | "createHistogram" | "createObservableGauge" | "createUpDownCounter"
|
|
29
|
+
>;
|
|
26
30
|
export type TelemetryTracer = Pick<Tracer, "startSpan">;
|
|
27
31
|
export type TelemetryLogger = Pick<Logger, "emit">;
|
|
28
32
|
export type Handler = (event: unknown, ctx: ObservMeHandlerContext) => Promise<void> | void;
|
|
@@ -88,6 +92,7 @@ export interface RegisterHandlersOptions {
|
|
|
88
92
|
readonly startTelemetry?: StartSessionTelemetry;
|
|
89
93
|
readonly env?: NodeJS.ProcessEnv;
|
|
90
94
|
readonly now?: () => number;
|
|
95
|
+
readonly wallClockNow?: () => number;
|
|
91
96
|
readonly configDirName?: string;
|
|
92
97
|
readonly trustedParentContext?: boolean;
|
|
93
98
|
readonly requireCompleteParentEnvelope?: boolean;
|
|
@@ -100,6 +105,7 @@ export interface StartSessionTelemetryOptions {
|
|
|
100
105
|
readonly config: ObservMeConfig;
|
|
101
106
|
readonly lineage: AgentLineageContext;
|
|
102
107
|
readonly now?: () => number;
|
|
108
|
+
readonly wallClockNow?: () => number;
|
|
103
109
|
}
|
|
104
110
|
|
|
105
111
|
export interface TurnSequenceRegistry {
|
|
@@ -119,6 +125,7 @@ export interface ObservMeTelemetrySession {
|
|
|
119
125
|
readonly logger: TelemetryLogger;
|
|
120
126
|
readonly metrics: ObservMeMetrics;
|
|
121
127
|
readonly spans: SpanRegistry;
|
|
128
|
+
readonly activeAgentLease?: ActiveAgentLeaseController;
|
|
122
129
|
agentTree: AgentTreeTracker;
|
|
123
130
|
sessionSpan?: Span;
|
|
124
131
|
sessionAttributes?: AttributeMap;
|
|
@@ -179,6 +186,7 @@ export interface ObservMeMetrics {
|
|
|
179
186
|
readonly eventsObserved: Counter;
|
|
180
187
|
readonly activeSpans: UpDownCounter;
|
|
181
188
|
readonly activeAgents: UpDownCounter;
|
|
189
|
+
readonly agentLeaseExpiresUnixTimeSeconds: ObservableGauge;
|
|
182
190
|
readonly workflowDurationMs: Histogram;
|
|
183
191
|
readonly agentRunDurationMs: Histogram;
|
|
184
192
|
readonly agentLifetimeDurationMs: Histogram;
|
|
@@ -33,12 +33,22 @@ interface IntegrationPiApi {
|
|
|
33
33
|
readonly events?: IntegrationEventBus;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
const integrationIdentifierPattern = /^[A-Za-z0-9._:-]{1,128}$/u;
|
|
37
|
+
const maximumIntegrationCommandLength = 4096;
|
|
38
|
+
const maximumIntegrationArgumentCount = 256;
|
|
39
|
+
const maximumIntegrationArgumentLength = 4096;
|
|
40
|
+
const maximumIntegrationEnvironmentEntries = 4096;
|
|
41
|
+
|
|
36
42
|
export function registerObservMeIntegration(pi: unknown, state: HandlerSessionState): (() => void) | undefined {
|
|
37
43
|
const events = resolveIntegrationEventBus(pi);
|
|
38
44
|
if (!events) return undefined;
|
|
39
45
|
|
|
40
46
|
const api = new SessionBackedObservMeIntegrationApi(state);
|
|
41
|
-
|
|
47
|
+
try {
|
|
48
|
+
return events.on(OBSERVME_INTEGRATION_CHANNEL, api.handleRequest.bind(api));
|
|
49
|
+
} catch {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
42
52
|
}
|
|
43
53
|
|
|
44
54
|
export class SessionBackedObservMeIntegrationApi implements ObservMeIntegrationApi {
|
|
@@ -50,8 +60,8 @@ export class SessionBackedObservMeIntegrationApi implements ObservMeIntegrationA
|
|
|
50
60
|
}
|
|
51
61
|
|
|
52
62
|
handleRequest(value: unknown): void {
|
|
53
|
-
if (!isCompatibleIntegrationRequest(value)) return;
|
|
54
63
|
try {
|
|
64
|
+
if (!isCompatibleIntegrationRequest(value)) return;
|
|
55
65
|
value.respond(this);
|
|
56
66
|
} catch {
|
|
57
67
|
return;
|
|
@@ -82,8 +92,12 @@ export class SessionBackedObservMeIntegrationApi implements ObservMeIntegrationA
|
|
|
82
92
|
startSubagent(options: ObservMeStartSubagentOptions = {}): ObservMeStartedSubagent | ObservMeIntegrationFailure {
|
|
83
93
|
const session = this.#state.session;
|
|
84
94
|
if (!session) return integrationFailure("session_unavailable");
|
|
85
|
-
|
|
86
95
|
try {
|
|
96
|
+
if (!isValidStartSubagentOptions(options)) return integrationFailure("invalid_request");
|
|
97
|
+
if (options.spawnId && session.spans.activeSubagentSpawns.has(options.spawnId)) {
|
|
98
|
+
return integrationFailure("spawn_already_exists");
|
|
99
|
+
}
|
|
100
|
+
|
|
87
101
|
const started = startSubagentSpawn(session, options);
|
|
88
102
|
return {
|
|
89
103
|
ok: true,
|
|
@@ -103,9 +117,12 @@ export class SessionBackedObservMeIntegrationApi implements ObservMeIntegrationA
|
|
|
103
117
|
): ObservMeIntegrationSuccess | ObservMeIntegrationFailure {
|
|
104
118
|
const session = this.#state.session;
|
|
105
119
|
if (!session) return integrationFailure("session_unavailable");
|
|
106
|
-
if (!session.spans.activeSubagentSpawns.get(spawnId)) return integrationFailure("spawn_not_found");
|
|
107
|
-
|
|
108
120
|
try {
|
|
121
|
+
if (!isValidIntegrationIdentifier(spawnId) || !isValidCompleteSubagentOptions(options)) {
|
|
122
|
+
return integrationFailure("invalid_request");
|
|
123
|
+
}
|
|
124
|
+
if (!session.spans.activeSubagentSpawns.has(spawnId)) return integrationFailure("spawn_not_found");
|
|
125
|
+
|
|
109
126
|
completeSubagentSpawn(session, spawnId, options);
|
|
110
127
|
return integrationSuccess();
|
|
111
128
|
} catch {
|
|
@@ -119,9 +136,12 @@ export class SessionBackedObservMeIntegrationApi implements ObservMeIntegrationA
|
|
|
119
136
|
): ObservMeIntegrationSuccess | ObservMeIntegrationFailure {
|
|
120
137
|
const session = this.#state.session;
|
|
121
138
|
if (!session) return integrationFailure("session_unavailable");
|
|
122
|
-
if (!session.spans.activeSubagentSpawns.get(spawnId)) return integrationFailure("spawn_not_found");
|
|
123
|
-
|
|
124
139
|
try {
|
|
140
|
+
if (!isValidIntegrationIdentifier(spawnId) || !isValidFailSubagentOptions(options)) {
|
|
141
|
+
return integrationFailure("invalid_request");
|
|
142
|
+
}
|
|
143
|
+
if (!session.spans.activeSubagentSpawns.has(spawnId)) return integrationFailure("spawn_not_found");
|
|
144
|
+
|
|
125
145
|
failSubagentSpawn(session, spawnId, options);
|
|
126
146
|
return integrationSuccess();
|
|
127
147
|
} catch {
|
|
@@ -157,8 +177,15 @@ export class SessionBackedObservMeIntegrationApi implements ObservMeIntegrationA
|
|
|
157
177
|
): ObservMeStartedWaitJoin | ObservMeIntegrationFailure {
|
|
158
178
|
const session = this.#state.session;
|
|
159
179
|
if (!session) return integrationFailure("session_unavailable");
|
|
160
|
-
|
|
161
180
|
try {
|
|
181
|
+
if (!isValidWaitJoinOptions(options)) return integrationFailure("invalid_request");
|
|
182
|
+
|
|
183
|
+
const requestedId = resolveRequestedWaitJoinId(options, kind);
|
|
184
|
+
const registry = kind === "wait" ? session.spans.activeAgentWaits : session.spans.activeAgentJoins;
|
|
185
|
+
if (requestedId && registry.has(requestedId)) {
|
|
186
|
+
return integrationFailure(kind === "wait" ? "wait_already_exists" : "join_already_exists");
|
|
187
|
+
}
|
|
188
|
+
|
|
162
189
|
const started = kind === "wait" ? startAgentWait(session, options) : startAgentJoin(session, options);
|
|
163
190
|
return { ok: true, id: started.id };
|
|
164
191
|
} catch {
|
|
@@ -173,10 +200,13 @@ export class SessionBackedObservMeIntegrationApi implements ObservMeIntegrationA
|
|
|
173
200
|
): ObservMeIntegrationSuccess | ObservMeIntegrationFailure {
|
|
174
201
|
const session = this.#state.session;
|
|
175
202
|
if (!session) return integrationFailure("session_unavailable");
|
|
176
|
-
const registry = kind === "wait" ? session.spans.activeAgentWaits : session.spans.activeAgentJoins;
|
|
177
|
-
if (!registry.get(id)) return integrationFailure(kind === "wait" ? "wait_not_found" : "join_not_found");
|
|
178
|
-
|
|
179
203
|
try {
|
|
204
|
+
if (!isValidIntegrationIdentifier(id) || !isValidWaitJoinOptions(options)) {
|
|
205
|
+
return integrationFailure("invalid_request");
|
|
206
|
+
}
|
|
207
|
+
const registry = kind === "wait" ? session.spans.activeAgentWaits : session.spans.activeAgentJoins;
|
|
208
|
+
if (!registry.has(id)) return integrationFailure(kind === "wait" ? "wait_not_found" : "join_not_found");
|
|
209
|
+
|
|
180
210
|
if (kind === "wait") endAgentWait(session, id, options);
|
|
181
211
|
else endAgentJoin(session, id, options);
|
|
182
212
|
return integrationSuccess();
|
|
@@ -188,8 +218,12 @@ export class SessionBackedObservMeIntegrationApi implements ObservMeIntegrationA
|
|
|
188
218
|
|
|
189
219
|
function resolveIntegrationEventBus(pi: unknown): IntegrationEventBus | undefined {
|
|
190
220
|
if (!pi || typeof pi !== "object") return undefined;
|
|
191
|
-
|
|
192
|
-
|
|
221
|
+
try {
|
|
222
|
+
const events = (pi as IntegrationPiApi).events;
|
|
223
|
+
return events && typeof events.on === "function" ? events : undefined;
|
|
224
|
+
} catch {
|
|
225
|
+
return undefined;
|
|
226
|
+
}
|
|
193
227
|
}
|
|
194
228
|
|
|
195
229
|
function isCompatibleIntegrationRequest(value: unknown): value is ObservMeIntegrationRequest {
|
|
@@ -202,6 +236,143 @@ function isCompatibleIntegrationRequest(value: unknown): value is ObservMeIntegr
|
|
|
202
236
|
);
|
|
203
237
|
}
|
|
204
238
|
|
|
239
|
+
function isValidStartSubagentOptions(value: unknown): value is ObservMeStartSubagentOptions {
|
|
240
|
+
if (!isIntegrationRecord(value)) return false;
|
|
241
|
+
const options = value as Partial<ObservMeStartSubagentOptions>;
|
|
242
|
+
return (
|
|
243
|
+
isOptionalIntegrationIdentifier(options.spawnId) &&
|
|
244
|
+
isOptionalIntegrationIdentifier(options.childAgentId) &&
|
|
245
|
+
isOptionalBoundedString(options.command, maximumIntegrationCommandLength) &&
|
|
246
|
+
isValidIntegrationArguments(options.args) &&
|
|
247
|
+
isOptionalSpawnType(options.spawnType) &&
|
|
248
|
+
isOptionalSpawnReason(options.spawnReason) &&
|
|
249
|
+
isOptionalIntegrationIdentifier(options.toolCallId) &&
|
|
250
|
+
isValidIntegrationEnvironment(options.env)
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function isValidCompleteSubagentOptions(value: unknown): value is ObservMeCompleteSubagentOptions {
|
|
255
|
+
if (!isIntegrationRecord(value)) return false;
|
|
256
|
+
const options = value as Partial<ObservMeCompleteSubagentOptions>;
|
|
257
|
+
return (
|
|
258
|
+
isOptionalIntegrationIdentifier(options.childAgentId) &&
|
|
259
|
+
isOptionalChildStatus(options.childStatus) &&
|
|
260
|
+
(options.outcome === undefined || options.outcome === "completed" || options.outcome === "failed" || options.outcome === "cancelled")
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function isValidFailSubagentOptions(value: unknown): value is ObservMeFailSubagentOptions {
|
|
265
|
+
if (!isIntegrationRecord(value)) return false;
|
|
266
|
+
const options = value as Partial<ObservMeFailSubagentOptions>;
|
|
267
|
+
return isOptionalIntegrationIdentifier(options.childAgentId) && isOptionalBoundedString(options.errorClass, 256);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function isValidWaitJoinOptions(value: unknown): value is ObservMeWaitJoinOptions {
|
|
271
|
+
if (!isIntegrationRecord(value)) return false;
|
|
272
|
+
const options = value as Partial<ObservMeWaitJoinOptions>;
|
|
273
|
+
return (
|
|
274
|
+
isOptionalIntegrationIdentifier(options.id) &&
|
|
275
|
+
isOptionalIntegrationIdentifier(options.spawnId) &&
|
|
276
|
+
isOptionalIntegrationIdentifier(options.childAgentId) &&
|
|
277
|
+
isOptionalChildStatus(options.childStatus) &&
|
|
278
|
+
isOptionalJoinStatus(options.joinStatus) &&
|
|
279
|
+
isOptionalWaitReason(options.reason) &&
|
|
280
|
+
(options.failurePropagated === undefined || typeof options.failurePropagated === "boolean") &&
|
|
281
|
+
(options.durationMs === undefined || isValidDuration(options.durationMs))
|
|
282
|
+
);
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function isIntegrationRecord(value: unknown): value is Record<string, unknown> {
|
|
286
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function isValidIntegrationIdentifier(value: unknown): value is string {
|
|
290
|
+
return typeof value === "string" && integrationIdentifierPattern.test(value);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function isOptionalIntegrationIdentifier(value: unknown): value is string | undefined {
|
|
294
|
+
return value === undefined || isValidIntegrationIdentifier(value);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function isOptionalBoundedString(value: unknown, maximumLength: number): value is string | undefined {
|
|
298
|
+
return value === undefined || (typeof value === "string" && value.length <= maximumLength);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function isValidIntegrationArguments(value: unknown): value is readonly string[] | undefined {
|
|
302
|
+
return (
|
|
303
|
+
value === undefined ||
|
|
304
|
+
(Array.isArray(value) &&
|
|
305
|
+
value.length <= maximumIntegrationArgumentCount &&
|
|
306
|
+
value.every(isValidIntegrationArgument))
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function isValidIntegrationArgument(value: unknown): value is string {
|
|
311
|
+
return typeof value === "string" && value.length <= maximumIntegrationArgumentLength;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function isValidIntegrationEnvironment(value: unknown): boolean {
|
|
315
|
+
if (value === undefined) return true;
|
|
316
|
+
if (!isIntegrationRecord(value)) return false;
|
|
317
|
+
const entries = Object.entries(value);
|
|
318
|
+
return entries.length <= maximumIntegrationEnvironmentEntries && entries.every(isValidIntegrationEnvironmentEntry);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function isValidIntegrationEnvironmentEntry(entry: [string, unknown]): boolean {
|
|
322
|
+
return entry[0].length > 0 && (typeof entry[1] === "string" || entry[1] === undefined);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function isOptionalSpawnType(value: unknown): boolean {
|
|
326
|
+
return value === undefined || value === "command" || value === "tool" || value === "extension" || value === "unknown";
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function isOptionalSpawnReason(value: unknown): boolean {
|
|
330
|
+
return (
|
|
331
|
+
value === undefined ||
|
|
332
|
+
value === "delegated_task" ||
|
|
333
|
+
value === "parallel_search" ||
|
|
334
|
+
value === "review" ||
|
|
335
|
+
value === "tool_wrapper" ||
|
|
336
|
+
value === "unknown"
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
function isOptionalChildStatus(value: unknown): boolean {
|
|
341
|
+
return (
|
|
342
|
+
value === undefined ||
|
|
343
|
+
value === "starting" ||
|
|
344
|
+
value === "active" ||
|
|
345
|
+
value === "completed" ||
|
|
346
|
+
value === "failed" ||
|
|
347
|
+
value === "cancelled" ||
|
|
348
|
+
value === "orphaned"
|
|
349
|
+
);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function isOptionalJoinStatus(value: unknown): boolean {
|
|
353
|
+
return (
|
|
354
|
+
value === undefined ||
|
|
355
|
+
value === "completed" ||
|
|
356
|
+
value === "failed" ||
|
|
357
|
+
value === "cancelled" ||
|
|
358
|
+
value === "timeout" ||
|
|
359
|
+
value === "unknown" ||
|
|
360
|
+
value === "waiting"
|
|
361
|
+
);
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
function isOptionalWaitReason(value: unknown): boolean {
|
|
365
|
+
return value === undefined || value === "dependency" || value === "rate_limit" || value === "child_running" || value === "unknown";
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function isValidDuration(value: unknown): value is number {
|
|
369
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= Number.MAX_SAFE_INTEGER;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
function resolveRequestedWaitJoinId(options: ObservMeWaitJoinOptions, kind: "wait" | "join"): string | undefined {
|
|
373
|
+
return options.id ?? (options.spawnId ? `${kind}-${options.spawnId}` : undefined);
|
|
374
|
+
}
|
|
375
|
+
|
|
205
376
|
function readSessionId(session: ObservMeTelemetrySession): string | undefined {
|
|
206
377
|
const value = session.sessionAttributes?.[SESSION_ATTRIBUTES.PI_SESSION_ID];
|
|
207
378
|
return typeof value === "string" ? value : undefined;
|
package/src/semconv/metrics.ts
CHANGED
|
@@ -66,6 +66,12 @@ export const OBSERVME_HISTOGRAM_METRIC_NAMES = {
|
|
|
66
66
|
export const OBSERVME_GAUGE_METRIC_NAMES = {
|
|
67
67
|
ACTIVE_SPANS: "observme_active_spans",
|
|
68
68
|
ACTIVE_AGENTS: "observme_active_agents",
|
|
69
|
+
AGENT_LEASE_EXPIRES_UNIXTIME_SECONDS: "observme_agent_lease_expires_unixtime_seconds",
|
|
70
|
+
} as const;
|
|
71
|
+
|
|
72
|
+
export const OBSERVME_AGENT_LEASE_METRIC_OPTIONS = {
|
|
73
|
+
unit: "s",
|
|
74
|
+
description: "Absolute Unix timestamp in seconds when the active Pi agent lease expires.",
|
|
69
75
|
} as const;
|
|
70
76
|
|
|
71
77
|
export const OFFICIAL_GENAI_METRIC_NAMES = {
|