@senad-d/observme 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +57 -0
- package/CHANGELOG.md +52 -0
- package/LICENSE +21 -0
- package/ObservMe-Production-Docs/00-README.md +79 -0
- package/ObservMe-Production-Docs/01-requirements-and-scope.md +207 -0
- package/ObservMe-Production-Docs/02-reference-architecture.md +306 -0
- package/ObservMe-Production-Docs/03-pi-event-and-session-model.md +266 -0
- package/ObservMe-Production-Docs/04-telemetry-semantic-conventions.md +722 -0
- package/ObservMe-Production-Docs/05-otel-pipeline-and-collector.md +355 -0
- package/ObservMe-Production-Docs/06-security-privacy-redaction.md +294 -0
- package/ObservMe-Production-Docs/07-extension-implementation-blueprint.md +447 -0
- package/ObservMe-Production-Docs/08-query-grafana-integration.md +276 -0
- package/ObservMe-Production-Docs/09-dashboards-alerts-slos.md +640 -0
- package/ObservMe-Production-Docs/10-testing-release-operations.md +319 -0
- package/ObservMe-Production-Docs/11-deployment-runbooks.md +203 -0
- package/ObservMe-Production-Docs/12-configuration-reference.md +337 -0
- package/ObservMe-Production-Docs/13-source-notes.md +33 -0
- package/ObservMe-Production-Docs/pi-session-format.md +427 -0
- package/README.md +356 -0
- package/SECURITY.md +45 -0
- package/dashboards/observme-agent-node-graphs.json +250 -0
- package/dashboards/observme-agents.json +1880 -0
- package/dashboards/observme-alerts.yaml +113 -0
- package/dashboards/observme-branches-compactions.json +1042 -0
- package/dashboards/observme-cost.json +1254 -0
- package/dashboards/observme-errors.json +1659 -0
- package/dashboards/observme-export-health.json +1802 -0
- package/dashboards/observme-latency.json +1494 -0
- package/dashboards/observme-llm-conversations.json +730 -0
- package/dashboards/observme-logs-llm.json +644 -0
- package/dashboards/observme-models.json +933 -0
- package/dashboards/observme-overview.json +2129 -0
- package/dashboards/observme-slo-health.json +737 -0
- package/dashboards/observme-slos.yaml +56 -0
- package/dashboards/observme-tools.json +902 -0
- package/dashboards/observme-trace-journey.json +1793 -0
- package/docs/STRUCTURE.md +49 -0
- package/docs/agent-subagent-observability-requirements.md +997 -0
- package/docs/compatibility-matrix.md +39 -0
- package/docs/configuration-tui-design-standard.md +767 -0
- package/docs/configuration.md +38 -0
- package/docs/review-validation.md +115 -0
- package/docs/validation-flow.md +117 -0
- package/examples/collector.yaml +123 -0
- package/examples/observme.yaml +131 -0
- package/img/demo.gif +0 -0
- package/img/icon.svg +47 -0
- package/package.json +103 -0
- package/src/commands/obs-agents-runtime.ts +150 -0
- package/src/commands/obs-agents.ts +491 -0
- package/src/commands/obs-args.ts +63 -0
- package/src/commands/obs-backfill.ts +1334 -0
- package/src/commands/obs-command-support.ts +43 -0
- package/src/commands/obs-cost.ts +228 -0
- package/src/commands/obs-diagnostics.ts +22 -0
- package/src/commands/obs-errors.ts +156 -0
- package/src/commands/obs-health.ts +301 -0
- package/src/commands/obs-link.ts +90 -0
- package/src/commands/obs-logs.ts +194 -0
- package/src/commands/obs-loki-summary.ts +184 -0
- package/src/commands/obs-session.ts +259 -0
- package/src/commands/obs-status.ts +359 -0
- package/src/commands/obs-tools.ts +274 -0
- package/src/commands/obs-trace.ts +411 -0
- package/src/commands/obs.ts +211 -0
- package/src/config/bootstrap-project-config.ts +300 -0
- package/src/config/defaults.ts +143 -0
- package/src/config/load-config.ts +631 -0
- package/src/config/project-paths.ts +61 -0
- package/src/config/schema.ts +405 -0
- package/src/config/validate.ts +456 -0
- package/src/constants.ts +4 -0
- package/src/diagnostics/sanitize.ts +6 -0
- package/src/extension.ts +38 -0
- package/src/otel/logs.ts +160 -0
- package/src/otel/metrics.ts +165 -0
- package/src/otel/otlp-endpoint.ts +10 -0
- package/src/otel/sdk.ts +114 -0
- package/src/otel/shutdown.ts +102 -0
- package/src/otel/traces.ts +166 -0
- package/src/pi/agent-lineage.ts +378 -0
- package/src/pi/agent-tree-tracker.ts +258 -0
- package/src/pi/event-handlers/agent-turn.ts +155 -0
- package/src/pi/event-handlers/lifecycle.ts +642 -0
- package/src/pi/event-handlers/llm.ts +115 -0
- package/src/pi/event-handlers/session-events.ts +159 -0
- package/src/pi/event-handlers/tool-bash.ts +275 -0
- package/src/pi/handler-internals.ts +2154 -0
- package/src/pi/handler-runtime.ts +633 -0
- package/src/pi/handler-types.ts +261 -0
- package/src/pi/handlers.ts +75 -0
- package/src/pi/subagent-spawn.ts +975 -0
- package/src/pi/subagent-types.ts +29 -0
- package/src/privacy/content-capture.ts +104 -0
- package/src/privacy/hash.ts +93 -0
- package/src/privacy/redact.ts +619 -0
- package/src/privacy/secret-patterns.ts +185 -0
- package/src/privacy/truncate.ts +69 -0
- package/src/query/grafana-readiness.ts +164 -0
- package/src/query/grafana-transport.ts +481 -0
- package/src/query/grafana.ts +371 -0
- package/src/query/loki.ts +332 -0
- package/src/query/prometheus.ts +388 -0
- package/src/query/tempo.ts +332 -0
- package/src/safety/sensitive-input.ts +208 -0
- package/src/semconv/attributes.ts +279 -0
- package/src/semconv/metrics.ts +146 -0
- package/src/semconv/spans.ts +19 -0
- package/src/semconv/values.ts +13 -0
- package/src/util/bounded-map.ts +97 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,491 @@
|
|
|
1
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import type { LoadSessionConfigOptions } from "../config/load-config.ts";
|
|
3
|
+
import type { ObservMeConfig } from "../config/schema.ts";
|
|
4
|
+
import type { AgentChildStatus, AgentTreeNode, AgentTreeSummary } from "../pi/agent-tree-tracker.ts";
|
|
5
|
+
import type { PrometheusFetch, PrometheusMetricSeries, QueryResult } from "../query/prometheus.ts";
|
|
6
|
+
import { createPrometheusQueryClient } from "../query/prometheus.ts";
|
|
7
|
+
import type { TimeRange, TraceSummary } from "../query/tempo.ts";
|
|
8
|
+
import { createTempoQueryClient } from "../query/tempo.ts";
|
|
9
|
+
import { COMMON_SPAN_ATTRIBUTES } from "../semconv/attributes.ts";
|
|
10
|
+
import { completeObsSubcommand, isExactObsSubcommandRequest } from "./obs-args.ts";
|
|
11
|
+
import { loadObsCommandConfig, notifyObsCommand } from "./obs-command-support.ts";
|
|
12
|
+
import { formatObsCommandFailure, readObsDiagnosticMessage, type ObsCommandRecoveryHint } from "./obs-diagnostics.ts";
|
|
13
|
+
import type { ObsAgentWaitJoinHint, ObsAgentsRuntimeSnapshot } from "./obs-agents-runtime.ts";
|
|
14
|
+
import { getLocalObsAgentsRuntimeSnapshot } from "./obs-agents-runtime.ts";
|
|
15
|
+
|
|
16
|
+
export interface ObsAgentsCommandContext {
|
|
17
|
+
readonly cwd?: string;
|
|
18
|
+
readonly ui: {
|
|
19
|
+
notify: (message: string, type?: "info" | "warning" | "error") => Promise<void> | void;
|
|
20
|
+
};
|
|
21
|
+
readonly isProjectTrusted?: () => boolean | Promise<boolean>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface ObsAgentChildRow {
|
|
25
|
+
readonly agentId: string;
|
|
26
|
+
readonly parentAgentId?: string;
|
|
27
|
+
readonly depth: number;
|
|
28
|
+
readonly role: string;
|
|
29
|
+
readonly capability?: string;
|
|
30
|
+
readonly status: AgentChildStatus;
|
|
31
|
+
readonly orphaned: boolean;
|
|
32
|
+
readonly activeChildren: number;
|
|
33
|
+
readonly fanoutCount: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface ObsAgentAggregateRow {
|
|
37
|
+
readonly labels: Record<string, string>;
|
|
38
|
+
readonly value: number;
|
|
39
|
+
readonly timestampUnixSeconds?: string;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface ObsAgentsAggregateRows {
|
|
43
|
+
readonly spawned: readonly ObsAgentAggregateRow[];
|
|
44
|
+
readonly fanoutP95: readonly ObsAgentAggregateRow[];
|
|
45
|
+
readonly orphaned: readonly ObsAgentAggregateRow[];
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface ObsAgentsSnapshot {
|
|
49
|
+
readonly workflowId?: string;
|
|
50
|
+
readonly workflowRootAgentId?: string;
|
|
51
|
+
readonly agentId?: string;
|
|
52
|
+
readonly parentAgentId?: string;
|
|
53
|
+
readonly rootAgentId?: string;
|
|
54
|
+
readonly role: string;
|
|
55
|
+
readonly capability?: string;
|
|
56
|
+
readonly depth: number;
|
|
57
|
+
readonly orphaned: boolean;
|
|
58
|
+
readonly sessionId?: string;
|
|
59
|
+
readonly traceId?: string;
|
|
60
|
+
readonly activeChildren: number;
|
|
61
|
+
readonly fanoutCount: number;
|
|
62
|
+
readonly treeDepth: number;
|
|
63
|
+
readonly treeWidth: number;
|
|
64
|
+
readonly orphanCount: number;
|
|
65
|
+
readonly children: readonly ObsAgentChildRow[];
|
|
66
|
+
readonly waitJoinHints: readonly ObsAgentWaitJoinHint[];
|
|
67
|
+
readonly aggregateQueries: readonly string[];
|
|
68
|
+
readonly aggregateRows: ObsAgentsAggregateRows;
|
|
69
|
+
readonly tempoSearchAttributes: Record<string, string>;
|
|
70
|
+
readonly traces: readonly TraceSummary[];
|
|
71
|
+
readonly recentChildrenLimit?: number;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export type ObsAgentsConfigLoader = (options: LoadSessionConfigOptions) => Promise<ObservMeConfig>;
|
|
75
|
+
export type ObsAgentsRuntimeProvider = (
|
|
76
|
+
ctx: ObsAgentsCommandContext,
|
|
77
|
+
) => Promise<ObsAgentsRuntimeSnapshot> | ObsAgentsRuntimeSnapshot;
|
|
78
|
+
export type ObsAgentsProvider = (ctx: ObsAgentsCommandContext) => Promise<ObsAgentsSnapshot> | ObsAgentsSnapshot;
|
|
79
|
+
|
|
80
|
+
export interface ObsAgentsSnapshotOptions {
|
|
81
|
+
readonly loadConfig?: ObsAgentsConfigLoader;
|
|
82
|
+
readonly fetch?: PrometheusFetch;
|
|
83
|
+
readonly env?: NodeJS.ProcessEnv;
|
|
84
|
+
readonly configDirName?: string;
|
|
85
|
+
readonly getRuntime?: ObsAgentsRuntimeProvider;
|
|
86
|
+
readonly searchRangeHours?: number;
|
|
87
|
+
readonly now?: () => Date;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface RegisterObsAgentsCommandOptions extends ObsAgentsSnapshotOptions {
|
|
91
|
+
readonly getAgents?: ObsAgentsProvider;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export const OBS_AGENTS_SPAWNED_PROMQL =
|
|
95
|
+
"sum(rate(observme_subagents_spawned_total[1h])) by (agent_role, subagent_depth, spawn_type, spawn_reason)";
|
|
96
|
+
export const OBS_AGENTS_FANOUT_P95_PROMQL =
|
|
97
|
+
"histogram_quantile(0.95, sum(rate(observme_agent_fanout_count_bucket[1h])) by (subagent_depth, le))";
|
|
98
|
+
export const OBS_AGENTS_ORPHAN_PROMQL = "sum(rate(observme_orphan_agents_total[1h])) by (agent_role, subagent_depth)";
|
|
99
|
+
export const OBS_AGENTS_TEMPO_DRILLDOWN_ATTRIBUTE_KEYS = [
|
|
100
|
+
COMMON_SPAN_ATTRIBUTES.PI_AGENT_ID,
|
|
101
|
+
COMMON_SPAN_ATTRIBUTES.PI_WORKFLOW_ID,
|
|
102
|
+
] as const;
|
|
103
|
+
|
|
104
|
+
const OBS_COMMAND_NAME = "obs";
|
|
105
|
+
const OBS_AGENTS_SUBCOMMAND = "agents";
|
|
106
|
+
const OBS_AGENTS_USAGE = "Usage: /obs agents";
|
|
107
|
+
const OBS_AGENTS_WINDOW = "1h";
|
|
108
|
+
const OBS_AGENTS_ERROR_NEXT_ACTION = "run /obs health and verify Grafana credentials, the Metrics datasource, and the Tempo datasource.";
|
|
109
|
+
const OBS_AGENTS_PROMETHEUS_NEXT_ACTION = "verify the Metrics datasource with /obs health, then rerun /obs agents.";
|
|
110
|
+
const OBS_AGENTS_TEMPO_NEXT_ACTION = "verify the Tempo datasource with /obs health, then rerun /obs agents.";
|
|
111
|
+
const DEFAULT_TRACE_SEARCH_RANGE_HOURS = 24;
|
|
112
|
+
const DEFAULT_RECENT_CHILDREN_RENDER_LIMIT = 10;
|
|
113
|
+
const millisecondsPerHour = 60 * 60 * 1000;
|
|
114
|
+
const emptyAgentTreeSummary = {
|
|
115
|
+
activeChildren: 0,
|
|
116
|
+
fanoutCount: 0,
|
|
117
|
+
treeDepth: 0,
|
|
118
|
+
treeWidth: 0,
|
|
119
|
+
orphanCount: 0,
|
|
120
|
+
childStatuses: {
|
|
121
|
+
starting: 0,
|
|
122
|
+
active: 0,
|
|
123
|
+
completed: 0,
|
|
124
|
+
failed: 0,
|
|
125
|
+
cancelled: 0,
|
|
126
|
+
orphaned: 0,
|
|
127
|
+
},
|
|
128
|
+
} as const satisfies AgentTreeSummary;
|
|
129
|
+
|
|
130
|
+
interface ObsAgentsQueryResults {
|
|
131
|
+
readonly spawned: QueryResult;
|
|
132
|
+
readonly fanoutP95: QueryResult;
|
|
133
|
+
readonly orphaned: QueryResult;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export function registerObsAgentsCommand(pi: ExtensionAPI, options: RegisterObsAgentsCommandOptions = {}): void {
|
|
137
|
+
const command = new ObsAgentsCommand(options);
|
|
138
|
+
|
|
139
|
+
pi.registerCommand(OBS_COMMAND_NAME, {
|
|
140
|
+
description: "Show ObservMe workflow and agent lineage. Usage: /obs agents",
|
|
141
|
+
getArgumentCompletions: getObsAgentsCommandArgumentCompletions,
|
|
142
|
+
handler: command.handle.bind(command),
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export async function handleObsAgentsCommand(
|
|
147
|
+
args: string,
|
|
148
|
+
ctx: ObsAgentsCommandContext,
|
|
149
|
+
options: RegisterObsAgentsCommandOptions = {},
|
|
150
|
+
): Promise<void> {
|
|
151
|
+
if (!isObsAgentsRequest(args)) {
|
|
152
|
+
await notifyObsCommand(ctx, OBS_AGENTS_USAGE, "warning");
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
try {
|
|
157
|
+
const snapshot = await resolveObsAgentsSnapshot(ctx, options);
|
|
158
|
+
await notifyObsCommand(ctx, renderObsAgents(snapshot), "info");
|
|
159
|
+
} catch (error) {
|
|
160
|
+
await notifyObsCommand(
|
|
161
|
+
ctx,
|
|
162
|
+
formatObsCommandFailure("ObservMe agents unavailable", error, resolveObsAgentsDiagnostic(error)),
|
|
163
|
+
"error",
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export function getObsAgentsCommandArgumentCompletions(prefix: string): Array<{ value: string; label: string }> | null {
|
|
169
|
+
return completeObsSubcommand(prefix, OBS_AGENTS_SUBCOMMAND);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export async function getObsAgentsSnapshot(
|
|
173
|
+
ctx: ObsAgentsCommandContext,
|
|
174
|
+
options: ObsAgentsSnapshotOptions = {},
|
|
175
|
+
): Promise<ObsAgentsSnapshot> {
|
|
176
|
+
const runtime = await resolveObsAgentsRuntime(ctx, options);
|
|
177
|
+
const config = await loadObsAgentsConfig(ctx, options);
|
|
178
|
+
const [aggregateResults, traces] = await Promise.all([
|
|
179
|
+
queryObsAgentsAggregates(config, options),
|
|
180
|
+
queryObsAgentsTempoTraces(config, runtime, options),
|
|
181
|
+
]);
|
|
182
|
+
|
|
183
|
+
return buildObsAgentsSnapshot(runtime, aggregateResults, traces, config.query.maxAgents);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
export function renderObsAgents(snapshot: ObsAgentsSnapshot): string {
|
|
187
|
+
const latestChild = readLatestChild(snapshot.children);
|
|
188
|
+
const lines = [
|
|
189
|
+
`Workflow: ${formatUnknown(snapshot.workflowId)} root=${formatUnknown(snapshot.workflowRootAgentId ?? snapshot.rootAgentId)}`,
|
|
190
|
+
`Agent: ${formatUnknown(snapshot.agentId)} (${snapshot.role} depth=${snapshot.depth})`,
|
|
191
|
+
`Session: ${formatUnknown(normalizeOptionalString(snapshot.sessionId))}`,
|
|
192
|
+
`Subagents spawned in current trace: ${snapshot.fanoutCount}`,
|
|
193
|
+
`Current tree: depth=${snapshot.treeDepth} width=${snapshot.treeWidth} active=${snapshot.activeChildren} orphaned=${snapshot.orphanCount}`,
|
|
194
|
+
`Recent children: ${renderRecentChildren(snapshot.children, snapshot.recentChildrenLimit)}`,
|
|
195
|
+
];
|
|
196
|
+
|
|
197
|
+
if (latestChild) lines.push(`Latest child: ${renderLatestChild(latestChild, snapshot.waitJoinHints)}`);
|
|
198
|
+
lines.push(
|
|
199
|
+
`Wait/join hints: ${renderWaitJoinHints(snapshot.waitJoinHints)}`,
|
|
200
|
+
`Aggregate agent metrics (last ${OBS_AGENTS_WINDOW}): ${renderAggregateRows(snapshot.aggregateRows)}`,
|
|
201
|
+
`Lineage drill-down: ${renderLineageDrilldown(snapshot)}`,
|
|
202
|
+
);
|
|
203
|
+
return lines.join("\n");
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
class ObsAgentsCommand {
|
|
207
|
+
readonly #options: RegisterObsAgentsCommandOptions;
|
|
208
|
+
|
|
209
|
+
constructor(options: RegisterObsAgentsCommandOptions) {
|
|
210
|
+
this.#options = options;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
async handle(args: string, ctx: ObsAgentsCommandContext): Promise<void> {
|
|
214
|
+
await handleObsAgentsCommand(args, ctx, this.#options);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async function resolveObsAgentsSnapshot(
|
|
219
|
+
ctx: ObsAgentsCommandContext,
|
|
220
|
+
options: RegisterObsAgentsCommandOptions,
|
|
221
|
+
): Promise<ObsAgentsSnapshot> {
|
|
222
|
+
if (options.getAgents) return options.getAgents(ctx);
|
|
223
|
+
return getObsAgentsSnapshot(ctx, options);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
async function resolveObsAgentsRuntime(
|
|
227
|
+
ctx: ObsAgentsCommandContext,
|
|
228
|
+
options: ObsAgentsSnapshotOptions,
|
|
229
|
+
): Promise<ObsAgentsRuntimeSnapshot> {
|
|
230
|
+
if (options.getRuntime) return options.getRuntime(ctx);
|
|
231
|
+
return getLocalObsAgentsRuntimeSnapshot();
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function loadObsAgentsConfig(
|
|
235
|
+
ctx: ObsAgentsCommandContext,
|
|
236
|
+
options: ObsAgentsSnapshotOptions,
|
|
237
|
+
): Promise<ObservMeConfig> {
|
|
238
|
+
return loadObsCommandConfig(ctx, options);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
async function queryObsAgentsAggregates(
|
|
242
|
+
config: ObservMeConfig,
|
|
243
|
+
options: ObsAgentsSnapshotOptions,
|
|
244
|
+
): Promise<ObsAgentsQueryResults> {
|
|
245
|
+
const client = createPrometheusQueryClient(config, { fetch: options.fetch });
|
|
246
|
+
const spawned = await client.queryPrometheus(OBS_AGENTS_SPAWNED_PROMQL, undefined, { resultLimit: "agents" });
|
|
247
|
+
const fanoutP95 = await client.queryPrometheus(OBS_AGENTS_FANOUT_P95_PROMQL, undefined, { resultLimit: "agents" });
|
|
248
|
+
const orphaned = await client.queryPrometheus(OBS_AGENTS_ORPHAN_PROMQL, undefined, { resultLimit: "agents" });
|
|
249
|
+
|
|
250
|
+
return { spawned, fanoutP95, orphaned };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async function queryObsAgentsTempoTraces(
|
|
254
|
+
config: ObservMeConfig,
|
|
255
|
+
runtime: ObsAgentsRuntimeSnapshot,
|
|
256
|
+
options: ObsAgentsSnapshotOptions,
|
|
257
|
+
): Promise<TraceSummary[]> {
|
|
258
|
+
const attrs = createTempoSearchAttributes(runtime);
|
|
259
|
+
if (Object.keys(attrs).length === 0) return [];
|
|
260
|
+
|
|
261
|
+
const client = createTempoQueryClient(config, { fetch: options.fetch });
|
|
262
|
+
return client.searchTempo(attrs, createObsAgentsSearchRange(options));
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function buildObsAgentsSnapshot(
|
|
266
|
+
runtime: ObsAgentsRuntimeSnapshot,
|
|
267
|
+
aggregateResults: ObsAgentsQueryResults,
|
|
268
|
+
traces: readonly TraceSummary[],
|
|
269
|
+
maxRecentChildren: number | undefined,
|
|
270
|
+
): ObsAgentsSnapshot {
|
|
271
|
+
const lineage = runtime.lineage;
|
|
272
|
+
const currentAgent = runtime.currentAgent;
|
|
273
|
+
const summary = runtime.summary ?? emptyAgentTreeSummary;
|
|
274
|
+
|
|
275
|
+
return {
|
|
276
|
+
workflowId: lineage?.workflowId ?? currentAgent?.workflowId,
|
|
277
|
+
workflowRootAgentId: lineage?.workflowRootAgentId ?? currentAgent?.rootAgentId,
|
|
278
|
+
agentId: lineage?.agentId ?? currentAgent?.agentId,
|
|
279
|
+
parentAgentId: lineage?.parentAgentId ?? currentAgent?.parentAgentId,
|
|
280
|
+
rootAgentId: lineage?.rootAgentId ?? currentAgent?.rootAgentId,
|
|
281
|
+
role: lineage?.role ?? currentAgent?.role ?? "unknown",
|
|
282
|
+
capability: lineage?.capability ?? currentAgent?.capability,
|
|
283
|
+
depth: normalizeCount(lineage?.depth ?? currentAgent?.depth),
|
|
284
|
+
orphaned: Boolean(lineage?.orphaned ?? currentAgent?.orphaned),
|
|
285
|
+
sessionId: normalizeOptionalString(runtime.sessionId),
|
|
286
|
+
traceId: normalizeOptionalString(runtime.traceId),
|
|
287
|
+
activeChildren: normalizeCount(currentAgent?.activeChildren ?? summary.activeChildren),
|
|
288
|
+
fanoutCount: normalizeCount(currentAgent?.fanoutCount ?? summary.fanoutCount),
|
|
289
|
+
treeDepth: normalizeCount(summary.treeDepth),
|
|
290
|
+
treeWidth: normalizeCount(summary.treeWidth),
|
|
291
|
+
orphanCount: normalizeCount(summary.orphanCount),
|
|
292
|
+
children: runtime.children.map(toObsAgentChildRow),
|
|
293
|
+
waitJoinHints: runtime.waitJoinHints,
|
|
294
|
+
aggregateQueries: [OBS_AGENTS_SPAWNED_PROMQL, OBS_AGENTS_FANOUT_P95_PROMQL, OBS_AGENTS_ORPHAN_PROMQL],
|
|
295
|
+
aggregateRows: {
|
|
296
|
+
spawned: aggregateResults.spawned.series.map(toObsAgentAggregateRow).filter(isObsAgentAggregateRow),
|
|
297
|
+
fanoutP95: aggregateResults.fanoutP95.series.map(toObsAgentAggregateRow).filter(isObsAgentAggregateRow),
|
|
298
|
+
orphaned: aggregateResults.orphaned.series.map(toObsAgentAggregateRow).filter(isObsAgentAggregateRow),
|
|
299
|
+
},
|
|
300
|
+
tempoSearchAttributes: createTempoSearchAttributes(runtime),
|
|
301
|
+
traces,
|
|
302
|
+
recentChildrenLimit: normalizeRecentChildrenLimit(maxRecentChildren),
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function toObsAgentChildRow(node: AgentTreeNode): ObsAgentChildRow {
|
|
307
|
+
return {
|
|
308
|
+
agentId: node.agentId,
|
|
309
|
+
parentAgentId: node.parentAgentId,
|
|
310
|
+
depth: normalizeCount(node.depth),
|
|
311
|
+
role: node.role,
|
|
312
|
+
capability: node.capability,
|
|
313
|
+
status: node.status,
|
|
314
|
+
orphaned: node.orphaned,
|
|
315
|
+
activeChildren: normalizeCount(node.activeChildren),
|
|
316
|
+
fanoutCount: normalizeCount(node.fanoutCount),
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function toObsAgentAggregateRow(series: PrometheusMetricSeries): ObsAgentAggregateRow | undefined {
|
|
321
|
+
const value = parseMetricValue(series.value?.value);
|
|
322
|
+
if (value === undefined) return undefined;
|
|
323
|
+
|
|
324
|
+
return {
|
|
325
|
+
labels: { ...series.metric },
|
|
326
|
+
value,
|
|
327
|
+
timestampUnixSeconds: series.value?.timestampUnixSeconds,
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function createTempoSearchAttributes(runtime: ObsAgentsRuntimeSnapshot): Record<string, string> {
|
|
332
|
+
const lineage = runtime.lineage;
|
|
333
|
+
if (!lineage) return {};
|
|
334
|
+
|
|
335
|
+
return {
|
|
336
|
+
[COMMON_SPAN_ATTRIBUTES.PI_AGENT_ID]: lineage.agentId,
|
|
337
|
+
[COMMON_SPAN_ATTRIBUTES.PI_WORKFLOW_ID]: lineage.workflowId,
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
function createObsAgentsSearchRange(options: ObsAgentsSnapshotOptions): TimeRange {
|
|
342
|
+
const to = options.now?.() ?? new Date();
|
|
343
|
+
const rangeHours = normalizeSearchRangeHours(options.searchRangeHours);
|
|
344
|
+
return { from: new Date(to.getTime() - rangeHours * millisecondsPerHour), to };
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function normalizeSearchRangeHours(value: number | undefined): number {
|
|
348
|
+
if (value === undefined || !Number.isFinite(value) || value <= 0) return DEFAULT_TRACE_SEARCH_RANGE_HOURS;
|
|
349
|
+
return value;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function readLatestChild(children: readonly ObsAgentChildRow[]): ObsAgentChildRow | undefined {
|
|
353
|
+
return children.at(-1);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function renderRecentChildren(children: readonly ObsAgentChildRow[], limit: number | undefined): string {
|
|
357
|
+
if (children.length === 0) return "none";
|
|
358
|
+
|
|
359
|
+
const selection = selectRecentChildrenForRender(children, limit);
|
|
360
|
+
const rendered = selection.children.map(renderRecentChild).join("; ");
|
|
361
|
+
if (selection.omittedCount === 0) return rendered;
|
|
362
|
+
return `${rendered}; omitted ${selection.omittedCount} child row(s)`;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
function selectRecentChildrenForRender(
|
|
366
|
+
children: readonly ObsAgentChildRow[],
|
|
367
|
+
limit: number | undefined,
|
|
368
|
+
): { readonly children: readonly ObsAgentChildRow[]; readonly omittedCount: number } {
|
|
369
|
+
const normalizedLimit = normalizeRecentChildrenLimit(limit);
|
|
370
|
+
if (children.length <= normalizedLimit) return { children, omittedCount: 0 };
|
|
371
|
+
|
|
372
|
+
if (normalizedLimit === 1) return { children: children.slice(-1), omittedCount: children.length - 1 };
|
|
373
|
+
|
|
374
|
+
const firstCount = Math.floor(normalizedLimit / 2);
|
|
375
|
+
const lastCount = normalizedLimit - firstCount;
|
|
376
|
+
const visibleChildren = [...children.slice(0, firstCount), ...children.slice(-lastCount)];
|
|
377
|
+
return { children: visibleChildren, omittedCount: children.length - visibleChildren.length };
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
function renderRecentChild(child: ObsAgentChildRow): string {
|
|
381
|
+
const orphan = child.orphaned ? " orphaned" : "";
|
|
382
|
+
return `${child.agentId} status=${child.status} depth=${child.depth}${orphan}`;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
function renderLatestChild(child: ObsAgentChildRow, hints: readonly ObsAgentWaitJoinHint[]): string {
|
|
386
|
+
const joinHint = readLatestHintForChild(child.agentId, hints, "join");
|
|
387
|
+
return `${child.agentId} status=${child.status} active=${child.activeChildren} join=${formatDuration(joinHint?.durationMs)}`;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function readLatestHintForChild(
|
|
391
|
+
childAgentId: string,
|
|
392
|
+
hints: readonly ObsAgentWaitJoinHint[],
|
|
393
|
+
kind: "join" | "wait",
|
|
394
|
+
): ObsAgentWaitJoinHint | undefined {
|
|
395
|
+
return hints.findLast(hint => hint.kind === kind && hint.childAgentId === childAgentId);
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
function renderWaitJoinHints(hints: readonly ObsAgentWaitJoinHint[]): string {
|
|
399
|
+
if (hints.length === 0) return "none";
|
|
400
|
+
|
|
401
|
+
const activeWaits = hints.filter(isActiveWaitHint).length;
|
|
402
|
+
const activeJoins = hints.filter(isActiveJoinHint).length;
|
|
403
|
+
const latest = hints.at(-1);
|
|
404
|
+
return `active_waits=${activeWaits} active_joins=${activeJoins} latest=${renderWaitJoinHint(latest)}`;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
function renderWaitJoinHint(hint: ObsAgentWaitJoinHint | undefined): string {
|
|
408
|
+
if (!hint) return "none";
|
|
409
|
+
|
|
410
|
+
const status = hint.joinStatus ?? hint.childStatus ?? (hint.active ? "waiting" : "complete");
|
|
411
|
+
return `${hint.kind}:${hint.childAgentId ?? hint.spawnId ?? hint.id} status=${status} duration=${formatDuration(hint.durationMs)}`;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
function renderAggregateRows(rows: ObsAgentsAggregateRows): string {
|
|
415
|
+
return `spawn_series=${rows.spawned.length} fanout_series=${rows.fanoutP95.length} orphan_series=${rows.orphaned.length}`;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function renderLineageDrilldown(snapshot: ObsAgentsSnapshot): string {
|
|
419
|
+
const attrs = Object.keys(snapshot.tempoSearchAttributes).join(", ") || "none";
|
|
420
|
+
const traceCount = snapshot.traces.length;
|
|
421
|
+
const latestTrace = normalizeOptionalString(snapshot.traces[0]?.traceId) ?? normalizeOptionalString(snapshot.traceId);
|
|
422
|
+
const traceSuffix = latestTrace ? ` latest_trace=${latestTrace}` : "";
|
|
423
|
+
return `Tempo attributes ${attrs} traces=${traceCount}${traceSuffix}`;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function isActiveWaitHint(hint: ObsAgentWaitJoinHint): boolean {
|
|
427
|
+
return hint.active && hint.kind === "wait";
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
function isActiveJoinHint(hint: ObsAgentWaitJoinHint): boolean {
|
|
431
|
+
return hint.active && hint.kind === "join";
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function isObsAgentAggregateRow(row: ObsAgentAggregateRow | undefined): row is ObsAgentAggregateRow {
|
|
435
|
+
return row !== undefined;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function parseMetricValue(value: string | number | undefined): number | undefined {
|
|
439
|
+
if (value === undefined) return undefined;
|
|
440
|
+
|
|
441
|
+
const metricValue = Number(value);
|
|
442
|
+
if (!Number.isFinite(metricValue) || metricValue < 0) return undefined;
|
|
443
|
+
return metricValue;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function normalizeCount(value: number | undefined): number {
|
|
447
|
+
if (value === undefined || !Number.isFinite(value) || value < 0) return 0;
|
|
448
|
+
return Math.trunc(value);
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
function normalizeRecentChildrenLimit(value: number | undefined): number {
|
|
452
|
+
if (value === undefined || !Number.isFinite(value) || value < 1) return DEFAULT_RECENT_CHILDREN_RENDER_LIMIT;
|
|
453
|
+
return Math.min(Math.trunc(value), DEFAULT_RECENT_CHILDREN_RENDER_LIMIT);
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
function normalizeOptionalString(value: string | undefined): string | undefined {
|
|
457
|
+
const trimmed = value?.trim();
|
|
458
|
+
return trimmed || undefined;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
function isObsAgentsRequest(args: string): boolean {
|
|
462
|
+
return isExactObsSubcommandRequest(args, OBS_AGENTS_SUBCOMMAND);
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
function formatUnknown(value: string | undefined): string {
|
|
466
|
+
return value ?? "unknown";
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function formatDuration(value: number | undefined): string {
|
|
470
|
+
if (value === undefined) return "n/a";
|
|
471
|
+
if (value < 1000) return `${value}ms`;
|
|
472
|
+
return `${trimTrailingFractionZeros((value / 1000).toFixed(2))}s`;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
function trimTrailingFractionZeros(value: string): string {
|
|
476
|
+
const decimalIndex = value.indexOf(".");
|
|
477
|
+
if (decimalIndex === -1) return value;
|
|
478
|
+
|
|
479
|
+
let end = value.length;
|
|
480
|
+
while (end > decimalIndex + 1 && value[end - 1] === "0") end -= 1;
|
|
481
|
+
if (end === decimalIndex + 1) return value.slice(0, decimalIndex);
|
|
482
|
+
return value.slice(0, end);
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
function resolveObsAgentsDiagnostic(error: unknown): ObsCommandRecoveryHint {
|
|
486
|
+
const message = readObsDiagnosticMessage(error);
|
|
487
|
+
|
|
488
|
+
if (message.includes("Prometheus")) return { subsystem: "Prometheus", nextAction: OBS_AGENTS_PROMETHEUS_NEXT_ACTION };
|
|
489
|
+
if (message.includes("Tempo")) return { subsystem: "Tempo", nextAction: OBS_AGENTS_TEMPO_NEXT_ACTION };
|
|
490
|
+
return { subsystem: "Agent telemetry", nextAction: OBS_AGENTS_ERROR_NEXT_ACTION };
|
|
491
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
export interface ObsCommandCompletion {
|
|
2
|
+
readonly value: string;
|
|
3
|
+
readonly label: string;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export interface ObsSubcommandArgsParseResult {
|
|
7
|
+
readonly matched: boolean;
|
|
8
|
+
readonly values: readonly string[];
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function tokenizeObsCommandArgs(args: string): string[] {
|
|
12
|
+
// /obs uses deterministic whitespace tokenization only; quoted-like text is not shell-parsed.
|
|
13
|
+
return args.trim().split(/\s+/u).filter(isNonEmptyString);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function firstObsCommandToken(args: string): string | undefined {
|
|
17
|
+
return tokenizeObsCommandArgs(args)[0]?.toLowerCase();
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function parseObsSubcommandArgs(args: string, subcommand: string): ObsSubcommandArgsParseResult {
|
|
21
|
+
const [rawSubcommand, ...values] = tokenizeObsCommandArgs(args);
|
|
22
|
+
return { matched: rawSubcommand?.toLowerCase() === subcommand, values };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function isExactObsSubcommandRequest(args: string, subcommand: string, options: { allowEmpty?: boolean } = {}): boolean {
|
|
26
|
+
const tokens = tokenizeObsCommandArgs(args);
|
|
27
|
+
if (tokens.length === 0) return options.allowEmpty === true;
|
|
28
|
+
const [rawSubcommand, ...rest] = tokens;
|
|
29
|
+
return rawSubcommand.toLowerCase() === subcommand && rest.length === 0;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function completeObsSubcommands(prefix: string, subcommands: readonly string[]): ObsCommandCompletion[] | null {
|
|
33
|
+
const normalizedPrefix = prefix.trim().toLowerCase();
|
|
34
|
+
const completions = subcommands
|
|
35
|
+
.filter(subcommand => subcommand.startsWith(normalizedPrefix))
|
|
36
|
+
.map(subcommand => ({ value: subcommand, label: subcommand }));
|
|
37
|
+
|
|
38
|
+
return completions.length > 0 ? completions : null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function completeObsSubcommand(prefix: string, subcommand: string): ObsCommandCompletion[] | null {
|
|
42
|
+
return completeObsSubcommands(prefix, [subcommand]);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function obsUsageWithError(usage: string, error: string | undefined): string {
|
|
46
|
+
return error ? `${usage}\n${error}` : usage;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function unknownObsArgumentMessage(token: string): string {
|
|
50
|
+
return `Unknown argument: ${token}.`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function unknownObsOptionMessage(token: string): string {
|
|
54
|
+
return `Unknown option: ${token}.`;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function missingObsOptionValueMessage(option: string): string {
|
|
58
|
+
return `Missing value for ${option}.`;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function isNonEmptyString(value: string): boolean {
|
|
62
|
+
return value.length > 0;
|
|
63
|
+
}
|