@obsunified/collector 2.0.1

package/src/plugins/action-routes.ts

@@ -0,0 +1,343 @@
+import {
+	DEFAULT_WINDOW_HOURS,
+	getConfiguredRetentionHours,
+} from "@obsunified/types/constants";
+import type { CollectorPlugin } from "../framework/collector";
+import {
+	getAutonomousReviewAggregates,
+	getCostAttributionAggregates,
+	getToolReliabilityAggregates,
+	getVersionDiffAggregates,
+} from "../lib/action-aggregates";
+import type {
+	ActionRef,
+	EntityManifestExtended,
+	EvalResultRef,
+	ToolCallRef,
+} from "../lib/identity-index";
+import { IdentityIndex } from "../lib/identity-index";
+import { sqlDbFor } from "../lib/sql-db";
+import { getProjectId } from "./_context";
+
+const parsePositiveInt = (
+	raw: string | undefined,
+	fallback: number,
+	min: number,
+	max: number,
+): number => {
+	const parsed = Number.parseInt(raw ?? "", 10);
+	if (!Number.isFinite(parsed)) return fallback;
+	return Math.min(max, Math.max(min, parsed));
+};
+
+const parseWindowHours = (
+	raw: string | undefined,
+	retentionHours: string | undefined,
+): number => {
+	const maxHours = getConfiguredRetentionHours(retentionHours);
+	return parsePositiveInt(raw, DEFAULT_WINDOW_HOURS, 1, maxHours);
+};
+
+interface CompareStep {
+	key: string;
+	index: number;
+	actionId: string;
+	parentActionId: string | null;
+	actionKind: string;
+	name: string | null;
+	status: string;
+	durationMs: number | null;
+	totalCostUsd: number | null;
+	toolName: string | null;
+	toolCallId: string | null;
+	evalPassed: boolean | null;
+	evalScore: number | null;
+	traceId: string | null;
+	spanId: string | null;
+	causalConfidence?: string;
+}
+
+const sortActionSteps = (actions: ActionRef[]): ActionRef[] =>
+	[...actions].sort((a, b) => {
+		const time = a.startedAt.localeCompare(b.startedAt);
+		if (time !== 0) return time;
+		return a.id.localeCompare(b.id);
+	});
+
+const stepBaseKey = (
+	action: ActionRef,
+	tool: ToolCallRef | undefined,
+): string => {
+	const name = tool?.toolName ?? action.stepId ?? action.name ?? "unnamed";
+	return `${action.actionKind}:${name}`.toLowerCase();
+};
+
+const buildCompareSteps = (manifest: EntityManifestExtended): CompareStep[] => {
+	const toolsByAction = new Map<string, ToolCallRef[]>();
+	for (const tool of manifest.toolCalls) {
+		const tools = toolsByAction.get(tool.actionId) ?? [];
+		tools.push(tool);
+		toolsByAction.set(tool.actionId, tools);
+	}
+
+	const evalsByAction = new Map<string, EvalResultRef[]>();
+	for (const evalResult of manifest.evalResults) {
+		const evals = evalsByAction.get(evalResult.actionId) ?? [];
+		evals.push(evalResult);
+		evalsByAction.set(evalResult.actionId, evals);
+	}
+
+	const seen = new Map<string, number>();
+	return sortActionSteps(manifest.actions).map((action, index) => {
+		const tool = toolsByAction.get(action.id)?.[0];
+		const evalResult = evalsByAction.get(action.id)?.[0];
+		const baseKey = stepBaseKey(action, tool);
+		const count = seen.get(baseKey) ?? 0;
+		seen.set(baseKey, count + 1);
+		return {
+			key: `${baseKey}#${count}`,
+			index,
+			actionId: action.id,
+			parentActionId: action.causedByActionId,
+			actionKind: action.actionKind,
+			name: action.name,
+			status: action.status,
+			durationMs: action.durationMs,
+			totalCostUsd: action.totalCostUsd,
+			toolName: tool?.toolName ?? null,
+			toolCallId: tool?.id ?? action.toolCallId,
+			evalPassed: evalResult ? evalResult.passed !== 0 : null,
+			evalScore: evalResult?.score ?? null,
+			traceId: action.traceId,
+			spanId: action.spanId,
+			causalConfidence: action.causalConfidence,
+		};
+	});
+};
+
+const changedFields = (left: CompareStep, right: CompareStep): string[] => {
+	const fields: (keyof CompareStep)[] = [
+		"parentActionId",
+		"actionKind",
+		"name",
+		"status",
+		"durationMs",
+		"totalCostUsd",
+		"toolName",
+		"evalPassed",
+		"evalScore",
+		"traceId",
+		"causalConfidence",
+	];
+	return fields.filter((field) => left[field] !== right[field]);
+};
+
+const compareStepSequences = (
+	leftSteps: CompareStep[],
+	rightSteps: CompareStep[],
+) => {
+	const leftByKey = new Map(leftSteps.map((step) => [step.key, step]));
+	const rightByKey = new Map(rightSteps.map((step) => [step.key, step]));
+	const keys = Array.from(new Set([...leftByKey.keys(), ...rightByKey.keys()]));
+	return keys.map((key) => {
+		const left = leftByKey.get(key) ?? null;
+		const right = rightByKey.get(key) ?? null;
+		const fields = left && right ? changedFields(left, right) : [];
+		return {
+			key,
+			changeType:
+				left && right
+					? fields.length > 0
+						? "changed"
+						: "same"
+					: left
+						? "removed"
+						: "added",
+			changedFields: fields,
+			left,
+			right,
+		};
+	});
+};
+
+const manifestForComparableId = async (
+	index: IdentityIndex,
+	projectId: string,
+	id: string,
+): Promise<EntityManifestExtended> => {
+	const byAction = await index.byAction(projectId, id);
+	if (byAction.actions.some((action) => action.id === id)) return byAction;
+	return index.byAgentRun(projectId, id);
+};
+
+export const actionRoutesPlugin: CollectorPlugin = {
+	name: "action-routes",
+	register(app) {
+		app.get("/internal/actions/aggregates/tool-reliability", async (c) => {
+			const projectId = getProjectId(c);
+			const hours = parseWindowHours(
+				c.req.query("hours"),
+				c.env.RETENTION_HOURS,
+			);
+			const limit = parsePositiveInt(c.req.query("limit"), 20, 1, 100);
+			const result = await getToolReliabilityAggregates(
+				sqlDbFor(c.env),
+				projectId,
+				hours,
+				limit,
+			);
+			return c.json(result);
+		});
+
+		app.get("/internal/actions/aggregates/cost-attribution", async (c) => {
+			const projectId = getProjectId(c);
+			const hours = parseWindowHours(
+				c.req.query("hours"),
+				c.env.RETENTION_HOURS,
+			);
+			const limit = parsePositiveInt(c.req.query("limit"), 20, 1, 100);
+			const result = await getCostAttributionAggregates(
+				sqlDbFor(c.env),
+				projectId,
+				hours,
+				limit,
+			);
+			return c.json(result);
+		});
+
+		app.get("/internal/actions/aggregates/autonomous-review", async (c) => {
+			const projectId = getProjectId(c);
+			const hours = parseWindowHours(
+				c.req.query("hours"),
+				c.env.RETENTION_HOURS,
+			);
+			const limit = parsePositiveInt(c.req.query("limit"), 50, 1, 200);
+			const result = await getAutonomousReviewAggregates(
+				sqlDbFor(c.env),
+				projectId,
+				hours,
+				limit,
+			);
+			return c.json(result);
+		});
+
+		app.get("/internal/actions/aggregates/version-diff", async (c) => {
+			const projectId = getProjectId(c);
+			const result = await getVersionDiffAggregates(
+				sqlDbFor(c.env),
+				projectId,
+				c.req.query("baseline"),
+				c.req.query("target"),
+			);
+			return c.json(result);
+		});
+
+		app.get("/internal/actions/compare", async (c) => {
+			const projectId = getProjectId(c);
+			const leftId = c.req.query("left");
+			const rightId = c.req.query("right");
+			if (!leftId || !rightId) {
+				return c.json(
+					{ error: "left and right query parameters required" },
+					400,
+				);
+			}
+
+			const index = new IdentityIndex(sqlDbFor(c.env));
+			const [leftManifest, rightManifest] = await Promise.all([
+				manifestForComparableId(index, projectId, leftId),
+				manifestForComparableId(index, projectId, rightId),
+			]);
+			const leftSteps = buildCompareSteps(leftManifest);
+			const rightSteps = buildCompareSteps(rightManifest);
+
+			if (leftSteps.length === 0 || rightSteps.length === 0) {
+				return c.json(
+					{
+						error: "Not Found",
+						message: "Both comparable action graphs must exist",
+					},
+					404,
+				);
+			}
+
+			return c.json({
+				projectId,
+				leftId,
+				rightId,
+				leftManifest,
+				rightManifest,
+				leftSteps,
+				rightSteps,
+				stepComparisons: compareStepSequences(leftSteps, rightSteps),
+				generatedAt: new Date().toISOString(),
+			});
+		});
+
+		app.get("/internal/actions/:id", async (c) => {
+			const projectId = getProjectId(c);
+			const id = c.req.param("id");
+			if (!id) return c.json({ error: "id required" }, 400);
+
+			const index = new IdentityIndex(sqlDbFor(c.env));
+			const manifest = await index.byAction(projectId, id);
+			const action = manifest.actions.find((a) => a.id === id);
+			if (!action) {
+				return c.json({ error: "Not Found", message: "Action not found" }, 404);
+			}
+
+			return c.json({ action, manifest });
+		});
+
+		app.get("/internal/agent-runs/:id", async (c) => {
+			const projectId = getProjectId(c);
+			const id = c.req.param("id");
+			if (!id) return c.json({ error: "id required" }, 400);
+
+			const index = new IdentityIndex(sqlDbFor(c.env));
+			const manifest = await index.byAgentRun(projectId, id);
+			const agentRun = manifest.agentRuns.find((r) => r.id === id);
+			if (!agentRun) {
+				return c.json(
+					{ error: "Not Found", message: "Agent run not found" },
+					404,
+				);
+			}
+
+			return c.json({ agentRun, manifest });
+		});
+
+		app.get("/internal/tool-calls/:id", async (c) => {
+			const projectId = getProjectId(c);
+			const id = c.req.param("id");
+			if (!id) return c.json({ error: "id required" }, 400);
+
+			const db = sqlDbFor(c.env);
+			const toolCallRow = await db
+				.prepare(
+					`SELECT action_id FROM tool_calls
+						WHERE project_id = ? AND id = ? LIMIT 1`,
+				)
+				.bind(projectId, id)
+				.first<{ action_id: string }>();
+			if (!toolCallRow) {
+				return c.json(
+					{ error: "Not Found", message: "Tool call not found" },
+					404,
+				);
+			}
+
+			const index = new IdentityIndex(db);
+			const manifest = await index.byAction(projectId, toolCallRow.action_id);
+			const toolCall = manifest.toolCalls.find((t) => t.id === id);
+			if (!toolCall) {
+				return c.json(
+					{ error: "Not Found", message: "Tool call not found" },
+					404,
+				);
+			}
+
+			return c.json({ toolCall, manifest });
+		});
+	},
+};

package/src/plugins/ai-receiver.ts

@@ -0,0 +1,254 @@
+import type {
+	AICallPayload,
+	AICallRecord,
+	AIEvaluationPayload,
+	ToolResponseContract,
+} from "@obsunified/types";
+import {
+	EVIDENCE_REFERENCE_CONTRACT,
+	EVIDENCE_REFERENCE_SCHEMA_VERSION,
+	EvidenceReferenceJsonSchema,
+	TOOL_RESPONSE_CONTRACT_SCHEMA_VERSION,
+} from "@obsunified/types";
+import { getConfiguredRetentionHours } from "@obsunified/types/constants";
+import type { CollectorPlugin } from "../framework/collector";
+import { AIStore, type IngestEvaluation } from "../lib/ai-store";
+import { sqlDbFor } from "../lib/sql-db";
+import { getProjectId } from "./_context";
+
+export const aiReceiverPlugin: CollectorPlugin = {
+	name: "ai-receiver",
+	register(app, _runtime) {
+		app.post("/v1/ai", async (c) => {
+			const projectId = getProjectId(c);
+			let payload: AICallPayload;
+			try {
+				payload = await c.req.json<AICallPayload>();
+			} catch {
+				return c.json({ error: "Invalid JSON body" }, 400);
+			}
+			if (!payload.calls || !Array.isArray(payload.calls)) {
+				return c.json({ error: "Missing calls array" }, 400);
+			}
+			if (payload.calls.length > 500) {
+				return c.json(
+					{ error: `Too many AI calls: ${payload.calls.length} (max 500)` },
+					413,
+				);
+			}
+			const store = new AIStore(sqlDbFor(c.env));
+
+			const retentionHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const now = new Date();
+			const nowStr = now.toISOString();
+			const expires = new Date(
+				now.getTime() + retentionHours * 60 * 60 * 1000,
+			).toISOString();
+
+			const records: AICallRecord[] = payload.calls.map((call) => ({
+				projectId,
+				callId: crypto.randomUUID(),
+				traceId: call.traceId || null,
+				spanId: call.spanId || null,
+				serviceName: call.serviceName || null,
+				modelName: call.modelName,
+				provider: call.provider,
+				callType: call.callType,
+				requestJson: call.request ? JSON.stringify(call.request) : null,
+				responseJson: call.response ? JSON.stringify(call.response) : null,
+				promptTokens: call.promptTokens || null,
+				completionTokens: call.completionTokens || null,
+				totalCostUsd: call.totalCostUsd || null,
+				latencyMs: call.latencyMs || null,
+				isError: call.isError || false,
+				errorMessage: call.errorMessage || null,
+				occurredAt: call.occurredAt || nowStr,
+				receivedAt: nowStr,
+				expiresAt: expires,
+				// RFC 0004 — denormalized from the active root span, when the
+				// SDK was told to propagate it.
+				sessionId: call.sessionId ?? null,
+				interactionId: call.interactionId ?? null,
+			}));
+
+			await store.ingestBatch(records);
+
+			return c.json({ accepted: records.length }, 202);
+		});
+
+		app.get("/internal/ai/overview", async (c) => {
+			const projectId = getProjectId(c);
+			const store = new AIStore(sqlDbFor(c.env));
+			const query = c.req.query();
+			const hours = parseInt(query.hours || "24", 10);
+
+			const response = await store.getAICalls({
+				projectId,
+				hours,
+				service: query.service,
+				model: query.model,
+				isError:
+					query.isError === "true"
+						? true
+						: query.isError === "false"
+							? false
+							: undefined,
+				traceId: query.traceId,
+				limit: parseInt(query.limit || "100", 10),
+			});
+
+			return c.json(response);
+		});
+
+		// OpenInference-kind spans (LLM/TOOL/RETRIEVER/CHAIN/AGENT/...) joined
+		// with their payload side-table. This is the replacement for the flat
+		// ai_calls endpoint — it returns typed spans that live inside the
+		// regular trace tree.
+		app.get("/internal/ai/spans", async (c) => {
+			const projectId = getProjectId(c);
+			const store = new AIStore(sqlDbFor(c.env));
+			const query = c.req.query();
+
+			const response = await store.getAISpans({
+				projectId,
+				hours: parseInt(query.hours || "24", 10),
+				kind: query.kind,
+				service: query.service,
+				traceId: query.traceId,
+				limit: parseInt(query.limit || "100", 10),
+			});
+
+			return c.json(response);
+		});
+
+		// Attach evaluations to AI spans. Accepts a batch of evaluation records
+		// keyed by (traceId, spanId). Caller supplies name (e.g. "hallucination"),
+		// one or both of score/label, and optional explanation + metadata.
+		app.post("/v1/ai/evaluations", async (c) => {
+			const projectId = getProjectId(c);
+			let payload: AIEvaluationPayload;
+			try {
+				payload = await c.req.json<AIEvaluationPayload>();
+			} catch {
+				return c.json({ error: "Invalid JSON body" }, 400);
+			}
+			if (!payload.evaluations || !Array.isArray(payload.evaluations)) {
+				return c.json({ error: "Missing evaluations array" }, 400);
+			}
+			if (payload.evaluations.length > 500) {
+				return c.json(
+					{
+						error: `Too many evaluations: ${payload.evaluations.length} (max 500)`,
+					},
+					413,
+				);
+			}
+
+			const retentionHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const now = new Date();
+			const nowStr = now.toISOString();
+			const expires = new Date(
+				now.getTime() + retentionHours * 60 * 60 * 1000,
+			).toISOString();
+
+			const records: IngestEvaluation[] = [];
+			for (const e of payload.evaluations) {
+				if (!e.traceId || !e.spanId || !e.name || !e.source) {
+					return c.json(
+						{ error: "Each evaluation needs traceId, spanId, name, source" },
+						400,
+					);
+				}
+				records.push({
+					projectId,
+					evaluationId: crypto.randomUUID(),
+					traceId: e.traceId,
+					spanId: e.spanId,
+					name: e.name,
+					score: typeof e.score === "number" ? e.score : null,
+					label: e.label ?? null,
+					explanation: e.explanation ?? null,
+					source: e.source,
+					metadataJson: e.metadata ? JSON.stringify(e.metadata) : null,
+					createdAt: nowStr,
+					expiresAt: expires,
+				});
+			}
+
+			const store = new AIStore(sqlDbFor(c.env));
+			await store.ingestEvaluations(records);
+			return c.json({ accepted: records.length }, 202);
+		});
+
+		// List conversation sessions — one row per distinct session.id with
+		// aggregated stats (span count, tokens, cost, error count, etc.).
+		app.get("/internal/ai/sessions", async (c) => {
+			const projectId = getProjectId(c);
+			const store = new AIStore(sqlDbFor(c.env));
+			const query = c.req.query();
+			const response = await store.listSessions({
+				projectId,
+				hours: parseInt(query.hours || "24", 10),
+				userId: query.userId,
+				limit: parseInt(query.limit || "100", 10),
+			});
+			return c.json(response);
+		});
+
+		// Full detail for a single session: every AI span in timestamp order
+		// (across one or more traces) plus any evaluations attached to them.
+		app.get("/internal/ai/sessions/:sessionId", async (c) => {
+			const projectId = getProjectId(c);
+			const store = new AIStore(sqlDbFor(c.env));
+			const sessionId = c.req.param("sessionId");
+			const response = await store.getSession(projectId, sessionId);
+			return c.json(response);
+		});
+
+		// List evaluations, optionally scoped to a single span or trace.
+		app.get("/internal/ai/evaluations", async (c) => {
+			const projectId = getProjectId(c);
+			const store = new AIStore(sqlDbFor(c.env));
+			const query = c.req.query();
+
+			const response = await store.listEvaluations({
+				projectId,
+				traceId: query.traceId,
+				spanId: query.spanId,
+				name: query.name,
+				limit: parseInt(query.limit || "200", 10),
+			});
+
+			return c.json(response);
+		});
+
+		app.get("/internal/ai/evaluations/:evaluationId", async (c) => {
+			const projectId = getProjectId(c);
+			const evaluationId = c.req.param("evaluationId");
+			const store = new AIStore(sqlDbFor(c.env));
+			const evaluation = await store.getEvaluation(projectId, evaluationId);
+			if (!evaluation) {
+				return c.json(
+					{ error: "Not Found", message: "Evaluation not found" },
+					404,
+				);
+			}
+			const contract: ToolResponseContract = {
+				schemaVersion: TOOL_RESPONSE_CONTRACT_SCHEMA_VERSION,
+				transport: "http",
+				tool: "GET /internal/ai/evaluations/:evaluationId",
+				params: { projectId, evaluationId },
+				returns:
+					"{ evaluation: AIEvaluationRecord, evidenceContract: EVIDENCE_REFERENCE_CONTRACT, timestamp: string }",
+				evidenceReferenceSchemaVersion: EVIDENCE_REFERENCE_SCHEMA_VERSION,
+				evidenceReferenceJsonSchema: EvidenceReferenceJsonSchema,
+			};
+			return c.json({
+				evaluation,
+				evidenceContract: EVIDENCE_REFERENCE_CONTRACT,
+				contract,
+				timestamp: new Date().toISOString(),
+			});
+		});
+	},
+};

package/src/plugins/ai-span-payloads-processor.test.ts

@@ -0,0 +1,90 @@
+import type { StoredSpan } from "@obsunified/types";
+import { describe, expect, it } from "vitest";
+import type {
+	CollectorRuntime,
+	SpanProcessorPlugin,
+} from "../framework/collector";
+import type { CollectorRouteContext } from "../framework/env";
+import { sha256Hex } from "../lib/hash";
+import { MemSqlDb } from "../lib/test-utils/mem-sql-db";
+import { aiSpanPayloadsProcessorPlugin } from "./ai-span-payloads-processor";
+
+const span = (): StoredSpan => ({
+	projectId: "proj-123",
+	spanId: "span-1",
+	parentSpanId: null,
+	traceId: "trace-1",
+	traceState: null,
+	serviceName: "agent-service",
+	scopeName: null,
+	scopeVersion: null,
+	spanName: "llm",
+	spanKind: 1,
+	statusCode: 1,
+	statusMessage: null,
+	startTime: "2026-05-22T00:00:00.000Z",
+	endTime: "2026-05-22T00:00:01.000Z",
+	durationMs: 1000,
+	attributesJson: JSON.stringify({
+		"openinference.span.kind": "LLM",
+		"ai.payload.input": "raw prompt",
+		"ai.payload.output": "raw completion",
+		"obs.action.id": "action-1",
+	}),
+	droppedAttributesCount: 0,
+	resourceAttributesJson: "{}",
+	eventsJson: "[]",
+	droppedEventsCount: 0,
+	linksJson: "[]",
+	droppedLinksCount: 0,
+	receivedAt: "2026-05-22T00:00:01.000Z",
+	expiresAt: "2026-05-23T00:00:01.000Z",
+	sessionId: null,
+	interactionId: null,
+});
+
+const processWith = async (db: MemSqlDb, env: Record<string, unknown> = {}) => {
+	const processors: SpanProcessorPlugin[] = [];
+	aiSpanPayloadsProcessorPlugin.register(
+		{} as Parameters<typeof aiSpanPayloadsProcessorPlugin.register>[0],
+		{
+			addSpanProcessor(p: SpanProcessorPlugin) {
+				processors.push(p);
+			},
+		} as unknown as CollectorRuntime,
+	);
+	const processFn = processors[0]?.process;
+	if (!processFn) throw new Error("span processor was not registered");
+	return processFn([span()], {
+		env: { DB: db, ...env },
+		now: new Date(),
+		logger: console,
+	} as unknown as CollectorRouteContext);
+};
+
+describe("aiSpanPayloadsProcessorPlugin payload capture policy", () => {
+	it("stores hashes but omits raw payloads when project capture is disabled", async () => {
+		const db = new MemSqlDb();
+		const processed = await processWith(db);
+
+		const insert = db.callsMatching("INSERT INTO ai_span_payloads")[0];
+		expect(insert.binds).toContain(null);
+		expect(insert.binds).toContain(await sha256Hex("raw prompt"));
+		expect(insert.binds).toContain(await sha256Hex("raw completion"));
+
+		const attrs = JSON.parse(processed[0].attributesJson);
+		expect(attrs["ai.payload.input"]).toBeUndefined();
+		expect(attrs["ai.payload.output"]).toBeUndefined();
+	});
+
+	it("stores raw payloads only when capture is explicitly enabled", async () => {
+		const db = new MemSqlDb();
+		await processWith(db, { OBS_PAYLOAD_CAPTURE_DEFAULT: "true" });
+
+		const insert = db.callsMatching("INSERT INTO ai_span_payloads")[0];
+		expect(insert.binds).toContain("raw prompt");
+		expect(insert.binds).toContain("raw completion");
+		expect(insert.binds).toContain(await sha256Hex("raw prompt"));
+		expect(insert.binds).toContain(await sha256Hex("raw completion"));
+	});
+});