@obsunified/collector 2.0.1

package/src/lib/json.ts

@@ -0,0 +1,39 @@
+import type { JsonValue } from "@obsunified/types";
+
+export const parseJsonRecord = (
+	value: string | null | undefined,
+): Record<string, JsonValue> => {
+	if (!value) {
+		return {};
+	}
+
+	try {
+		const parsed = JSON.parse(value) as JsonValue;
+		return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+			? (parsed as Record<string, JsonValue>)
+			: {};
+	} catch {
+		return {};
+	}
+};
+
+export const parseJsonValue = (value: string): JsonValue => {
+	try {
+		return JSON.parse(value) as JsonValue;
+	} catch {
+		return null;
+	}
+};
+
+export const parseJsonArray = (value: string | null | undefined): unknown[] => {
+	if (!value) {
+		return [];
+	}
+
+	try {
+		const parsed = JSON.parse(value) as unknown;
+		return Array.isArray(parsed) ? parsed : [];
+	} catch {
+		return [];
+	}
+};

package/src/lib/llm.ts

@@ -0,0 +1,263 @@
+/**
+ * RFC 0002 Stage 3 — narrative LLM adapter.
+ *
+ * Single-provider for v1 (Anthropic). The shape is "stuff in a system
+ * prompt that bakes the rendering rules, ask for one declarative
+ * sentence, return text" — a different provider plugs in here without
+ * changing callers.
+ *
+ * Rendering rules from the RFC's UX section, enforced at the system
+ * prompt layer so the model can't accidentally drift:
+ *
+ *   1. Never start with "I" or "Here's" — declarative, not chatbot.
+ *   2. Always cite — if the prompt mentions "the trace", a trace_id
+ *      must follow inline.
+ *   3. Include a time anchor ("starting 8 min ago", "since 14:32").
+ *   4. ≤2 sentences. We're labelling tiles, not writing essays.
+ */
+
+import type { AnalysisDefinition, AnalysisResult } from "@obsunified/types";
+import type { ChildSpanRunner } from "../framework/logger";
+
+export interface NarrativeRequest {
+	definition: AnalysisDefinition;
+	current: AnalysisResult;
+	previous: AnalysisResult | null;
+}
+
+/**
+ * Provider discriminator. The collector picks one based on which API key
+ * env var is set (OpenAI takes precedence when both are set, since users
+ * who configure both presumably did so deliberately for OpenAI).
+ *
+ *   anthropic  → POST https://api.anthropic.com/v1/messages
+ *   openai     → POST https://api.openai.com/v1/chat/completions
+ *                (apiUrl override lets you point at openrouter, vLLM,
+ *                 azure-openai-compat, ollama, etc.)
+ */
+export type LlmProvider = "anthropic" | "openai";
+
+export interface LlmConfig {
+	provider: LlmProvider;
+	apiKey: string;
+	/** Model id. Provider-specific. e.g. "claude-haiku-4-5" / "gpt-4o-mini". */
+	model: string;
+	/** Override base URL — Anthropic-compatible or OpenAI-compatible. */
+	apiUrl?: string;
+	/** Optional abort signal used to cap provider calls from schedulers. */
+	signal?: AbortSignal;
+	/**
+	 * Optional child-span runner. When set, every LLM API call (and tool
+	 * dispatch in the agent loop) is wrapped in an OpenInference-shaped span
+	 * with `gen_ai.system`, `gen_ai.request.model`, and post-call usage
+	 * attributes. Wired by the worker entrypoint via `runtime.withChildSpan`.
+	 */
+	tracer?: ChildSpanRunner;
+}
+
+const DEFAULT_API_URL = "https://api.anthropic.com/v1/messages";
+
+const SYSTEM_PROMPT = `You write one-sentence operational narratives for a self-hosted observability dashboard.
+
+Hard rules — violations are unusable:
+1. Never start with "I", "Here's", "We", "It looks", or any first-person/chatbot framing. Write declarative statements about the system.
+2. If you reference a trace, span, log, or service, include the supplied identifier inline (e.g. "trace abc12345").
+3. Include a time anchor: "starting Nm ago", "for the last Nm", "since HH:MM" — pull from the inputs.
+4. ≤ 2 sentences total, ≤ 220 characters. Numbers belong in the sentence, not in a separate label.
+5. If the inputs are insufficient to write a useful sentence, output exactly the string NO_NARRATIVE (no quotes).
+
+Tone: like a senior on-call writing a Slack message. Specific, blunt, no hedging, no advice ("you should..."), no questions.`;
+
+const renderTemplate = (
+	template: string,
+	def: AnalysisDefinition,
+	r: AnalysisResult,
+): string => {
+	const scope = (def.scope ?? {}) as Record<string, unknown>;
+	const traceIds = Array.isArray(r.payload?.trace_ids)
+		? (r.payload.trace_ids as unknown[])
+				.filter((x): x is string => typeof x === "string")
+				.slice(0, 5)
+				.join(", ")
+		: "";
+	const subs: Record<string, string> = {
+		title: def.title,
+		status: r.status,
+		primary: r.primaryValue === null ? "n/a" : String(r.primaryValue),
+		baseline: r.baselineValue === null ? "n/a" : String(r.baselineValue),
+		delta_pct: r.deltaPct === null ? "n/a" : `${r.deltaPct.toFixed(1)}%`,
+		service: typeof scope.service === "string" ? scope.service : "",
+		trace_ids: traceIds,
+	};
+	return template.replace(/\{\{(\w+)\}\}/g, (_, k) => subs[k] ?? "");
+};
+
+const buildUserPrompt = ({
+	definition,
+	current,
+	previous,
+}: NarrativeRequest): string => {
+	const lines: string[] = [];
+	const tmplRendered = renderTemplate(
+		definition.narrate?.prompt ?? "",
+		definition,
+		current,
+	).trim();
+	if (tmplRendered) lines.push(tmplRendered);
+
+	lines.push("");
+	lines.push("Current state:");
+	lines.push(`- panel: ${definition.title}`);
+	lines.push(`- status: ${current.status}`);
+	if (current.primaryValue !== null)
+		lines.push(`- primary: ${current.primaryValue}`);
+	if (current.baselineValue !== null)
+		lines.push(`- baseline (1h): ${current.baselineValue}`);
+	if (current.deltaPct !== null)
+		lines.push(`- delta vs baseline: ${current.deltaPct.toFixed(1)}%`);
+
+	const since = previous
+		? Math.max(
+				1,
+				Math.round(
+					(new Date(current.generatedAt).getTime() -
+						new Date(previous.generatedAt).getTime()) /
+						60000,
+				),
+			)
+		: null;
+	if (since !== null) {
+		lines.push(
+			`- previous run was ${since} minute${since === 1 ? "" : "s"} ago, status=${previous?.status ?? "n/a"}`,
+		);
+	}
+
+	const payloadKeys = Object.keys(current.payload ?? {}).filter(
+		(k) => k !== "sparkline",
+	);
+	if (payloadKeys.length > 0) {
+		lines.push("");
+		lines.push("Payload details:");
+		for (const k of payloadKeys) {
+			const v = current.payload[k];
+			const rendered =
+				typeof v === "string" || typeof v === "number" || typeof v === "boolean"
+					? String(v)
+					: JSON.stringify(v);
+			if (rendered.length <= 200) lines.push(`- ${k}: ${rendered}`);
+		}
+	}
+
+	lines.push("");
+	lines.push(
+		"Output: a single sentence (max two), no preamble, no quotes, no Markdown.",
+	);
+	return lines.join("\n");
+};
+
+export class LlmCallError extends Error {
+	readonly status: number | null;
+	constructor(message: string, status: number | null = null) {
+		super(message);
+		this.name = "LlmCallError";
+		this.status = status;
+	}
+}
+
+/**
+ * Provider dispatcher for narrative generation.
+ *
+ * Returns `null` when:
+ *   - the model returns the sentinel NO_NARRATIVE
+ *   - the response is empty
+ * Throws `LlmCallError` on non-2xx HTTP responses so the runner can
+ * count the failure against the budget without writing a bogus narrative.
+ */
+export async function generateNarrative(
+	req: NarrativeRequest,
+	config: LlmConfig,
+): Promise<string | null> {
+	if (config.provider === "openai") {
+		// Lazy import to keep the Anthropic-only path zero-cost on the worker
+		// boot and avoid a circular import between llm.ts and openai.ts.
+		const { generateNarrativeOpenAI } = await import("./openai");
+		return generateNarrativeOpenAI(req, config);
+	}
+	return generateNarrativeAnthropic(req, config);
+}
+
+async function generateNarrativeAnthropic(
+	req: NarrativeRequest,
+	config: LlmConfig,
+): Promise<string | null> {
+	const url = config.apiUrl ?? DEFAULT_API_URL;
+	const userPrompt = buildUserPrompt(req);
+
+	const tracer = config.tracer;
+	const callLLM = async (
+		span: { setAttribute(k: string, v: unknown): void } | null,
+	): Promise<string | null> => {
+		const response = await fetch(url, {
+			method: "POST",
+			headers: {
+				"x-api-key": config.apiKey,
+				"anthropic-version": "2023-06-01",
+				"content-type": "application/json",
+			},
+			body: JSON.stringify({
+				model: config.model,
+				max_tokens: 200,
+				system: SYSTEM_PROMPT,
+				messages: [{ role: "user", content: userPrompt }],
+			}),
+			signal: config.signal,
+		});
+		if (span) {
+			span.setAttribute("http.response.status_code", response.status);
+		}
+
+		if (!response.ok) {
+			const text = await response.text().catch(() => "");
+			throw new LlmCallError(
+				`anthropic ${response.status}: ${text.slice(0, 200)}`,
+				response.status,
+			);
+		}
+
+		const body = (await response.json()) as {
+			content?: Array<{ type?: string; text?: string }>;
+			usage?: { input_tokens?: number; output_tokens?: number };
+			stop_reason?: string;
+		};
+		if (body.stop_reason === "max_tokens") {
+			throw new LlmCallError("anthropic response truncated by max_tokens");
+		}
+		if (span && body.usage) {
+			if (typeof body.usage.input_tokens === "number")
+				span.setAttribute("gen_ai.usage.input_tokens", body.usage.input_tokens);
+			if (typeof body.usage.output_tokens === "number")
+				span.setAttribute(
+					"gen_ai.usage.output_tokens",
+					body.usage.output_tokens,
+				);
+		}
+		const text = (body.content ?? [])
+			.filter((block) => block?.type === "text")
+			.map((block) => block?.text ?? "")
+			.join("")
+			.trim();
+
+		if (!text || text === "NO_NARRATIVE") return null;
+		// Strip any accidental wrapping quotes the model might add despite the
+		// system prompt.
+		return text.replace(/^["']|["']$/g, "").trim();
+	};
+
+	if (!tracer) return callLLM(null);
+	return tracer("llm.anthropic.messages", async (span) => callLLM(span), {
+		"openinference.span.kind": "LLM",
+		"gen_ai.system": "anthropic",
+		"gen_ai.request.model": config.model,
+		"gen_ai.request.max_tokens": 200,
+	});
+}

package/src/lib/logs-store.ts

@@ -0,0 +1,188 @@
+import type {
+	LogRecord,
+	LogSeverity,
+	LogsOverviewOptions,
+	LogsOverviewResponse,
+} from "@obsunified/types";
+import { dialectFor, type SqlDb } from "./sql-db";
+
+const LOG_SEVERITIES = new Set<LogSeverity>([
+	"DEBUG",
+	"INFO",
+	"WARN",
+	"ERROR",
+	"FATAL",
+]);
+
+const toLogSeverity = (severity: string): LogSeverity =>
+	LOG_SEVERITIES.has(severity as LogSeverity)
+		? (severity as LogSeverity)
+		: "INFO";
+
+/** Clamp an integer to a safe range */
+const clampInt = (
+	value: unknown,
+	min: number,
+	max: number,
+	fallback: number,
+): number => {
+	const n = typeof value === "number" ? value : parseInt(String(value), 10);
+	if (!Number.isFinite(n)) return fallback;
+	return Math.max(min, Math.min(max, n));
+};
+
+interface LogRow {
+	project_id?: string;
+	log_id: string;
+	trace_id: string | null;
+	span_id: string | null;
+	service_name: string | null;
+	severity: string;
+	severity_number: number;
+	logger_name: string | null;
+	message: string;
+	attributes_json: string | null;
+	flags: number | null;
+	dropped_attributes_count: number | null;
+	occurred_at: string;
+	received_at: string;
+	expires_at: string;
+}
+
+interface LogsSummaryRow {
+	totalLogs?: number;
+	errorLogs?: number;
+	warnLogs?: number;
+}
+
+export class LogsStore {
+	constructor(private readonly db: SqlDb) {}
+
+	async ingestBatch(logs: LogRecord[]): Promise<void> {
+		if (logs.length === 0) return;
+
+		const stmt = this.db.prepare(`
+      INSERT OR IGNORE INTO logs (
+        project_id, log_id, trace_id, span_id, service_name, severity, severity_number,
+        logger_name, message, attributes_json, flags, dropped_attributes_count,
+        occurred_at, received_at, expires_at, session_id, interaction_id
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+		const batch = logs.map((l) => {
+			if (!l.projectId)
+				throw new Error("LogsStore.ingestBatch: log.projectId is required");
+			return stmt.bind(
+				l.projectId,
+				l.logId,
+				l.traceId,
+				l.spanId,
+				l.serviceName,
+				l.severity,
+				l.severityNumber,
+				l.loggerName,
+				l.message,
+				l.attributesJson,
+				l.flags,
+				l.droppedAttributesCount,
+				l.occurredAt,
+				l.receivedAt,
+				l.expiresAt,
+				l.sessionId ?? null,
+				l.interactionId ?? null,
+			);
+		});
+
+		await this.db.batch(batch);
+	}
+
+	async getLogs(options: LogsOverviewOptions): Promise<LogsOverviewResponse> {
+		if (!options.projectId)
+			throw new Error("LogsStore.getLogs: projectId is required");
+		const dialect = dialectFor(this.db);
+		const hours = clampInt(options.hours, 1, 720, 24);
+		const limit = clampInt(options.limit, 1, 1000, 100);
+
+		let sql = `SELECT * FROM logs WHERE project_id = ? AND received_at >= ${dialect.sinceHours("?")}`;
+		const params: unknown[] = [options.projectId, hours];
+
+		if (options.service) {
+			sql += ` AND service_name = ?`;
+			params.push(options.service);
+		}
+		if (options.severity) {
+			sql += ` AND severity = ?`;
+			params.push(options.severity);
+		}
+		if (options.traceId) {
+			sql += ` AND trace_id = ?`;
+			params.push(options.traceId);
+		}
+		if (options.search) {
+			sql += ` AND message LIKE ?`;
+			params.push(`%${options.search}%`);
+		}
+
+		sql += ` ORDER BY received_at DESC LIMIT ?`;
+		params.push(limit);
+
+		const results = await this.db
+			.prepare(sql)
+			.bind(...params)
+			.all<LogRow>();
+
+		// Simple summary stats
+		const summarySql = `
+      SELECT
+        COUNT(*) as totalLogs,
+        SUM(CASE WHEN severity = 'ERROR' OR severity = 'FATAL' THEN 1 ELSE 0 END) as errorLogs,
+        SUM(CASE WHEN severity = 'WARN' THEN 1 ELSE 0 END) as warnLogs
+      FROM logs WHERE project_id = ? AND received_at >= ${dialect.sinceHours("?")}
+      ${options.service ? "AND service_name = ?" : ""}
+    `;
+		const summaryParams: unknown[] = [options.projectId, hours];
+		if (options.service) summaryParams.push(options.service);
+		const summaryResult =
+			(await this.db
+				.prepare(summarySql)
+				.bind(...summaryParams)
+				.first<LogsSummaryRow>()) || {};
+
+		const logs: LogRecord[] = (results.results || []).map((r) => ({
+			projectId: r.project_id ?? "default",
+			logId: r.log_id,
+			traceId: r.trace_id,
+			spanId: r.span_id,
+			serviceName: r.service_name,
+			severity: toLogSeverity(r.severity),
+			severityNumber: r.severity_number,
+			loggerName: r.logger_name,
+			message: r.message,
+			attributesJson: r.attributes_json,
+			flags: r.flags ?? 0,
+			droppedAttributesCount: r.dropped_attributes_count ?? 0,
+			occurredAt: r.occurred_at,
+			receivedAt: r.received_at,
+			expiresAt: r.expires_at,
+		}));
+
+		return {
+			logs,
+			summary: {
+				totalLogs: summaryResult.totalLogs || 0,
+				errorLogs: summaryResult.errorLogs || 0,
+				warnLogs: summaryResult.warnLogs || 0,
+			},
+			windowHours: hours,
+			timestamp: new Date().toISOString(),
+		};
+	}
+
+	async purgeExpired(): Promise<number> {
+		const dialect = dialectFor(this.db);
+		const { meta } = await this.db
+			.prepare(`DELETE FROM logs WHERE expires_at < ${dialect.now()}`)
+			.run();
+		return meta.changes;
+	}
+}

package/src/lib/metrics-store.test.ts

@@ -0,0 +1,189 @@
+import { describe, expect, it } from "vitest";
+import type { DecodedMetricPoint } from "../otlp/decode";
+import { MetricsStore } from "./metrics-store";
+import { MemSqlDb } from "./test-utils/mem-sql-db";
+
+const point = (
+	overrides: Partial<DecodedMetricPoint> = {},
+): DecodedMetricPoint => ({
+	name: "http.server.duration",
+	description: null,
+	unit: "ms",
+	type: "histogram",
+	isMonotonic: null,
+	temporality: 2,
+	scopeName: "@opentelemetry/instrumentation-http",
+	scopeVersion: "1.0.0",
+	serviceName: "checkout",
+	resourceAttrsJson: JSON.stringify({ "service.name": "checkout" }),
+	attributesJson: JSON.stringify({ route: "/checkout" }),
+	identity: "metric-identity",
+	tsNs: "1800000000000000000",
+	startTsNs: null,
+	value: null,
+	count: 1,
+	sum: 42,
+	min: null,
+	max: null,
+	boundsJson: JSON.stringify([50]),
+	bucketCountsJson: JSON.stringify([1]),
+	extraJson: null,
+	exemplarsJson: null,
+	...overrides,
+});
+
+describe("MetricsStore", () => {
+	it("indexes metric exemplars by trace/span during ingest", async () => {
+		const db = new MemSqlDb({
+			all: (sql, binds) => {
+				if (sql.includes("FROM metric_series")) {
+					return [{ id: "series-1", identity: binds[1] }];
+				}
+				return [];
+			},
+		});
+		const store = new MetricsStore(db);
+
+		await store.ingestBatch({
+			projectId: "p1",
+			receivedAt: "2026-06-01T00:00:00.000Z",
+			expiresAt: "2026-06-02T00:00:00.000Z",
+			points: [
+				point({
+					exemplarsJson: JSON.stringify([
+						{
+							value: 42,
+							traceId: "01010101010101010101010101010101",
+							spanId: "0202020202020202",
+							tsNs: "1800000000000000001",
+						},
+						{
+							value: 41,
+							traceId: null,
+							spanId: null,
+							tsNs: "1800000000000000002",
+						},
+						{
+							value: 40,
+							traceId: "03030303030303030303030303030303",
+							spanId: null,
+							tsNs: "1800000000000000003",
+						},
+					]),
+				}),
+			],
+		});
+
+		expect(db.callsMatching("INSERT INTO metric_point")).toHaveLength(1);
+		const exemplarInserts = db.callsMatching("INSERT INTO metric_exemplars");
+		expect(exemplarInserts).toHaveLength(2);
+		expect(exemplarInserts[0].binds.slice(3)).toEqual([
+			"p1",
+			"http.server.duration",
+			"checkout",
+			"01010101010101010101010101010101",
+			"0202020202020202",
+			"1800000000000000001",
+			42,
+			"2026-06-01T00:00:00.000Z",
+			"2026-06-02T00:00:00.000Z",
+		]);
+		expect(exemplarInserts[1].binds[6]).toBe(
+			"03030303030303030303030303030303",
+		);
+		expect(exemplarInserts[1].binds[7]).toBeNull();
+	});
+
+	it("queries exemplars for a trace", async () => {
+		const db = new MemSqlDb({
+			all: (sql) => {
+				if (sql.includes("FROM metric_exemplars")) {
+					return [
+						{
+							id: "ex-1",
+							point_id: "point-1",
+							series_id: "series-1",
+							metric_name: "http.server.duration",
+							service_name: "checkout",
+							trace_id: "trace-1",
+							span_id: "span-1",
+							ts_ns: "1800000000000000001",
+							value: 42,
+							received_at: "2026-06-01T00:00:00.000Z",
+						},
+					];
+				}
+				return [];
+			},
+		});
+		const store = new MetricsStore(db);
+
+		const rows = await store.exemplarsForTrace("p1", "trace-1", 5);
+
+		expect(rows).toEqual([
+			{
+				id: "ex-1",
+				pointId: "point-1",
+				seriesId: "series-1",
+				metricName: "http.server.duration",
+				serviceName: "checkout",
+				traceId: "trace-1",
+				spanId: "span-1",
+				tsNs: "1800000000000000001",
+				value: 42,
+				receivedAt: "2026-06-01T00:00:00.000Z",
+			},
+		]);
+		expect(db.calls[0].binds).toEqual(["p1", "trace-1", 5]);
+	});
+
+	it("queries recent exemplars for resource metric pivots", async () => {
+		const db = new MemSqlDb({
+			all: (sql) => {
+				if (sql.includes("FROM metric_exemplars")) {
+					expect(sql).toContain("service_name = ?");
+					expect(sql).toContain("metric_name LIKE ?");
+					return [
+						{
+							id: "ex-2",
+							point_id: "point-2",
+							series_id: "series-2",
+							metric_name: "system.cpu.utilization",
+							service_name: "checkout",
+							trace_id: "trace-2",
+							span_id: "span-2",
+							ts_ns: "1800000000000000002",
+							value: 0.91,
+							received_at: "2026-06-01T00:01:00.000Z",
+						},
+					];
+				}
+				return [];
+			},
+		});
+		const store = new MetricsStore(db);
+
+		const rows = await store.recentExemplars({
+			projectId: "p1",
+			serviceName: "checkout",
+			metricPrefix: "system.",
+			limit: 3,
+		});
+
+		expect(rows).toEqual([
+			{
+				id: "ex-2",
+				pointId: "point-2",
+				seriesId: "series-2",
+				metricName: "system.cpu.utilization",
+				serviceName: "checkout",
+				traceId: "trace-2",
+				spanId: "span-2",
+				tsNs: "1800000000000000002",
+				value: 0.91,
+				receivedAt: "2026-06-01T00:01:00.000Z",
+			},
+		]);
+		expect(db.calls[0].binds).toEqual(["p1", "checkout", "system.%", 3]);
+	});
+});