@obsunified/collector 2.0.1 → 2.2.0

package/src/framework/collector.ts

@@ -2,9 +2,18 @@ import type { StoredSpan, UsageEventRecord } from "@obsunified/types";
 import { getConfiguredRetentionHours } from "@obsunified/types/constants";
 import type { MiddlewareHandler } from "hono";
 import { Hono } from "hono";
+import { bodyLimit } from "hono/body-limit";
+import {
+	type ActionGraphPurgeSummary,
+	purgeExpiredActionGraph,
+} from "../lib/action-retention";
 import { AIStore } from "../lib/ai-store";
 import { runAllDueAnalyses } from "../lib/analyses-runner";
 import { AnalysesStore } from "../lib/analyses-store";
+import {
+	type HousekeepingSummary,
+	purgeSecondaryRows,
+} from "../lib/housekeeping";
 import { LogsStore } from "../lib/logs-store";
 import { MetricsStore } from "../lib/metrics-store";
 import { aggregatePropagation } from "../lib/propagation-metric";
@@ -12,6 +21,11 @@ import { D1Adapter, dialectFor, type SqlDb, sqlDbFor } from "../lib/sql-db";
 import { TelemetryStore } from "../lib/store";
 import { UsageStore } from "../lib/usage-store";
 import type { CollectorEnv, CollectorRouteContext } from "./env";
+import {
+	DEFAULT_MAX_AUTH_BODY_BYTES,
+	DEFAULT_MAX_BODY_BYTES,
+	parseByteLimit,
+} from "./limits";
 import {
 	type ChildSpanRunner,
 	consoleLogger,
@@ -103,6 +117,18 @@ export interface CollectorConfig {
 	 * a tracing layer at the storage seam.
 	 */
 	sqlDb?: SqlDbFactory;
+	/**
+	 * Raw request body cap for `/v1/*` and `/internal/*` in bytes. Defaults
+	 * to `MAX_BODY_BYTES` from the environment, then 10 MiB. Oversized
+	 * requests get a 413 before any handler buffers the body.
+	 */
+	maxBodyBytes?: number;
+	/**
+	 * Body cap for the unauthenticated `/auth/*` routes (login/check/logout).
+	 * Defaults to 16 KiB and is independent of `maxBodyBytes`, so an anonymous
+	 * client can never make the host buffer an ingest-sized body.
+	 */
+	maxAuthBodyBytes?: number;
 }
 
 export class CollectorRuntime implements CollectorPluginContext {
@@ -221,6 +247,10 @@ export class CollectorRuntime implements CollectorPluginContext {
 	}
 }
 
+/** `/dashboard/login` and anything beneath it (the SPA's login route). */
+export const isDashboardLoginPath = (path: string): boolean =>
+	path === "/dashboard/login" || path.startsWith("/dashboard/login/");
+
 export const createTelemetryCollectorApp = (
 	pluginsOrConfig: CollectorPlugin[] | CollectorConfig,
 ): Hono<{ Bindings: CollectorEnv }> => {
@@ -285,6 +315,35 @@ export const createTelemetryCollectorApp = (
 		}
 	});
 
+	// Body size cap — before auth and every handler, so an oversized body is
+	// refused by Content-Length or cut off mid-stream instead of buffered.
+	const bodyCap: MiddlewareHandler<{ Bindings: CollectorEnv }> = (c, next) => {
+		const maxSize =
+			config.maxBodyBytes ??
+			parseByteLimit(c.env?.MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES);
+		return bodyLimit({
+			maxSize,
+			onError: (ctx) =>
+				ctx.json({ error: `Payload too large (max ${maxSize} bytes)` }, 413),
+		})(c, next);
+	};
+	app.use("/v1/*", bodyCap);
+	app.use("/internal/*", bodyCap);
+	// `/auth/*` is reachable without any credential, so it gets its own,
+	// much smaller cap regardless of what MAX_BODY_BYTES allows for ingest.
+	const authMaxSize = config.maxAuthBodyBytes ?? DEFAULT_MAX_AUTH_BODY_BYTES;
+	app.use(
+		"/auth/*",
+		bodyLimit({
+			maxSize: authMaxSize,
+			onError: (ctx) =>
+				ctx.json(
+					{ error: `Payload too large (max ${authMaxSize} bytes)` },
+					413,
+				),
+		}),
+	);
+
 	// Ingest auth (API key) — always on /v1/* when configured.
 	if (config.auth?.middleware) {
 		app.use("/v1/*", config.auth.middleware);
@@ -293,20 +352,28 @@ export const createTelemetryCollectorApp = (
 		app.use("/v1/*", config.auth.ingest);
 	}
 
-	// Dashboard auth (session cookie) — owns /internal/* when configured.
+	// Dashboard auth (session cookie or programmatic bearer token) — owns /internal/* and /dashboard/* when configured.
 	// Routes for login/check/logout are always registered so the dashboard
 	// can authenticate regardless of subsequent middleware placement.
 	if (config.dashboardAuth) {
+		const { middleware } = config.dashboardAuth;
 		config.dashboardAuth.registerRoutes(app);
-		app.use("/dashboard/*", config.dashboardAuth.middleware);
-		app.use("/internal/*", config.dashboardAuth.middleware);
-	} else if (config.auth?.middleware) {
-		// No dashboard auth configured — fall back to applying the unified
-		// ingest auth to /internal/* too, preserving the earlier single-auth
-		// contract for deployments that haven't adopted dashboard auth yet.
-		app.use("/internal/*", config.auth.middleware);
-	} else if (config.auth?.query) {
-		app.use("/internal/*", config.auth.query);
+		// The login page is where unauthenticated HTML requests get sent;
+		// guarding it with the same middleware redirects it to itself.
+		const guard: MiddlewareHandler<{
+			Bindings: CollectorEnv;
+			// biome-ignore lint/suspicious/noExplicitAny: same contravariant Variables shape as CollectorConfig.dashboardAuth.middleware
+			Variables: any;
+		}> = async (c, next) =>
+			isDashboardLoginPath(c.req.path) ? next() : middleware(c, next);
+		app.use("/dashboard/*", guard);
+		app.use("/internal/*", middleware);
+	} else {
+		// Fail closed for /internal/* and /dashboard/*: ingest keys must never grant read/admin access
+		app.use("/internal/*", async (c) => c.json({ error: "Unauthorized" }, 401));
+		app.use("/dashboard/*", async (c) =>
+			c.json({ error: "Unauthorized" }, 401),
+		);
 	}
 
 	for (const plugin of config.plugins) {
@@ -324,100 +391,260 @@ export const createTelemetryCollectorApp = (
 	return app;
 };
 
+/**
+ * Runtime-neutral entry point for a background maintenance task.
+ *
+ * `run(env)` does the work and needs nothing but a `CollectorEnv`, so any
+ * host (Cloudflare cron, a Node `setInterval`, a Kubernetes CronJob hitting
+ * a one-shot script) can drive it. `scheduled(event, env, ctx)` is the thin
+ * Cloudflare Workers adapter over `run` — the event and context are accepted
+ * for signature compatibility with `ExportedHandler.scheduled` but are never
+ * consulted, so non-Workers hosts should call `run` directly rather than
+ * fabricating placeholder event objects.
+ */
+export interface ScheduledTaskHandler<TSummary = void> {
+	run(env: CollectorEnv): Promise<TSummary>;
+	scheduled(
+		event: ScheduledEvent,
+		env: CollectorEnv,
+		ctx: ExecutionContext,
+	): Promise<void>;
+}
+
+/** Per-table purge counts reported by the retention cleanup task. */
+export interface RetentionCleanupSummary {
+	spans: number;
+	usage: number;
+	logs: number;
+	ai_calls: number;
+	metric_points: number;
+	analysis_results: number;
+	profile_blobs: number;
+	/** RFC 0010 action spine rows removed (trees older than RETENTION_HOURS). */
+	actions: number;
+	/** Leaf rows removed with their actions: agent_runs, tool_calls,
+	 *  retrieval_events, eval_results and artifacts. */
+	action_leaves: number;
+	/** Secondary tables (see lib/housekeeping.ts); zeros when that step failed. */
+	housekeeping: HousekeepingSummary;
+}
+
 /**
  * Scheduled handler that purges expired telemetry and usage rows.
- * Wire into wrangler.toml: `[triggers] crons = ["0 * * * *"]`
+ * Wire into wrangler.toml: `[triggers] crons = ["0 * * * *"]`, or call
+ * `.run(env)` from a non-Workers host scheduler.
  * Source: AgentOwl + DecisionOps
  */
+/** Expired profile rows examined per sweep: pages × page size. */
+const PROFILE_PURGE_PAGE_SIZE = 500;
+const PROFILE_PURGE_MAX_PAGES = 10;
+
 export const createRetentionCleanupHandler = (options?: {
 	logger?: Logger;
-}) => {
+}): ScheduledTaskHandler<RetentionCleanupSummary> => {
 	const logger = options?.logger ?? consoleLogger;
-	return {
-		async scheduled(
-			_event: ScheduledEvent,
-			env: CollectorEnv,
-			_ctx: ExecutionContext,
-		): Promise<void> {
-			const db = sqlDbFor(env);
-			const telemetryStore = new TelemetryStore(db);
-			const usageStore = new UsageStore(db);
-			const logsStore = new LogsStore(db);
-			const aiStore = new AIStore(db);
-			const metricsStore = new MetricsStore(db);
-			const analysesStore = new AnalysesStore(db);
-
-			const [
-				telemetryPurged,
-				usagePurged,
-				logsPurged,
-				aiPurged,
-				metricsPurged,
-				analysesPurged,
-			] = await Promise.all([
-				telemetryStore.purgeExpired(),
-				usageStore.purgeExpired(),
-				logsStore.purgeExpired(),
-				aiStore.purgeExpired(),
-				metricsStore.purgeExpired(),
-				analysesStore.purgeExpired(),
-			]);
-
-			// RFC 0007 Phase 4.10 — profile_blobs retention. Cascade
-			// foreign key on profile_trace_index handles the join rows;
-			// we still need to delete the R2 blobs, which we do
-			// best-effort by reading the URLs first.
-			let profilesPurged = 0;
-			try {
-				const dialect = dialectFor(db);
-				const expiredProfiles = await db
+	const run = async (env: CollectorEnv): Promise<RetentionCleanupSummary> => {
+		const db = sqlDbFor(env);
+		const telemetryStore = new TelemetryStore(db);
+		const usageStore = new UsageStore(db);
+		const logsStore = new LogsStore(db);
+		const aiStore = new AIStore(db);
+		const metricsStore = new MetricsStore(db);
+		const analysesStore = new AnalysesStore(db);
+
+		const [
+			telemetryPurged,
+			usagePurged,
+			logsPurged,
+			aiPurged,
+			metricsPurged,
+			analysesPurged,
+		] = await Promise.all([
+			telemetryStore.purgeExpired(),
+			usageStore.purgeExpired(),
+			logsStore.purgeExpired(),
+			aiStore.purgeExpired(),
+			metricsStore.purgeExpired(),
+			analysesStore.purgeExpired(),
+		]);
+
+		// RFC 0007 Phase 4.10 — profile_blobs retention. Cascade
+		// foreign key on profile_trace_index handles the join rows;
+		// we still need to delete the R2 blobs, which we do
+		// best-effort by reading the URLs first.
+		// Rows go only for blobs whose delete settled, in pages, so a row never
+		// outlives its blob reference while the object is still in storage
+		// (previously one page of blobs was deleted and then every expired
+		// row, orphaning anything past the page).
+		let profilesPurged = 0;
+		try {
+			const dialect = dialectFor(db);
+			if (!env.PROFILES_BUCKET) {
+				logger.warn(
+					"[retention-cleanup] PROFILES_BUCKET is not bound; expired profile rows are removed without deleting their blobs",
+				);
+			}
+			for (let page = 0; page < PROFILE_PURGE_MAX_PAGES; page += 1) {
+				const expired = await db
 					.prepare(
-						`SELECT id, blob_url FROM profile_blobs WHERE expires_at < ${dialect.now()} LIMIT 500`,
+						`SELECT id, blob_url FROM profile_blobs WHERE expires_at < ${dialect.now()} LIMIT ${PROFILE_PURGE_PAGE_SIZE}`,
 					)
 					.all<{ id: string; blob_url: string }>();
-				if (expiredProfiles.results.length > 0 && env.PROFILES_BUCKET) {
-					await Promise.allSettled(
-						expiredProfiles.results.map((p) =>
-							env.PROFILES_BUCKET?.delete(p.blob_url),
-						),
+				if (expired.results.length === 0) break;
+				const removable: string[] = [];
+				if (env.PROFILES_BUCKET) {
+					const bucket = env.PROFILES_BUCKET;
+					const settled = await Promise.allSettled(
+						expired.results.map((p) => bucket.delete(p.blob_url)),
 					);
+					settled.forEach((outcome, i) => {
+						const row = expired.results[i];
+						if (!row) return;
+						if (outcome.status === "fulfilled") removable.push(row.id);
+						else
+							logger.warn(
+								"[retention-cleanup] profile blob delete failed; row kept for retry",
+								{
+									profile_id: row.id,
+									error:
+										outcome.reason instanceof Error
+											? outcome.reason.message
+											: String(outcome.reason),
+								},
+							);
+					});
+				} else {
+					removable.push(...expired.results.map((p) => p.id));
 				}
-				const deleted = await db
-					.prepare(
-						`DELETE FROM profile_blobs WHERE expires_at < ${dialect.now()}`,
-					)
-					.run();
-				profilesPurged = deleted.meta.changes;
-			} catch (err) {
-				logger.error("[retention-cleanup] profile purge failed", {
-					error: err instanceof Error ? err.message : String(err),
-				});
+				for (let i = 0; i < removable.length; i += 90) {
+					const chunk = removable.slice(i, i + 90);
+					const deleted = await db
+						.prepare(
+							`DELETE FROM profile_blobs WHERE id IN (${chunk.map(() => "?").join(", ")})`,
+						)
+						.bind(...chunk)
+						.run();
+					profilesPurged += deleted.meta.changes;
+				}
+				// A page with kept rows would be re-selected forever; stop here
+				// and let the next sweep retry the failed blobs.
+				if (
+					expired.results.length < PROFILE_PURGE_PAGE_SIZE ||
+					removable.length < expired.results.length
+				)
+					break;
 			}
+		} catch (err) {
+			logger.error("[retention-cleanup] profile purge failed", {
+				error: err instanceof Error ? err.message : String(err),
+			});
+		}
 
-			logger.info("[retention-cleanup] purged expired rows", {
-				spans: telemetryPurged,
-				usage: usagePurged,
-				logs: logsPurged,
-				ai_calls: aiPurged,
-				metric_points: metricsPurged,
-				analysis_results: analysesPurged,
-				profile_blobs: profilesPurged,
+		let housekeeping: HousekeepingSummary = {
+			alert_evaluations: 0,
+			ask_evidence_events: 0,
+			metric_series: 0,
+			evidence_refs: 0,
+			evidence_ref_expansions: 0,
+		};
+		try {
+			housekeeping = await purgeSecondaryRows(db, {
+				retentionHours: getConfiguredRetentionHours(env.RETENTION_HOURS),
+			});
+		} catch (err) {
+			logger.error("[retention-cleanup] housekeeping purge failed", {
+				error: err instanceof Error ? err.message : String(err),
 			});
+		}
 
-			// RFC 0004 Phase 1.8 — emit interaction_id propagation counters.
-			// Hourly aggregation rides the retention cron because cadence
-			// matches and both touch the same projects' tables.
-			try {
-				const propagation = await aggregatePropagation(db, new Date(), logger);
-				logger.info("[propagation-metric] hourly aggregation complete", {
-					projects: propagation.projects,
-					points_written: propagation.pointsWritten,
-				});
-			} catch (err) {
-				logger.error("[propagation-metric] aggregation failed", {
-					error: err instanceof Error ? err.message : String(err),
-				});
+		// RFC 0010 — action graph retention. Derived from `actions.started_at`
+		// because the spine has no `expires_at`; whole causal trees go at once,
+		// in bounded batches, and in-progress or eval-case-pinned trees stay.
+		// Runs after the per-table purges so a slow sweep never delays them.
+		let actionGraph: ActionGraphPurgeSummary | undefined;
+		try {
+			actionGraph = await purgeExpiredActionGraph(db, {
+				retentionHours: getConfiguredRetentionHours(env.RETENTION_HOURS),
+			});
+			if (actionGraph.truncated) {
+				logger.warn(
+					"[retention-cleanup] action graph purge hit its batch cap; remaining expired trees wait for the next run",
+					{ batches: actionGraph.batches, trees: actionGraph.trees },
+				);
 			}
+			if (actionGraph.oversized_trees > 0) {
+				// Retained on purpose: deleting them would exceed the per-batch
+				// bound, and a partial tree is worse than an old one. Operators
+				// can raise `maxActionsPerBatch` or remove them by hand.
+				logger.warn(
+					"[retention-cleanup] expired action trees retained because they exceed the per-batch action cap",
+					{
+						oversized_trees: actionGraph.oversized_trees,
+						trees: actionGraph.oversized,
+					},
+				);
+			}
+		} catch (err) {
+			logger.error("[retention-cleanup] action graph purge failed", {
+				error: err instanceof Error ? err.message : String(err),
+			});
+		}
+
+		const summary: RetentionCleanupSummary = {
+			spans: telemetryPurged,
+			usage: usagePurged,
+			logs: logsPurged,
+			ai_calls: aiPurged,
+			metric_points: metricsPurged,
+			analysis_results: analysesPurged,
+			profile_blobs: profilesPurged,
+			housekeeping,
+			actions: actionGraph?.actions ?? 0,
+			action_leaves: actionGraph
+				? actionGraph.agent_runs +
+					actionGraph.tool_calls +
+					actionGraph.retrieval_events +
+					actionGraph.eval_results +
+					actionGraph.artifacts
+				: 0,
+		};
+		logger.info("[retention-cleanup] purged expired rows", {
+			...summary,
+			...(actionGraph
+				? {
+						action_agent_runs: actionGraph.agent_runs,
+						action_tool_calls: actionGraph.tool_calls,
+						action_retrieval_events: actionGraph.retrieval_events,
+						action_eval_results: actionGraph.eval_results,
+						action_artifacts: actionGraph.artifacts,
+						action_trees: actionGraph.trees,
+						action_batches: actionGraph.batches,
+						action_oversized_trees: actionGraph.oversized_trees,
+					}
+				: {}),
+		});
+
+		// RFC 0004 Phase 1.8 — emit interaction_id propagation counters.
+		// Hourly aggregation rides the retention cron because cadence
+		// matches and both touch the same projects' tables.
+		try {
+			const propagation = await aggregatePropagation(db, new Date(), logger);
+			logger.info("[propagation-metric] hourly aggregation complete", {
+				projects: propagation.projects,
+				points_written: propagation.pointsWritten,
+			});
+		} catch (err) {
+			logger.error("[propagation-metric] aggregation failed", {
+				error: err instanceof Error ? err.message : String(err),
+			});
+		}
+
+		return summary;
+	};
+	return {
+		run,
+		scheduled: async (_event, env, _ctx) => {
+			await run(env);
 		},
 	};
 };
@@ -427,41 +654,54 @@ export const createRetentionCleanupHandler = (options?: {
  * `runAllDueAnalyses` filters by `last_run + refreshSeconds < now` before
  * touching D1.
  *
- * Wire to a per-minute trigger in wrangler.toml: `crons = ["* * * * *"]`.
- * Kept distinct from retention so the per-minute tick doesn't sweep every
- * retention table every cycle.
+ * Wire to a per-minute trigger in wrangler.toml: `crons = ["* * * * *"]`,
+ * or call `.run(env)` from a non-Workers host scheduler. Kept distinct from
+ * retention so the per-minute tick doesn't sweep every retention table every
+ * cycle.
  */
 export const createAnalysesRunHandler = (options?: {
 	logger?: Logger;
 	tracer?: ChildSpanRunner;
-}) => {
+}): ScheduledTaskHandler<AnalysesRunSummary | undefined> => {
 	const logger = options?.logger ?? consoleLogger;
 	const tracer = options?.tracer;
+	/**
+	 * Runs every due analysis. Resolves to the run summary, or `undefined`
+	 * when the run failed — failures are logged rather than thrown so a
+	 * transient DB hiccup never breaks a host's cron loop.
+	 */
+	const run = async (
+		env: CollectorEnv,
+	): Promise<AnalysesRunSummary | undefined> => {
+		try {
+			const retentionHours = getConfiguredRetentionHours(env.RETENTION_HOURS);
+			const summary = await runAllDueAnalyses({
+				env,
+				retentionHours,
+				logger,
+				tracer,
+			});
+			logger.info("[analyses] scheduled run summary", {
+				refreshed: summary.refreshed,
+				ran: summary.ran,
+				failed: summary.failed,
+				narrated: summary.narrated,
+			});
+			return summary;
+		} catch (error) {
+			logger.error("[analyses] scheduled run failed", {
+				error: error instanceof Error ? error.message : String(error),
+			});
+			return undefined;
+		}
+	};
 	return {
-		async scheduled(
-			_event: ScheduledEvent,
-			env: CollectorEnv,
-			_ctx: ExecutionContext,
-		): Promise<void> {
-			try {
-				const retentionHours = getConfiguredRetentionHours(env.RETENTION_HOURS);
-				const summary = await runAllDueAnalyses({
-					env,
-					retentionHours,
-					logger,
-					tracer,
-				});
-				logger.info("[analyses] scheduled run summary", {
-					refreshed: summary.refreshed,
-					ran: summary.ran,
-					failed: summary.failed,
-					narrated: summary.narrated,
-				});
-			} catch (error) {
-				logger.error("[analyses] scheduled run failed", {
-					error: error instanceof Error ? error.message : String(error),
-				});
-			}
+		run,
+		scheduled: async (_event, env, _ctx) => {
+			await run(env);
 		},
 	};
 };
+
+/** Shape returned by `runAllDueAnalyses` and surfaced via the run handler. */
+export type AnalysesRunSummary = Awaited<ReturnType<typeof runAllDueAnalyses>>;

package/src/framework/env.ts

@@ -26,10 +26,19 @@ export interface CollectorEnv {
 	INGEST_KEY?: string;
 	/** Password for dashboard login */
 	DASHBOARD_PASSWORD?: string;
+	/** Programmatic read/admin bearer token for /internal/* endpoints (e.g. MCP server) */
+	DASHBOARD_TOKEN?: string;
 	/** Comma-separated allowed origins for CORS (e.g. "https://my-app.com,https://staging.my-app.com") */
 	ALLOWED_ORIGINS?: string;
 	/** Set to "true" to allow unauthenticated ingest (local dev only) */
 	ALLOW_UNAUTHENTICATED?: string;
+	/** Raw request body cap in bytes for /v1/* and /internal/* (default 10 MiB). */
+	MAX_BODY_BYTES?: string;
+	/**
+	 * Set to "true" to let alert webhooks target loopback / private / link-local
+	 * hosts (self-hosted receivers). Off by default to block SSRF.
+	 */
+	ALERT_WEBHOOK_ALLOW_PRIVATE?: string;
 	TELEMETRY_REDACT_FIELDS?: string;
 	/**
 	 * Local/test escape hatch for payload capture. Production should prefer the

package/src/framework/limits.ts

@@ -0,0 +1,31 @@
+/**
+ * Request body limits for ingest (`/v1/*`) and admin (`/internal/*`) routes.
+ *
+ * Without a cap the collector buffered whole bodies (and fully gunzipped
+ * OTLP payloads) before the per-item caps applied. Cloudflare bounds the raw
+ * request size per plan; the Node host has no platform limit, so a single
+ * ingest key could exhaust memory with one request or one gzip bomb.
+ */
+
+/** Raw request body cap. Overridable via `MAX_BODY_BYTES` or `CollectorConfig.maxBodyBytes`. */
+export const DEFAULT_MAX_BODY_BYTES = 10 * 1024 * 1024;
+
+/**
+ * Cap for the unauthenticated `/auth/*` routes. `POST /auth/login` carries a
+ * JSON object with one password; anything near this size is not a login.
+ * Deliberately small and separate from `MAX_BODY_BYTES`: raising the ingest
+ * cap for large OTLP batches must not widen what an anonymous client can
+ * make the host buffer.
+ */
+export const DEFAULT_MAX_AUTH_BODY_BYTES = 16 * 1024;
+
+/** Cap on the gunzipped OTLP payload (a 10 MiB gzip can expand far beyond this). */
+export const DEFAULT_MAX_DECOMPRESSED_BYTES = 32 * 1024 * 1024;
+
+export const parseByteLimit = (
+	raw: string | undefined,
+	fallback: number,
+): number => {
+	const parsed = Number.parseInt(raw ?? "", 10);
+	return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};