@obsunified/collector 2.0.1 → 2.1.1

package/src/lib/test-utils/node-sqlite-db.ts

@@ -0,0 +1,188 @@
+/**
+ * `SqlDb` over Node's built-in `node:sqlite` for tests that need real SQL
+ * execution against the shipped `migrations/` (as opposed to `MemSqlDb`,
+ * which only records which statements a store chose to issue).
+ *
+ * `batch` runs the statements inside one transaction, matching what D1's
+ * `batch` and `PostgresAdapter.batch` guarantee.
+ */
+
+import { readdirSync, readFileSync } from "node:fs";
+import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { fileURLToPath } from "node:url";
+import { type SqlDb, type SqlStatement, sqliteDialect } from "../sql-db";
+
+type SqliteBind = string | number | bigint | null | Uint8Array;
+
+/** D1 accepts `undefined` and booleans in `bind`; node:sqlite does not. */
+const toBind = (value: unknown): SqliteBind => {
+	if (value === undefined || value === null) return null;
+	if (typeof value === "boolean") return value ? 1 : 0;
+	if (
+		typeof value === "number" ||
+		typeof value === "bigint" ||
+		typeof value === "string" ||
+		value instanceof Uint8Array
+	) {
+		return value;
+	}
+	return JSON.stringify(value);
+};
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+export const SQLITE_MIGRATIONS_DIR = path.resolve(here, "../../migrations");
+
+/** Sorted migration file names, optionally only those up to `through`. */
+export const listSqliteMigrations = (through?: string): string[] =>
+	readdirSync(SQLITE_MIGRATIONS_DIR)
+		.filter((f) => f.endsWith(".sql"))
+		.filter((f) => through === undefined || f <= through)
+		.sort();
+
+export interface ApplyMigrationsOptions {
+	/** Apply files whose numeric prefix is `>= from` (inclusive, e.g. "045"). */
+	from?: string;
+	/** Apply files whose numeric prefix is `<= upTo` (inclusive, e.g. "044"). */
+	upTo?: string;
+}
+
+/** Apply `migrations/*.sql` in sorted order; returns the file names applied. */
+export function applySqliteMigrations(
+	db: DatabaseSync,
+	options: ApplyMigrationsOptions = {},
+): string[] {
+	const applied: string[] = [];
+	for (const file of listSqliteMigrations()) {
+		const prefix = file.slice(0, 3);
+		if (options.from && prefix < options.from) continue;
+		if (options.upTo && prefix > options.upTo) continue;
+		db.exec(readFileSync(path.join(SQLITE_MIGRATIONS_DIR, file), "utf8"));
+		applied.push(file);
+	}
+	return applied;
+}
+
+/** Fresh in-memory database with the selected D1 migrations applied. */
+export const createMigratedSqliteDb = (
+	options: ApplyMigrationsOptions = {},
+): NodeSqliteDb => {
+	const db = new NodeSqliteDb();
+	applySqliteMigrations(db.raw, options);
+	return db;
+};
+
+/** Column names of a table's PRIMARY KEY in key order. */
+export const primaryKeyColumns = (db: DatabaseSync, table: string): string[] =>
+	(
+		db.prepare(`PRAGMA table_info(${table})`).all() as Array<{
+			name: string;
+			pk: number;
+		}>
+	)
+		.filter((c) => c.pk > 0)
+		.sort((a, b) => a.pk - b.pk)
+		.map((c) => c.name);
+
+/** Index names declared on a table (SQLite auto-indexes excluded). */
+export const indexNames = (db: DatabaseSync, table: string): string[] =>
+	(db.prepare(`PRAGMA index_list(${table})`).all() as Array<{ name: string }>)
+		.map((i) => i.name)
+		.filter((n) => !n.startsWith("sqlite_autoindex"))
+		.sort();
+
+export class NodeSqliteDb implements SqlDb {
+	readonly dialect = sqliteDialect;
+
+	constructor(readonly raw: DatabaseSync = new DatabaseSync(":memory:")) {}
+
+	close(): void {
+		this.raw.close();
+	}
+
+	/** Apply `migrations/*.sql` in order. `through` applies up to that file. */
+	static migrated(through?: string): NodeSqliteDb {
+		const db = new NodeSqliteDb();
+		db.applyMigrations(listSqliteMigrations(through));
+		return db;
+	}
+
+	applyMigrations(files: readonly string[]): void {
+		for (const file of files) {
+			this.raw.exec(
+				readFileSync(path.join(SQLITE_MIGRATIONS_DIR, file), "utf8"),
+			);
+		}
+	}
+
+	prepare(sql: string): SqlStatement {
+		return new NodeSqliteStatement(this.raw, sql, []);
+	}
+
+	/** Batches are serialized and run without yielding, like D1's atomic batch. */
+	private batchQueue: Promise<unknown> = Promise.resolve();
+
+	batch(
+		statements: SqlStatement[],
+	): Promise<Array<{ meta: { changes: number } }>> {
+		const run = () => {
+			const results: Array<{ meta: { changes: number } }> = [];
+			this.raw.exec("BEGIN");
+			try {
+				for (const stmt of statements) {
+					if (!(stmt instanceof NodeSqliteStatement)) {
+						throw new Error("NodeSqliteDb.batch: foreign statement");
+					}
+					results.push(stmt.runSync());
+				}
+				this.raw.exec("COMMIT");
+			} catch (err) {
+				this.raw.exec("ROLLBACK");
+				throw err;
+			}
+			return results;
+		};
+		const next = this.batchQueue.then(run, run);
+		this.batchQueue = next.catch(() => undefined);
+		return next;
+	}
+
+	/** Convenience for assertions: `SELECT COUNT(*)` with an optional filter. */
+	count(table: string, where = "1 = 1", ...binds: SqliteBind[]): number {
+		const row = this.raw
+			.prepare(`SELECT COUNT(*) AS n FROM ${table} WHERE ${where}`)
+			.get(...binds) as { n: number | bigint };
+		return Number(row.n);
+	}
+}
+
+class NodeSqliteStatement implements SqlStatement {
+	constructor(
+		private readonly db: DatabaseSync,
+		private readonly sql: string,
+		private readonly binds: readonly unknown[],
+	) {}
+
+	bind(...args: unknown[]): SqlStatement {
+		return new NodeSqliteStatement(this.db, this.sql, args.map(toBind));
+	}
+
+	async first<T = Record<string, unknown>>(): Promise<T | null> {
+		const row = this.db.prepare(this.sql).get(...(this.binds as SqliteBind[]));
+		return (row ?? null) as T | null;
+	}
+
+	async all<T = Record<string, unknown>>(): Promise<{ results: T[] }> {
+		const rows = this.db.prepare(this.sql).all(...(this.binds as SqliteBind[]));
+		return { results: rows as T[] };
+	}
+
+	runSync(): { meta: { changes: number } } {
+		const info = this.db.prepare(this.sql).run(...(this.binds as SqliteBind[]));
+		return { meta: { changes: Number(info.changes) } };
+	}
+
+	async run(): Promise<{ meta: { changes: number } }> {
+		return this.runSync();
+	}
+}

package/src/lib/webhook-url.test.ts

@@ -0,0 +1,122 @@
+import { describe, expect, it } from "vitest";
+import {
+	assertSafeWebhookUrl,
+	isPrivateWebhookHost,
+	redactWebhookUrl,
+	resolveWebhookHost,
+	WebhookUrlError,
+} from "./webhook-url";
+
+describe("assertSafeWebhookUrl", () => {
+	it("accepts public http(s) hooks", () => {
+		for (const url of [
+			"https://hooks.slack.com/services/T000/B000/XXXX",
+			"https://events.pagerduty.com/v2/enqueue",
+			"http://alerts.example.com:8080/hook?team=1",
+			"https://8.8.8.8/hook",
+		]) {
+			expect(() => assertSafeWebhookUrl(url)).not.toThrow();
+		}
+	});
+
+	it("rejects non-http schemes, relative URLs and embedded credentials", () => {
+		for (const url of [
+			"ftp://example.com/x",
+			"file:///etc/passwd",
+			"javascript:alert(1)",
+			"/relative/hook",
+			"https://user:pass@example.com/hook",
+		]) {
+			expect(() => assertSafeWebhookUrl(url), url).toThrow(WebhookUrlError);
+		}
+	});
+
+	it("rejects loopback, private, link-local and metadata hosts", () => {
+		for (const url of [
+			"http://localhost/hook",
+			"http://app.localhost/hook",
+			"http://127.0.0.1:9000/hook",
+			"http://0.0.0.0/hook",
+			"http://10.1.2.3/hook",
+			"http://172.16.0.1/hook",
+			"http://192.168.1.1/hook",
+			"http://100.64.0.1/hook",
+			"http://169.254.169.254/latest/meta-data/",
+			"http://[::1]/hook",
+			"http://[fe80::1]/hook",
+			"http://[fd00::1]/hook",
+			"http://[::ffff:10.0.0.1]/hook",
+			"http://collector/hook",
+			"http://db.internal/hook",
+		]) {
+			expect(() => assertSafeWebhookUrl(url), url).toThrow(WebhookUrlError);
+		}
+	});
+
+	it("allows private hosts only with the explicit opt-in", () => {
+		const url = "http://alertmanager.internal:9093/api/v2/alerts";
+		expect(() => assertSafeWebhookUrl(url)).toThrow(WebhookUrlError);
+		expect(
+			assertSafeWebhookUrl(url, { allowPrivateNetworks: true }).hostname,
+		).toBe("alertmanager.internal");
+		// The opt-in never unlocks credentials or other schemes.
+		expect(() =>
+			assertSafeWebhookUrl("ftp://10.0.0.1/x", { allowPrivateNetworks: true }),
+		).toThrow(WebhookUrlError);
+	});
+
+	it("isPrivateWebhookHost treats public names as public", () => {
+		expect(isPrivateWebhookHost("hooks.slack.com")).toBe(false);
+		expect(isPrivateWebhookHost("hooks.slack.com.")).toBe(false);
+		expect(isPrivateWebhookHost("1.1.1.1")).toBe(false);
+	});
+});
+
+describe("redactWebhookUrl", () => {
+	it("keeps only the origin: any path segment or query may be the credential", () => {
+		expect(
+			redactWebhookUrl("https://hooks.slack.com/services/T000/B000/SECRET"),
+		).toBe("https://hooks.slack.com/…");
+		// A generic receiver whose only secret is the single path segment.
+		expect(redactWebhookUrl("https://hooks.example.com/TOKEN123")).toBe(
+			"https://hooks.example.com/…",
+		);
+		expect(redactWebhookUrl("https://example.com/?token=abc")).toBe(
+			"https://example.com/…",
+		);
+		expect(redactWebhookUrl("https://user:pw@example.com:8443/")).toBe(
+			"https://example.com:8443",
+		);
+		expect(redactWebhookUrl("not a url")).toBe("<invalid url>");
+		for (const url of [
+			"https://hooks.example.com/TOKEN123",
+			"https://example.com/a/b?token=abc#frag",
+			"https://user:pw@example.com/",
+		]) {
+			expect(redactWebhookUrl(url)).not.toMatch(/TOKEN123|token=|pw|abc|frag/);
+		}
+	});
+
+	it("resolveWebhookHost classifies answers and fails closed when it cannot resolve", async () => {
+		const lookup = async (host: string) =>
+			host === "evil.example"
+				? ["93.184.216.34", "10.0.0.8"]
+				: ["93.184.216.34"];
+		expect(await resolveWebhookHost("evil.example", lookup)).toBe("private");
+		expect(await resolveWebhookHost("hooks.example", lookup)).toBe("public");
+		expect(
+			await resolveWebhookHost("v6.example", async () => ["fd00::1"]),
+		).toBe("private");
+		// IP literals were judged already by the literal check.
+		expect(await resolveWebhookHost("10.0.0.8", lookup)).toBe("public");
+		// No answer is never "safe": the delivery fetch would resolve on its own.
+		expect(
+			await resolveWebhookHost("down.example", async () => {
+				throw new Error("ENOTFOUND");
+			}),
+		).toBe("unresolvable");
+		expect(await resolveWebhookHost("empty.example", async () => [])).toBe(
+			"unresolvable",
+		);
+	});
+});

package/src/lib/webhook-url.ts

@@ -0,0 +1,155 @@
+/**
+ * Alert webhook destinations are admin-supplied URLs the collector POSTs to
+ * from inside its own network. Two guards live here:
+ *
+ *  - `assertSafeWebhookUrl` rejects anything that is not plain http(s) to a
+ *    public host, so a rule cannot be pointed at loopback, RFC 1918 ranges,
+ *    link-local (cloud metadata) or embedded credentials. Self-hosted
+ *    installs that legitimately post to an internal receiver can opt in
+ *    with `allowPrivateNetworks` (env `ALERT_WEBHOOK_ALLOW_PRIVATE=true`).
+ *  - `redactWebhookUrl` renders a URL for logs and span attributes. Slack
+ *    and PagerDuty-style hooks carry their secret in the path, so only the
+ *    origin and the first path segment survive; query and userinfo never do.
+ *
+ * Hostnames are checked as written; DNS resolution happens at fetch time
+ * and a name that resolves to a private address is not caught here.
+ */
+
+export interface WebhookUrlOptions {
+	allowPrivateNetworks?: boolean;
+}
+
+export class WebhookUrlError extends Error {}
+
+const parseIpv4 = (host: string): number[] | null => {
+	const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
+	if (!m) return null;
+	const octets = m.slice(1).map((o) => Number.parseInt(o, 10));
+	return octets.every((o) => o <= 255) ? octets : null;
+};
+
+const isPrivateIpv4 = (o: number[]): boolean =>
+	o[0] === 0 || // "this" network / unspecified
+	o[0] === 10 ||
+	o[0] === 127 ||
+	(o[0] === 100 && o[1] >= 64 && o[1] <= 127) || // CGNAT
+	(o[0] === 169 && o[1] === 254) || // link-local incl. cloud metadata
+	(o[0] === 172 && o[1] >= 16 && o[1] <= 31) ||
+	(o[0] === 192 && o[1] === 168) ||
+	o[0] >= 224; // multicast, reserved, broadcast
+
+const isPrivateIpv6 = (host: string): boolean => {
+	// URL.hostname keeps the brackets for IPv6 literals.
+	const literal = host.replace(/^\[|\]$/g, "").toLowerCase();
+	if (literal === "::" || literal === "::1") return true;
+	// IPv4-mapped (::ffff:a.b.c.d). The WHATWG URL parser serializes the
+	// embedded address as two hex groups (::ffff:a00:1), so accept both.
+	const dotted = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(literal);
+	if (dotted) {
+		const v4 = parseIpv4(dotted[1]);
+		return v4 ? isPrivateIpv4(v4) : true;
+	}
+	const hex = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(literal);
+	if (hex) {
+		const hi = Number.parseInt(hex[1], 16);
+		const lo = Number.parseInt(hex[2], 16);
+		return isPrivateIpv4([hi >> 8, hi & 0xff, lo >> 8, lo & 0xff]);
+	}
+	const first = Number.parseInt(literal.split(":")[0] || "0", 16);
+	if (Number.isNaN(first)) return true;
+	if ((first & 0xfe00) === 0xfc00) return true; // fc00::/7 unique local
+	if ((first & 0xffc0) === 0xfe80) return true; // fe80::/10 link-local
+	if ((first & 0xff00) === 0xff00) return true; // ff00::/8 multicast
+	return false;
+};
+
+/** True for hosts that name the collector itself or a non-public network. */
+export const isPrivateWebhookHost = (hostname: string): boolean => {
+	const host = hostname.toLowerCase().replace(/\.$/, "");
+	if (!host) return true;
+	if (host === "localhost" || host.endsWith(".localhost")) return true;
+	if (host.endsWith(".local") || host.endsWith(".internal")) return true;
+	if (host.startsWith("[")) return isPrivateIpv6(host);
+	const v4 = parseIpv4(host);
+	if (v4) return isPrivateIpv4(v4);
+	// Bare hostnames without a dot are almost always intranet names.
+	return !host.includes(".");
+};
+
+/** Parse and validate a webhook destination; returns the normalized URL. */
+export const assertSafeWebhookUrl = (
+	raw: string,
+	options: WebhookUrlOptions = {},
+): URL => {
+	let url: URL;
+	try {
+		url = new URL(raw);
+	} catch {
+		throw new WebhookUrlError("channel.url must be an absolute http(s) URL");
+	}
+	if (url.protocol !== "https:" && url.protocol !== "http:") {
+		throw new WebhookUrlError("channel.url must be http(s)://");
+	}
+	if (url.username || url.password) {
+		throw new WebhookUrlError(
+			"channel.url must not embed credentials; use channel.headers",
+		);
+	}
+	if (!options.allowPrivateNetworks && isPrivateWebhookHost(url.hostname)) {
+		throw new WebhookUrlError(
+			`channel.url host "${url.hostname}" is not a public address (set ALERT_WEBHOOK_ALLOW_PRIVATE=true to allow internal receivers)`,
+		);
+	}
+	return url;
+};
+
+/** Resolve a hostname to its addresses (e.g. `dns.lookup(host, { all: true })`). */
+export type HostLookup = (hostname: string) => Promise<string[]>;
+
+export type HostResolution = "public" | "private" | "unresolvable";
+
+/**
+ * Second line of defence for hosts that pass the literal check: resolve the
+ * name and classify it. Any private / loopback / link-local answer makes it
+ * `private`. A lookup that throws or returns nothing is `unresolvable` —
+ * callers must treat that as a refusal too (fail closed): the delivery
+ * fetch resolves on its own, so "we could not see where this goes" must
+ * never become "send it anyway". The fetch's own resolution can still
+ * differ from this one (rebinding); that window is not closed here.
+ */
+export const resolveWebhookHost = async (
+	hostname: string,
+	lookup: HostLookup,
+): Promise<HostResolution> => {
+	// Literals were already judged by isPrivateWebhookHost.
+	if (parseIpv4(hostname) || hostname.startsWith("[")) return "public";
+	let addresses: string[];
+	try {
+		addresses = await lookup(hostname);
+	} catch {
+		return "unresolvable";
+	}
+	if (addresses.length === 0) return "unresolvable";
+	const isPrivate = addresses.some((address) => {
+		const v4 = parseIpv4(address);
+		if (v4) return isPrivateIpv4(v4);
+		return isPrivateIpv6(`[${address}]`);
+	});
+	return isPrivate ? "private" : "public";
+};
+
+/**
+ * Origin only. Any path segment or query parameter of a webhook URL may be
+ * the credential (Slack nests it, generic receivers often use a single
+ * token segment), so nothing past the host is ever rendered; `/…` marks
+ * that a path existed. Userinfo is dropped with the rest.
+ */
+export const redactWebhookUrl = (raw: string): string => {
+	try {
+		const url = new URL(raw);
+		const hasPath = url.pathname !== "/" || url.search !== "";
+		return `${url.protocol}//${url.host}${hasPath ? "/…" : ""}`;
+	} catch {
+		return "<invalid url>";
+	}
+};

package/src/migrations/041_ingest_key_hash_text.sql

@@ -0,0 +1,5 @@
+-- No-op on SQLite/D1: ingest_keys.key_hash has been TEXT since 009.
+-- Present only to keep migration numbering in parity with
+-- migrations-postgres/041_ingest_key_hash_text.sql, which converts the
+-- Postgres column from BYTEA to TEXT.
+SELECT 1;

package/src/migrations/042_session_replay_project_composite_pk.sql

@@ -0,0 +1,52 @@
+-- Migration 042: session replay composite primary key (project_id, session_id)
+--
+-- Fixes cross-project session replay metadata collisions. Previously,
+-- session_replay_metadata used session_id alone as primary key while
+-- project_id was added in migration 010. Replay receiver upserts used
+-- ON CONFLICT(session_id), causing identical session IDs from different
+-- projects to overwrite or increment each other.
+--
+-- This forward migration recreates the table with PRIMARY KEY (project_id, session_id),
+-- preserving all existing rows and column defaults, and recreates associated indexes.
+
+DROP TABLE IF EXISTS session_replay_metadata_new;
+
+CREATE TABLE session_replay_metadata_new (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  session_id TEXT NOT NULL,
+  visitor_id TEXT NOT NULL,
+  first_chunk_at TEXT NOT NULL,
+  last_chunk_at TEXT NOT NULL,
+  chunk_count INTEGER DEFAULT 0,
+  events_count INTEGER DEFAULT 0,
+  storage_bytes INTEGER NOT NULL DEFAULT 0,
+  PRIMARY KEY (project_id, session_id)
+);
+
+INSERT INTO session_replay_metadata_new (
+  project_id,
+  session_id,
+  visitor_id,
+  first_chunk_at,
+  last_chunk_at,
+  chunk_count,
+  events_count,
+  storage_bytes
+)
+SELECT
+  COALESCE(project_id, 'default'),
+  session_id,
+  visitor_id,
+  first_chunk_at,
+  last_chunk_at,
+  COALESCE(chunk_count, 0),
+  COALESCE(events_count, 0),
+  COALESCE(storage_bytes, 0)
+FROM session_replay_metadata;
+
+DROP TABLE session_replay_metadata;
+
+ALTER TABLE session_replay_metadata_new RENAME TO session_replay_metadata;
+
+CREATE INDEX IF NOT EXISTS idx_replay_visitor ON session_replay_metadata (visitor_id);
+CREATE INDEX IF NOT EXISTS idx_replay_project_last ON session_replay_metadata (project_id, last_chunk_at DESC);

package/src/migrations/043_metric_exemplars_text_timestamps.sql

@@ -0,0 +1,5 @@
+-- No-op on SQLite/D1: metric_exemplars.received_at and expires_at have been
+-- TEXT since 035. Present only to keep migration numbering in parity with
+-- migrations-postgres/043_metric_exemplars_text_timestamps.sql, which
+-- converts the Postgres columns from TIMESTAMPTZ to ISO TEXT.
+SELECT 1;

package/src/migrations/044_actions_started_at_index.sql

@@ -0,0 +1,10 @@
+-- RFC 0010 — action graph retention.
+--
+-- The hourly retention sweep expires action trees by `actions.started_at`
+-- (the spine has no `expires_at` column). Existing indexes all lead with
+-- `project_id` plus another key, so a cross-project "oldest actions first"
+-- scan would read the whole table every hour. This index lets the sweep
+-- pick its next batch of candidates in index order.
+
+CREATE INDEX IF NOT EXISTS idx_actions_started_at
+  ON actions (started_at);

package/src/migrations/045_user_profiles_project_composite_pk.sql

@@ -0,0 +1,40 @@
+-- Migration 045: user_profiles composite primary key (project_id, user_id)
+--
+-- user_profiles was created in 006 with `user_id TEXT PRIMARY KEY` and only
+-- gained project_id in 010. /v1/identify upserted on ON CONFLICT(user_id),
+-- so an ingest key for one project could read or rewrite another project's
+-- profile by posting the same userId (commonly an email address).
+--
+-- Rebuilds the table with PRIMARY KEY (project_id, user_id), preserving
+-- every row and the two indexes. The old key was unique, so the composite
+-- key cannot collide. Re-running the file is a no-op in effect.
+
+DROP TABLE IF EXISTS user_profiles_new;
+
+CREATE TABLE user_profiles_new (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  user_id TEXT NOT NULL,
+  visitor_id TEXT NOT NULL,
+  email TEXT,
+  name TEXT,
+  properties_json TEXT,
+  first_seen_at TEXT NOT NULL,
+  last_seen_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, user_id)
+);
+
+INSERT INTO user_profiles_new (
+  project_id, user_id, visitor_id, email, name, properties_json,
+  first_seen_at, last_seen_at
+)
+SELECT
+  COALESCE(project_id, 'default'), user_id, visitor_id, email, name,
+  properties_json, first_seen_at, last_seen_at
+FROM user_profiles;
+
+DROP TABLE user_profiles;
+
+ALTER TABLE user_profiles_new RENAME TO user_profiles;
+
+CREATE INDEX IF NOT EXISTS idx_user_visitor ON user_profiles (visitor_id);
+CREATE INDEX IF NOT EXISTS idx_users_project_last ON user_profiles (project_id, last_seen_at DESC);