@obsunified/collector 2.0.1 → 2.1.1

package/src/lib/identity-index/key-lookups.ts

@@ -53,9 +53,9 @@ export async function manifestBySession(
 			.prepare(
 				`SELECT first_chunk_at, last_chunk_at, chunk_count, events_count
 					FROM session_replay_metadata
-					WHERE session_id = ?`,
+					WHERE project_id = ? AND session_id = ?`,
 			)
-			.bind(sessionId)
+			.bind(projectId, sessionId)
 			.first<{
 				first_chunk_at: string;
 				last_chunk_at: string;

package/src/lib/identity-index/user-lookup.ts

@@ -13,10 +13,13 @@ export async function manifestByUser(
 	const sessionLimit = Math.max(1, Math.min(opts.sessions ?? 5, 20));
 
 	// Step 1 — resolve user_id → visitor_id. user_profiles is the only
-	// table that owns the user_id ↔ visitor_id binding.
+	// table that owns the user_id ↔ visitor_id binding, keyed per project
+	// (the same user_id can exist in several projects — migration 045).
 	const profile = await db
-		.prepare(`SELECT visitor_id FROM user_profiles WHERE user_id = ? LIMIT 1`)
-		.bind(userId)
+		.prepare(
+			`SELECT visitor_id FROM user_profiles WHERE project_id = ? AND user_id = ? LIMIT 1`,
+		)
+		.bind(projectId, userId)
 		.first<{ visitor_id: string }>();
 	if (!profile) {
 		return {
@@ -105,10 +108,10 @@ export async function manifestByUser(
 			.prepare(
 				`SELECT session_id, first_chunk_at, last_chunk_at, chunk_count, events_count
 					FROM session_replay_metadata
-					WHERE session_id IN (${placeholders})
+					WHERE project_id = ? AND session_id IN (${placeholders})
 					ORDER BY last_chunk_at DESC LIMIT 1`,
 			)
-			.bind(...sessionIds)
+			.bind(projectId, ...sessionIds)
 			.first<{
 				session_id: string;
 				first_chunk_at: string;

package/src/lib/identity-index.test.ts

@@ -81,13 +81,12 @@ describe("IdentityIndex.bySession", () => {
 			eventsCount: 42,
 		});
 
-		// All five queries pin (project_id, session_id) — replay lookup uses
-		// just session_id since replay is global.
+		// All five queries pin (project_id, session_id).
 		const allCalls = db.calls.filter((c) => c.op !== "first" || c.binds.length);
 		const sessionQueries = allCalls.filter(
 			(c) => c.sql.includes("session_id") && c.binds.includes("proj-1"),
 		);
-		expect(sessionQueries.length).toBeGreaterThanOrEqual(4);
+		expect(sessionQueries.length).toBe(5);
 	});
 
 	it("returns null replay when session_replay_metadata is empty", async () => {

package/src/lib/metrics-store-series.test.ts

@@ -0,0 +1,99 @@
+/**
+ * Two ingest batches that both see a brand-new metric identity used to race
+ * on `UNIQUE(project_id, identity)`: the loser's whole batch failed and its
+ * points were lost. The insert is now conflict-tolerant and the loser adopts
+ * the winner's series id. Executed against the real migrated schema.
+ */
+
+import { describe, expect, it } from "vitest";
+import type { DecodedMetricPoint } from "../otlp/decode";
+import { MetricsStore } from "./metrics-store";
+import { createMigratedSqliteDb } from "./test-utils/node-sqlite-db";
+
+const point = (over: Partial<DecodedMetricPoint> = {}): DecodedMetricPoint => ({
+	name: "http.server.duration",
+	description: null,
+	unit: "ms",
+	type: "gauge",
+	isMonotonic: null,
+	temporality: null,
+	scopeName: "test",
+	scopeVersion: "1",
+	serviceName: "checkout",
+	resourceAttrsJson: "{}",
+	attributesJson: "{}",
+	identity: "identity-shared",
+	tsNs: "1800000000000000000",
+	startTsNs: null,
+	value: 1,
+	count: null,
+	sum: null,
+	min: null,
+	max: null,
+	boundsJson: null,
+	bucketCountsJson: null,
+	extraJson: null,
+	exemplarsJson: null,
+	...over,
+});
+
+describe("MetricsStore series resolution under concurrency", () => {
+	it("lets concurrent first-sight ingests share one series row", async () => {
+		const db = createMigratedSqliteDb();
+		const store = new MetricsStore(db);
+		const now = new Date().toISOString();
+		const later = new Date(Date.now() + 3_600_000).toISOString();
+
+		// Both calls run their lookup (miss) before either inserts.
+		await Promise.all([
+			store.ingestBatch({
+				projectId: "alpha",
+				points: [point({ value: 1 })],
+				receivedAt: now,
+				expiresAt: later,
+			}),
+			store.ingestBatch({
+				projectId: "alpha",
+				points: [point({ value: 2 })],
+				receivedAt: now,
+				expiresAt: later,
+			}),
+		]);
+
+		const series = await db
+			.prepare(`SELECT id FROM metric_series WHERE project_id = ?`)
+			.bind("alpha")
+			.all<{ id: string }>();
+		expect(series.results).toHaveLength(1);
+		const points = await db
+			.prepare(
+				`SELECT series_id, value FROM metric_point WHERE project_id = ? ORDER BY value`,
+			)
+			.bind("alpha")
+			.all<{ series_id: string; value: number }>();
+		expect(points.results.map((p) => p.value)).toEqual([1, 2]);
+		expect(new Set(points.results.map((p) => p.series_id))).toEqual(
+			new Set([series.results[0]?.id]),
+		);
+		db.close();
+	});
+
+	it("keeps series separate per project", async () => {
+		const db = createMigratedSqliteDb();
+		const store = new MetricsStore(db);
+		const now = new Date().toISOString();
+		for (const projectId of ["alpha", "beta"]) {
+			await store.ingestBatch({
+				projectId,
+				points: [point()],
+				receivedAt: now,
+				expiresAt: now,
+			});
+		}
+		const rows = await db
+			.prepare(`SELECT project_id FROM metric_series ORDER BY project_id`)
+			.all<{ project_id: string }>();
+		expect(rows.results.map((r) => r.project_id)).toEqual(["alpha", "beta"]);
+		db.close();
+	});
+});

package/src/lib/metrics-store.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from "vitest";
 import type { DecodedMetricPoint } from "../otlp/decode";
 import { MetricsStore } from "./metrics-store";
+import { postgresDialect } from "./sql-db";
 import { MemSqlDb } from "./test-utils/mem-sql-db";
 
 const point = (
@@ -33,6 +34,28 @@ const point = (
 });
 
 describe("MetricsStore", () => {
+	it("purges expired exemplars and points with the dialect's text-rendered now()", async () => {
+		const sqlite = new MemSqlDb({ run: () => ({ changes: 2 }) });
+		expect(await new MetricsStore(sqlite).purgeExpired()).toBe(4);
+		expect(sqlite.calls.map((c) => c.sql)).toEqual([
+			"DELETE FROM metric_exemplars WHERE expires_at < strftime('%Y-%m-%dT%H:%M:%fZ', 'now')",
+			"DELETE FROM metric_point WHERE expires_at < strftime('%Y-%m-%dT%H:%M:%fZ', 'now')",
+		]);
+
+		// metric_exemplars.received_at / expires_at are TEXT ISO on Postgres
+		// too (035 for fresh installs, 043 for databases created with the
+		// TIMESTAMPTZ version), so the comparison must be text < text.
+		const postgres = Object.assign(
+			new MemSqlDb({ run: () => ({ changes: 1 }) }),
+			{ dialect: postgresDialect },
+		);
+		expect(await new MetricsStore(postgres).purgeExpired()).toBe(2);
+		expect(postgres.calls.map((c) => c.sql)).toEqual([
+			`DELETE FROM metric_exemplars WHERE expires_at < to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`,
+			`DELETE FROM metric_point WHERE expires_at < to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`,
+		]);
+	});
+
 	it("indexes metric exemplars by trace/span during ingest", async () => {
 		const db = new MemSqlDb({
 			all: (sql, binds) => {

package/src/lib/metrics-store.ts

@@ -272,20 +272,25 @@ export class MetricsStore {
 			}
 		}
 
-		// Insert missing series.
+		// Insert missing series. Two concurrent batches can both miss the
+		// lookup for a brand-new identity; UNIQUE(project_id, identity) would
+		// then fail the loser's whole batch. INSERT OR IGNORE lets the loser
+		// proceed, and the re-select below swaps in the winner's series id.
 		const insertStmt = this.db.prepare(`
-			INSERT INTO metric_series (
+			INSERT OR IGNORE INTO metric_series (
 				id, project_id, name, description, unit, type,
 				is_monotonic, temporality, scope_name, scope_version, service_name,
 				resource_attrs_json, attributes_json, identity, created_at
 			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 		`);
 		const toInsert: SqlStatement[] = [];
+		const attemptedIds = new Set<string>();
 		const createdAt = new Date().toISOString();
 		for (const [identity, p] of byIdentity.entries()) {
 			if (existing.has(identity)) continue;
 			const id = crypto.randomUUID();
 			existing.set(identity, { id, identity });
+			attemptedIds.add(identity);
 			toInsert.push(
 				insertStmt.bind(
 					id,
@@ -306,7 +311,26 @@ export class MetricsStore {
 				),
 			);
 		}
-		if (toInsert.length) await this.db.batch(toInsert);
+		if (toInsert.length === 0) return existing;
+		await this.db.batch(toInsert);
+
+		// Re-read the identities we tried to insert: an ignored insert means
+		// another writer won, and points must reference the stored id.
+		const attempted = identities.filter((identity) =>
+			attemptedIds.has(identity),
+		);
+		for (let i = 0; i < attempted.length; i += 50) {
+			const chunk = attempted.slice(i, i + 50);
+			const placeholders = chunk.map(() => "?").join(",");
+			const rows = await this.db
+				.prepare(
+					`SELECT id, identity FROM metric_series
+					 WHERE project_id = ? AND identity IN (${placeholders})`,
+				)
+				.bind(projectId, ...chunk)
+				.all<SeriesRow>();
+			for (const row of rows.results) existing.set(row.identity, row);
+		}
 		return existing;
 	}
 }

package/src/lib/projects-store.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "vitest";
+import { sha256Hex } from "./hash";
+import { ProjectsStore } from "./projects-store";
+import { createMigratedSqliteDb } from "./test-utils/node-sqlite-db";
+
+describe("ProjectsStore.revokeKey", () => {
+	it("only revokes a key through its own project", async () => {
+		const db = createMigratedSqliteDb();
+		const store = new ProjectsStore(db);
+		const alpha = await store.createProject({ name: "Alpha", slug: "alpha" });
+		const beta = await store.createProject({ name: "Beta", slug: "beta" });
+		const betaKey = await store.createKey(beta.id, "ci");
+		const hash = await sha256Hex(betaKey.key);
+
+		// Alpha's admin cannot revoke beta's key by guessing its id.
+		expect(await store.revokeKey(betaKey.id, alpha.id)).toBe(false);
+		expect((await store.findByKeyHash(hash))?.projectId).toBe(beta.id);
+
+		expect(await store.revokeKey(betaKey.id, beta.id)).toBe(true);
+		expect(await store.findByKeyHash(hash)).toBeNull();
+		// Already revoked: a second revoke reports nothing changed.
+		expect(await store.revokeKey(betaKey.id, beta.id)).toBe(false);
+		db.close();
+	});
+});

package/src/lib/projects-store.ts

@@ -158,12 +158,13 @@ export class ProjectsStore {
 		};
 	}
 
-	async revokeKey(keyId: string): Promise<boolean> {
+	async revokeKey(keyId: string, projectId: string): Promise<boolean> {
 		const result = await this.db
 			.prepare(
-				`UPDATE ingest_keys SET revoked_at = ? WHERE id = ? AND revoked_at IS NULL`,
+				`UPDATE ingest_keys SET revoked_at = ?
+				 WHERE id = ? AND project_id = ? AND revoked_at IS NULL`,
 			)
-			.bind(new Date().toISOString(), keyId)
+			.bind(new Date().toISOString(), keyId, projectId)
 			.run();
 		return (result.meta?.changes ?? 0) > 0;
 	}

package/src/lib/sql-db-dialect.test.ts

@@ -4,16 +4,25 @@ import { postgresDialect, sqliteDialect } from "./sql-db";
 describe("SqlDialect", () => {
 	it("renders native SQLite expressions", () => {
 		expect(sqliteDialect.sinceHours("?")).toBe(
-			"datetime('now', '-' || ? || ' hours')",
+			"strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-' || ? || ' hours')",
 		);
 		expect(
 			sqliteDialect.jsonText("attributes_json", '$."llm.cost.total_usd"'),
 		).toBe(`json_extract(attributes_json, '$."llm.cost.total_usd"')`);
 	});
 
-	it("renders native Postgres expressions without adapter regex translation", () => {
+	it("renders Postgres time windows as ISO text (timestamp columns are TEXT)", () => {
+		// `received_at` / `expires_at` are TEXT on Postgres too, and Postgres
+		// refuses `text >= timestamptz`, so windows must come back as text in
+		// the same `YYYY-MM-DDTHH:MM:SS.mmmZ` shape `Date#toISOString` writes.
 		expect(postgresDialect.sinceHours("$1")).toBe(
-			"(CURRENT_TIMESTAMP - ($1::text || ' hours')::interval)",
+			`to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - ($1::text || ' hours')::interval, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`,
+		);
+		expect(postgresDialect.sinceMinutes("$2")).toBe(
+			`to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - ($2::text || ' minutes')::interval, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`,
+		);
+		expect(postgresDialect.now()).toBe(
+			`to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`,
 		);
 		expect(
 			postgresDialect.jsonText("attributes_json", '$."llm.cost.total_usd"'),