@obsunified/collector 2.2.0 → 2.2.1

package/src/lib/analyses-runner.ts

@@ -36,7 +36,11 @@ import {
 	type Logger,
 } from "../framework/logger";
 import { AnalysesStore } from "./analyses-store";
-import { openOrDeduplicateIncident } from "./autosre-remediation-gate";
+import {
+	evaluateRemediationWatchdogs,
+	openOrDeduplicateIncident,
+	resolveServiceIncidentTarget,
+} from "./autosre-remediation-gate";
 import { generateNarrative, LlmCallError, type LlmConfig } from "./llm";
 import {
 	computeSignature,
@@ -292,6 +296,20 @@ export async function runAllDueAnalyses(
 		totals.narrated += summary.narrated;
 		totals.skipped += summary.skipped ?? 0;
 	}
+
+	// RFC 0013 § 5.B: the post-remediation watchdog (grant/claim timeouts and the
+	// +2m/+5m/+15m recovery checkpoints) rides the 1-minute tick so checkpoints
+	// are evaluated close to when they come due. Cross-project by design.
+	try {
+		const wd = await evaluateRemediationWatchdogs(db, { logger });
+		if (wd.evaluated + wd.timedOutClaims + wd.expiredGrants + wd.failed > 0) {
+			logger.info("[analyses] remediation watchdog summary", { ...wd });
+		}
+	} catch (error) {
+		logger.error("[analyses] remediation watchdog failed", {
+			error: error instanceof Error ? error.message : String(error),
+		});
+	}
 	return totals;
 }
 
@@ -477,12 +495,31 @@ async function runDueAnalysesForProject(
 						}
 						await store.insertResult(result, expiresAt);
 						if (result.status === "critical") {
-							await openOrDeduplicateIncident(db, projectId, {
-								environment: "default",
-								targetNodeId: `analysis:${def.id}`,
-								symptomAnchorKind: "analysis",
-								symptomAnchorId: def.id,
-							}).catch(() => undefined);
+							try {
+								const scopeService = def.scope?.service;
+								const target = await resolveServiceIncidentTarget(
+									db,
+									projectId,
+									typeof scopeService === "string" ? scopeService : null,
+									`analysis:${def.id}`,
+									logger,
+								);
+								await openOrDeduplicateIncident(db, projectId, {
+									environment: target.environment,
+									targetNodeId: target.targetNodeId,
+									symptomAnchorKind: "analysis",
+									symptomAnchorId: def.id,
+								});
+							} catch (incidentError) {
+								// Incident bookkeeping must not fail the analysis run.
+								logger.error("[analyses] failed to open incident", {
+									analysis_id: def.id,
+									error:
+										incidentError instanceof Error
+											? incidentError.message
+											: String(incidentError),
+								});
+							}
 						}
 						// markRan releases the claim as a side-effect.
 						await store.markRan(projectId, def.id, result.generatedAt);

package/src/lib/autosre-change-correlation.ts

@@ -12,6 +12,7 @@ import type {
 	RelationshipCertainty,
 } from "@obsunified/types";
 import { buildCanonicalNodeId } from "@obsunified/types";
+import { normalizeEnvironment } from "@obsunified/types/lineage";
 import { randomHex } from "./hash";
 import { parseJsonArray, parseJsonRecord } from "./json";
 import type { SqlDb } from "./sql-db";
@@ -73,14 +74,30 @@ export const mapChangeEventRow = (row: ChangeEventRow): ChangeEvent => ({
 	expiresAt: row.expires_at,
 });
 
+export class InvalidTimestampError extends Error {
+	constructor(field: string, value: string) {
+		super(`${field} must be a valid ISO-8601 timestamp, got "${value}"`);
+		this.name = "InvalidTimestampError";
+	}
+}
+
+/** Canonicalizes a client timestamp to `toISOString()` form so string comparison in SQL is sound. */
+const normalizeTimestamp = (field: string, value: string): string => {
+	const ms = new Date(value).getTime();
+	if (!Number.isFinite(ms)) throw new InvalidTimestampError(field, value);
+	return new Date(ms).toISOString();
+};
+
 export async function recordChangeEvent(
 	db: SqlDb,
 	projectId: string,
 	input: ChangeEventInput,
 ): Promise<ChangeEvent> {
 	const nowIso = new Date().toISOString();
-	const occurredAt = input.occurredAt || nowIso;
-	const environment = (input.environment || "default").trim().toLowerCase();
+	const occurredAt = input.occurredAt
+		? normalizeTimestamp("occurredAt", input.occurredAt)
+		: nowIso;
+	const environment = normalizeEnvironment(input.environment);
 	const namespace = (input.namespace || "default").trim();
 	const serviceName = input.serviceName?.trim() || null;
 
@@ -257,7 +274,7 @@ export async function listChangeEvents(
 	}
 	if (options.since) {
 		conditions.push("occurred_at >= ?");
-		binds.push(options.since);
+		binds.push(normalizeTimestamp("since", options.since));
 	}
 
 	const limit = Math.max(1, Math.min(200, options.limit ?? 50));
@@ -346,8 +363,10 @@ const summarizeCohort = (
 		requestCount > 0 ? downstreamCount / requestCount : 0;
 	const avgDownstreamDurationMs =
 		downstreamCount > 0
-			? downstreamSpans.reduce((acc, s) => acc + Number(s.duration_ms ?? 0), 0) /
-				downstreamCount
+			? downstreamSpans.reduce(
+					(acc, s) => acc + Number(s.duration_ms ?? 0),
+					0,
+				) / downstreamCount
 			: null;
 
 	return {
@@ -433,7 +452,9 @@ export async function findCrossTraceContention(
 			}>(),
 	]);
 
-	const isScoped = nodeRow ? nodeRow.is_scoped_identity === 1 : !options.targetNodeId.includes(":unscoped/");
+	const isScoped = nodeRow
+		? nodeRow.is_scoped_identity === 1
+		: !options.targetNodeId.includes(":unscoped/");
 	const relationshipCertainty: RelationshipCertainty = isScoped
 		? "exact"
 		: "inferred";
@@ -455,7 +476,11 @@ export async function findCrossTraceContention(
 const extractFilePathsFromSpan = (span: SpanCohortSampleRow): string[] => {
 	const found: string[] = [];
 	const attrs = parseJsonRecord(span.attributes_json ?? "{}");
-	for (const key of ["code.filepath", "obs.code.filepath", "exception.stacktrace"]) {
+	for (const key of [
+		"code.filepath",
+		"obs.code.filepath",
+		"exception.stacktrace",
+	]) {
 		const val = attrs[key];
 		if (typeof val === "string") {
 			found.push(val);
@@ -556,9 +581,9 @@ export async function computeChangeBlastRadius(
 
 	const flagKey =
 		change.kind === "feature_flag"
-			? (typeof change.attributes.flagKey === "string"
+			? ((typeof change.attributes.flagKey === "string"
 					? change.attributes.flagKey
-					: change.serviceName) ?? "flag"
+					: change.serviceName) ?? "flag")
 			: null;
 
 	const newCohortKey =
@@ -577,7 +602,10 @@ export async function computeChangeBlastRadius(
 			const attrs = parseJsonRecord(s.attributes_json ?? "{}");
 			const direct = attrs[`obs.feature_flags.${flagKey}`];
 			if (direct !== undefined && direct !== null) return String(direct);
-			if (attrs["feature_flag.key"] === flagKey && attrs["feature_flag.variant"] !== undefined) {
+			if (
+				attrs["feature_flag.key"] === flagKey &&
+				attrs["feature_flag.variant"] !== undefined
+			) {
 				return String(attrs["feature_flag.variant"]);
 			}
 			const nested = attrs["obs.feature_flags"];
@@ -732,8 +760,7 @@ export async function computeChangeBlastRadius(
 	} else if (preBaseline.requestCount >= minSample) {
 		// Instant 100% cutover: compare post-change v_new vs pre-change baseline
 		rateDifference = newCohort.errorRate - preBaseline.errorRate;
-		relativeRisk =
-			newCohort.errorRate / Math.max(preBaseline.errorRate, 0.001);
+		relativeRisk = newCohort.errorRate / Math.max(preBaseline.errorRate, 0.001);
 		const p95Ratio =
 			newCohort.p95DurationMs !== null &&
 			preBaseline.p95DurationMs !== null &&

package/src/lib/autosre-findings.ts

@@ -12,7 +12,12 @@ import type {
 	RecoveryStatus,
 	TelemetryCodeReference,
 } from "@obsunified/types";
-import { buildCanonicalNodeId, buildFindingFingerprint } from "@obsunified/types";
+import {
+	buildCanonicalNodeId,
+	buildFindingFingerprint,
+} from "@obsunified/types";
+import { consoleLogger, type Logger } from "../framework/logger";
+import { computeChangeBlastRadius } from "./autosre-change-correlation";
 import { randomHex } from "./hash";
 import { parseJsonArray, parseJsonRecord } from "./json";
 import type { SqlDb } from "./sql-db";
@@ -148,14 +153,15 @@ export const mapFindingRow = (row: FindingRow): InvestigationFinding => {
 		cohortVerdict:
 			(row.cohort_verdict as CohortAttributionVerdict | null) ?? null,
 		contendingTraceId: row.contending_trace_id,
-		causalChain: parseJsonArray(row.causal_chain_json) as unknown as CausalTriple[],
+		causalChain: parseJsonArray(
+			row.causal_chain_json,
+		) as unknown as CausalTriple[],
 		citedEvidenceIds: parseJsonArray(row.cited_evidence_ids_json).map(String),
 		codeReferences: parseJsonArray(
 			row.code_references_json ?? "[]",
 		) as unknown as TelemetryCodeReference[],
 		counterfactualProof,
-		recoveryStatus: (row.recovery_status ||
-			"not_applicable") as RecoveryStatus,
+		recoveryStatus: (row.recovery_status || "not_applicable") as RecoveryStatus,
 		causalVerificationStatus: (row.causal_verification_status ||
 			"unverified") as CausalVerificationStatus,
 		watchdogStep: Number(row.watchdog_step ?? 0),
@@ -174,17 +180,24 @@ export const mapFindingRow = (row: FindingRow): InvestigationFinding => {
 };
 
 /**
- * Enforces P1 #1: Structural overlap (`touched_same_code_path` or `originated_fault_in_span`)
- * can never be promoted to `cohort_rate_isolated` or `counterfactual_verified` without
- * actual cohort isolation or SQL-verified counterfactual proof.
+ * Enforces P1 #1: the causal proof tier is server-derived. Nothing the client
+ * asserts (`cohortVerdict`, `causalProofTier`, an `isolated_to_version_cohort`
+ * triple) can raise the tier above `corroborated_candidate`. Only a verdict the
+ * collector recomputed itself from telemetry (`serverCohortVerdict`, produced by
+ * `computeChangeBlastRadius`) may yield `cohort_rate_isolated`, and only
+ * `verifyPairedCounterfactualProof` can reach `counterfactual_verified`.
  */
 function resolveSafeCausalProofTier(
 	input: InvestigationFindingInput,
+	serverCohortVerdict: CohortAttributionVerdict | null,
 ): CausalProofTier {
-	if (input.cohortVerdict === "isolated_to_new_version") {
+	if (serverCohortVerdict === "isolated_to_new_version") {
 		return "cohort_rate_isolated";
 	}
 	if (
+		serverCohortVerdict === "sequential_post_cutover_regression" ||
+		serverCohortVerdict === "shared_degradation_new_version_aggressor" ||
+		input.cohortVerdict === "isolated_to_new_version" ||
 		input.cohortVerdict === "sequential_post_cutover_regression" ||
 		input.cohortVerdict === "shared_degradation_new_version_aggressor"
 	) {
@@ -195,17 +208,12 @@ function resolveSafeCausalProofTier(
 			t.predicate === "touched_same_code_path" ||
 			t.predicate === "contended_on_shared_resource" ||
 			t.predicate === "induced_shared_resource_contention" ||
-			t.predicate === "degraded_after_change",
+			t.predicate === "degraded_after_change" ||
+			t.predicate === "isolated_to_version_cohort",
 	);
-	if (input.causalProofTier === "cohort_rate_isolated") {
-		// Only allow cohort_rate_isolated if a change or isolated_to_version_cohort triple is present
-		const hasIsolatedTriple = input.causalChain.some(
-			(t) => t.predicate === "isolated_to_version_cohort",
-		);
-		if (hasIsolatedTriple) return "cohort_rate_isolated";
-	}
 	if (
 		hasCodeOverlapOrContention ||
+		input.causalProofTier === "cohort_rate_isolated" ||
 		input.causalProofTier === "corroborated_candidate"
 	) {
 		return "corroborated_candidate";
@@ -217,7 +225,9 @@ export async function recordInvestigationFinding(
 	db: SqlDb,
 	projectId: string,
 	input: InvestigationFindingInput,
+	options: { logger?: Logger } = {},
 ): Promise<InvestigationFinding> {
+	const logger = options.logger ?? consoleLogger;
 	const nowIso = new Date().toISOString();
 	const environment = (
 		input.environment ??
@@ -256,7 +266,40 @@ export async function recordInvestigationFinding(
 		.bind(projectId, environment, fpResult.fingerprint)
 		.first<{ id: string; causal_verification_status: string }>();
 
-	const causalProofTier = resolveSafeCausalProofTier(input);
+	// Recompute the cohort verdict server-side; never trust the client's claim.
+	let serverCohortVerdict: CohortAttributionVerdict | null = null;
+	if (input.contributingChangeId) {
+		const blastRadius = await computeChangeBlastRadius(
+			db,
+			projectId,
+			input.contributingChangeId,
+			{ environment },
+		);
+		serverCohortVerdict = blastRadius?.cohortVerdict ?? null;
+	}
+	const causalProofTier = resolveSafeCausalProofTier(
+		input,
+		serverCohortVerdict,
+	);
+	if (
+		input.cohortVerdict === "isolated_to_new_version" &&
+		serverCohortVerdict !== "isolated_to_new_version"
+	) {
+		logger.warn(
+			"[autosre] client-asserted cohortVerdict=isolated_to_new_version was not corroborated server-side; capping proof tier",
+			{
+				projectId,
+				contributingChangeId: input.contributingChangeId ?? null,
+				serverCohortVerdict,
+			},
+		);
+	}
+	// An unverified client claim of isolation is not persisted as a verdict.
+	const storedCohortVerdict =
+		serverCohortVerdict ??
+		(input.cohortVerdict === "isolated_to_new_version"
+			? null
+			: (input.cohortVerdict ?? null));
 
 	const relationshipScore = input.causalChain.some(
 		(c) => c.relationshipCertainty === "exact",
@@ -304,9 +347,7 @@ export async function recordInvestigationFinding(
 	};
 
 	const id = input.id?.trim() || `fnd_${randomHex(10)}`;
-	const expiresAt = new Date(
-		Date.now() + 90 * 24 * 3600_000,
-	).toISOString();
+	const expiresAt = new Date(Date.now() + 90 * 24 * 3600_000).toISOString();
 
 	await db
 		.prepare(
@@ -352,7 +393,7 @@ export async function recordInvestigationFinding(
 			input.rootCauseEntityKind ?? null,
 			input.rootCauseEntityId ?? null,
 			input.contributingChangeId ?? null,
-			input.cohortVerdict ?? null,
+			storedCohortVerdict,
 			input.contendingTraceId ?? null,
 			JSON.stringify(input.causalChain),
 			JSON.stringify(input.citedEvidenceIds),
@@ -429,9 +470,7 @@ export async function searchInvestigationFindings(
 		binds.push(options.symptomEntityId);
 	}
 	if (options.rootCauseEntityId) {
-		conditions.push(
-			"(root_cause_entity_id = ? OR fault_locus_entity_id = ?)",
-		);
+		conditions.push("(root_cause_entity_id = ? OR fault_locus_entity_id = ?)");
 		binds.push(options.rootCauseEntityId, options.rootCauseEntityId);
 	}
 	if (options.causalVerificationStatus) {
@@ -443,7 +482,7 @@ export async function searchInvestigationFindings(
 		binds.push(options.incidentId);
 	}
 
-	const limit = Math.max(1, Math.min(100, options.limit ?? 25));
+	const limit = Math.max(1, Math.min(200, options.limit ?? 25));
 	binds.push(limit);
 
 	const rows = await db
@@ -575,7 +614,8 @@ export async function verifyPairedCounterfactualProof(
 	const failingPassed =
 		failingRow.result_passed !== null
 			? Number(failingRow.result_passed) === 1
-			: Number(failingRow.fail_count) === 0 && Number(failingRow.pass_count) > 0;
+			: Number(failingRow.fail_count) === 0 &&
+				Number(failingRow.pass_count) > 0;
 	if (failingPassed) {
 		throw new CounterfactualVerificationError(
 			`Baseline run ${input.failingRunId} did not fail (passed=1)`,
@@ -590,8 +630,7 @@ export async function verifyPairedCounterfactualProof(
 		);
 	}
 
-	const failingFp =
-		failingRow.result_failure_fp ?? failingRow.run_failure_fp;
+	const failingFp = failingRow.result_failure_fp ?? failingRow.run_failure_fp;
 	if (failingFp !== finding.fingerprint) {
 		throw new CounterfactualVerificationError(
 			`Failing run fingerprint (${failingFp ?? "null"}) does not match finding fingerprint (${finding.fingerprint})`,
@@ -627,8 +666,7 @@ export async function verifyPairedCounterfactualProof(
 				`Candidate run ${passRunId} did not pass`,
 			);
 		}
-		const passInputHash =
-			passRow.result_input_hash ?? passRow.run_input_hash;
+		const passInputHash = passRow.result_input_hash ?? passRow.run_input_hash;
 		if (passInputHash !== failingInputHash) {
 			throw new CounterfactualVerificationError(
 				`Candidate run ${passRunId} input_payload_hash (${passInputHash ?? "null"}) does not match failing run (${failingInputHash})`,

package/src/lib/autosre-ontology-cache.test.ts

@@ -0,0 +1,78 @@
+import type { StoredSpan } from "@obsunified/types";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+	clearOntologyUpsertCache,
+	extractSpanLineageAndTopology,
+	upsertOntologyFromSpans,
+} from "./autosre-ontology";
+import type { SqlDb } from "./sql-db";
+
+const dbSpan = (i: number, environment = "prod"): StoredSpan =>
+	({
+		projectId: "p",
+		traceId: `t${i}`,
+		spanId: `s${i}`,
+		environment,
+		startTime: "2026-10-15T00:00:00.000Z",
+		receivedAt: "2026-10-15T00:00:00.000Z",
+		attributesJson: JSON.stringify({
+			"db.system": "postgresql",
+			"db.name": `db${i}`,
+			"server.address": "pg.internal",
+			"server.port": 5432,
+		}),
+		resourceAttributesJson: "{}",
+	}) as unknown as StoredSpan;
+
+const fakeDb = (batch: SqlDb["batch"]): SqlDb => {
+	const stmt = { bind: () => stmt };
+	return { prepare: () => stmt, batch } as unknown as SqlDb;
+};
+
+describe("ontology env normalization", () => {
+	afterEach(() => clearOntologyUpsertCache());
+
+	it("maps production-tagged spans to the prod environment", () => {
+		const span = {
+			...dbSpan(1, ""),
+			environment: undefined,
+			resourceAttributesJson: JSON.stringify({
+				"deployment.environment.name": "Production",
+			}),
+		} as unknown as StoredSpan;
+		expect(extractSpanLineageAndTopology(span).environment).toBe("prod");
+	});
+});
+
+describe("NODE_UPSERT_CACHE", () => {
+	afterEach(() => clearOntologyUpsertCache());
+
+	it("does not remember keys when the batch fails", async () => {
+		const failing = vi.fn(async () => {
+			throw new Error("boom");
+		});
+		await expect(
+			upsertOntologyFromSpans(fakeDb(failing as never), [dbSpan(1)]),
+		).rejects.toThrow("boom");
+		const ok = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(ok as never), [dbSpan(1)]);
+		expect(ok).toHaveBeenCalledTimes(1);
+		// Now cached: a third call writes nothing.
+		const again = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(again as never), [dbSpan(1)]);
+		expect(again).not.toHaveBeenCalled();
+	});
+
+	it("caps the cache and evicts the oldest entries", async () => {
+		const spans = Array.from({ length: 5_002 }, (_, i) => dbSpan(i));
+		const batch = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(batch as never), spans);
+		// Oldest keys were evicted so they are written again; newest stay cached.
+		const second = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(second as never), [dbSpan(0)]);
+		expect(second).toHaveBeenCalledTimes(1);
+		const third = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(third as never), [dbSpan(5_001)]);
+		expect(third).not.toHaveBeenCalled();
+	});
+});

package/src/lib/autosre-ontology.ts

@@ -9,7 +9,11 @@ import type {
 	RelationshipCertainty,
 	StoredSpan,
 } from "@obsunified/types";
-import { buildCanonicalNodeId, normalizeErrorSignature } from "@obsunified/types";
+import {
+	buildCanonicalNodeId,
+	normalizeErrorSignature,
+} from "@obsunified/types";
+import { normalizeEnvironment } from "@obsunified/types/lineage";
 import { randomHex } from "./hash";
 import { parseJsonArray, parseJsonRecord } from "./json";
 import type { SqlDb, SqlStatement } from "./sql-db";
@@ -53,22 +57,22 @@ export function extractSpanLineageAndTopology(
 	const attrs = parseJsonRecord(span.attributesJson);
 	const resource = parseJsonRecord(span.resourceAttributesJson);
 
-	const environment = (
+	const environment = normalizeEnvironment(
 		span.environment ??
-		pickStr(
-			resource,
-			"deployment.environment.name",
-			"deployment.environment",
-			"obs.environment",
-		) ??
-		pickStr(
-			attrs,
-			"deployment.environment.name",
-			"deployment.environment",
-			"obs.environment",
-		) ??
-		"default"
-	).toLowerCase();
+			pickStr(
+				resource,
+				"deployment.environment.name",
+				"deployment.environment",
+				"obs.environment",
+			) ??
+			pickStr(
+				attrs,
+				"deployment.environment.name",
+				"deployment.environment",
+				"obs.environment",
+			) ??
+			undefined,
+	);
 
 	const namespace =
 		pickStr(resource, "service.namespace") ??
@@ -144,11 +148,8 @@ export function extractSpanLineageAndTopology(
 			environment,
 			messagingSystem,
 			messagingDestination:
-				pickStr(
-					attrs,
-					"messaging.destination.name",
-					"messaging.destination",
-				) ?? "default",
+				pickStr(attrs, "messaging.destination.name", "messaging.destination") ??
+				"default",
 			serverAddress,
 		});
 		edgePredicate =
@@ -207,8 +208,7 @@ export function extractSpanLineageAndTopology(
 								? (evObj.attributes as Record<string, JsonValue>)
 								: {};
 						rawErr =
-							pickStr(evAttrs, "exception.type", "exception.message") ??
-							rawErr;
+							pickStr(evAttrs, "exception.type", "exception.message") ?? rawErr;
 					}
 				}
 			}
@@ -236,6 +236,20 @@ export function extractSpanLineageAndTopology(
 // 60-second isolate cache to avoid repeating identical ontology node upserts on every span
 const NODE_UPSERT_CACHE = new Map<string, number>();
 const CACHE_TTL_MS = 60_000;
+const NODE_UPSERT_CACHE_MAX = 5_000;
+
+function rememberUpserts(keys: Iterable<string>, nowMs: number): void {
+	for (const key of keys) {
+		// Re-insert so Map iteration order tracks recency; evict oldest past the cap.
+		NODE_UPSERT_CACHE.delete(key);
+		NODE_UPSERT_CACHE.set(key, nowMs);
+	}
+	while (NODE_UPSERT_CACHE.size > NODE_UPSERT_CACHE_MAX) {
+		const oldest = NODE_UPSERT_CACHE.keys().next().value;
+		if (oldest === undefined) break;
+		NODE_UPSERT_CACHE.delete(oldest);
+	}
+}
 
 export function clearOntologyUpsertCache(): void {
 	NODE_UPSERT_CACHE.clear();
@@ -254,6 +268,8 @@ export async function upsertOntologyFromSpans(
 
 	const nowMs = Date.now();
 	const statements: SqlStatement[] = [];
+	// Cache keys are only remembered once the batch has been written successfully.
+	const pendingCacheKeys = new Set<string>();
 	const edgeAggregates = new Map<
 		string,
 		{
@@ -293,8 +309,11 @@ export async function upsertOntologyFromSpans(
 		if (lineage.sourceNode) {
 			const cacheKey = `${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}::${lineage.serviceVersion ?? ""}::${lineage.commitSha ?? ""}::${lineage.configDigest ?? ""}`;
 			const cachedAt = NODE_UPSERT_CACHE.get(cacheKey);
-			if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
-				NODE_UPSERT_CACHE.set(cacheKey, nowMs);
+			if (
+				!pendingCacheKeys.has(cacheKey) &&
+				(options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS)
+			) {
+				pendingCacheKeys.add(cacheKey);
 				serviceNodesToCheck.set(
 					`${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}`,
 					{
@@ -313,8 +332,11 @@ export async function upsertOntologyFromSpans(
 		if (lineage.targetNode) {
 			const targetCacheKey = `${span.projectId}::${lineage.environment}::${lineage.targetNode.id}`;
 			const cachedAt = NODE_UPSERT_CACHE.get(targetCacheKey);
-			if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
-				NODE_UPSERT_CACHE.set(targetCacheKey, nowMs);
+			if (
+				!pendingCacheKeys.has(targetCacheKey) &&
+				(options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS)
+			) {
+				pendingCacheKeys.add(targetCacheKey);
 				statements.push(
 					db
 						.prepare(
@@ -511,6 +533,7 @@ export async function upsertOntologyFromSpans(
 	if (statements.length > 0) {
 		await db.batch(statements);
 	}
+	rememberUpserts(pendingCacheKeys, nowMs);
 }
 
 /**
@@ -577,7 +600,8 @@ export function computeTarjanScc(
 		if (lowlink.get(nodeId) === indices.get(nodeId)) {
 			const component: string[] = [];
 			while (stack.length > 0) {
-				const w = stack.pop()!;
+				const w = stack.pop();
+				if (w === undefined) break;
 				onStack.delete(w);
 				component.push(w);
 				if (w === nodeId) break;