@obsunified/collector 2.2.0 → 2.2.3

package/src/lib/autosre-findings.ts

@@ -12,8 +12,14 @@ import type {
 	RecoveryStatus,
 	TelemetryCodeReference,
 } from "@obsunified/types";
-import { buildCanonicalNodeId, buildFindingFingerprint } from "@obsunified/types";
-import { randomHex } from "./hash";
+import {
+	buildCanonicalNodeId,
+	buildFindingFingerprint,
+	OntologyNodeKindSchema,
+} from "@obsunified/types";
+import { consoleLogger, type Logger } from "../framework/logger";
+import { computeChangeBlastRadius } from "./autosre-change-correlation";
+import { randomHex, sha256Hex } from "./hash";
 import { parseJsonArray, parseJsonRecord } from "./json";
 import type { SqlDb } from "./sql-db";
 
@@ -148,14 +154,15 @@ export const mapFindingRow = (row: FindingRow): InvestigationFinding => {
 		cohortVerdict:
 			(row.cohort_verdict as CohortAttributionVerdict | null) ?? null,
 		contendingTraceId: row.contending_trace_id,
-		causalChain: parseJsonArray(row.causal_chain_json) as unknown as CausalTriple[],
+		causalChain: parseJsonArray(
+			row.causal_chain_json,
+		) as unknown as CausalTriple[],
 		citedEvidenceIds: parseJsonArray(row.cited_evidence_ids_json).map(String),
 		codeReferences: parseJsonArray(
 			row.code_references_json ?? "[]",
 		) as unknown as TelemetryCodeReference[],
 		counterfactualProof,
-		recoveryStatus: (row.recovery_status ||
-			"not_applicable") as RecoveryStatus,
+		recoveryStatus: (row.recovery_status || "not_applicable") as RecoveryStatus,
 		causalVerificationStatus: (row.causal_verification_status ||
 			"unverified") as CausalVerificationStatus,
 		watchdogStep: Number(row.watchdog_step ?? 0),
@@ -174,17 +181,24 @@ export const mapFindingRow = (row: FindingRow): InvestigationFinding => {
 };
 
 /**
- * Enforces P1 #1: Structural overlap (`touched_same_code_path` or `originated_fault_in_span`)
- * can never be promoted to `cohort_rate_isolated` or `counterfactual_verified` without
- * actual cohort isolation or SQL-verified counterfactual proof.
+ * Enforces P1 #1: the causal proof tier is server-derived. Nothing the client
+ * asserts (`cohortVerdict`, `causalProofTier`, an `isolated_to_version_cohort`
+ * triple) can raise the tier above `corroborated_candidate`. Only a verdict the
+ * collector recomputed itself from telemetry (`serverCohortVerdict`, produced by
+ * `computeChangeBlastRadius`) may yield `cohort_rate_isolated`, and only
+ * `verifyPairedCounterfactualProof` can reach `counterfactual_verified`.
  */
 function resolveSafeCausalProofTier(
 	input: InvestigationFindingInput,
+	serverCohortVerdict: CohortAttributionVerdict | null,
 ): CausalProofTier {
-	if (input.cohortVerdict === "isolated_to_new_version") {
+	if (serverCohortVerdict === "isolated_to_new_version") {
 		return "cohort_rate_isolated";
 	}
 	if (
+		serverCohortVerdict === "sequential_post_cutover_regression" ||
+		serverCohortVerdict === "shared_degradation_new_version_aggressor" ||
+		input.cohortVerdict === "isolated_to_new_version" ||
 		input.cohortVerdict === "sequential_post_cutover_regression" ||
 		input.cohortVerdict === "shared_degradation_new_version_aggressor"
 	) {
@@ -195,17 +209,12 @@ function resolveSafeCausalProofTier(
 			t.predicate === "touched_same_code_path" ||
 			t.predicate === "contended_on_shared_resource" ||
 			t.predicate === "induced_shared_resource_contention" ||
-			t.predicate === "degraded_after_change",
+			t.predicate === "degraded_after_change" ||
+			t.predicate === "isolated_to_version_cohort",
 	);
-	if (input.causalProofTier === "cohort_rate_isolated") {
-		// Only allow cohort_rate_isolated if a change or isolated_to_version_cohort triple is present
-		const hasIsolatedTriple = input.causalChain.some(
-			(t) => t.predicate === "isolated_to_version_cohort",
-		);
-		if (hasIsolatedTriple) return "cohort_rate_isolated";
-	}
 	if (
 		hasCodeOverlapOrContention ||
+		input.causalProofTier === "cohort_rate_isolated" ||
 		input.causalProofTier === "corroborated_candidate"
 	) {
 		return "corroborated_candidate";
@@ -213,11 +222,104 @@ function resolveSafeCausalProofTier(
 	return "unverified_locus";
 }
 
+export interface FaultLocusSpanRow {
+	service_name: string | null;
+	span_name: string;
+	status_message: string | null;
+	error_signature?: string | null;
+}
+
+/** Looks up the span a finding's fault locus refers to (newest match wins). */
+export async function loadFaultLocusSpan(
+	db: SqlDb,
+	projectId: string,
+	environment: string,
+	spanId: string,
+	traceId?: string,
+): Promise<FaultLocusSpanRow | null> {
+	if (!spanId) return null;
+	return db
+		.prepare(
+			`SELECT service_name, span_name, status_message, error_signature
+			 FROM telemetry_spans
+			 WHERE project_id = ? AND environment = ? AND span_id = ?
+			 ${traceId ? "AND trace_id = ?" : ""}
+			 ORDER BY start_time DESC
+			 LIMIT 1`,
+		)
+		.bind(
+			...(traceId
+				? [projectId, environment, spanId, traceId]
+				: [projectId, environment, spanId]),
+		)
+		.first<FaultLocusSpanRow>();
+}
+
+/** Fingerprint inputs derivable from a span; empty fields are left undefined. */
+export type SpanFingerprintInput = Pick<
+	FindingFingerprintInput,
+	"environment" | "originatingNodeId" | "downstreamTargetNodeId"
+> &
+	Partial<
+		Pick<FindingFingerprintInput, "normalizedOperation" | "errorSignature">
+	>;
+
+/** True for `kind:env:ns:name...` ids built by buildCanonicalNodeId. */
+function looksLikeCanonicalNodeId(id: string): boolean {
+	const parts = id.split(":");
+	return (
+		parts.length >= 4 &&
+		(OntologyNodeKindSchema.options as readonly string[]).includes(parts[0])
+	);
+}
+
+/**
+ * Splits a `<traceId>:<spanId>` span locus. Returns null for anything that
+ * is a canonical node id or does not have exactly two non-empty parts.
+ */
+function splitTraceSpanLocus(
+	id: string,
+): { traceId: string; spanId: string } | null {
+	if (looksLikeCanonicalNodeId(id)) return null;
+	const parts = id.split(":");
+	if (parts.length !== 2 || !parts[0] || !parts[1]) return null;
+	return { traceId: parts[0], spanId: parts[1] };
+}
+
+/**
+ * Stable recurrence-fingerprint inputs for a fault-locus span: the service node,
+ * the span name as the operation and the span's error signature (falling back
+ * to its status message). No instance ids participate, so two traces failing
+ * the same way on the same service/route share a fingerprint. Used by both
+ * finding creation and the evidence bundle's prior-finding lookup.
+ */
+export function fingerprintInputFromSpan(
+	span: FaultLocusSpanRow,
+	environment: string,
+): SpanFingerprintInput {
+	return {
+		environment,
+		originatingNodeId: buildCanonicalNodeId({
+			kind: "service",
+			environment,
+			serviceName: span.service_name || "unknown-service",
+		}).id,
+		// Empty strings are "absent": returning undefined lets the caller's
+		// `??` chain reach its title/summary fallbacks.
+		normalizedOperation: span.span_name?.trim() || undefined,
+		errorSignature:
+			span.error_signature?.trim() || span.status_message?.trim() || undefined,
+		downstreamTargetNodeId: null,
+	};
+}
+
 export async function recordInvestigationFinding(
 	db: SqlDb,
 	projectId: string,
 	input: InvestigationFindingInput,
+	options: { logger?: Logger } = {},
 ): Promise<InvestigationFinding> {
+	const logger = options.logger ?? consoleLogger;
 	const nowIso = new Date().toISOString();
 	const environment = (
 		input.environment ??
@@ -227,23 +329,86 @@ export async function recordInvestigationFinding(
 		.trim()
 		.toLowerCase();
 
-	const defaultOriginNode = input.faultLocusEntityId.includes(":")
-		? input.faultLocusEntityId
+	// Idempotent retries: one key maps to one finding id per project and
+	// environment, and a replay returns the stored finding untouched.
+	const idempotencyKey = input.idempotencyKey?.trim();
+	let idempotencyId: string | undefined;
+	if (idempotencyKey) {
+		idempotencyId = `fnd_k${(await sha256Hex(`${projectId}\u0000${environment}\u0000${idempotencyKey}`)).slice(0, 20)}`;
+		const replay = await getInvestigationFinding(
+			db,
+			projectId,
+			idempotencyId,
+			environment,
+		);
+		if (replay) return replay;
+	}
+
+	// Findings pointing at a span id (no colon) must not mint a per-incident
+	// fingerprint. Resolve the span to its service/operation/error signature so
+	// every incident on the same service + route + error shares one identity.
+	// Caller-supplied fingerprint fields always win; unknown spans fall back to
+	// treating the locus id as a service name.
+	const supplied: Partial<FindingFingerprintInput> =
+		input.fingerprintInput ?? {};
+	const locusId = input.faultLocusEntityId.trim();
+	let spanDerived: Partial<FindingFingerprintInput> = {};
+	const traceSpan =
+		input.faultLocusEntityKind === "span" ? splitTraceSpanLocus(locusId) : null;
+	if (traceSpan) {
+		const spanRow = await loadFaultLocusSpan(
+			db,
+			projectId,
+			environment,
+			traceSpan.spanId,
+			traceSpan.traceId,
+		);
+		if (spanRow) spanDerived = fingerprintInputFromSpan(spanRow, environment);
+	} else if (
+		!locusId.includes(":") &&
+		input.faultLocusEntityKind !== "service"
+	) {
+		const spanRow = await loadFaultLocusSpan(
+			db,
+			projectId,
+			environment,
+			locusId,
+		);
+		if (spanRow) spanDerived = fingerprintInputFromSpan(spanRow, environment);
+	}
+	const defaultOriginNode = locusId.includes(":")
+		? traceSpan
+			? buildCanonicalNodeId({
+					kind: "service",
+					environment,
+					serviceName: locusId,
+				}).id
+			: locusId
 		: buildCanonicalNodeId({
 				kind: "service",
 				environment,
-				serviceName: input.faultLocusEntityId,
+				serviceName: locusId,
 			}).id;
 
-	const fpResult = buildFindingFingerprint(
-		input.fingerprintInput ?? {
-			environment,
-			originatingNodeId: defaultOriginNode,
-			normalizedOperation: input.title,
-			errorSignature: input.summary.slice(0, 120),
-			downstreamTargetNodeId: null,
-		},
-	);
+	const fpResult = buildFindingFingerprint({
+		environment: supplied.environment ?? environment,
+		originatingNodeId:
+			supplied.originatingNodeId ??
+			spanDerived.originatingNodeId ??
+			defaultOriginNode,
+		normalizedOperation:
+			supplied.normalizedOperation ??
+			spanDerived.normalizedOperation ??
+			input.title,
+		errorSignature:
+			supplied.errorSignature ??
+			spanDerived.errorSignature ??
+			input.summary.slice(0, 120),
+		downstreamTargetNodeId:
+			supplied.downstreamTargetNodeId ??
+			spanDerived.downstreamTargetNodeId ??
+			null,
+	});
 
 	// Check for prior verified findings with the same stable fingerprint
 	const priorRow = await db
@@ -256,7 +421,40 @@ export async function recordInvestigationFinding(
 		.bind(projectId, environment, fpResult.fingerprint)
 		.first<{ id: string; causal_verification_status: string }>();
 
-	const causalProofTier = resolveSafeCausalProofTier(input);
+	// Recompute the cohort verdict server-side; never trust the client's claim.
+	let serverCohortVerdict: CohortAttributionVerdict | null = null;
+	if (input.contributingChangeId) {
+		const blastRadius = await computeChangeBlastRadius(
+			db,
+			projectId,
+			input.contributingChangeId,
+			{ environment },
+		);
+		serverCohortVerdict = blastRadius?.cohortVerdict ?? null;
+	}
+	const causalProofTier = resolveSafeCausalProofTier(
+		input,
+		serverCohortVerdict,
+	);
+	if (
+		input.cohortVerdict === "isolated_to_new_version" &&
+		serverCohortVerdict !== "isolated_to_new_version"
+	) {
+		logger.warn(
+			"[autosre] client-asserted cohortVerdict=isolated_to_new_version was not corroborated server-side; capping proof tier",
+			{
+				projectId,
+				contributingChangeId: input.contributingChangeId ?? null,
+				serverCohortVerdict,
+			},
+		);
+	}
+	// An unverified client claim of isolation is not persisted as a verdict.
+	const storedCohortVerdict =
+		serverCohortVerdict ??
+		(input.cohortVerdict === "isolated_to_new_version"
+			? null
+			: (input.cohortVerdict ?? null));
 
 	const relationshipScore = input.causalChain.some(
 		(c) => c.relationshipCertainty === "exact",
@@ -303,10 +501,8 @@ export async function recordInvestigationFinding(
 		finalConfidence: confidence,
 	};
 
-	const id = input.id?.trim() || `fnd_${randomHex(10)}`;
-	const expiresAt = new Date(
-		Date.now() + 90 * 24 * 3600_000,
-	).toISOString();
+	const id = idempotencyId ?? (input.id?.trim() || `fnd_${randomHex(10)}`);
+	const expiresAt = new Date(Date.now() + 90 * 24 * 3600_000).toISOString();
 
 	await db
 		.prepare(
@@ -352,7 +548,7 @@ export async function recordInvestigationFinding(
 			input.rootCauseEntityKind ?? null,
 			input.rootCauseEntityId ?? null,
 			input.contributingChangeId ?? null,
-			input.cohortVerdict ?? null,
+			storedCohortVerdict,
 			input.contendingTraceId ?? null,
 			JSON.stringify(input.causalChain),
 			JSON.stringify(input.citedEvidenceIds),
@@ -429,9 +625,7 @@ export async function searchInvestigationFindings(
 		binds.push(options.symptomEntityId);
 	}
 	if (options.rootCauseEntityId) {
-		conditions.push(
-			"(root_cause_entity_id = ? OR fault_locus_entity_id = ?)",
-		);
+		conditions.push("(root_cause_entity_id = ? OR fault_locus_entity_id = ?)");
 		binds.push(options.rootCauseEntityId, options.rootCauseEntityId);
 	}
 	if (options.causalVerificationStatus) {
@@ -443,7 +637,7 @@ export async function searchInvestigationFindings(
 		binds.push(options.incidentId);
 	}
 
-	const limit = Math.max(1, Math.min(100, options.limit ?? 25));
+	const limit = Math.max(1, Math.min(200, options.limit ?? 25));
 	binds.push(limit);
 
 	const rows = await db
@@ -575,7 +769,8 @@ export async function verifyPairedCounterfactualProof(
 	const failingPassed =
 		failingRow.result_passed !== null
 			? Number(failingRow.result_passed) === 1
-			: Number(failingRow.fail_count) === 0 && Number(failingRow.pass_count) > 0;
+			: Number(failingRow.fail_count) === 0 &&
+				Number(failingRow.pass_count) > 0;
 	if (failingPassed) {
 		throw new CounterfactualVerificationError(
 			`Baseline run ${input.failingRunId} did not fail (passed=1)`,
@@ -590,8 +785,7 @@ export async function verifyPairedCounterfactualProof(
 		);
 	}
 
-	const failingFp =
-		failingRow.result_failure_fp ?? failingRow.run_failure_fp;
+	const failingFp = failingRow.result_failure_fp ?? failingRow.run_failure_fp;
 	if (failingFp !== finding.fingerprint) {
 		throw new CounterfactualVerificationError(
 			`Failing run fingerprint (${failingFp ?? "null"}) does not match finding fingerprint (${finding.fingerprint})`,
@@ -627,8 +821,7 @@ export async function verifyPairedCounterfactualProof(
 				`Candidate run ${passRunId} did not pass`,
 			);
 		}
-		const passInputHash =
-			passRow.result_input_hash ?? passRow.run_input_hash;
+		const passInputHash = passRow.result_input_hash ?? passRow.run_input_hash;
 		if (passInputHash !== failingInputHash) {
 			throw new CounterfactualVerificationError(
 				`Candidate run ${passRunId} input_payload_hash (${passInputHash ?? "null"}) does not match failing run (${failingInputHash})`,

package/src/lib/autosre-ontology-cache.test.ts

@@ -0,0 +1,78 @@
+import type { StoredSpan } from "@obsunified/types";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+	clearOntologyUpsertCache,
+	extractSpanLineageAndTopology,
+	upsertOntologyFromSpans,
+} from "./autosre-ontology";
+import type { SqlDb } from "./sql-db";
+
+const dbSpan = (i: number, environment = "prod"): StoredSpan =>
+	({
+		projectId: "p",
+		traceId: `t${i}`,
+		spanId: `s${i}`,
+		environment,
+		startTime: "2026-10-15T00:00:00.000Z",
+		receivedAt: "2026-10-15T00:00:00.000Z",
+		attributesJson: JSON.stringify({
+			"db.system": "postgresql",
+			"db.name": `db${i}`,
+			"server.address": "pg.internal",
+			"server.port": 5432,
+		}),
+		resourceAttributesJson: "{}",
+	}) as unknown as StoredSpan;
+
+const fakeDb = (batch: SqlDb["batch"]): SqlDb => {
+	const stmt = { bind: () => stmt };
+	return { prepare: () => stmt, batch } as unknown as SqlDb;
+};
+
+describe("ontology env normalization", () => {
+	afterEach(() => clearOntologyUpsertCache());
+
+	it("maps production-tagged spans to the prod environment", () => {
+		const span = {
+			...dbSpan(1, ""),
+			environment: undefined,
+			resourceAttributesJson: JSON.stringify({
+				"deployment.environment.name": "Production",
+			}),
+		} as unknown as StoredSpan;
+		expect(extractSpanLineageAndTopology(span).environment).toBe("prod");
+	});
+});
+
+describe("NODE_UPSERT_CACHE", () => {
+	afterEach(() => clearOntologyUpsertCache());
+
+	it("does not remember keys when the batch fails", async () => {
+		const failing = vi.fn(async () => {
+			throw new Error("boom");
+		});
+		await expect(
+			upsertOntologyFromSpans(fakeDb(failing as never), [dbSpan(1)]),
+		).rejects.toThrow("boom");
+		const ok = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(ok as never), [dbSpan(1)]);
+		expect(ok).toHaveBeenCalledTimes(1);
+		// Now cached: a third call writes nothing.
+		const again = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(again as never), [dbSpan(1)]);
+		expect(again).not.toHaveBeenCalled();
+	});
+
+	it("caps the cache and evicts the oldest entries", async () => {
+		const spans = Array.from({ length: 5_002 }, (_, i) => dbSpan(i));
+		const batch = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(batch as never), spans);
+		// Oldest keys were evicted so they are written again; newest stay cached.
+		const second = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(second as never), [dbSpan(0)]);
+		expect(second).toHaveBeenCalledTimes(1);
+		const third = vi.fn(async () => []);
+		await upsertOntologyFromSpans(fakeDb(third as never), [dbSpan(5_001)]);
+		expect(third).not.toHaveBeenCalled();
+	});
+});

package/src/lib/autosre-ontology.ts

@@ -9,7 +9,11 @@ import type {
 	RelationshipCertainty,
 	StoredSpan,
 } from "@obsunified/types";
-import { buildCanonicalNodeId, normalizeErrorSignature } from "@obsunified/types";
+import {
+	buildCanonicalNodeId,
+	normalizeErrorSignature,
+} from "@obsunified/types";
+import { normalizeEnvironment } from "@obsunified/types/lineage";
 import { randomHex } from "./hash";
 import { parseJsonArray, parseJsonRecord } from "./json";
 import type { SqlDb, SqlStatement } from "./sql-db";
@@ -53,22 +57,22 @@ export function extractSpanLineageAndTopology(
 	const attrs = parseJsonRecord(span.attributesJson);
 	const resource = parseJsonRecord(span.resourceAttributesJson);
 
-	const environment = (
+	const environment = normalizeEnvironment(
 		span.environment ??
-		pickStr(
-			resource,
-			"deployment.environment.name",
-			"deployment.environment",
-			"obs.environment",
-		) ??
-		pickStr(
-			attrs,
-			"deployment.environment.name",
-			"deployment.environment",
-			"obs.environment",
-		) ??
-		"default"
-	).toLowerCase();
+			pickStr(
+				resource,
+				"deployment.environment.name",
+				"deployment.environment",
+				"obs.environment",
+			) ??
+			pickStr(
+				attrs,
+				"deployment.environment.name",
+				"deployment.environment",
+				"obs.environment",
+			) ??
+			undefined,
+	);
 
 	const namespace =
 		pickStr(resource, "service.namespace") ??
@@ -144,11 +148,8 @@ export function extractSpanLineageAndTopology(
 			environment,
 			messagingSystem,
 			messagingDestination:
-				pickStr(
-					attrs,
-					"messaging.destination.name",
-					"messaging.destination",
-				) ?? "default",
+				pickStr(attrs, "messaging.destination.name", "messaging.destination") ??
+				"default",
 			serverAddress,
 		});
 		edgePredicate =
@@ -207,8 +208,7 @@ export function extractSpanLineageAndTopology(
 								? (evObj.attributes as Record<string, JsonValue>)
 								: {};
 						rawErr =
-							pickStr(evAttrs, "exception.type", "exception.message") ??
-							rawErr;
+							pickStr(evAttrs, "exception.type", "exception.message") ?? rawErr;
 					}
 				}
 			}
@@ -236,6 +236,20 @@ export function extractSpanLineageAndTopology(
 // 60-second isolate cache to avoid repeating identical ontology node upserts on every span
 const NODE_UPSERT_CACHE = new Map<string, number>();
 const CACHE_TTL_MS = 60_000;
+const NODE_UPSERT_CACHE_MAX = 5_000;
+
+function rememberUpserts(keys: Iterable<string>, nowMs: number): void {
+	for (const key of keys) {
+		// Re-insert so Map iteration order tracks recency; evict oldest past the cap.
+		NODE_UPSERT_CACHE.delete(key);
+		NODE_UPSERT_CACHE.set(key, nowMs);
+	}
+	while (NODE_UPSERT_CACHE.size > NODE_UPSERT_CACHE_MAX) {
+		const oldest = NODE_UPSERT_CACHE.keys().next().value;
+		if (oldest === undefined) break;
+		NODE_UPSERT_CACHE.delete(oldest);
+	}
+}
 
 export function clearOntologyUpsertCache(): void {
 	NODE_UPSERT_CACHE.clear();
@@ -254,6 +268,8 @@ export async function upsertOntologyFromSpans(
 
 	const nowMs = Date.now();
 	const statements: SqlStatement[] = [];
+	// Cache keys are only remembered once the batch has been written successfully.
+	const pendingCacheKeys = new Set<string>();
 	const edgeAggregates = new Map<
 		string,
 		{
@@ -293,8 +309,11 @@ export async function upsertOntologyFromSpans(
 		if (lineage.sourceNode) {
 			const cacheKey = `${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}::${lineage.serviceVersion ?? ""}::${lineage.commitSha ?? ""}::${lineage.configDigest ?? ""}`;
 			const cachedAt = NODE_UPSERT_CACHE.get(cacheKey);
-			if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
-				NODE_UPSERT_CACHE.set(cacheKey, nowMs);
+			if (
+				!pendingCacheKeys.has(cacheKey) &&
+				(options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS)
+			) {
+				pendingCacheKeys.add(cacheKey);
 				serviceNodesToCheck.set(
 					`${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}`,
 					{
@@ -313,8 +332,11 @@ export async function upsertOntologyFromSpans(
 		if (lineage.targetNode) {
 			const targetCacheKey = `${span.projectId}::${lineage.environment}::${lineage.targetNode.id}`;
 			const cachedAt = NODE_UPSERT_CACHE.get(targetCacheKey);
-			if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
-				NODE_UPSERT_CACHE.set(targetCacheKey, nowMs);
+			if (
+				!pendingCacheKeys.has(targetCacheKey) &&
+				(options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS)
+			) {
+				pendingCacheKeys.add(targetCacheKey);
 				statements.push(
 					db
 						.prepare(
@@ -511,6 +533,7 @@ export async function upsertOntologyFromSpans(
 	if (statements.length > 0) {
 		await db.batch(statements);
 	}
+	rememberUpserts(pendingCacheKeys, nowMs);
 }
 
 /**
@@ -577,7 +600,8 @@ export function computeTarjanScc(
 		if (lowlink.get(nodeId) === indices.get(nodeId)) {
 			const component: string[] = [];
 			while (stack.length > 0) {
-				const w = stack.pop()!;
+				const w = stack.pop();
+				if (w === undefined) break;
 				onStack.delete(w);
 				component.push(w);
 				if (w === nodeId) break;