@obsunified/collector 2.1.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
  2. package/dist/chunk-ZSF5XP3B.js.map +1 -0
  3. package/dist/index.d.ts +309 -2
  4. package/dist/index.js +4159 -256
  5. package/dist/index.js.map +1 -1
  6. package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
  7. package/package.json +2 -2
  8. package/src/__tests__/autosre-closed-loop.test.ts +1275 -0
  9. package/src/index.ts +43 -0
  10. package/src/lib/analyses-runner.ts +9 -0
  11. package/src/lib/autosre-change-correlation.ts +780 -0
  12. package/src/lib/autosre-findings.ts +725 -0
  13. package/src/lib/autosre-ontology.ts +914 -0
  14. package/src/lib/autosre-remediation-gate.ts +1663 -0
  15. package/src/lib/eval-cases-store.ts +63 -1
  16. package/src/lib/evidence-references.ts +6 -0
  17. package/src/lib/housekeeping.test.ts +80 -0
  18. package/src/lib/housekeeping.ts +101 -1
  19. package/src/lib/store/ingest.ts +94 -6
  20. package/src/migrations/050_autosre_causal_ontology.sql +216 -0
  21. package/src/migrations/alert-state-notified.test.ts +3 -3
  22. package/src/migrations/parity.test.ts +4 -0
  23. package/src/migrations/project-scoped-keys.test.ts +1 -1
  24. package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
  25. package/src/plugins/alerts-evaluator.ts +17 -1
  26. package/src/plugins/autosre-routes.ts +1019 -0
  27. package/src/plugins/connected-routes/manifest.ts +13 -1
  28. package/src/plugins/connected-routes.ts +271 -0
  29. package/src/plugins/default-span-enrichment.ts +13 -1
  30. package/src/plugins/eval-cases-routes.ts +57 -2
  31. package/src/plugins/evidence-retrieval-routes.ts +300 -4
  32. package/dist/chunk-XKQTVIRE.js.map +0 -1
  33. /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
@@ -0,0 +1,725 @@
1
+ import type {
2
+ CausalProofTier,
3
+ CausalTriple,
4
+ CausalVerificationStatus,
5
+ CohortAttributionVerdict,
6
+ ConfidenceBreakdown,
7
+ EvidenceEntityKind,
8
+ FindingFingerprintInput,
9
+ InvestigationFinding,
10
+ InvestigationFindingInput,
11
+ PairedCounterfactualProof,
12
+ RecoveryStatus,
13
+ TelemetryCodeReference,
14
+ } from "@obsunified/types";
15
+ import { buildCanonicalNodeId, buildFindingFingerprint } from "@obsunified/types";
16
+ import { randomHex } from "./hash";
17
+ import { parseJsonArray, parseJsonRecord } from "./json";
18
+ import type { SqlDb } from "./sql-db";
19
+
20
+ interface FindingRow {
21
+ project_id: string;
22
+ environment: string;
23
+ id: string;
24
+ incident_id: string | null;
25
+ fingerprint: string;
26
+ fingerprint_input_json: string;
27
+ title: string;
28
+ summary: string;
29
+ causal_proof_tier: string;
30
+ confidence: number;
31
+ confidence_breakdown_json: string;
32
+ symptom_entity_kind: string;
33
+ symptom_entity_id: string;
34
+ fault_locus_entity_kind: string;
35
+ fault_locus_entity_id: string;
36
+ root_cause_entity_kind: string | null;
37
+ root_cause_entity_id: string | null;
38
+ contributing_change_id: string | null;
39
+ cohort_verdict: string | null;
40
+ contending_trace_id: string | null;
41
+ causal_chain_json: string;
42
+ cited_evidence_ids_json: string;
43
+ code_references_json: string | null;
44
+ counterfactual_proof_json: string | null;
45
+ recovery_status: string;
46
+ causal_verification_status: string;
47
+ watchdog_step: number;
48
+ baseline_request_rate_per_min: number | null;
49
+ next_watchdog_check_at: string | null;
50
+ actor_type: string;
51
+ actor_id: string;
52
+ agent_run_id: string | null;
53
+ created_at: string;
54
+ updated_at: string;
55
+ expires_at: string | null;
56
+ }
57
+
58
+ export const mapFindingRow = (row: FindingRow): InvestigationFinding => {
59
+ const rawFpInput = parseJsonRecord(row.fingerprint_input_json);
60
+ const fingerprintInput: FindingFingerprintInput = {
61
+ environment:
62
+ typeof rawFpInput.environment === "string"
63
+ ? rawFpInput.environment
64
+ : row.environment,
65
+ originatingNodeId:
66
+ typeof rawFpInput.originatingNodeId === "string"
67
+ ? rawFpInput.originatingNodeId
68
+ : row.fault_locus_entity_id,
69
+ normalizedOperation:
70
+ typeof rawFpInput.normalizedOperation === "string"
71
+ ? rawFpInput.normalizedOperation
72
+ : row.title,
73
+ errorSignature:
74
+ typeof rawFpInput.errorSignature === "string"
75
+ ? rawFpInput.errorSignature
76
+ : "ERROR",
77
+ downstreamTargetNodeId:
78
+ typeof rawFpInput.downstreamTargetNodeId === "string"
79
+ ? rawFpInput.downstreamTargetNodeId
80
+ : null,
81
+ };
82
+
83
+ const rawBreakdown = parseJsonRecord(row.confidence_breakdown_json);
84
+ const confidenceBreakdown: ConfidenceBreakdown = {
85
+ relationshipScore: Number(rawBreakdown.relationshipScore ?? 0.5),
86
+ cohortLiftScore: Number(rawBreakdown.cohortLiftScore ?? 0),
87
+ temporalProximityScore: Number(rawBreakdown.temporalProximityScore ?? 0.5),
88
+ counterfactualBonus: Number(rawBreakdown.counterfactualBonus ?? 0),
89
+ priorRecurrenceBonus: Number(rawBreakdown.priorRecurrenceBonus ?? 0),
90
+ finalConfidence: Number(rawBreakdown.finalConfidence ?? row.confidence),
91
+ };
92
+
93
+ let counterfactualProof: PairedCounterfactualProof | null = null;
94
+ if (row.counterfactual_proof_json) {
95
+ const rawCf = parseJsonRecord(row.counterfactual_proof_json);
96
+ if (typeof rawCf.evalCaseId === "string") {
97
+ counterfactualProof = {
98
+ evalCaseId: rawCf.evalCaseId,
99
+ inputPayloadHash: String(rawCf.inputPayloadHash ?? ""),
100
+ failingRunId: String(rawCf.failingRunId ?? ""),
101
+ failingVersion:
102
+ typeof rawCf.failingVersion === "string"
103
+ ? rawCf.failingVersion
104
+ : null,
105
+ failingCommitSha:
106
+ typeof rawCf.failingCommitSha === "string"
107
+ ? rawCf.failingCommitSha
108
+ : null,
109
+ failureFingerprint: String(rawCf.failureFingerprint ?? row.fingerprint),
110
+ passingRunIds: Array.isArray(rawCf.passingRunIds)
111
+ ? rawCf.passingRunIds.map(String)
112
+ : [],
113
+ passingVersion:
114
+ typeof rawCf.passingVersion === "string"
115
+ ? rawCf.passingVersion
116
+ : null,
117
+ passingCommitSha:
118
+ typeof rawCf.passingCommitSha === "string"
119
+ ? rawCf.passingCommitSha
120
+ : null,
121
+ requiredTrials: Number(rawCf.requiredTrials ?? 1),
122
+ verifiedAt: String(rawCf.verifiedAt ?? row.updated_at),
123
+ };
124
+ }
125
+ }
126
+
127
+ return {
128
+ id: row.id,
129
+ projectId: row.project_id,
130
+ environment: row.environment,
131
+ incidentId: row.incident_id,
132
+ fingerprint: row.fingerprint,
133
+ fingerprintInput,
134
+ title: row.title,
135
+ summary: row.summary,
136
+ causalProofTier: (row.causal_proof_tier ||
137
+ "unverified_locus") as CausalProofTier,
138
+ confidence: Number(row.confidence),
139
+ confidenceBreakdown,
140
+ symptomEntityKind: row.symptom_entity_kind as EvidenceEntityKind,
141
+ symptomEntityId: row.symptom_entity_id,
142
+ faultLocusEntityKind: row.fault_locus_entity_kind as EvidenceEntityKind,
143
+ faultLocusEntityId: row.fault_locus_entity_id,
144
+ rootCauseEntityKind:
145
+ (row.root_cause_entity_kind as EvidenceEntityKind | null) ?? null,
146
+ rootCauseEntityId: row.root_cause_entity_id,
147
+ contributingChangeId: row.contributing_change_id,
148
+ cohortVerdict:
149
+ (row.cohort_verdict as CohortAttributionVerdict | null) ?? null,
150
+ contendingTraceId: row.contending_trace_id,
151
+ causalChain: parseJsonArray(row.causal_chain_json) as unknown as CausalTriple[],
152
+ citedEvidenceIds: parseJsonArray(row.cited_evidence_ids_json).map(String),
153
+ codeReferences: parseJsonArray(
154
+ row.code_references_json ?? "[]",
155
+ ) as unknown as TelemetryCodeReference[],
156
+ counterfactualProof,
157
+ recoveryStatus: (row.recovery_status ||
158
+ "not_applicable") as RecoveryStatus,
159
+ causalVerificationStatus: (row.causal_verification_status ||
160
+ "unverified") as CausalVerificationStatus,
161
+ watchdogStep: Number(row.watchdog_step ?? 0),
162
+ baselineRequestRatePerMin:
163
+ row.baseline_request_rate_per_min === null
164
+ ? null
165
+ : Number(row.baseline_request_rate_per_min),
166
+ nextWatchdogCheckAt: row.next_watchdog_check_at,
167
+ actorType: row.actor_type,
168
+ actorId: row.actor_id,
169
+ agentRunId: row.agent_run_id,
170
+ createdAt: row.created_at,
171
+ updatedAt: row.updated_at,
172
+ expiresAt: row.expires_at,
173
+ };
174
+ };
175
+
176
+ /**
177
+ * Enforces P1 #1: Structural overlap (`touched_same_code_path` or `originated_fault_in_span`)
178
+ * can never be promoted to `cohort_rate_isolated` or `counterfactual_verified` without
179
+ * actual cohort isolation or SQL-verified counterfactual proof.
180
+ */
181
+ function resolveSafeCausalProofTier(
182
+ input: InvestigationFindingInput,
183
+ ): CausalProofTier {
184
+ if (input.cohortVerdict === "isolated_to_new_version") {
185
+ return "cohort_rate_isolated";
186
+ }
187
+ if (
188
+ input.cohortVerdict === "sequential_post_cutover_regression" ||
189
+ input.cohortVerdict === "shared_degradation_new_version_aggressor"
190
+ ) {
191
+ return "corroborated_candidate";
192
+ }
193
+ const hasCodeOverlapOrContention = input.causalChain.some(
194
+ (t) =>
195
+ t.predicate === "touched_same_code_path" ||
196
+ t.predicate === "contended_on_shared_resource" ||
197
+ t.predicate === "induced_shared_resource_contention" ||
198
+ t.predicate === "degraded_after_change",
199
+ );
200
+ if (input.causalProofTier === "cohort_rate_isolated") {
201
+ // Only allow cohort_rate_isolated if a change or isolated_to_version_cohort triple is present
202
+ const hasIsolatedTriple = input.causalChain.some(
203
+ (t) => t.predicate === "isolated_to_version_cohort",
204
+ );
205
+ if (hasIsolatedTriple) return "cohort_rate_isolated";
206
+ }
207
+ if (
208
+ hasCodeOverlapOrContention ||
209
+ input.causalProofTier === "corroborated_candidate"
210
+ ) {
211
+ return "corroborated_candidate";
212
+ }
213
+ return "unverified_locus";
214
+ }
215
+
216
+ export async function recordInvestigationFinding(
217
+ db: SqlDb,
218
+ projectId: string,
219
+ input: InvestigationFindingInput,
220
+ ): Promise<InvestigationFinding> {
221
+ const nowIso = new Date().toISOString();
222
+ const environment = (
223
+ input.environment ??
224
+ input.fingerprintInput?.environment ??
225
+ "default"
226
+ )
227
+ .trim()
228
+ .toLowerCase();
229
+
230
+ const defaultOriginNode = input.faultLocusEntityId.includes(":")
231
+ ? input.faultLocusEntityId
232
+ : buildCanonicalNodeId({
233
+ kind: "service",
234
+ environment,
235
+ serviceName: input.faultLocusEntityId,
236
+ }).id;
237
+
238
+ const fpResult = buildFindingFingerprint(
239
+ input.fingerprintInput ?? {
240
+ environment,
241
+ originatingNodeId: defaultOriginNode,
242
+ normalizedOperation: input.title,
243
+ errorSignature: input.summary.slice(0, 120),
244
+ downstreamTargetNodeId: null,
245
+ },
246
+ );
247
+
248
+ // Check for prior verified findings with the same stable fingerprint
249
+ const priorRow = await db
250
+ .prepare(
251
+ `SELECT id, causal_verification_status FROM investigation_findings
252
+ WHERE project_id = ? AND environment = ? AND fingerprint = ?
253
+ ORDER BY created_at DESC
254
+ LIMIT 1`,
255
+ )
256
+ .bind(projectId, environment, fpResult.fingerprint)
257
+ .first<{ id: string; causal_verification_status: string }>();
258
+
259
+ const causalProofTier = resolveSafeCausalProofTier(input);
260
+
261
+ const relationshipScore = input.causalChain.some(
262
+ (c) => c.relationshipCertainty === "exact",
263
+ )
264
+ ? 0.9
265
+ : input.causalChain.some((c) => c.relationshipCertainty === "inferred")
266
+ ? 0.65
267
+ : 0.4;
268
+ const cohortLiftScore =
269
+ causalProofTier === "cohort_rate_isolated"
270
+ ? 0.9
271
+ : causalProofTier === "corroborated_candidate"
272
+ ? 0.6
273
+ : 0.2;
274
+ const priorBonus =
275
+ priorRow?.causal_verification_status === "counterfactual_verified" ||
276
+ priorRow?.causal_verification_status === "operationally_corroborated"
277
+ ? 0.1
278
+ : 0;
279
+
280
+ const computedConfidence = Math.min(
281
+ 0.99,
282
+ Number(
283
+ (
284
+ 0.4 * relationshipScore +
285
+ 0.4 * cohortLiftScore +
286
+ 0.2 * 0.7 +
287
+ priorBonus
288
+ ).toFixed(2),
289
+ ),
290
+ );
291
+ const confidence = input.confidence ?? computedConfidence;
292
+
293
+ const confidenceBreakdown: ConfidenceBreakdown = {
294
+ relationshipScore:
295
+ input.confidenceBreakdown?.relationshipScore ?? relationshipScore,
296
+ cohortLiftScore:
297
+ input.confidenceBreakdown?.cohortLiftScore ?? cohortLiftScore,
298
+ temporalProximityScore:
299
+ input.confidenceBreakdown?.temporalProximityScore ?? 0.7,
300
+ counterfactualBonus: input.confidenceBreakdown?.counterfactualBonus ?? 0,
301
+ priorRecurrenceBonus:
302
+ input.confidenceBreakdown?.priorRecurrenceBonus ?? priorBonus,
303
+ finalConfidence: confidence,
304
+ };
305
+
306
+ const id = input.id?.trim() || `fnd_${randomHex(10)}`;
307
+ const expiresAt = new Date(
308
+ Date.now() + 90 * 24 * 3600_000,
309
+ ).toISOString();
310
+
311
+ await db
312
+ .prepare(
313
+ `INSERT INTO investigation_findings (
314
+ project_id, environment, id, incident_id, fingerprint,
315
+ fingerprint_input_json, title, summary, causal_proof_tier,
316
+ confidence, confidence_breakdown_json, symptom_entity_kind,
317
+ symptom_entity_id, fault_locus_entity_kind, fault_locus_entity_id,
318
+ root_cause_entity_kind, root_cause_entity_id, contributing_change_id,
319
+ cohort_verdict, contending_trace_id, causal_chain_json,
320
+ cited_evidence_ids_json, code_references_json,
321
+ recovery_status, causal_verification_status, watchdog_step,
322
+ actor_type, actor_id, agent_run_id, created_at, updated_at, expires_at
323
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'not_applicable', 'unverified', 0, ?, ?, ?, ?, ?, ?)
324
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
325
+ title = excluded.title,
326
+ summary = excluded.summary,
327
+ causal_proof_tier = excluded.causal_proof_tier,
328
+ confidence = excluded.confidence,
329
+ confidence_breakdown_json = excluded.confidence_breakdown_json,
330
+ contributing_change_id = COALESCE(excluded.contributing_change_id, investigation_findings.contributing_change_id),
331
+ cohort_verdict = COALESCE(excluded.cohort_verdict, investigation_findings.cohort_verdict),
332
+ causal_chain_json = excluded.causal_chain_json,
333
+ cited_evidence_ids_json = excluded.cited_evidence_ids_json,
334
+ updated_at = excluded.updated_at`,
335
+ )
336
+ .bind(
337
+ projectId,
338
+ environment,
339
+ id,
340
+ input.incidentId ?? null,
341
+ fpResult.fingerprint,
342
+ JSON.stringify(fpResult.normalizedInput),
343
+ input.title,
344
+ input.summary,
345
+ causalProofTier,
346
+ confidence,
347
+ JSON.stringify(confidenceBreakdown),
348
+ input.symptomEntityKind,
349
+ input.symptomEntityId,
350
+ input.faultLocusEntityKind,
351
+ input.faultLocusEntityId,
352
+ input.rootCauseEntityKind ?? null,
353
+ input.rootCauseEntityId ?? null,
354
+ input.contributingChangeId ?? null,
355
+ input.cohortVerdict ?? null,
356
+ input.contendingTraceId ?? null,
357
+ JSON.stringify(input.causalChain),
358
+ JSON.stringify(input.citedEvidenceIds),
359
+ JSON.stringify(input.codeReferences ?? []),
360
+ input.actorType ?? "agent",
361
+ input.actorId ?? "autosre-agent",
362
+ input.agentRunId ?? null,
363
+ nowIso,
364
+ nowIso,
365
+ expiresAt,
366
+ )
367
+ .run();
368
+
369
+ const created = await getInvestigationFinding(db, projectId, id, environment);
370
+ if (!created) {
371
+ throw new Error(`Failed to load created finding ${id}`);
372
+ }
373
+ return created;
374
+ }
375
+
376
+ export async function getInvestigationFinding(
377
+ db: SqlDb,
378
+ projectId: string,
379
+ findingId: string,
380
+ environment?: string,
381
+ ): Promise<InvestigationFinding | null> {
382
+ const row = environment
383
+ ? await db
384
+ .prepare(
385
+ `SELECT * FROM investigation_findings
386
+ WHERE project_id = ? AND environment = ? AND id = ?`,
387
+ )
388
+ .bind(projectId, environment.toLowerCase(), findingId)
389
+ .first<FindingRow>()
390
+ : await db
391
+ .prepare(
392
+ `SELECT * FROM investigation_findings
393
+ WHERE project_id = ? AND id = ?
394
+ ORDER BY created_at DESC
395
+ LIMIT 1`,
396
+ )
397
+ .bind(projectId, findingId)
398
+ .first<FindingRow>();
399
+
400
+ return row ? mapFindingRow(row) : null;
401
+ }
402
+
403
+ export async function searchInvestigationFindings(
404
+ db: SqlDb,
405
+ options: {
406
+ projectId: string;
407
+ environment?: string;
408
+ fingerprint?: string;
409
+ symptomEntityId?: string;
410
+ rootCauseEntityId?: string;
411
+ causalVerificationStatus?: string;
412
+ incidentId?: string;
413
+ limit?: number;
414
+ },
415
+ ): Promise<InvestigationFinding[]> {
416
+ const conditions: string[] = ["project_id = ?"];
417
+ const binds: unknown[] = [options.projectId];
418
+
419
+ if (options.environment) {
420
+ conditions.push("environment = ?");
421
+ binds.push(options.environment.toLowerCase());
422
+ }
423
+ if (options.fingerprint) {
424
+ conditions.push("fingerprint = ?");
425
+ binds.push(options.fingerprint);
426
+ }
427
+ if (options.symptomEntityId) {
428
+ conditions.push("symptom_entity_id = ?");
429
+ binds.push(options.symptomEntityId);
430
+ }
431
+ if (options.rootCauseEntityId) {
432
+ conditions.push(
433
+ "(root_cause_entity_id = ? OR fault_locus_entity_id = ?)",
434
+ );
435
+ binds.push(options.rootCauseEntityId, options.rootCauseEntityId);
436
+ }
437
+ if (options.causalVerificationStatus) {
438
+ conditions.push("causal_verification_status = ?");
439
+ binds.push(options.causalVerificationStatus);
440
+ }
441
+ if (options.incidentId) {
442
+ conditions.push("incident_id = ?");
443
+ binds.push(options.incidentId);
444
+ }
445
+
446
+ const limit = Math.max(1, Math.min(100, options.limit ?? 25));
447
+ binds.push(limit);
448
+
449
+ const rows = await db
450
+ .prepare(
451
+ `SELECT * FROM investigation_findings
452
+ WHERE ${conditions.join(" AND ")}
453
+ ORDER BY created_at DESC
454
+ LIMIT ?`,
455
+ )
456
+ .bind(...binds)
457
+ .all<FindingRow>();
458
+
459
+ return rows.results.map(mapFindingRow);
460
+ }
461
+
462
+ export class CounterfactualVerificationError extends Error {
463
+ constructor(message: string) {
464
+ super(message);
465
+ this.name = "CounterfactualVerificationError";
466
+ }
467
+ }
468
+
469
+ interface JoinedEvalRunResultRow {
470
+ run_id: string;
471
+ eval_case_id: string | null;
472
+ service_version: string | null;
473
+ commit_sha: string | null;
474
+ candidate_agent_version: string | null;
475
+ candidate_prompt_version: string | null;
476
+ run_input_hash: string | null;
477
+ run_failure_fp: string | null;
478
+ pass_count: number;
479
+ fail_count: number;
480
+ result_passed: number | null;
481
+ result_input_hash: string | null;
482
+ result_failure_fp: string | null;
483
+ }
484
+
485
+ /**
486
+ * Verifies a PairedCounterfactualProof in SQL against `eval_runs` and `eval_case_results`
487
+ * (RFC 0013 § 5.A & P1 #3).
488
+ *
489
+ * Requires:
490
+ * 1. `failingRunId` and all `passingRunIds` belong to `projectId` and ran against `evalCaseId`
491
+ * with the identical non-null `input_payload_hash`.
492
+ * 2. `failingRunId` failed (`passed = 0`) and its `failure_fingerprint` matches `finding.fingerprint`.
493
+ * 3. All `passingRunIds` passed (`passed = 1`) and `passingRunIds.length >= requiredTrials`.
494
+ * 4. Failing and passing runs have distinct recorded version/commit/prompt lineage.
495
+ */
496
+ export async function verifyPairedCounterfactualProof(
497
+ db: SqlDb,
498
+ projectId: string,
499
+ findingId: string,
500
+ input: {
501
+ environment?: string;
502
+ evalCaseId: string;
503
+ failingRunId: string;
504
+ passingRunIds: string[];
505
+ requiredTrials?: number;
506
+ },
507
+ ): Promise<InvestigationFinding> {
508
+ const finding = await getInvestigationFinding(
509
+ db,
510
+ projectId,
511
+ findingId,
512
+ input.environment,
513
+ );
514
+ if (!finding) {
515
+ throw new CounterfactualVerificationError(
516
+ `Finding ${findingId} not found in project ${projectId}`,
517
+ );
518
+ }
519
+
520
+ const requiredTrials = Math.max(1, input.requiredTrials ?? 1);
521
+ const uniquePassingIds = Array.from(new Set(input.passingRunIds));
522
+ if (uniquePassingIds.length < requiredTrials) {
523
+ throw new CounterfactualVerificationError(
524
+ `Counterfactual requires at least ${requiredTrials} passing trial run(s), got ${uniquePassingIds.length}`,
525
+ );
526
+ }
527
+ if (uniquePassingIds.includes(input.failingRunId)) {
528
+ throw new CounterfactualVerificationError(
529
+ "failingRunId cannot also be listed in passingRunIds",
530
+ );
531
+ }
532
+
533
+ const loadRunVerification = async (
534
+ runId: string,
535
+ ): Promise<JoinedEvalRunResultRow | null> => {
536
+ const row = await db
537
+ .prepare(
538
+ `SELECT
539
+ r.id AS run_id,
540
+ COALESCE(cr.eval_case_id, r.eval_case_id) AS eval_case_id,
541
+ r.service_version,
542
+ r.commit_sha,
543
+ r.candidate_agent_version,
544
+ r.candidate_prompt_version,
545
+ r.input_payload_hash AS run_input_hash,
546
+ r.failure_fingerprint AS run_failure_fp,
547
+ r.pass_count,
548
+ r.fail_count,
549
+ cr.passed AS result_passed,
550
+ cr.input_payload_hash AS result_input_hash,
551
+ cr.failure_fingerprint AS result_failure_fp
552
+ FROM eval_runs r
553
+ LEFT JOIN eval_case_results cr
554
+ ON cr.project_id = r.project_id AND cr.run_id = r.id AND cr.eval_case_id = ?
555
+ WHERE r.project_id = ? AND r.id = ?
556
+ LIMIT 1`,
557
+ )
558
+ .bind(input.evalCaseId, projectId, runId)
559
+ .first<JoinedEvalRunResultRow>();
560
+ return row;
561
+ };
562
+
563
+ const failingRow = await loadRunVerification(input.failingRunId);
564
+ if (!failingRow) {
565
+ throw new CounterfactualVerificationError(
566
+ `Failing eval run ${input.failingRunId} not found in project ${projectId}`,
567
+ );
568
+ }
569
+ if (failingRow.eval_case_id !== input.evalCaseId) {
570
+ throw new CounterfactualVerificationError(
571
+ `Failing eval run ${input.failingRunId} does not match evalCaseId ${input.evalCaseId}`,
572
+ );
573
+ }
574
+
575
+ const failingPassed =
576
+ failingRow.result_passed !== null
577
+ ? Number(failingRow.result_passed) === 1
578
+ : Number(failingRow.fail_count) === 0 && Number(failingRow.pass_count) > 0;
579
+ if (failingPassed) {
580
+ throw new CounterfactualVerificationError(
581
+ `Baseline run ${input.failingRunId} did not fail (passed=1)`,
582
+ );
583
+ }
584
+
585
+ const failingInputHash =
586
+ failingRow.result_input_hash ?? failingRow.run_input_hash;
587
+ if (!failingInputHash) {
588
+ throw new CounterfactualVerificationError(
589
+ `Failing eval run ${input.failingRunId} is missing input_payload_hash`,
590
+ );
591
+ }
592
+
593
+ const failingFp =
594
+ failingRow.result_failure_fp ?? failingRow.run_failure_fp;
595
+ if (failingFp !== finding.fingerprint) {
596
+ throw new CounterfactualVerificationError(
597
+ `Failing run fingerprint (${failingFp ?? "null"}) does not match finding fingerprint (${finding.fingerprint})`,
598
+ );
599
+ }
600
+
601
+ const failingVersion =
602
+ failingRow.service_version ??
603
+ failingRow.candidate_agent_version ??
604
+ failingRow.candidate_prompt_version;
605
+
606
+ let passingVersion: string | null = null;
607
+ let passingCommitSha: string | null = null;
608
+
609
+ for (const passRunId of uniquePassingIds) {
610
+ const passRow = await loadRunVerification(passRunId);
611
+ if (!passRow) {
612
+ throw new CounterfactualVerificationError(
613
+ `Passing eval run ${passRunId} not found in project ${projectId}`,
614
+ );
615
+ }
616
+ if (passRow.eval_case_id !== input.evalCaseId) {
617
+ throw new CounterfactualVerificationError(
618
+ `Passing eval run ${passRunId} does not match evalCaseId ${input.evalCaseId}`,
619
+ );
620
+ }
621
+ const didPass =
622
+ passRow.result_passed !== null
623
+ ? Number(passRow.result_passed) === 1
624
+ : Number(passRow.pass_count) > 0 && Number(passRow.fail_count) === 0;
625
+ if (!didPass) {
626
+ throw new CounterfactualVerificationError(
627
+ `Candidate run ${passRunId} did not pass`,
628
+ );
629
+ }
630
+ const passInputHash =
631
+ passRow.result_input_hash ?? passRow.run_input_hash;
632
+ if (passInputHash !== failingInputHash) {
633
+ throw new CounterfactualVerificationError(
634
+ `Candidate run ${passRunId} input_payload_hash (${passInputHash ?? "null"}) does not match failing run (${failingInputHash})`,
635
+ );
636
+ }
637
+ passingVersion =
638
+ passRow.service_version ??
639
+ passRow.candidate_agent_version ??
640
+ passRow.candidate_prompt_version ??
641
+ passingVersion;
642
+ passingCommitSha = passRow.commit_sha ?? passingCommitSha;
643
+ }
644
+
645
+ if (
646
+ !failingVersion &&
647
+ !failingRow.commit_sha &&
648
+ !passingVersion &&
649
+ !passingCommitSha
650
+ ) {
651
+ throw new CounterfactualVerificationError(
652
+ "Counterfactual runs must record version, commit_sha, or candidate version lineage",
653
+ );
654
+ }
655
+
656
+ const nowIso = new Date().toISOString();
657
+ const proof: PairedCounterfactualProof = {
658
+ evalCaseId: input.evalCaseId,
659
+ inputPayloadHash: failingInputHash,
660
+ failingRunId: input.failingRunId,
661
+ failingVersion: failingVersion ?? null,
662
+ failingCommitSha: failingRow.commit_sha ?? null,
663
+ failureFingerprint: finding.fingerprint,
664
+ passingRunIds: uniquePassingIds,
665
+ passingVersion,
666
+ passingCommitSha,
667
+ requiredTrials,
668
+ verifiedAt: nowIso,
669
+ };
670
+
671
+ const updatedChain: CausalTriple[] = [
672
+ ...finding.causalChain,
673
+ {
674
+ subjectKind: "finding",
675
+ subjectId: finding.id,
676
+ predicate: "verified_by_counterfactual",
677
+ objectKind: "eval",
678
+ objectId: input.evalCaseId,
679
+ relationshipCertainty: "exact",
680
+ causalProofTier: "counterfactual_verified",
681
+ confidence: 0.99,
682
+ reason: `Paired counterfactual verified on evalCase ${input.evalCaseId}: run ${input.failingRunId} (${failingVersion ?? "baseline"}) failed with ${finding.fingerprint} and ${uniquePassingIds.join(", ")} (${passingVersion ?? "candidate"}) passed on payload ${failingInputHash}`,
683
+ },
684
+ ];
685
+
686
+ const updatedBreakdown: ConfidenceBreakdown = {
687
+ ...finding.confidenceBreakdown,
688
+ counterfactualBonus: 0.25,
689
+ finalConfidence: 0.99,
690
+ };
691
+
692
+ await db
693
+ .prepare(
694
+ `UPDATE investigation_findings
695
+ SET causal_proof_tier = 'counterfactual_verified',
696
+ causal_verification_status = 'counterfactual_verified',
697
+ confidence = 0.99,
698
+ confidence_breakdown_json = ?,
699
+ counterfactual_proof_json = ?,
700
+ causal_chain_json = ?,
701
+ updated_at = ?
702
+ WHERE project_id = ? AND environment = ? AND id = ?`,
703
+ )
704
+ .bind(
705
+ JSON.stringify(updatedBreakdown),
706
+ JSON.stringify(proof),
707
+ JSON.stringify(updatedChain),
708
+ nowIso,
709
+ projectId,
710
+ finding.environment,
711
+ finding.id,
712
+ )
713
+ .run();
714
+
715
+ const updated = await getInvestigationFinding(
716
+ db,
717
+ projectId,
718
+ finding.id,
719
+ finding.environment,
720
+ );
721
+ if (!updated) {
722
+ throw new Error(`Failed to reload finding ${finding.id}`);
723
+ }
724
+ return updated;
725
+ }