@obsunified/collector 2.1.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
  2. package/dist/chunk-ZSF5XP3B.js.map +1 -0
  3. package/dist/index.d.ts +309 -2
  4. package/dist/index.js +4159 -256
  5. package/dist/index.js.map +1 -1
  6. package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
  7. package/package.json +2 -2
  8. package/src/__tests__/autosre-closed-loop.test.ts +1275 -0
  9. package/src/index.ts +43 -0
  10. package/src/lib/analyses-runner.ts +9 -0
  11. package/src/lib/autosre-change-correlation.ts +780 -0
  12. package/src/lib/autosre-findings.ts +725 -0
  13. package/src/lib/autosre-ontology.ts +914 -0
  14. package/src/lib/autosre-remediation-gate.ts +1663 -0
  15. package/src/lib/eval-cases-store.ts +63 -1
  16. package/src/lib/evidence-references.ts +6 -0
  17. package/src/lib/housekeeping.test.ts +80 -0
  18. package/src/lib/housekeeping.ts +101 -1
  19. package/src/lib/store/ingest.ts +94 -6
  20. package/src/migrations/050_autosre_causal_ontology.sql +216 -0
  21. package/src/migrations/alert-state-notified.test.ts +3 -3
  22. package/src/migrations/parity.test.ts +4 -0
  23. package/src/migrations/project-scoped-keys.test.ts +1 -1
  24. package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
  25. package/src/plugins/alerts-evaluator.ts +17 -1
  26. package/src/plugins/autosre-routes.ts +1019 -0
  27. package/src/plugins/connected-routes/manifest.ts +13 -1
  28. package/src/plugins/connected-routes.ts +271 -0
  29. package/src/plugins/default-span-enrichment.ts +13 -1
  30. package/src/plugins/eval-cases-routes.ts +57 -2
  31. package/src/plugins/evidence-retrieval-routes.ts +300 -4
  32. package/dist/chunk-XKQTVIRE.js.map +0 -1
  33. /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
package/dist/index.js CHANGED
@@ -18,7 +18,7 @@ import {
18
18
  askEvidenceReferences,
19
19
  generateNarrative,
20
20
  sourceLinkEvidenceReferences
21
- } from "./chunk-XKQTVIRE.js";
21
+ } from "./chunk-ZSF5XP3B.js";
22
22
 
23
23
  // src/framework/collector.ts
24
24
  import { getConfiguredRetentionHours } from "@obsunified/types/constants";
@@ -2487,6 +2487,1569 @@ var NOOP_HANDLE = {
2487
2487
  };
2488
2488
  var passthroughChildSpan = async (_name, fn) => fn(NOOP_HANDLE);
2489
2489
 
2490
+ // src/lib/autosre-remediation-gate.ts
2491
+ import { AgentAutonomyLevel } from "@obsunified/types/constants";
2492
+
2493
+ // src/lib/autosre-findings.ts
2494
+ import { buildCanonicalNodeId, buildFindingFingerprint } from "@obsunified/types";
2495
+ var mapFindingRow = (row) => {
2496
+ const rawFpInput = parseJsonRecord(row.fingerprint_input_json);
2497
+ const fingerprintInput = {
2498
+ environment: typeof rawFpInput.environment === "string" ? rawFpInput.environment : row.environment,
2499
+ originatingNodeId: typeof rawFpInput.originatingNodeId === "string" ? rawFpInput.originatingNodeId : row.fault_locus_entity_id,
2500
+ normalizedOperation: typeof rawFpInput.normalizedOperation === "string" ? rawFpInput.normalizedOperation : row.title,
2501
+ errorSignature: typeof rawFpInput.errorSignature === "string" ? rawFpInput.errorSignature : "ERROR",
2502
+ downstreamTargetNodeId: typeof rawFpInput.downstreamTargetNodeId === "string" ? rawFpInput.downstreamTargetNodeId : null
2503
+ };
2504
+ const rawBreakdown = parseJsonRecord(row.confidence_breakdown_json);
2505
+ const confidenceBreakdown = {
2506
+ relationshipScore: Number(rawBreakdown.relationshipScore ?? 0.5),
2507
+ cohortLiftScore: Number(rawBreakdown.cohortLiftScore ?? 0),
2508
+ temporalProximityScore: Number(rawBreakdown.temporalProximityScore ?? 0.5),
2509
+ counterfactualBonus: Number(rawBreakdown.counterfactualBonus ?? 0),
2510
+ priorRecurrenceBonus: Number(rawBreakdown.priorRecurrenceBonus ?? 0),
2511
+ finalConfidence: Number(rawBreakdown.finalConfidence ?? row.confidence)
2512
+ };
2513
+ let counterfactualProof = null;
2514
+ if (row.counterfactual_proof_json) {
2515
+ const rawCf = parseJsonRecord(row.counterfactual_proof_json);
2516
+ if (typeof rawCf.evalCaseId === "string") {
2517
+ counterfactualProof = {
2518
+ evalCaseId: rawCf.evalCaseId,
2519
+ inputPayloadHash: String(rawCf.inputPayloadHash ?? ""),
2520
+ failingRunId: String(rawCf.failingRunId ?? ""),
2521
+ failingVersion: typeof rawCf.failingVersion === "string" ? rawCf.failingVersion : null,
2522
+ failingCommitSha: typeof rawCf.failingCommitSha === "string" ? rawCf.failingCommitSha : null,
2523
+ failureFingerprint: String(rawCf.failureFingerprint ?? row.fingerprint),
2524
+ passingRunIds: Array.isArray(rawCf.passingRunIds) ? rawCf.passingRunIds.map(String) : [],
2525
+ passingVersion: typeof rawCf.passingVersion === "string" ? rawCf.passingVersion : null,
2526
+ passingCommitSha: typeof rawCf.passingCommitSha === "string" ? rawCf.passingCommitSha : null,
2527
+ requiredTrials: Number(rawCf.requiredTrials ?? 1),
2528
+ verifiedAt: String(rawCf.verifiedAt ?? row.updated_at)
2529
+ };
2530
+ }
2531
+ }
2532
+ return {
2533
+ id: row.id,
2534
+ projectId: row.project_id,
2535
+ environment: row.environment,
2536
+ incidentId: row.incident_id,
2537
+ fingerprint: row.fingerprint,
2538
+ fingerprintInput,
2539
+ title: row.title,
2540
+ summary: row.summary,
2541
+ causalProofTier: row.causal_proof_tier || "unverified_locus",
2542
+ confidence: Number(row.confidence),
2543
+ confidenceBreakdown,
2544
+ symptomEntityKind: row.symptom_entity_kind,
2545
+ symptomEntityId: row.symptom_entity_id,
2546
+ faultLocusEntityKind: row.fault_locus_entity_kind,
2547
+ faultLocusEntityId: row.fault_locus_entity_id,
2548
+ rootCauseEntityKind: row.root_cause_entity_kind ?? null,
2549
+ rootCauseEntityId: row.root_cause_entity_id,
2550
+ contributingChangeId: row.contributing_change_id,
2551
+ cohortVerdict: row.cohort_verdict ?? null,
2552
+ contendingTraceId: row.contending_trace_id,
2553
+ causalChain: parseJsonArray(row.causal_chain_json),
2554
+ citedEvidenceIds: parseJsonArray(row.cited_evidence_ids_json).map(String),
2555
+ codeReferences: parseJsonArray(
2556
+ row.code_references_json ?? "[]"
2557
+ ),
2558
+ counterfactualProof,
2559
+ recoveryStatus: row.recovery_status || "not_applicable",
2560
+ causalVerificationStatus: row.causal_verification_status || "unverified",
2561
+ watchdogStep: Number(row.watchdog_step ?? 0),
2562
+ baselineRequestRatePerMin: row.baseline_request_rate_per_min === null ? null : Number(row.baseline_request_rate_per_min),
2563
+ nextWatchdogCheckAt: row.next_watchdog_check_at,
2564
+ actorType: row.actor_type,
2565
+ actorId: row.actor_id,
2566
+ agentRunId: row.agent_run_id,
2567
+ createdAt: row.created_at,
2568
+ updatedAt: row.updated_at,
2569
+ expiresAt: row.expires_at
2570
+ };
2571
+ };
2572
+ function resolveSafeCausalProofTier(input) {
2573
+ if (input.cohortVerdict === "isolated_to_new_version") {
2574
+ return "cohort_rate_isolated";
2575
+ }
2576
+ if (input.cohortVerdict === "sequential_post_cutover_regression" || input.cohortVerdict === "shared_degradation_new_version_aggressor") {
2577
+ return "corroborated_candidate";
2578
+ }
2579
+ const hasCodeOverlapOrContention = input.causalChain.some(
2580
+ (t) => t.predicate === "touched_same_code_path" || t.predicate === "contended_on_shared_resource" || t.predicate === "induced_shared_resource_contention" || t.predicate === "degraded_after_change"
2581
+ );
2582
+ if (input.causalProofTier === "cohort_rate_isolated") {
2583
+ const hasIsolatedTriple = input.causalChain.some(
2584
+ (t) => t.predicate === "isolated_to_version_cohort"
2585
+ );
2586
+ if (hasIsolatedTriple) return "cohort_rate_isolated";
2587
+ }
2588
+ if (hasCodeOverlapOrContention || input.causalProofTier === "corroborated_candidate") {
2589
+ return "corroborated_candidate";
2590
+ }
2591
+ return "unverified_locus";
2592
+ }
2593
+ async function recordInvestigationFinding(db, projectId, input) {
2594
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
2595
+ const environment = (input.environment ?? input.fingerprintInput?.environment ?? "default").trim().toLowerCase();
2596
+ const defaultOriginNode = input.faultLocusEntityId.includes(":") ? input.faultLocusEntityId : buildCanonicalNodeId({
2597
+ kind: "service",
2598
+ environment,
2599
+ serviceName: input.faultLocusEntityId
2600
+ }).id;
2601
+ const fpResult = buildFindingFingerprint(
2602
+ input.fingerprintInput ?? {
2603
+ environment,
2604
+ originatingNodeId: defaultOriginNode,
2605
+ normalizedOperation: input.title,
2606
+ errorSignature: input.summary.slice(0, 120),
2607
+ downstreamTargetNodeId: null
2608
+ }
2609
+ );
2610
+ const priorRow = await db.prepare(
2611
+ `SELECT id, causal_verification_status FROM investigation_findings
2612
+ WHERE project_id = ? AND environment = ? AND fingerprint = ?
2613
+ ORDER BY created_at DESC
2614
+ LIMIT 1`
2615
+ ).bind(projectId, environment, fpResult.fingerprint).first();
2616
+ const causalProofTier = resolveSafeCausalProofTier(input);
2617
+ const relationshipScore = input.causalChain.some(
2618
+ (c) => c.relationshipCertainty === "exact"
2619
+ ) ? 0.9 : input.causalChain.some((c) => c.relationshipCertainty === "inferred") ? 0.65 : 0.4;
2620
+ const cohortLiftScore = causalProofTier === "cohort_rate_isolated" ? 0.9 : causalProofTier === "corroborated_candidate" ? 0.6 : 0.2;
2621
+ const priorBonus = priorRow?.causal_verification_status === "counterfactual_verified" || priorRow?.causal_verification_status === "operationally_corroborated" ? 0.1 : 0;
2622
+ const computedConfidence = Math.min(
2623
+ 0.99,
2624
+ Number(
2625
+ (0.4 * relationshipScore + 0.4 * cohortLiftScore + 0.2 * 0.7 + priorBonus).toFixed(2)
2626
+ )
2627
+ );
2628
+ const confidence = input.confidence ?? computedConfidence;
2629
+ const confidenceBreakdown = {
2630
+ relationshipScore: input.confidenceBreakdown?.relationshipScore ?? relationshipScore,
2631
+ cohortLiftScore: input.confidenceBreakdown?.cohortLiftScore ?? cohortLiftScore,
2632
+ temporalProximityScore: input.confidenceBreakdown?.temporalProximityScore ?? 0.7,
2633
+ counterfactualBonus: input.confidenceBreakdown?.counterfactualBonus ?? 0,
2634
+ priorRecurrenceBonus: input.confidenceBreakdown?.priorRecurrenceBonus ?? priorBonus,
2635
+ finalConfidence: confidence
2636
+ };
2637
+ const id = input.id?.trim() || `fnd_${randomHex(10)}`;
2638
+ const expiresAt = new Date(
2639
+ Date.now() + 90 * 24 * 36e5
2640
+ ).toISOString();
2641
+ await db.prepare(
2642
+ `INSERT INTO investigation_findings (
2643
+ project_id, environment, id, incident_id, fingerprint,
2644
+ fingerprint_input_json, title, summary, causal_proof_tier,
2645
+ confidence, confidence_breakdown_json, symptom_entity_kind,
2646
+ symptom_entity_id, fault_locus_entity_kind, fault_locus_entity_id,
2647
+ root_cause_entity_kind, root_cause_entity_id, contributing_change_id,
2648
+ cohort_verdict, contending_trace_id, causal_chain_json,
2649
+ cited_evidence_ids_json, code_references_json,
2650
+ recovery_status, causal_verification_status, watchdog_step,
2651
+ actor_type, actor_id, agent_run_id, created_at, updated_at, expires_at
2652
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'not_applicable', 'unverified', 0, ?, ?, ?, ?, ?, ?)
2653
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
2654
+ title = excluded.title,
2655
+ summary = excluded.summary,
2656
+ causal_proof_tier = excluded.causal_proof_tier,
2657
+ confidence = excluded.confidence,
2658
+ confidence_breakdown_json = excluded.confidence_breakdown_json,
2659
+ contributing_change_id = COALESCE(excluded.contributing_change_id, investigation_findings.contributing_change_id),
2660
+ cohort_verdict = COALESCE(excluded.cohort_verdict, investigation_findings.cohort_verdict),
2661
+ causal_chain_json = excluded.causal_chain_json,
2662
+ cited_evidence_ids_json = excluded.cited_evidence_ids_json,
2663
+ updated_at = excluded.updated_at`
2664
+ ).bind(
2665
+ projectId,
2666
+ environment,
2667
+ id,
2668
+ input.incidentId ?? null,
2669
+ fpResult.fingerprint,
2670
+ JSON.stringify(fpResult.normalizedInput),
2671
+ input.title,
2672
+ input.summary,
2673
+ causalProofTier,
2674
+ confidence,
2675
+ JSON.stringify(confidenceBreakdown),
2676
+ input.symptomEntityKind,
2677
+ input.symptomEntityId,
2678
+ input.faultLocusEntityKind,
2679
+ input.faultLocusEntityId,
2680
+ input.rootCauseEntityKind ?? null,
2681
+ input.rootCauseEntityId ?? null,
2682
+ input.contributingChangeId ?? null,
2683
+ input.cohortVerdict ?? null,
2684
+ input.contendingTraceId ?? null,
2685
+ JSON.stringify(input.causalChain),
2686
+ JSON.stringify(input.citedEvidenceIds),
2687
+ JSON.stringify(input.codeReferences ?? []),
2688
+ input.actorType ?? "agent",
2689
+ input.actorId ?? "autosre-agent",
2690
+ input.agentRunId ?? null,
2691
+ nowIso,
2692
+ nowIso,
2693
+ expiresAt
2694
+ ).run();
2695
+ const created = await getInvestigationFinding(db, projectId, id, environment);
2696
+ if (!created) {
2697
+ throw new Error(`Failed to load created finding ${id}`);
2698
+ }
2699
+ return created;
2700
+ }
2701
+ async function getInvestigationFinding(db, projectId, findingId, environment) {
2702
+ const row = environment ? await db.prepare(
2703
+ `SELECT * FROM investigation_findings
2704
+ WHERE project_id = ? AND environment = ? AND id = ?`
2705
+ ).bind(projectId, environment.toLowerCase(), findingId).first() : await db.prepare(
2706
+ `SELECT * FROM investigation_findings
2707
+ WHERE project_id = ? AND id = ?
2708
+ ORDER BY created_at DESC
2709
+ LIMIT 1`
2710
+ ).bind(projectId, findingId).first();
2711
+ return row ? mapFindingRow(row) : null;
2712
+ }
2713
+ async function searchInvestigationFindings(db, options) {
2714
+ const conditions = ["project_id = ?"];
2715
+ const binds = [options.projectId];
2716
+ if (options.environment) {
2717
+ conditions.push("environment = ?");
2718
+ binds.push(options.environment.toLowerCase());
2719
+ }
2720
+ if (options.fingerprint) {
2721
+ conditions.push("fingerprint = ?");
2722
+ binds.push(options.fingerprint);
2723
+ }
2724
+ if (options.symptomEntityId) {
2725
+ conditions.push("symptom_entity_id = ?");
2726
+ binds.push(options.symptomEntityId);
2727
+ }
2728
+ if (options.rootCauseEntityId) {
2729
+ conditions.push(
2730
+ "(root_cause_entity_id = ? OR fault_locus_entity_id = ?)"
2731
+ );
2732
+ binds.push(options.rootCauseEntityId, options.rootCauseEntityId);
2733
+ }
2734
+ if (options.causalVerificationStatus) {
2735
+ conditions.push("causal_verification_status = ?");
2736
+ binds.push(options.causalVerificationStatus);
2737
+ }
2738
+ if (options.incidentId) {
2739
+ conditions.push("incident_id = ?");
2740
+ binds.push(options.incidentId);
2741
+ }
2742
+ const limit = Math.max(1, Math.min(100, options.limit ?? 25));
2743
+ binds.push(limit);
2744
+ const rows = await db.prepare(
2745
+ `SELECT * FROM investigation_findings
2746
+ WHERE ${conditions.join(" AND ")}
2747
+ ORDER BY created_at DESC
2748
+ LIMIT ?`
2749
+ ).bind(...binds).all();
2750
+ return rows.results.map(mapFindingRow);
2751
+ }
2752
+ var CounterfactualVerificationError = class extends Error {
2753
+ constructor(message) {
2754
+ super(message);
2755
+ this.name = "CounterfactualVerificationError";
2756
+ }
2757
+ };
2758
+ async function verifyPairedCounterfactualProof(db, projectId, findingId, input) {
2759
+ const finding = await getInvestigationFinding(
2760
+ db,
2761
+ projectId,
2762
+ findingId,
2763
+ input.environment
2764
+ );
2765
+ if (!finding) {
2766
+ throw new CounterfactualVerificationError(
2767
+ `Finding ${findingId} not found in project ${projectId}`
2768
+ );
2769
+ }
2770
+ const requiredTrials = Math.max(1, input.requiredTrials ?? 1);
2771
+ const uniquePassingIds = Array.from(new Set(input.passingRunIds));
2772
+ if (uniquePassingIds.length < requiredTrials) {
2773
+ throw new CounterfactualVerificationError(
2774
+ `Counterfactual requires at least ${requiredTrials} passing trial run(s), got ${uniquePassingIds.length}`
2775
+ );
2776
+ }
2777
+ if (uniquePassingIds.includes(input.failingRunId)) {
2778
+ throw new CounterfactualVerificationError(
2779
+ "failingRunId cannot also be listed in passingRunIds"
2780
+ );
2781
+ }
2782
+ const loadRunVerification = async (runId) => {
2783
+ const row = await db.prepare(
2784
+ `SELECT
2785
+ r.id AS run_id,
2786
+ COALESCE(cr.eval_case_id, r.eval_case_id) AS eval_case_id,
2787
+ r.service_version,
2788
+ r.commit_sha,
2789
+ r.candidate_agent_version,
2790
+ r.candidate_prompt_version,
2791
+ r.input_payload_hash AS run_input_hash,
2792
+ r.failure_fingerprint AS run_failure_fp,
2793
+ r.pass_count,
2794
+ r.fail_count,
2795
+ cr.passed AS result_passed,
2796
+ cr.input_payload_hash AS result_input_hash,
2797
+ cr.failure_fingerprint AS result_failure_fp
2798
+ FROM eval_runs r
2799
+ LEFT JOIN eval_case_results cr
2800
+ ON cr.project_id = r.project_id AND cr.run_id = r.id AND cr.eval_case_id = ?
2801
+ WHERE r.project_id = ? AND r.id = ?
2802
+ LIMIT 1`
2803
+ ).bind(input.evalCaseId, projectId, runId).first();
2804
+ return row;
2805
+ };
2806
+ const failingRow = await loadRunVerification(input.failingRunId);
2807
+ if (!failingRow) {
2808
+ throw new CounterfactualVerificationError(
2809
+ `Failing eval run ${input.failingRunId} not found in project ${projectId}`
2810
+ );
2811
+ }
2812
+ if (failingRow.eval_case_id !== input.evalCaseId) {
2813
+ throw new CounterfactualVerificationError(
2814
+ `Failing eval run ${input.failingRunId} does not match evalCaseId ${input.evalCaseId}`
2815
+ );
2816
+ }
2817
+ const failingPassed = failingRow.result_passed !== null ? Number(failingRow.result_passed) === 1 : Number(failingRow.fail_count) === 0 && Number(failingRow.pass_count) > 0;
2818
+ if (failingPassed) {
2819
+ throw new CounterfactualVerificationError(
2820
+ `Baseline run ${input.failingRunId} did not fail (passed=1)`
2821
+ );
2822
+ }
2823
+ const failingInputHash = failingRow.result_input_hash ?? failingRow.run_input_hash;
2824
+ if (!failingInputHash) {
2825
+ throw new CounterfactualVerificationError(
2826
+ `Failing eval run ${input.failingRunId} is missing input_payload_hash`
2827
+ );
2828
+ }
2829
+ const failingFp = failingRow.result_failure_fp ?? failingRow.run_failure_fp;
2830
+ if (failingFp !== finding.fingerprint) {
2831
+ throw new CounterfactualVerificationError(
2832
+ `Failing run fingerprint (${failingFp ?? "null"}) does not match finding fingerprint (${finding.fingerprint})`
2833
+ );
2834
+ }
2835
+ const failingVersion = failingRow.service_version ?? failingRow.candidate_agent_version ?? failingRow.candidate_prompt_version;
2836
+ let passingVersion = null;
2837
+ let passingCommitSha = null;
2838
+ for (const passRunId of uniquePassingIds) {
2839
+ const passRow = await loadRunVerification(passRunId);
2840
+ if (!passRow) {
2841
+ throw new CounterfactualVerificationError(
2842
+ `Passing eval run ${passRunId} not found in project ${projectId}`
2843
+ );
2844
+ }
2845
+ if (passRow.eval_case_id !== input.evalCaseId) {
2846
+ throw new CounterfactualVerificationError(
2847
+ `Passing eval run ${passRunId} does not match evalCaseId ${input.evalCaseId}`
2848
+ );
2849
+ }
2850
+ const didPass = passRow.result_passed !== null ? Number(passRow.result_passed) === 1 : Number(passRow.pass_count) > 0 && Number(passRow.fail_count) === 0;
2851
+ if (!didPass) {
2852
+ throw new CounterfactualVerificationError(
2853
+ `Candidate run ${passRunId} did not pass`
2854
+ );
2855
+ }
2856
+ const passInputHash = passRow.result_input_hash ?? passRow.run_input_hash;
2857
+ if (passInputHash !== failingInputHash) {
2858
+ throw new CounterfactualVerificationError(
2859
+ `Candidate run ${passRunId} input_payload_hash (${passInputHash ?? "null"}) does not match failing run (${failingInputHash})`
2860
+ );
2861
+ }
2862
+ passingVersion = passRow.service_version ?? passRow.candidate_agent_version ?? passRow.candidate_prompt_version ?? passingVersion;
2863
+ passingCommitSha = passRow.commit_sha ?? passingCommitSha;
2864
+ }
2865
+ if (!failingVersion && !failingRow.commit_sha && !passingVersion && !passingCommitSha) {
2866
+ throw new CounterfactualVerificationError(
2867
+ "Counterfactual runs must record version, commit_sha, or candidate version lineage"
2868
+ );
2869
+ }
2870
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
2871
+ const proof = {
2872
+ evalCaseId: input.evalCaseId,
2873
+ inputPayloadHash: failingInputHash,
2874
+ failingRunId: input.failingRunId,
2875
+ failingVersion: failingVersion ?? null,
2876
+ failingCommitSha: failingRow.commit_sha ?? null,
2877
+ failureFingerprint: finding.fingerprint,
2878
+ passingRunIds: uniquePassingIds,
2879
+ passingVersion,
2880
+ passingCommitSha,
2881
+ requiredTrials,
2882
+ verifiedAt: nowIso
2883
+ };
2884
+ const updatedChain = [
2885
+ ...finding.causalChain,
2886
+ {
2887
+ subjectKind: "finding",
2888
+ subjectId: finding.id,
2889
+ predicate: "verified_by_counterfactual",
2890
+ objectKind: "eval",
2891
+ objectId: input.evalCaseId,
2892
+ relationshipCertainty: "exact",
2893
+ causalProofTier: "counterfactual_verified",
2894
+ confidence: 0.99,
2895
+ reason: `Paired counterfactual verified on evalCase ${input.evalCaseId}: run ${input.failingRunId} (${failingVersion ?? "baseline"}) failed with ${finding.fingerprint} and ${uniquePassingIds.join(", ")} (${passingVersion ?? "candidate"}) passed on payload ${failingInputHash}`
2896
+ }
2897
+ ];
2898
+ const updatedBreakdown = {
2899
+ ...finding.confidenceBreakdown,
2900
+ counterfactualBonus: 0.25,
2901
+ finalConfidence: 0.99
2902
+ };
2903
+ await db.prepare(
2904
+ `UPDATE investigation_findings
2905
+ SET causal_proof_tier = 'counterfactual_verified',
2906
+ causal_verification_status = 'counterfactual_verified',
2907
+ confidence = 0.99,
2908
+ confidence_breakdown_json = ?,
2909
+ counterfactual_proof_json = ?,
2910
+ causal_chain_json = ?,
2911
+ updated_at = ?
2912
+ WHERE project_id = ? AND environment = ? AND id = ?`
2913
+ ).bind(
2914
+ JSON.stringify(updatedBreakdown),
2915
+ JSON.stringify(proof),
2916
+ JSON.stringify(updatedChain),
2917
+ nowIso,
2918
+ projectId,
2919
+ finding.environment,
2920
+ finding.id
2921
+ ).run();
2922
+ const updated = await getInvestigationFinding(
2923
+ db,
2924
+ projectId,
2925
+ finding.id,
2926
+ finding.environment
2927
+ );
2928
+ if (!updated) {
2929
+ throw new Error(`Failed to reload finding ${finding.id}`);
2930
+ }
2931
+ return updated;
2932
+ }
2933
+
2934
+ // src/lib/webhook-url.ts
2935
+ var WebhookUrlError = class extends Error {
2936
+ };
2937
+ var parseIpv4 = (host) => {
2938
+ const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
2939
+ if (!m) return null;
2940
+ const octets = m.slice(1).map((o) => Number.parseInt(o, 10));
2941
+ return octets.every((o) => o <= 255) ? octets : null;
2942
+ };
2943
+ var isPrivateIpv4 = (o) => o[0] === 0 || // "this" network / unspecified
2944
+ o[0] === 10 || o[0] === 127 || o[0] === 100 && o[1] >= 64 && o[1] <= 127 || // CGNAT
2945
+ o[0] === 169 && o[1] === 254 || // link-local incl. cloud metadata
2946
+ o[0] === 172 && o[1] >= 16 && o[1] <= 31 || o[0] === 192 && o[1] === 168 || o[0] >= 224;
2947
+ var isPrivateIpv6 = (host) => {
2948
+ const literal = host.replace(/^\[|\]$/g, "").toLowerCase();
2949
+ if (literal === "::" || literal === "::1") return true;
2950
+ const dotted = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(literal);
2951
+ if (dotted) {
2952
+ const v4 = parseIpv4(dotted[1]);
2953
+ return v4 ? isPrivateIpv4(v4) : true;
2954
+ }
2955
+ const hex = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(literal);
2956
+ if (hex) {
2957
+ const hi = Number.parseInt(hex[1], 16);
2958
+ const lo = Number.parseInt(hex[2], 16);
2959
+ return isPrivateIpv4([hi >> 8, hi & 255, lo >> 8, lo & 255]);
2960
+ }
2961
+ const first = Number.parseInt(literal.split(":")[0] || "0", 16);
2962
+ if (Number.isNaN(first)) return true;
2963
+ if ((first & 65024) === 64512) return true;
2964
+ if ((first & 65472) === 65152) return true;
2965
+ if ((first & 65280) === 65280) return true;
2966
+ return false;
2967
+ };
2968
+ var isPrivateWebhookHost = (hostname) => {
2969
+ const host = hostname.toLowerCase().replace(/\.$/, "");
2970
+ if (!host) return true;
2971
+ if (host === "localhost" || host.endsWith(".localhost")) return true;
2972
+ if (host.endsWith(".local") || host.endsWith(".internal")) return true;
2973
+ if (host.startsWith("[")) return isPrivateIpv6(host);
2974
+ const v4 = parseIpv4(host);
2975
+ if (v4) return isPrivateIpv4(v4);
2976
+ return !host.includes(".");
2977
+ };
2978
+ var assertSafeWebhookUrl = (raw, options = {}) => {
2979
+ let url;
2980
+ try {
2981
+ url = new URL(raw);
2982
+ } catch {
2983
+ throw new WebhookUrlError("channel.url must be an absolute http(s) URL");
2984
+ }
2985
+ if (url.protocol !== "https:" && url.protocol !== "http:") {
2986
+ throw new WebhookUrlError("channel.url must be http(s)://");
2987
+ }
2988
+ if (url.username || url.password) {
2989
+ throw new WebhookUrlError(
2990
+ "channel.url must not embed credentials; use channel.headers"
2991
+ );
2992
+ }
2993
+ if (!options.allowPrivateNetworks && isPrivateWebhookHost(url.hostname)) {
2994
+ throw new WebhookUrlError(
2995
+ `channel.url host "${url.hostname}" is not a public address (set ALERT_WEBHOOK_ALLOW_PRIVATE=true to allow internal receivers)`
2996
+ );
2997
+ }
2998
+ return url;
2999
+ };
3000
+ var resolveWebhookHost = async (hostname, lookup) => {
3001
+ if (parseIpv4(hostname) || hostname.startsWith("[")) return "public";
3002
+ let addresses;
3003
+ try {
3004
+ addresses = await lookup(hostname);
3005
+ } catch {
3006
+ return "unresolvable";
3007
+ }
3008
+ if (addresses.length === 0) return "unresolvable";
3009
+ const isPrivate = addresses.some((address) => {
3010
+ const v4 = parseIpv4(address);
3011
+ if (v4) return isPrivateIpv4(v4);
3012
+ return isPrivateIpv6(`[${address}]`);
3013
+ });
3014
+ return isPrivate ? "private" : "public";
3015
+ };
3016
+ var redactWebhookUrl = (raw) => {
3017
+ try {
3018
+ const url = new URL(raw);
3019
+ const hasPath = url.pathname !== "/" || url.search !== "";
3020
+ return `${url.protocol}//${url.host}${hasPath ? "/\u2026" : ""}`;
3021
+ } catch {
3022
+ return "<invalid url>";
3023
+ }
3024
+ };
3025
+
3026
+ // src/lib/autosre-remediation-gate.ts
3027
+ var mapIncidentRow = (row) => ({
3028
+ id: row.id,
3029
+ projectId: row.project_id,
3030
+ environment: row.environment,
3031
+ dedupKey: row.dedup_key,
3032
+ status: row.status,
3033
+ targetNodeId: row.target_node_id,
3034
+ symptomAnchorKind: row.symptom_anchor_kind,
3035
+ symptomAnchorId: row.symptom_anchor_id,
3036
+ symptomFingerprint: row.symptom_fingerprint,
3037
+ alertCount: Number(row.alert_count ?? 1),
3038
+ latestBundle: row.latest_bundle_json ? parseJsonRecord(row.latest_bundle_json) : null,
3039
+ leaseOwnerId: row.lease_owner_id,
3040
+ leaseExpiresAt: row.lease_expires_at,
3041
+ dispatchAttempts: Number(row.dispatch_attempts ?? 0),
3042
+ lastDispatchedAt: row.last_dispatched_at,
3043
+ openedAt: row.opened_at,
3044
+ updatedAt: row.updated_at,
3045
+ resolvedAt: row.resolved_at,
3046
+ expiresAt: row.expires_at
3047
+ });
3048
+ async function openOrDeduplicateIncident(db, projectId, input) {
3049
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
3050
+ const environment = (input.environment || "default").trim().toLowerCase();
3051
+ const category = (input.symptomCategory ?? input.symptomFingerprint ?? `${input.symptomAnchorKind}:${input.symptomAnchorId}`).trim();
3052
+ const dedupKey = await sha256Hex(
3053
+ `${projectId}::${environment}::${input.targetNodeId}::${category}`
3054
+ );
3055
+ const existing = await db.prepare(
3056
+ `SELECT * FROM incident_investigations
3057
+ WHERE project_id = ?
3058
+ AND environment = ?
3059
+ AND dedup_key = ?
3060
+ AND status IN ('open', 'dispatched', 'leased', 'remediating', 'verifying')
3061
+ ORDER BY opened_at DESC
3062
+ LIMIT 1`
3063
+ ).bind(projectId, environment, dedupKey).first();
3064
+ if (existing) {
3065
+ await db.prepare(
3066
+ `UPDATE incident_investigations
3067
+ SET alert_count = alert_count + 1,
3068
+ latest_bundle_json = COALESCE(?, latest_bundle_json),
3069
+ symptom_fingerprint = COALESCE(?, symptom_fingerprint),
3070
+ updated_at = ?
3071
+ WHERE project_id = ? AND environment = ? AND id = ?`
3072
+ ).bind(
3073
+ input.latestBundle ? JSON.stringify(input.latestBundle) : null,
3074
+ input.symptomFingerprint ?? null,
3075
+ nowIso,
3076
+ projectId,
3077
+ environment,
3078
+ existing.id
3079
+ ).run();
3080
+ const updated = await getIncidentInvestigation(
3081
+ db,
3082
+ projectId,
3083
+ existing.id,
3084
+ environment
3085
+ );
3086
+ return {
3087
+ incident: updated ?? mapIncidentRow(existing),
3088
+ deduplicated: true
3089
+ };
3090
+ }
3091
+ const id = input.id?.trim() || `inc_${randomHex(10)}`;
3092
+ const expiresAt = new Date(
3093
+ Date.now() + 30 * 24 * 36e5
3094
+ ).toISOString();
3095
+ await db.prepare(
3096
+ `INSERT INTO incident_investigations (
3097
+ project_id, environment, id, dedup_key, status, target_node_id,
3098
+ symptom_anchor_kind, symptom_anchor_id, symptom_fingerprint,
3099
+ alert_count, latest_bundle_json, dispatch_attempts,
3100
+ opened_at, updated_at, expires_at
3101
+ ) VALUES (?, ?, ?, ?, 'open', ?, ?, ?, ?, 1, ?, 0, ?, ?, ?)`
3102
+ ).bind(
3103
+ projectId,
3104
+ environment,
3105
+ id,
3106
+ dedupKey,
3107
+ input.targetNodeId,
3108
+ input.symptomAnchorKind,
3109
+ input.symptomAnchorId,
3110
+ input.symptomFingerprint ?? null,
3111
+ input.latestBundle ? JSON.stringify(input.latestBundle) : null,
3112
+ nowIso,
3113
+ nowIso,
3114
+ expiresAt
3115
+ ).run();
3116
+ const created = await getIncidentInvestigation(db, projectId, id, environment);
3117
+ if (!created) {
3118
+ throw new Error(`Failed to load created incident ${id}`);
3119
+ }
3120
+ return { incident: created, deduplicated: false };
3121
+ }
3122
+ async function getIncidentInvestigation(db, projectId, incidentId, environment) {
3123
+ const row = environment ? await db.prepare(
3124
+ `SELECT * FROM incident_investigations
3125
+ WHERE project_id = ? AND environment = ? AND id = ?`
3126
+ ).bind(projectId, environment.toLowerCase(), incidentId).first() : await db.prepare(
3127
+ `SELECT * FROM incident_investigations
3128
+ WHERE project_id = ? AND id = ?
3129
+ ORDER BY opened_at DESC
3130
+ LIMIT 1`
3131
+ ).bind(projectId, incidentId).first();
3132
+ return row ? mapIncidentRow(row) : null;
3133
+ }
3134
+ async function listIncidentInvestigations(db, options) {
3135
+ const conditions = ["project_id = ?"];
3136
+ const binds = [options.projectId];
3137
+ if (options.environment) {
3138
+ conditions.push("environment = ?");
3139
+ binds.push(options.environment.toLowerCase());
3140
+ }
3141
+ if (options.status) {
3142
+ conditions.push("status = ?");
3143
+ binds.push(options.status);
3144
+ }
3145
+ const limit = Math.max(1, Math.min(100, options.limit ?? 50));
3146
+ binds.push(limit);
3147
+ const rows = await db.prepare(
3148
+ `SELECT * FROM incident_investigations
3149
+ WHERE ${conditions.join(" AND ")}
3150
+ ORDER BY opened_at DESC
3151
+ LIMIT ?`
3152
+ ).bind(...binds).all();
3153
+ return rows.results.map(mapIncidentRow);
3154
+ }
3155
+ async function acquireIncidentLease(db, projectId, incidentId, input) {
3156
+ const nowIso = input.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
3157
+ const incident = await getIncidentInvestigation(
3158
+ db,
3159
+ projectId,
3160
+ incidentId,
3161
+ input.environment
3162
+ );
3163
+ if (!incident) {
3164
+ return {
3165
+ acquired: false,
3166
+ reason: `Incident ${incidentId} not found`,
3167
+ conflictOwnerId: null,
3168
+ leaseExpiresAt: null
3169
+ };
3170
+ }
3171
+ if (incident.leaseOwnerId && incident.leaseOwnerId !== input.runnerId && incident.leaseExpiresAt && incident.leaseExpiresAt > nowIso) {
3172
+ return {
3173
+ acquired: false,
3174
+ reason: `Incident ${incidentId} is already leased by runner ${incident.leaseOwnerId} until ${incident.leaseExpiresAt}`,
3175
+ conflictOwnerId: incident.leaseOwnerId,
3176
+ leaseExpiresAt: incident.leaseExpiresAt
3177
+ };
3178
+ }
3179
+ const durationSec = Math.max(
3180
+ 15,
3181
+ Math.min(600, input.leaseDurationSeconds ?? 120)
3182
+ );
3183
+ const leaseExpiresAt = new Date(
3184
+ Date.parse(nowIso) + durationSec * 1e3
3185
+ ).toISOString();
3186
+ const nextStatus = incident.status === "open" || incident.status === "dispatched" ? "leased" : incident.status;
3187
+ await db.prepare(
3188
+ `UPDATE incident_investigations
3189
+ SET lease_owner_id = ?,
3190
+ lease_expires_at = ?,
3191
+ status = ?,
3192
+ updated_at = ?
3193
+ WHERE project_id = ? AND environment = ? AND id = ?`
3194
+ ).bind(
3195
+ input.runnerId,
3196
+ leaseExpiresAt,
3197
+ nextStatus,
3198
+ nowIso,
3199
+ projectId,
3200
+ incident.environment,
3201
+ incident.id
3202
+ ).run();
3203
+ const updated = await getIncidentInvestigation(
3204
+ db,
3205
+ projectId,
3206
+ incident.id,
3207
+ incident.environment
3208
+ );
3209
+ return {
3210
+ acquired: true,
3211
+ incident: updated ?? {
3212
+ ...incident,
3213
+ leaseOwnerId: input.runnerId,
3214
+ leaseExpiresAt,
3215
+ status: nextStatus,
3216
+ updatedAt: nowIso
3217
+ }
3218
+ };
3219
+ }
3220
+ async function computeHmacSha256Hex(secret, body) {
3221
+ const enc = new TextEncoder();
3222
+ const key = await crypto.subtle.importKey(
3223
+ "raw",
3224
+ enc.encode(secret),
3225
+ { name: "HMAC", hash: "SHA-256" },
3226
+ false,
3227
+ ["sign"]
3228
+ );
3229
+ const sig = await crypto.subtle.sign("HMAC", key, enc.encode(body));
3230
+ const bytes = new Uint8Array(sig);
3231
+ let hex = "";
3232
+ for (const b of bytes) hex += b.toString(16).padStart(2, "0");
3233
+ return hex;
3234
+ }
3235
+ async function dispatchIncidentWebhook(db, projectId, incidentId, webhook) {
3236
+ const incident = await getIncidentInvestigation(db, projectId, incidentId);
3237
+ if (!incident) {
3238
+ return { delivered: false, error: "Incident not found" };
3239
+ }
3240
+ if (incident.dispatchAttempts >= 5) {
3241
+ return { delivered: false, error: "MAX_DISPATCH_ATTEMPTS (5) exceeded" };
3242
+ }
3243
+ const safeUrl = assertSafeWebhookUrl(webhook.url, {
3244
+ allowPrivateNetworks: webhook.allowPrivateNetworks
3245
+ });
3246
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
3247
+ const body = JSON.stringify({
3248
+ event: "autosre.incident.opened",
3249
+ incidentId: incident.id,
3250
+ projectId: incident.projectId,
3251
+ environment: incident.environment,
3252
+ targetNodeId: incident.targetNodeId,
3253
+ dedupKey: incident.dedupKey,
3254
+ symptomAnchor: {
3255
+ kind: incident.symptomAnchorKind,
3256
+ id: incident.symptomAnchorId
3257
+ },
3258
+ bundle: incident.latestBundle,
3259
+ dispatchedAt: nowIso
3260
+ });
3261
+ const headers = {
3262
+ "Content-Type": "application/json",
3263
+ "X-Obs-Incident-Id": incident.id,
3264
+ "X-Obs-Deduplication-Key": incident.dedupKey
3265
+ };
3266
+ if (webhook.secret) {
3267
+ const hmac = await computeHmacSha256Hex(webhook.secret, body);
3268
+ headers["X-Obs-Signature-256"] = `sha256=${hmac}`;
3269
+ }
3270
+ const fetchFn = webhook.fetchImpl ?? fetch;
3271
+ let delivered = false;
3272
+ let status;
3273
+ let errorMsg;
3274
+ try {
3275
+ const res = await fetchFn(safeUrl.toString(), {
3276
+ method: "POST",
3277
+ headers,
3278
+ body
3279
+ });
3280
+ status = res.status;
3281
+ delivered = res.ok;
3282
+ if (!res.ok) {
3283
+ errorMsg = `HTTP ${res.status}`;
3284
+ }
3285
+ } catch (err) {
3286
+ errorMsg = err instanceof Error ? err.message : String(err);
3287
+ }
3288
+ await db.prepare(
3289
+ `UPDATE incident_investigations
3290
+ SET dispatch_attempts = dispatch_attempts + 1,
3291
+ last_dispatched_at = ?,
3292
+ status = CASE WHEN ? = 1 AND status = 'open' THEN 'dispatched' ELSE status END,
3293
+ updated_at = ?
3294
+ WHERE project_id = ? AND environment = ? AND id = ?`
3295
+ ).bind(
3296
+ nowIso,
3297
+ delivered ? 1 : 0,
3298
+ nowIso,
3299
+ projectId,
3300
+ incident.environment,
3301
+ incident.id
3302
+ ).run();
3303
+ return { delivered, status, error: errorMsg };
3304
+ }
3305
+ var mapProposalRow = (row, grantToken) => ({
3306
+ id: row.id,
3307
+ projectId: row.project_id,
3308
+ environment: row.environment,
3309
+ incidentId: row.incident_id,
3310
+ findingId: row.finding_id,
3311
+ idempotencyKey: row.idempotency_key,
3312
+ actionKind: row.action_kind,
3313
+ targetNodeId: row.target_node_id,
3314
+ expectedCurrentState: parseJsonRecord(
3315
+ row.expected_current_state_json
3316
+ ),
3317
+ desiredTargetState: parseJsonRecord(
3318
+ row.desired_target_state_json
3319
+ ),
3320
+ requestedAutonomy: row.requested_autonomy,
3321
+ effectiveAutonomy: row.effective_autonomy,
3322
+ policyDecisionReason: row.policy_decision_reason,
3323
+ status: row.status,
3324
+ grantExpiresAt: row.grant_expires_at,
3325
+ approvedByUserId: row.approved_by_user_id,
3326
+ approvedAt: row.approved_at,
3327
+ claimedByRunnerId: row.claimed_by_runner_id,
3328
+ claimedAt: row.claimed_at,
3329
+ claimExpiresAt: row.claim_expires_at,
3330
+ remediationAgentRunId: row.remediation_agent_run_id,
3331
+ resultingChangeId: row.resulting_change_id,
3332
+ failureReason: row.failure_reason,
3333
+ createdAt: row.created_at,
3334
+ updatedAt: row.updated_at,
3335
+ expiresAt: row.expires_at,
3336
+ ...grantToken ? { grantToken } : {}
3337
+ });
3338
+ function evaluateActionPolicyMatrix(actionKind, causalProofTier, recentProposalsLastHour) {
3339
+ if (recentProposalsLastHour >= 3) {
3340
+ return {
3341
+ effectiveAutonomy: AgentAutonomyLevel.BlockedByPolicy,
3342
+ reason: `Circuit breaker tripped: ${recentProposalsLastHour} remediation proposals already recorded in the last hour for this target/fingerprint (max 3/hr)`
3343
+ };
3344
+ }
3345
+ switch (actionKind) {
3346
+ case "open_code_pr": {
3347
+ if (causalProofTier === "counterfactual_verified" || causalProofTier === "cohort_rate_isolated" || causalProofTier === "corroborated_candidate") {
3348
+ return {
3349
+ effectiveAutonomy: AgentAutonomyLevel.AutonomousWrite,
3350
+ reason: `Non-merging PR creation permitted as autonomous_write for causalProofTier=${causalProofTier}`
3351
+ };
3352
+ }
3353
+ return {
3354
+ effectiveAutonomy: AgentAutonomyLevel.HumanApprovedWrite,
3355
+ reason: `open_code_pr requires human_approved_write when causalProofTier=${causalProofTier}`
3356
+ };
3357
+ }
3358
+ case "toggle_feature_flag":
3359
+ case "rollback_deploy": {
3360
+ if (causalProofTier === "counterfactual_verified" || causalProofTier === "cohort_rate_isolated") {
3361
+ return {
3362
+ effectiveAutonomy: AgentAutonomyLevel.AutonomousWrite,
3363
+ reason: `${actionKind} authorized as autonomous_write with causalProofTier=${causalProofTier}`
3364
+ };
3365
+ }
3366
+ if (causalProofTier === "corroborated_candidate") {
3367
+ return {
3368
+ effectiveAutonomy: AgentAutonomyLevel.HumanApprovedWrite,
3369
+ reason: `${actionKind} with causalProofTier=corroborated_candidate (structural overlap or sequential cutover without concurrent cohort isolation) requires human_approved_write`
3370
+ };
3371
+ }
3372
+ return {
3373
+ effectiveAutonomy: AgentAutonomyLevel.BlockedByPolicy,
3374
+ reason: `${actionKind} blocked_by_policy: causalProofTier=${causalProofTier} (unverified fault locus cannot authorize production ${actionKind})`
3375
+ };
3376
+ }
3377
+ case "apply_config":
3378
+ case "promote_agent_prompt": {
3379
+ if (causalProofTier === "counterfactual_verified") {
3380
+ return {
3381
+ effectiveAutonomy: AgentAutonomyLevel.AutonomousWrite,
3382
+ reason: `${actionKind} authorized as autonomous_write with counterfactual_verified proof`
3383
+ };
3384
+ }
3385
+ if (causalProofTier === "cohort_rate_isolated" || causalProofTier === "corroborated_candidate") {
3386
+ return {
3387
+ effectiveAutonomy: AgentAutonomyLevel.HumanApprovedWrite,
3388
+ reason: `${actionKind} requires counterfactual_verified for autonomous_write; parked for human_approved_write (tier=${causalProofTier})`
3389
+ };
3390
+ }
3391
+ return {
3392
+ effectiveAutonomy: AgentAutonomyLevel.BlockedByPolicy,
3393
+ reason: `${actionKind} blocked_by_policy: causalProofTier=${causalProofTier}`
3394
+ };
3395
+ }
3396
+ }
3397
+ }
3398
+ var RemediationGateError = class extends Error {
3399
+ constructor(message, statusCode = 400) {
3400
+ super(message);
3401
+ this.statusCode = statusCode;
3402
+ this.name = "RemediationGateError";
3403
+ }
3404
+ };
3405
+ async function proposeRemediationAction(db, projectId, input) {
3406
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
3407
+ const environment = (input.environment || "default").trim().toLowerCase();
3408
+ const existing = await db.prepare(
3409
+ `SELECT * FROM remediation_proposals
3410
+ WHERE project_id = ? AND environment = ? AND idempotency_key = ?`
3411
+ ).bind(projectId, environment, input.idempotencyKey).first();
3412
+ if (existing) {
3413
+ if (existing.action_kind !== input.actionKind || existing.target_node_id !== input.targetNodeId) {
3414
+ throw new RemediationGateError(
3415
+ `Idempotency key ${input.idempotencyKey} already used for a different action or target node`,
3416
+ 409
3417
+ );
3418
+ }
3419
+ return mapProposalRow(existing);
3420
+ }
3421
+ const finding = await getInvestigationFinding(
3422
+ db,
3423
+ projectId,
3424
+ input.findingId,
3425
+ environment
3426
+ );
3427
+ if (!finding) {
3428
+ throw new RemediationGateError(
3429
+ `Finding ${input.findingId} not found in environment ${environment}`,
3430
+ 404
3431
+ );
3432
+ }
3433
+ const oneHourAgo = new Date(Date.now() - 36e5).toISOString();
3434
+ const recentCountRow = await db.prepare(
3435
+ `SELECT COUNT(*) AS cnt FROM remediation_proposals
3436
+ WHERE project_id = ?
3437
+ AND environment = ?
3438
+ AND target_node_id = ?
3439
+ AND created_at >= ?`
3440
+ ).bind(projectId, environment, input.targetNodeId, oneHourAgo).first();
3441
+ const recentCount = Number(recentCountRow?.cnt ?? 0);
3442
+ const requestedAutonomy = input.requestedAutonomy ?? AgentAutonomyLevel.AutonomousWrite;
3443
+ const policy = evaluateActionPolicyMatrix(
3444
+ input.actionKind,
3445
+ finding.causalProofTier,
3446
+ recentCount
3447
+ );
3448
+ let effectiveAutonomy = policy.effectiveAutonomy;
3449
+ let policyReason = policy.reason;
3450
+ if (requestedAutonomy === AgentAutonomyLevel.HumanApprovedWrite && effectiveAutonomy === AgentAutonomyLevel.AutonomousWrite) {
3451
+ effectiveAutonomy = AgentAutonomyLevel.HumanApprovedWrite;
3452
+ policyReason = `Caller explicitly requested human_approved_write (${policy.reason})`;
3453
+ }
3454
+ let status = "pending_human_approval";
3455
+ let grantToken;
3456
+ let grantTokenHash = null;
3457
+ let grantExpiresAt = null;
3458
+ if (effectiveAutonomy === AgentAutonomyLevel.AutonomousWrite) {
3459
+ status = "grant_issued";
3460
+ grantToken = `rgt_${randomHex(16)}`;
3461
+ grantTokenHash = await sha256Hex(grantToken);
3462
+ grantExpiresAt = new Date(Date.now() + 5 * 6e4).toISOString();
3463
+ } else if (effectiveAutonomy === AgentAutonomyLevel.BlockedByPolicy) {
3464
+ status = "rejected";
3465
+ }
3466
+ const id = `rem_${randomHex(10)}`;
3467
+ const expiresAt = new Date(
3468
+ Date.now() + 30 * 24 * 36e5
3469
+ ).toISOString();
3470
+ await db.prepare(
3471
+ `INSERT INTO remediation_proposals (
3472
+ project_id, environment, id, incident_id, finding_id,
3473
+ idempotency_key, action_kind, target_node_id,
3474
+ expected_current_state_json, desired_target_state_json,
3475
+ requested_autonomy, effective_autonomy, policy_decision_reason,
3476
+ status, grant_token_hash, grant_expires_at,
3477
+ created_at, updated_at, expires_at
3478
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
3479
+ ).bind(
3480
+ projectId,
3481
+ environment,
3482
+ id,
3483
+ input.incidentId,
3484
+ input.findingId,
3485
+ input.idempotencyKey,
3486
+ input.actionKind,
3487
+ input.targetNodeId,
3488
+ JSON.stringify(input.expectedCurrentState),
3489
+ JSON.stringify(input.desiredTargetState),
3490
+ requestedAutonomy,
3491
+ effectiveAutonomy,
3492
+ policyReason,
3493
+ status,
3494
+ grantTokenHash,
3495
+ grantExpiresAt,
3496
+ nowIso,
3497
+ nowIso,
3498
+ expiresAt
3499
+ ).run();
3500
+ const created = await db.prepare(
3501
+ `SELECT * FROM remediation_proposals
3502
+ WHERE project_id = ? AND environment = ? AND id = ?`
3503
+ ).bind(projectId, environment, id).first();
3504
+ if (!created) {
3505
+ throw new Error(`Failed to load created remediation proposal ${id}`);
3506
+ }
3507
+ return mapProposalRow(created, grantToken);
3508
+ }
3509
+ async function getRemediationProposal(db, projectId, proposalId, environment) {
3510
+ const row = environment ? await db.prepare(
3511
+ `SELECT * FROM remediation_proposals
3512
+ WHERE project_id = ? AND environment = ? AND id = ?`
3513
+ ).bind(projectId, environment.toLowerCase(), proposalId).first() : await db.prepare(
3514
+ `SELECT * FROM remediation_proposals
3515
+ WHERE project_id = ? AND id = ?
3516
+ ORDER BY created_at DESC
3517
+ LIMIT 1`
3518
+ ).bind(projectId, proposalId).first();
3519
+ return row ? mapProposalRow(row) : null;
3520
+ }
3521
+ async function listRemediationProposals(db, options) {
3522
+ const conditions = ["project_id = ?"];
3523
+ const binds = [options.projectId];
3524
+ if (options.environment) {
3525
+ conditions.push("environment = ?");
3526
+ binds.push(options.environment.toLowerCase());
3527
+ }
3528
+ if (options.status) {
3529
+ conditions.push("status = ?");
3530
+ binds.push(options.status);
3531
+ }
3532
+ if (options.incidentId) {
3533
+ conditions.push("incident_id = ?");
3534
+ binds.push(options.incidentId);
3535
+ }
3536
+ if (options.findingId) {
3537
+ conditions.push("finding_id = ?");
3538
+ binds.push(options.findingId);
3539
+ }
3540
+ const limit = Math.max(1, Math.min(100, options.limit ?? 50));
3541
+ binds.push(limit);
3542
+ const rows = await db.prepare(
3543
+ `SELECT * FROM remediation_proposals
3544
+ WHERE ${conditions.join(" AND ")}
3545
+ ORDER BY created_at DESC
3546
+ LIMIT ?`
3547
+ ).bind(...binds).all();
3548
+ return rows.results.map((r) => mapProposalRow(r));
3549
+ }
3550
+ async function approveRemediationProposal(db, projectId, proposalId, options) {
3551
+ const row = options.environment ? await db.prepare(
3552
+ `SELECT * FROM remediation_proposals
3553
+ WHERE project_id = ? AND environment = ? AND id = ?`
3554
+ ).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
3555
+ `SELECT * FROM remediation_proposals
3556
+ WHERE project_id = ? AND id = ?
3557
+ LIMIT 1`
3558
+ ).bind(projectId, proposalId).first();
3559
+ if (!row) {
3560
+ throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
3561
+ }
3562
+ if (row.status !== "pending_human_approval") {
3563
+ throw new RemediationGateError(
3564
+ `Proposal ${proposalId} cannot be approved from status=${row.status}`,
3565
+ 409
3566
+ );
3567
+ }
3568
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
3569
+ const grantToken = `rgt_${randomHex(16)}`;
3570
+ const grantTokenHash = await sha256Hex(grantToken);
3571
+ const grantExpiresAt = new Date(Date.now() + 5 * 6e4).toISOString();
3572
+ await db.prepare(
3573
+ `UPDATE remediation_proposals
3574
+ SET status = 'grant_issued',
3575
+ grant_token_hash = ?,
3576
+ grant_expires_at = ?,
3577
+ approved_by_user_id = ?,
3578
+ approved_at = ?,
3579
+ updated_at = ?
3580
+ WHERE project_id = ? AND environment = ? AND id = ?`
3581
+ ).bind(
3582
+ grantTokenHash,
3583
+ grantExpiresAt,
3584
+ options.approvedByUserId,
3585
+ nowIso,
3586
+ nowIso,
3587
+ projectId,
3588
+ row.environment,
3589
+ row.id
3590
+ ).run();
3591
+ const updated = await db.prepare(
3592
+ `SELECT * FROM remediation_proposals
3593
+ WHERE project_id = ? AND environment = ? AND id = ?`
3594
+ ).bind(projectId, row.environment, row.id).first();
3595
+ return mapProposalRow(updated ?? row, grantToken);
3596
+ }
3597
+ async function rejectRemediationProposal(db, projectId, proposalId, options) {
3598
+ const row = options.environment ? await db.prepare(
3599
+ `SELECT * FROM remediation_proposals
3600
+ WHERE project_id = ? AND environment = ? AND id = ?`
3601
+ ).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
3602
+ `SELECT * FROM remediation_proposals
3603
+ WHERE project_id = ? AND id = ?
3604
+ LIMIT 1`
3605
+ ).bind(projectId, proposalId).first();
3606
+ if (!row) {
3607
+ throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
3608
+ }
3609
+ if (row.status !== "pending_human_approval" && row.status !== "grant_issued") {
3610
+ throw new RemediationGateError(
3611
+ `Proposal ${proposalId} cannot be rejected from status=${row.status}`,
3612
+ 409
3613
+ );
3614
+ }
3615
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
3616
+ const reason = options.reason ? `Rejected by ${options.rejectedByUserId}: ${options.reason}` : `Rejected by ${options.rejectedByUserId}`;
3617
+ await db.prepare(
3618
+ `UPDATE remediation_proposals
3619
+ SET status = 'rejected',
3620
+ grant_token_hash = NULL,
3621
+ failure_reason = ?,
3622
+ updated_at = ?
3623
+ WHERE project_id = ? AND environment = ? AND id = ?`
3624
+ ).bind(reason, nowIso, projectId, row.environment, row.id).run();
3625
+ const updated = await db.prepare(
3626
+ `SELECT * FROM remediation_proposals
3627
+ WHERE project_id = ? AND environment = ? AND id = ?`
3628
+ ).bind(projectId, row.environment, row.id).first();
3629
+ return mapProposalRow(updated ?? row);
3630
+ }
3631
+ async function claimRemediationGrant(db, projectId, proposalId, options) {
3632
+ const nowIso = options.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
3633
+ const row = options.environment ? await db.prepare(
3634
+ `SELECT * FROM remediation_proposals
3635
+ WHERE project_id = ? AND environment = ? AND id = ?`
3636
+ ).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
3637
+ `SELECT * FROM remediation_proposals
3638
+ WHERE project_id = ? AND id = ?
3639
+ LIMIT 1`
3640
+ ).bind(projectId, proposalId).first();
3641
+ if (!row) {
3642
+ throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
3643
+ }
3644
+ if (row.status !== "grant_issued") {
3645
+ throw new RemediationGateError(
3646
+ `Grant cannot be claimed from status=${row.status} (grants are single-use)`,
3647
+ 409
3648
+ );
3649
+ }
3650
+ if (row.grant_expires_at && row.grant_expires_at <= nowIso) {
3651
+ await db.prepare(
3652
+ `UPDATE remediation_proposals
3653
+ SET status = 'expired', updated_at = ?
3654
+ WHERE project_id = ? AND environment = ? AND id = ?`
3655
+ ).bind(nowIso, projectId, row.environment, row.id).run();
3656
+ throw new RemediationGateError(
3657
+ `Grant for proposal ${proposalId} expired at ${row.grant_expires_at}`,
3658
+ 410
3659
+ );
3660
+ }
3661
+ const incomingHash = await sha256Hex(options.grantToken);
3662
+ if (!row.grant_token_hash || incomingHash !== row.grant_token_hash) {
3663
+ throw new RemediationGateError("Invalid grantToken", 403);
3664
+ }
3665
+ const expected = parseJsonRecord(
3666
+ row.expected_current_state_json
3667
+ );
3668
+ const targetNode = await db.prepare(
3669
+ `SELECT current_version, current_commit_sha, current_config_digest
3670
+ FROM ontology_nodes
3671
+ WHERE project_id = ? AND environment = ? AND id = ?`
3672
+ ).bind(projectId, row.environment, row.target_node_id).first();
3673
+ if (targetNode) {
3674
+ if (expected.version && targetNode.current_version && targetNode.current_version !== expected.version) {
3675
+ throw new RemediationGateError(
3676
+ `Optimistic version lock failed on ${row.target_node_id}: expected version "${expected.version}", but current version is "${targetNode.current_version}"`,
3677
+ 409
3678
+ );
3679
+ }
3680
+ if (expected.commitSha && targetNode.current_commit_sha && targetNode.current_commit_sha !== expected.commitSha) {
3681
+ throw new RemediationGateError(
3682
+ `Optimistic commit lock failed on ${row.target_node_id}: expected commit "${expected.commitSha}", but current commit is "${targetNode.current_commit_sha}"`,
3683
+ 409
3684
+ );
3685
+ }
3686
+ if (expected.configDigest && targetNode.current_config_digest && targetNode.current_config_digest !== expected.configDigest) {
3687
+ throw new RemediationGateError(
3688
+ `Optimistic config lock failed on ${row.target_node_id}: expected digest "${expected.configDigest}", but current digest is "${targetNode.current_config_digest}"`,
3689
+ 409
3690
+ );
3691
+ }
3692
+ }
3693
+ const claimExpiresAt = new Date(
3694
+ Date.parse(nowIso) + 5 * 6e4
3695
+ ).toISOString();
3696
+ await db.batch([
3697
+ db.prepare(
3698
+ `UPDATE remediation_proposals
3699
+ SET status = 'claimed_executing',
3700
+ grant_token_hash = NULL,
3701
+ claimed_by_runner_id = ?,
3702
+ claimed_at = ?,
3703
+ claim_expires_at = ?,
3704
+ updated_at = ?
3705
+ WHERE project_id = ? AND environment = ? AND id = ?`
3706
+ ).bind(
3707
+ options.runnerId ?? "autosre-runner",
3708
+ nowIso,
3709
+ claimExpiresAt,
3710
+ nowIso,
3711
+ projectId,
3712
+ row.environment,
3713
+ row.id
3714
+ ),
3715
+ db.prepare(
3716
+ `UPDATE incident_investigations
3717
+ SET status = 'remediating', updated_at = ?
3718
+ WHERE project_id = ? AND environment = ? AND id = ?`
3719
+ ).bind(nowIso, projectId, row.environment, row.incident_id)
3720
+ ]);
3721
+ const updated = await db.prepare(
3722
+ `SELECT * FROM remediation_proposals
3723
+ WHERE project_id = ? AND environment = ? AND id = ?`
3724
+ ).bind(projectId, row.environment, row.id).first();
3725
+ return mapProposalRow(updated ?? row);
3726
+ }
3727
+ async function completeRemediationProposal(db, projectId, proposalId, options) {
3728
+ const nowIso = options.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
3729
+ const row = options.environment ? await db.prepare(
3730
+ `SELECT * FROM remediation_proposals
3731
+ WHERE project_id = ? AND environment = ? AND id = ?`
3732
+ ).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
3733
+ `SELECT * FROM remediation_proposals
3734
+ WHERE project_id = ? AND id = ?
3735
+ LIMIT 1`
3736
+ ).bind(projectId, proposalId).first();
3737
+ if (!row) {
3738
+ throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
3739
+ }
3740
+ if (row.status !== "claimed_executing") {
3741
+ throw new RemediationGateError(
3742
+ `Proposal ${proposalId} cannot be completed from status=${row.status} (must be claimed_executing)`,
3743
+ 409
3744
+ );
3745
+ }
3746
+ const desired = parseJsonRecord(
3747
+ row.desired_target_state_json
3748
+ );
3749
+ const nextWatchdogAt = new Date(
3750
+ Date.parse(nowIso) + 2 * 6e4
3751
+ ).toISOString();
3752
+ const finding = await getInvestigationFinding(
3753
+ db,
3754
+ projectId,
3755
+ row.finding_id,
3756
+ row.environment
3757
+ );
3758
+ const updatedChain = finding ? [...finding.causalChain] : [];
3759
+ if (options.resultingChangeId) {
3760
+ updatedChain.push({
3761
+ subjectKind: "finding",
3762
+ subjectId: row.finding_id,
3763
+ predicate: "mitigated_by_change",
3764
+ objectKind: "change",
3765
+ objectId: options.resultingChangeId,
3766
+ relationshipCertainty: "exact",
3767
+ causalProofTier: finding?.causalProofTier ?? "cohort_rate_isolated",
3768
+ confidence: 0.9,
3769
+ reason: `Remediation proposal ${row.id} (${row.action_kind}) applied change ${options.resultingChangeId}`
3770
+ });
3771
+ }
3772
+ if (options.remediationAgentRunId) {
3773
+ updatedChain.push({
3774
+ subjectKind: "finding",
3775
+ subjectId: row.finding_id,
3776
+ predicate: "remediated_by_agent_run",
3777
+ objectKind: "agent_run",
3778
+ objectId: options.remediationAgentRunId,
3779
+ relationshipCertainty: "exact",
3780
+ causalProofTier: finding?.causalProofTier ?? "cohort_rate_isolated",
3781
+ confidence: 0.9,
3782
+ reason: `Remediation executed by agent run ${options.remediationAgentRunId}`
3783
+ });
3784
+ }
3785
+ await db.batch([
3786
+ db.prepare(
3787
+ `UPDATE remediation_proposals
3788
+ SET status = 'completed',
3789
+ resulting_change_id = COALESCE(?, resulting_change_id),
3790
+ remediation_agent_run_id = COALESCE(?, remediation_agent_run_id),
3791
+ updated_at = ?
3792
+ WHERE project_id = ? AND environment = ? AND id = ?`
3793
+ ).bind(
3794
+ options.resultingChangeId ?? null,
3795
+ options.remediationAgentRunId ?? null,
3796
+ nowIso,
3797
+ projectId,
3798
+ row.environment,
3799
+ row.id
3800
+ ),
3801
+ db.prepare(
3802
+ `UPDATE ontology_nodes
3803
+ SET current_version = COALESCE(?, current_version),
3804
+ current_commit_sha = COALESCE(?, current_commit_sha),
3805
+ current_config_digest = COALESCE(?, current_config_digest),
3806
+ last_seen_at = ?
3807
+ WHERE project_id = ? AND environment = ? AND id = ?`
3808
+ ).bind(
3809
+ desired.version ?? null,
3810
+ desired.commitSha ?? null,
3811
+ desired.configDigest ?? null,
3812
+ nowIso,
3813
+ projectId,
3814
+ row.environment,
3815
+ row.target_node_id
3816
+ ),
3817
+ db.prepare(
3818
+ `UPDATE investigation_findings
3819
+ SET recovery_status = 'monitoring',
3820
+ watchdog_step = 1,
3821
+ baseline_request_rate_per_min = COALESCE(?, baseline_request_rate_per_min),
3822
+ next_watchdog_check_at = ?,
3823
+ causal_chain_json = ?,
3824
+ updated_at = ?
3825
+ WHERE project_id = ? AND environment = ? AND id = ?`
3826
+ ).bind(
3827
+ options.baselineRequestRatePerMin ?? null,
3828
+ nextWatchdogAt,
3829
+ JSON.stringify(updatedChain),
3830
+ nowIso,
3831
+ projectId,
3832
+ row.environment,
3833
+ row.finding_id
3834
+ ),
3835
+ db.prepare(
3836
+ `UPDATE incident_investigations
3837
+ SET status = 'verifying', updated_at = ?
3838
+ WHERE project_id = ? AND environment = ? AND id = ?`
3839
+ ).bind(nowIso, projectId, row.environment, row.incident_id)
3840
+ ]);
3841
+ const updated = await db.prepare(
3842
+ `SELECT * FROM remediation_proposals
3843
+ WHERE project_id = ? AND environment = ? AND id = ?`
3844
+ ).bind(projectId, row.environment, row.id).first();
3845
+ return mapProposalRow(updated ?? row);
3846
+ }
3847
+ async function failRemediationProposal(db, projectId, proposalId, options) {
3848
+ const nowIso = options.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
3849
+ const row = options.environment ? await db.prepare(
3850
+ `SELECT * FROM remediation_proposals
3851
+ WHERE project_id = ? AND environment = ? AND id = ?`
3852
+ ).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
3853
+ `SELECT * FROM remediation_proposals
3854
+ WHERE project_id = ? AND id = ?
3855
+ LIMIT 1`
3856
+ ).bind(projectId, proposalId).first();
3857
+ if (!row) {
3858
+ throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
3859
+ }
3860
+ await db.batch([
3861
+ db.prepare(
3862
+ `UPDATE remediation_proposals
3863
+ SET status = 'failed_execution',
3864
+ failure_reason = ?,
3865
+ updated_at = ?
3866
+ WHERE project_id = ? AND environment = ? AND id = ?`
3867
+ ).bind(options.failureReason, nowIso, projectId, row.environment, row.id),
3868
+ db.prepare(
3869
+ `UPDATE incident_investigations
3870
+ SET status = 'escalated', updated_at = ?
3871
+ WHERE project_id = ? AND environment = ? AND id = ?`
3872
+ ).bind(nowIso, projectId, row.environment, row.incident_id)
3873
+ ]);
3874
+ const updated = await db.prepare(
3875
+ `SELECT * FROM remediation_proposals
3876
+ WHERE project_id = ? AND environment = ? AND id = ?`
3877
+ ).bind(projectId, row.environment, row.id).first();
3878
+ return mapProposalRow(updated ?? row);
3879
+ }
3880
+ async function evaluateRemediationWatchdogs(db, options = {}) {
3881
+ const nowIso = options.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
3882
+ const minComparableRequests = options.minComparableRequests ?? 50;
3883
+ const expiredGrantsRes = await db.prepare(
3884
+ `UPDATE remediation_proposals
3885
+ SET status = 'expired', grant_token_hash = NULL, updated_at = ?
3886
+ WHERE status = 'grant_issued'
3887
+ AND grant_expires_at IS NOT NULL
3888
+ AND grant_expires_at <= ?`
3889
+ ).bind(nowIso, nowIso).run();
3890
+ const timedOutRows = await db.prepare(
3891
+ `SELECT project_id, environment, id, incident_id
3892
+ FROM remediation_proposals
3893
+ WHERE status = 'claimed_executing'
3894
+ AND claim_expires_at IS NOT NULL
3895
+ AND claim_expires_at <= ?`
3896
+ ).bind(nowIso).all();
3897
+ for (const t of timedOutRows.results) {
3898
+ await db.batch([
3899
+ db.prepare(
3900
+ `UPDATE remediation_proposals
3901
+ SET status = 'execution_timed_out',
3902
+ failure_reason = 'Runner did not complete or fail within 5m claim window',
3903
+ updated_at = ?
3904
+ WHERE project_id = ? AND environment = ? AND id = ?`
3905
+ ).bind(nowIso, t.project_id, t.environment, t.id),
3906
+ db.prepare(
3907
+ `UPDATE incident_investigations
3908
+ SET status = 'escalated', updated_at = ?
3909
+ WHERE project_id = ? AND environment = ? AND id = ?`
3910
+ ).bind(nowIso, t.project_id, t.environment, t.incident_id)
3911
+ ]);
3912
+ }
3913
+ const dueFindings = await db.prepare(
3914
+ `SELECT
3915
+ project_id, environment, id, incident_id,
3916
+ fault_locus_entity_id, root_cause_entity_id,
3917
+ watchdog_step, baseline_request_rate_per_min,
3918
+ causal_verification_status, updated_at
3919
+ FROM investigation_findings
3920
+ WHERE recovery_status = 'monitoring'
3921
+ AND next_watchdog_check_at IS NOT NULL
3922
+ AND next_watchdog_check_at <= ?
3923
+ LIMIT 50`
3924
+ ).bind(nowIso).all();
3925
+ let recovered = 0;
3926
+ let inconclusiveLowTraffic = 0;
3927
+ let stillDegraded = 0;
3928
+ for (const f of dueFindings.results) {
3929
+ const completedProposal = await db.prepare(
3930
+ `SELECT
3931
+ p.updated_at,
3932
+ p.target_node_id,
3933
+ p.resulting_change_id,
3934
+ c.occurred_at AS change_occurred_at
3935
+ FROM remediation_proposals p
3936
+ LEFT JOIN change_events c
3937
+ ON c.project_id = p.project_id
3938
+ AND c.environment = p.environment
3939
+ AND c.id = p.resulting_change_id
3940
+ WHERE p.project_id = ? AND p.environment = ? AND p.finding_id = ? AND p.status = 'completed'
3941
+ ORDER BY p.updated_at DESC
3942
+ LIMIT 1`
3943
+ ).bind(f.project_id, f.environment, f.id).first();
3944
+ const sinceTs = completedProposal?.change_occurred_at ?? completedProposal?.updated_at ?? f.updated_at;
3945
+ const targetNodeId = completedProposal?.target_node_id ?? f.fault_locus_entity_id;
3946
+ const parts = targetNodeId.split(":");
3947
+ const serviceName = parts[0] === "service" && parts.length >= 4 ? parts.slice(3).join(":") : null;
3948
+ const stats = serviceName ? await db.prepare(
3949
+ `SELECT
3950
+ COUNT(*) AS total_requests,
3951
+ SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) AS error_requests
3952
+ FROM telemetry_spans
3953
+ WHERE project_id = ?
3954
+ AND environment = ?
3955
+ AND service_name = ?
3956
+ AND (parent_span_id IS NULL OR span_kind = 1)
3957
+ AND (start_time >= ? OR received_at >= ?)`
3958
+ ).bind(f.project_id, f.environment, serviceName, sinceTs, sinceTs).first() : await db.prepare(
3959
+ `SELECT
3960
+ COUNT(*) AS total_requests,
3961
+ SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) AS error_requests
3962
+ FROM telemetry_spans
3963
+ WHERE project_id = ?
3964
+ AND environment = ?
3965
+ AND (parent_span_id IS NULL OR span_kind = 1)
3966
+ AND (start_time >= ? OR received_at >= ?)`
3967
+ ).bind(f.project_id, f.environment, sinceTs, sinceTs).first();
3968
+ const totalReq = Number(stats?.total_requests ?? 0);
3969
+ const errorReq = Number(stats?.error_requests ?? 0);
3970
+ const errorRate = totalReq > 0 ? errorReq / totalReq : 0;
3971
+ const requiredTraffic = Math.max(
3972
+ minComparableRequests,
3973
+ f.baseline_request_rate_per_min ? Math.floor(0.5 * f.baseline_request_rate_per_min * 5) : minComparableRequests
3974
+ );
3975
+ if (totalReq < requiredTraffic) {
3976
+ if (f.watchdog_step < 3) {
3977
+ const nextStep = f.watchdog_step + 1;
3978
+ const offsetMins = nextStep === 2 ? 5 : 15;
3979
+ const nextCheck = new Date(
3980
+ Date.parse(nowIso) + offsetMins * 6e4
3981
+ ).toISOString();
3982
+ await db.prepare(
3983
+ `UPDATE investigation_findings
3984
+ SET watchdog_step = ?, next_watchdog_check_at = ?, updated_at = ?
3985
+ WHERE project_id = ? AND environment = ? AND id = ?`
3986
+ ).bind(nextStep, nextCheck, nowIso, f.project_id, f.environment, f.id).run();
3987
+ } else {
3988
+ inconclusiveLowTraffic += 1;
3989
+ await db.prepare(
3990
+ `UPDATE investigation_findings
3991
+ SET recovery_status = 'inconclusive_low_traffic',
3992
+ next_watchdog_check_at = NULL,
3993
+ updated_at = ?
3994
+ WHERE project_id = ? AND environment = ? AND id = ?`
3995
+ ).bind(nowIso, f.project_id, f.environment, f.id).run();
3996
+ }
3997
+ continue;
3998
+ }
3999
+ if (errorRate >= 0.05) {
4000
+ stillDegraded += 1;
4001
+ await db.prepare(
4002
+ `UPDATE investigation_findings
4003
+ SET recovery_status = 'still_degraded',
4004
+ causal_verification_status = 'refuted',
4005
+ next_watchdog_check_at = NULL,
4006
+ updated_at = ?
4007
+ WHERE project_id = ? AND environment = ? AND id = ?`
4008
+ ).bind(nowIso, f.project_id, f.environment, f.id).run();
4009
+ if (f.incident_id) {
4010
+ await db.prepare(
4011
+ `UPDATE incident_investigations
4012
+ SET status = 'escalated', updated_at = ?
4013
+ WHERE project_id = ? AND environment = ? AND id = ?`
4014
+ ).bind(nowIso, f.project_id, f.environment, f.incident_id).run();
4015
+ }
4016
+ } else {
4017
+ recovered += 1;
4018
+ const nextCausalVerification = f.causal_verification_status === "counterfactual_verified" ? "counterfactual_verified" : "operationally_corroborated";
4019
+ await db.prepare(
4020
+ `UPDATE investigation_findings
4021
+ SET recovery_status = 'recovered',
4022
+ causal_verification_status = ?,
4023
+ watchdog_step = 3,
4024
+ next_watchdog_check_at = NULL,
4025
+ updated_at = ?
4026
+ WHERE project_id = ? AND environment = ? AND id = ?`
4027
+ ).bind(
4028
+ nextCausalVerification,
4029
+ nowIso,
4030
+ f.project_id,
4031
+ f.environment,
4032
+ f.id
4033
+ ).run();
4034
+ if (f.incident_id) {
4035
+ await db.prepare(
4036
+ `UPDATE incident_investigations
4037
+ SET status = 'resolved', resolved_at = ?, updated_at = ?
4038
+ WHERE project_id = ? AND environment = ? AND id = ?`
4039
+ ).bind(nowIso, nowIso, f.project_id, f.environment, f.incident_id).run();
4040
+ }
4041
+ }
4042
+ }
4043
+ return {
4044
+ evaluated: dueFindings.results.length,
4045
+ recovered,
4046
+ inconclusiveLowTraffic,
4047
+ stillDegraded,
4048
+ timedOutClaims: timedOutRows.results.length,
4049
+ expiredGrants: expiredGrantsRes.meta.changes
4050
+ };
4051
+ }
4052
+
2490
4053
  // src/lib/narrate-gate.ts
2491
4054
  var COMPARATORS = ["<=", ">=", "==", "!=", "<", ">"];
2492
4055
  var stripWhitespace = (s) => s.replace(/\s+/g, "");
@@ -2855,6 +4418,14 @@ async function runDueAnalysesForProject(projectId, deps) {
2855
4418
  if (result.narrative) narrated += 1;
2856
4419
  }
2857
4420
  await store.insertResult(result, expiresAt);
4421
+ if (result.status === "critical") {
4422
+ await openOrDeduplicateIncident(db, projectId, {
4423
+ environment: "default",
4424
+ targetNodeId: `analysis:${def.id}`,
4425
+ symptomAnchorKind: "analysis",
4426
+ symptomAnchorId: def.id
4427
+ }).catch(() => void 0);
4428
+ }
2858
4429
  await store.markRan(projectId, def.id, result.generatedAt);
2859
4430
  ran += 1;
2860
4431
  } catch (error) {
@@ -2880,6 +4451,7 @@ var changes = (r) => r.meta?.changes ?? 0;
2880
4451
  async function purgeSecondaryRows(db, options) {
2881
4452
  const nowMs = options.now?.() ?? Date.now();
2882
4453
  const isoBefore = (hours) => new Date(nowMs - hours * 36e5).toISOString();
4454
+ const nowIso = isoBefore(0);
2883
4455
  const alertEvaluations = await db.prepare(`DELETE FROM alert_evaluations WHERE evaluated_at < ?`).bind(isoBefore(ALERT_EVALUATIONS_MAX_AGE_HOURS)).run();
2884
4456
  const askEvents = await db.prepare(`DELETE FROM ask_evidence_events WHERE asked_at < ?`).bind(nowMs - ASK_EVIDENCE_EVENTS_MAX_AGE_HOURS * 36e5).run();
2885
4457
  const series = await db.prepare(
@@ -2892,7 +4464,7 @@ async function purgeSecondaryRows(db, options) {
2892
4464
  const refs = await db.prepare(
2893
4465
  `DELETE FROM evidence_retrieval_refs
2894
4466
  WHERE expires_at IS NOT NULL AND expires_at < ?`
2895
- ).bind(isoBefore(0)).run();
4467
+ ).bind(nowIso).run();
2896
4468
  const expansions = await db.prepare(
2897
4469
  `DELETE FROM evidence_ref_expansions
2898
4470
  WHERE NOT EXISTS (
@@ -2900,12 +4472,75 @@ async function purgeSecondaryRows(db, options) {
2900
4472
  WHERE r.ref_id = evidence_ref_expansions.ref_id
2901
4473
  )`
2902
4474
  ).run();
4475
+ const thirtyDaysAgoIso = isoBefore(30 * 24);
4476
+ const ninetyDaysAgoIso = isoBefore(90 * 24);
4477
+ let ontologyEdgesPurged = 0;
4478
+ let ontologyNodesPurged = 0;
4479
+ let changeEventsPurged = 0;
4480
+ let incidentInvestigationsPurged = 0;
4481
+ let investigationFindingsPurged = 0;
4482
+ let remediationProposalsPurged = 0;
4483
+ try {
4484
+ const edgesRes = await db.prepare(`DELETE FROM ontology_edges WHERE last_seen_at < ?`).bind(thirtyDaysAgoIso).run();
4485
+ ontologyEdgesPurged = changes(edgesRes);
4486
+ const nodesRes = await db.prepare(
4487
+ `DELETE FROM ontology_nodes
4488
+ WHERE last_seen_at < ?
4489
+ AND NOT EXISTS (
4490
+ SELECT 1 FROM ontology_edges e
4491
+ WHERE e.project_id = ontology_nodes.project_id
4492
+ AND e.environment = ontology_nodes.environment
4493
+ AND (e.source_node_id = ontology_nodes.id OR e.target_node_id = ontology_nodes.id)
4494
+ )`
4495
+ ).bind(thirtyDaysAgoIso).run();
4496
+ ontologyNodesPurged = changes(nodesRes);
4497
+ const proposalsRes = await db.prepare(
4498
+ `DELETE FROM remediation_proposals
4499
+ WHERE (expires_at IS NOT NULL AND expires_at < ?)
4500
+ OR (expires_at IS NULL AND created_at < ?)`
4501
+ ).bind(nowIso, ninetyDaysAgoIso).run();
4502
+ remediationProposalsPurged = changes(proposalsRes);
4503
+ const findingsRes = await db.prepare(
4504
+ `DELETE FROM investigation_findings
4505
+ WHERE causal_verification_status NOT IN ('counterfactual_verified', 'operationally_corroborated')
4506
+ AND ((expires_at IS NOT NULL AND expires_at < ?)
4507
+ OR (expires_at IS NULL AND created_at < ?))`
4508
+ ).bind(nowIso, ninetyDaysAgoIso).run();
4509
+ investigationFindingsPurged = changes(findingsRes);
4510
+ const changesRes = await db.prepare(
4511
+ `DELETE FROM change_events
4512
+ WHERE ((expires_at IS NOT NULL AND expires_at < ?)
4513
+ OR (expires_at IS NULL AND occurred_at < ?))
4514
+ AND NOT EXISTS (
4515
+ SELECT 1 FROM investigation_findings f
4516
+ WHERE f.project_id = change_events.project_id
4517
+ AND f.environment = change_events.environment
4518
+ AND f.contributing_change_id = change_events.id
4519
+ )`
4520
+ ).bind(nowIso, ninetyDaysAgoIso).run();
4521
+ changeEventsPurged = changes(changesRes);
4522
+ const incidentsRes = await db.prepare(
4523
+ `DELETE FROM incident_investigations
4524
+ WHERE status IN ('resolved', 'escalated')
4525
+ AND ((expires_at IS NOT NULL AND expires_at < ?)
4526
+ OR (expires_at IS NULL AND updated_at < ?))`
4527
+ ).bind(nowIso, ninetyDaysAgoIso).run();
4528
+ incidentInvestigationsPurged = changes(incidentsRes);
4529
+ await evaluateRemediationWatchdogs(db, { nowIso });
4530
+ } catch {
4531
+ }
2903
4532
  return {
2904
4533
  alert_evaluations: changes(alertEvaluations),
2905
4534
  ask_evidence_events: changes(askEvents),
2906
4535
  metric_series: changes(series),
2907
4536
  evidence_refs: changes(refs),
2908
- evidence_ref_expansions: changes(expansions)
4537
+ evidence_ref_expansions: changes(expansions),
4538
+ ontology_edges: ontologyEdgesPurged,
4539
+ ontology_nodes: ontologyNodesPurged,
4540
+ change_events: changeEventsPurged,
4541
+ incident_investigations: incidentInvestigationsPurged,
4542
+ investigation_findings: investigationFindingsPurged,
4543
+ remediation_proposals: remediationProposalsPurged
2909
4544
  };
2910
4545
  }
2911
4546
 
@@ -3641,13 +5276,630 @@ async function getTelemetryExportRows(db, options) {
3641
5276
  ).join("\n");
3642
5277
  }
3643
5278
 
5279
+ // src/lib/autosre-ontology.ts
5280
+ import { buildCanonicalNodeId as buildCanonicalNodeId2, normalizeErrorSignature } from "@obsunified/types";
5281
+ var pickStr = (record, ...keys) => {
5282
+ for (const key of keys) {
5283
+ const val = record[key];
5284
+ if (typeof val === "string" && val.trim().length > 0) {
5285
+ return val.trim();
5286
+ }
5287
+ if (typeof val === "number") {
5288
+ return String(val);
5289
+ }
5290
+ }
5291
+ return null;
5292
+ };
5293
+ function extractSpanLineageAndTopology(span) {
5294
+ const attrs = parseJsonRecord(span.attributesJson);
5295
+ const resource = parseJsonRecord(span.resourceAttributesJson);
5296
+ const environment = (span.environment ?? pickStr(
5297
+ resource,
5298
+ "deployment.environment.name",
5299
+ "deployment.environment",
5300
+ "obs.environment"
5301
+ ) ?? pickStr(
5302
+ attrs,
5303
+ "deployment.environment.name",
5304
+ "deployment.environment",
5305
+ "obs.environment"
5306
+ ) ?? "default").toLowerCase();
5307
+ const namespace = pickStr(resource, "service.namespace") ?? pickStr(attrs, "service.namespace") ?? "default";
5308
+ const serviceName = span.serviceName ?? pickStr(resource, "service.name") ?? pickStr(attrs, "service.name");
5309
+ const serviceVersion = span.serviceVersion ?? pickStr(resource, "service.version", "obs.service.version") ?? pickStr(attrs, "service.version", "obs.service.version");
5310
+ const commitSha = span.commitSha ?? pickStr(
5311
+ resource,
5312
+ "vcs.commit.id",
5313
+ "git.commit.sha",
5314
+ "obs.commit.sha",
5315
+ "service.commit.sha"
5316
+ ) ?? pickStr(attrs, "vcs.commit.id", "git.commit.sha", "obs.commit.sha");
5317
+ const configDigest = pickStr(resource, "obs.config.digest") ?? pickStr(attrs, "obs.config.digest");
5318
+ const sourceNode = serviceName ? buildCanonicalNodeId2({
5319
+ kind: "service",
5320
+ environment,
5321
+ namespace,
5322
+ serviceName
5323
+ }) : null;
5324
+ let targetNode = null;
5325
+ let edgePredicate = null;
5326
+ const dbSystem = pickStr(attrs, "db.system");
5327
+ const messagingSystem = pickStr(attrs, "messaging.system");
5328
+ const peerService = pickStr(attrs, "peer.service", "obs.target.service");
5329
+ const serverAddress = pickStr(
5330
+ attrs,
5331
+ "server.address",
5332
+ "net.peer.name",
5333
+ "db.connection_string"
5334
+ );
5335
+ const serverPort = pickStr(attrs, "server.port", "net.peer.port");
5336
+ const toolName = pickStr(attrs, "obs.tool.name", "tool.name");
5337
+ if (dbSystem) {
5338
+ const isCache = dbSystem.toLowerCase() === "redis" || dbSystem.toLowerCase() === "memcached";
5339
+ targetNode = buildCanonicalNodeId2({
5340
+ kind: isCache ? "cache" : "database",
5341
+ environment,
5342
+ dbSystem,
5343
+ dbName: pickStr(attrs, "db.namespace", "db.name") ?? dbSystem,
5344
+ serverAddress,
5345
+ serverPort
5346
+ });
5347
+ edgePredicate = isCache ? "queries_cache" : "queries_database";
5348
+ } else if (messagingSystem) {
5349
+ targetNode = buildCanonicalNodeId2({
5350
+ kind: "queue_topic",
5351
+ environment,
5352
+ messagingSystem,
5353
+ messagingDestination: pickStr(
5354
+ attrs,
5355
+ "messaging.destination.name",
5356
+ "messaging.destination"
5357
+ ) ?? "default",
5358
+ serverAddress
5359
+ });
5360
+ edgePredicate = span.spanKind === 4 ? "publishes_to_queue" : "consumes_from_queue";
5361
+ } else if (peerService) {
5362
+ targetNode = buildCanonicalNodeId2({
5363
+ kind: "service",
5364
+ environment,
5365
+ namespace: pickStr(attrs, "peer.namespace") ?? namespace,
5366
+ serviceName: peerService
5367
+ });
5368
+ edgePredicate = "calls_service";
5369
+ } else if (span.spanKind === 2 && serverAddress) {
5370
+ targetNode = buildCanonicalNodeId2({
5371
+ kind: "external_api",
5372
+ environment,
5373
+ serverAddress,
5374
+ serverPort: serverPort ?? "443"
5375
+ });
5376
+ edgePredicate = "calls_external_api";
5377
+ } else if (toolName) {
5378
+ targetNode = buildCanonicalNodeId2({
5379
+ kind: "tool",
5380
+ environment,
5381
+ serviceName: pickStr(attrs, "obs.agent.id", "agent.id") ?? serviceName ?? "default",
5382
+ toolName
5383
+ });
5384
+ edgePredicate = "invokes_tool";
5385
+ }
5386
+ let errorSignature = span.errorSignature ?? null;
5387
+ if (!errorSignature && span.statusCode === 2) {
5388
+ let rawErr = pickStr(
5389
+ attrs,
5390
+ "exception.type",
5391
+ "error.type",
5392
+ "obs.tool.error_type",
5393
+ "http.response.status_code",
5394
+ "http.status_code",
5395
+ "rpc.grpc.status_code"
5396
+ ) ?? span.statusMessage;
5397
+ if (!rawErr && span.eventsJson) {
5398
+ const events = parseJsonArray(span.eventsJson);
5399
+ for (const ev of events) {
5400
+ if (ev && typeof ev === "object" && !Array.isArray(ev)) {
5401
+ const evObj = ev;
5402
+ if (evObj.name === "exception" && evObj.attributes) {
5403
+ const evAttrs = typeof evObj.attributes === "object" && !Array.isArray(evObj.attributes) ? evObj.attributes : {};
5404
+ rawErr = pickStr(evAttrs, "exception.type", "exception.message") ?? rawErr;
5405
+ }
5406
+ }
5407
+ }
5408
+ }
5409
+ errorSignature = normalizeErrorSignature(
5410
+ rawErr ?? `ERROR:${span.spanName}`
5411
+ );
5412
+ }
5413
+ return {
5414
+ environment,
5415
+ namespace,
5416
+ serviceName,
5417
+ serviceVersion,
5418
+ commitSha,
5419
+ configDigest,
5420
+ sourceNode,
5421
+ targetNode,
5422
+ edgePredicate,
5423
+ errorSignature
5424
+ };
5425
+ }
5426
+ var NODE_UPSERT_CACHE = /* @__PURE__ */ new Map();
5427
+ var CACHE_TTL_MS = 6e4;
5428
+ async function upsertOntologyFromSpans(db, spans, options = {}) {
5429
+ if (spans.length === 0) return;
5430
+ const nowMs = Date.now();
5431
+ const statements = [];
5432
+ const edgeAggregates = /* @__PURE__ */ new Map();
5433
+ const serviceNodesToCheck = /* @__PURE__ */ new Map();
5434
+ for (const span of spans) {
5435
+ const lineage = extractSpanLineageAndTopology(span);
5436
+ const ts = span.receivedAt || span.startTime || (/* @__PURE__ */ new Date()).toISOString();
5437
+ if (lineage.sourceNode) {
5438
+ const cacheKey = `${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}::${lineage.serviceVersion ?? ""}::${lineage.commitSha ?? ""}::${lineage.configDigest ?? ""}`;
5439
+ const cachedAt = NODE_UPSERT_CACHE.get(cacheKey);
5440
+ if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
5441
+ NODE_UPSERT_CACHE.set(cacheKey, nowMs);
5442
+ serviceNodesToCheck.set(
5443
+ `${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}`,
5444
+ {
5445
+ projectId: span.projectId,
5446
+ environment: lineage.environment,
5447
+ node: lineage.sourceNode,
5448
+ version: lineage.serviceVersion,
5449
+ commitSha: lineage.commitSha,
5450
+ configDigest: lineage.configDigest,
5451
+ timestamp: ts
5452
+ }
5453
+ );
5454
+ }
5455
+ }
5456
+ if (lineage.targetNode) {
5457
+ const targetCacheKey = `${span.projectId}::${lineage.environment}::${lineage.targetNode.id}`;
5458
+ const cachedAt = NODE_UPSERT_CACHE.get(targetCacheKey);
5459
+ if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
5460
+ NODE_UPSERT_CACHE.set(targetCacheKey, nowMs);
5461
+ statements.push(
5462
+ db.prepare(
5463
+ `INSERT INTO ontology_nodes (
5464
+ project_id, environment, id, kind, name, namespace_or_endpoint,
5465
+ is_scoped_identity, attributes_json, first_seen_at, last_seen_at
5466
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
5467
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
5468
+ last_seen_at = excluded.last_seen_at`
5469
+ ).bind(
5470
+ span.projectId,
5471
+ lineage.environment,
5472
+ lineage.targetNode.id,
5473
+ lineage.targetNode.kind,
5474
+ lineage.targetNode.name,
5475
+ lineage.targetNode.namespaceOrEndpoint,
5476
+ lineage.targetNode.isScopedIdentity ? 1 : 0,
5477
+ ts,
5478
+ ts
5479
+ )
5480
+ );
5481
+ }
5482
+ }
5483
+ if (lineage.sourceNode && lineage.targetNode && lineage.edgePredicate) {
5484
+ const edgeId = `${lineage.sourceNode.id}--${lineage.edgePredicate}-->${lineage.targetNode.id}`;
5485
+ const aggKey = `${span.projectId}::${lineage.environment}::${edgeId}`;
5486
+ const certainty = lineage.sourceNode.isScopedIdentity && lineage.targetNode.isScopedIdentity ? "exact" : "inferred";
5487
+ const existing = edgeAggregates.get(aggKey);
5488
+ const isErr = span.statusCode === 2 ? 1 : 0;
5489
+ if (existing) {
5490
+ existing.callCount += 1;
5491
+ existing.errorCount += isErr;
5492
+ existing.lastDurationMs = span.durationMs;
5493
+ if (ts > existing.lastSeenAt) existing.lastSeenAt = ts;
5494
+ } else {
5495
+ edgeAggregates.set(aggKey, {
5496
+ projectId: span.projectId,
5497
+ environment: lineage.environment,
5498
+ id: edgeId,
5499
+ sourceNodeId: lineage.sourceNode.id,
5500
+ predicate: lineage.edgePredicate,
5501
+ targetNodeId: lineage.targetNode.id,
5502
+ certainty,
5503
+ callCount: 1,
5504
+ errorCount: isErr,
5505
+ lastDurationMs: span.durationMs,
5506
+ firstSeenAt: ts,
5507
+ lastSeenAt: ts
5508
+ });
5509
+ }
5510
+ }
5511
+ }
5512
+ for (const entry of serviceNodesToCheck.values()) {
5513
+ if (entry.version) {
5514
+ const existingNode = await db.prepare(
5515
+ `SELECT current_version, current_commit_sha
5516
+ FROM ontology_nodes
5517
+ WHERE project_id = ? AND environment = ? AND id = ?`
5518
+ ).bind(entry.projectId, entry.environment, entry.node.id).first();
5519
+ if (existingNode?.current_version && existingNode.current_version !== entry.version) {
5520
+ const existingChange = await db.prepare(
5521
+ `SELECT id FROM change_events
5522
+ WHERE project_id = ? AND environment = ? AND target_node_id = ? AND version = ?
5523
+ LIMIT 1`
5524
+ ).bind(
5525
+ entry.projectId,
5526
+ entry.environment,
5527
+ entry.node.id,
5528
+ entry.version
5529
+ ).first();
5530
+ if (!existingChange) {
5531
+ const changeId = `chg_inf_${randomHex(8)}`;
5532
+ const expiresAt = new Date(
5533
+ Date.now() + 30 * 24 * 36e5
5534
+ ).toISOString();
5535
+ statements.push(
5536
+ db.prepare(
5537
+ `INSERT INTO change_events (
5538
+ project_id, environment, id, target_node_id, kind, source,
5539
+ relationship_certainty, summary, service_name, version,
5540
+ previous_version, commit_sha, previous_commit_sha, config_digest,
5541
+ touched_paths_json, attributes_json, occurred_at, received_at, expires_at
5542
+ ) VALUES (?, ?, ?, ?, 'deploy', 'inferred_telemetry', 'inferred', ?, ?, ?, ?, ?, ?, ?, '[]', '{}', ?, ?, ?)
5543
+ ON CONFLICT (project_id, environment, id) DO NOTHING`
5544
+ ).bind(
5545
+ entry.projectId,
5546
+ entry.environment,
5547
+ changeId,
5548
+ entry.node.id,
5549
+ `Inferred version transition ${existingNode.current_version} -> ${entry.version} on ${entry.node.name}`,
5550
+ entry.node.name,
5551
+ entry.version,
5552
+ existingNode.current_version,
5553
+ entry.commitSha,
5554
+ existingNode.current_commit_sha,
5555
+ entry.configDigest,
5556
+ entry.timestamp,
5557
+ entry.timestamp,
5558
+ expiresAt
5559
+ )
5560
+ );
5561
+ }
5562
+ }
5563
+ }
5564
+ statements.push(
5565
+ db.prepare(
5566
+ `INSERT INTO ontology_nodes (
5567
+ project_id, environment, id, kind, name, namespace_or_endpoint,
5568
+ is_scoped_identity, current_version, current_commit_sha,
5569
+ current_config_digest, attributes_json, first_seen_at, last_seen_at
5570
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
5571
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
5572
+ current_version = COALESCE(excluded.current_version, ontology_nodes.current_version),
5573
+ current_commit_sha = COALESCE(excluded.current_commit_sha, ontology_nodes.current_commit_sha),
5574
+ current_config_digest = COALESCE(excluded.current_config_digest, ontology_nodes.current_config_digest),
5575
+ last_seen_at = excluded.last_seen_at`
5576
+ ).bind(
5577
+ entry.projectId,
5578
+ entry.environment,
5579
+ entry.node.id,
5580
+ entry.node.kind,
5581
+ entry.node.name,
5582
+ entry.node.namespaceOrEndpoint,
5583
+ entry.node.isScopedIdentity ? 1 : 0,
5584
+ entry.version,
5585
+ entry.commitSha,
5586
+ entry.configDigest,
5587
+ entry.timestamp,
5588
+ entry.timestamp
5589
+ )
5590
+ );
5591
+ }
5592
+ for (const edge of edgeAggregates.values()) {
5593
+ statements.push(
5594
+ db.prepare(
5595
+ `INSERT INTO ontology_edges (
5596
+ project_id, environment, id, source_node_id, predicate, target_node_id,
5597
+ relationship_certainty, call_count, error_count, last_duration_ms,
5598
+ first_seen_at, last_seen_at
5599
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
5600
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
5601
+ call_count = ontology_edges.call_count + excluded.call_count,
5602
+ error_count = ontology_edges.error_count + excluded.error_count,
5603
+ last_duration_ms = excluded.last_duration_ms,
5604
+ last_seen_at = excluded.last_seen_at`
5605
+ ).bind(
5606
+ edge.projectId,
5607
+ edge.environment,
5608
+ edge.id,
5609
+ edge.sourceNodeId,
5610
+ edge.predicate,
5611
+ edge.targetNodeId,
5612
+ edge.certainty,
5613
+ edge.callCount,
5614
+ edge.errorCount,
5615
+ edge.lastDurationMs,
5616
+ edge.firstSeenAt,
5617
+ edge.lastSeenAt
5618
+ )
5619
+ );
5620
+ }
5621
+ if (statements.length > 0) {
5622
+ await db.batch(statements);
5623
+ }
5624
+ }
5625
+ function computeTarjanScc(nodes, edges) {
5626
+ const adjacency = /* @__PURE__ */ new Map();
5627
+ const selfLoops = /* @__PURE__ */ new Set();
5628
+ for (const node of nodes) {
5629
+ adjacency.set(node.id, []);
5630
+ }
5631
+ for (const edge of edges) {
5632
+ if (!adjacency.has(edge.sourceNodeId)) {
5633
+ adjacency.set(edge.sourceNodeId, []);
5634
+ }
5635
+ if (!adjacency.has(edge.targetNodeId)) {
5636
+ adjacency.set(edge.targetNodeId, []);
5637
+ }
5638
+ adjacency.get(edge.sourceNodeId)?.push(edge.targetNodeId);
5639
+ if (edge.sourceNodeId === edge.targetNodeId) {
5640
+ selfLoops.add(edge.sourceNodeId);
5641
+ }
5642
+ }
5643
+ let index = 0;
5644
+ const indices = /* @__PURE__ */ new Map();
5645
+ const lowlink = /* @__PURE__ */ new Map();
5646
+ const onStack = /* @__PURE__ */ new Set();
5647
+ const stack = [];
5648
+ const rawComponents = [];
5649
+ const strongConnect = (nodeId) => {
5650
+ indices.set(nodeId, index);
5651
+ lowlink.set(nodeId, index);
5652
+ index += 1;
5653
+ stack.push(nodeId);
5654
+ onStack.add(nodeId);
5655
+ const neighbors = adjacency.get(nodeId) ?? [];
5656
+ for (const targetId of neighbors) {
5657
+ if (!indices.has(targetId)) {
5658
+ strongConnect(targetId);
5659
+ lowlink.set(
5660
+ nodeId,
5661
+ Math.min(lowlink.get(nodeId) ?? 0, lowlink.get(targetId) ?? 0)
5662
+ );
5663
+ } else if (onStack.has(targetId)) {
5664
+ lowlink.set(
5665
+ nodeId,
5666
+ Math.min(lowlink.get(nodeId) ?? 0, indices.get(targetId) ?? 0)
5667
+ );
5668
+ }
5669
+ }
5670
+ if (lowlink.get(nodeId) === indices.get(nodeId)) {
5671
+ const component = [];
5672
+ while (stack.length > 0) {
5673
+ const w = stack.pop();
5674
+ onStack.delete(w);
5675
+ component.push(w);
5676
+ if (w === nodeId) break;
5677
+ }
5678
+ component.sort();
5679
+ rawComponents.push(component);
5680
+ }
5681
+ };
5682
+ for (const nodeId of adjacency.keys()) {
5683
+ if (!indices.has(nodeId)) {
5684
+ strongConnect(nodeId);
5685
+ }
5686
+ }
5687
+ const nodeSccMap = /* @__PURE__ */ new Map();
5688
+ const components = rawComponents.map((nodeIds, idx) => {
5689
+ const sccId = `scc_${idx + 1}`;
5690
+ const isCyclic = nodeIds.length > 1 || nodeIds.length === 1 && selfLoops.has(nodeIds[0]);
5691
+ for (const id of nodeIds) {
5692
+ nodeSccMap.set(id, { sccId, isCyclic });
5693
+ }
5694
+ return { sccId, nodeIds, isCyclic };
5695
+ });
5696
+ const annotatedNodes = nodes.map((n) => {
5697
+ const scc = nodeSccMap.get(n.id);
5698
+ return {
5699
+ ...n,
5700
+ sccId: scc?.sccId ?? "scc_0",
5701
+ inCycle: scc?.isCyclic ?? false
5702
+ };
5703
+ });
5704
+ return { annotatedNodes, components };
5705
+ }
5706
+ function localizeTraceFaultLocus(traceId, spans) {
5707
+ if (spans.length === 0) {
5708
+ return {
5709
+ faultLocusSpan: null,
5710
+ faultLocusNodeId: null,
5711
+ relationshipCertainty: "exact",
5712
+ causalProofTier: "unverified_locus",
5713
+ causalTriples: []
5714
+ };
5715
+ }
5716
+ const spanById = /* @__PURE__ */ new Map();
5717
+ const childrenByParent = /* @__PURE__ */ new Map();
5718
+ for (const s of spans) {
5719
+ spanById.set(s.span_id, s);
5720
+ if (s.parent_span_id) {
5721
+ const list = childrenByParent.get(s.parent_span_id) ?? [];
5722
+ list.push(s);
5723
+ childrenByParent.set(s.parent_span_id, list);
5724
+ }
5725
+ }
5726
+ for (const list of childrenByParent.values()) {
5727
+ list.sort((a, b) => a.start_time.localeCompare(b.start_time));
5728
+ }
5729
+ const roots = spans.filter((s) => !s.parent_span_id || !spanById.has(s.parent_span_id)).sort((a, b) => a.start_time.localeCompare(b.start_time));
5730
+ const visited = /* @__PURE__ */ new Set();
5731
+ const causalTriples = [];
5732
+ const leafErrorSpans = [];
5733
+ const MAX_DEPTH = 256;
5734
+ const walk = (span, depth) => {
5735
+ if (visited.has(span.span_id) || depth > MAX_DEPTH) return;
5736
+ visited.add(span.span_id);
5737
+ const children = (childrenByParent.get(span.span_id) ?? []).filter(
5738
+ (c) => !visited.has(c.span_id)
5739
+ );
5740
+ const errorChildren = children.filter((c) => c.status_code === 2);
5741
+ if (span.status_code === 2) {
5742
+ if (errorChildren.length === 0) {
5743
+ leafErrorSpans.push(span);
5744
+ } else {
5745
+ for (const errChild of errorChildren) {
5746
+ causalTriples.push({
5747
+ subjectKind: "span",
5748
+ subjectId: span.span_id,
5749
+ predicate: "failed_due_to_child_span",
5750
+ objectKind: "span",
5751
+ objectId: errChild.span_id,
5752
+ relationshipCertainty: "exact",
5753
+ causalProofTier: "unverified_locus",
5754
+ confidence: 0.85,
5755
+ reason: `Parent span ${span.span_name} failed while child span ${errChild.span_name} returned ERROR`
5756
+ });
5757
+ }
5758
+ }
5759
+ }
5760
+ for (const child of children) {
5761
+ walk(child, depth + 1);
5762
+ }
5763
+ };
5764
+ for (const root of roots) {
5765
+ walk(root, 0);
5766
+ }
5767
+ for (const s of spans) {
5768
+ if (!visited.has(s.span_id)) {
5769
+ walk(s, 0);
5770
+ }
5771
+ }
5772
+ leafErrorSpans.sort((a, b) => a.start_time.localeCompare(b.start_time));
5773
+ const faultLocusSpan = leafErrorSpans[0] ?? spans.find((s) => s.status_code === 2) ?? null;
5774
+ let faultLocusNodeId = null;
5775
+ if (faultLocusSpan) {
5776
+ const env = (faultLocusSpan.environment ?? "default").toLowerCase();
5777
+ faultLocusNodeId = faultLocusSpan.target_node_id ?? (faultLocusSpan.service_name ? buildCanonicalNodeId2({
5778
+ kind: "service",
5779
+ environment: env,
5780
+ serviceName: faultLocusSpan.service_name
5781
+ }).id : null);
5782
+ causalTriples.unshift({
5783
+ subjectKind: "trace",
5784
+ subjectId: traceId,
5785
+ predicate: "originated_fault_in_span",
5786
+ objectKind: "span",
5787
+ objectId: faultLocusSpan.span_id,
5788
+ relationshipCertainty: "exact",
5789
+ causalProofTier: "unverified_locus",
5790
+ confidence: 0.9,
5791
+ reason: `Deepest/earliest failing span ${faultLocusSpan.span_name} (${faultLocusSpan.service_name ?? "unknown"}) is the trace fault locus`
5792
+ });
5793
+ }
5794
+ return {
5795
+ faultLocusSpan,
5796
+ faultLocusNodeId,
5797
+ relationshipCertainty: "exact",
5798
+ causalProofTier: "unverified_locus",
5799
+ causalTriples
5800
+ };
5801
+ }
5802
+ async function loadTraceFaultLocusFromDb(db, projectId, traceId) {
5803
+ const rows = await db.prepare(
5804
+ `SELECT
5805
+ span_id, parent_span_id, service_name, span_name, span_kind,
5806
+ status_code, status_message, start_time, end_time, duration_ms,
5807
+ environment, target_node_id, error_signature
5808
+ FROM telemetry_spans
5809
+ WHERE project_id = ? AND trace_id = ?
5810
+ ORDER BY start_time ASC
5811
+ LIMIT 500`
5812
+ ).bind(projectId, traceId).all();
5813
+ return localizeTraceFaultLocus(traceId, rows.results);
5814
+ }
5815
+ async function getOntologyGraph(db, options) {
5816
+ const env = (options.environment ?? "default").toLowerCase();
5817
+ const [nodeRows, edgeRows] = await Promise.all([
5818
+ db.prepare(
5819
+ `SELECT * FROM ontology_nodes
5820
+ WHERE project_id = ? AND environment = ?
5821
+ ORDER BY id ASC
5822
+ LIMIT 500`
5823
+ ).bind(options.projectId, env).all(),
5824
+ db.prepare(
5825
+ `SELECT * FROM ontology_edges
5826
+ WHERE project_id = ? AND environment = ?
5827
+ ORDER BY call_count DESC
5828
+ LIMIT 1000`
5829
+ ).bind(options.projectId, env).all()
5830
+ ]);
5831
+ const rawNodes = nodeRows.results.map((r) => ({
5832
+ projectId: r.project_id,
5833
+ environment: r.environment,
5834
+ id: r.id,
5835
+ kind: r.kind,
5836
+ name: r.name,
5837
+ namespaceOrEndpoint: r.namespace_or_endpoint,
5838
+ isScopedIdentity: r.is_scoped_identity === 1,
5839
+ currentVersion: r.current_version,
5840
+ currentCommitSha: r.current_commit_sha,
5841
+ currentConfigDigest: r.current_config_digest,
5842
+ attributes: parseJsonRecord(r.attributes_json ?? "{}"),
5843
+ firstSeenAt: r.first_seen_at,
5844
+ lastSeenAt: r.last_seen_at
5845
+ }));
5846
+ const edges = edgeRows.results.map((r) => ({
5847
+ projectId: r.project_id,
5848
+ environment: r.environment,
5849
+ id: r.id,
5850
+ sourceNodeId: r.source_node_id,
5851
+ predicate: r.predicate,
5852
+ targetNodeId: r.target_node_id,
5853
+ relationshipCertainty: r.relationship_certainty || "exact",
5854
+ callCount: Number(r.call_count ?? 0),
5855
+ errorCount: Number(r.error_count ?? 0),
5856
+ lastDurationMs: r.last_duration_ms === null ? null : Number(r.last_duration_ms),
5857
+ firstSeenAt: r.first_seen_at,
5858
+ lastSeenAt: r.last_seen_at
5859
+ }));
5860
+ const { annotatedNodes, components } = computeTarjanScc(rawNodes, edges);
5861
+ const inboundErrorsByTarget = /* @__PURE__ */ new Map();
5862
+ for (const e of edges) {
5863
+ if (e.errorCount > 0) {
5864
+ inboundErrorsByTarget.set(
5865
+ e.targetNodeId,
5866
+ (inboundErrorsByTarget.get(e.targetNodeId) ?? 0) + e.errorCount
5867
+ );
5868
+ }
5869
+ }
5870
+ const nodes = annotatedNodes.map((n) => ({
5871
+ ...n,
5872
+ isBlackoutSuspected: (inboundErrorsByTarget.get(n.id) ?? 0) >= 3 && edges.filter((e) => e.sourceNodeId === n.id).length === 0
5873
+ }));
5874
+ return {
5875
+ environment: env,
5876
+ nodes,
5877
+ edges,
5878
+ stronglyConnectedComponents: components
5879
+ };
5880
+ }
5881
+
3644
5882
  // src/lib/store/ingest.ts
3645
5883
  async function ingestTelemetrySpans(db, spans) {
3646
5884
  if (spans.length === 0) return { inserted: 0, traceCount: 0 };
3647
- const statements = spans.map((span) => {
5885
+ const enrichedSpans = spans.map((span) => {
3648
5886
  if (!span.projectId)
3649
5887
  throw new Error("TelemetryStore.ingest: span.projectId is required");
3650
- return db.prepare(`
5888
+ if (span.environment !== void 0 && span.serviceVersion !== void 0 && span.targetNodeId !== void 0) {
5889
+ return span;
5890
+ }
5891
+ const lineage = extractSpanLineageAndTopology(span);
5892
+ return {
5893
+ ...span,
5894
+ environment: span.environment ?? lineage.environment,
5895
+ serviceVersion: span.serviceVersion ?? lineage.serviceVersion,
5896
+ commitSha: span.commitSha ?? lineage.commitSha,
5897
+ targetNodeId: span.targetNodeId ?? lineage.targetNode?.id ?? null,
5898
+ errorSignature: span.errorSignature ?? lineage.errorSignature
5899
+ };
5900
+ });
5901
+ const statements = enrichedSpans.map(
5902
+ (span) => db.prepare(`
3651
5903
  INSERT OR IGNORE INTO telemetry_spans (
3652
5904
  project_id, trace_id, span_id, parent_span_id, trace_state,
3653
5905
  service_name, scope_name, scope_version, span_name, span_kind,
@@ -3655,8 +5907,9 @@ async function ingestTelemetrySpans(db, spans) {
3655
5907
  attributes_json, dropped_attributes_count,
3656
5908
  resource_attributes_json, events_json, dropped_events_count,
3657
5909
  links_json, dropped_links_count, received_at, expires_at,
3658
- session_id, interaction_id, telemetry_sdk_name
3659
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
5910
+ session_id, interaction_id, telemetry_sdk_name,
5911
+ environment, service_version, commit_sha, target_node_id, error_signature
5912
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
3660
5913
  `).bind(
3661
5914
  span.projectId,
3662
5915
  span.traceId,
@@ -3684,10 +5937,69 @@ async function ingestTelemetrySpans(db, spans) {
3684
5937
  span.expiresAt,
3685
5938
  span.sessionId ?? null,
3686
5939
  span.interactionId ?? null,
3687
- span.telemetrySdkName ?? null
3688
- );
3689
- });
3690
- await db.batch(statements);
5940
+ span.telemetrySdkName ?? null,
5941
+ span.environment ?? "default",
5942
+ span.serviceVersion ?? null,
5943
+ span.commitSha ?? null,
5944
+ span.targetNodeId ?? null,
5945
+ span.errorSignature ?? null
5946
+ )
5947
+ );
5948
+ try {
5949
+ await db.batch(statements);
5950
+ } catch (err) {
5951
+ const msg = err instanceof Error ? err.message : String(err);
5952
+ if (msg.includes("no such column: environment")) {
5953
+ const legacyStatements = enrichedSpans.map(
5954
+ (span) => db.prepare(`
5955
+ INSERT OR IGNORE INTO telemetry_spans (
5956
+ project_id, trace_id, span_id, parent_span_id, trace_state,
5957
+ service_name, scope_name, scope_version, span_name, span_kind,
5958
+ status_code, status_message, start_time, end_time, duration_ms,
5959
+ attributes_json, dropped_attributes_count,
5960
+ resource_attributes_json, events_json, dropped_events_count,
5961
+ links_json, dropped_links_count, received_at, expires_at,
5962
+ session_id, interaction_id, telemetry_sdk_name
5963
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
5964
+ `).bind(
5965
+ span.projectId,
5966
+ span.traceId,
5967
+ span.spanId,
5968
+ span.parentSpanId,
5969
+ span.traceState,
5970
+ span.serviceName,
5971
+ span.scopeName,
5972
+ span.scopeVersion,
5973
+ span.spanName,
5974
+ span.spanKind,
5975
+ span.statusCode,
5976
+ span.statusMessage,
5977
+ span.startTime,
5978
+ span.endTime,
5979
+ span.durationMs,
5980
+ span.attributesJson,
5981
+ span.droppedAttributesCount,
5982
+ span.resourceAttributesJson,
5983
+ span.eventsJson,
5984
+ span.droppedEventsCount,
5985
+ span.linksJson,
5986
+ span.droppedLinksCount,
5987
+ span.receivedAt,
5988
+ span.expiresAt,
5989
+ span.sessionId ?? null,
5990
+ span.interactionId ?? null,
5991
+ span.telemetrySdkName ?? null
5992
+ )
5993
+ );
5994
+ await db.batch(legacyStatements);
5995
+ } else {
5996
+ throw err;
5997
+ }
5998
+ }
5999
+ try {
6000
+ await upsertOntologyFromSpans(db, enrichedSpans);
6001
+ } catch {
6002
+ }
3691
6003
  const uniqueTraces = new Set(spans.map((s) => `${s.projectId}:${s.traceId}`));
3692
6004
  return {
3693
6005
  inserted: spans.length,
@@ -8533,118 +10845,24 @@ var aiSpanPayloadsProcessorPlugin = {
8533
10845
  r.receivedAt,
8534
10846
  r.expiresAt,
8535
10847
  r.interactionId,
8536
- r.actionId
8537
- )
8538
- )
8539
- );
8540
- } catch (err) {
8541
- context.logger.error("[ai-span-payloads-processor] write failed", {
8542
- error: err instanceof Error ? err.message : String(err)
8543
- });
8544
- }
8545
- }
8546
- return transformed;
8547
- }
8548
- });
8549
- }
8550
- };
8551
-
8552
- // src/lib/alerts-store.ts
8553
- import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT4 } from "@obsunified/types";
8554
-
8555
- // src/lib/webhook-url.ts
8556
- var WebhookUrlError = class extends Error {
8557
- };
8558
- var parseIpv4 = (host) => {
8559
- const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
8560
- if (!m) return null;
8561
- const octets = m.slice(1).map((o) => Number.parseInt(o, 10));
8562
- return octets.every((o) => o <= 255) ? octets : null;
8563
- };
8564
- var isPrivateIpv4 = (o) => o[0] === 0 || // "this" network / unspecified
8565
- o[0] === 10 || o[0] === 127 || o[0] === 100 && o[1] >= 64 && o[1] <= 127 || // CGNAT
8566
- o[0] === 169 && o[1] === 254 || // link-local incl. cloud metadata
8567
- o[0] === 172 && o[1] >= 16 && o[1] <= 31 || o[0] === 192 && o[1] === 168 || o[0] >= 224;
8568
- var isPrivateIpv6 = (host) => {
8569
- const literal = host.replace(/^\[|\]$/g, "").toLowerCase();
8570
- if (literal === "::" || literal === "::1") return true;
8571
- const dotted = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(literal);
8572
- if (dotted) {
8573
- const v4 = parseIpv4(dotted[1]);
8574
- return v4 ? isPrivateIpv4(v4) : true;
8575
- }
8576
- const hex = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(literal);
8577
- if (hex) {
8578
- const hi = Number.parseInt(hex[1], 16);
8579
- const lo = Number.parseInt(hex[2], 16);
8580
- return isPrivateIpv4([hi >> 8, hi & 255, lo >> 8, lo & 255]);
8581
- }
8582
- const first = Number.parseInt(literal.split(":")[0] || "0", 16);
8583
- if (Number.isNaN(first)) return true;
8584
- if ((first & 65024) === 64512) return true;
8585
- if ((first & 65472) === 65152) return true;
8586
- if ((first & 65280) === 65280) return true;
8587
- return false;
8588
- };
8589
- var isPrivateWebhookHost = (hostname) => {
8590
- const host = hostname.toLowerCase().replace(/\.$/, "");
8591
- if (!host) return true;
8592
- if (host === "localhost" || host.endsWith(".localhost")) return true;
8593
- if (host.endsWith(".local") || host.endsWith(".internal")) return true;
8594
- if (host.startsWith("[")) return isPrivateIpv6(host);
8595
- const v4 = parseIpv4(host);
8596
- if (v4) return isPrivateIpv4(v4);
8597
- return !host.includes(".");
8598
- };
8599
- var assertSafeWebhookUrl = (raw, options = {}) => {
8600
- let url;
8601
- try {
8602
- url = new URL(raw);
8603
- } catch {
8604
- throw new WebhookUrlError("channel.url must be an absolute http(s) URL");
8605
- }
8606
- if (url.protocol !== "https:" && url.protocol !== "http:") {
8607
- throw new WebhookUrlError("channel.url must be http(s)://");
8608
- }
8609
- if (url.username || url.password) {
8610
- throw new WebhookUrlError(
8611
- "channel.url must not embed credentials; use channel.headers"
8612
- );
8613
- }
8614
- if (!options.allowPrivateNetworks && isPrivateWebhookHost(url.hostname)) {
8615
- throw new WebhookUrlError(
8616
- `channel.url host "${url.hostname}" is not a public address (set ALERT_WEBHOOK_ALLOW_PRIVATE=true to allow internal receivers)`
8617
- );
8618
- }
8619
- return url;
8620
- };
8621
- var resolveWebhookHost = async (hostname, lookup) => {
8622
- if (parseIpv4(hostname) || hostname.startsWith("[")) return "public";
8623
- let addresses;
8624
- try {
8625
- addresses = await lookup(hostname);
8626
- } catch {
8627
- return "unresolvable";
8628
- }
8629
- if (addresses.length === 0) return "unresolvable";
8630
- const isPrivate = addresses.some((address) => {
8631
- const v4 = parseIpv4(address);
8632
- if (v4) return isPrivateIpv4(v4);
8633
- return isPrivateIpv6(`[${address}]`);
8634
- });
8635
- return isPrivate ? "private" : "public";
8636
- };
8637
- var redactWebhookUrl = (raw) => {
8638
- try {
8639
- const url = new URL(raw);
8640
- const hasPath = url.pathname !== "/" || url.search !== "";
8641
- return `${url.protocol}//${url.host}${hasPath ? "/\u2026" : ""}`;
8642
- } catch {
8643
- return "<invalid url>";
10848
+ r.actionId
10849
+ )
10850
+ )
10851
+ );
10852
+ } catch (err) {
10853
+ context.logger.error("[ai-span-payloads-processor] write failed", {
10854
+ error: err instanceof Error ? err.message : String(err)
10855
+ });
10856
+ }
10857
+ }
10858
+ return transformed;
10859
+ }
10860
+ });
8644
10861
  }
8645
10862
  };
8646
10863
 
8647
10864
  // src/lib/alerts-store.ts
10865
+ import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT4 } from "@obsunified/types";
8648
10866
  var parseJsonField = (value) => {
8649
10867
  if (typeof value === "string") return JSON.parse(value);
8650
10868
  return value;
@@ -9120,7 +11338,7 @@ async function mapWithConcurrency(items, concurrency, mapper) {
9120
11338
  );
9121
11339
  return results;
9122
11340
  }
9123
- async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery) {
11341
+ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery, onAlertFiring) {
9124
11342
  const none = { fired: 0, resolved: 0, redelivered: 0 };
9125
11343
  try {
9126
11344
  const value = await withTimeout(
@@ -9158,6 +11376,9 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
9158
11376
  );
9159
11377
  if (previous !== next) {
9160
11378
  await store.transitionState(rule.id, rule.projectId, next, now);
11379
+ if (next === "firing" && onAlertFiring) {
11380
+ await onAlertFiring(rule).catch(() => void 0);
11381
+ }
9161
11382
  const ok = await deliver(next);
9162
11383
  if (ok) {
9163
11384
  await store.markNotified(rule.id, rule.projectId, next, now);
@@ -9226,7 +11447,8 @@ async function evaluateRuleBatch(store, rules, options) {
9226
11447
  logger,
9227
11448
  options?.tracer,
9228
11449
  ruleTimeoutMs,
9229
- delivery
11450
+ delivery,
11451
+ options?.onAlertFiring
9230
11452
  )
9231
11453
  );
9232
11454
  return {
@@ -9240,7 +11462,8 @@ async function evaluateAllRules(env, options) {
9240
11462
  const logger = options?.logger ?? consoleLogger;
9241
11463
  const tracer = options?.tracer;
9242
11464
  const allowPrivateWebhookTargets = env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true";
9243
- const store = new AlertsStore(sqlDbFor(env), { allowPrivateWebhookTargets });
11465
+ const db = sqlDbFor(env);
11466
+ const store = new AlertsStore(db, { allowPrivateWebhookTargets });
9244
11467
  const rules = await store.listEnabledRules();
9245
11468
  return evaluateRuleBatch(store, rules, {
9246
11469
  logger,
@@ -9248,7 +11471,15 @@ async function evaluateAllRules(env, options) {
9248
11471
  concurrency: options?.concurrency,
9249
11472
  ruleTimeoutMs: options?.ruleTimeoutMs,
9250
11473
  allowPrivateWebhookTargets,
9251
- hostLookup: options?.hostLookup
11474
+ hostLookup: options?.hostLookup,
11475
+ onAlertFiring: async (rule) => {
11476
+ await openOrDeduplicateIncident(db, rule.projectId, {
11477
+ environment: "default",
11478
+ targetNodeId: `alert:${rule.id}`,
11479
+ symptomAnchorKind: "alert",
11480
+ symptomAnchorId: rule.id
11481
+ });
11482
+ }
9252
11483
  });
9253
11484
  }
9254
11485
  function createAlertEvaluatorHandler() {
@@ -9542,7 +11773,7 @@ var summarizeResult = (def, result) => {
9542
11773
  };
9543
11774
  async function runAsk(question, deps) {
9544
11775
  if (deps.llm.provider === "openai") {
9545
- const { runAskOpenAI } = await import("./openai-FJYOQ7WO.js");
11776
+ const { runAskOpenAI } = await import("./openai-GSJZGFGL.js");
9546
11777
  return runAskOpenAI(question, deps);
9547
11778
  }
9548
11779
  return runAskAnthropic(question, deps);
@@ -9717,149 +11948,1288 @@ async function callAnthropic(config, messages, turnIndex) {
9717
11948
  json.usage.output_tokens
9718
11949
  );
9719
11950
  }
9720
- if (span && json.stop_reason)
9721
- span.setAttribute("gen_ai.response.finish_reason", json.stop_reason);
9722
- return json;
9723
- };
9724
- if (!config.tracer) return exec(null);
9725
- const attrs = {
9726
- "openinference.span.kind": "LLM",
9727
- "gen_ai.system": "anthropic",
9728
- "gen_ai.request.model": config.model,
9729
- "gen_ai.request.max_tokens": 1024,
9730
- "http.url": url
11951
+ if (span && json.stop_reason)
11952
+ span.setAttribute("gen_ai.response.finish_reason", json.stop_reason);
11953
+ return json;
11954
+ };
11955
+ if (!config.tracer) return exec(null);
11956
+ const attrs = {
11957
+ "openinference.span.kind": "LLM",
11958
+ "gen_ai.system": "anthropic",
11959
+ "gen_ai.request.model": config.model,
11960
+ "gen_ai.request.max_tokens": 1024,
11961
+ "http.url": url
11962
+ };
11963
+ if (typeof turnIndex === "number") attrs["llm.turn"] = turnIndex;
11964
+ return config.tracer(
11965
+ "llm.anthropic.messages",
11966
+ async (span) => exec(span),
11967
+ attrs
11968
+ );
11969
+ }
11970
+
11971
+ // src/plugins/ask-routes.ts
11972
+ var MAX_QUESTION_CHARS = 1e3;
11973
+ var askRoutesPlugin = {
11974
+ name: "ask-routes",
11975
+ register(app, runtime) {
11976
+ app.post("/internal/ask", async (c) => {
11977
+ const body = await readJsonObject(c);
11978
+ if (!body) {
11979
+ return c.json(
11980
+ { error: "Bad Request", message: "body must be a JSON object" },
11981
+ 400
11982
+ );
11983
+ }
11984
+ const question = (body.question ?? "").trim();
11985
+ if (!question) {
11986
+ return c.json(
11987
+ { error: "Bad Request", message: "question is required" },
11988
+ 400
11989
+ );
11990
+ }
11991
+ if (question.length > MAX_QUESTION_CHARS) {
11992
+ return c.json(
11993
+ {
11994
+ error: "Bad Request",
11995
+ message: `question exceeds ${MAX_QUESTION_CHARS} chars`
11996
+ },
11997
+ 400
11998
+ );
11999
+ }
12000
+ const projectId = getProjectId(c);
12001
+ const openaiBase = c.env.OPENAI_BASE_URL?.trim() ? c.env.OPENAI_BASE_URL : void 0;
12002
+ const narrativeModel = c.env.NARRATIVE_MODEL?.trim() ? c.env.NARRATIVE_MODEL : void 0;
12003
+ const llm = c.env.OPENAI_API_KEY?.trim() ? {
12004
+ provider: "openai",
12005
+ apiKey: c.env.OPENAI_API_KEY,
12006
+ model: narrativeModel ?? "gpt-4o-mini",
12007
+ apiUrl: openaiBase,
12008
+ tracer: runtime.withChildSpan
12009
+ } : c.env.ANTHROPIC_API_KEY?.trim() ? {
12010
+ provider: "anthropic",
12011
+ apiKey: c.env.ANTHROPIC_API_KEY,
12012
+ model: narrativeModel ?? "claude-haiku-4-5",
12013
+ tracer: runtime.withChildSpan
12014
+ } : null;
12015
+ if (!llm) {
12016
+ const response = {
12017
+ answer: null,
12018
+ evidence: [],
12019
+ queries: [],
12020
+ error: "Ask is not configured \u2014 set OPENAI_API_KEY or ANTHROPIC_API_KEY on the collector to enable it.",
12021
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
12022
+ };
12023
+ return c.json(response, 503);
12024
+ }
12025
+ const store = new AnalysesStore(sqlDbFor(c.env));
12026
+ try {
12027
+ const startedAt = Date.now();
12028
+ runtime.logger.info("[ask] llm call starting", {
12029
+ project_id: projectId,
12030
+ provider: llm.provider,
12031
+ model: llm.model,
12032
+ question_chars: question.length
12033
+ });
12034
+ const result = await runtime.withChildSpan(
12035
+ "ask.runAsk",
12036
+ () => runAsk(question, {
12037
+ llm,
12038
+ listAnalyses: async (filters) => {
12039
+ const all = await getAllAnalysesForProject(projectId, {
12040
+ db: sqlDbFor(c.env)
12041
+ });
12042
+ return all.filter((d) => {
12043
+ if (filters?.group && d.group !== filters.group) return false;
12044
+ if (filters?.view && d.view !== filters.view) return false;
12045
+ return true;
12046
+ });
12047
+ },
12048
+ getLatestResult: async (id) => {
12049
+ const definitions = await store.listDefinitions(projectId);
12050
+ const definition = definitions.find((d) => d.id === id);
12051
+ if (!definition) return null;
12052
+ const r = await store.getLatestResult(projectId, id);
12053
+ return { definition, result: r };
12054
+ }
12055
+ }),
12056
+ {
12057
+ "llm.provider": llm.provider,
12058
+ "llm.model": llm.model,
12059
+ "openinference.span.kind": "LLM"
12060
+ }
12061
+ );
12062
+ runtime.logger.info("[ask] llm call finished", {
12063
+ project_id: projectId,
12064
+ provider: llm.provider,
12065
+ model: llm.model,
12066
+ latency_ms: Date.now() - startedAt,
12067
+ evidence_count: result.evidence.length,
12068
+ query_count: result.queries.length,
12069
+ answered: Boolean(result.answer)
12070
+ });
12071
+ if (result.answer && result.evidence.length > 0) {
12072
+ await store.recordAskEvidence(
12073
+ projectId,
12074
+ result.evidence.map((e) => e.analysisId)
12075
+ ).catch(
12076
+ (err) => runtime.logger.warn("[ask] recordAskEvidence failed", {
12077
+ error: err instanceof Error ? err.message : String(err)
12078
+ })
12079
+ );
12080
+ }
12081
+ return c.json(result);
12082
+ } catch (error) {
12083
+ const message = error instanceof Error ? error.message : String(error);
12084
+ runtime.logger.error("[ask] failed", { error: message });
12085
+ const response = {
12086
+ answer: null,
12087
+ evidence: [],
12088
+ queries: [],
12089
+ error: message,
12090
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
12091
+ };
12092
+ return c.json(response, 500);
12093
+ }
12094
+ });
12095
+ }
12096
+ };
12097
+
12098
+ // src/plugins/autosre-routes.ts
12099
+ import { AgentAutonomyLevel as AgentAutonomyLevel2 } from "@obsunified/types/constants";
12100
+
12101
+ // src/lib/autosre-change-correlation.ts
12102
+ import { buildCanonicalNodeId as buildCanonicalNodeId3 } from "@obsunified/types";
12103
+ var mapChangeEventRow = (row) => ({
12104
+ id: row.id,
12105
+ projectId: row.project_id,
12106
+ environment: row.environment,
12107
+ targetNodeId: row.target_node_id,
12108
+ kind: row.kind,
12109
+ source: row.source,
12110
+ relationshipCertainty: row.relationship_certainty || "exact",
12111
+ summary: row.summary,
12112
+ serviceName: row.service_name,
12113
+ version: row.version,
12114
+ previousVersion: row.previous_version,
12115
+ commitSha: row.commit_sha,
12116
+ previousCommitSha: row.previous_commit_sha,
12117
+ configDigest: row.config_digest,
12118
+ repositoryUrl: row.repository_url,
12119
+ pullRequestUrl: row.pull_request_url,
12120
+ actorId: row.actor_id,
12121
+ causedByActionId: row.caused_by_action_id,
12122
+ agentRunId: row.agent_run_id,
12123
+ touchedPaths: parseJsonArray(row.touched_paths_json ?? "[]").filter(
12124
+ (p) => typeof p === "string"
12125
+ ),
12126
+ attributes: parseJsonRecord(row.attributes_json ?? "{}"),
12127
+ occurredAt: row.occurred_at,
12128
+ receivedAt: row.received_at,
12129
+ expiresAt: row.expires_at
12130
+ });
12131
+ async function recordChangeEvent(db, projectId, input) {
12132
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
12133
+ const occurredAt = input.occurredAt || nowIso;
12134
+ const environment = (input.environment || "default").trim().toLowerCase();
12135
+ const namespace = (input.namespace || "default").trim();
12136
+ const serviceName = input.serviceName?.trim() || null;
12137
+ const canonicalNode = input.targetNodeId ? {
12138
+ id: input.targetNodeId,
12139
+ environment,
12140
+ kind: input.kind === "feature_flag" ? "feature_flag" : "service",
12141
+ name: serviceName ?? input.targetNodeId,
12142
+ namespaceOrEndpoint: namespace,
12143
+ isScopedIdentity: true
12144
+ } : input.kind === "feature_flag" ? buildCanonicalNodeId3({
12145
+ kind: "feature_flag",
12146
+ environment,
12147
+ flagKey: (typeof input.attributes?.flagKey === "string" ? input.attributes.flagKey : serviceName) ?? "flag"
12148
+ }) : buildCanonicalNodeId3({
12149
+ kind: "service",
12150
+ environment,
12151
+ namespace,
12152
+ serviceName: serviceName ?? "unknown-service"
12153
+ });
12154
+ const id = input.id?.trim() || `chg_${randomHex(10)}`;
12155
+ const source = input.source ?? "explicit_api";
12156
+ const certainty = input.relationshipCertainty ?? (source === "inferred_telemetry" ? "inferred" : "exact");
12157
+ const touchedPaths = (input.touchedPaths ?? []).slice(0, 500);
12158
+ const expiresAt = new Date(
12159
+ Date.parse(occurredAt) + 30 * 24 * 36e5
12160
+ ).toISOString();
12161
+ await db.batch([
12162
+ db.prepare(
12163
+ `INSERT INTO ontology_nodes (
12164
+ project_id, environment, id, kind, name, namespace_or_endpoint,
12165
+ is_scoped_identity, current_version, current_commit_sha,
12166
+ current_config_digest, attributes_json, first_seen_at, last_seen_at
12167
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
12168
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
12169
+ current_version = COALESCE(excluded.current_version, ontology_nodes.current_version),
12170
+ current_commit_sha = COALESCE(excluded.current_commit_sha, ontology_nodes.current_commit_sha),
12171
+ current_config_digest = COALESCE(excluded.current_config_digest, ontology_nodes.current_config_digest),
12172
+ last_seen_at = excluded.last_seen_at`
12173
+ ).bind(
12174
+ projectId,
12175
+ environment,
12176
+ canonicalNode.id,
12177
+ canonicalNode.kind,
12178
+ canonicalNode.name,
12179
+ canonicalNode.namespaceOrEndpoint,
12180
+ canonicalNode.isScopedIdentity ? 1 : 0,
12181
+ input.version ?? null,
12182
+ input.commitSha ?? null,
12183
+ input.configDigest ?? null,
12184
+ occurredAt,
12185
+ nowIso
12186
+ ),
12187
+ db.prepare(
12188
+ `INSERT INTO change_events (
12189
+ project_id, environment, id, target_node_id, kind, source,
12190
+ relationship_certainty, summary, service_name, version,
12191
+ previous_version, commit_sha, previous_commit_sha, config_digest,
12192
+ repository_url, pull_request_url, actor_id, caused_by_action_id,
12193
+ agent_run_id, touched_paths_json, attributes_json,
12194
+ occurred_at, received_at, expires_at
12195
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
12196
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
12197
+ summary = excluded.summary,
12198
+ version = COALESCE(excluded.version, change_events.version),
12199
+ previous_version = COALESCE(excluded.previous_version, change_events.previous_version),
12200
+ commit_sha = COALESCE(excluded.commit_sha, change_events.commit_sha),
12201
+ touched_paths_json = excluded.touched_paths_json,
12202
+ attributes_json = excluded.attributes_json,
12203
+ occurred_at = excluded.occurred_at`
12204
+ ).bind(
12205
+ projectId,
12206
+ environment,
12207
+ id,
12208
+ canonicalNode.id,
12209
+ input.kind,
12210
+ source,
12211
+ certainty,
12212
+ input.summary,
12213
+ serviceName,
12214
+ input.version ?? null,
12215
+ input.previousVersion ?? null,
12216
+ input.commitSha ?? null,
12217
+ input.previousCommitSha ?? null,
12218
+ input.configDigest ?? null,
12219
+ input.repositoryUrl ?? null,
12220
+ input.pullRequestUrl ?? null,
12221
+ input.actorId ?? null,
12222
+ input.causedByActionId ?? null,
12223
+ input.agentRunId ?? null,
12224
+ JSON.stringify(touchedPaths),
12225
+ JSON.stringify(input.attributes ?? {}),
12226
+ occurredAt,
12227
+ nowIso,
12228
+ expiresAt
12229
+ )
12230
+ ]);
12231
+ return {
12232
+ id,
12233
+ projectId,
12234
+ environment,
12235
+ targetNodeId: canonicalNode.id,
12236
+ kind: input.kind,
12237
+ source,
12238
+ relationshipCertainty: certainty,
12239
+ summary: input.summary,
12240
+ serviceName,
12241
+ version: input.version ?? null,
12242
+ previousVersion: input.previousVersion ?? null,
12243
+ commitSha: input.commitSha ?? null,
12244
+ previousCommitSha: input.previousCommitSha ?? null,
12245
+ configDigest: input.configDigest ?? null,
12246
+ repositoryUrl: input.repositoryUrl ?? null,
12247
+ pullRequestUrl: input.pullRequestUrl ?? null,
12248
+ actorId: input.actorId ?? null,
12249
+ causedByActionId: input.causedByActionId ?? null,
12250
+ agentRunId: input.agentRunId ?? null,
12251
+ touchedPaths,
12252
+ attributes: input.attributes ?? {},
12253
+ occurredAt,
12254
+ receivedAt: nowIso,
12255
+ expiresAt
12256
+ };
12257
+ }
12258
+ async function listChangeEvents(db, options) {
12259
+ const conditions = ["project_id = ?"];
12260
+ const binds = [options.projectId];
12261
+ if (options.environment) {
12262
+ conditions.push("environment = ?");
12263
+ binds.push(options.environment.toLowerCase());
12264
+ }
12265
+ if (options.serviceName) {
12266
+ conditions.push("service_name = ?");
12267
+ binds.push(options.serviceName);
12268
+ }
12269
+ if (options.targetNodeId) {
12270
+ conditions.push("target_node_id = ?");
12271
+ binds.push(options.targetNodeId);
12272
+ }
12273
+ if (options.kind) {
12274
+ conditions.push("kind = ?");
12275
+ binds.push(options.kind);
12276
+ }
12277
+ if (options.since) {
12278
+ conditions.push("occurred_at >= ?");
12279
+ binds.push(options.since);
12280
+ }
12281
+ const limit = Math.max(1, Math.min(200, options.limit ?? 50));
12282
+ binds.push(limit);
12283
+ const rows = await db.prepare(
12284
+ `SELECT * FROM change_events
12285
+ WHERE ${conditions.join(" AND ")}
12286
+ ORDER BY occurred_at DESC
12287
+ LIMIT ?`
12288
+ ).bind(...binds).all();
12289
+ return rows.results.map(mapChangeEventRow);
12290
+ }
12291
+ async function getChangeEvent(db, projectId, changeId, environment) {
12292
+ const row = environment ? await db.prepare(
12293
+ `SELECT * FROM change_events
12294
+ WHERE project_id = ? AND environment = ? AND id = ?`
12295
+ ).bind(projectId, environment.toLowerCase(), changeId).first() : await db.prepare(
12296
+ `SELECT * FROM change_events
12297
+ WHERE project_id = ? AND id = ?
12298
+ ORDER BY occurred_at DESC
12299
+ LIMIT 1`
12300
+ ).bind(projectId, changeId).first();
12301
+ return row ? mapChangeEventRow(row) : null;
12302
+ }
12303
+ var computePercentile = (sorted, p) => {
12304
+ if (sorted.length === 0) return null;
12305
+ const idx = Math.min(
12306
+ sorted.length - 1,
12307
+ Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
12308
+ );
12309
+ return sorted[idx];
12310
+ };
12311
+ var summarizeCohort = (cohortKey, requestSpans, downstreamSpans) => {
12312
+ const requestCount = requestSpans.length;
12313
+ const errorCount = requestSpans.filter((s) => s.status_code === 2).length;
12314
+ const errorRate = requestCount > 0 ? errorCount / requestCount : 0;
12315
+ const durations = requestSpans.map((s) => Number(s.duration_ms ?? 0)).sort((a, b) => a - b);
12316
+ const p95DurationMs = computePercentile(durations, 95);
12317
+ const downstreamCount = downstreamSpans.length;
12318
+ const avgDownstreamCallsPerRequest = requestCount > 0 ? downstreamCount / requestCount : 0;
12319
+ const avgDownstreamDurationMs = downstreamCount > 0 ? downstreamSpans.reduce((acc, s) => acc + Number(s.duration_ms ?? 0), 0) / downstreamCount : null;
12320
+ return {
12321
+ cohortKey,
12322
+ requestCount,
12323
+ errorCount,
12324
+ errorRate,
12325
+ p95DurationMs,
12326
+ avgDownstreamCallsPerRequest,
12327
+ avgDownstreamDurationMs
12328
+ };
12329
+ };
12330
+ async function findCrossTraceContention(db, options) {
12331
+ const env = options.environment.toLowerCase();
12332
+ const limit = Math.max(1, Math.min(20, options.limit ?? 5));
12333
+ const [nodeRow, contentionRows] = await Promise.all([
12334
+ db.prepare(
12335
+ `SELECT is_scoped_identity FROM ontology_nodes
12336
+ WHERE project_id = ? AND environment = ? AND id = ?`
12337
+ ).bind(options.projectId, env, options.targetNodeId).first(),
12338
+ db.prepare(
12339
+ `SELECT
12340
+ trace_id,
12341
+ span_id,
12342
+ service_name,
12343
+ service_version,
12344
+ span_name,
12345
+ COUNT(*) AS concurrent_calls,
12346
+ MAX(duration_ms) AS max_duration_ms,
12347
+ SUM(duration_ms) AS total_resource_time_ms
12348
+ FROM telemetry_spans
12349
+ WHERE project_id = ?
12350
+ AND environment = ?
12351
+ AND target_node_id = ?
12352
+ AND trace_id <> ?
12353
+ AND start_time <= ?
12354
+ AND end_time >= ?
12355
+ GROUP BY trace_id, span_id, service_name, service_version, span_name
12356
+ ORDER BY total_resource_time_ms DESC
12357
+ LIMIT ?`
12358
+ ).bind(
12359
+ options.projectId,
12360
+ env,
12361
+ options.targetNodeId,
12362
+ options.excludeTraceId,
12363
+ options.endTime,
12364
+ options.startTime,
12365
+ limit
12366
+ ).all()
12367
+ ]);
12368
+ const isScoped = nodeRow ? nodeRow.is_scoped_identity === 1 : !options.targetNodeId.includes(":unscoped/");
12369
+ const relationshipCertainty = isScoped ? "exact" : "inferred";
12370
+ return contentionRows.results.map((r) => ({
12371
+ aggressorTraceId: r.trace_id,
12372
+ aggressorSpanId: r.span_id,
12373
+ aggressorServiceName: r.service_name,
12374
+ aggressorServiceVersion: r.service_version,
12375
+ aggressorSpanName: r.span_name,
12376
+ sharedTargetNodeId: options.targetNodeId,
12377
+ relationshipCertainty,
12378
+ concurrentCalls: Number(r.concurrent_calls ?? 1),
12379
+ maxDurationMs: Number(r.max_duration_ms ?? 0),
12380
+ totalResourceTimeMs: Number(r.total_resource_time_ms ?? 0)
12381
+ }));
12382
+ }
12383
+ var extractFilePathsFromSpan = (span) => {
12384
+ const found = [];
12385
+ const attrs = parseJsonRecord(span.attributes_json ?? "{}");
12386
+ for (const key of ["code.filepath", "obs.code.filepath", "exception.stacktrace"]) {
12387
+ const val = attrs[key];
12388
+ if (typeof val === "string") {
12389
+ found.push(val);
12390
+ }
12391
+ }
12392
+ if (span.events_json) {
12393
+ const events = parseJsonArray(span.events_json);
12394
+ for (const ev of events) {
12395
+ if (ev && typeof ev === "object" && !Array.isArray(ev)) {
12396
+ const evAttrs = parseJsonRecord(
12397
+ JSON.stringify(ev.attributes ?? {})
12398
+ );
12399
+ for (const key of ["code.filepath", "exception.stacktrace"]) {
12400
+ if (typeof evAttrs[key] === "string") {
12401
+ found.push(evAttrs[key]);
12402
+ }
12403
+ }
12404
+ }
12405
+ }
12406
+ }
12407
+ return found;
12408
+ };
12409
+ async function computeChangeBlastRadius(db, projectId, changeId, options = {}) {
12410
+ const change = await getChangeEvent(
12411
+ db,
12412
+ projectId,
12413
+ changeId,
12414
+ options.environment
12415
+ );
12416
+ if (!change) return null;
12417
+ const minSample = options.minComparableSample ?? 50;
12418
+ const preWindowStart = new Date(
12419
+ Date.parse(change.occurredAt) - 30 * 6e4
12420
+ ).toISOString();
12421
+ const serviceFilter = change.serviceName;
12422
+ const spansResult = serviceFilter ? await db.prepare(
12423
+ `SELECT
12424
+ trace_id, span_id, parent_span_id, span_kind, status_code,
12425
+ duration_ms, service_version, target_node_id, attributes_json,
12426
+ events_json, start_time, end_time, received_at
12427
+ FROM telemetry_spans
12428
+ WHERE project_id = ?
12429
+ AND environment = ?
12430
+ AND service_name = ?
12431
+ AND received_at >= ?
12432
+ ORDER BY received_at DESC
12433
+ LIMIT 5000`
12434
+ ).bind(projectId, change.environment, serviceFilter, preWindowStart).all() : await db.prepare(
12435
+ `SELECT
12436
+ trace_id, span_id, parent_span_id, span_kind, status_code,
12437
+ duration_ms, service_version, target_node_id, attributes_json,
12438
+ events_json, start_time, end_time, received_at
12439
+ FROM telemetry_spans
12440
+ WHERE project_id = ?
12441
+ AND environment = ?
12442
+ AND received_at >= ?
12443
+ ORDER BY received_at DESC
12444
+ LIMIT 5000`
12445
+ ).bind(projectId, change.environment, preWindowStart).all();
12446
+ const allSpans = spansResult.results;
12447
+ const postSpans = allSpans.filter(
12448
+ (s) => (s.start_time || s.received_at) >= change.occurredAt
12449
+ );
12450
+ const preSpans = allSpans.filter(
12451
+ (s) => (s.start_time || s.received_at) < change.occurredAt
12452
+ );
12453
+ const isRootOrServer = (s) => s.parent_span_id === null || s.span_kind === 1;
12454
+ const isDownstreamCall = (s) => Boolean(s.target_node_id) || s.span_kind === 2 || s.span_kind === 4;
12455
+ const flagKey = change.kind === "feature_flag" ? (typeof change.attributes.flagKey === "string" ? change.attributes.flagKey : change.serviceName) ?? "flag" : null;
12456
+ const newCohortKey = change.version ?? (typeof change.attributes.newVariant === "string" ? change.attributes.newVariant : "true");
12457
+ const oldCohortKey = change.previousVersion ?? (typeof change.attributes.previousVariant === "string" ? change.attributes.previousVariant : "false");
12458
+ const getCohortOfSpan = (s) => {
12459
+ if (flagKey) {
12460
+ const attrs = parseJsonRecord(s.attributes_json ?? "{}");
12461
+ const direct = attrs[`obs.feature_flags.${flagKey}`];
12462
+ if (direct !== void 0 && direct !== null) return String(direct);
12463
+ if (attrs["feature_flag.key"] === flagKey && attrs["feature_flag.variant"] !== void 0) {
12464
+ return String(attrs["feature_flag.variant"]);
12465
+ }
12466
+ const nested = attrs["obs.feature_flags"];
12467
+ if (nested && typeof nested === "object" && !Array.isArray(nested)) {
12468
+ const v = nested[flagKey];
12469
+ if (v !== void 0 && v !== null) return String(v);
12470
+ }
12471
+ return null;
12472
+ }
12473
+ return s.service_version;
12474
+ };
12475
+ const postNewReq = postSpans.filter(
12476
+ (s) => isRootOrServer(s) && getCohortOfSpan(s) === newCohortKey
12477
+ );
12478
+ const postNewDown = postSpans.filter(
12479
+ (s) => isDownstreamCall(s) && getCohortOfSpan(s) === newCohortKey
12480
+ );
12481
+ const postOldReq = postSpans.filter(
12482
+ (s) => isRootOrServer(s) && (oldCohortKey ? getCohortOfSpan(s) === oldCohortKey : getCohortOfSpan(s) !== null && getCohortOfSpan(s) !== newCohortKey)
12483
+ );
12484
+ const postOldDown = postSpans.filter(
12485
+ (s) => isDownstreamCall(s) && (oldCohortKey ? getCohortOfSpan(s) === oldCohortKey : getCohortOfSpan(s) !== null && getCohortOfSpan(s) !== newCohortKey)
12486
+ );
12487
+ const preOldReq = preSpans.filter(
12488
+ (s) => isRootOrServer(s) && (!oldCohortKey || getCohortOfSpan(s) === oldCohortKey)
12489
+ );
12490
+ const preOldDown = preSpans.filter(
12491
+ (s) => isDownstreamCall(s) && (!oldCohortKey || getCohortOfSpan(s) === oldCohortKey)
12492
+ );
12493
+ const newCohort = summarizeCohort(newCohortKey, postNewReq, postNewDown);
12494
+ const oldCohort = summarizeCohort(oldCohortKey, postOldReq, postOldDown);
12495
+ const preBaseline = summarizeCohort(
12496
+ `${oldCohortKey}:pre`,
12497
+ preOldReq,
12498
+ preOldDown
12499
+ );
12500
+ const overlappingSet = /* @__PURE__ */ new Set();
12501
+ const matchingErrorSpanIds = [];
12502
+ if (change.touchedPaths.length > 0) {
12503
+ for (const s of postNewReq.filter((sp) => sp.status_code === 2)) {
12504
+ const rawPaths = extractFilePathsFromSpan(s);
12505
+ let matchedSpan = false;
12506
+ for (const touched of change.touchedPaths) {
12507
+ if (rawPaths.some((raw) => raw.includes(touched))) {
12508
+ overlappingSet.add(touched);
12509
+ matchedSpan = true;
12510
+ }
12511
+ }
12512
+ if (matchedSpan) {
12513
+ matchingErrorSpanIds.push(s.span_id);
12514
+ }
12515
+ }
12516
+ }
12517
+ const overlappingCodePaths = Array.from(overlappingSet);
12518
+ let cohortVerdict = "insufficient_sample";
12519
+ let causalProofTier = "unverified_locus";
12520
+ let rateDifference = 0;
12521
+ let relativeRisk = 1;
12522
+ const sharedResourceAggressors = [];
12523
+ if (newCohort.requestCount < minSample) {
12524
+ cohortVerdict = "insufficient_sample";
12525
+ causalProofTier = overlappingCodePaths.length > 0 ? "corroborated_candidate" : "unverified_locus";
12526
+ } else if (oldCohort.requestCount >= minSample) {
12527
+ rateDifference = newCohort.errorRate - oldCohort.errorRate;
12528
+ relativeRisk = newCohort.errorRate / Math.max(oldCohort.errorRate, 1e-3);
12529
+ const fanoutRatio = newCohort.avgDownstreamCallsPerRequest / Math.max(oldCohort.avgDownstreamCallsPerRequest, 0.25);
12530
+ const durationRatio = (newCohort.avgDownstreamDurationMs ?? 0) / Math.max(oldCohort.avgDownstreamDurationMs ?? 1, 1);
12531
+ const hasNewVersionAggressorSignal = fanoutRatio >= 3 || durationRatio >= 3;
12532
+ const hasPreChangeDegradationSignal = preBaseline.requestCount >= 10 && oldCohort.errorRate > 1.5 * Math.max(preBaseline.errorRate, 0.01);
12533
+ const bothDegraded = newCohort.errorRate >= 0.05 && oldCohort.errorRate >= 0.05 && (hasNewVersionAggressorSignal || hasPreChangeDegradationSignal);
12534
+ if (bothDegraded) {
12535
+ if (hasNewVersionAggressorSignal) {
12536
+ cohortVerdict = "shared_degradation_new_version_aggressor";
12537
+ causalProofTier = "corroborated_candidate";
12538
+ const sampleTarget = postNewDown.find(
12539
+ (d) => d.target_node_id
12540
+ )?.target_node_id;
12541
+ if (sampleTarget) {
12542
+ const victimTrace = postOldReq.find((r) => r.status_code === 2)?.trace_id ?? "victim";
12543
+ const candidates = await findCrossTraceContention(db, {
12544
+ projectId,
12545
+ environment: change.environment,
12546
+ targetNodeId: sampleTarget,
12547
+ excludeTraceId: victimTrace,
12548
+ startTime: change.occurredAt,
12549
+ endTime: (/* @__PURE__ */ new Date()).toISOString()
12550
+ });
12551
+ sharedResourceAggressors.push(...candidates);
12552
+ }
12553
+ } else {
12554
+ cohortVerdict = "shared_degradation_external_cause";
12555
+ causalProofTier = "unverified_locus";
12556
+ }
12557
+ } else {
12558
+ const p95Ratio = newCohort.p95DurationMs !== null && oldCohort.p95DurationMs !== null && oldCohort.p95DurationMs > 0 ? newCohort.p95DurationMs / oldCohort.p95DurationMs : 1;
12559
+ if ((rateDifference >= 0.05 || p95Ratio >= 2.5) && (relativeRisk >= 5 || p95Ratio >= 2.5)) {
12560
+ cohortVerdict = "isolated_to_new_version";
12561
+ causalProofTier = "cohort_rate_isolated";
12562
+ } else {
12563
+ cohortVerdict = "no_regression_detected";
12564
+ causalProofTier = "unverified_locus";
12565
+ }
12566
+ }
12567
+ } else if (preBaseline.requestCount >= minSample) {
12568
+ rateDifference = newCohort.errorRate - preBaseline.errorRate;
12569
+ relativeRisk = newCohort.errorRate / Math.max(preBaseline.errorRate, 1e-3);
12570
+ const p95Ratio = newCohort.p95DurationMs !== null && preBaseline.p95DurationMs !== null && preBaseline.p95DurationMs > 0 ? newCohort.p95DurationMs / preBaseline.p95DurationMs : 1;
12571
+ if ((rateDifference >= 0.05 || p95Ratio >= 2.5) && (relativeRisk >= 5 || p95Ratio >= 2.5)) {
12572
+ cohortVerdict = "sequential_post_cutover_regression";
12573
+ causalProofTier = "corroborated_candidate";
12574
+ } else {
12575
+ cohortVerdict = "no_regression_detected";
12576
+ causalProofTier = "unverified_locus";
12577
+ }
12578
+ } else if (overlappingCodePaths.length > 0) {
12579
+ cohortVerdict = "insufficient_sample";
12580
+ causalProofTier = "corroborated_candidate";
12581
+ }
12582
+ const relationshipCertainty = overlappingCodePaths.length > 0 ? "exact" : change.relationshipCertainty;
12583
+ const summary = `Change ${change.id} (${change.kind} ${newCohortKey} in ${change.environment}): verdict=${cohortVerdict}, proofTier=${causalProofTier}, newErrorRate=${(newCohort.errorRate * 100).toFixed(1)}% (N=${newCohort.requestCount}), oldErrorRate=${(oldCohort.errorRate * 100).toFixed(1)}% (N=${oldCohort.requestCount})`;
12584
+ return {
12585
+ change,
12586
+ cohortVerdict,
12587
+ causalProofTier,
12588
+ relationshipCertainty,
12589
+ minComparableSample: minSample,
12590
+ newCohort,
12591
+ oldCohort,
12592
+ preChangeBaselineCohort: preBaseline,
12593
+ rateDifference,
12594
+ relativeRisk,
12595
+ overlappingCodePaths,
12596
+ matchingErrorSpanIds,
12597
+ sharedResourceAggressors,
12598
+ summary
9731
12599
  };
9732
- if (typeof turnIndex === "number") attrs["llm.turn"] = turnIndex;
9733
- return config.tracer(
9734
- "llm.anthropic.messages",
9735
- async (span) => exec(span),
9736
- attrs
9737
- );
9738
12600
  }
9739
12601
 
9740
- // src/plugins/ask-routes.ts
9741
- var MAX_QUESTION_CHARS = 1e3;
9742
- var askRoutesPlugin = {
9743
- name: "ask-routes",
12602
+ // src/plugins/autosre-routes.ts
12603
+ var CHANGE_KINDS = /* @__PURE__ */ new Set([
12604
+ "deploy",
12605
+ "rollback",
12606
+ "feature_flag",
12607
+ "config_change",
12608
+ "prompt_rollout",
12609
+ "agent_version",
12610
+ "schema_migration",
12611
+ "infra_scale"
12612
+ ]);
12613
+ var REMEDIATION_ACTION_KINDS = /* @__PURE__ */ new Set([
12614
+ "open_code_pr",
12615
+ "toggle_feature_flag",
12616
+ "rollback_deploy",
12617
+ "apply_config",
12618
+ "promote_agent_prompt"
12619
+ ]);
12620
+ var parseLimit = (raw, fallback = 50) => {
12621
+ const parsed = Number.parseInt(raw ?? String(fallback), 10);
12622
+ if (!Number.isFinite(parsed)) return fallback;
12623
+ return Math.max(1, Math.min(200, parsed));
12624
+ };
12625
+ var isAgentAutonomyLevel = (value) => typeof value === "string" && Object.values(AgentAutonomyLevel2).includes(value);
12626
+ function requireHumanDashboardSession(c) {
12627
+ const authHeader = c.req.header("Authorization");
12628
+ if (authHeader) {
12629
+ return {
12630
+ ok: false,
12631
+ message: "Human approval requires an authenticated human dashboard session (obs_session cookie); agent/API bearer tokens cannot self-approve or reject remediation proposals."
12632
+ };
12633
+ }
12634
+ const cookieHeader = c.req.header("Cookie") ?? "";
12635
+ const match = cookieHeader.match(/(?:^|;\s*)obs_session=([^;]*)/);
12636
+ if (!match || !match[1]) {
12637
+ return {
12638
+ ok: false,
12639
+ message: "Human approval requires an authenticated human dashboard session (obs_session cookie)."
12640
+ };
12641
+ }
12642
+ const explicitActor = c.req.header("X-Obs-User-Actor")?.trim();
12643
+ const rawSession = decodeURIComponent(match[1]);
12644
+ const sessionActor = explicitActor || `dashboard_session:${rawSession.slice(0, 24)}`;
12645
+ return { ok: true, sessionActor };
12646
+ }
12647
+ var autosreRoutesPlugin = {
12648
+ name: "autosre-routes",
9744
12649
  register(app, runtime) {
9745
- app.post("/internal/ask", async (c) => {
12650
+ app.post("/v1/changes", async (c) => {
12651
+ const projectId = getProjectId(c);
9746
12652
  const body = await readJsonObject(c);
9747
12653
  if (!body) {
12654
+ return c.json({ error: "Invalid JSON body" }, 400);
12655
+ }
12656
+ const rawKind = typeof body.kind === "string" ? body.kind.trim() : "deploy";
12657
+ const normalizedKind = rawKind === "agent_prompt_release" ? "prompt_rollout" : rawKind;
12658
+ if (!CHANGE_KINDS.has(normalizedKind)) {
9748
12659
  return c.json(
9749
- { error: "Bad Request", message: "body must be a JSON object" },
12660
+ {
12661
+ error: `Invalid change kind: ${String(body.kind)}. Must be one of: ${Array.from(CHANGE_KINDS).join(", ")}`
12662
+ },
9750
12663
  400
9751
12664
  );
9752
12665
  }
9753
- const question = (body.question ?? "").trim();
9754
- if (!question) {
12666
+ const summary = typeof body.summary === "string" ? body.summary.trim() : "";
12667
+ if (!summary) {
12668
+ return c.json({ error: "summary is required" }, 400);
12669
+ }
12670
+ const db = runtime.getSqlDb(c.env);
12671
+ const touchedPaths = Array.isArray(body.touchedPaths) ? body.touchedPaths.filter((p) => typeof p === "string") : void 0;
12672
+ const rawAttrs = body.attributes ?? body.metadata;
12673
+ const attributes = rawAttrs && typeof rawAttrs === "object" && !Array.isArray(rawAttrs) ? rawAttrs : void 0;
12674
+ const serviceName = typeof body.serviceName === "string" ? body.serviceName : typeof body.service === "string" ? body.service : void 0;
12675
+ const namespace = typeof body.namespace === "string" ? body.namespace : typeof body.serviceNamespace === "string" ? body.serviceNamespace : void 0;
12676
+ const repositoryUrl = typeof body.repositoryUrl === "string" ? body.repositoryUrl : typeof body.repoUrl === "string" ? body.repoUrl : null;
12677
+ const pullRequestUrl = typeof body.pullRequestUrl === "string" ? body.pullRequestUrl : typeof body.prUrl === "string" ? body.prUrl : null;
12678
+ const actorId = typeof body.actorId === "string" ? body.actorId : typeof body.actor === "string" ? body.actor : null;
12679
+ const changeEvent = await recordChangeEvent(db, projectId, {
12680
+ id: typeof body.id === "string" ? body.id : void 0,
12681
+ environment: typeof body.environment === "string" ? body.environment : void 0,
12682
+ kind: normalizedKind,
12683
+ source: body.source === "inferred_telemetry" || body.source === "autosre_remediation" ? body.source : "explicit_api",
12684
+ targetNodeId: typeof body.targetNodeId === "string" ? body.targetNodeId : void 0,
12685
+ serviceName,
12686
+ namespace,
12687
+ version: typeof body.version === "string" ? body.version : null,
12688
+ previousVersion: typeof body.previousVersion === "string" ? body.previousVersion : null,
12689
+ commitSha: typeof body.commitSha === "string" ? body.commitSha : null,
12690
+ previousCommitSha: typeof body.previousCommitSha === "string" ? body.previousCommitSha : null,
12691
+ configDigest: typeof body.configDigest === "string" ? body.configDigest : null,
12692
+ summary,
12693
+ actorId,
12694
+ repositoryUrl,
12695
+ pullRequestUrl,
12696
+ causedByActionId: typeof body.causedByActionId === "string" ? body.causedByActionId : null,
12697
+ agentRunId: typeof body.agentRunId === "string" ? body.agentRunId : null,
12698
+ touchedPaths,
12699
+ attributes,
12700
+ occurredAt: typeof body.occurredAt === "string" ? body.occurredAt : void 0
12701
+ });
12702
+ return c.json({ changeEvent, change: changeEvent }, 201);
12703
+ });
12704
+ app.get("/internal/ontology", async (c) => {
12705
+ const projectId = getProjectId(c);
12706
+ const db = sqlDbFor(c.env);
12707
+ const environment = c.req.query("environment") || void 0;
12708
+ const traceId = c.req.query("traceId") || void 0;
12709
+ const graph = await getOntologyGraph(db, {
12710
+ projectId,
12711
+ environment
12712
+ });
12713
+ const faultLocus = traceId ? await loadTraceFaultLocusFromDb(db, projectId, traceId) : null;
12714
+ return c.json({
12715
+ ...graph,
12716
+ faultLocus
12717
+ });
12718
+ });
12719
+ app.get("/internal/changes", async (c) => {
12720
+ const projectId = getProjectId(c);
12721
+ const db = sqlDbFor(c.env);
12722
+ const rawKind = c.req.query("kind");
12723
+ const normalizedKind = rawKind === "agent_prompt_release" ? "prompt_rollout" : rawKind;
12724
+ if (normalizedKind && !CHANGE_KINDS.has(normalizedKind)) {
12725
+ return c.json({ error: `Invalid kind: ${rawKind}` }, 400);
12726
+ }
12727
+ const changes2 = await listChangeEvents(db, {
12728
+ projectId,
12729
+ environment: c.req.query("environment") || void 0,
12730
+ serviceName: c.req.query("serviceName") || c.req.query("service") || void 0,
12731
+ targetNodeId: c.req.query("targetNodeId") || void 0,
12732
+ kind: normalizedKind || void 0,
12733
+ since: c.req.query("since") || void 0,
12734
+ limit: parseLimit(c.req.query("limit"), 50)
12735
+ });
12736
+ return c.json({ changes: changes2 });
12737
+ });
12738
+ app.get("/internal/changes/:id/blast-radius", async (c) => {
12739
+ const projectId = getProjectId(c);
12740
+ const changeId = c.req.param("id");
12741
+ if (!changeId) {
12742
+ return c.json({ error: "changeId is required" }, 400);
12743
+ }
12744
+ const db = sqlDbFor(c.env);
12745
+ const minSampleRaw = c.req.query("minSample") ?? c.req.query("minComparableSample");
12746
+ const minSample = minSampleRaw ? Number.parseInt(minSampleRaw, 10) : void 0;
12747
+ const blastRadius = await computeChangeBlastRadius(
12748
+ db,
12749
+ projectId,
12750
+ changeId,
12751
+ {
12752
+ environment: c.req.query("environment") || void 0,
12753
+ minComparableSample: minSample !== void 0 && Number.isFinite(minSample) ? minSample : void 0
12754
+ }
12755
+ );
12756
+ if (!blastRadius) {
12757
+ return c.json({ error: "Change event not found" }, 404);
12758
+ }
12759
+ return c.json({ blastRadius, ...blastRadius });
12760
+ });
12761
+ app.get("/internal/incidents", async (c) => {
12762
+ const projectId = getProjectId(c);
12763
+ const db = sqlDbFor(c.env);
12764
+ const incidents = await listIncidentInvestigations(db, {
12765
+ projectId,
12766
+ environment: c.req.query("environment") || void 0,
12767
+ status: c.req.query("status") || void 0,
12768
+ limit: parseLimit(c.req.query("limit"), 50)
12769
+ });
12770
+ return c.json({ incidents });
12771
+ });
12772
+ app.post("/internal/incidents", async (c) => {
12773
+ const projectId = getProjectId(c);
12774
+ const body = await readJsonObject(c);
12775
+ if (!body) {
12776
+ return c.json({ error: "Invalid JSON body" }, 400);
12777
+ }
12778
+ const targetNodeId = typeof body.targetNodeId === "string" ? body.targetNodeId.trim() : typeof body.symptomNodeId === "string" ? body.symptomNodeId.trim() : "";
12779
+ if (!targetNodeId) {
12780
+ return c.json({ error: "targetNodeId is required" }, 400);
12781
+ }
12782
+ const symptomAnchorKind = typeof body.symptomAnchorKind === "string" ? body.symptomAnchorKind : typeof body.triggerKind === "string" ? body.triggerKind : "alert";
12783
+ const symptomAnchorId = typeof body.symptomAnchorId === "string" ? body.symptomAnchorId : typeof body.triggerEntityId === "string" ? body.triggerEntityId : targetNodeId;
12784
+ const db = sqlDbFor(c.env);
12785
+ const result = await openOrDeduplicateIncident(db, projectId, {
12786
+ id: typeof body.id === "string" ? body.id : void 0,
12787
+ environment: typeof body.environment === "string" ? body.environment : void 0,
12788
+ targetNodeId,
12789
+ symptomCategory: typeof body.symptomCategory === "string" ? body.symptomCategory : void 0,
12790
+ symptomAnchorKind,
12791
+ symptomAnchorId,
12792
+ symptomFingerprint: typeof body.symptomFingerprint === "string" ? body.symptomFingerprint : typeof body.fingerprint === "string" ? body.fingerprint : null,
12793
+ latestBundle: body.latestBundle && typeof body.latestBundle === "object" && !Array.isArray(body.latestBundle) ? body.latestBundle : null
12794
+ });
12795
+ return c.json(result, result.deduplicated ? 200 : 201);
12796
+ });
12797
+ app.post("/internal/incidents/:id/lease", async (c) => {
12798
+ const projectId = getProjectId(c);
12799
+ const incidentId = c.req.param("id");
12800
+ const body = await readJsonObject(c);
12801
+ if (!body) {
12802
+ return c.json({ error: "Invalid JSON body" }, 400);
12803
+ }
12804
+ const runnerId = typeof body.runnerId === "string" ? body.runnerId.trim() : typeof body.leaseOwner === "string" ? body.leaseOwner.trim() : "";
12805
+ if (!runnerId) {
12806
+ return c.json({ error: "runnerId is required" }, 400);
12807
+ }
12808
+ const db = sqlDbFor(c.env);
12809
+ const leaseDurationSeconds = typeof body.leaseDurationSeconds === "number" && Number.isFinite(body.leaseDurationSeconds) ? body.leaseDurationSeconds : typeof body.leaseDurationMs === "number" && Number.isFinite(body.leaseDurationMs) ? Math.ceil(body.leaseDurationMs / 1e3) : void 0;
12810
+ const result = await acquireIncidentLease(db, projectId, incidentId, {
12811
+ environment: typeof body.environment === "string" ? body.environment : void 0,
12812
+ runnerId,
12813
+ leaseDurationSeconds
12814
+ });
12815
+ return c.json(result, result.acquired ? 200 : 409);
12816
+ });
12817
+ app.post("/internal/incidents/:id/dispatch", async (c) => {
12818
+ const projectId = getProjectId(c);
12819
+ const incidentId = c.req.param("id");
12820
+ const body = await readJsonObject(c);
12821
+ const webhookUrl = typeof body?.webhookUrl === "string" ? body.webhookUrl : typeof body?.url === "string" ? body.url : "";
12822
+ if (!webhookUrl) {
12823
+ return c.json({ error: "webhookUrl is required" }, 400);
12824
+ }
12825
+ const db = sqlDbFor(c.env);
12826
+ const result = await dispatchIncidentWebhook(db, projectId, incidentId, {
12827
+ url: webhookUrl,
12828
+ secret: typeof body?.secret === "string" ? body.secret : void 0,
12829
+ allowPrivateNetworks: Boolean(
12830
+ body?.allowPrivateNetworks ?? body?.allowPrivateWebhookTargets
12831
+ )
12832
+ });
12833
+ return c.json(result, result.delivered ? 200 : 502);
12834
+ });
12835
+ app.get("/internal/findings", async (c) => {
12836
+ const projectId = getProjectId(c);
12837
+ const db = sqlDbFor(c.env);
12838
+ const findings = await searchInvestigationFindings(db, {
12839
+ projectId,
12840
+ environment: c.req.query("environment") || void 0,
12841
+ fingerprint: c.req.query("fingerprint") || void 0,
12842
+ symptomEntityId: c.req.query("symptomEntityId") || void 0,
12843
+ rootCauseEntityId: c.req.query("rootCauseEntityId") || c.req.query("faultLocusEntityId") || void 0,
12844
+ causalVerificationStatus: c.req.query("causalVerificationStatus") || void 0,
12845
+ incidentId: c.req.query("incidentId") || void 0,
12846
+ limit: parseLimit(c.req.query("limit"), 20)
12847
+ });
12848
+ return c.json({ findings });
12849
+ });
12850
+ app.post("/internal/findings", async (c) => {
12851
+ const projectId = getProjectId(c);
12852
+ const body = await readJsonObject(c);
12853
+ if (!body) {
12854
+ return c.json({ error: "Invalid JSON body" }, 400);
12855
+ }
12856
+ const title = typeof body.title === "string" ? body.title.trim() : "";
12857
+ const summary = typeof body.summary === "string" ? body.summary.trim() : "";
12858
+ const symptomEntityKind = typeof body.symptomEntityKind === "string" ? body.symptomEntityKind.trim() : "";
12859
+ const symptomEntityId = typeof body.symptomEntityId === "string" ? body.symptomEntityId.trim() : "";
12860
+ const faultLocusEntityKind = typeof body.faultLocusEntityKind === "string" ? body.faultLocusEntityKind.trim() : "";
12861
+ const faultLocusEntityId = typeof body.faultLocusEntityId === "string" ? body.faultLocusEntityId.trim() : "";
12862
+ if (!title || !summary || !symptomEntityKind || !symptomEntityId || !faultLocusEntityKind || !faultLocusEntityId) {
9755
12863
  return c.json(
9756
- { error: "Bad Request", message: "question is required" },
12864
+ {
12865
+ error: "title, summary, symptomEntityKind, symptomEntityId, faultLocusEntityKind, and faultLocusEntityId are required"
12866
+ },
9757
12867
  400
9758
12868
  );
9759
12869
  }
9760
- if (question.length > MAX_QUESTION_CHARS) {
12870
+ const db = sqlDbFor(c.env);
12871
+ const input = {
12872
+ id: typeof body.id === "string" ? body.id : void 0,
12873
+ environment: typeof body.environment === "string" ? body.environment : void 0,
12874
+ incidentId: typeof body.incidentId === "string" ? body.incidentId : null,
12875
+ fingerprintInput: body.fingerprintInput && typeof body.fingerprintInput === "object" ? body.fingerprintInput : void 0,
12876
+ title,
12877
+ summary,
12878
+ causalProofTier: typeof body.causalProofTier === "string" ? body.causalProofTier : void 0,
12879
+ confidence: typeof body.confidence === "number" ? body.confidence : void 0,
12880
+ symptomEntityKind,
12881
+ symptomEntityId,
12882
+ faultLocusEntityKind,
12883
+ faultLocusEntityId,
12884
+ rootCauseEntityKind: typeof body.rootCauseEntityKind === "string" ? body.rootCauseEntityKind : null,
12885
+ rootCauseEntityId: typeof body.rootCauseEntityId === "string" ? body.rootCauseEntityId : null,
12886
+ contributingChangeId: typeof body.contributingChangeId === "string" ? body.contributingChangeId : null,
12887
+ cohortVerdict: typeof body.cohortVerdict === "string" ? body.cohortVerdict : null,
12888
+ contendingTraceId: typeof body.contendingTraceId === "string" ? body.contendingTraceId : null,
12889
+ causalChain: Array.isArray(body.causalChain) ? body.causalChain : [],
12890
+ citedEvidenceIds: Array.isArray(body.citedEvidenceIds) ? body.citedEvidenceIds.filter(
12891
+ (id) => typeof id === "string"
12892
+ ) : [],
12893
+ agentRunId: typeof body.agentRunId === "string" ? body.agentRunId : null
12894
+ };
12895
+ const finding = await recordInvestigationFinding(db, projectId, input);
12896
+ return c.json({ finding, id: finding.id }, 201);
12897
+ });
12898
+ app.post("/internal/findings/:id/counterfactual", async (c) => {
12899
+ const projectId = getProjectId(c);
12900
+ const findingId = c.req.param("id");
12901
+ const body = await readJsonObject(c);
12902
+ if (!body) {
12903
+ return c.json({ error: "Invalid JSON body" }, 400);
12904
+ }
12905
+ const evalCaseId = typeof body.evalCaseId === "string" ? body.evalCaseId.trim() : "";
12906
+ const failingRunId = typeof body.failingRunId === "string" ? body.failingRunId.trim() : "";
12907
+ const passingRunIds = Array.isArray(body.passingRunIds) ? body.passingRunIds.filter(
12908
+ (id) => typeof id === "string" && id.trim() !== ""
12909
+ ) : [];
12910
+ if (!evalCaseId || !failingRunId) {
9761
12911
  return c.json(
12912
+ { error: "evalCaseId and failingRunId are required" },
12913
+ 400
12914
+ );
12915
+ }
12916
+ const db = sqlDbFor(c.env);
12917
+ try {
12918
+ const finding = await verifyPairedCounterfactualProof(
12919
+ db,
12920
+ projectId,
12921
+ findingId,
9762
12922
  {
9763
- error: "Bad Request",
9764
- message: `question exceeds ${MAX_QUESTION_CHARS} chars`
12923
+ environment: typeof body.environment === "string" ? body.environment : void 0,
12924
+ evalCaseId,
12925
+ failingRunId,
12926
+ passingRunIds,
12927
+ requiredTrials: typeof body.requiredTrials === "number" && Number.isFinite(body.requiredTrials) ? body.requiredTrials : void 0
12928
+ }
12929
+ );
12930
+ return c.json({ verified: true, finding });
12931
+ } catch (err) {
12932
+ const message = err instanceof Error ? err.message : String(err);
12933
+ if (err instanceof CounterfactualVerificationError) {
12934
+ const isNotFound2 = message.includes("not found in project");
12935
+ return c.json(
12936
+ { verified: false, error: message, reason: message },
12937
+ isNotFound2 ? 404 : 422
12938
+ );
12939
+ }
12940
+ return c.json({ verified: false, error: message }, 400);
12941
+ }
12942
+ });
12943
+ app.get("/internal/remediations", async (c) => {
12944
+ const projectId = getProjectId(c);
12945
+ const db = sqlDbFor(c.env);
12946
+ const proposals = await listRemediationProposals(db, {
12947
+ projectId,
12948
+ environment: c.req.query("environment") || void 0,
12949
+ incidentId: c.req.query("incidentId") || void 0,
12950
+ findingId: c.req.query("findingId") || void 0,
12951
+ status: c.req.query("status") || void 0,
12952
+ limit: parseLimit(c.req.query("limit"), 50)
12953
+ });
12954
+ return c.json({ proposals });
12955
+ });
12956
+ app.post("/internal/remediations/propose", async (c) => {
12957
+ const projectId = getProjectId(c);
12958
+ const body = await readJsonObject(c);
12959
+ if (!body) {
12960
+ return c.json({ error: "Invalid JSON body" }, 400);
12961
+ }
12962
+ const incidentId = typeof body.incidentId === "string" ? body.incidentId.trim() : "";
12963
+ const findingId = typeof body.findingId === "string" ? body.findingId.trim() : "";
12964
+ const idempotencyKey = typeof body.idempotencyKey === "string" ? body.idempotencyKey.trim() : "";
12965
+ const actionKind = typeof body.actionKind === "string" ? body.actionKind.trim() : "";
12966
+ const targetNodeId = typeof body.targetNodeId === "string" ? body.targetNodeId.trim() : "";
12967
+ if (!incidentId || !findingId || !idempotencyKey || !REMEDIATION_ACTION_KINDS.has(actionKind) || !targetNodeId) {
12968
+ return c.json(
12969
+ {
12970
+ error: "incidentId, findingId, idempotencyKey, valid actionKind, and targetNodeId are required"
9765
12971
  },
9766
12972
  400
9767
12973
  );
9768
12974
  }
12975
+ const expectedCurrentState = body.expectedCurrentState && typeof body.expectedCurrentState === "object" && !Array.isArray(body.expectedCurrentState) ? body.expectedCurrentState : {};
12976
+ const desiredTargetState = body.desiredTargetState && typeof body.desiredTargetState === "object" && !Array.isArray(body.desiredTargetState) ? body.desiredTargetState : {};
12977
+ const requestedAutonomy = isAgentAutonomyLevel(
12978
+ body.requestedAutonomy
12979
+ ) ? body.requestedAutonomy : "autonomous_write";
12980
+ const db = sqlDbFor(c.env);
12981
+ const input = {
12982
+ environment: typeof body.environment === "string" ? body.environment : void 0,
12983
+ incidentId,
12984
+ findingId,
12985
+ idempotencyKey,
12986
+ actionKind,
12987
+ targetNodeId,
12988
+ expectedCurrentState,
12989
+ desiredTargetState,
12990
+ requestedAutonomy
12991
+ };
12992
+ try {
12993
+ const proposal = await proposeRemediationAction(db, projectId, input);
12994
+ return c.json(
12995
+ {
12996
+ proposal,
12997
+ ...proposal,
12998
+ grantToken: proposal.grantToken ?? null
12999
+ },
13000
+ 201
13001
+ );
13002
+ } catch (err) {
13003
+ const message = err instanceof Error ? err.message : String(err);
13004
+ const status = err instanceof RemediationGateError ? err.statusCode : 400;
13005
+ return c.json({ error: message }, status);
13006
+ }
13007
+ });
13008
+ app.post("/internal/remediations/:id/approve", async (c) => {
13009
+ const sessionCheck = requireHumanDashboardSession(c);
13010
+ if (!sessionCheck.ok) {
13011
+ return c.json(
13012
+ { error: "Forbidden", message: sessionCheck.message },
13013
+ 403
13014
+ );
13015
+ }
9769
13016
  const projectId = getProjectId(c);
9770
- const openaiBase = c.env.OPENAI_BASE_URL?.trim() ? c.env.OPENAI_BASE_URL : void 0;
9771
- const narrativeModel = c.env.NARRATIVE_MODEL?.trim() ? c.env.NARRATIVE_MODEL : void 0;
9772
- const llm = c.env.OPENAI_API_KEY?.trim() ? {
9773
- provider: "openai",
9774
- apiKey: c.env.OPENAI_API_KEY,
9775
- model: narrativeModel ?? "gpt-4o-mini",
9776
- apiUrl: openaiBase,
9777
- tracer: runtime.withChildSpan
9778
- } : c.env.ANTHROPIC_API_KEY?.trim() ? {
9779
- provider: "anthropic",
9780
- apiKey: c.env.ANTHROPIC_API_KEY,
9781
- model: narrativeModel ?? "claude-haiku-4-5",
9782
- tracer: runtime.withChildSpan
9783
- } : null;
9784
- if (!llm) {
9785
- const response = {
9786
- answer: null,
9787
- evidence: [],
9788
- queries: [],
9789
- error: "Ask is not configured \u2014 set OPENAI_API_KEY or ANTHROPIC_API_KEY on the collector to enable it.",
9790
- timestamp: (/* @__PURE__ */ new Date()).toISOString()
9791
- };
9792
- return c.json(response, 503);
13017
+ const proposalId = c.req.param("id");
13018
+ const body = await readJsonObject(c).catch(() => null) ?? {};
13019
+ const environment = typeof body.environment === "string" ? body.environment : void 0;
13020
+ const approvedByUserId = typeof body.approvedByUserId === "string" && body.approvedByUserId.trim() ? body.approvedByUserId.trim() : typeof body.approvedBySession === "string" && body.approvedBySession.trim() ? body.approvedBySession.trim() : sessionCheck.sessionActor;
13021
+ const db = sqlDbFor(c.env);
13022
+ try {
13023
+ const proposal = await approveRemediationProposal(
13024
+ db,
13025
+ projectId,
13026
+ proposalId,
13027
+ {
13028
+ environment,
13029
+ approvedByUserId
13030
+ }
13031
+ );
13032
+ return c.json({
13033
+ proposal,
13034
+ ...proposal,
13035
+ grantToken: proposal.grantToken ?? null
13036
+ });
13037
+ } catch (err) {
13038
+ const message = err instanceof Error ? err.message : String(err);
13039
+ const status = err instanceof RemediationGateError ? err.statusCode : 400;
13040
+ return c.json({ error: message }, status);
9793
13041
  }
9794
- const store = new AnalysesStore(sqlDbFor(c.env));
13042
+ });
13043
+ app.post("/internal/remediations/:id/reject", async (c) => {
13044
+ const sessionCheck = requireHumanDashboardSession(c);
13045
+ if (!sessionCheck.ok) {
13046
+ return c.json(
13047
+ { error: "Forbidden", message: sessionCheck.message },
13048
+ 403
13049
+ );
13050
+ }
13051
+ const projectId = getProjectId(c);
13052
+ const proposalId = c.req.param("id");
13053
+ const body = await readJsonObject(c).catch(() => null) ?? {};
13054
+ const environment = typeof body.environment === "string" ? body.environment : void 0;
13055
+ const reason = typeof body.reason === "string" ? body.reason : void 0;
13056
+ const db = sqlDbFor(c.env);
13057
+ try {
13058
+ const proposal = await rejectRemediationProposal(
13059
+ db,
13060
+ projectId,
13061
+ proposalId,
13062
+ {
13063
+ environment,
13064
+ rejectedByUserId: sessionCheck.sessionActor,
13065
+ reason
13066
+ }
13067
+ );
13068
+ return c.json({ proposal, ...proposal });
13069
+ } catch (err) {
13070
+ const message = err instanceof Error ? err.message : String(err);
13071
+ const status = err instanceof RemediationGateError ? err.statusCode : 400;
13072
+ return c.json({ error: message }, status);
13073
+ }
13074
+ });
13075
+ app.post("/internal/remediations/:id/claim", async (c) => {
13076
+ const projectId = getProjectId(c);
13077
+ const proposalId = c.req.param("id");
13078
+ const body = await readJsonObject(c);
13079
+ if (!body) {
13080
+ return c.json({ error: "Invalid JSON body" }, 400);
13081
+ }
13082
+ const grantToken = typeof body.grantToken === "string" ? body.grantToken.trim() : "";
13083
+ if (!grantToken) {
13084
+ return c.json({ error: "grantToken is required" }, 400);
13085
+ }
13086
+ const db = sqlDbFor(c.env);
13087
+ try {
13088
+ const proposal = await claimRemediationGrant(
13089
+ db,
13090
+ projectId,
13091
+ proposalId,
13092
+ {
13093
+ environment: typeof body.environment === "string" ? body.environment : void 0,
13094
+ grantToken,
13095
+ runnerId: typeof body.runnerId === "string" ? body.runnerId : void 0
13096
+ }
13097
+ );
13098
+ return c.json({ claimed: true, proposal, ...proposal });
13099
+ } catch (err) {
13100
+ const message = err instanceof Error ? err.message : String(err);
13101
+ const status = err instanceof RemediationGateError ? err.statusCode : 409;
13102
+ return c.json({ claimed: false, error: message, reason: message }, status);
13103
+ }
13104
+ });
13105
+ app.post("/internal/remediations/:id/complete", async (c) => {
13106
+ const projectId = getProjectId(c);
13107
+ const proposalId = c.req.param("id");
13108
+ const body = await readJsonObject(c).catch(() => null) ?? {};
13109
+ const db = sqlDbFor(c.env);
13110
+ try {
13111
+ const proposal = await completeRemediationProposal(
13112
+ db,
13113
+ projectId,
13114
+ proposalId,
13115
+ {
13116
+ environment: typeof body.environment === "string" ? body.environment : void 0,
13117
+ resultingChangeId: typeof body.resultingChangeId === "string" ? body.resultingChangeId : null,
13118
+ remediationAgentRunId: typeof body.remediationAgentRunId === "string" ? body.remediationAgentRunId : null
13119
+ }
13120
+ );
13121
+ return c.json({ proposal, ...proposal });
13122
+ } catch (err) {
13123
+ const message = err instanceof Error ? err.message : String(err);
13124
+ const status = err instanceof RemediationGateError ? err.statusCode : 400;
13125
+ return c.json({ error: message }, status);
13126
+ }
13127
+ });
13128
+ app.post("/internal/remediations/:id/fail", async (c) => {
13129
+ const projectId = getProjectId(c);
13130
+ const proposalId = c.req.param("id");
13131
+ const body = await readJsonObject(c).catch(() => null) ?? {};
13132
+ const db = sqlDbFor(c.env);
9795
13133
  try {
9796
- const startedAt = Date.now();
9797
- runtime.logger.info("[ask] llm call starting", {
9798
- project_id: projectId,
9799
- provider: llm.provider,
9800
- model: llm.model,
9801
- question_chars: question.length
9802
- });
9803
- const result = await runtime.withChildSpan(
9804
- "ask.runAsk",
9805
- () => runAsk(question, {
9806
- llm,
9807
- listAnalyses: async (filters) => {
9808
- const all = await getAllAnalysesForProject(projectId, {
9809
- db: sqlDbFor(c.env)
9810
- });
9811
- return all.filter((d) => {
9812
- if (filters?.group && d.group !== filters.group) return false;
9813
- if (filters?.view && d.view !== filters.view) return false;
9814
- return true;
9815
- });
9816
- },
9817
- getLatestResult: async (id) => {
9818
- const definitions = await store.listDefinitions(projectId);
9819
- const definition = definitions.find((d) => d.id === id);
9820
- if (!definition) return null;
9821
- const r = await store.getLatestResult(projectId, id);
9822
- return { definition, result: r };
9823
- }
9824
- }),
13134
+ const proposal = await failRemediationProposal(
13135
+ db,
13136
+ projectId,
13137
+ proposalId,
9825
13138
  {
9826
- "llm.provider": llm.provider,
9827
- "llm.model": llm.model,
9828
- "openinference.span.kind": "LLM"
13139
+ environment: typeof body.environment === "string" ? body.environment : void 0,
13140
+ failureReason: typeof body.failureReason === "string" ? body.failureReason : "Execution failed"
9829
13141
  }
9830
13142
  );
9831
- runtime.logger.info("[ask] llm call finished", {
9832
- project_id: projectId,
9833
- provider: llm.provider,
9834
- model: llm.model,
9835
- latency_ms: Date.now() - startedAt,
9836
- evidence_count: result.evidence.length,
9837
- query_count: result.queries.length,
9838
- answered: Boolean(result.answer)
9839
- });
9840
- if (result.answer && result.evidence.length > 0) {
9841
- await store.recordAskEvidence(
9842
- projectId,
9843
- result.evidence.map((e) => e.analysisId)
9844
- ).catch(
9845
- (err) => runtime.logger.warn("[ask] recordAskEvidence failed", {
9846
- error: err instanceof Error ? err.message : String(err)
9847
- })
9848
- );
9849
- }
9850
- return c.json(result);
9851
- } catch (error) {
9852
- const message = error instanceof Error ? error.message : String(error);
9853
- runtime.logger.error("[ask] failed", { error: message });
9854
- const response = {
9855
- answer: null,
9856
- evidence: [],
9857
- queries: [],
9858
- error: message,
9859
- timestamp: (/* @__PURE__ */ new Date()).toISOString()
9860
- };
9861
- return c.json(response, 500);
13143
+ return c.json({ proposal, ...proposal });
13144
+ } catch (err) {
13145
+ const message = err instanceof Error ? err.message : String(err);
13146
+ const status = err instanceof RemediationGateError ? err.statusCode : 400;
13147
+ return c.json({ error: message }, status);
13148
+ }
13149
+ });
13150
+ app.post("/internal/remediations/watchdog", async (c) => {
13151
+ const db = sqlDbFor(c.env);
13152
+ const body = await readJsonObject(c).catch(() => null) ?? {};
13153
+ const summary = await evaluateRemediationWatchdogs(db, {
13154
+ nowIso: typeof body.nowIso === "string" ? body.nowIso : void 0,
13155
+ minComparableRequests: typeof body.minComparableRequests === "number" ? body.minComparableRequests : void 0
13156
+ });
13157
+ return c.json({ summary, ...summary });
13158
+ });
13159
+ app.get("/internal/remediations/:id", async (c) => {
13160
+ const projectId = getProjectId(c);
13161
+ const proposalId = c.req.param("id");
13162
+ const db = sqlDbFor(c.env);
13163
+ const proposal = await getRemediationProposal(
13164
+ db,
13165
+ projectId,
13166
+ proposalId,
13167
+ c.req.query("environment") || void 0
13168
+ );
13169
+ if (!proposal) {
13170
+ return c.json({ error: "Proposal not found" }, 404);
13171
+ }
13172
+ return c.json({ proposal });
13173
+ });
13174
+ app.get("/internal/incidents/:id", async (c) => {
13175
+ const projectId = getProjectId(c);
13176
+ const incidentId = c.req.param("id");
13177
+ const environment = c.req.query("environment") || void 0;
13178
+ const db = sqlDbFor(c.env);
13179
+ const incident = await getIncidentInvestigation(
13180
+ db,
13181
+ projectId,
13182
+ incidentId,
13183
+ environment
13184
+ );
13185
+ if (!incident) {
13186
+ return c.json({ error: "Incident not found" }, 404);
13187
+ }
13188
+ const [findings, proposals] = await Promise.all([
13189
+ searchInvestigationFindings(db, {
13190
+ projectId,
13191
+ environment: incident.environment,
13192
+ incidentId: incident.id,
13193
+ limit: 20
13194
+ }),
13195
+ listRemediationProposals(db, {
13196
+ projectId,
13197
+ environment: incident.environment,
13198
+ incidentId: incident.id,
13199
+ limit: 20
13200
+ })
13201
+ ]);
13202
+ return c.json({ incident, findings, proposals });
13203
+ });
13204
+ app.get("/internal/findings/:id", async (c) => {
13205
+ const projectId = getProjectId(c);
13206
+ const findingId = c.req.param("id");
13207
+ const db = sqlDbFor(c.env);
13208
+ const finding = await getInvestigationFinding(
13209
+ db,
13210
+ projectId,
13211
+ findingId,
13212
+ c.req.query("environment") || void 0
13213
+ );
13214
+ if (!finding) {
13215
+ return c.json({ error: "Finding not found" }, 404);
13216
+ }
13217
+ return c.json({ finding });
13218
+ });
13219
+ app.get("/internal/changes/:id", async (c) => {
13220
+ const projectId = getProjectId(c);
13221
+ const changeId = c.req.param("id");
13222
+ const db = sqlDbFor(c.env);
13223
+ const changeEvent = await getChangeEvent(
13224
+ db,
13225
+ projectId,
13226
+ changeId,
13227
+ c.req.query("environment") || void 0
13228
+ );
13229
+ if (!changeEvent) {
13230
+ return c.json({ error: "Change event not found" }, 404);
9862
13231
  }
13232
+ return c.json({ changeEvent, change: changeEvent });
9863
13233
  });
9864
13234
  }
9865
13235
  };
@@ -9940,7 +13310,10 @@ var KNOWN_KINDS = /* @__PURE__ */ new Set([
9940
13310
  "user",
9941
13311
  "action",
9942
13312
  "agent_run",
9943
- "tool_call"
13313
+ "tool_call",
13314
+ "change",
13315
+ "finding",
13316
+ "incident"
9944
13317
  ]);
9945
13318
  var MAX_LINKS_INLINE = 5;
9946
13319
  var truncate = (s, n = 80) => s.length > n ? `${s.slice(0, n - 1)}\u2026` : s;
@@ -11038,6 +14411,186 @@ var connectedRoutesPlugin = {
11038
14411
  manifest.rawManifest = actionManifest;
11039
14412
  }
11040
14413
  }
14414
+ } else if (kind === "change") {
14415
+ const db = sqlDbFor(c.env);
14416
+ const changeRow = await db.prepare(
14417
+ `SELECT * FROM change_events WHERE project_id = ? AND id = ? LIMIT 1`
14418
+ ).bind(projectId, id).first();
14419
+ if (changeRow) {
14420
+ manifest.up = [
14421
+ {
14422
+ label: "Target Ontology Node",
14423
+ links: [
14424
+ {
14425
+ label: `${changeRow.target_node_id} (${changeRow.environment})`,
14426
+ href: `#/health?node=${encodeURIComponent(changeRow.target_node_id)}`,
14427
+ relationshipCertainty: "exact",
14428
+ reason: "Target node mutated by this change event."
14429
+ }
14430
+ ]
14431
+ }
14432
+ ];
14433
+ const findings = await db.prepare(
14434
+ `SELECT id, title, causal_proof_tier, confidence, incident_id
14435
+ FROM investigation_findings
14436
+ WHERE project_id = ? AND environment = ? AND contributing_change_id = ?
14437
+ ORDER BY created_at DESC LIMIT 10`
14438
+ ).bind(projectId, changeRow.environment, changeRow.id).all();
14439
+ manifest.down = [
14440
+ {
14441
+ label: "Attributed Findings",
14442
+ links: findings.results.map((f) => ({
14443
+ label: f.title,
14444
+ href: `#/findings/${encodeURIComponent(f.id)}`,
14445
+ entityKind: "finding",
14446
+ entityId: f.id,
14447
+ relationshipCertainty: "exact",
14448
+ causalProofTier: f.causal_proof_tier,
14449
+ confidence: f.confidence,
14450
+ reason: "Finding attributes this change event."
14451
+ })),
14452
+ emptyReason: findings.results.length === 0 ? "No investigation findings cite this change event." : void 0
14453
+ }
14454
+ ];
14455
+ const spans = changeRow.version ? await db.prepare(
14456
+ `SELECT DISTINCT trace_id, span_id, service_name, span_name
14457
+ FROM telemetry_spans
14458
+ WHERE project_id = ? AND environment = ? AND service_version = ? AND status_code = 2
14459
+ ORDER BY start_time DESC LIMIT 5`
14460
+ ).bind(projectId, changeRow.environment, changeRow.version).all() : { results: [] };
14461
+ manifest.across = [
14462
+ {
14463
+ label: "Failing Spans on Cohort Version",
14464
+ links: spans.results.map((s) => ({
14465
+ label: `${s.service_name} \xB7 ${s.span_name}`,
14466
+ href: `#/traces/${s.trace_id}#span=${s.span_id}`,
14467
+ entityKind: "span",
14468
+ entityId: `${s.trace_id}:${s.span_id}`,
14469
+ relationshipCertainty: "exact",
14470
+ reason: `Span executed on version ${changeRow.version}.`
14471
+ })),
14472
+ emptyReason: spans.results.length === 0 ? "No failing spans recorded on this change version." : void 0
14473
+ }
14474
+ ];
14475
+ }
14476
+ } else if (kind === "finding") {
14477
+ const db = sqlDbFor(c.env);
14478
+ const findingRow = await db.prepare(
14479
+ `SELECT * FROM investigation_findings WHERE project_id = ? AND id = ? LIMIT 1`
14480
+ ).bind(projectId, id).first();
14481
+ if (findingRow) {
14482
+ const proofTier = findingRow.causal_proof_tier;
14483
+ manifest.up = [];
14484
+ if (findingRow.incident_id) {
14485
+ manifest.up.push({
14486
+ label: "Incident Investigation",
14487
+ links: [
14488
+ {
14489
+ label: `incident ${findingRow.incident_id}`,
14490
+ href: `#/incidents/${encodeURIComponent(findingRow.incident_id)}`,
14491
+ entityKind: "incident",
14492
+ entityId: findingRow.incident_id,
14493
+ relationshipCertainty: "exact",
14494
+ causalProofTier: proofTier
14495
+ }
14496
+ ]
14497
+ });
14498
+ }
14499
+ if (findingRow.contributing_change_id) {
14500
+ manifest.up.push({
14501
+ label: "Contributing Change",
14502
+ links: [
14503
+ {
14504
+ label: `change ${findingRow.contributing_change_id}`,
14505
+ href: `#/changes/${encodeURIComponent(findingRow.contributing_change_id)}`,
14506
+ entityKind: "change",
14507
+ entityId: findingRow.contributing_change_id,
14508
+ relationshipCertainty: "exact",
14509
+ causalProofTier: proofTier,
14510
+ confidence: findingRow.confidence
14511
+ }
14512
+ ]
14513
+ });
14514
+ }
14515
+ manifest.across = [
14516
+ {
14517
+ label: "Fault Locus & Symptom",
14518
+ links: [
14519
+ {
14520
+ label: `fault locus: ${findingRow.fault_locus_entity_id}`,
14521
+ href: `#/traces?q=${encodeURIComponent(findingRow.fault_locus_entity_id)}`,
14522
+ relationshipCertainty: "exact",
14523
+ causalProofTier: proofTier
14524
+ },
14525
+ ...findingRow.contending_trace_id ? [
14526
+ {
14527
+ label: `contending trace ${findingRow.contending_trace_id}`,
14528
+ href: `#/traces/${encodeURIComponent(findingRow.contending_trace_id)}`,
14529
+ entityKind: "trace",
14530
+ entityId: findingRow.contending_trace_id,
14531
+ relationshipCertainty: "inferred",
14532
+ causalProofTier: proofTier
14533
+ }
14534
+ ] : []
14535
+ ]
14536
+ }
14537
+ ];
14538
+ manifest.down = [
14539
+ {
14540
+ label: "Agent Lineage",
14541
+ links: [
14542
+ ...findingRow.agent_run_id ? [
14543
+ {
14544
+ ...linkToAgentRun(findingRow.agent_run_id),
14545
+ relationshipCertainty: "exact",
14546
+ causalProofTier: proofTier
14547
+ }
14548
+ ] : []
14549
+ ],
14550
+ emptyReason: !findingRow.agent_run_id ? "No agent run linked to this finding." : void 0
14551
+ }
14552
+ ];
14553
+ }
14554
+ } else if (kind === "incident") {
14555
+ const db = sqlDbFor(c.env);
14556
+ const incRow = await db.prepare(
14557
+ `SELECT * FROM incident_investigations WHERE project_id = ? AND id = ? LIMIT 1`
14558
+ ).bind(projectId, id).first();
14559
+ if (incRow) {
14560
+ manifest.up = [
14561
+ {
14562
+ label: "Trigger Signal",
14563
+ links: [
14564
+ {
14565
+ label: `${incRow.symptom_anchor_kind}: ${incRow.symptom_anchor_id}`,
14566
+ href: `#/alerts?id=${encodeURIComponent(incRow.symptom_anchor_id)}`,
14567
+ entityKind: incRow.symptom_anchor_kind === "analysis" ? "analysis" : "alert",
14568
+ entityId: incRow.symptom_anchor_id,
14569
+ relationshipCertainty: "exact"
14570
+ }
14571
+ ]
14572
+ }
14573
+ ];
14574
+ const incFindings = await db.prepare(
14575
+ `SELECT id, title, causal_proof_tier FROM investigation_findings
14576
+ WHERE project_id = ? AND incident_id = ?
14577
+ ORDER BY created_at DESC LIMIT 5`
14578
+ ).bind(projectId, incRow.id).all();
14579
+ manifest.across = [
14580
+ {
14581
+ label: "Findings",
14582
+ links: incFindings.results.map((f) => ({
14583
+ label: f.title,
14584
+ href: `#/findings/${encodeURIComponent(f.id)}`,
14585
+ entityKind: "finding",
14586
+ entityId: f.id,
14587
+ relationshipCertainty: "exact",
14588
+ causalProofTier: f.causal_proof_tier
14589
+ })),
14590
+ emptyReason: incFindings.results.length > 0 ? void 0 : "No finding recorded for this incident yet."
14591
+ }
14592
+ ];
14593
+ }
11041
14594
  } else if (kind === "alert" || kind === "analysis") {
11042
14595
  manifest.related = [
11043
14596
  {
@@ -11143,7 +14696,7 @@ var defaultSpanEnrichmentPlugin = {
11143
14696
  const sessionId = typeof attributes[SESSION_ID_KEY2] === "string" && attributes[SESSION_ID_KEY2].length > 0 ? attributes[SESSION_ID_KEY2] : null;
11144
14697
  const interactionId = typeof attributes[INTERACTION_ID_KEY2] === "string" && attributes[INTERACTION_ID_KEY2].length > 0 ? attributes[INTERACTION_ID_KEY2] : null;
11145
14698
  const telemetrySdkName = typeof resourceAttributes["telemetry.sdk.name"] === "string" && resourceAttributes["telemetry.sdk.name"].length > 0 ? resourceAttributes["telemetry.sdk.name"] : null;
11146
- return {
14699
+ const candidate = {
11147
14700
  ...span,
11148
14701
  serviceName,
11149
14702
  attributesJson: JSON.stringify(normalizedAttributes),
@@ -11151,6 +14704,15 @@ var defaultSpanEnrichmentPlugin = {
11151
14704
  interactionId,
11152
14705
  telemetrySdkName
11153
14706
  };
14707
+ const lineage = extractSpanLineageAndTopology(candidate);
14708
+ return {
14709
+ ...candidate,
14710
+ environment: lineage.environment,
14711
+ serviceVersion: lineage.serviceVersion,
14712
+ commitSha: lineage.commitSha,
14713
+ targetNodeId: lineage.targetNode?.id ?? null,
14714
+ errorSignature: lineage.errorSignature
14715
+ };
11154
14716
  });
11155
14717
  }
11156
14718
  });
@@ -11311,8 +14873,18 @@ var rowToEvalRun = (row) => {
11311
14873
  promptVersion: row.candidate_prompt_version ?? null,
11312
14874
  modelProvider: row.candidate_model_provider ?? null,
11313
14875
  model: row.candidate_model ?? null,
11314
- modelVersion: row.candidate_model_version ?? null
14876
+ modelVersion: row.candidate_model_version ?? null,
14877
+ environment: row.environment ?? "default",
14878
+ serviceVersion: row.service_version ?? null,
14879
+ commitSha: row.commit_sha ?? null,
14880
+ configDigest: row.config_digest ?? null
11315
14881
  },
14882
+ environment: row.environment ?? "default",
14883
+ serviceVersion: row.service_version ?? null,
14884
+ commitSha: row.commit_sha ?? null,
14885
+ configDigest: row.config_digest ?? null,
14886
+ inputPayloadHash: row.input_payload_hash ?? null,
14887
+ failureFingerprint: row.failure_fingerprint ?? null,
11316
14888
  startedAt: row.started_at ?? null,
11317
14889
  endedAt: row.ended_at ?? null,
11318
14890
  totalCount: row.total_count ?? 0,
@@ -11443,6 +15015,12 @@ var EvalCasesStore = class {
11443
15015
  if (!evalCase) throw new Error("evalCaseId does not reference a case");
11444
15016
  }
11445
15017
  const candidate = input.candidate ?? {};
15018
+ const environment = input.environment?.trim() || candidate.environment?.trim() || "default";
15019
+ const serviceVersion = input.serviceVersion?.trim() || candidate.serviceVersion?.trim() || null;
15020
+ const commitSha = input.commitSha?.trim() || candidate.commitSha?.trim() || null;
15021
+ const configDigest = input.configDigest?.trim() || candidate.configDigest?.trim() || null;
15022
+ const inputPayloadHash = input.inputPayloadHash?.trim() || null;
15023
+ const failureFingerprint = input.failureFingerprint?.trim() || null;
11446
15024
  const totalCount = nonNegativeInteger(input.totalCount);
11447
15025
  const passCount = nonNegativeInteger(input.passCount);
11448
15026
  const failCount = nonNegativeInteger(input.failCount);
@@ -11456,10 +15034,12 @@ var EvalCasesStore = class {
11456
15034
  candidate_agent_id, candidate_agent_version,
11457
15035
  candidate_prompt_id, candidate_prompt_version,
11458
15036
  candidate_model_provider, candidate_model, candidate_model_version,
15037
+ environment, service_version, commit_sha, config_digest,
15038
+ input_payload_hash, failure_fingerprint,
11459
15039
  status, started_at, ended_at,
11460
15040
  total_count, pass_count, fail_count, average_score,
11461
15041
  metadata_json, created_at
11462
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
15042
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
11463
15043
  ).bind(
11464
15044
  id,
11465
15045
  projectId,
@@ -11471,6 +15051,12 @@ var EvalCasesStore = class {
11471
15051
  candidate.modelProvider ?? null,
11472
15052
  candidate.model ?? null,
11473
15053
  candidate.modelVersion ?? null,
15054
+ environment,
15055
+ serviceVersion,
15056
+ commitSha,
15057
+ configDigest,
15058
+ inputPayloadHash,
15059
+ failureFingerprint,
11474
15060
  status,
11475
15061
  startedAt,
11476
15062
  endedAt,
@@ -11537,6 +15123,14 @@ var EvalCasesStore = class {
11537
15123
  average_score = (
11538
15124
  SELECT AVG(score) FROM eval_case_results
11539
15125
  WHERE project_id = ? AND run_id = ? AND score IS NOT NULL
15126
+ ),
15127
+ input_payload_hash = COALESCE(
15128
+ input_payload_hash,
15129
+ (SELECT input_payload_hash FROM eval_case_results WHERE project_id = ? AND run_id = ? AND input_payload_hash IS NOT NULL LIMIT 1)
15130
+ ),
15131
+ failure_fingerprint = COALESCE(
15132
+ failure_fingerprint,
15133
+ (SELECT failure_fingerprint FROM eval_case_results WHERE project_id = ? AND run_id = ? AND failure_fingerprint IS NOT NULL LIMIT 1)
11540
15134
  )
11541
15135
  WHERE project_id = ? AND id = ?`
11542
15136
  ).bind(
@@ -11549,6 +15143,10 @@ var EvalCasesStore = class {
11549
15143
  projectId,
11550
15144
  id,
11551
15145
  projectId,
15146
+ id,
15147
+ projectId,
15148
+ id,
15149
+ projectId,
11552
15150
  id
11553
15151
  ).run();
11554
15152
  }
@@ -11829,7 +15427,7 @@ var badRequest = (message) => ({
11829
15427
  error: "Bad Request",
11830
15428
  message
11831
15429
  });
11832
- var parseLimit = (value) => {
15430
+ var parseLimit2 = (value) => {
11833
15431
  const parsed = Number.parseInt(value ?? "50", 10);
11834
15432
  if (!Number.isFinite(parsed)) return 50;
11835
15433
  return Math.min(200, Math.max(1, parsed));
@@ -11928,7 +15526,7 @@ var evalCasesRoutesPlugin = {
11928
15526
  projectId: getProjectId(c),
11929
15527
  sourceEntityType,
11930
15528
  sourceEntityId: c.req.query("sourceEntityId"),
11931
- limit: parseLimit(c.req.query("limit"))
15529
+ limit: parseLimit2(c.req.query("limit"))
11932
15530
  });
11933
15531
  return c.json({ evalCases });
11934
15532
  });
@@ -12000,6 +15598,12 @@ var evalCasesRoutesPlugin = {
12000
15598
  id: id ?? void 0,
12001
15599
  evalCaseId,
12002
15600
  status,
15601
+ environment: asOptionalString(body.environment) ?? asOptionalString(candidate.environment),
15602
+ serviceVersion: asOptionalString(body.serviceVersion) ?? asOptionalString(candidate.serviceVersion),
15603
+ commitSha: asOptionalString(body.commitSha) ?? asOptionalString(candidate.commitSha),
15604
+ configDigest: asOptionalString(body.configDigest) ?? asOptionalString(candidate.configDigest),
15605
+ inputPayloadHash: asOptionalString(body.inputPayloadHash),
15606
+ failureFingerprint: asOptionalString(body.failureFingerprint),
12003
15607
  candidate: {
12004
15608
  agentId: asOptionalString(candidate.agentId),
12005
15609
  agentVersion: asOptionalString(candidate.agentVersion),
@@ -12007,7 +15611,11 @@ var evalCasesRoutesPlugin = {
12007
15611
  promptVersion: asOptionalString(candidate.promptVersion),
12008
15612
  modelProvider: asOptionalString(candidate.modelProvider),
12009
15613
  model: asOptionalString(candidate.model),
12010
- modelVersion: asOptionalString(candidate.modelVersion)
15614
+ modelVersion: asOptionalString(candidate.modelVersion),
15615
+ environment: asOptionalString(candidate.environment) ?? asOptionalString(body.environment),
15616
+ serviceVersion: asOptionalString(candidate.serviceVersion) ?? asOptionalString(body.serviceVersion),
15617
+ commitSha: asOptionalString(candidate.commitSha) ?? asOptionalString(body.commitSha),
15618
+ configDigest: asOptionalString(candidate.configDigest) ?? asOptionalString(body.configDigest)
12011
15619
  },
12012
15620
  startedAt: asOptionalString(body.startedAt),
12013
15621
  endedAt: asOptionalString(body.endedAt),
@@ -12040,7 +15648,7 @@ var evalCasesRoutesPlugin = {
12040
15648
  projectId: getProjectId(c),
12041
15649
  evalCaseId: c.req.query("evalCaseId"),
12042
15650
  status,
12043
- limit: parseLimit(c.req.query("limit"))
15651
+ limit: parseLimit2(c.req.query("limit"))
12044
15652
  });
12045
15653
  return c.json({ evalRuns });
12046
15654
  });
@@ -12089,12 +15697,15 @@ var evalCasesRoutesPlugin = {
12089
15697
  const passed = body.passed ? 1 : 0;
12090
15698
  const score = typeof body.score === "number" && Number.isFinite(body.score) ? body.score : null;
12091
15699
  const actualOutcome = typeof body.actualOutcome === "string" ? body.actualOutcome : null;
15700
+ const inputPayloadHash = typeof body.inputPayloadHash === "string" && body.inputPayloadHash.trim() ? body.inputPayloadHash.trim() : null;
15701
+ const failureFingerprint = typeof body.failureFingerprint === "string" && body.failureFingerprint.trim() ? body.failureFingerprint.trim() : null;
12092
15702
  const details_json = body.details === void 0 ? null : JSON.stringify(body.details);
12093
15703
  const now = (/* @__PURE__ */ new Date()).toISOString();
12094
15704
  await db.prepare(
12095
15705
  `INSERT INTO eval_case_results (
12096
- id, project_id, eval_case_id, run_id, passed, score, actual_outcome, details_json, created_at
12097
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
15706
+ id, project_id, eval_case_id, run_id, passed, score, actual_outcome,
15707
+ input_payload_hash, failure_fingerprint, details_json, created_at
15708
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
12098
15709
  ).bind(
12099
15710
  id,
12100
15711
  projectId,
@@ -12103,6 +15714,8 @@ var evalCasesRoutesPlugin = {
12103
15714
  passed,
12104
15715
  score,
12105
15716
  actualOutcome,
15717
+ inputPayloadHash,
15718
+ failureFingerprint,
12106
15719
  details_json,
12107
15720
  now
12108
15721
  ).run();
@@ -12116,6 +15729,8 @@ var evalCasesRoutesPlugin = {
12116
15729
  passed: body.passed,
12117
15730
  score,
12118
15731
  actualOutcome,
15732
+ inputPayloadHash,
15733
+ failureFingerprint,
12119
15734
  details: body.details ?? null,
12120
15735
  createdAt: now,
12121
15736
  evidenceReferences: evalCaseResultEvidenceReferences(evalCase, id),
@@ -12155,6 +15770,8 @@ var evalCasesRoutesPlugin = {
12155
15770
  passed: row.passed === 1,
12156
15771
  score: row.score,
12157
15772
  actualOutcome: row.actual_outcome,
15773
+ inputPayloadHash: row.input_payload_hash ?? null,
15774
+ failureFingerprint: row.failure_fingerprint ?? null,
12158
15775
  details: parseJsonMaybe(row.details_json),
12159
15776
  createdAt: row.created_at,
12160
15777
  evidenceReferences: evalCaseResultEvidenceReferences(
@@ -12417,6 +16034,7 @@ var DEFAULT_PROFILE_FRAME_LIMIT = 50;
12417
16034
  var MAX_PROFILE_FRAME_LIMIT = 200;
12418
16035
  var EVIDENCE_INTENTS = /* @__PURE__ */ new Set([
12419
16036
  "debug_failure",
16037
+ "triage_incident",
12420
16038
  "explain_latency",
12421
16039
  "explain_cost",
12422
16040
  "inspect_agent_run",
@@ -13464,6 +17082,156 @@ var buildActionGraphBundle = (projectId, anchorKind, anchorId, manifest, profile
13464
17082
  };
13465
17083
  return bundle;
13466
17084
  };
17085
+ var attachAutoSreTriageContext = async (db, projectId, traceId, bundle) => {
17086
+ try {
17087
+ const faultLocus = traceId ? await loadTraceFaultLocusFromDb(db, projectId, traceId) : null;
17088
+ const environment = (faultLocus?.faultLocusSpan?.environment ?? bundle.environment ?? "default").toLowerCase();
17089
+ const recentChanges = await listChangeEvents(db, {
17090
+ projectId,
17091
+ environment,
17092
+ limit: 5
17093
+ });
17094
+ const blastRadii = (await Promise.all(
17095
+ recentChanges.slice(0, 3).map(
17096
+ (ch) => computeChangeBlastRadius(db, projectId, ch.id, { environment })
17097
+ )
17098
+ )).filter((br) => Boolean(br));
17099
+ const candidateChanges = blastRadii.map((br) => ({
17100
+ changeId: br.change.id,
17101
+ kind: br.change.kind,
17102
+ version: br.change.version,
17103
+ previousVersion: br.change.previousVersion,
17104
+ occurredAt: br.change.occurredAt,
17105
+ cohortVerdict: br.cohortVerdict,
17106
+ causalProofTier: br.causalProofTier,
17107
+ rateNew: br.newCohort.errorRate,
17108
+ rateOld: br.oldCohort.errorRate,
17109
+ sampleNew: br.newCohort.requestCount,
17110
+ sampleOld: br.oldCohort.requestCount,
17111
+ overlappingCodePaths: br.overlappingCodePaths
17112
+ }));
17113
+ const contentionCandidates = blastRadii.flatMap((br) => br.sharedResourceAggressors).map((ag) => ({
17114
+ aggressorTraceId: ag.aggressorTraceId,
17115
+ aggressorSpanId: ag.aggressorSpanId,
17116
+ aggressorServiceName: ag.aggressorServiceName,
17117
+ sharedTargetNodeId: ag.sharedTargetNodeId,
17118
+ relationshipCertainty: ag.relationshipCertainty,
17119
+ totalResourceTimeMs: ag.totalResourceTimeMs
17120
+ }));
17121
+ let priorFindingsList = await searchInvestigationFindings(db, {
17122
+ projectId,
17123
+ environment,
17124
+ rootCauseEntityId: faultLocus?.faultLocusNodeId ?? faultLocus?.faultLocusSpan?.span_id ?? void 0,
17125
+ limit: 5
17126
+ });
17127
+ if (priorFindingsList.length === 0 && faultLocus?.faultLocusSpan?.span_id) {
17128
+ priorFindingsList = await searchInvestigationFindings(db, {
17129
+ projectId,
17130
+ environment,
17131
+ rootCauseEntityId: faultLocus.faultLocusSpan.span_id,
17132
+ limit: 5
17133
+ });
17134
+ }
17135
+ if (priorFindingsList.length === 0) {
17136
+ priorFindingsList = await searchInvestigationFindings(db, {
17137
+ projectId,
17138
+ environment,
17139
+ limit: 5
17140
+ });
17141
+ }
17142
+ const priorFindings = priorFindingsList.map((f) => ({
17143
+ findingId: f.id,
17144
+ title: f.title,
17145
+ causalProofTier: f.causalProofTier,
17146
+ causalVerificationStatus: f.causalVerificationStatus,
17147
+ recoveryStatus: f.recoveryStatus,
17148
+ contributingChangeId: f.contributingChangeId,
17149
+ occurredAt: f.createdAt
17150
+ }));
17151
+ for (const br of blastRadii) {
17152
+ const changeId = br.change.id;
17153
+ const changeRoute = `#/changes/${encodeURIComponent(changeId)}`;
17154
+ bundle.evidenceReferences.push({
17155
+ evidenceId: `change:${changeId}`,
17156
+ entityKind: "change",
17157
+ entityId: changeId,
17158
+ route: changeRoute,
17159
+ source: br.overlappingCodePaths.length ? "stack_frame_overlap" : "cohort_comparison",
17160
+ confidence: br.cohortVerdict === "isolated_to_new_version" ? 0.9 : 0.65,
17161
+ relationshipCertainty: br.relationshipCertainty,
17162
+ causalProofTier: br.causalProofTier,
17163
+ reason: br.summary,
17164
+ citations: [
17165
+ {
17166
+ label: `change ${changeId} (${br.change.version ?? br.change.kind})`,
17167
+ entityKind: "change",
17168
+ entityId: changeId,
17169
+ route: changeRoute
17170
+ }
17171
+ ],
17172
+ suggestedNextPivots: [
17173
+ {
17174
+ label: "Inspect change blast radius",
17175
+ entityKind: "change",
17176
+ entityId: changeId,
17177
+ route: changeRoute,
17178
+ reason: "Inspect traffic-normalized cohort lift and contention."
17179
+ }
17180
+ ]
17181
+ });
17182
+ }
17183
+ for (const finding of priorFindingsList) {
17184
+ const findingRoute = `#/findings/${encodeURIComponent(finding.id)}`;
17185
+ bundle.evidenceReferences.push({
17186
+ evidenceId: `finding:${finding.id}`,
17187
+ entityKind: "finding",
17188
+ entityId: finding.id,
17189
+ route: findingRoute,
17190
+ source: "incident_memory_fingerprint",
17191
+ confidence: finding.confidence,
17192
+ relationshipCertainty: "exact",
17193
+ causalProofTier: finding.causalProofTier,
17194
+ reason: `Prior finding (${finding.causalProofTier}): ${finding.title}`,
17195
+ citations: [
17196
+ {
17197
+ label: finding.title,
17198
+ entityKind: "finding",
17199
+ entityId: finding.id,
17200
+ route: findingRoute
17201
+ }
17202
+ ],
17203
+ suggestedNextPivots: [
17204
+ {
17205
+ label: "Open prior finding",
17206
+ entityKind: "finding",
17207
+ entityId: finding.id,
17208
+ route: findingRoute,
17209
+ reason: "Search incident memory for similar recurring findings."
17210
+ }
17211
+ ]
17212
+ });
17213
+ }
17214
+ bundle.autosre = {
17215
+ faultLocus: faultLocus?.faultLocusNodeId ? {
17216
+ nodeId: faultLocus.faultLocusNodeId,
17217
+ spanId: faultLocus.faultLocusSpan?.span_id,
17218
+ relationshipCertainty: faultLocus.relationshipCertainty,
17219
+ isBlackoutSuspected: false
17220
+ } : void 0,
17221
+ candidateChanges,
17222
+ contentionCandidates,
17223
+ priorFindings
17224
+ };
17225
+ if (bundle.budget) {
17226
+ bundle.budget = {
17227
+ ...bundle.budget,
17228
+ estimatedTokens: estimateTokens(bundle)
17229
+ };
17230
+ }
17231
+ } catch {
17232
+ }
17233
+ return bundle;
17234
+ };
13467
17235
  var parseBundleRequest = async (request) => {
13468
17236
  try {
13469
17237
  const body = await request.json();
@@ -13529,7 +17297,7 @@ var evidenceRetrievalRoutesPlugin = {
13529
17297
  )).flat().filter(
13530
17298
  (profile, index, all) => all.findIndex((candidate) => candidate.id === profile.id) === index
13531
17299
  );
13532
- const bundle2 = buildActionGraphBundle(
17300
+ let bundle2 = buildActionGraphBundle(
13533
17301
  projectId,
13534
17302
  anchor.entityKind,
13535
17303
  anchor.entityId,
@@ -13539,13 +17307,107 @@ var evidenceRetrievalRoutesPlugin = {
13539
17307
  targetTokens,
13540
17308
  detailLevel
13541
17309
  );
17310
+ if (intent === "triage_incident") {
17311
+ bundle2 = await attachAutoSreTriageContext(
17312
+ db,
17313
+ projectId,
17314
+ traceIds[0] ?? null,
17315
+ bundle2
17316
+ );
17317
+ }
13542
17318
  return c.json(await materializeBundle(db, projectId, bundle2));
13543
17319
  }
17320
+ if (anchor.entityKind === "incident") {
17321
+ const incident = await getIncidentInvestigation(
17322
+ db,
17323
+ projectId,
17324
+ anchor.entityId
17325
+ );
17326
+ if (!incident) {
17327
+ return c.json(
17328
+ { error: "Not Found", message: "Incident not found" },
17329
+ 404
17330
+ );
17331
+ }
17332
+ const recentErrSpan = await db.prepare(
17333
+ `SELECT trace_id FROM telemetry_spans
17334
+ WHERE project_id = ? AND environment = ? AND status_code = 2
17335
+ ORDER BY start_time DESC LIMIT 1`
17336
+ ).bind(projectId, incident.environment).first();
17337
+ const incRoute = `#/incidents/${encodeURIComponent(incident.id)}`;
17338
+ const baseBundle = {
17339
+ schemaVersion: BUNDLE_SCHEMA_VERSION,
17340
+ intent,
17341
+ environment: incident.environment,
17342
+ incidentId: incident.id,
17343
+ symptomFingerprint: incident.symptomFingerprint ?? void 0,
17344
+ anchor: { entityKind: "incident", entityId: incident.id },
17345
+ budget: { targetTokens, detailLevel },
17346
+ summary: `Incident ${incident.id} (${incident.status}) on ${incident.targetNodeId} in environment ${incident.environment}.`,
17347
+ derivedSummaries: [
17348
+ {
17349
+ title: `Incident ${incident.id}`,
17350
+ reason: `Trigger ${incident.symptomAnchorKind}:${incident.symptomAnchorId} on ${incident.targetNodeId} (status=${incident.status}).`,
17351
+ confidence: 1,
17352
+ evidenceIds: [`incident:${incident.id}`],
17353
+ retrievalRefIds: []
17354
+ }
17355
+ ],
17356
+ findings: [],
17357
+ compactions: [],
17358
+ evidenceReferences: [
17359
+ {
17360
+ evidenceId: `incident:${incident.id}`,
17361
+ entityKind: "incident",
17362
+ entityId: incident.id,
17363
+ route: incRoute,
17364
+ source: "incident_investigations",
17365
+ confidence: 1,
17366
+ relationshipCertainty: "exact",
17367
+ reason: `Incident triggered by ${incident.symptomAnchorKind}:${incident.symptomAnchorId}.`,
17368
+ citations: [
17369
+ {
17370
+ label: incident.targetNodeId,
17371
+ entityKind: "incident",
17372
+ entityId: incident.id,
17373
+ route: incRoute
17374
+ }
17375
+ ],
17376
+ suggestedNextPivots: [
17377
+ {
17378
+ label: "Open incident",
17379
+ entityKind: "incident",
17380
+ entityId: incident.id,
17381
+ route: incRoute,
17382
+ reason: "Inspect incident investigation and candidate changes."
17383
+ }
17384
+ ]
17385
+ }
17386
+ ],
17387
+ retrievalRefs: [],
17388
+ suggestedNextPivots: [
17389
+ {
17390
+ label: "Open incident",
17391
+ entityKind: "incident",
17392
+ entityId: incident.id,
17393
+ route: incRoute,
17394
+ reason: "Inspect incident investigation and candidate changes."
17395
+ }
17396
+ ]
17397
+ };
17398
+ const enriched = await attachAutoSreTriageContext(
17399
+ db,
17400
+ projectId,
17401
+ recentErrSpan?.trace_id ?? null,
17402
+ baseBundle
17403
+ );
17404
+ return c.json(await materializeBundle(db, projectId, enriched));
17405
+ }
13544
17406
  if (anchor.entityKind !== "trace") {
13545
17407
  return c.json(
13546
17408
  {
13547
17409
  error: "Bad Request",
13548
- message: "Supported evidence bundle anchors are trace, action, agent_run, and tool_call."
17410
+ message: "Supported evidence bundle anchors are trace, action, agent_run, tool_call, and incident."
13549
17411
  },
13550
17412
  400
13551
17413
  );
@@ -13563,7 +17425,7 @@ var evidenceRetrievalRoutesPlugin = {
13563
17425
  limit: MAX_LOGS_FOR_BUNDLE
13564
17426
  });
13565
17427
  const profiles = await profilesForTrace(db, projectId, traceId);
13566
- const bundle = await buildTraceBundle(
17428
+ let bundle = await buildTraceBundle(
13567
17429
  projectId,
13568
17430
  detail,
13569
17431
  logs.logs,
@@ -13573,6 +17435,14 @@ var evidenceRetrievalRoutesPlugin = {
13573
17435
  detailLevel,
13574
17436
  hours
13575
17437
  );
17438
+ if (intent === "triage_incident" || intent === "debug_failure") {
17439
+ bundle = await attachAutoSreTriageContext(
17440
+ db,
17441
+ projectId,
17442
+ traceId,
17443
+ bundle
17444
+ );
17445
+ }
13576
17446
  return c.json(await materializeBundle(db, projectId, bundle));
13577
17447
  });
13578
17448
  app.get("/internal/evidence/stats", async (c) => {
@@ -14492,7 +18362,7 @@ var parseHours = (rawValue) => Math.min(
14492
18362
  Number.parseInt(rawValue || String(DEFAULT_WINDOW_HOURS2), 10) || DEFAULT_WINDOW_HOURS2
14493
18363
  )
14494
18364
  );
14495
- var parseLimit2 = (rawValue) => Math.min(
18365
+ var parseLimit3 = (rawValue) => Math.min(
14496
18366
  MAX_ISSUE_ROWS,
14497
18367
  Math.max(
14498
18368
  1,
@@ -14511,7 +18381,7 @@ var issueInsightsPlugin = {
14511
18381
  service: c.req.query("service") || void 0,
14512
18382
  category: c.req.query("category") === "error" || c.req.query("category") === "latency" || c.req.query("category") === "dependency" ? c.req.query("category") : "all",
14513
18383
  includeInternal: c.req.query("includeInternal") === "true",
14514
- limit: parseLimit2(c.req.query("limit"))
18384
+ limit: parseLimit3(c.req.query("limit"))
14515
18385
  });
14516
18386
  return c.json({
14517
18387
  ...overview,
@@ -18682,6 +22552,7 @@ var allPlugins = [
18682
22552
  analysesRoutesPlugin,
18683
22553
  actionRoutesPlugin,
18684
22554
  evalCasesRoutesPlugin,
22555
+ autosreRoutesPlugin,
18685
22556
  evidenceRetrievalRoutesPlugin,
18686
22557
  askRoutesPlugin,
18687
22558
  tailRoutesPlugin,
@@ -18704,6 +22575,7 @@ export {
18704
22575
  ActionStore,
18705
22576
  AlertsStore,
18706
22577
  BlobStoreToR2Adapter,
22578
+ CounterfactualVerificationError,
18707
22579
  D1Adapter,
18708
22580
  EvalCaseSourceNotFoundError,
18709
22581
  EvalCasesStore,
@@ -18711,8 +22583,10 @@ export {
18711
22583
  PostgresAdapter,
18712
22584
  ProjectsStore,
18713
22585
  R2BlobStore,
22586
+ RemediationGateError,
18714
22587
  S3BlobStore,
18715
22588
  TailHub,
22589
+ acquireIncidentLease,
18716
22590
  actionRetentionCutoff,
18717
22591
  actionRoutesPlugin,
18718
22592
  aiReceiverPlugin,
@@ -18720,9 +22594,15 @@ export {
18720
22594
  alertsRoutesPlugin,
18721
22595
  allPlugins,
18722
22596
  analysesRoutesPlugin,
22597
+ approveRemediationProposal,
18723
22598
  askRoutesPlugin,
22599
+ autosreRoutesPlugin,
18724
22600
  botFilterPlugin,
22601
+ claimRemediationGrant,
18725
22602
  compareValue,
22603
+ completeRemediationProposal,
22604
+ computeChangeBlastRadius,
22605
+ computeTarjanScc,
18726
22606
  configurePostgresTypeParsers,
18727
22607
  connectedRoutesPlugin,
18728
22608
  consoleLogger,
@@ -18736,35 +22616,58 @@ export {
18736
22616
  dashboardRoutesPlugin,
18737
22617
  src_default as default,
18738
22618
  defaultSpanEnrichmentPlugin,
22619
+ dispatchIncidentWebhook,
18739
22620
  evalCasesRoutesPlugin,
18740
22621
  evaluateAllRules,
22622
+ evaluateRemediationWatchdogs,
18741
22623
  evidenceRetrievalRoutesPlugin,
22624
+ extractSpanLineageAndTopology,
22625
+ failRemediationProposal,
22626
+ findCrossTraceContention,
18742
22627
  genAiNormalizerPlugin,
22628
+ getChangeEvent,
22629
+ getIncidentInvestigation,
22630
+ getInvestigationFinding,
22631
+ getOntologyGraph,
22632
+ getRemediationProposal,
18743
22633
  identityReceiverPlugin,
18744
22634
  isEvalCaseSourceType,
18745
22635
  isEvalRunStatus,
18746
22636
  issueEnrichmentPlugin,
18747
22637
  issueInsightsPlugin,
22638
+ listChangeEvents,
22639
+ listIncidentInvestigations,
22640
+ listRemediationProposals,
22641
+ loadTraceFaultLocusFromDb,
22642
+ localizeTraceFaultLocus,
18748
22643
  logsReceiverPlugin,
18749
22644
  metricsReceiverPlugin,
18750
22645
  onboardingRoutesPlugin,
22646
+ openOrDeduplicateIncident,
18751
22647
  otlpReceiverPlugin,
18752
22648
  platformRoutesPlugin,
18753
22649
  profileRoutesPlugin,
18754
22650
  projectsRoutesPlugin,
22651
+ proposeRemediationAction,
18755
22652
  purgeExpiredActionGraph,
18756
22653
  queryRoutesPlugin,
22654
+ recordChangeEvent,
22655
+ recordInvestigationFinding,
18757
22656
  redactionProcessorPlugin,
22657
+ rejectRemediationProposal,
18758
22658
  replayQueryRoutesPlugin,
18759
22659
  replayReceiverPlugin,
22660
+ searchInvestigationFindings,
18760
22661
  tailRoutesPlugin,
18761
22662
  timelineRoutesPlugin,
18762
22663
  uaEnrichmentPlugin,
22664
+ upsertOntologyFromSpans,
18763
22665
  usageNormalizationPlugin,
18764
22666
  usagePrivacyPlugin,
18765
22667
  usageQueryRoutesPlugin,
18766
22668
  usageReceiverPlugin,
18767
22669
  usersQueryRoutesPlugin,
18768
- utmEnrichmentPlugin
22670
+ utmEnrichmentPlugin,
22671
+ verifyPairedCounterfactualProof
18769
22672
  };
18770
22673
  //# sourceMappingURL=index.js.map