@obsunified/collector 2.1.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
- package/dist/chunk-ZSF5XP3B.js.map +1 -0
- package/dist/index.d.ts +309 -2
- package/dist/index.js +4159 -256
- package/dist/index.js.map +1 -1
- package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
- package/package.json +2 -2
- package/src/__tests__/autosre-closed-loop.test.ts +1275 -0
- package/src/index.ts +43 -0
- package/src/lib/analyses-runner.ts +9 -0
- package/src/lib/autosre-change-correlation.ts +780 -0
- package/src/lib/autosre-findings.ts +725 -0
- package/src/lib/autosre-ontology.ts +914 -0
- package/src/lib/autosre-remediation-gate.ts +1663 -0
- package/src/lib/eval-cases-store.ts +63 -1
- package/src/lib/evidence-references.ts +6 -0
- package/src/lib/housekeeping.test.ts +80 -0
- package/src/lib/housekeeping.ts +101 -1
- package/src/lib/store/ingest.ts +94 -6
- package/src/migrations/050_autosre_causal_ontology.sql +216 -0
- package/src/migrations/alert-state-notified.test.ts +3 -3
- package/src/migrations/parity.test.ts +4 -0
- package/src/migrations/project-scoped-keys.test.ts +1 -1
- package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
- package/src/plugins/alerts-evaluator.ts +17 -1
- package/src/plugins/autosre-routes.ts +1019 -0
- package/src/plugins/connected-routes/manifest.ts +13 -1
- package/src/plugins/connected-routes.ts +271 -0
- package/src/plugins/default-span-enrichment.ts +13 -1
- package/src/plugins/eval-cases-routes.ts +57 -2
- package/src/plugins/evidence-retrieval-routes.ts +300 -4
- package/dist/chunk-XKQTVIRE.js.map +0 -1
- /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
askEvidenceReferences,
|
|
19
19
|
generateNarrative,
|
|
20
20
|
sourceLinkEvidenceReferences
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-ZSF5XP3B.js";
|
|
22
22
|
|
|
23
23
|
// src/framework/collector.ts
|
|
24
24
|
import { getConfiguredRetentionHours } from "@obsunified/types/constants";
|
|
@@ -2487,6 +2487,1569 @@ var NOOP_HANDLE = {
|
|
|
2487
2487
|
};
|
|
2488
2488
|
var passthroughChildSpan = async (_name, fn) => fn(NOOP_HANDLE);
|
|
2489
2489
|
|
|
2490
|
+
// src/lib/autosre-remediation-gate.ts
|
|
2491
|
+
import { AgentAutonomyLevel } from "@obsunified/types/constants";
|
|
2492
|
+
|
|
2493
|
+
// src/lib/autosre-findings.ts
|
|
2494
|
+
import { buildCanonicalNodeId, buildFindingFingerprint } from "@obsunified/types";
|
|
2495
|
+
var mapFindingRow = (row) => {
|
|
2496
|
+
const rawFpInput = parseJsonRecord(row.fingerprint_input_json);
|
|
2497
|
+
const fingerprintInput = {
|
|
2498
|
+
environment: typeof rawFpInput.environment === "string" ? rawFpInput.environment : row.environment,
|
|
2499
|
+
originatingNodeId: typeof rawFpInput.originatingNodeId === "string" ? rawFpInput.originatingNodeId : row.fault_locus_entity_id,
|
|
2500
|
+
normalizedOperation: typeof rawFpInput.normalizedOperation === "string" ? rawFpInput.normalizedOperation : row.title,
|
|
2501
|
+
errorSignature: typeof rawFpInput.errorSignature === "string" ? rawFpInput.errorSignature : "ERROR",
|
|
2502
|
+
downstreamTargetNodeId: typeof rawFpInput.downstreamTargetNodeId === "string" ? rawFpInput.downstreamTargetNodeId : null
|
|
2503
|
+
};
|
|
2504
|
+
const rawBreakdown = parseJsonRecord(row.confidence_breakdown_json);
|
|
2505
|
+
const confidenceBreakdown = {
|
|
2506
|
+
relationshipScore: Number(rawBreakdown.relationshipScore ?? 0.5),
|
|
2507
|
+
cohortLiftScore: Number(rawBreakdown.cohortLiftScore ?? 0),
|
|
2508
|
+
temporalProximityScore: Number(rawBreakdown.temporalProximityScore ?? 0.5),
|
|
2509
|
+
counterfactualBonus: Number(rawBreakdown.counterfactualBonus ?? 0),
|
|
2510
|
+
priorRecurrenceBonus: Number(rawBreakdown.priorRecurrenceBonus ?? 0),
|
|
2511
|
+
finalConfidence: Number(rawBreakdown.finalConfidence ?? row.confidence)
|
|
2512
|
+
};
|
|
2513
|
+
let counterfactualProof = null;
|
|
2514
|
+
if (row.counterfactual_proof_json) {
|
|
2515
|
+
const rawCf = parseJsonRecord(row.counterfactual_proof_json);
|
|
2516
|
+
if (typeof rawCf.evalCaseId === "string") {
|
|
2517
|
+
counterfactualProof = {
|
|
2518
|
+
evalCaseId: rawCf.evalCaseId,
|
|
2519
|
+
inputPayloadHash: String(rawCf.inputPayloadHash ?? ""),
|
|
2520
|
+
failingRunId: String(rawCf.failingRunId ?? ""),
|
|
2521
|
+
failingVersion: typeof rawCf.failingVersion === "string" ? rawCf.failingVersion : null,
|
|
2522
|
+
failingCommitSha: typeof rawCf.failingCommitSha === "string" ? rawCf.failingCommitSha : null,
|
|
2523
|
+
failureFingerprint: String(rawCf.failureFingerprint ?? row.fingerprint),
|
|
2524
|
+
passingRunIds: Array.isArray(rawCf.passingRunIds) ? rawCf.passingRunIds.map(String) : [],
|
|
2525
|
+
passingVersion: typeof rawCf.passingVersion === "string" ? rawCf.passingVersion : null,
|
|
2526
|
+
passingCommitSha: typeof rawCf.passingCommitSha === "string" ? rawCf.passingCommitSha : null,
|
|
2527
|
+
requiredTrials: Number(rawCf.requiredTrials ?? 1),
|
|
2528
|
+
verifiedAt: String(rawCf.verifiedAt ?? row.updated_at)
|
|
2529
|
+
};
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
return {
|
|
2533
|
+
id: row.id,
|
|
2534
|
+
projectId: row.project_id,
|
|
2535
|
+
environment: row.environment,
|
|
2536
|
+
incidentId: row.incident_id,
|
|
2537
|
+
fingerprint: row.fingerprint,
|
|
2538
|
+
fingerprintInput,
|
|
2539
|
+
title: row.title,
|
|
2540
|
+
summary: row.summary,
|
|
2541
|
+
causalProofTier: row.causal_proof_tier || "unverified_locus",
|
|
2542
|
+
confidence: Number(row.confidence),
|
|
2543
|
+
confidenceBreakdown,
|
|
2544
|
+
symptomEntityKind: row.symptom_entity_kind,
|
|
2545
|
+
symptomEntityId: row.symptom_entity_id,
|
|
2546
|
+
faultLocusEntityKind: row.fault_locus_entity_kind,
|
|
2547
|
+
faultLocusEntityId: row.fault_locus_entity_id,
|
|
2548
|
+
rootCauseEntityKind: row.root_cause_entity_kind ?? null,
|
|
2549
|
+
rootCauseEntityId: row.root_cause_entity_id,
|
|
2550
|
+
contributingChangeId: row.contributing_change_id,
|
|
2551
|
+
cohortVerdict: row.cohort_verdict ?? null,
|
|
2552
|
+
contendingTraceId: row.contending_trace_id,
|
|
2553
|
+
causalChain: parseJsonArray(row.causal_chain_json),
|
|
2554
|
+
citedEvidenceIds: parseJsonArray(row.cited_evidence_ids_json).map(String),
|
|
2555
|
+
codeReferences: parseJsonArray(
|
|
2556
|
+
row.code_references_json ?? "[]"
|
|
2557
|
+
),
|
|
2558
|
+
counterfactualProof,
|
|
2559
|
+
recoveryStatus: row.recovery_status || "not_applicable",
|
|
2560
|
+
causalVerificationStatus: row.causal_verification_status || "unverified",
|
|
2561
|
+
watchdogStep: Number(row.watchdog_step ?? 0),
|
|
2562
|
+
baselineRequestRatePerMin: row.baseline_request_rate_per_min === null ? null : Number(row.baseline_request_rate_per_min),
|
|
2563
|
+
nextWatchdogCheckAt: row.next_watchdog_check_at,
|
|
2564
|
+
actorType: row.actor_type,
|
|
2565
|
+
actorId: row.actor_id,
|
|
2566
|
+
agentRunId: row.agent_run_id,
|
|
2567
|
+
createdAt: row.created_at,
|
|
2568
|
+
updatedAt: row.updated_at,
|
|
2569
|
+
expiresAt: row.expires_at
|
|
2570
|
+
};
|
|
2571
|
+
};
|
|
2572
|
+
function resolveSafeCausalProofTier(input) {
|
|
2573
|
+
if (input.cohortVerdict === "isolated_to_new_version") {
|
|
2574
|
+
return "cohort_rate_isolated";
|
|
2575
|
+
}
|
|
2576
|
+
if (input.cohortVerdict === "sequential_post_cutover_regression" || input.cohortVerdict === "shared_degradation_new_version_aggressor") {
|
|
2577
|
+
return "corroborated_candidate";
|
|
2578
|
+
}
|
|
2579
|
+
const hasCodeOverlapOrContention = input.causalChain.some(
|
|
2580
|
+
(t) => t.predicate === "touched_same_code_path" || t.predicate === "contended_on_shared_resource" || t.predicate === "induced_shared_resource_contention" || t.predicate === "degraded_after_change"
|
|
2581
|
+
);
|
|
2582
|
+
if (input.causalProofTier === "cohort_rate_isolated") {
|
|
2583
|
+
const hasIsolatedTriple = input.causalChain.some(
|
|
2584
|
+
(t) => t.predicate === "isolated_to_version_cohort"
|
|
2585
|
+
);
|
|
2586
|
+
if (hasIsolatedTriple) return "cohort_rate_isolated";
|
|
2587
|
+
}
|
|
2588
|
+
if (hasCodeOverlapOrContention || input.causalProofTier === "corroborated_candidate") {
|
|
2589
|
+
return "corroborated_candidate";
|
|
2590
|
+
}
|
|
2591
|
+
return "unverified_locus";
|
|
2592
|
+
}
|
|
2593
|
+
async function recordInvestigationFinding(db, projectId, input) {
|
|
2594
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
2595
|
+
const environment = (input.environment ?? input.fingerprintInput?.environment ?? "default").trim().toLowerCase();
|
|
2596
|
+
const defaultOriginNode = input.faultLocusEntityId.includes(":") ? input.faultLocusEntityId : buildCanonicalNodeId({
|
|
2597
|
+
kind: "service",
|
|
2598
|
+
environment,
|
|
2599
|
+
serviceName: input.faultLocusEntityId
|
|
2600
|
+
}).id;
|
|
2601
|
+
const fpResult = buildFindingFingerprint(
|
|
2602
|
+
input.fingerprintInput ?? {
|
|
2603
|
+
environment,
|
|
2604
|
+
originatingNodeId: defaultOriginNode,
|
|
2605
|
+
normalizedOperation: input.title,
|
|
2606
|
+
errorSignature: input.summary.slice(0, 120),
|
|
2607
|
+
downstreamTargetNodeId: null
|
|
2608
|
+
}
|
|
2609
|
+
);
|
|
2610
|
+
const priorRow = await db.prepare(
|
|
2611
|
+
`SELECT id, causal_verification_status FROM investigation_findings
|
|
2612
|
+
WHERE project_id = ? AND environment = ? AND fingerprint = ?
|
|
2613
|
+
ORDER BY created_at DESC
|
|
2614
|
+
LIMIT 1`
|
|
2615
|
+
).bind(projectId, environment, fpResult.fingerprint).first();
|
|
2616
|
+
const causalProofTier = resolveSafeCausalProofTier(input);
|
|
2617
|
+
const relationshipScore = input.causalChain.some(
|
|
2618
|
+
(c) => c.relationshipCertainty === "exact"
|
|
2619
|
+
) ? 0.9 : input.causalChain.some((c) => c.relationshipCertainty === "inferred") ? 0.65 : 0.4;
|
|
2620
|
+
const cohortLiftScore = causalProofTier === "cohort_rate_isolated" ? 0.9 : causalProofTier === "corroborated_candidate" ? 0.6 : 0.2;
|
|
2621
|
+
const priorBonus = priorRow?.causal_verification_status === "counterfactual_verified" || priorRow?.causal_verification_status === "operationally_corroborated" ? 0.1 : 0;
|
|
2622
|
+
const computedConfidence = Math.min(
|
|
2623
|
+
0.99,
|
|
2624
|
+
Number(
|
|
2625
|
+
(0.4 * relationshipScore + 0.4 * cohortLiftScore + 0.2 * 0.7 + priorBonus).toFixed(2)
|
|
2626
|
+
)
|
|
2627
|
+
);
|
|
2628
|
+
const confidence = input.confidence ?? computedConfidence;
|
|
2629
|
+
const confidenceBreakdown = {
|
|
2630
|
+
relationshipScore: input.confidenceBreakdown?.relationshipScore ?? relationshipScore,
|
|
2631
|
+
cohortLiftScore: input.confidenceBreakdown?.cohortLiftScore ?? cohortLiftScore,
|
|
2632
|
+
temporalProximityScore: input.confidenceBreakdown?.temporalProximityScore ?? 0.7,
|
|
2633
|
+
counterfactualBonus: input.confidenceBreakdown?.counterfactualBonus ?? 0,
|
|
2634
|
+
priorRecurrenceBonus: input.confidenceBreakdown?.priorRecurrenceBonus ?? priorBonus,
|
|
2635
|
+
finalConfidence: confidence
|
|
2636
|
+
};
|
|
2637
|
+
const id = input.id?.trim() || `fnd_${randomHex(10)}`;
|
|
2638
|
+
const expiresAt = new Date(
|
|
2639
|
+
Date.now() + 90 * 24 * 36e5
|
|
2640
|
+
).toISOString();
|
|
2641
|
+
await db.prepare(
|
|
2642
|
+
`INSERT INTO investigation_findings (
|
|
2643
|
+
project_id, environment, id, incident_id, fingerprint,
|
|
2644
|
+
fingerprint_input_json, title, summary, causal_proof_tier,
|
|
2645
|
+
confidence, confidence_breakdown_json, symptom_entity_kind,
|
|
2646
|
+
symptom_entity_id, fault_locus_entity_kind, fault_locus_entity_id,
|
|
2647
|
+
root_cause_entity_kind, root_cause_entity_id, contributing_change_id,
|
|
2648
|
+
cohort_verdict, contending_trace_id, causal_chain_json,
|
|
2649
|
+
cited_evidence_ids_json, code_references_json,
|
|
2650
|
+
recovery_status, causal_verification_status, watchdog_step,
|
|
2651
|
+
actor_type, actor_id, agent_run_id, created_at, updated_at, expires_at
|
|
2652
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'not_applicable', 'unverified', 0, ?, ?, ?, ?, ?, ?)
|
|
2653
|
+
ON CONFLICT (project_id, environment, id) DO UPDATE SET
|
|
2654
|
+
title = excluded.title,
|
|
2655
|
+
summary = excluded.summary,
|
|
2656
|
+
causal_proof_tier = excluded.causal_proof_tier,
|
|
2657
|
+
confidence = excluded.confidence,
|
|
2658
|
+
confidence_breakdown_json = excluded.confidence_breakdown_json,
|
|
2659
|
+
contributing_change_id = COALESCE(excluded.contributing_change_id, investigation_findings.contributing_change_id),
|
|
2660
|
+
cohort_verdict = COALESCE(excluded.cohort_verdict, investigation_findings.cohort_verdict),
|
|
2661
|
+
causal_chain_json = excluded.causal_chain_json,
|
|
2662
|
+
cited_evidence_ids_json = excluded.cited_evidence_ids_json,
|
|
2663
|
+
updated_at = excluded.updated_at`
|
|
2664
|
+
).bind(
|
|
2665
|
+
projectId,
|
|
2666
|
+
environment,
|
|
2667
|
+
id,
|
|
2668
|
+
input.incidentId ?? null,
|
|
2669
|
+
fpResult.fingerprint,
|
|
2670
|
+
JSON.stringify(fpResult.normalizedInput),
|
|
2671
|
+
input.title,
|
|
2672
|
+
input.summary,
|
|
2673
|
+
causalProofTier,
|
|
2674
|
+
confidence,
|
|
2675
|
+
JSON.stringify(confidenceBreakdown),
|
|
2676
|
+
input.symptomEntityKind,
|
|
2677
|
+
input.symptomEntityId,
|
|
2678
|
+
input.faultLocusEntityKind,
|
|
2679
|
+
input.faultLocusEntityId,
|
|
2680
|
+
input.rootCauseEntityKind ?? null,
|
|
2681
|
+
input.rootCauseEntityId ?? null,
|
|
2682
|
+
input.contributingChangeId ?? null,
|
|
2683
|
+
input.cohortVerdict ?? null,
|
|
2684
|
+
input.contendingTraceId ?? null,
|
|
2685
|
+
JSON.stringify(input.causalChain),
|
|
2686
|
+
JSON.stringify(input.citedEvidenceIds),
|
|
2687
|
+
JSON.stringify(input.codeReferences ?? []),
|
|
2688
|
+
input.actorType ?? "agent",
|
|
2689
|
+
input.actorId ?? "autosre-agent",
|
|
2690
|
+
input.agentRunId ?? null,
|
|
2691
|
+
nowIso,
|
|
2692
|
+
nowIso,
|
|
2693
|
+
expiresAt
|
|
2694
|
+
).run();
|
|
2695
|
+
const created = await getInvestigationFinding(db, projectId, id, environment);
|
|
2696
|
+
if (!created) {
|
|
2697
|
+
throw new Error(`Failed to load created finding ${id}`);
|
|
2698
|
+
}
|
|
2699
|
+
return created;
|
|
2700
|
+
}
|
|
2701
|
+
async function getInvestigationFinding(db, projectId, findingId, environment) {
|
|
2702
|
+
const row = environment ? await db.prepare(
|
|
2703
|
+
`SELECT * FROM investigation_findings
|
|
2704
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
2705
|
+
).bind(projectId, environment.toLowerCase(), findingId).first() : await db.prepare(
|
|
2706
|
+
`SELECT * FROM investigation_findings
|
|
2707
|
+
WHERE project_id = ? AND id = ?
|
|
2708
|
+
ORDER BY created_at DESC
|
|
2709
|
+
LIMIT 1`
|
|
2710
|
+
).bind(projectId, findingId).first();
|
|
2711
|
+
return row ? mapFindingRow(row) : null;
|
|
2712
|
+
}
|
|
2713
|
+
async function searchInvestigationFindings(db, options) {
|
|
2714
|
+
const conditions = ["project_id = ?"];
|
|
2715
|
+
const binds = [options.projectId];
|
|
2716
|
+
if (options.environment) {
|
|
2717
|
+
conditions.push("environment = ?");
|
|
2718
|
+
binds.push(options.environment.toLowerCase());
|
|
2719
|
+
}
|
|
2720
|
+
if (options.fingerprint) {
|
|
2721
|
+
conditions.push("fingerprint = ?");
|
|
2722
|
+
binds.push(options.fingerprint);
|
|
2723
|
+
}
|
|
2724
|
+
if (options.symptomEntityId) {
|
|
2725
|
+
conditions.push("symptom_entity_id = ?");
|
|
2726
|
+
binds.push(options.symptomEntityId);
|
|
2727
|
+
}
|
|
2728
|
+
if (options.rootCauseEntityId) {
|
|
2729
|
+
conditions.push(
|
|
2730
|
+
"(root_cause_entity_id = ? OR fault_locus_entity_id = ?)"
|
|
2731
|
+
);
|
|
2732
|
+
binds.push(options.rootCauseEntityId, options.rootCauseEntityId);
|
|
2733
|
+
}
|
|
2734
|
+
if (options.causalVerificationStatus) {
|
|
2735
|
+
conditions.push("causal_verification_status = ?");
|
|
2736
|
+
binds.push(options.causalVerificationStatus);
|
|
2737
|
+
}
|
|
2738
|
+
if (options.incidentId) {
|
|
2739
|
+
conditions.push("incident_id = ?");
|
|
2740
|
+
binds.push(options.incidentId);
|
|
2741
|
+
}
|
|
2742
|
+
const limit = Math.max(1, Math.min(100, options.limit ?? 25));
|
|
2743
|
+
binds.push(limit);
|
|
2744
|
+
const rows = await db.prepare(
|
|
2745
|
+
`SELECT * FROM investigation_findings
|
|
2746
|
+
WHERE ${conditions.join(" AND ")}
|
|
2747
|
+
ORDER BY created_at DESC
|
|
2748
|
+
LIMIT ?`
|
|
2749
|
+
).bind(...binds).all();
|
|
2750
|
+
return rows.results.map(mapFindingRow);
|
|
2751
|
+
}
|
|
2752
|
+
var CounterfactualVerificationError = class extends Error {
|
|
2753
|
+
constructor(message) {
|
|
2754
|
+
super(message);
|
|
2755
|
+
this.name = "CounterfactualVerificationError";
|
|
2756
|
+
}
|
|
2757
|
+
};
|
|
2758
|
+
async function verifyPairedCounterfactualProof(db, projectId, findingId, input) {
|
|
2759
|
+
const finding = await getInvestigationFinding(
|
|
2760
|
+
db,
|
|
2761
|
+
projectId,
|
|
2762
|
+
findingId,
|
|
2763
|
+
input.environment
|
|
2764
|
+
);
|
|
2765
|
+
if (!finding) {
|
|
2766
|
+
throw new CounterfactualVerificationError(
|
|
2767
|
+
`Finding ${findingId} not found in project ${projectId}`
|
|
2768
|
+
);
|
|
2769
|
+
}
|
|
2770
|
+
const requiredTrials = Math.max(1, input.requiredTrials ?? 1);
|
|
2771
|
+
const uniquePassingIds = Array.from(new Set(input.passingRunIds));
|
|
2772
|
+
if (uniquePassingIds.length < requiredTrials) {
|
|
2773
|
+
throw new CounterfactualVerificationError(
|
|
2774
|
+
`Counterfactual requires at least ${requiredTrials} passing trial run(s), got ${uniquePassingIds.length}`
|
|
2775
|
+
);
|
|
2776
|
+
}
|
|
2777
|
+
if (uniquePassingIds.includes(input.failingRunId)) {
|
|
2778
|
+
throw new CounterfactualVerificationError(
|
|
2779
|
+
"failingRunId cannot also be listed in passingRunIds"
|
|
2780
|
+
);
|
|
2781
|
+
}
|
|
2782
|
+
const loadRunVerification = async (runId) => {
|
|
2783
|
+
const row = await db.prepare(
|
|
2784
|
+
`SELECT
|
|
2785
|
+
r.id AS run_id,
|
|
2786
|
+
COALESCE(cr.eval_case_id, r.eval_case_id) AS eval_case_id,
|
|
2787
|
+
r.service_version,
|
|
2788
|
+
r.commit_sha,
|
|
2789
|
+
r.candidate_agent_version,
|
|
2790
|
+
r.candidate_prompt_version,
|
|
2791
|
+
r.input_payload_hash AS run_input_hash,
|
|
2792
|
+
r.failure_fingerprint AS run_failure_fp,
|
|
2793
|
+
r.pass_count,
|
|
2794
|
+
r.fail_count,
|
|
2795
|
+
cr.passed AS result_passed,
|
|
2796
|
+
cr.input_payload_hash AS result_input_hash,
|
|
2797
|
+
cr.failure_fingerprint AS result_failure_fp
|
|
2798
|
+
FROM eval_runs r
|
|
2799
|
+
LEFT JOIN eval_case_results cr
|
|
2800
|
+
ON cr.project_id = r.project_id AND cr.run_id = r.id AND cr.eval_case_id = ?
|
|
2801
|
+
WHERE r.project_id = ? AND r.id = ?
|
|
2802
|
+
LIMIT 1`
|
|
2803
|
+
).bind(input.evalCaseId, projectId, runId).first();
|
|
2804
|
+
return row;
|
|
2805
|
+
};
|
|
2806
|
+
const failingRow = await loadRunVerification(input.failingRunId);
|
|
2807
|
+
if (!failingRow) {
|
|
2808
|
+
throw new CounterfactualVerificationError(
|
|
2809
|
+
`Failing eval run ${input.failingRunId} not found in project ${projectId}`
|
|
2810
|
+
);
|
|
2811
|
+
}
|
|
2812
|
+
if (failingRow.eval_case_id !== input.evalCaseId) {
|
|
2813
|
+
throw new CounterfactualVerificationError(
|
|
2814
|
+
`Failing eval run ${input.failingRunId} does not match evalCaseId ${input.evalCaseId}`
|
|
2815
|
+
);
|
|
2816
|
+
}
|
|
2817
|
+
const failingPassed = failingRow.result_passed !== null ? Number(failingRow.result_passed) === 1 : Number(failingRow.fail_count) === 0 && Number(failingRow.pass_count) > 0;
|
|
2818
|
+
if (failingPassed) {
|
|
2819
|
+
throw new CounterfactualVerificationError(
|
|
2820
|
+
`Baseline run ${input.failingRunId} did not fail (passed=1)`
|
|
2821
|
+
);
|
|
2822
|
+
}
|
|
2823
|
+
const failingInputHash = failingRow.result_input_hash ?? failingRow.run_input_hash;
|
|
2824
|
+
if (!failingInputHash) {
|
|
2825
|
+
throw new CounterfactualVerificationError(
|
|
2826
|
+
`Failing eval run ${input.failingRunId} is missing input_payload_hash`
|
|
2827
|
+
);
|
|
2828
|
+
}
|
|
2829
|
+
const failingFp = failingRow.result_failure_fp ?? failingRow.run_failure_fp;
|
|
2830
|
+
if (failingFp !== finding.fingerprint) {
|
|
2831
|
+
throw new CounterfactualVerificationError(
|
|
2832
|
+
`Failing run fingerprint (${failingFp ?? "null"}) does not match finding fingerprint (${finding.fingerprint})`
|
|
2833
|
+
);
|
|
2834
|
+
}
|
|
2835
|
+
const failingVersion = failingRow.service_version ?? failingRow.candidate_agent_version ?? failingRow.candidate_prompt_version;
|
|
2836
|
+
let passingVersion = null;
|
|
2837
|
+
let passingCommitSha = null;
|
|
2838
|
+
for (const passRunId of uniquePassingIds) {
|
|
2839
|
+
const passRow = await loadRunVerification(passRunId);
|
|
2840
|
+
if (!passRow) {
|
|
2841
|
+
throw new CounterfactualVerificationError(
|
|
2842
|
+
`Passing eval run ${passRunId} not found in project ${projectId}`
|
|
2843
|
+
);
|
|
2844
|
+
}
|
|
2845
|
+
if (passRow.eval_case_id !== input.evalCaseId) {
|
|
2846
|
+
throw new CounterfactualVerificationError(
|
|
2847
|
+
`Passing eval run ${passRunId} does not match evalCaseId ${input.evalCaseId}`
|
|
2848
|
+
);
|
|
2849
|
+
}
|
|
2850
|
+
const didPass = passRow.result_passed !== null ? Number(passRow.result_passed) === 1 : Number(passRow.pass_count) > 0 && Number(passRow.fail_count) === 0;
|
|
2851
|
+
if (!didPass) {
|
|
2852
|
+
throw new CounterfactualVerificationError(
|
|
2853
|
+
`Candidate run ${passRunId} did not pass`
|
|
2854
|
+
);
|
|
2855
|
+
}
|
|
2856
|
+
const passInputHash = passRow.result_input_hash ?? passRow.run_input_hash;
|
|
2857
|
+
if (passInputHash !== failingInputHash) {
|
|
2858
|
+
throw new CounterfactualVerificationError(
|
|
2859
|
+
`Candidate run ${passRunId} input_payload_hash (${passInputHash ?? "null"}) does not match failing run (${failingInputHash})`
|
|
2860
|
+
);
|
|
2861
|
+
}
|
|
2862
|
+
passingVersion = passRow.service_version ?? passRow.candidate_agent_version ?? passRow.candidate_prompt_version ?? passingVersion;
|
|
2863
|
+
passingCommitSha = passRow.commit_sha ?? passingCommitSha;
|
|
2864
|
+
}
|
|
2865
|
+
if (!failingVersion && !failingRow.commit_sha && !passingVersion && !passingCommitSha) {
|
|
2866
|
+
throw new CounterfactualVerificationError(
|
|
2867
|
+
"Counterfactual runs must record version, commit_sha, or candidate version lineage"
|
|
2868
|
+
);
|
|
2869
|
+
}
|
|
2870
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
2871
|
+
const proof = {
|
|
2872
|
+
evalCaseId: input.evalCaseId,
|
|
2873
|
+
inputPayloadHash: failingInputHash,
|
|
2874
|
+
failingRunId: input.failingRunId,
|
|
2875
|
+
failingVersion: failingVersion ?? null,
|
|
2876
|
+
failingCommitSha: failingRow.commit_sha ?? null,
|
|
2877
|
+
failureFingerprint: finding.fingerprint,
|
|
2878
|
+
passingRunIds: uniquePassingIds,
|
|
2879
|
+
passingVersion,
|
|
2880
|
+
passingCommitSha,
|
|
2881
|
+
requiredTrials,
|
|
2882
|
+
verifiedAt: nowIso
|
|
2883
|
+
};
|
|
2884
|
+
const updatedChain = [
|
|
2885
|
+
...finding.causalChain,
|
|
2886
|
+
{
|
|
2887
|
+
subjectKind: "finding",
|
|
2888
|
+
subjectId: finding.id,
|
|
2889
|
+
predicate: "verified_by_counterfactual",
|
|
2890
|
+
objectKind: "eval",
|
|
2891
|
+
objectId: input.evalCaseId,
|
|
2892
|
+
relationshipCertainty: "exact",
|
|
2893
|
+
causalProofTier: "counterfactual_verified",
|
|
2894
|
+
confidence: 0.99,
|
|
2895
|
+
reason: `Paired counterfactual verified on evalCase ${input.evalCaseId}: run ${input.failingRunId} (${failingVersion ?? "baseline"}) failed with ${finding.fingerprint} and ${uniquePassingIds.join(", ")} (${passingVersion ?? "candidate"}) passed on payload ${failingInputHash}`
|
|
2896
|
+
}
|
|
2897
|
+
];
|
|
2898
|
+
const updatedBreakdown = {
|
|
2899
|
+
...finding.confidenceBreakdown,
|
|
2900
|
+
counterfactualBonus: 0.25,
|
|
2901
|
+
finalConfidence: 0.99
|
|
2902
|
+
};
|
|
2903
|
+
await db.prepare(
|
|
2904
|
+
`UPDATE investigation_findings
|
|
2905
|
+
SET causal_proof_tier = 'counterfactual_verified',
|
|
2906
|
+
causal_verification_status = 'counterfactual_verified',
|
|
2907
|
+
confidence = 0.99,
|
|
2908
|
+
confidence_breakdown_json = ?,
|
|
2909
|
+
counterfactual_proof_json = ?,
|
|
2910
|
+
causal_chain_json = ?,
|
|
2911
|
+
updated_at = ?
|
|
2912
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
2913
|
+
).bind(
|
|
2914
|
+
JSON.stringify(updatedBreakdown),
|
|
2915
|
+
JSON.stringify(proof),
|
|
2916
|
+
JSON.stringify(updatedChain),
|
|
2917
|
+
nowIso,
|
|
2918
|
+
projectId,
|
|
2919
|
+
finding.environment,
|
|
2920
|
+
finding.id
|
|
2921
|
+
).run();
|
|
2922
|
+
const updated = await getInvestigationFinding(
|
|
2923
|
+
db,
|
|
2924
|
+
projectId,
|
|
2925
|
+
finding.id,
|
|
2926
|
+
finding.environment
|
|
2927
|
+
);
|
|
2928
|
+
if (!updated) {
|
|
2929
|
+
throw new Error(`Failed to reload finding ${finding.id}`);
|
|
2930
|
+
}
|
|
2931
|
+
return updated;
|
|
2932
|
+
}
|
|
2933
|
+
|
|
2934
|
+
// src/lib/webhook-url.ts
|
|
2935
|
+
var WebhookUrlError = class extends Error {
|
|
2936
|
+
};
|
|
2937
|
+
var parseIpv4 = (host) => {
|
|
2938
|
+
const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
|
|
2939
|
+
if (!m) return null;
|
|
2940
|
+
const octets = m.slice(1).map((o) => Number.parseInt(o, 10));
|
|
2941
|
+
return octets.every((o) => o <= 255) ? octets : null;
|
|
2942
|
+
};
|
|
2943
|
+
var isPrivateIpv4 = (o) => o[0] === 0 || // "this" network / unspecified
|
|
2944
|
+
o[0] === 10 || o[0] === 127 || o[0] === 100 && o[1] >= 64 && o[1] <= 127 || // CGNAT
|
|
2945
|
+
o[0] === 169 && o[1] === 254 || // link-local incl. cloud metadata
|
|
2946
|
+
o[0] === 172 && o[1] >= 16 && o[1] <= 31 || o[0] === 192 && o[1] === 168 || o[0] >= 224;
|
|
2947
|
+
var isPrivateIpv6 = (host) => {
|
|
2948
|
+
const literal = host.replace(/^\[|\]$/g, "").toLowerCase();
|
|
2949
|
+
if (literal === "::" || literal === "::1") return true;
|
|
2950
|
+
const dotted = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(literal);
|
|
2951
|
+
if (dotted) {
|
|
2952
|
+
const v4 = parseIpv4(dotted[1]);
|
|
2953
|
+
return v4 ? isPrivateIpv4(v4) : true;
|
|
2954
|
+
}
|
|
2955
|
+
const hex = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(literal);
|
|
2956
|
+
if (hex) {
|
|
2957
|
+
const hi = Number.parseInt(hex[1], 16);
|
|
2958
|
+
const lo = Number.parseInt(hex[2], 16);
|
|
2959
|
+
return isPrivateIpv4([hi >> 8, hi & 255, lo >> 8, lo & 255]);
|
|
2960
|
+
}
|
|
2961
|
+
const first = Number.parseInt(literal.split(":")[0] || "0", 16);
|
|
2962
|
+
if (Number.isNaN(first)) return true;
|
|
2963
|
+
if ((first & 65024) === 64512) return true;
|
|
2964
|
+
if ((first & 65472) === 65152) return true;
|
|
2965
|
+
if ((first & 65280) === 65280) return true;
|
|
2966
|
+
return false;
|
|
2967
|
+
};
|
|
2968
|
+
var isPrivateWebhookHost = (hostname) => {
|
|
2969
|
+
const host = hostname.toLowerCase().replace(/\.$/, "");
|
|
2970
|
+
if (!host) return true;
|
|
2971
|
+
if (host === "localhost" || host.endsWith(".localhost")) return true;
|
|
2972
|
+
if (host.endsWith(".local") || host.endsWith(".internal")) return true;
|
|
2973
|
+
if (host.startsWith("[")) return isPrivateIpv6(host);
|
|
2974
|
+
const v4 = parseIpv4(host);
|
|
2975
|
+
if (v4) return isPrivateIpv4(v4);
|
|
2976
|
+
return !host.includes(".");
|
|
2977
|
+
};
|
|
2978
|
+
var assertSafeWebhookUrl = (raw, options = {}) => {
|
|
2979
|
+
let url;
|
|
2980
|
+
try {
|
|
2981
|
+
url = new URL(raw);
|
|
2982
|
+
} catch {
|
|
2983
|
+
throw new WebhookUrlError("channel.url must be an absolute http(s) URL");
|
|
2984
|
+
}
|
|
2985
|
+
if (url.protocol !== "https:" && url.protocol !== "http:") {
|
|
2986
|
+
throw new WebhookUrlError("channel.url must be http(s)://");
|
|
2987
|
+
}
|
|
2988
|
+
if (url.username || url.password) {
|
|
2989
|
+
throw new WebhookUrlError(
|
|
2990
|
+
"channel.url must not embed credentials; use channel.headers"
|
|
2991
|
+
);
|
|
2992
|
+
}
|
|
2993
|
+
if (!options.allowPrivateNetworks && isPrivateWebhookHost(url.hostname)) {
|
|
2994
|
+
throw new WebhookUrlError(
|
|
2995
|
+
`channel.url host "${url.hostname}" is not a public address (set ALERT_WEBHOOK_ALLOW_PRIVATE=true to allow internal receivers)`
|
|
2996
|
+
);
|
|
2997
|
+
}
|
|
2998
|
+
return url;
|
|
2999
|
+
};
|
|
3000
|
+
var resolveWebhookHost = async (hostname, lookup) => {
|
|
3001
|
+
if (parseIpv4(hostname) || hostname.startsWith("[")) return "public";
|
|
3002
|
+
let addresses;
|
|
3003
|
+
try {
|
|
3004
|
+
addresses = await lookup(hostname);
|
|
3005
|
+
} catch {
|
|
3006
|
+
return "unresolvable";
|
|
3007
|
+
}
|
|
3008
|
+
if (addresses.length === 0) return "unresolvable";
|
|
3009
|
+
const isPrivate = addresses.some((address) => {
|
|
3010
|
+
const v4 = parseIpv4(address);
|
|
3011
|
+
if (v4) return isPrivateIpv4(v4);
|
|
3012
|
+
return isPrivateIpv6(`[${address}]`);
|
|
3013
|
+
});
|
|
3014
|
+
return isPrivate ? "private" : "public";
|
|
3015
|
+
};
|
|
3016
|
+
var redactWebhookUrl = (raw) => {
|
|
3017
|
+
try {
|
|
3018
|
+
const url = new URL(raw);
|
|
3019
|
+
const hasPath = url.pathname !== "/" || url.search !== "";
|
|
3020
|
+
return `${url.protocol}//${url.host}${hasPath ? "/\u2026" : ""}`;
|
|
3021
|
+
} catch {
|
|
3022
|
+
return "<invalid url>";
|
|
3023
|
+
}
|
|
3024
|
+
};
|
|
3025
|
+
|
|
3026
|
+
// src/lib/autosre-remediation-gate.ts
|
|
3027
|
+
var mapIncidentRow = (row) => ({
|
|
3028
|
+
id: row.id,
|
|
3029
|
+
projectId: row.project_id,
|
|
3030
|
+
environment: row.environment,
|
|
3031
|
+
dedupKey: row.dedup_key,
|
|
3032
|
+
status: row.status,
|
|
3033
|
+
targetNodeId: row.target_node_id,
|
|
3034
|
+
symptomAnchorKind: row.symptom_anchor_kind,
|
|
3035
|
+
symptomAnchorId: row.symptom_anchor_id,
|
|
3036
|
+
symptomFingerprint: row.symptom_fingerprint,
|
|
3037
|
+
alertCount: Number(row.alert_count ?? 1),
|
|
3038
|
+
latestBundle: row.latest_bundle_json ? parseJsonRecord(row.latest_bundle_json) : null,
|
|
3039
|
+
leaseOwnerId: row.lease_owner_id,
|
|
3040
|
+
leaseExpiresAt: row.lease_expires_at,
|
|
3041
|
+
dispatchAttempts: Number(row.dispatch_attempts ?? 0),
|
|
3042
|
+
lastDispatchedAt: row.last_dispatched_at,
|
|
3043
|
+
openedAt: row.opened_at,
|
|
3044
|
+
updatedAt: row.updated_at,
|
|
3045
|
+
resolvedAt: row.resolved_at,
|
|
3046
|
+
expiresAt: row.expires_at
|
|
3047
|
+
});
|
|
3048
|
+
async function openOrDeduplicateIncident(db, projectId, input) {
|
|
3049
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
3050
|
+
const environment = (input.environment || "default").trim().toLowerCase();
|
|
3051
|
+
const category = (input.symptomCategory ?? input.symptomFingerprint ?? `${input.symptomAnchorKind}:${input.symptomAnchorId}`).trim();
|
|
3052
|
+
const dedupKey = await sha256Hex(
|
|
3053
|
+
`${projectId}::${environment}::${input.targetNodeId}::${category}`
|
|
3054
|
+
);
|
|
3055
|
+
const existing = await db.prepare(
|
|
3056
|
+
`SELECT * FROM incident_investigations
|
|
3057
|
+
WHERE project_id = ?
|
|
3058
|
+
AND environment = ?
|
|
3059
|
+
AND dedup_key = ?
|
|
3060
|
+
AND status IN ('open', 'dispatched', 'leased', 'remediating', 'verifying')
|
|
3061
|
+
ORDER BY opened_at DESC
|
|
3062
|
+
LIMIT 1`
|
|
3063
|
+
).bind(projectId, environment, dedupKey).first();
|
|
3064
|
+
if (existing) {
|
|
3065
|
+
await db.prepare(
|
|
3066
|
+
`UPDATE incident_investigations
|
|
3067
|
+
SET alert_count = alert_count + 1,
|
|
3068
|
+
latest_bundle_json = COALESCE(?, latest_bundle_json),
|
|
3069
|
+
symptom_fingerprint = COALESCE(?, symptom_fingerprint),
|
|
3070
|
+
updated_at = ?
|
|
3071
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3072
|
+
).bind(
|
|
3073
|
+
input.latestBundle ? JSON.stringify(input.latestBundle) : null,
|
|
3074
|
+
input.symptomFingerprint ?? null,
|
|
3075
|
+
nowIso,
|
|
3076
|
+
projectId,
|
|
3077
|
+
environment,
|
|
3078
|
+
existing.id
|
|
3079
|
+
).run();
|
|
3080
|
+
const updated = await getIncidentInvestigation(
|
|
3081
|
+
db,
|
|
3082
|
+
projectId,
|
|
3083
|
+
existing.id,
|
|
3084
|
+
environment
|
|
3085
|
+
);
|
|
3086
|
+
return {
|
|
3087
|
+
incident: updated ?? mapIncidentRow(existing),
|
|
3088
|
+
deduplicated: true
|
|
3089
|
+
};
|
|
3090
|
+
}
|
|
3091
|
+
const id = input.id?.trim() || `inc_${randomHex(10)}`;
|
|
3092
|
+
const expiresAt = new Date(
|
|
3093
|
+
Date.now() + 30 * 24 * 36e5
|
|
3094
|
+
).toISOString();
|
|
3095
|
+
await db.prepare(
|
|
3096
|
+
`INSERT INTO incident_investigations (
|
|
3097
|
+
project_id, environment, id, dedup_key, status, target_node_id,
|
|
3098
|
+
symptom_anchor_kind, symptom_anchor_id, symptom_fingerprint,
|
|
3099
|
+
alert_count, latest_bundle_json, dispatch_attempts,
|
|
3100
|
+
opened_at, updated_at, expires_at
|
|
3101
|
+
) VALUES (?, ?, ?, ?, 'open', ?, ?, ?, ?, 1, ?, 0, ?, ?, ?)`
|
|
3102
|
+
).bind(
|
|
3103
|
+
projectId,
|
|
3104
|
+
environment,
|
|
3105
|
+
id,
|
|
3106
|
+
dedupKey,
|
|
3107
|
+
input.targetNodeId,
|
|
3108
|
+
input.symptomAnchorKind,
|
|
3109
|
+
input.symptomAnchorId,
|
|
3110
|
+
input.symptomFingerprint ?? null,
|
|
3111
|
+
input.latestBundle ? JSON.stringify(input.latestBundle) : null,
|
|
3112
|
+
nowIso,
|
|
3113
|
+
nowIso,
|
|
3114
|
+
expiresAt
|
|
3115
|
+
).run();
|
|
3116
|
+
const created = await getIncidentInvestigation(db, projectId, id, environment);
|
|
3117
|
+
if (!created) {
|
|
3118
|
+
throw new Error(`Failed to load created incident ${id}`);
|
|
3119
|
+
}
|
|
3120
|
+
return { incident: created, deduplicated: false };
|
|
3121
|
+
}
|
|
3122
|
+
async function getIncidentInvestigation(db, projectId, incidentId, environment) {
|
|
3123
|
+
const row = environment ? await db.prepare(
|
|
3124
|
+
`SELECT * FROM incident_investigations
|
|
3125
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3126
|
+
).bind(projectId, environment.toLowerCase(), incidentId).first() : await db.prepare(
|
|
3127
|
+
`SELECT * FROM incident_investigations
|
|
3128
|
+
WHERE project_id = ? AND id = ?
|
|
3129
|
+
ORDER BY opened_at DESC
|
|
3130
|
+
LIMIT 1`
|
|
3131
|
+
).bind(projectId, incidentId).first();
|
|
3132
|
+
return row ? mapIncidentRow(row) : null;
|
|
3133
|
+
}
|
|
3134
|
+
async function listIncidentInvestigations(db, options) {
|
|
3135
|
+
const conditions = ["project_id = ?"];
|
|
3136
|
+
const binds = [options.projectId];
|
|
3137
|
+
if (options.environment) {
|
|
3138
|
+
conditions.push("environment = ?");
|
|
3139
|
+
binds.push(options.environment.toLowerCase());
|
|
3140
|
+
}
|
|
3141
|
+
if (options.status) {
|
|
3142
|
+
conditions.push("status = ?");
|
|
3143
|
+
binds.push(options.status);
|
|
3144
|
+
}
|
|
3145
|
+
const limit = Math.max(1, Math.min(100, options.limit ?? 50));
|
|
3146
|
+
binds.push(limit);
|
|
3147
|
+
const rows = await db.prepare(
|
|
3148
|
+
`SELECT * FROM incident_investigations
|
|
3149
|
+
WHERE ${conditions.join(" AND ")}
|
|
3150
|
+
ORDER BY opened_at DESC
|
|
3151
|
+
LIMIT ?`
|
|
3152
|
+
).bind(...binds).all();
|
|
3153
|
+
return rows.results.map(mapIncidentRow);
|
|
3154
|
+
}
|
|
3155
|
+
async function acquireIncidentLease(db, projectId, incidentId, input) {
|
|
3156
|
+
const nowIso = input.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
3157
|
+
const incident = await getIncidentInvestigation(
|
|
3158
|
+
db,
|
|
3159
|
+
projectId,
|
|
3160
|
+
incidentId,
|
|
3161
|
+
input.environment
|
|
3162
|
+
);
|
|
3163
|
+
if (!incident) {
|
|
3164
|
+
return {
|
|
3165
|
+
acquired: false,
|
|
3166
|
+
reason: `Incident ${incidentId} not found`,
|
|
3167
|
+
conflictOwnerId: null,
|
|
3168
|
+
leaseExpiresAt: null
|
|
3169
|
+
};
|
|
3170
|
+
}
|
|
3171
|
+
if (incident.leaseOwnerId && incident.leaseOwnerId !== input.runnerId && incident.leaseExpiresAt && incident.leaseExpiresAt > nowIso) {
|
|
3172
|
+
return {
|
|
3173
|
+
acquired: false,
|
|
3174
|
+
reason: `Incident ${incidentId} is already leased by runner ${incident.leaseOwnerId} until ${incident.leaseExpiresAt}`,
|
|
3175
|
+
conflictOwnerId: incident.leaseOwnerId,
|
|
3176
|
+
leaseExpiresAt: incident.leaseExpiresAt
|
|
3177
|
+
};
|
|
3178
|
+
}
|
|
3179
|
+
const durationSec = Math.max(
|
|
3180
|
+
15,
|
|
3181
|
+
Math.min(600, input.leaseDurationSeconds ?? 120)
|
|
3182
|
+
);
|
|
3183
|
+
const leaseExpiresAt = new Date(
|
|
3184
|
+
Date.parse(nowIso) + durationSec * 1e3
|
|
3185
|
+
).toISOString();
|
|
3186
|
+
const nextStatus = incident.status === "open" || incident.status === "dispatched" ? "leased" : incident.status;
|
|
3187
|
+
await db.prepare(
|
|
3188
|
+
`UPDATE incident_investigations
|
|
3189
|
+
SET lease_owner_id = ?,
|
|
3190
|
+
lease_expires_at = ?,
|
|
3191
|
+
status = ?,
|
|
3192
|
+
updated_at = ?
|
|
3193
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3194
|
+
).bind(
|
|
3195
|
+
input.runnerId,
|
|
3196
|
+
leaseExpiresAt,
|
|
3197
|
+
nextStatus,
|
|
3198
|
+
nowIso,
|
|
3199
|
+
projectId,
|
|
3200
|
+
incident.environment,
|
|
3201
|
+
incident.id
|
|
3202
|
+
).run();
|
|
3203
|
+
const updated = await getIncidentInvestigation(
|
|
3204
|
+
db,
|
|
3205
|
+
projectId,
|
|
3206
|
+
incident.id,
|
|
3207
|
+
incident.environment
|
|
3208
|
+
);
|
|
3209
|
+
return {
|
|
3210
|
+
acquired: true,
|
|
3211
|
+
incident: updated ?? {
|
|
3212
|
+
...incident,
|
|
3213
|
+
leaseOwnerId: input.runnerId,
|
|
3214
|
+
leaseExpiresAt,
|
|
3215
|
+
status: nextStatus,
|
|
3216
|
+
updatedAt: nowIso
|
|
3217
|
+
}
|
|
3218
|
+
};
|
|
3219
|
+
}
|
|
3220
|
+
async function computeHmacSha256Hex(secret, body) {
|
|
3221
|
+
const enc = new TextEncoder();
|
|
3222
|
+
const key = await crypto.subtle.importKey(
|
|
3223
|
+
"raw",
|
|
3224
|
+
enc.encode(secret),
|
|
3225
|
+
{ name: "HMAC", hash: "SHA-256" },
|
|
3226
|
+
false,
|
|
3227
|
+
["sign"]
|
|
3228
|
+
);
|
|
3229
|
+
const sig = await crypto.subtle.sign("HMAC", key, enc.encode(body));
|
|
3230
|
+
const bytes = new Uint8Array(sig);
|
|
3231
|
+
let hex = "";
|
|
3232
|
+
for (const b of bytes) hex += b.toString(16).padStart(2, "0");
|
|
3233
|
+
return hex;
|
|
3234
|
+
}
|
|
3235
|
+
async function dispatchIncidentWebhook(db, projectId, incidentId, webhook) {
|
|
3236
|
+
const incident = await getIncidentInvestigation(db, projectId, incidentId);
|
|
3237
|
+
if (!incident) {
|
|
3238
|
+
return { delivered: false, error: "Incident not found" };
|
|
3239
|
+
}
|
|
3240
|
+
if (incident.dispatchAttempts >= 5) {
|
|
3241
|
+
return { delivered: false, error: "MAX_DISPATCH_ATTEMPTS (5) exceeded" };
|
|
3242
|
+
}
|
|
3243
|
+
const safeUrl = assertSafeWebhookUrl(webhook.url, {
|
|
3244
|
+
allowPrivateNetworks: webhook.allowPrivateNetworks
|
|
3245
|
+
});
|
|
3246
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
3247
|
+
const body = JSON.stringify({
|
|
3248
|
+
event: "autosre.incident.opened",
|
|
3249
|
+
incidentId: incident.id,
|
|
3250
|
+
projectId: incident.projectId,
|
|
3251
|
+
environment: incident.environment,
|
|
3252
|
+
targetNodeId: incident.targetNodeId,
|
|
3253
|
+
dedupKey: incident.dedupKey,
|
|
3254
|
+
symptomAnchor: {
|
|
3255
|
+
kind: incident.symptomAnchorKind,
|
|
3256
|
+
id: incident.symptomAnchorId
|
|
3257
|
+
},
|
|
3258
|
+
bundle: incident.latestBundle,
|
|
3259
|
+
dispatchedAt: nowIso
|
|
3260
|
+
});
|
|
3261
|
+
const headers = {
|
|
3262
|
+
"Content-Type": "application/json",
|
|
3263
|
+
"X-Obs-Incident-Id": incident.id,
|
|
3264
|
+
"X-Obs-Deduplication-Key": incident.dedupKey
|
|
3265
|
+
};
|
|
3266
|
+
if (webhook.secret) {
|
|
3267
|
+
const hmac = await computeHmacSha256Hex(webhook.secret, body);
|
|
3268
|
+
headers["X-Obs-Signature-256"] = `sha256=${hmac}`;
|
|
3269
|
+
}
|
|
3270
|
+
const fetchFn = webhook.fetchImpl ?? fetch;
|
|
3271
|
+
let delivered = false;
|
|
3272
|
+
let status;
|
|
3273
|
+
let errorMsg;
|
|
3274
|
+
try {
|
|
3275
|
+
const res = await fetchFn(safeUrl.toString(), {
|
|
3276
|
+
method: "POST",
|
|
3277
|
+
headers,
|
|
3278
|
+
body
|
|
3279
|
+
});
|
|
3280
|
+
status = res.status;
|
|
3281
|
+
delivered = res.ok;
|
|
3282
|
+
if (!res.ok) {
|
|
3283
|
+
errorMsg = `HTTP ${res.status}`;
|
|
3284
|
+
}
|
|
3285
|
+
} catch (err) {
|
|
3286
|
+
errorMsg = err instanceof Error ? err.message : String(err);
|
|
3287
|
+
}
|
|
3288
|
+
await db.prepare(
|
|
3289
|
+
`UPDATE incident_investigations
|
|
3290
|
+
SET dispatch_attempts = dispatch_attempts + 1,
|
|
3291
|
+
last_dispatched_at = ?,
|
|
3292
|
+
status = CASE WHEN ? = 1 AND status = 'open' THEN 'dispatched' ELSE status END,
|
|
3293
|
+
updated_at = ?
|
|
3294
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3295
|
+
).bind(
|
|
3296
|
+
nowIso,
|
|
3297
|
+
delivered ? 1 : 0,
|
|
3298
|
+
nowIso,
|
|
3299
|
+
projectId,
|
|
3300
|
+
incident.environment,
|
|
3301
|
+
incident.id
|
|
3302
|
+
).run();
|
|
3303
|
+
return { delivered, status, error: errorMsg };
|
|
3304
|
+
}
|
|
3305
|
+
var mapProposalRow = (row, grantToken) => ({
|
|
3306
|
+
id: row.id,
|
|
3307
|
+
projectId: row.project_id,
|
|
3308
|
+
environment: row.environment,
|
|
3309
|
+
incidentId: row.incident_id,
|
|
3310
|
+
findingId: row.finding_id,
|
|
3311
|
+
idempotencyKey: row.idempotency_key,
|
|
3312
|
+
actionKind: row.action_kind,
|
|
3313
|
+
targetNodeId: row.target_node_id,
|
|
3314
|
+
expectedCurrentState: parseJsonRecord(
|
|
3315
|
+
row.expected_current_state_json
|
|
3316
|
+
),
|
|
3317
|
+
desiredTargetState: parseJsonRecord(
|
|
3318
|
+
row.desired_target_state_json
|
|
3319
|
+
),
|
|
3320
|
+
requestedAutonomy: row.requested_autonomy,
|
|
3321
|
+
effectiveAutonomy: row.effective_autonomy,
|
|
3322
|
+
policyDecisionReason: row.policy_decision_reason,
|
|
3323
|
+
status: row.status,
|
|
3324
|
+
grantExpiresAt: row.grant_expires_at,
|
|
3325
|
+
approvedByUserId: row.approved_by_user_id,
|
|
3326
|
+
approvedAt: row.approved_at,
|
|
3327
|
+
claimedByRunnerId: row.claimed_by_runner_id,
|
|
3328
|
+
claimedAt: row.claimed_at,
|
|
3329
|
+
claimExpiresAt: row.claim_expires_at,
|
|
3330
|
+
remediationAgentRunId: row.remediation_agent_run_id,
|
|
3331
|
+
resultingChangeId: row.resulting_change_id,
|
|
3332
|
+
failureReason: row.failure_reason,
|
|
3333
|
+
createdAt: row.created_at,
|
|
3334
|
+
updatedAt: row.updated_at,
|
|
3335
|
+
expiresAt: row.expires_at,
|
|
3336
|
+
...grantToken ? { grantToken } : {}
|
|
3337
|
+
});
|
|
3338
|
+
function evaluateActionPolicyMatrix(actionKind, causalProofTier, recentProposalsLastHour) {
|
|
3339
|
+
if (recentProposalsLastHour >= 3) {
|
|
3340
|
+
return {
|
|
3341
|
+
effectiveAutonomy: AgentAutonomyLevel.BlockedByPolicy,
|
|
3342
|
+
reason: `Circuit breaker tripped: ${recentProposalsLastHour} remediation proposals already recorded in the last hour for this target/fingerprint (max 3/hr)`
|
|
3343
|
+
};
|
|
3344
|
+
}
|
|
3345
|
+
switch (actionKind) {
|
|
3346
|
+
case "open_code_pr": {
|
|
3347
|
+
if (causalProofTier === "counterfactual_verified" || causalProofTier === "cohort_rate_isolated" || causalProofTier === "corroborated_candidate") {
|
|
3348
|
+
return {
|
|
3349
|
+
effectiveAutonomy: AgentAutonomyLevel.AutonomousWrite,
|
|
3350
|
+
reason: `Non-merging PR creation permitted as autonomous_write for causalProofTier=${causalProofTier}`
|
|
3351
|
+
};
|
|
3352
|
+
}
|
|
3353
|
+
return {
|
|
3354
|
+
effectiveAutonomy: AgentAutonomyLevel.HumanApprovedWrite,
|
|
3355
|
+
reason: `open_code_pr requires human_approved_write when causalProofTier=${causalProofTier}`
|
|
3356
|
+
};
|
|
3357
|
+
}
|
|
3358
|
+
case "toggle_feature_flag":
|
|
3359
|
+
case "rollback_deploy": {
|
|
3360
|
+
if (causalProofTier === "counterfactual_verified" || causalProofTier === "cohort_rate_isolated") {
|
|
3361
|
+
return {
|
|
3362
|
+
effectiveAutonomy: AgentAutonomyLevel.AutonomousWrite,
|
|
3363
|
+
reason: `${actionKind} authorized as autonomous_write with causalProofTier=${causalProofTier}`
|
|
3364
|
+
};
|
|
3365
|
+
}
|
|
3366
|
+
if (causalProofTier === "corroborated_candidate") {
|
|
3367
|
+
return {
|
|
3368
|
+
effectiveAutonomy: AgentAutonomyLevel.HumanApprovedWrite,
|
|
3369
|
+
reason: `${actionKind} with causalProofTier=corroborated_candidate (structural overlap or sequential cutover without concurrent cohort isolation) requires human_approved_write`
|
|
3370
|
+
};
|
|
3371
|
+
}
|
|
3372
|
+
return {
|
|
3373
|
+
effectiveAutonomy: AgentAutonomyLevel.BlockedByPolicy,
|
|
3374
|
+
reason: `${actionKind} blocked_by_policy: causalProofTier=${causalProofTier} (unverified fault locus cannot authorize production ${actionKind})`
|
|
3375
|
+
};
|
|
3376
|
+
}
|
|
3377
|
+
case "apply_config":
|
|
3378
|
+
case "promote_agent_prompt": {
|
|
3379
|
+
if (causalProofTier === "counterfactual_verified") {
|
|
3380
|
+
return {
|
|
3381
|
+
effectiveAutonomy: AgentAutonomyLevel.AutonomousWrite,
|
|
3382
|
+
reason: `${actionKind} authorized as autonomous_write with counterfactual_verified proof`
|
|
3383
|
+
};
|
|
3384
|
+
}
|
|
3385
|
+
if (causalProofTier === "cohort_rate_isolated" || causalProofTier === "corroborated_candidate") {
|
|
3386
|
+
return {
|
|
3387
|
+
effectiveAutonomy: AgentAutonomyLevel.HumanApprovedWrite,
|
|
3388
|
+
reason: `${actionKind} requires counterfactual_verified for autonomous_write; parked for human_approved_write (tier=${causalProofTier})`
|
|
3389
|
+
};
|
|
3390
|
+
}
|
|
3391
|
+
return {
|
|
3392
|
+
effectiveAutonomy: AgentAutonomyLevel.BlockedByPolicy,
|
|
3393
|
+
reason: `${actionKind} blocked_by_policy: causalProofTier=${causalProofTier}`
|
|
3394
|
+
};
|
|
3395
|
+
}
|
|
3396
|
+
}
|
|
3397
|
+
}
|
|
3398
|
+
var RemediationGateError = class extends Error {
|
|
3399
|
+
constructor(message, statusCode = 400) {
|
|
3400
|
+
super(message);
|
|
3401
|
+
this.statusCode = statusCode;
|
|
3402
|
+
this.name = "RemediationGateError";
|
|
3403
|
+
}
|
|
3404
|
+
};
|
|
3405
|
+
async function proposeRemediationAction(db, projectId, input) {
|
|
3406
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
3407
|
+
const environment = (input.environment || "default").trim().toLowerCase();
|
|
3408
|
+
const existing = await db.prepare(
|
|
3409
|
+
`SELECT * FROM remediation_proposals
|
|
3410
|
+
WHERE project_id = ? AND environment = ? AND idempotency_key = ?`
|
|
3411
|
+
).bind(projectId, environment, input.idempotencyKey).first();
|
|
3412
|
+
if (existing) {
|
|
3413
|
+
if (existing.action_kind !== input.actionKind || existing.target_node_id !== input.targetNodeId) {
|
|
3414
|
+
throw new RemediationGateError(
|
|
3415
|
+
`Idempotency key ${input.idempotencyKey} already used for a different action or target node`,
|
|
3416
|
+
409
|
|
3417
|
+
);
|
|
3418
|
+
}
|
|
3419
|
+
return mapProposalRow(existing);
|
|
3420
|
+
}
|
|
3421
|
+
const finding = await getInvestigationFinding(
|
|
3422
|
+
db,
|
|
3423
|
+
projectId,
|
|
3424
|
+
input.findingId,
|
|
3425
|
+
environment
|
|
3426
|
+
);
|
|
3427
|
+
if (!finding) {
|
|
3428
|
+
throw new RemediationGateError(
|
|
3429
|
+
`Finding ${input.findingId} not found in environment ${environment}`,
|
|
3430
|
+
404
|
|
3431
|
+
);
|
|
3432
|
+
}
|
|
3433
|
+
const oneHourAgo = new Date(Date.now() - 36e5).toISOString();
|
|
3434
|
+
const recentCountRow = await db.prepare(
|
|
3435
|
+
`SELECT COUNT(*) AS cnt FROM remediation_proposals
|
|
3436
|
+
WHERE project_id = ?
|
|
3437
|
+
AND environment = ?
|
|
3438
|
+
AND target_node_id = ?
|
|
3439
|
+
AND created_at >= ?`
|
|
3440
|
+
).bind(projectId, environment, input.targetNodeId, oneHourAgo).first();
|
|
3441
|
+
const recentCount = Number(recentCountRow?.cnt ?? 0);
|
|
3442
|
+
const requestedAutonomy = input.requestedAutonomy ?? AgentAutonomyLevel.AutonomousWrite;
|
|
3443
|
+
const policy = evaluateActionPolicyMatrix(
|
|
3444
|
+
input.actionKind,
|
|
3445
|
+
finding.causalProofTier,
|
|
3446
|
+
recentCount
|
|
3447
|
+
);
|
|
3448
|
+
let effectiveAutonomy = policy.effectiveAutonomy;
|
|
3449
|
+
let policyReason = policy.reason;
|
|
3450
|
+
if (requestedAutonomy === AgentAutonomyLevel.HumanApprovedWrite && effectiveAutonomy === AgentAutonomyLevel.AutonomousWrite) {
|
|
3451
|
+
effectiveAutonomy = AgentAutonomyLevel.HumanApprovedWrite;
|
|
3452
|
+
policyReason = `Caller explicitly requested human_approved_write (${policy.reason})`;
|
|
3453
|
+
}
|
|
3454
|
+
let status = "pending_human_approval";
|
|
3455
|
+
let grantToken;
|
|
3456
|
+
let grantTokenHash = null;
|
|
3457
|
+
let grantExpiresAt = null;
|
|
3458
|
+
if (effectiveAutonomy === AgentAutonomyLevel.AutonomousWrite) {
|
|
3459
|
+
status = "grant_issued";
|
|
3460
|
+
grantToken = `rgt_${randomHex(16)}`;
|
|
3461
|
+
grantTokenHash = await sha256Hex(grantToken);
|
|
3462
|
+
grantExpiresAt = new Date(Date.now() + 5 * 6e4).toISOString();
|
|
3463
|
+
} else if (effectiveAutonomy === AgentAutonomyLevel.BlockedByPolicy) {
|
|
3464
|
+
status = "rejected";
|
|
3465
|
+
}
|
|
3466
|
+
const id = `rem_${randomHex(10)}`;
|
|
3467
|
+
const expiresAt = new Date(
|
|
3468
|
+
Date.now() + 30 * 24 * 36e5
|
|
3469
|
+
).toISOString();
|
|
3470
|
+
await db.prepare(
|
|
3471
|
+
`INSERT INTO remediation_proposals (
|
|
3472
|
+
project_id, environment, id, incident_id, finding_id,
|
|
3473
|
+
idempotency_key, action_kind, target_node_id,
|
|
3474
|
+
expected_current_state_json, desired_target_state_json,
|
|
3475
|
+
requested_autonomy, effective_autonomy, policy_decision_reason,
|
|
3476
|
+
status, grant_token_hash, grant_expires_at,
|
|
3477
|
+
created_at, updated_at, expires_at
|
|
3478
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3479
|
+
).bind(
|
|
3480
|
+
projectId,
|
|
3481
|
+
environment,
|
|
3482
|
+
id,
|
|
3483
|
+
input.incidentId,
|
|
3484
|
+
input.findingId,
|
|
3485
|
+
input.idempotencyKey,
|
|
3486
|
+
input.actionKind,
|
|
3487
|
+
input.targetNodeId,
|
|
3488
|
+
JSON.stringify(input.expectedCurrentState),
|
|
3489
|
+
JSON.stringify(input.desiredTargetState),
|
|
3490
|
+
requestedAutonomy,
|
|
3491
|
+
effectiveAutonomy,
|
|
3492
|
+
policyReason,
|
|
3493
|
+
status,
|
|
3494
|
+
grantTokenHash,
|
|
3495
|
+
grantExpiresAt,
|
|
3496
|
+
nowIso,
|
|
3497
|
+
nowIso,
|
|
3498
|
+
expiresAt
|
|
3499
|
+
).run();
|
|
3500
|
+
const created = await db.prepare(
|
|
3501
|
+
`SELECT * FROM remediation_proposals
|
|
3502
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3503
|
+
).bind(projectId, environment, id).first();
|
|
3504
|
+
if (!created) {
|
|
3505
|
+
throw new Error(`Failed to load created remediation proposal ${id}`);
|
|
3506
|
+
}
|
|
3507
|
+
return mapProposalRow(created, grantToken);
|
|
3508
|
+
}
|
|
3509
|
+
async function getRemediationProposal(db, projectId, proposalId, environment) {
|
|
3510
|
+
const row = environment ? await db.prepare(
|
|
3511
|
+
`SELECT * FROM remediation_proposals
|
|
3512
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3513
|
+
).bind(projectId, environment.toLowerCase(), proposalId).first() : await db.prepare(
|
|
3514
|
+
`SELECT * FROM remediation_proposals
|
|
3515
|
+
WHERE project_id = ? AND id = ?
|
|
3516
|
+
ORDER BY created_at DESC
|
|
3517
|
+
LIMIT 1`
|
|
3518
|
+
).bind(projectId, proposalId).first();
|
|
3519
|
+
return row ? mapProposalRow(row) : null;
|
|
3520
|
+
}
|
|
3521
|
+
async function listRemediationProposals(db, options) {
|
|
3522
|
+
const conditions = ["project_id = ?"];
|
|
3523
|
+
const binds = [options.projectId];
|
|
3524
|
+
if (options.environment) {
|
|
3525
|
+
conditions.push("environment = ?");
|
|
3526
|
+
binds.push(options.environment.toLowerCase());
|
|
3527
|
+
}
|
|
3528
|
+
if (options.status) {
|
|
3529
|
+
conditions.push("status = ?");
|
|
3530
|
+
binds.push(options.status);
|
|
3531
|
+
}
|
|
3532
|
+
if (options.incidentId) {
|
|
3533
|
+
conditions.push("incident_id = ?");
|
|
3534
|
+
binds.push(options.incidentId);
|
|
3535
|
+
}
|
|
3536
|
+
if (options.findingId) {
|
|
3537
|
+
conditions.push("finding_id = ?");
|
|
3538
|
+
binds.push(options.findingId);
|
|
3539
|
+
}
|
|
3540
|
+
const limit = Math.max(1, Math.min(100, options.limit ?? 50));
|
|
3541
|
+
binds.push(limit);
|
|
3542
|
+
const rows = await db.prepare(
|
|
3543
|
+
`SELECT * FROM remediation_proposals
|
|
3544
|
+
WHERE ${conditions.join(" AND ")}
|
|
3545
|
+
ORDER BY created_at DESC
|
|
3546
|
+
LIMIT ?`
|
|
3547
|
+
).bind(...binds).all();
|
|
3548
|
+
return rows.results.map((r) => mapProposalRow(r));
|
|
3549
|
+
}
|
|
3550
|
+
async function approveRemediationProposal(db, projectId, proposalId, options) {
|
|
3551
|
+
const row = options.environment ? await db.prepare(
|
|
3552
|
+
`SELECT * FROM remediation_proposals
|
|
3553
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3554
|
+
).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
|
|
3555
|
+
`SELECT * FROM remediation_proposals
|
|
3556
|
+
WHERE project_id = ? AND id = ?
|
|
3557
|
+
LIMIT 1`
|
|
3558
|
+
).bind(projectId, proposalId).first();
|
|
3559
|
+
if (!row) {
|
|
3560
|
+
throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
|
|
3561
|
+
}
|
|
3562
|
+
if (row.status !== "pending_human_approval") {
|
|
3563
|
+
throw new RemediationGateError(
|
|
3564
|
+
`Proposal ${proposalId} cannot be approved from status=${row.status}`,
|
|
3565
|
+
409
|
|
3566
|
+
);
|
|
3567
|
+
}
|
|
3568
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
3569
|
+
const grantToken = `rgt_${randomHex(16)}`;
|
|
3570
|
+
const grantTokenHash = await sha256Hex(grantToken);
|
|
3571
|
+
const grantExpiresAt = new Date(Date.now() + 5 * 6e4).toISOString();
|
|
3572
|
+
await db.prepare(
|
|
3573
|
+
`UPDATE remediation_proposals
|
|
3574
|
+
SET status = 'grant_issued',
|
|
3575
|
+
grant_token_hash = ?,
|
|
3576
|
+
grant_expires_at = ?,
|
|
3577
|
+
approved_by_user_id = ?,
|
|
3578
|
+
approved_at = ?,
|
|
3579
|
+
updated_at = ?
|
|
3580
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3581
|
+
).bind(
|
|
3582
|
+
grantTokenHash,
|
|
3583
|
+
grantExpiresAt,
|
|
3584
|
+
options.approvedByUserId,
|
|
3585
|
+
nowIso,
|
|
3586
|
+
nowIso,
|
|
3587
|
+
projectId,
|
|
3588
|
+
row.environment,
|
|
3589
|
+
row.id
|
|
3590
|
+
).run();
|
|
3591
|
+
const updated = await db.prepare(
|
|
3592
|
+
`SELECT * FROM remediation_proposals
|
|
3593
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3594
|
+
).bind(projectId, row.environment, row.id).first();
|
|
3595
|
+
return mapProposalRow(updated ?? row, grantToken);
|
|
3596
|
+
}
|
|
3597
|
+
async function rejectRemediationProposal(db, projectId, proposalId, options) {
|
|
3598
|
+
const row = options.environment ? await db.prepare(
|
|
3599
|
+
`SELECT * FROM remediation_proposals
|
|
3600
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3601
|
+
).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
|
|
3602
|
+
`SELECT * FROM remediation_proposals
|
|
3603
|
+
WHERE project_id = ? AND id = ?
|
|
3604
|
+
LIMIT 1`
|
|
3605
|
+
).bind(projectId, proposalId).first();
|
|
3606
|
+
if (!row) {
|
|
3607
|
+
throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
|
|
3608
|
+
}
|
|
3609
|
+
if (row.status !== "pending_human_approval" && row.status !== "grant_issued") {
|
|
3610
|
+
throw new RemediationGateError(
|
|
3611
|
+
`Proposal ${proposalId} cannot be rejected from status=${row.status}`,
|
|
3612
|
+
409
|
|
3613
|
+
);
|
|
3614
|
+
}
|
|
3615
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
3616
|
+
const reason = options.reason ? `Rejected by ${options.rejectedByUserId}: ${options.reason}` : `Rejected by ${options.rejectedByUserId}`;
|
|
3617
|
+
await db.prepare(
|
|
3618
|
+
`UPDATE remediation_proposals
|
|
3619
|
+
SET status = 'rejected',
|
|
3620
|
+
grant_token_hash = NULL,
|
|
3621
|
+
failure_reason = ?,
|
|
3622
|
+
updated_at = ?
|
|
3623
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3624
|
+
).bind(reason, nowIso, projectId, row.environment, row.id).run();
|
|
3625
|
+
const updated = await db.prepare(
|
|
3626
|
+
`SELECT * FROM remediation_proposals
|
|
3627
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3628
|
+
).bind(projectId, row.environment, row.id).first();
|
|
3629
|
+
return mapProposalRow(updated ?? row);
|
|
3630
|
+
}
|
|
3631
|
+
async function claimRemediationGrant(db, projectId, proposalId, options) {
|
|
3632
|
+
const nowIso = options.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
3633
|
+
const row = options.environment ? await db.prepare(
|
|
3634
|
+
`SELECT * FROM remediation_proposals
|
|
3635
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3636
|
+
).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
|
|
3637
|
+
`SELECT * FROM remediation_proposals
|
|
3638
|
+
WHERE project_id = ? AND id = ?
|
|
3639
|
+
LIMIT 1`
|
|
3640
|
+
).bind(projectId, proposalId).first();
|
|
3641
|
+
if (!row) {
|
|
3642
|
+
throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
|
|
3643
|
+
}
|
|
3644
|
+
if (row.status !== "grant_issued") {
|
|
3645
|
+
throw new RemediationGateError(
|
|
3646
|
+
`Grant cannot be claimed from status=${row.status} (grants are single-use)`,
|
|
3647
|
+
409
|
|
3648
|
+
);
|
|
3649
|
+
}
|
|
3650
|
+
if (row.grant_expires_at && row.grant_expires_at <= nowIso) {
|
|
3651
|
+
await db.prepare(
|
|
3652
|
+
`UPDATE remediation_proposals
|
|
3653
|
+
SET status = 'expired', updated_at = ?
|
|
3654
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3655
|
+
).bind(nowIso, projectId, row.environment, row.id).run();
|
|
3656
|
+
throw new RemediationGateError(
|
|
3657
|
+
`Grant for proposal ${proposalId} expired at ${row.grant_expires_at}`,
|
|
3658
|
+
410
|
|
3659
|
+
);
|
|
3660
|
+
}
|
|
3661
|
+
const incomingHash = await sha256Hex(options.grantToken);
|
|
3662
|
+
if (!row.grant_token_hash || incomingHash !== row.grant_token_hash) {
|
|
3663
|
+
throw new RemediationGateError("Invalid grantToken", 403);
|
|
3664
|
+
}
|
|
3665
|
+
const expected = parseJsonRecord(
|
|
3666
|
+
row.expected_current_state_json
|
|
3667
|
+
);
|
|
3668
|
+
const targetNode = await db.prepare(
|
|
3669
|
+
`SELECT current_version, current_commit_sha, current_config_digest
|
|
3670
|
+
FROM ontology_nodes
|
|
3671
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3672
|
+
).bind(projectId, row.environment, row.target_node_id).first();
|
|
3673
|
+
if (targetNode) {
|
|
3674
|
+
if (expected.version && targetNode.current_version && targetNode.current_version !== expected.version) {
|
|
3675
|
+
throw new RemediationGateError(
|
|
3676
|
+
`Optimistic version lock failed on ${row.target_node_id}: expected version "${expected.version}", but current version is "${targetNode.current_version}"`,
|
|
3677
|
+
409
|
|
3678
|
+
);
|
|
3679
|
+
}
|
|
3680
|
+
if (expected.commitSha && targetNode.current_commit_sha && targetNode.current_commit_sha !== expected.commitSha) {
|
|
3681
|
+
throw new RemediationGateError(
|
|
3682
|
+
`Optimistic commit lock failed on ${row.target_node_id}: expected commit "${expected.commitSha}", but current commit is "${targetNode.current_commit_sha}"`,
|
|
3683
|
+
409
|
|
3684
|
+
);
|
|
3685
|
+
}
|
|
3686
|
+
if (expected.configDigest && targetNode.current_config_digest && targetNode.current_config_digest !== expected.configDigest) {
|
|
3687
|
+
throw new RemediationGateError(
|
|
3688
|
+
`Optimistic config lock failed on ${row.target_node_id}: expected digest "${expected.configDigest}", but current digest is "${targetNode.current_config_digest}"`,
|
|
3689
|
+
409
|
|
3690
|
+
);
|
|
3691
|
+
}
|
|
3692
|
+
}
|
|
3693
|
+
const claimExpiresAt = new Date(
|
|
3694
|
+
Date.parse(nowIso) + 5 * 6e4
|
|
3695
|
+
).toISOString();
|
|
3696
|
+
await db.batch([
|
|
3697
|
+
db.prepare(
|
|
3698
|
+
`UPDATE remediation_proposals
|
|
3699
|
+
SET status = 'claimed_executing',
|
|
3700
|
+
grant_token_hash = NULL,
|
|
3701
|
+
claimed_by_runner_id = ?,
|
|
3702
|
+
claimed_at = ?,
|
|
3703
|
+
claim_expires_at = ?,
|
|
3704
|
+
updated_at = ?
|
|
3705
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3706
|
+
).bind(
|
|
3707
|
+
options.runnerId ?? "autosre-runner",
|
|
3708
|
+
nowIso,
|
|
3709
|
+
claimExpiresAt,
|
|
3710
|
+
nowIso,
|
|
3711
|
+
projectId,
|
|
3712
|
+
row.environment,
|
|
3713
|
+
row.id
|
|
3714
|
+
),
|
|
3715
|
+
db.prepare(
|
|
3716
|
+
`UPDATE incident_investigations
|
|
3717
|
+
SET status = 'remediating', updated_at = ?
|
|
3718
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3719
|
+
).bind(nowIso, projectId, row.environment, row.incident_id)
|
|
3720
|
+
]);
|
|
3721
|
+
const updated = await db.prepare(
|
|
3722
|
+
`SELECT * FROM remediation_proposals
|
|
3723
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3724
|
+
).bind(projectId, row.environment, row.id).first();
|
|
3725
|
+
return mapProposalRow(updated ?? row);
|
|
3726
|
+
}
|
|
3727
|
+
async function completeRemediationProposal(db, projectId, proposalId, options) {
|
|
3728
|
+
const nowIso = options.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
3729
|
+
const row = options.environment ? await db.prepare(
|
|
3730
|
+
`SELECT * FROM remediation_proposals
|
|
3731
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3732
|
+
).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
|
|
3733
|
+
`SELECT * FROM remediation_proposals
|
|
3734
|
+
WHERE project_id = ? AND id = ?
|
|
3735
|
+
LIMIT 1`
|
|
3736
|
+
).bind(projectId, proposalId).first();
|
|
3737
|
+
if (!row) {
|
|
3738
|
+
throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
|
|
3739
|
+
}
|
|
3740
|
+
if (row.status !== "claimed_executing") {
|
|
3741
|
+
throw new RemediationGateError(
|
|
3742
|
+
`Proposal ${proposalId} cannot be completed from status=${row.status} (must be claimed_executing)`,
|
|
3743
|
+
409
|
|
3744
|
+
);
|
|
3745
|
+
}
|
|
3746
|
+
const desired = parseJsonRecord(
|
|
3747
|
+
row.desired_target_state_json
|
|
3748
|
+
);
|
|
3749
|
+
const nextWatchdogAt = new Date(
|
|
3750
|
+
Date.parse(nowIso) + 2 * 6e4
|
|
3751
|
+
).toISOString();
|
|
3752
|
+
const finding = await getInvestigationFinding(
|
|
3753
|
+
db,
|
|
3754
|
+
projectId,
|
|
3755
|
+
row.finding_id,
|
|
3756
|
+
row.environment
|
|
3757
|
+
);
|
|
3758
|
+
const updatedChain = finding ? [...finding.causalChain] : [];
|
|
3759
|
+
if (options.resultingChangeId) {
|
|
3760
|
+
updatedChain.push({
|
|
3761
|
+
subjectKind: "finding",
|
|
3762
|
+
subjectId: row.finding_id,
|
|
3763
|
+
predicate: "mitigated_by_change",
|
|
3764
|
+
objectKind: "change",
|
|
3765
|
+
objectId: options.resultingChangeId,
|
|
3766
|
+
relationshipCertainty: "exact",
|
|
3767
|
+
causalProofTier: finding?.causalProofTier ?? "cohort_rate_isolated",
|
|
3768
|
+
confidence: 0.9,
|
|
3769
|
+
reason: `Remediation proposal ${row.id} (${row.action_kind}) applied change ${options.resultingChangeId}`
|
|
3770
|
+
});
|
|
3771
|
+
}
|
|
3772
|
+
if (options.remediationAgentRunId) {
|
|
3773
|
+
updatedChain.push({
|
|
3774
|
+
subjectKind: "finding",
|
|
3775
|
+
subjectId: row.finding_id,
|
|
3776
|
+
predicate: "remediated_by_agent_run",
|
|
3777
|
+
objectKind: "agent_run",
|
|
3778
|
+
objectId: options.remediationAgentRunId,
|
|
3779
|
+
relationshipCertainty: "exact",
|
|
3780
|
+
causalProofTier: finding?.causalProofTier ?? "cohort_rate_isolated",
|
|
3781
|
+
confidence: 0.9,
|
|
3782
|
+
reason: `Remediation executed by agent run ${options.remediationAgentRunId}`
|
|
3783
|
+
});
|
|
3784
|
+
}
|
|
3785
|
+
await db.batch([
|
|
3786
|
+
db.prepare(
|
|
3787
|
+
`UPDATE remediation_proposals
|
|
3788
|
+
SET status = 'completed',
|
|
3789
|
+
resulting_change_id = COALESCE(?, resulting_change_id),
|
|
3790
|
+
remediation_agent_run_id = COALESCE(?, remediation_agent_run_id),
|
|
3791
|
+
updated_at = ?
|
|
3792
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3793
|
+
).bind(
|
|
3794
|
+
options.resultingChangeId ?? null,
|
|
3795
|
+
options.remediationAgentRunId ?? null,
|
|
3796
|
+
nowIso,
|
|
3797
|
+
projectId,
|
|
3798
|
+
row.environment,
|
|
3799
|
+
row.id
|
|
3800
|
+
),
|
|
3801
|
+
db.prepare(
|
|
3802
|
+
`UPDATE ontology_nodes
|
|
3803
|
+
SET current_version = COALESCE(?, current_version),
|
|
3804
|
+
current_commit_sha = COALESCE(?, current_commit_sha),
|
|
3805
|
+
current_config_digest = COALESCE(?, current_config_digest),
|
|
3806
|
+
last_seen_at = ?
|
|
3807
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3808
|
+
).bind(
|
|
3809
|
+
desired.version ?? null,
|
|
3810
|
+
desired.commitSha ?? null,
|
|
3811
|
+
desired.configDigest ?? null,
|
|
3812
|
+
nowIso,
|
|
3813
|
+
projectId,
|
|
3814
|
+
row.environment,
|
|
3815
|
+
row.target_node_id
|
|
3816
|
+
),
|
|
3817
|
+
db.prepare(
|
|
3818
|
+
`UPDATE investigation_findings
|
|
3819
|
+
SET recovery_status = 'monitoring',
|
|
3820
|
+
watchdog_step = 1,
|
|
3821
|
+
baseline_request_rate_per_min = COALESCE(?, baseline_request_rate_per_min),
|
|
3822
|
+
next_watchdog_check_at = ?,
|
|
3823
|
+
causal_chain_json = ?,
|
|
3824
|
+
updated_at = ?
|
|
3825
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3826
|
+
).bind(
|
|
3827
|
+
options.baselineRequestRatePerMin ?? null,
|
|
3828
|
+
nextWatchdogAt,
|
|
3829
|
+
JSON.stringify(updatedChain),
|
|
3830
|
+
nowIso,
|
|
3831
|
+
projectId,
|
|
3832
|
+
row.environment,
|
|
3833
|
+
row.finding_id
|
|
3834
|
+
),
|
|
3835
|
+
db.prepare(
|
|
3836
|
+
`UPDATE incident_investigations
|
|
3837
|
+
SET status = 'verifying', updated_at = ?
|
|
3838
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3839
|
+
).bind(nowIso, projectId, row.environment, row.incident_id)
|
|
3840
|
+
]);
|
|
3841
|
+
const updated = await db.prepare(
|
|
3842
|
+
`SELECT * FROM remediation_proposals
|
|
3843
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3844
|
+
).bind(projectId, row.environment, row.id).first();
|
|
3845
|
+
return mapProposalRow(updated ?? row);
|
|
3846
|
+
}
|
|
3847
|
+
async function failRemediationProposal(db, projectId, proposalId, options) {
|
|
3848
|
+
const nowIso = options.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
3849
|
+
const row = options.environment ? await db.prepare(
|
|
3850
|
+
`SELECT * FROM remediation_proposals
|
|
3851
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3852
|
+
).bind(projectId, options.environment.toLowerCase(), proposalId).first() : await db.prepare(
|
|
3853
|
+
`SELECT * FROM remediation_proposals
|
|
3854
|
+
WHERE project_id = ? AND id = ?
|
|
3855
|
+
LIMIT 1`
|
|
3856
|
+
).bind(projectId, proposalId).first();
|
|
3857
|
+
if (!row) {
|
|
3858
|
+
throw new RemediationGateError(`Proposal ${proposalId} not found`, 404);
|
|
3859
|
+
}
|
|
3860
|
+
await db.batch([
|
|
3861
|
+
db.prepare(
|
|
3862
|
+
`UPDATE remediation_proposals
|
|
3863
|
+
SET status = 'failed_execution',
|
|
3864
|
+
failure_reason = ?,
|
|
3865
|
+
updated_at = ?
|
|
3866
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3867
|
+
).bind(options.failureReason, nowIso, projectId, row.environment, row.id),
|
|
3868
|
+
db.prepare(
|
|
3869
|
+
`UPDATE incident_investigations
|
|
3870
|
+
SET status = 'escalated', updated_at = ?
|
|
3871
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3872
|
+
).bind(nowIso, projectId, row.environment, row.incident_id)
|
|
3873
|
+
]);
|
|
3874
|
+
const updated = await db.prepare(
|
|
3875
|
+
`SELECT * FROM remediation_proposals
|
|
3876
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3877
|
+
).bind(projectId, row.environment, row.id).first();
|
|
3878
|
+
return mapProposalRow(updated ?? row);
|
|
3879
|
+
}
|
|
3880
|
+
async function evaluateRemediationWatchdogs(db, options = {}) {
|
|
3881
|
+
const nowIso = options.nowIso ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
3882
|
+
const minComparableRequests = options.minComparableRequests ?? 50;
|
|
3883
|
+
const expiredGrantsRes = await db.prepare(
|
|
3884
|
+
`UPDATE remediation_proposals
|
|
3885
|
+
SET status = 'expired', grant_token_hash = NULL, updated_at = ?
|
|
3886
|
+
WHERE status = 'grant_issued'
|
|
3887
|
+
AND grant_expires_at IS NOT NULL
|
|
3888
|
+
AND grant_expires_at <= ?`
|
|
3889
|
+
).bind(nowIso, nowIso).run();
|
|
3890
|
+
const timedOutRows = await db.prepare(
|
|
3891
|
+
`SELECT project_id, environment, id, incident_id
|
|
3892
|
+
FROM remediation_proposals
|
|
3893
|
+
WHERE status = 'claimed_executing'
|
|
3894
|
+
AND claim_expires_at IS NOT NULL
|
|
3895
|
+
AND claim_expires_at <= ?`
|
|
3896
|
+
).bind(nowIso).all();
|
|
3897
|
+
for (const t of timedOutRows.results) {
|
|
3898
|
+
await db.batch([
|
|
3899
|
+
db.prepare(
|
|
3900
|
+
`UPDATE remediation_proposals
|
|
3901
|
+
SET status = 'execution_timed_out',
|
|
3902
|
+
failure_reason = 'Runner did not complete or fail within 5m claim window',
|
|
3903
|
+
updated_at = ?
|
|
3904
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3905
|
+
).bind(nowIso, t.project_id, t.environment, t.id),
|
|
3906
|
+
db.prepare(
|
|
3907
|
+
`UPDATE incident_investigations
|
|
3908
|
+
SET status = 'escalated', updated_at = ?
|
|
3909
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3910
|
+
).bind(nowIso, t.project_id, t.environment, t.incident_id)
|
|
3911
|
+
]);
|
|
3912
|
+
}
|
|
3913
|
+
const dueFindings = await db.prepare(
|
|
3914
|
+
`SELECT
|
|
3915
|
+
project_id, environment, id, incident_id,
|
|
3916
|
+
fault_locus_entity_id, root_cause_entity_id,
|
|
3917
|
+
watchdog_step, baseline_request_rate_per_min,
|
|
3918
|
+
causal_verification_status, updated_at
|
|
3919
|
+
FROM investigation_findings
|
|
3920
|
+
WHERE recovery_status = 'monitoring'
|
|
3921
|
+
AND next_watchdog_check_at IS NOT NULL
|
|
3922
|
+
AND next_watchdog_check_at <= ?
|
|
3923
|
+
LIMIT 50`
|
|
3924
|
+
).bind(nowIso).all();
|
|
3925
|
+
let recovered = 0;
|
|
3926
|
+
let inconclusiveLowTraffic = 0;
|
|
3927
|
+
let stillDegraded = 0;
|
|
3928
|
+
for (const f of dueFindings.results) {
|
|
3929
|
+
const completedProposal = await db.prepare(
|
|
3930
|
+
`SELECT
|
|
3931
|
+
p.updated_at,
|
|
3932
|
+
p.target_node_id,
|
|
3933
|
+
p.resulting_change_id,
|
|
3934
|
+
c.occurred_at AS change_occurred_at
|
|
3935
|
+
FROM remediation_proposals p
|
|
3936
|
+
LEFT JOIN change_events c
|
|
3937
|
+
ON c.project_id = p.project_id
|
|
3938
|
+
AND c.environment = p.environment
|
|
3939
|
+
AND c.id = p.resulting_change_id
|
|
3940
|
+
WHERE p.project_id = ? AND p.environment = ? AND p.finding_id = ? AND p.status = 'completed'
|
|
3941
|
+
ORDER BY p.updated_at DESC
|
|
3942
|
+
LIMIT 1`
|
|
3943
|
+
).bind(f.project_id, f.environment, f.id).first();
|
|
3944
|
+
const sinceTs = completedProposal?.change_occurred_at ?? completedProposal?.updated_at ?? f.updated_at;
|
|
3945
|
+
const targetNodeId = completedProposal?.target_node_id ?? f.fault_locus_entity_id;
|
|
3946
|
+
const parts = targetNodeId.split(":");
|
|
3947
|
+
const serviceName = parts[0] === "service" && parts.length >= 4 ? parts.slice(3).join(":") : null;
|
|
3948
|
+
const stats = serviceName ? await db.prepare(
|
|
3949
|
+
`SELECT
|
|
3950
|
+
COUNT(*) AS total_requests,
|
|
3951
|
+
SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) AS error_requests
|
|
3952
|
+
FROM telemetry_spans
|
|
3953
|
+
WHERE project_id = ?
|
|
3954
|
+
AND environment = ?
|
|
3955
|
+
AND service_name = ?
|
|
3956
|
+
AND (parent_span_id IS NULL OR span_kind = 1)
|
|
3957
|
+
AND (start_time >= ? OR received_at >= ?)`
|
|
3958
|
+
).bind(f.project_id, f.environment, serviceName, sinceTs, sinceTs).first() : await db.prepare(
|
|
3959
|
+
`SELECT
|
|
3960
|
+
COUNT(*) AS total_requests,
|
|
3961
|
+
SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) AS error_requests
|
|
3962
|
+
FROM telemetry_spans
|
|
3963
|
+
WHERE project_id = ?
|
|
3964
|
+
AND environment = ?
|
|
3965
|
+
AND (parent_span_id IS NULL OR span_kind = 1)
|
|
3966
|
+
AND (start_time >= ? OR received_at >= ?)`
|
|
3967
|
+
).bind(f.project_id, f.environment, sinceTs, sinceTs).first();
|
|
3968
|
+
const totalReq = Number(stats?.total_requests ?? 0);
|
|
3969
|
+
const errorReq = Number(stats?.error_requests ?? 0);
|
|
3970
|
+
const errorRate = totalReq > 0 ? errorReq / totalReq : 0;
|
|
3971
|
+
const requiredTraffic = Math.max(
|
|
3972
|
+
minComparableRequests,
|
|
3973
|
+
f.baseline_request_rate_per_min ? Math.floor(0.5 * f.baseline_request_rate_per_min * 5) : minComparableRequests
|
|
3974
|
+
);
|
|
3975
|
+
if (totalReq < requiredTraffic) {
|
|
3976
|
+
if (f.watchdog_step < 3) {
|
|
3977
|
+
const nextStep = f.watchdog_step + 1;
|
|
3978
|
+
const offsetMins = nextStep === 2 ? 5 : 15;
|
|
3979
|
+
const nextCheck = new Date(
|
|
3980
|
+
Date.parse(nowIso) + offsetMins * 6e4
|
|
3981
|
+
).toISOString();
|
|
3982
|
+
await db.prepare(
|
|
3983
|
+
`UPDATE investigation_findings
|
|
3984
|
+
SET watchdog_step = ?, next_watchdog_check_at = ?, updated_at = ?
|
|
3985
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3986
|
+
).bind(nextStep, nextCheck, nowIso, f.project_id, f.environment, f.id).run();
|
|
3987
|
+
} else {
|
|
3988
|
+
inconclusiveLowTraffic += 1;
|
|
3989
|
+
await db.prepare(
|
|
3990
|
+
`UPDATE investigation_findings
|
|
3991
|
+
SET recovery_status = 'inconclusive_low_traffic',
|
|
3992
|
+
next_watchdog_check_at = NULL,
|
|
3993
|
+
updated_at = ?
|
|
3994
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
3995
|
+
).bind(nowIso, f.project_id, f.environment, f.id).run();
|
|
3996
|
+
}
|
|
3997
|
+
continue;
|
|
3998
|
+
}
|
|
3999
|
+
if (errorRate >= 0.05) {
|
|
4000
|
+
stillDegraded += 1;
|
|
4001
|
+
await db.prepare(
|
|
4002
|
+
`UPDATE investigation_findings
|
|
4003
|
+
SET recovery_status = 'still_degraded',
|
|
4004
|
+
causal_verification_status = 'refuted',
|
|
4005
|
+
next_watchdog_check_at = NULL,
|
|
4006
|
+
updated_at = ?
|
|
4007
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
4008
|
+
).bind(nowIso, f.project_id, f.environment, f.id).run();
|
|
4009
|
+
if (f.incident_id) {
|
|
4010
|
+
await db.prepare(
|
|
4011
|
+
`UPDATE incident_investigations
|
|
4012
|
+
SET status = 'escalated', updated_at = ?
|
|
4013
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
4014
|
+
).bind(nowIso, f.project_id, f.environment, f.incident_id).run();
|
|
4015
|
+
}
|
|
4016
|
+
} else {
|
|
4017
|
+
recovered += 1;
|
|
4018
|
+
const nextCausalVerification = f.causal_verification_status === "counterfactual_verified" ? "counterfactual_verified" : "operationally_corroborated";
|
|
4019
|
+
await db.prepare(
|
|
4020
|
+
`UPDATE investigation_findings
|
|
4021
|
+
SET recovery_status = 'recovered',
|
|
4022
|
+
causal_verification_status = ?,
|
|
4023
|
+
watchdog_step = 3,
|
|
4024
|
+
next_watchdog_check_at = NULL,
|
|
4025
|
+
updated_at = ?
|
|
4026
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
4027
|
+
).bind(
|
|
4028
|
+
nextCausalVerification,
|
|
4029
|
+
nowIso,
|
|
4030
|
+
f.project_id,
|
|
4031
|
+
f.environment,
|
|
4032
|
+
f.id
|
|
4033
|
+
).run();
|
|
4034
|
+
if (f.incident_id) {
|
|
4035
|
+
await db.prepare(
|
|
4036
|
+
`UPDATE incident_investigations
|
|
4037
|
+
SET status = 'resolved', resolved_at = ?, updated_at = ?
|
|
4038
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
4039
|
+
).bind(nowIso, nowIso, f.project_id, f.environment, f.incident_id).run();
|
|
4040
|
+
}
|
|
4041
|
+
}
|
|
4042
|
+
}
|
|
4043
|
+
return {
|
|
4044
|
+
evaluated: dueFindings.results.length,
|
|
4045
|
+
recovered,
|
|
4046
|
+
inconclusiveLowTraffic,
|
|
4047
|
+
stillDegraded,
|
|
4048
|
+
timedOutClaims: timedOutRows.results.length,
|
|
4049
|
+
expiredGrants: expiredGrantsRes.meta.changes
|
|
4050
|
+
};
|
|
4051
|
+
}
|
|
4052
|
+
|
|
2490
4053
|
// src/lib/narrate-gate.ts
|
|
2491
4054
|
var COMPARATORS = ["<=", ">=", "==", "!=", "<", ">"];
|
|
2492
4055
|
var stripWhitespace = (s) => s.replace(/\s+/g, "");
|
|
@@ -2855,6 +4418,14 @@ async function runDueAnalysesForProject(projectId, deps) {
|
|
|
2855
4418
|
if (result.narrative) narrated += 1;
|
|
2856
4419
|
}
|
|
2857
4420
|
await store.insertResult(result, expiresAt);
|
|
4421
|
+
if (result.status === "critical") {
|
|
4422
|
+
await openOrDeduplicateIncident(db, projectId, {
|
|
4423
|
+
environment: "default",
|
|
4424
|
+
targetNodeId: `analysis:${def.id}`,
|
|
4425
|
+
symptomAnchorKind: "analysis",
|
|
4426
|
+
symptomAnchorId: def.id
|
|
4427
|
+
}).catch(() => void 0);
|
|
4428
|
+
}
|
|
2858
4429
|
await store.markRan(projectId, def.id, result.generatedAt);
|
|
2859
4430
|
ran += 1;
|
|
2860
4431
|
} catch (error) {
|
|
@@ -2880,6 +4451,7 @@ var changes = (r) => r.meta?.changes ?? 0;
|
|
|
2880
4451
|
async function purgeSecondaryRows(db, options) {
|
|
2881
4452
|
const nowMs = options.now?.() ?? Date.now();
|
|
2882
4453
|
const isoBefore = (hours) => new Date(nowMs - hours * 36e5).toISOString();
|
|
4454
|
+
const nowIso = isoBefore(0);
|
|
2883
4455
|
const alertEvaluations = await db.prepare(`DELETE FROM alert_evaluations WHERE evaluated_at < ?`).bind(isoBefore(ALERT_EVALUATIONS_MAX_AGE_HOURS)).run();
|
|
2884
4456
|
const askEvents = await db.prepare(`DELETE FROM ask_evidence_events WHERE asked_at < ?`).bind(nowMs - ASK_EVIDENCE_EVENTS_MAX_AGE_HOURS * 36e5).run();
|
|
2885
4457
|
const series = await db.prepare(
|
|
@@ -2892,7 +4464,7 @@ async function purgeSecondaryRows(db, options) {
|
|
|
2892
4464
|
const refs = await db.prepare(
|
|
2893
4465
|
`DELETE FROM evidence_retrieval_refs
|
|
2894
4466
|
WHERE expires_at IS NOT NULL AND expires_at < ?`
|
|
2895
|
-
).bind(
|
|
4467
|
+
).bind(nowIso).run();
|
|
2896
4468
|
const expansions = await db.prepare(
|
|
2897
4469
|
`DELETE FROM evidence_ref_expansions
|
|
2898
4470
|
WHERE NOT EXISTS (
|
|
@@ -2900,12 +4472,75 @@ async function purgeSecondaryRows(db, options) {
|
|
|
2900
4472
|
WHERE r.ref_id = evidence_ref_expansions.ref_id
|
|
2901
4473
|
)`
|
|
2902
4474
|
).run();
|
|
4475
|
+
const thirtyDaysAgoIso = isoBefore(30 * 24);
|
|
4476
|
+
const ninetyDaysAgoIso = isoBefore(90 * 24);
|
|
4477
|
+
let ontologyEdgesPurged = 0;
|
|
4478
|
+
let ontologyNodesPurged = 0;
|
|
4479
|
+
let changeEventsPurged = 0;
|
|
4480
|
+
let incidentInvestigationsPurged = 0;
|
|
4481
|
+
let investigationFindingsPurged = 0;
|
|
4482
|
+
let remediationProposalsPurged = 0;
|
|
4483
|
+
try {
|
|
4484
|
+
const edgesRes = await db.prepare(`DELETE FROM ontology_edges WHERE last_seen_at < ?`).bind(thirtyDaysAgoIso).run();
|
|
4485
|
+
ontologyEdgesPurged = changes(edgesRes);
|
|
4486
|
+
const nodesRes = await db.prepare(
|
|
4487
|
+
`DELETE FROM ontology_nodes
|
|
4488
|
+
WHERE last_seen_at < ?
|
|
4489
|
+
AND NOT EXISTS (
|
|
4490
|
+
SELECT 1 FROM ontology_edges e
|
|
4491
|
+
WHERE e.project_id = ontology_nodes.project_id
|
|
4492
|
+
AND e.environment = ontology_nodes.environment
|
|
4493
|
+
AND (e.source_node_id = ontology_nodes.id OR e.target_node_id = ontology_nodes.id)
|
|
4494
|
+
)`
|
|
4495
|
+
).bind(thirtyDaysAgoIso).run();
|
|
4496
|
+
ontologyNodesPurged = changes(nodesRes);
|
|
4497
|
+
const proposalsRes = await db.prepare(
|
|
4498
|
+
`DELETE FROM remediation_proposals
|
|
4499
|
+
WHERE (expires_at IS NOT NULL AND expires_at < ?)
|
|
4500
|
+
OR (expires_at IS NULL AND created_at < ?)`
|
|
4501
|
+
).bind(nowIso, ninetyDaysAgoIso).run();
|
|
4502
|
+
remediationProposalsPurged = changes(proposalsRes);
|
|
4503
|
+
const findingsRes = await db.prepare(
|
|
4504
|
+
`DELETE FROM investigation_findings
|
|
4505
|
+
WHERE causal_verification_status NOT IN ('counterfactual_verified', 'operationally_corroborated')
|
|
4506
|
+
AND ((expires_at IS NOT NULL AND expires_at < ?)
|
|
4507
|
+
OR (expires_at IS NULL AND created_at < ?))`
|
|
4508
|
+
).bind(nowIso, ninetyDaysAgoIso).run();
|
|
4509
|
+
investigationFindingsPurged = changes(findingsRes);
|
|
4510
|
+
const changesRes = await db.prepare(
|
|
4511
|
+
`DELETE FROM change_events
|
|
4512
|
+
WHERE ((expires_at IS NOT NULL AND expires_at < ?)
|
|
4513
|
+
OR (expires_at IS NULL AND occurred_at < ?))
|
|
4514
|
+
AND NOT EXISTS (
|
|
4515
|
+
SELECT 1 FROM investigation_findings f
|
|
4516
|
+
WHERE f.project_id = change_events.project_id
|
|
4517
|
+
AND f.environment = change_events.environment
|
|
4518
|
+
AND f.contributing_change_id = change_events.id
|
|
4519
|
+
)`
|
|
4520
|
+
).bind(nowIso, ninetyDaysAgoIso).run();
|
|
4521
|
+
changeEventsPurged = changes(changesRes);
|
|
4522
|
+
const incidentsRes = await db.prepare(
|
|
4523
|
+
`DELETE FROM incident_investigations
|
|
4524
|
+
WHERE status IN ('resolved', 'escalated')
|
|
4525
|
+
AND ((expires_at IS NOT NULL AND expires_at < ?)
|
|
4526
|
+
OR (expires_at IS NULL AND updated_at < ?))`
|
|
4527
|
+
).bind(nowIso, ninetyDaysAgoIso).run();
|
|
4528
|
+
incidentInvestigationsPurged = changes(incidentsRes);
|
|
4529
|
+
await evaluateRemediationWatchdogs(db, { nowIso });
|
|
4530
|
+
} catch {
|
|
4531
|
+
}
|
|
2903
4532
|
return {
|
|
2904
4533
|
alert_evaluations: changes(alertEvaluations),
|
|
2905
4534
|
ask_evidence_events: changes(askEvents),
|
|
2906
4535
|
metric_series: changes(series),
|
|
2907
4536
|
evidence_refs: changes(refs),
|
|
2908
|
-
evidence_ref_expansions: changes(expansions)
|
|
4537
|
+
evidence_ref_expansions: changes(expansions),
|
|
4538
|
+
ontology_edges: ontologyEdgesPurged,
|
|
4539
|
+
ontology_nodes: ontologyNodesPurged,
|
|
4540
|
+
change_events: changeEventsPurged,
|
|
4541
|
+
incident_investigations: incidentInvestigationsPurged,
|
|
4542
|
+
investigation_findings: investigationFindingsPurged,
|
|
4543
|
+
remediation_proposals: remediationProposalsPurged
|
|
2909
4544
|
};
|
|
2910
4545
|
}
|
|
2911
4546
|
|
|
@@ -3641,13 +5276,630 @@ async function getTelemetryExportRows(db, options) {
|
|
|
3641
5276
|
).join("\n");
|
|
3642
5277
|
}
|
|
3643
5278
|
|
|
5279
|
+
// src/lib/autosre-ontology.ts
|
|
5280
|
+
import { buildCanonicalNodeId as buildCanonicalNodeId2, normalizeErrorSignature } from "@obsunified/types";
|
|
5281
|
+
var pickStr = (record, ...keys) => {
|
|
5282
|
+
for (const key of keys) {
|
|
5283
|
+
const val = record[key];
|
|
5284
|
+
if (typeof val === "string" && val.trim().length > 0) {
|
|
5285
|
+
return val.trim();
|
|
5286
|
+
}
|
|
5287
|
+
if (typeof val === "number") {
|
|
5288
|
+
return String(val);
|
|
5289
|
+
}
|
|
5290
|
+
}
|
|
5291
|
+
return null;
|
|
5292
|
+
};
|
|
5293
|
+
function extractSpanLineageAndTopology(span) {
|
|
5294
|
+
const attrs = parseJsonRecord(span.attributesJson);
|
|
5295
|
+
const resource = parseJsonRecord(span.resourceAttributesJson);
|
|
5296
|
+
const environment = (span.environment ?? pickStr(
|
|
5297
|
+
resource,
|
|
5298
|
+
"deployment.environment.name",
|
|
5299
|
+
"deployment.environment",
|
|
5300
|
+
"obs.environment"
|
|
5301
|
+
) ?? pickStr(
|
|
5302
|
+
attrs,
|
|
5303
|
+
"deployment.environment.name",
|
|
5304
|
+
"deployment.environment",
|
|
5305
|
+
"obs.environment"
|
|
5306
|
+
) ?? "default").toLowerCase();
|
|
5307
|
+
const namespace = pickStr(resource, "service.namespace") ?? pickStr(attrs, "service.namespace") ?? "default";
|
|
5308
|
+
const serviceName = span.serviceName ?? pickStr(resource, "service.name") ?? pickStr(attrs, "service.name");
|
|
5309
|
+
const serviceVersion = span.serviceVersion ?? pickStr(resource, "service.version", "obs.service.version") ?? pickStr(attrs, "service.version", "obs.service.version");
|
|
5310
|
+
const commitSha = span.commitSha ?? pickStr(
|
|
5311
|
+
resource,
|
|
5312
|
+
"vcs.commit.id",
|
|
5313
|
+
"git.commit.sha",
|
|
5314
|
+
"obs.commit.sha",
|
|
5315
|
+
"service.commit.sha"
|
|
5316
|
+
) ?? pickStr(attrs, "vcs.commit.id", "git.commit.sha", "obs.commit.sha");
|
|
5317
|
+
const configDigest = pickStr(resource, "obs.config.digest") ?? pickStr(attrs, "obs.config.digest");
|
|
5318
|
+
const sourceNode = serviceName ? buildCanonicalNodeId2({
|
|
5319
|
+
kind: "service",
|
|
5320
|
+
environment,
|
|
5321
|
+
namespace,
|
|
5322
|
+
serviceName
|
|
5323
|
+
}) : null;
|
|
5324
|
+
let targetNode = null;
|
|
5325
|
+
let edgePredicate = null;
|
|
5326
|
+
const dbSystem = pickStr(attrs, "db.system");
|
|
5327
|
+
const messagingSystem = pickStr(attrs, "messaging.system");
|
|
5328
|
+
const peerService = pickStr(attrs, "peer.service", "obs.target.service");
|
|
5329
|
+
const serverAddress = pickStr(
|
|
5330
|
+
attrs,
|
|
5331
|
+
"server.address",
|
|
5332
|
+
"net.peer.name",
|
|
5333
|
+
"db.connection_string"
|
|
5334
|
+
);
|
|
5335
|
+
const serverPort = pickStr(attrs, "server.port", "net.peer.port");
|
|
5336
|
+
const toolName = pickStr(attrs, "obs.tool.name", "tool.name");
|
|
5337
|
+
if (dbSystem) {
|
|
5338
|
+
const isCache = dbSystem.toLowerCase() === "redis" || dbSystem.toLowerCase() === "memcached";
|
|
5339
|
+
targetNode = buildCanonicalNodeId2({
|
|
5340
|
+
kind: isCache ? "cache" : "database",
|
|
5341
|
+
environment,
|
|
5342
|
+
dbSystem,
|
|
5343
|
+
dbName: pickStr(attrs, "db.namespace", "db.name") ?? dbSystem,
|
|
5344
|
+
serverAddress,
|
|
5345
|
+
serverPort
|
|
5346
|
+
});
|
|
5347
|
+
edgePredicate = isCache ? "queries_cache" : "queries_database";
|
|
5348
|
+
} else if (messagingSystem) {
|
|
5349
|
+
targetNode = buildCanonicalNodeId2({
|
|
5350
|
+
kind: "queue_topic",
|
|
5351
|
+
environment,
|
|
5352
|
+
messagingSystem,
|
|
5353
|
+
messagingDestination: pickStr(
|
|
5354
|
+
attrs,
|
|
5355
|
+
"messaging.destination.name",
|
|
5356
|
+
"messaging.destination"
|
|
5357
|
+
) ?? "default",
|
|
5358
|
+
serverAddress
|
|
5359
|
+
});
|
|
5360
|
+
edgePredicate = span.spanKind === 4 ? "publishes_to_queue" : "consumes_from_queue";
|
|
5361
|
+
} else if (peerService) {
|
|
5362
|
+
targetNode = buildCanonicalNodeId2({
|
|
5363
|
+
kind: "service",
|
|
5364
|
+
environment,
|
|
5365
|
+
namespace: pickStr(attrs, "peer.namespace") ?? namespace,
|
|
5366
|
+
serviceName: peerService
|
|
5367
|
+
});
|
|
5368
|
+
edgePredicate = "calls_service";
|
|
5369
|
+
} else if (span.spanKind === 2 && serverAddress) {
|
|
5370
|
+
targetNode = buildCanonicalNodeId2({
|
|
5371
|
+
kind: "external_api",
|
|
5372
|
+
environment,
|
|
5373
|
+
serverAddress,
|
|
5374
|
+
serverPort: serverPort ?? "443"
|
|
5375
|
+
});
|
|
5376
|
+
edgePredicate = "calls_external_api";
|
|
5377
|
+
} else if (toolName) {
|
|
5378
|
+
targetNode = buildCanonicalNodeId2({
|
|
5379
|
+
kind: "tool",
|
|
5380
|
+
environment,
|
|
5381
|
+
serviceName: pickStr(attrs, "obs.agent.id", "agent.id") ?? serviceName ?? "default",
|
|
5382
|
+
toolName
|
|
5383
|
+
});
|
|
5384
|
+
edgePredicate = "invokes_tool";
|
|
5385
|
+
}
|
|
5386
|
+
let errorSignature = span.errorSignature ?? null;
|
|
5387
|
+
if (!errorSignature && span.statusCode === 2) {
|
|
5388
|
+
let rawErr = pickStr(
|
|
5389
|
+
attrs,
|
|
5390
|
+
"exception.type",
|
|
5391
|
+
"error.type",
|
|
5392
|
+
"obs.tool.error_type",
|
|
5393
|
+
"http.response.status_code",
|
|
5394
|
+
"http.status_code",
|
|
5395
|
+
"rpc.grpc.status_code"
|
|
5396
|
+
) ?? span.statusMessage;
|
|
5397
|
+
if (!rawErr && span.eventsJson) {
|
|
5398
|
+
const events = parseJsonArray(span.eventsJson);
|
|
5399
|
+
for (const ev of events) {
|
|
5400
|
+
if (ev && typeof ev === "object" && !Array.isArray(ev)) {
|
|
5401
|
+
const evObj = ev;
|
|
5402
|
+
if (evObj.name === "exception" && evObj.attributes) {
|
|
5403
|
+
const evAttrs = typeof evObj.attributes === "object" && !Array.isArray(evObj.attributes) ? evObj.attributes : {};
|
|
5404
|
+
rawErr = pickStr(evAttrs, "exception.type", "exception.message") ?? rawErr;
|
|
5405
|
+
}
|
|
5406
|
+
}
|
|
5407
|
+
}
|
|
5408
|
+
}
|
|
5409
|
+
errorSignature = normalizeErrorSignature(
|
|
5410
|
+
rawErr ?? `ERROR:${span.spanName}`
|
|
5411
|
+
);
|
|
5412
|
+
}
|
|
5413
|
+
return {
|
|
5414
|
+
environment,
|
|
5415
|
+
namespace,
|
|
5416
|
+
serviceName,
|
|
5417
|
+
serviceVersion,
|
|
5418
|
+
commitSha,
|
|
5419
|
+
configDigest,
|
|
5420
|
+
sourceNode,
|
|
5421
|
+
targetNode,
|
|
5422
|
+
edgePredicate,
|
|
5423
|
+
errorSignature
|
|
5424
|
+
};
|
|
5425
|
+
}
|
|
5426
|
+
var NODE_UPSERT_CACHE = /* @__PURE__ */ new Map();
|
|
5427
|
+
var CACHE_TTL_MS = 6e4;
|
|
5428
|
+
async function upsertOntologyFromSpans(db, spans, options = {}) {
|
|
5429
|
+
if (spans.length === 0) return;
|
|
5430
|
+
const nowMs = Date.now();
|
|
5431
|
+
const statements = [];
|
|
5432
|
+
const edgeAggregates = /* @__PURE__ */ new Map();
|
|
5433
|
+
const serviceNodesToCheck = /* @__PURE__ */ new Map();
|
|
5434
|
+
for (const span of spans) {
|
|
5435
|
+
const lineage = extractSpanLineageAndTopology(span);
|
|
5436
|
+
const ts = span.receivedAt || span.startTime || (/* @__PURE__ */ new Date()).toISOString();
|
|
5437
|
+
if (lineage.sourceNode) {
|
|
5438
|
+
const cacheKey = `${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}::${lineage.serviceVersion ?? ""}::${lineage.commitSha ?? ""}::${lineage.configDigest ?? ""}`;
|
|
5439
|
+
const cachedAt = NODE_UPSERT_CACHE.get(cacheKey);
|
|
5440
|
+
if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
|
|
5441
|
+
NODE_UPSERT_CACHE.set(cacheKey, nowMs);
|
|
5442
|
+
serviceNodesToCheck.set(
|
|
5443
|
+
`${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}`,
|
|
5444
|
+
{
|
|
5445
|
+
projectId: span.projectId,
|
|
5446
|
+
environment: lineage.environment,
|
|
5447
|
+
node: lineage.sourceNode,
|
|
5448
|
+
version: lineage.serviceVersion,
|
|
5449
|
+
commitSha: lineage.commitSha,
|
|
5450
|
+
configDigest: lineage.configDigest,
|
|
5451
|
+
timestamp: ts
|
|
5452
|
+
}
|
|
5453
|
+
);
|
|
5454
|
+
}
|
|
5455
|
+
}
|
|
5456
|
+
if (lineage.targetNode) {
|
|
5457
|
+
const targetCacheKey = `${span.projectId}::${lineage.environment}::${lineage.targetNode.id}`;
|
|
5458
|
+
const cachedAt = NODE_UPSERT_CACHE.get(targetCacheKey);
|
|
5459
|
+
if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
|
|
5460
|
+
NODE_UPSERT_CACHE.set(targetCacheKey, nowMs);
|
|
5461
|
+
statements.push(
|
|
5462
|
+
db.prepare(
|
|
5463
|
+
`INSERT INTO ontology_nodes (
|
|
5464
|
+
project_id, environment, id, kind, name, namespace_or_endpoint,
|
|
5465
|
+
is_scoped_identity, attributes_json, first_seen_at, last_seen_at
|
|
5466
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
|
|
5467
|
+
ON CONFLICT (project_id, environment, id) DO UPDATE SET
|
|
5468
|
+
last_seen_at = excluded.last_seen_at`
|
|
5469
|
+
).bind(
|
|
5470
|
+
span.projectId,
|
|
5471
|
+
lineage.environment,
|
|
5472
|
+
lineage.targetNode.id,
|
|
5473
|
+
lineage.targetNode.kind,
|
|
5474
|
+
lineage.targetNode.name,
|
|
5475
|
+
lineage.targetNode.namespaceOrEndpoint,
|
|
5476
|
+
lineage.targetNode.isScopedIdentity ? 1 : 0,
|
|
5477
|
+
ts,
|
|
5478
|
+
ts
|
|
5479
|
+
)
|
|
5480
|
+
);
|
|
5481
|
+
}
|
|
5482
|
+
}
|
|
5483
|
+
if (lineage.sourceNode && lineage.targetNode && lineage.edgePredicate) {
|
|
5484
|
+
const edgeId = `${lineage.sourceNode.id}--${lineage.edgePredicate}-->${lineage.targetNode.id}`;
|
|
5485
|
+
const aggKey = `${span.projectId}::${lineage.environment}::${edgeId}`;
|
|
5486
|
+
const certainty = lineage.sourceNode.isScopedIdentity && lineage.targetNode.isScopedIdentity ? "exact" : "inferred";
|
|
5487
|
+
const existing = edgeAggregates.get(aggKey);
|
|
5488
|
+
const isErr = span.statusCode === 2 ? 1 : 0;
|
|
5489
|
+
if (existing) {
|
|
5490
|
+
existing.callCount += 1;
|
|
5491
|
+
existing.errorCount += isErr;
|
|
5492
|
+
existing.lastDurationMs = span.durationMs;
|
|
5493
|
+
if (ts > existing.lastSeenAt) existing.lastSeenAt = ts;
|
|
5494
|
+
} else {
|
|
5495
|
+
edgeAggregates.set(aggKey, {
|
|
5496
|
+
projectId: span.projectId,
|
|
5497
|
+
environment: lineage.environment,
|
|
5498
|
+
id: edgeId,
|
|
5499
|
+
sourceNodeId: lineage.sourceNode.id,
|
|
5500
|
+
predicate: lineage.edgePredicate,
|
|
5501
|
+
targetNodeId: lineage.targetNode.id,
|
|
5502
|
+
certainty,
|
|
5503
|
+
callCount: 1,
|
|
5504
|
+
errorCount: isErr,
|
|
5505
|
+
lastDurationMs: span.durationMs,
|
|
5506
|
+
firstSeenAt: ts,
|
|
5507
|
+
lastSeenAt: ts
|
|
5508
|
+
});
|
|
5509
|
+
}
|
|
5510
|
+
}
|
|
5511
|
+
}
|
|
5512
|
+
for (const entry of serviceNodesToCheck.values()) {
|
|
5513
|
+
if (entry.version) {
|
|
5514
|
+
const existingNode = await db.prepare(
|
|
5515
|
+
`SELECT current_version, current_commit_sha
|
|
5516
|
+
FROM ontology_nodes
|
|
5517
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
5518
|
+
).bind(entry.projectId, entry.environment, entry.node.id).first();
|
|
5519
|
+
if (existingNode?.current_version && existingNode.current_version !== entry.version) {
|
|
5520
|
+
const existingChange = await db.prepare(
|
|
5521
|
+
`SELECT id FROM change_events
|
|
5522
|
+
WHERE project_id = ? AND environment = ? AND target_node_id = ? AND version = ?
|
|
5523
|
+
LIMIT 1`
|
|
5524
|
+
).bind(
|
|
5525
|
+
entry.projectId,
|
|
5526
|
+
entry.environment,
|
|
5527
|
+
entry.node.id,
|
|
5528
|
+
entry.version
|
|
5529
|
+
).first();
|
|
5530
|
+
if (!existingChange) {
|
|
5531
|
+
const changeId = `chg_inf_${randomHex(8)}`;
|
|
5532
|
+
const expiresAt = new Date(
|
|
5533
|
+
Date.now() + 30 * 24 * 36e5
|
|
5534
|
+
).toISOString();
|
|
5535
|
+
statements.push(
|
|
5536
|
+
db.prepare(
|
|
5537
|
+
`INSERT INTO change_events (
|
|
5538
|
+
project_id, environment, id, target_node_id, kind, source,
|
|
5539
|
+
relationship_certainty, summary, service_name, version,
|
|
5540
|
+
previous_version, commit_sha, previous_commit_sha, config_digest,
|
|
5541
|
+
touched_paths_json, attributes_json, occurred_at, received_at, expires_at
|
|
5542
|
+
) VALUES (?, ?, ?, ?, 'deploy', 'inferred_telemetry', 'inferred', ?, ?, ?, ?, ?, ?, ?, '[]', '{}', ?, ?, ?)
|
|
5543
|
+
ON CONFLICT (project_id, environment, id) DO NOTHING`
|
|
5544
|
+
).bind(
|
|
5545
|
+
entry.projectId,
|
|
5546
|
+
entry.environment,
|
|
5547
|
+
changeId,
|
|
5548
|
+
entry.node.id,
|
|
5549
|
+
`Inferred version transition ${existingNode.current_version} -> ${entry.version} on ${entry.node.name}`,
|
|
5550
|
+
entry.node.name,
|
|
5551
|
+
entry.version,
|
|
5552
|
+
existingNode.current_version,
|
|
5553
|
+
entry.commitSha,
|
|
5554
|
+
existingNode.current_commit_sha,
|
|
5555
|
+
entry.configDigest,
|
|
5556
|
+
entry.timestamp,
|
|
5557
|
+
entry.timestamp,
|
|
5558
|
+
expiresAt
|
|
5559
|
+
)
|
|
5560
|
+
);
|
|
5561
|
+
}
|
|
5562
|
+
}
|
|
5563
|
+
}
|
|
5564
|
+
statements.push(
|
|
5565
|
+
db.prepare(
|
|
5566
|
+
`INSERT INTO ontology_nodes (
|
|
5567
|
+
project_id, environment, id, kind, name, namespace_or_endpoint,
|
|
5568
|
+
is_scoped_identity, current_version, current_commit_sha,
|
|
5569
|
+
current_config_digest, attributes_json, first_seen_at, last_seen_at
|
|
5570
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
|
|
5571
|
+
ON CONFLICT (project_id, environment, id) DO UPDATE SET
|
|
5572
|
+
current_version = COALESCE(excluded.current_version, ontology_nodes.current_version),
|
|
5573
|
+
current_commit_sha = COALESCE(excluded.current_commit_sha, ontology_nodes.current_commit_sha),
|
|
5574
|
+
current_config_digest = COALESCE(excluded.current_config_digest, ontology_nodes.current_config_digest),
|
|
5575
|
+
last_seen_at = excluded.last_seen_at`
|
|
5576
|
+
).bind(
|
|
5577
|
+
entry.projectId,
|
|
5578
|
+
entry.environment,
|
|
5579
|
+
entry.node.id,
|
|
5580
|
+
entry.node.kind,
|
|
5581
|
+
entry.node.name,
|
|
5582
|
+
entry.node.namespaceOrEndpoint,
|
|
5583
|
+
entry.node.isScopedIdentity ? 1 : 0,
|
|
5584
|
+
entry.version,
|
|
5585
|
+
entry.commitSha,
|
|
5586
|
+
entry.configDigest,
|
|
5587
|
+
entry.timestamp,
|
|
5588
|
+
entry.timestamp
|
|
5589
|
+
)
|
|
5590
|
+
);
|
|
5591
|
+
}
|
|
5592
|
+
for (const edge of edgeAggregates.values()) {
|
|
5593
|
+
statements.push(
|
|
5594
|
+
db.prepare(
|
|
5595
|
+
`INSERT INTO ontology_edges (
|
|
5596
|
+
project_id, environment, id, source_node_id, predicate, target_node_id,
|
|
5597
|
+
relationship_certainty, call_count, error_count, last_duration_ms,
|
|
5598
|
+
first_seen_at, last_seen_at
|
|
5599
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
5600
|
+
ON CONFLICT (project_id, environment, id) DO UPDATE SET
|
|
5601
|
+
call_count = ontology_edges.call_count + excluded.call_count,
|
|
5602
|
+
error_count = ontology_edges.error_count + excluded.error_count,
|
|
5603
|
+
last_duration_ms = excluded.last_duration_ms,
|
|
5604
|
+
last_seen_at = excluded.last_seen_at`
|
|
5605
|
+
).bind(
|
|
5606
|
+
edge.projectId,
|
|
5607
|
+
edge.environment,
|
|
5608
|
+
edge.id,
|
|
5609
|
+
edge.sourceNodeId,
|
|
5610
|
+
edge.predicate,
|
|
5611
|
+
edge.targetNodeId,
|
|
5612
|
+
edge.certainty,
|
|
5613
|
+
edge.callCount,
|
|
5614
|
+
edge.errorCount,
|
|
5615
|
+
edge.lastDurationMs,
|
|
5616
|
+
edge.firstSeenAt,
|
|
5617
|
+
edge.lastSeenAt
|
|
5618
|
+
)
|
|
5619
|
+
);
|
|
5620
|
+
}
|
|
5621
|
+
if (statements.length > 0) {
|
|
5622
|
+
await db.batch(statements);
|
|
5623
|
+
}
|
|
5624
|
+
}
|
|
5625
|
+
function computeTarjanScc(nodes, edges) {
|
|
5626
|
+
const adjacency = /* @__PURE__ */ new Map();
|
|
5627
|
+
const selfLoops = /* @__PURE__ */ new Set();
|
|
5628
|
+
for (const node of nodes) {
|
|
5629
|
+
adjacency.set(node.id, []);
|
|
5630
|
+
}
|
|
5631
|
+
for (const edge of edges) {
|
|
5632
|
+
if (!adjacency.has(edge.sourceNodeId)) {
|
|
5633
|
+
adjacency.set(edge.sourceNodeId, []);
|
|
5634
|
+
}
|
|
5635
|
+
if (!adjacency.has(edge.targetNodeId)) {
|
|
5636
|
+
adjacency.set(edge.targetNodeId, []);
|
|
5637
|
+
}
|
|
5638
|
+
adjacency.get(edge.sourceNodeId)?.push(edge.targetNodeId);
|
|
5639
|
+
if (edge.sourceNodeId === edge.targetNodeId) {
|
|
5640
|
+
selfLoops.add(edge.sourceNodeId);
|
|
5641
|
+
}
|
|
5642
|
+
}
|
|
5643
|
+
let index = 0;
|
|
5644
|
+
const indices = /* @__PURE__ */ new Map();
|
|
5645
|
+
const lowlink = /* @__PURE__ */ new Map();
|
|
5646
|
+
const onStack = /* @__PURE__ */ new Set();
|
|
5647
|
+
const stack = [];
|
|
5648
|
+
const rawComponents = [];
|
|
5649
|
+
const strongConnect = (nodeId) => {
|
|
5650
|
+
indices.set(nodeId, index);
|
|
5651
|
+
lowlink.set(nodeId, index);
|
|
5652
|
+
index += 1;
|
|
5653
|
+
stack.push(nodeId);
|
|
5654
|
+
onStack.add(nodeId);
|
|
5655
|
+
const neighbors = adjacency.get(nodeId) ?? [];
|
|
5656
|
+
for (const targetId of neighbors) {
|
|
5657
|
+
if (!indices.has(targetId)) {
|
|
5658
|
+
strongConnect(targetId);
|
|
5659
|
+
lowlink.set(
|
|
5660
|
+
nodeId,
|
|
5661
|
+
Math.min(lowlink.get(nodeId) ?? 0, lowlink.get(targetId) ?? 0)
|
|
5662
|
+
);
|
|
5663
|
+
} else if (onStack.has(targetId)) {
|
|
5664
|
+
lowlink.set(
|
|
5665
|
+
nodeId,
|
|
5666
|
+
Math.min(lowlink.get(nodeId) ?? 0, indices.get(targetId) ?? 0)
|
|
5667
|
+
);
|
|
5668
|
+
}
|
|
5669
|
+
}
|
|
5670
|
+
if (lowlink.get(nodeId) === indices.get(nodeId)) {
|
|
5671
|
+
const component = [];
|
|
5672
|
+
while (stack.length > 0) {
|
|
5673
|
+
const w = stack.pop();
|
|
5674
|
+
onStack.delete(w);
|
|
5675
|
+
component.push(w);
|
|
5676
|
+
if (w === nodeId) break;
|
|
5677
|
+
}
|
|
5678
|
+
component.sort();
|
|
5679
|
+
rawComponents.push(component);
|
|
5680
|
+
}
|
|
5681
|
+
};
|
|
5682
|
+
for (const nodeId of adjacency.keys()) {
|
|
5683
|
+
if (!indices.has(nodeId)) {
|
|
5684
|
+
strongConnect(nodeId);
|
|
5685
|
+
}
|
|
5686
|
+
}
|
|
5687
|
+
const nodeSccMap = /* @__PURE__ */ new Map();
|
|
5688
|
+
const components = rawComponents.map((nodeIds, idx) => {
|
|
5689
|
+
const sccId = `scc_${idx + 1}`;
|
|
5690
|
+
const isCyclic = nodeIds.length > 1 || nodeIds.length === 1 && selfLoops.has(nodeIds[0]);
|
|
5691
|
+
for (const id of nodeIds) {
|
|
5692
|
+
nodeSccMap.set(id, { sccId, isCyclic });
|
|
5693
|
+
}
|
|
5694
|
+
return { sccId, nodeIds, isCyclic };
|
|
5695
|
+
});
|
|
5696
|
+
const annotatedNodes = nodes.map((n) => {
|
|
5697
|
+
const scc = nodeSccMap.get(n.id);
|
|
5698
|
+
return {
|
|
5699
|
+
...n,
|
|
5700
|
+
sccId: scc?.sccId ?? "scc_0",
|
|
5701
|
+
inCycle: scc?.isCyclic ?? false
|
|
5702
|
+
};
|
|
5703
|
+
});
|
|
5704
|
+
return { annotatedNodes, components };
|
|
5705
|
+
}
|
|
5706
|
+
function localizeTraceFaultLocus(traceId, spans) {
|
|
5707
|
+
if (spans.length === 0) {
|
|
5708
|
+
return {
|
|
5709
|
+
faultLocusSpan: null,
|
|
5710
|
+
faultLocusNodeId: null,
|
|
5711
|
+
relationshipCertainty: "exact",
|
|
5712
|
+
causalProofTier: "unverified_locus",
|
|
5713
|
+
causalTriples: []
|
|
5714
|
+
};
|
|
5715
|
+
}
|
|
5716
|
+
const spanById = /* @__PURE__ */ new Map();
|
|
5717
|
+
const childrenByParent = /* @__PURE__ */ new Map();
|
|
5718
|
+
for (const s of spans) {
|
|
5719
|
+
spanById.set(s.span_id, s);
|
|
5720
|
+
if (s.parent_span_id) {
|
|
5721
|
+
const list = childrenByParent.get(s.parent_span_id) ?? [];
|
|
5722
|
+
list.push(s);
|
|
5723
|
+
childrenByParent.set(s.parent_span_id, list);
|
|
5724
|
+
}
|
|
5725
|
+
}
|
|
5726
|
+
for (const list of childrenByParent.values()) {
|
|
5727
|
+
list.sort((a, b) => a.start_time.localeCompare(b.start_time));
|
|
5728
|
+
}
|
|
5729
|
+
const roots = spans.filter((s) => !s.parent_span_id || !spanById.has(s.parent_span_id)).sort((a, b) => a.start_time.localeCompare(b.start_time));
|
|
5730
|
+
const visited = /* @__PURE__ */ new Set();
|
|
5731
|
+
const causalTriples = [];
|
|
5732
|
+
const leafErrorSpans = [];
|
|
5733
|
+
const MAX_DEPTH = 256;
|
|
5734
|
+
const walk = (span, depth) => {
|
|
5735
|
+
if (visited.has(span.span_id) || depth > MAX_DEPTH) return;
|
|
5736
|
+
visited.add(span.span_id);
|
|
5737
|
+
const children = (childrenByParent.get(span.span_id) ?? []).filter(
|
|
5738
|
+
(c) => !visited.has(c.span_id)
|
|
5739
|
+
);
|
|
5740
|
+
const errorChildren = children.filter((c) => c.status_code === 2);
|
|
5741
|
+
if (span.status_code === 2) {
|
|
5742
|
+
if (errorChildren.length === 0) {
|
|
5743
|
+
leafErrorSpans.push(span);
|
|
5744
|
+
} else {
|
|
5745
|
+
for (const errChild of errorChildren) {
|
|
5746
|
+
causalTriples.push({
|
|
5747
|
+
subjectKind: "span",
|
|
5748
|
+
subjectId: span.span_id,
|
|
5749
|
+
predicate: "failed_due_to_child_span",
|
|
5750
|
+
objectKind: "span",
|
|
5751
|
+
objectId: errChild.span_id,
|
|
5752
|
+
relationshipCertainty: "exact",
|
|
5753
|
+
causalProofTier: "unverified_locus",
|
|
5754
|
+
confidence: 0.85,
|
|
5755
|
+
reason: `Parent span ${span.span_name} failed while child span ${errChild.span_name} returned ERROR`
|
|
5756
|
+
});
|
|
5757
|
+
}
|
|
5758
|
+
}
|
|
5759
|
+
}
|
|
5760
|
+
for (const child of children) {
|
|
5761
|
+
walk(child, depth + 1);
|
|
5762
|
+
}
|
|
5763
|
+
};
|
|
5764
|
+
for (const root of roots) {
|
|
5765
|
+
walk(root, 0);
|
|
5766
|
+
}
|
|
5767
|
+
for (const s of spans) {
|
|
5768
|
+
if (!visited.has(s.span_id)) {
|
|
5769
|
+
walk(s, 0);
|
|
5770
|
+
}
|
|
5771
|
+
}
|
|
5772
|
+
leafErrorSpans.sort((a, b) => a.start_time.localeCompare(b.start_time));
|
|
5773
|
+
const faultLocusSpan = leafErrorSpans[0] ?? spans.find((s) => s.status_code === 2) ?? null;
|
|
5774
|
+
let faultLocusNodeId = null;
|
|
5775
|
+
if (faultLocusSpan) {
|
|
5776
|
+
const env = (faultLocusSpan.environment ?? "default").toLowerCase();
|
|
5777
|
+
faultLocusNodeId = faultLocusSpan.target_node_id ?? (faultLocusSpan.service_name ? buildCanonicalNodeId2({
|
|
5778
|
+
kind: "service",
|
|
5779
|
+
environment: env,
|
|
5780
|
+
serviceName: faultLocusSpan.service_name
|
|
5781
|
+
}).id : null);
|
|
5782
|
+
causalTriples.unshift({
|
|
5783
|
+
subjectKind: "trace",
|
|
5784
|
+
subjectId: traceId,
|
|
5785
|
+
predicate: "originated_fault_in_span",
|
|
5786
|
+
objectKind: "span",
|
|
5787
|
+
objectId: faultLocusSpan.span_id,
|
|
5788
|
+
relationshipCertainty: "exact",
|
|
5789
|
+
causalProofTier: "unverified_locus",
|
|
5790
|
+
confidence: 0.9,
|
|
5791
|
+
reason: `Deepest/earliest failing span ${faultLocusSpan.span_name} (${faultLocusSpan.service_name ?? "unknown"}) is the trace fault locus`
|
|
5792
|
+
});
|
|
5793
|
+
}
|
|
5794
|
+
return {
|
|
5795
|
+
faultLocusSpan,
|
|
5796
|
+
faultLocusNodeId,
|
|
5797
|
+
relationshipCertainty: "exact",
|
|
5798
|
+
causalProofTier: "unverified_locus",
|
|
5799
|
+
causalTriples
|
|
5800
|
+
};
|
|
5801
|
+
}
|
|
5802
|
+
async function loadTraceFaultLocusFromDb(db, projectId, traceId) {
|
|
5803
|
+
const rows = await db.prepare(
|
|
5804
|
+
`SELECT
|
|
5805
|
+
span_id, parent_span_id, service_name, span_name, span_kind,
|
|
5806
|
+
status_code, status_message, start_time, end_time, duration_ms,
|
|
5807
|
+
environment, target_node_id, error_signature
|
|
5808
|
+
FROM telemetry_spans
|
|
5809
|
+
WHERE project_id = ? AND trace_id = ?
|
|
5810
|
+
ORDER BY start_time ASC
|
|
5811
|
+
LIMIT 500`
|
|
5812
|
+
).bind(projectId, traceId).all();
|
|
5813
|
+
return localizeTraceFaultLocus(traceId, rows.results);
|
|
5814
|
+
}
|
|
5815
|
+
async function getOntologyGraph(db, options) {
|
|
5816
|
+
const env = (options.environment ?? "default").toLowerCase();
|
|
5817
|
+
const [nodeRows, edgeRows] = await Promise.all([
|
|
5818
|
+
db.prepare(
|
|
5819
|
+
`SELECT * FROM ontology_nodes
|
|
5820
|
+
WHERE project_id = ? AND environment = ?
|
|
5821
|
+
ORDER BY id ASC
|
|
5822
|
+
LIMIT 500`
|
|
5823
|
+
).bind(options.projectId, env).all(),
|
|
5824
|
+
db.prepare(
|
|
5825
|
+
`SELECT * FROM ontology_edges
|
|
5826
|
+
WHERE project_id = ? AND environment = ?
|
|
5827
|
+
ORDER BY call_count DESC
|
|
5828
|
+
LIMIT 1000`
|
|
5829
|
+
).bind(options.projectId, env).all()
|
|
5830
|
+
]);
|
|
5831
|
+
const rawNodes = nodeRows.results.map((r) => ({
|
|
5832
|
+
projectId: r.project_id,
|
|
5833
|
+
environment: r.environment,
|
|
5834
|
+
id: r.id,
|
|
5835
|
+
kind: r.kind,
|
|
5836
|
+
name: r.name,
|
|
5837
|
+
namespaceOrEndpoint: r.namespace_or_endpoint,
|
|
5838
|
+
isScopedIdentity: r.is_scoped_identity === 1,
|
|
5839
|
+
currentVersion: r.current_version,
|
|
5840
|
+
currentCommitSha: r.current_commit_sha,
|
|
5841
|
+
currentConfigDigest: r.current_config_digest,
|
|
5842
|
+
attributes: parseJsonRecord(r.attributes_json ?? "{}"),
|
|
5843
|
+
firstSeenAt: r.first_seen_at,
|
|
5844
|
+
lastSeenAt: r.last_seen_at
|
|
5845
|
+
}));
|
|
5846
|
+
const edges = edgeRows.results.map((r) => ({
|
|
5847
|
+
projectId: r.project_id,
|
|
5848
|
+
environment: r.environment,
|
|
5849
|
+
id: r.id,
|
|
5850
|
+
sourceNodeId: r.source_node_id,
|
|
5851
|
+
predicate: r.predicate,
|
|
5852
|
+
targetNodeId: r.target_node_id,
|
|
5853
|
+
relationshipCertainty: r.relationship_certainty || "exact",
|
|
5854
|
+
callCount: Number(r.call_count ?? 0),
|
|
5855
|
+
errorCount: Number(r.error_count ?? 0),
|
|
5856
|
+
lastDurationMs: r.last_duration_ms === null ? null : Number(r.last_duration_ms),
|
|
5857
|
+
firstSeenAt: r.first_seen_at,
|
|
5858
|
+
lastSeenAt: r.last_seen_at
|
|
5859
|
+
}));
|
|
5860
|
+
const { annotatedNodes, components } = computeTarjanScc(rawNodes, edges);
|
|
5861
|
+
const inboundErrorsByTarget = /* @__PURE__ */ new Map();
|
|
5862
|
+
for (const e of edges) {
|
|
5863
|
+
if (e.errorCount > 0) {
|
|
5864
|
+
inboundErrorsByTarget.set(
|
|
5865
|
+
e.targetNodeId,
|
|
5866
|
+
(inboundErrorsByTarget.get(e.targetNodeId) ?? 0) + e.errorCount
|
|
5867
|
+
);
|
|
5868
|
+
}
|
|
5869
|
+
}
|
|
5870
|
+
const nodes = annotatedNodes.map((n) => ({
|
|
5871
|
+
...n,
|
|
5872
|
+
isBlackoutSuspected: (inboundErrorsByTarget.get(n.id) ?? 0) >= 3 && edges.filter((e) => e.sourceNodeId === n.id).length === 0
|
|
5873
|
+
}));
|
|
5874
|
+
return {
|
|
5875
|
+
environment: env,
|
|
5876
|
+
nodes,
|
|
5877
|
+
edges,
|
|
5878
|
+
stronglyConnectedComponents: components
|
|
5879
|
+
};
|
|
5880
|
+
}
|
|
5881
|
+
|
|
3644
5882
|
// src/lib/store/ingest.ts
|
|
3645
5883
|
async function ingestTelemetrySpans(db, spans) {
|
|
3646
5884
|
if (spans.length === 0) return { inserted: 0, traceCount: 0 };
|
|
3647
|
-
const
|
|
5885
|
+
const enrichedSpans = spans.map((span) => {
|
|
3648
5886
|
if (!span.projectId)
|
|
3649
5887
|
throw new Error("TelemetryStore.ingest: span.projectId is required");
|
|
3650
|
-
|
|
5888
|
+
if (span.environment !== void 0 && span.serviceVersion !== void 0 && span.targetNodeId !== void 0) {
|
|
5889
|
+
return span;
|
|
5890
|
+
}
|
|
5891
|
+
const lineage = extractSpanLineageAndTopology(span);
|
|
5892
|
+
return {
|
|
5893
|
+
...span,
|
|
5894
|
+
environment: span.environment ?? lineage.environment,
|
|
5895
|
+
serviceVersion: span.serviceVersion ?? lineage.serviceVersion,
|
|
5896
|
+
commitSha: span.commitSha ?? lineage.commitSha,
|
|
5897
|
+
targetNodeId: span.targetNodeId ?? lineage.targetNode?.id ?? null,
|
|
5898
|
+
errorSignature: span.errorSignature ?? lineage.errorSignature
|
|
5899
|
+
};
|
|
5900
|
+
});
|
|
5901
|
+
const statements = enrichedSpans.map(
|
|
5902
|
+
(span) => db.prepare(`
|
|
3651
5903
|
INSERT OR IGNORE INTO telemetry_spans (
|
|
3652
5904
|
project_id, trace_id, span_id, parent_span_id, trace_state,
|
|
3653
5905
|
service_name, scope_name, scope_version, span_name, span_kind,
|
|
@@ -3655,8 +5907,9 @@ async function ingestTelemetrySpans(db, spans) {
|
|
|
3655
5907
|
attributes_json, dropped_attributes_count,
|
|
3656
5908
|
resource_attributes_json, events_json, dropped_events_count,
|
|
3657
5909
|
links_json, dropped_links_count, received_at, expires_at,
|
|
3658
|
-
session_id, interaction_id, telemetry_sdk_name
|
|
3659
|
-
|
|
5910
|
+
session_id, interaction_id, telemetry_sdk_name,
|
|
5911
|
+
environment, service_version, commit_sha, target_node_id, error_signature
|
|
5912
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
3660
5913
|
`).bind(
|
|
3661
5914
|
span.projectId,
|
|
3662
5915
|
span.traceId,
|
|
@@ -3684,10 +5937,69 @@ async function ingestTelemetrySpans(db, spans) {
|
|
|
3684
5937
|
span.expiresAt,
|
|
3685
5938
|
span.sessionId ?? null,
|
|
3686
5939
|
span.interactionId ?? null,
|
|
3687
|
-
span.telemetrySdkName ?? null
|
|
3688
|
-
|
|
3689
|
-
|
|
3690
|
-
|
|
5940
|
+
span.telemetrySdkName ?? null,
|
|
5941
|
+
span.environment ?? "default",
|
|
5942
|
+
span.serviceVersion ?? null,
|
|
5943
|
+
span.commitSha ?? null,
|
|
5944
|
+
span.targetNodeId ?? null,
|
|
5945
|
+
span.errorSignature ?? null
|
|
5946
|
+
)
|
|
5947
|
+
);
|
|
5948
|
+
try {
|
|
5949
|
+
await db.batch(statements);
|
|
5950
|
+
} catch (err) {
|
|
5951
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
5952
|
+
if (msg.includes("no such column: environment")) {
|
|
5953
|
+
const legacyStatements = enrichedSpans.map(
|
|
5954
|
+
(span) => db.prepare(`
|
|
5955
|
+
INSERT OR IGNORE INTO telemetry_spans (
|
|
5956
|
+
project_id, trace_id, span_id, parent_span_id, trace_state,
|
|
5957
|
+
service_name, scope_name, scope_version, span_name, span_kind,
|
|
5958
|
+
status_code, status_message, start_time, end_time, duration_ms,
|
|
5959
|
+
attributes_json, dropped_attributes_count,
|
|
5960
|
+
resource_attributes_json, events_json, dropped_events_count,
|
|
5961
|
+
links_json, dropped_links_count, received_at, expires_at,
|
|
5962
|
+
session_id, interaction_id, telemetry_sdk_name
|
|
5963
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
5964
|
+
`).bind(
|
|
5965
|
+
span.projectId,
|
|
5966
|
+
span.traceId,
|
|
5967
|
+
span.spanId,
|
|
5968
|
+
span.parentSpanId,
|
|
5969
|
+
span.traceState,
|
|
5970
|
+
span.serviceName,
|
|
5971
|
+
span.scopeName,
|
|
5972
|
+
span.scopeVersion,
|
|
5973
|
+
span.spanName,
|
|
5974
|
+
span.spanKind,
|
|
5975
|
+
span.statusCode,
|
|
5976
|
+
span.statusMessage,
|
|
5977
|
+
span.startTime,
|
|
5978
|
+
span.endTime,
|
|
5979
|
+
span.durationMs,
|
|
5980
|
+
span.attributesJson,
|
|
5981
|
+
span.droppedAttributesCount,
|
|
5982
|
+
span.resourceAttributesJson,
|
|
5983
|
+
span.eventsJson,
|
|
5984
|
+
span.droppedEventsCount,
|
|
5985
|
+
span.linksJson,
|
|
5986
|
+
span.droppedLinksCount,
|
|
5987
|
+
span.receivedAt,
|
|
5988
|
+
span.expiresAt,
|
|
5989
|
+
span.sessionId ?? null,
|
|
5990
|
+
span.interactionId ?? null,
|
|
5991
|
+
span.telemetrySdkName ?? null
|
|
5992
|
+
)
|
|
5993
|
+
);
|
|
5994
|
+
await db.batch(legacyStatements);
|
|
5995
|
+
} else {
|
|
5996
|
+
throw err;
|
|
5997
|
+
}
|
|
5998
|
+
}
|
|
5999
|
+
try {
|
|
6000
|
+
await upsertOntologyFromSpans(db, enrichedSpans);
|
|
6001
|
+
} catch {
|
|
6002
|
+
}
|
|
3691
6003
|
const uniqueTraces = new Set(spans.map((s) => `${s.projectId}:${s.traceId}`));
|
|
3692
6004
|
return {
|
|
3693
6005
|
inserted: spans.length,
|
|
@@ -8533,118 +10845,24 @@ var aiSpanPayloadsProcessorPlugin = {
|
|
|
8533
10845
|
r.receivedAt,
|
|
8534
10846
|
r.expiresAt,
|
|
8535
10847
|
r.interactionId,
|
|
8536
|
-
r.actionId
|
|
8537
|
-
)
|
|
8538
|
-
)
|
|
8539
|
-
);
|
|
8540
|
-
} catch (err) {
|
|
8541
|
-
context.logger.error("[ai-span-payloads-processor] write failed", {
|
|
8542
|
-
error: err instanceof Error ? err.message : String(err)
|
|
8543
|
-
});
|
|
8544
|
-
}
|
|
8545
|
-
}
|
|
8546
|
-
return transformed;
|
|
8547
|
-
}
|
|
8548
|
-
});
|
|
8549
|
-
}
|
|
8550
|
-
};
|
|
8551
|
-
|
|
8552
|
-
// src/lib/alerts-store.ts
|
|
8553
|
-
import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT4 } from "@obsunified/types";
|
|
8554
|
-
|
|
8555
|
-
// src/lib/webhook-url.ts
|
|
8556
|
-
var WebhookUrlError = class extends Error {
|
|
8557
|
-
};
|
|
8558
|
-
var parseIpv4 = (host) => {
|
|
8559
|
-
const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
|
|
8560
|
-
if (!m) return null;
|
|
8561
|
-
const octets = m.slice(1).map((o) => Number.parseInt(o, 10));
|
|
8562
|
-
return octets.every((o) => o <= 255) ? octets : null;
|
|
8563
|
-
};
|
|
8564
|
-
var isPrivateIpv4 = (o) => o[0] === 0 || // "this" network / unspecified
|
|
8565
|
-
o[0] === 10 || o[0] === 127 || o[0] === 100 && o[1] >= 64 && o[1] <= 127 || // CGNAT
|
|
8566
|
-
o[0] === 169 && o[1] === 254 || // link-local incl. cloud metadata
|
|
8567
|
-
o[0] === 172 && o[1] >= 16 && o[1] <= 31 || o[0] === 192 && o[1] === 168 || o[0] >= 224;
|
|
8568
|
-
var isPrivateIpv6 = (host) => {
|
|
8569
|
-
const literal = host.replace(/^\[|\]$/g, "").toLowerCase();
|
|
8570
|
-
if (literal === "::" || literal === "::1") return true;
|
|
8571
|
-
const dotted = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(literal);
|
|
8572
|
-
if (dotted) {
|
|
8573
|
-
const v4 = parseIpv4(dotted[1]);
|
|
8574
|
-
return v4 ? isPrivateIpv4(v4) : true;
|
|
8575
|
-
}
|
|
8576
|
-
const hex = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(literal);
|
|
8577
|
-
if (hex) {
|
|
8578
|
-
const hi = Number.parseInt(hex[1], 16);
|
|
8579
|
-
const lo = Number.parseInt(hex[2], 16);
|
|
8580
|
-
return isPrivateIpv4([hi >> 8, hi & 255, lo >> 8, lo & 255]);
|
|
8581
|
-
}
|
|
8582
|
-
const first = Number.parseInt(literal.split(":")[0] || "0", 16);
|
|
8583
|
-
if (Number.isNaN(first)) return true;
|
|
8584
|
-
if ((first & 65024) === 64512) return true;
|
|
8585
|
-
if ((first & 65472) === 65152) return true;
|
|
8586
|
-
if ((first & 65280) === 65280) return true;
|
|
8587
|
-
return false;
|
|
8588
|
-
};
|
|
8589
|
-
var isPrivateWebhookHost = (hostname) => {
|
|
8590
|
-
const host = hostname.toLowerCase().replace(/\.$/, "");
|
|
8591
|
-
if (!host) return true;
|
|
8592
|
-
if (host === "localhost" || host.endsWith(".localhost")) return true;
|
|
8593
|
-
if (host.endsWith(".local") || host.endsWith(".internal")) return true;
|
|
8594
|
-
if (host.startsWith("[")) return isPrivateIpv6(host);
|
|
8595
|
-
const v4 = parseIpv4(host);
|
|
8596
|
-
if (v4) return isPrivateIpv4(v4);
|
|
8597
|
-
return !host.includes(".");
|
|
8598
|
-
};
|
|
8599
|
-
var assertSafeWebhookUrl = (raw, options = {}) => {
|
|
8600
|
-
let url;
|
|
8601
|
-
try {
|
|
8602
|
-
url = new URL(raw);
|
|
8603
|
-
} catch {
|
|
8604
|
-
throw new WebhookUrlError("channel.url must be an absolute http(s) URL");
|
|
8605
|
-
}
|
|
8606
|
-
if (url.protocol !== "https:" && url.protocol !== "http:") {
|
|
8607
|
-
throw new WebhookUrlError("channel.url must be http(s)://");
|
|
8608
|
-
}
|
|
8609
|
-
if (url.username || url.password) {
|
|
8610
|
-
throw new WebhookUrlError(
|
|
8611
|
-
"channel.url must not embed credentials; use channel.headers"
|
|
8612
|
-
);
|
|
8613
|
-
}
|
|
8614
|
-
if (!options.allowPrivateNetworks && isPrivateWebhookHost(url.hostname)) {
|
|
8615
|
-
throw new WebhookUrlError(
|
|
8616
|
-
`channel.url host "${url.hostname}" is not a public address (set ALERT_WEBHOOK_ALLOW_PRIVATE=true to allow internal receivers)`
|
|
8617
|
-
);
|
|
8618
|
-
}
|
|
8619
|
-
return url;
|
|
8620
|
-
};
|
|
8621
|
-
var resolveWebhookHost = async (hostname, lookup) => {
|
|
8622
|
-
if (parseIpv4(hostname) || hostname.startsWith("[")) return "public";
|
|
8623
|
-
let addresses;
|
|
8624
|
-
try {
|
|
8625
|
-
addresses = await lookup(hostname);
|
|
8626
|
-
} catch {
|
|
8627
|
-
return "unresolvable";
|
|
8628
|
-
}
|
|
8629
|
-
if (addresses.length === 0) return "unresolvable";
|
|
8630
|
-
const isPrivate = addresses.some((address) => {
|
|
8631
|
-
const v4 = parseIpv4(address);
|
|
8632
|
-
if (v4) return isPrivateIpv4(v4);
|
|
8633
|
-
return isPrivateIpv6(`[${address}]`);
|
|
8634
|
-
});
|
|
8635
|
-
return isPrivate ? "private" : "public";
|
|
8636
|
-
};
|
|
8637
|
-
var redactWebhookUrl = (raw) => {
|
|
8638
|
-
try {
|
|
8639
|
-
const url = new URL(raw);
|
|
8640
|
-
const hasPath = url.pathname !== "/" || url.search !== "";
|
|
8641
|
-
return `${url.protocol}//${url.host}${hasPath ? "/\u2026" : ""}`;
|
|
8642
|
-
} catch {
|
|
8643
|
-
return "<invalid url>";
|
|
10848
|
+
r.actionId
|
|
10849
|
+
)
|
|
10850
|
+
)
|
|
10851
|
+
);
|
|
10852
|
+
} catch (err) {
|
|
10853
|
+
context.logger.error("[ai-span-payloads-processor] write failed", {
|
|
10854
|
+
error: err instanceof Error ? err.message : String(err)
|
|
10855
|
+
});
|
|
10856
|
+
}
|
|
10857
|
+
}
|
|
10858
|
+
return transformed;
|
|
10859
|
+
}
|
|
10860
|
+
});
|
|
8644
10861
|
}
|
|
8645
10862
|
};
|
|
8646
10863
|
|
|
8647
10864
|
// src/lib/alerts-store.ts
|
|
10865
|
+
import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT4 } from "@obsunified/types";
|
|
8648
10866
|
var parseJsonField = (value) => {
|
|
8649
10867
|
if (typeof value === "string") return JSON.parse(value);
|
|
8650
10868
|
return value;
|
|
@@ -9120,7 +11338,7 @@ async function mapWithConcurrency(items, concurrency, mapper) {
|
|
|
9120
11338
|
);
|
|
9121
11339
|
return results;
|
|
9122
11340
|
}
|
|
9123
|
-
async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery) {
|
|
11341
|
+
async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery, onAlertFiring) {
|
|
9124
11342
|
const none = { fired: 0, resolved: 0, redelivered: 0 };
|
|
9125
11343
|
try {
|
|
9126
11344
|
const value = await withTimeout(
|
|
@@ -9158,6 +11376,9 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
|
|
|
9158
11376
|
);
|
|
9159
11377
|
if (previous !== next) {
|
|
9160
11378
|
await store.transitionState(rule.id, rule.projectId, next, now);
|
|
11379
|
+
if (next === "firing" && onAlertFiring) {
|
|
11380
|
+
await onAlertFiring(rule).catch(() => void 0);
|
|
11381
|
+
}
|
|
9161
11382
|
const ok = await deliver(next);
|
|
9162
11383
|
if (ok) {
|
|
9163
11384
|
await store.markNotified(rule.id, rule.projectId, next, now);
|
|
@@ -9226,7 +11447,8 @@ async function evaluateRuleBatch(store, rules, options) {
|
|
|
9226
11447
|
logger,
|
|
9227
11448
|
options?.tracer,
|
|
9228
11449
|
ruleTimeoutMs,
|
|
9229
|
-
delivery
|
|
11450
|
+
delivery,
|
|
11451
|
+
options?.onAlertFiring
|
|
9230
11452
|
)
|
|
9231
11453
|
);
|
|
9232
11454
|
return {
|
|
@@ -9240,7 +11462,8 @@ async function evaluateAllRules(env, options) {
|
|
|
9240
11462
|
const logger = options?.logger ?? consoleLogger;
|
|
9241
11463
|
const tracer = options?.tracer;
|
|
9242
11464
|
const allowPrivateWebhookTargets = env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true";
|
|
9243
|
-
const
|
|
11465
|
+
const db = sqlDbFor(env);
|
|
11466
|
+
const store = new AlertsStore(db, { allowPrivateWebhookTargets });
|
|
9244
11467
|
const rules = await store.listEnabledRules();
|
|
9245
11468
|
return evaluateRuleBatch(store, rules, {
|
|
9246
11469
|
logger,
|
|
@@ -9248,7 +11471,15 @@ async function evaluateAllRules(env, options) {
|
|
|
9248
11471
|
concurrency: options?.concurrency,
|
|
9249
11472
|
ruleTimeoutMs: options?.ruleTimeoutMs,
|
|
9250
11473
|
allowPrivateWebhookTargets,
|
|
9251
|
-
hostLookup: options?.hostLookup
|
|
11474
|
+
hostLookup: options?.hostLookup,
|
|
11475
|
+
onAlertFiring: async (rule) => {
|
|
11476
|
+
await openOrDeduplicateIncident(db, rule.projectId, {
|
|
11477
|
+
environment: "default",
|
|
11478
|
+
targetNodeId: `alert:${rule.id}`,
|
|
11479
|
+
symptomAnchorKind: "alert",
|
|
11480
|
+
symptomAnchorId: rule.id
|
|
11481
|
+
});
|
|
11482
|
+
}
|
|
9252
11483
|
});
|
|
9253
11484
|
}
|
|
9254
11485
|
function createAlertEvaluatorHandler() {
|
|
@@ -9542,7 +11773,7 @@ var summarizeResult = (def, result) => {
|
|
|
9542
11773
|
};
|
|
9543
11774
|
async function runAsk(question, deps) {
|
|
9544
11775
|
if (deps.llm.provider === "openai") {
|
|
9545
|
-
const { runAskOpenAI } = await import("./openai-
|
|
11776
|
+
const { runAskOpenAI } = await import("./openai-GSJZGFGL.js");
|
|
9546
11777
|
return runAskOpenAI(question, deps);
|
|
9547
11778
|
}
|
|
9548
11779
|
return runAskAnthropic(question, deps);
|
|
@@ -9717,149 +11948,1288 @@ async function callAnthropic(config, messages, turnIndex) {
|
|
|
9717
11948
|
json.usage.output_tokens
|
|
9718
11949
|
);
|
|
9719
11950
|
}
|
|
9720
|
-
if (span && json.stop_reason)
|
|
9721
|
-
span.setAttribute("gen_ai.response.finish_reason", json.stop_reason);
|
|
9722
|
-
return json;
|
|
9723
|
-
};
|
|
9724
|
-
if (!config.tracer) return exec(null);
|
|
9725
|
-
const attrs = {
|
|
9726
|
-
"openinference.span.kind": "LLM",
|
|
9727
|
-
"gen_ai.system": "anthropic",
|
|
9728
|
-
"gen_ai.request.model": config.model,
|
|
9729
|
-
"gen_ai.request.max_tokens": 1024,
|
|
9730
|
-
"http.url": url
|
|
11951
|
+
if (span && json.stop_reason)
|
|
11952
|
+
span.setAttribute("gen_ai.response.finish_reason", json.stop_reason);
|
|
11953
|
+
return json;
|
|
11954
|
+
};
|
|
11955
|
+
if (!config.tracer) return exec(null);
|
|
11956
|
+
const attrs = {
|
|
11957
|
+
"openinference.span.kind": "LLM",
|
|
11958
|
+
"gen_ai.system": "anthropic",
|
|
11959
|
+
"gen_ai.request.model": config.model,
|
|
11960
|
+
"gen_ai.request.max_tokens": 1024,
|
|
11961
|
+
"http.url": url
|
|
11962
|
+
};
|
|
11963
|
+
if (typeof turnIndex === "number") attrs["llm.turn"] = turnIndex;
|
|
11964
|
+
return config.tracer(
|
|
11965
|
+
"llm.anthropic.messages",
|
|
11966
|
+
async (span) => exec(span),
|
|
11967
|
+
attrs
|
|
11968
|
+
);
|
|
11969
|
+
}
|
|
11970
|
+
|
|
11971
|
+
// src/plugins/ask-routes.ts
|
|
11972
|
+
var MAX_QUESTION_CHARS = 1e3;
|
|
11973
|
+
var askRoutesPlugin = {
|
|
11974
|
+
name: "ask-routes",
|
|
11975
|
+
register(app, runtime) {
|
|
11976
|
+
app.post("/internal/ask", async (c) => {
|
|
11977
|
+
const body = await readJsonObject(c);
|
|
11978
|
+
if (!body) {
|
|
11979
|
+
return c.json(
|
|
11980
|
+
{ error: "Bad Request", message: "body must be a JSON object" },
|
|
11981
|
+
400
|
|
11982
|
+
);
|
|
11983
|
+
}
|
|
11984
|
+
const question = (body.question ?? "").trim();
|
|
11985
|
+
if (!question) {
|
|
11986
|
+
return c.json(
|
|
11987
|
+
{ error: "Bad Request", message: "question is required" },
|
|
11988
|
+
400
|
|
11989
|
+
);
|
|
11990
|
+
}
|
|
11991
|
+
if (question.length > MAX_QUESTION_CHARS) {
|
|
11992
|
+
return c.json(
|
|
11993
|
+
{
|
|
11994
|
+
error: "Bad Request",
|
|
11995
|
+
message: `question exceeds ${MAX_QUESTION_CHARS} chars`
|
|
11996
|
+
},
|
|
11997
|
+
400
|
|
11998
|
+
);
|
|
11999
|
+
}
|
|
12000
|
+
const projectId = getProjectId(c);
|
|
12001
|
+
const openaiBase = c.env.OPENAI_BASE_URL?.trim() ? c.env.OPENAI_BASE_URL : void 0;
|
|
12002
|
+
const narrativeModel = c.env.NARRATIVE_MODEL?.trim() ? c.env.NARRATIVE_MODEL : void 0;
|
|
12003
|
+
const llm = c.env.OPENAI_API_KEY?.trim() ? {
|
|
12004
|
+
provider: "openai",
|
|
12005
|
+
apiKey: c.env.OPENAI_API_KEY,
|
|
12006
|
+
model: narrativeModel ?? "gpt-4o-mini",
|
|
12007
|
+
apiUrl: openaiBase,
|
|
12008
|
+
tracer: runtime.withChildSpan
|
|
12009
|
+
} : c.env.ANTHROPIC_API_KEY?.trim() ? {
|
|
12010
|
+
provider: "anthropic",
|
|
12011
|
+
apiKey: c.env.ANTHROPIC_API_KEY,
|
|
12012
|
+
model: narrativeModel ?? "claude-haiku-4-5",
|
|
12013
|
+
tracer: runtime.withChildSpan
|
|
12014
|
+
} : null;
|
|
12015
|
+
if (!llm) {
|
|
12016
|
+
const response = {
|
|
12017
|
+
answer: null,
|
|
12018
|
+
evidence: [],
|
|
12019
|
+
queries: [],
|
|
12020
|
+
error: "Ask is not configured \u2014 set OPENAI_API_KEY or ANTHROPIC_API_KEY on the collector to enable it.",
|
|
12021
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
12022
|
+
};
|
|
12023
|
+
return c.json(response, 503);
|
|
12024
|
+
}
|
|
12025
|
+
const store = new AnalysesStore(sqlDbFor(c.env));
|
|
12026
|
+
try {
|
|
12027
|
+
const startedAt = Date.now();
|
|
12028
|
+
runtime.logger.info("[ask] llm call starting", {
|
|
12029
|
+
project_id: projectId,
|
|
12030
|
+
provider: llm.provider,
|
|
12031
|
+
model: llm.model,
|
|
12032
|
+
question_chars: question.length
|
|
12033
|
+
});
|
|
12034
|
+
const result = await runtime.withChildSpan(
|
|
12035
|
+
"ask.runAsk",
|
|
12036
|
+
() => runAsk(question, {
|
|
12037
|
+
llm,
|
|
12038
|
+
listAnalyses: async (filters) => {
|
|
12039
|
+
const all = await getAllAnalysesForProject(projectId, {
|
|
12040
|
+
db: sqlDbFor(c.env)
|
|
12041
|
+
});
|
|
12042
|
+
return all.filter((d) => {
|
|
12043
|
+
if (filters?.group && d.group !== filters.group) return false;
|
|
12044
|
+
if (filters?.view && d.view !== filters.view) return false;
|
|
12045
|
+
return true;
|
|
12046
|
+
});
|
|
12047
|
+
},
|
|
12048
|
+
getLatestResult: async (id) => {
|
|
12049
|
+
const definitions = await store.listDefinitions(projectId);
|
|
12050
|
+
const definition = definitions.find((d) => d.id === id);
|
|
12051
|
+
if (!definition) return null;
|
|
12052
|
+
const r = await store.getLatestResult(projectId, id);
|
|
12053
|
+
return { definition, result: r };
|
|
12054
|
+
}
|
|
12055
|
+
}),
|
|
12056
|
+
{
|
|
12057
|
+
"llm.provider": llm.provider,
|
|
12058
|
+
"llm.model": llm.model,
|
|
12059
|
+
"openinference.span.kind": "LLM"
|
|
12060
|
+
}
|
|
12061
|
+
);
|
|
12062
|
+
runtime.logger.info("[ask] llm call finished", {
|
|
12063
|
+
project_id: projectId,
|
|
12064
|
+
provider: llm.provider,
|
|
12065
|
+
model: llm.model,
|
|
12066
|
+
latency_ms: Date.now() - startedAt,
|
|
12067
|
+
evidence_count: result.evidence.length,
|
|
12068
|
+
query_count: result.queries.length,
|
|
12069
|
+
answered: Boolean(result.answer)
|
|
12070
|
+
});
|
|
12071
|
+
if (result.answer && result.evidence.length > 0) {
|
|
12072
|
+
await store.recordAskEvidence(
|
|
12073
|
+
projectId,
|
|
12074
|
+
result.evidence.map((e) => e.analysisId)
|
|
12075
|
+
).catch(
|
|
12076
|
+
(err) => runtime.logger.warn("[ask] recordAskEvidence failed", {
|
|
12077
|
+
error: err instanceof Error ? err.message : String(err)
|
|
12078
|
+
})
|
|
12079
|
+
);
|
|
12080
|
+
}
|
|
12081
|
+
return c.json(result);
|
|
12082
|
+
} catch (error) {
|
|
12083
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
12084
|
+
runtime.logger.error("[ask] failed", { error: message });
|
|
12085
|
+
const response = {
|
|
12086
|
+
answer: null,
|
|
12087
|
+
evidence: [],
|
|
12088
|
+
queries: [],
|
|
12089
|
+
error: message,
|
|
12090
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
12091
|
+
};
|
|
12092
|
+
return c.json(response, 500);
|
|
12093
|
+
}
|
|
12094
|
+
});
|
|
12095
|
+
}
|
|
12096
|
+
};
|
|
12097
|
+
|
|
12098
|
+
// src/plugins/autosre-routes.ts
|
|
12099
|
+
import { AgentAutonomyLevel as AgentAutonomyLevel2 } from "@obsunified/types/constants";
|
|
12100
|
+
|
|
12101
|
+
// src/lib/autosre-change-correlation.ts
|
|
12102
|
+
import { buildCanonicalNodeId as buildCanonicalNodeId3 } from "@obsunified/types";
|
|
12103
|
+
var mapChangeEventRow = (row) => ({
|
|
12104
|
+
id: row.id,
|
|
12105
|
+
projectId: row.project_id,
|
|
12106
|
+
environment: row.environment,
|
|
12107
|
+
targetNodeId: row.target_node_id,
|
|
12108
|
+
kind: row.kind,
|
|
12109
|
+
source: row.source,
|
|
12110
|
+
relationshipCertainty: row.relationship_certainty || "exact",
|
|
12111
|
+
summary: row.summary,
|
|
12112
|
+
serviceName: row.service_name,
|
|
12113
|
+
version: row.version,
|
|
12114
|
+
previousVersion: row.previous_version,
|
|
12115
|
+
commitSha: row.commit_sha,
|
|
12116
|
+
previousCommitSha: row.previous_commit_sha,
|
|
12117
|
+
configDigest: row.config_digest,
|
|
12118
|
+
repositoryUrl: row.repository_url,
|
|
12119
|
+
pullRequestUrl: row.pull_request_url,
|
|
12120
|
+
actorId: row.actor_id,
|
|
12121
|
+
causedByActionId: row.caused_by_action_id,
|
|
12122
|
+
agentRunId: row.agent_run_id,
|
|
12123
|
+
touchedPaths: parseJsonArray(row.touched_paths_json ?? "[]").filter(
|
|
12124
|
+
(p) => typeof p === "string"
|
|
12125
|
+
),
|
|
12126
|
+
attributes: parseJsonRecord(row.attributes_json ?? "{}"),
|
|
12127
|
+
occurredAt: row.occurred_at,
|
|
12128
|
+
receivedAt: row.received_at,
|
|
12129
|
+
expiresAt: row.expires_at
|
|
12130
|
+
});
|
|
12131
|
+
async function recordChangeEvent(db, projectId, input) {
|
|
12132
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
12133
|
+
const occurredAt = input.occurredAt || nowIso;
|
|
12134
|
+
const environment = (input.environment || "default").trim().toLowerCase();
|
|
12135
|
+
const namespace = (input.namespace || "default").trim();
|
|
12136
|
+
const serviceName = input.serviceName?.trim() || null;
|
|
12137
|
+
const canonicalNode = input.targetNodeId ? {
|
|
12138
|
+
id: input.targetNodeId,
|
|
12139
|
+
environment,
|
|
12140
|
+
kind: input.kind === "feature_flag" ? "feature_flag" : "service",
|
|
12141
|
+
name: serviceName ?? input.targetNodeId,
|
|
12142
|
+
namespaceOrEndpoint: namespace,
|
|
12143
|
+
isScopedIdentity: true
|
|
12144
|
+
} : input.kind === "feature_flag" ? buildCanonicalNodeId3({
|
|
12145
|
+
kind: "feature_flag",
|
|
12146
|
+
environment,
|
|
12147
|
+
flagKey: (typeof input.attributes?.flagKey === "string" ? input.attributes.flagKey : serviceName) ?? "flag"
|
|
12148
|
+
}) : buildCanonicalNodeId3({
|
|
12149
|
+
kind: "service",
|
|
12150
|
+
environment,
|
|
12151
|
+
namespace,
|
|
12152
|
+
serviceName: serviceName ?? "unknown-service"
|
|
12153
|
+
});
|
|
12154
|
+
const id = input.id?.trim() || `chg_${randomHex(10)}`;
|
|
12155
|
+
const source = input.source ?? "explicit_api";
|
|
12156
|
+
const certainty = input.relationshipCertainty ?? (source === "inferred_telemetry" ? "inferred" : "exact");
|
|
12157
|
+
const touchedPaths = (input.touchedPaths ?? []).slice(0, 500);
|
|
12158
|
+
const expiresAt = new Date(
|
|
12159
|
+
Date.parse(occurredAt) + 30 * 24 * 36e5
|
|
12160
|
+
).toISOString();
|
|
12161
|
+
await db.batch([
|
|
12162
|
+
db.prepare(
|
|
12163
|
+
`INSERT INTO ontology_nodes (
|
|
12164
|
+
project_id, environment, id, kind, name, namespace_or_endpoint,
|
|
12165
|
+
is_scoped_identity, current_version, current_commit_sha,
|
|
12166
|
+
current_config_digest, attributes_json, first_seen_at, last_seen_at
|
|
12167
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
|
|
12168
|
+
ON CONFLICT (project_id, environment, id) DO UPDATE SET
|
|
12169
|
+
current_version = COALESCE(excluded.current_version, ontology_nodes.current_version),
|
|
12170
|
+
current_commit_sha = COALESCE(excluded.current_commit_sha, ontology_nodes.current_commit_sha),
|
|
12171
|
+
current_config_digest = COALESCE(excluded.current_config_digest, ontology_nodes.current_config_digest),
|
|
12172
|
+
last_seen_at = excluded.last_seen_at`
|
|
12173
|
+
).bind(
|
|
12174
|
+
projectId,
|
|
12175
|
+
environment,
|
|
12176
|
+
canonicalNode.id,
|
|
12177
|
+
canonicalNode.kind,
|
|
12178
|
+
canonicalNode.name,
|
|
12179
|
+
canonicalNode.namespaceOrEndpoint,
|
|
12180
|
+
canonicalNode.isScopedIdentity ? 1 : 0,
|
|
12181
|
+
input.version ?? null,
|
|
12182
|
+
input.commitSha ?? null,
|
|
12183
|
+
input.configDigest ?? null,
|
|
12184
|
+
occurredAt,
|
|
12185
|
+
nowIso
|
|
12186
|
+
),
|
|
12187
|
+
db.prepare(
|
|
12188
|
+
`INSERT INTO change_events (
|
|
12189
|
+
project_id, environment, id, target_node_id, kind, source,
|
|
12190
|
+
relationship_certainty, summary, service_name, version,
|
|
12191
|
+
previous_version, commit_sha, previous_commit_sha, config_digest,
|
|
12192
|
+
repository_url, pull_request_url, actor_id, caused_by_action_id,
|
|
12193
|
+
agent_run_id, touched_paths_json, attributes_json,
|
|
12194
|
+
occurred_at, received_at, expires_at
|
|
12195
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
12196
|
+
ON CONFLICT (project_id, environment, id) DO UPDATE SET
|
|
12197
|
+
summary = excluded.summary,
|
|
12198
|
+
version = COALESCE(excluded.version, change_events.version),
|
|
12199
|
+
previous_version = COALESCE(excluded.previous_version, change_events.previous_version),
|
|
12200
|
+
commit_sha = COALESCE(excluded.commit_sha, change_events.commit_sha),
|
|
12201
|
+
touched_paths_json = excluded.touched_paths_json,
|
|
12202
|
+
attributes_json = excluded.attributes_json,
|
|
12203
|
+
occurred_at = excluded.occurred_at`
|
|
12204
|
+
).bind(
|
|
12205
|
+
projectId,
|
|
12206
|
+
environment,
|
|
12207
|
+
id,
|
|
12208
|
+
canonicalNode.id,
|
|
12209
|
+
input.kind,
|
|
12210
|
+
source,
|
|
12211
|
+
certainty,
|
|
12212
|
+
input.summary,
|
|
12213
|
+
serviceName,
|
|
12214
|
+
input.version ?? null,
|
|
12215
|
+
input.previousVersion ?? null,
|
|
12216
|
+
input.commitSha ?? null,
|
|
12217
|
+
input.previousCommitSha ?? null,
|
|
12218
|
+
input.configDigest ?? null,
|
|
12219
|
+
input.repositoryUrl ?? null,
|
|
12220
|
+
input.pullRequestUrl ?? null,
|
|
12221
|
+
input.actorId ?? null,
|
|
12222
|
+
input.causedByActionId ?? null,
|
|
12223
|
+
input.agentRunId ?? null,
|
|
12224
|
+
JSON.stringify(touchedPaths),
|
|
12225
|
+
JSON.stringify(input.attributes ?? {}),
|
|
12226
|
+
occurredAt,
|
|
12227
|
+
nowIso,
|
|
12228
|
+
expiresAt
|
|
12229
|
+
)
|
|
12230
|
+
]);
|
|
12231
|
+
return {
|
|
12232
|
+
id,
|
|
12233
|
+
projectId,
|
|
12234
|
+
environment,
|
|
12235
|
+
targetNodeId: canonicalNode.id,
|
|
12236
|
+
kind: input.kind,
|
|
12237
|
+
source,
|
|
12238
|
+
relationshipCertainty: certainty,
|
|
12239
|
+
summary: input.summary,
|
|
12240
|
+
serviceName,
|
|
12241
|
+
version: input.version ?? null,
|
|
12242
|
+
previousVersion: input.previousVersion ?? null,
|
|
12243
|
+
commitSha: input.commitSha ?? null,
|
|
12244
|
+
previousCommitSha: input.previousCommitSha ?? null,
|
|
12245
|
+
configDigest: input.configDigest ?? null,
|
|
12246
|
+
repositoryUrl: input.repositoryUrl ?? null,
|
|
12247
|
+
pullRequestUrl: input.pullRequestUrl ?? null,
|
|
12248
|
+
actorId: input.actorId ?? null,
|
|
12249
|
+
causedByActionId: input.causedByActionId ?? null,
|
|
12250
|
+
agentRunId: input.agentRunId ?? null,
|
|
12251
|
+
touchedPaths,
|
|
12252
|
+
attributes: input.attributes ?? {},
|
|
12253
|
+
occurredAt,
|
|
12254
|
+
receivedAt: nowIso,
|
|
12255
|
+
expiresAt
|
|
12256
|
+
};
|
|
12257
|
+
}
|
|
12258
|
+
async function listChangeEvents(db, options) {
|
|
12259
|
+
const conditions = ["project_id = ?"];
|
|
12260
|
+
const binds = [options.projectId];
|
|
12261
|
+
if (options.environment) {
|
|
12262
|
+
conditions.push("environment = ?");
|
|
12263
|
+
binds.push(options.environment.toLowerCase());
|
|
12264
|
+
}
|
|
12265
|
+
if (options.serviceName) {
|
|
12266
|
+
conditions.push("service_name = ?");
|
|
12267
|
+
binds.push(options.serviceName);
|
|
12268
|
+
}
|
|
12269
|
+
if (options.targetNodeId) {
|
|
12270
|
+
conditions.push("target_node_id = ?");
|
|
12271
|
+
binds.push(options.targetNodeId);
|
|
12272
|
+
}
|
|
12273
|
+
if (options.kind) {
|
|
12274
|
+
conditions.push("kind = ?");
|
|
12275
|
+
binds.push(options.kind);
|
|
12276
|
+
}
|
|
12277
|
+
if (options.since) {
|
|
12278
|
+
conditions.push("occurred_at >= ?");
|
|
12279
|
+
binds.push(options.since);
|
|
12280
|
+
}
|
|
12281
|
+
const limit = Math.max(1, Math.min(200, options.limit ?? 50));
|
|
12282
|
+
binds.push(limit);
|
|
12283
|
+
const rows = await db.prepare(
|
|
12284
|
+
`SELECT * FROM change_events
|
|
12285
|
+
WHERE ${conditions.join(" AND ")}
|
|
12286
|
+
ORDER BY occurred_at DESC
|
|
12287
|
+
LIMIT ?`
|
|
12288
|
+
).bind(...binds).all();
|
|
12289
|
+
return rows.results.map(mapChangeEventRow);
|
|
12290
|
+
}
|
|
12291
|
+
async function getChangeEvent(db, projectId, changeId, environment) {
|
|
12292
|
+
const row = environment ? await db.prepare(
|
|
12293
|
+
`SELECT * FROM change_events
|
|
12294
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
12295
|
+
).bind(projectId, environment.toLowerCase(), changeId).first() : await db.prepare(
|
|
12296
|
+
`SELECT * FROM change_events
|
|
12297
|
+
WHERE project_id = ? AND id = ?
|
|
12298
|
+
ORDER BY occurred_at DESC
|
|
12299
|
+
LIMIT 1`
|
|
12300
|
+
).bind(projectId, changeId).first();
|
|
12301
|
+
return row ? mapChangeEventRow(row) : null;
|
|
12302
|
+
}
|
|
12303
|
+
var computePercentile = (sorted, p) => {
|
|
12304
|
+
if (sorted.length === 0) return null;
|
|
12305
|
+
const idx = Math.min(
|
|
12306
|
+
sorted.length - 1,
|
|
12307
|
+
Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
|
|
12308
|
+
);
|
|
12309
|
+
return sorted[idx];
|
|
12310
|
+
};
|
|
12311
|
+
var summarizeCohort = (cohortKey, requestSpans, downstreamSpans) => {
|
|
12312
|
+
const requestCount = requestSpans.length;
|
|
12313
|
+
const errorCount = requestSpans.filter((s) => s.status_code === 2).length;
|
|
12314
|
+
const errorRate = requestCount > 0 ? errorCount / requestCount : 0;
|
|
12315
|
+
const durations = requestSpans.map((s) => Number(s.duration_ms ?? 0)).sort((a, b) => a - b);
|
|
12316
|
+
const p95DurationMs = computePercentile(durations, 95);
|
|
12317
|
+
const downstreamCount = downstreamSpans.length;
|
|
12318
|
+
const avgDownstreamCallsPerRequest = requestCount > 0 ? downstreamCount / requestCount : 0;
|
|
12319
|
+
const avgDownstreamDurationMs = downstreamCount > 0 ? downstreamSpans.reduce((acc, s) => acc + Number(s.duration_ms ?? 0), 0) / downstreamCount : null;
|
|
12320
|
+
return {
|
|
12321
|
+
cohortKey,
|
|
12322
|
+
requestCount,
|
|
12323
|
+
errorCount,
|
|
12324
|
+
errorRate,
|
|
12325
|
+
p95DurationMs,
|
|
12326
|
+
avgDownstreamCallsPerRequest,
|
|
12327
|
+
avgDownstreamDurationMs
|
|
12328
|
+
};
|
|
12329
|
+
};
|
|
12330
|
+
async function findCrossTraceContention(db, options) {
|
|
12331
|
+
const env = options.environment.toLowerCase();
|
|
12332
|
+
const limit = Math.max(1, Math.min(20, options.limit ?? 5));
|
|
12333
|
+
const [nodeRow, contentionRows] = await Promise.all([
|
|
12334
|
+
db.prepare(
|
|
12335
|
+
`SELECT is_scoped_identity FROM ontology_nodes
|
|
12336
|
+
WHERE project_id = ? AND environment = ? AND id = ?`
|
|
12337
|
+
).bind(options.projectId, env, options.targetNodeId).first(),
|
|
12338
|
+
db.prepare(
|
|
12339
|
+
`SELECT
|
|
12340
|
+
trace_id,
|
|
12341
|
+
span_id,
|
|
12342
|
+
service_name,
|
|
12343
|
+
service_version,
|
|
12344
|
+
span_name,
|
|
12345
|
+
COUNT(*) AS concurrent_calls,
|
|
12346
|
+
MAX(duration_ms) AS max_duration_ms,
|
|
12347
|
+
SUM(duration_ms) AS total_resource_time_ms
|
|
12348
|
+
FROM telemetry_spans
|
|
12349
|
+
WHERE project_id = ?
|
|
12350
|
+
AND environment = ?
|
|
12351
|
+
AND target_node_id = ?
|
|
12352
|
+
AND trace_id <> ?
|
|
12353
|
+
AND start_time <= ?
|
|
12354
|
+
AND end_time >= ?
|
|
12355
|
+
GROUP BY trace_id, span_id, service_name, service_version, span_name
|
|
12356
|
+
ORDER BY total_resource_time_ms DESC
|
|
12357
|
+
LIMIT ?`
|
|
12358
|
+
).bind(
|
|
12359
|
+
options.projectId,
|
|
12360
|
+
env,
|
|
12361
|
+
options.targetNodeId,
|
|
12362
|
+
options.excludeTraceId,
|
|
12363
|
+
options.endTime,
|
|
12364
|
+
options.startTime,
|
|
12365
|
+
limit
|
|
12366
|
+
).all()
|
|
12367
|
+
]);
|
|
12368
|
+
const isScoped = nodeRow ? nodeRow.is_scoped_identity === 1 : !options.targetNodeId.includes(":unscoped/");
|
|
12369
|
+
const relationshipCertainty = isScoped ? "exact" : "inferred";
|
|
12370
|
+
return contentionRows.results.map((r) => ({
|
|
12371
|
+
aggressorTraceId: r.trace_id,
|
|
12372
|
+
aggressorSpanId: r.span_id,
|
|
12373
|
+
aggressorServiceName: r.service_name,
|
|
12374
|
+
aggressorServiceVersion: r.service_version,
|
|
12375
|
+
aggressorSpanName: r.span_name,
|
|
12376
|
+
sharedTargetNodeId: options.targetNodeId,
|
|
12377
|
+
relationshipCertainty,
|
|
12378
|
+
concurrentCalls: Number(r.concurrent_calls ?? 1),
|
|
12379
|
+
maxDurationMs: Number(r.max_duration_ms ?? 0),
|
|
12380
|
+
totalResourceTimeMs: Number(r.total_resource_time_ms ?? 0)
|
|
12381
|
+
}));
|
|
12382
|
+
}
|
|
12383
|
+
var extractFilePathsFromSpan = (span) => {
|
|
12384
|
+
const found = [];
|
|
12385
|
+
const attrs = parseJsonRecord(span.attributes_json ?? "{}");
|
|
12386
|
+
for (const key of ["code.filepath", "obs.code.filepath", "exception.stacktrace"]) {
|
|
12387
|
+
const val = attrs[key];
|
|
12388
|
+
if (typeof val === "string") {
|
|
12389
|
+
found.push(val);
|
|
12390
|
+
}
|
|
12391
|
+
}
|
|
12392
|
+
if (span.events_json) {
|
|
12393
|
+
const events = parseJsonArray(span.events_json);
|
|
12394
|
+
for (const ev of events) {
|
|
12395
|
+
if (ev && typeof ev === "object" && !Array.isArray(ev)) {
|
|
12396
|
+
const evAttrs = parseJsonRecord(
|
|
12397
|
+
JSON.stringify(ev.attributes ?? {})
|
|
12398
|
+
);
|
|
12399
|
+
for (const key of ["code.filepath", "exception.stacktrace"]) {
|
|
12400
|
+
if (typeof evAttrs[key] === "string") {
|
|
12401
|
+
found.push(evAttrs[key]);
|
|
12402
|
+
}
|
|
12403
|
+
}
|
|
12404
|
+
}
|
|
12405
|
+
}
|
|
12406
|
+
}
|
|
12407
|
+
return found;
|
|
12408
|
+
};
|
|
12409
|
+
async function computeChangeBlastRadius(db, projectId, changeId, options = {}) {
|
|
12410
|
+
const change = await getChangeEvent(
|
|
12411
|
+
db,
|
|
12412
|
+
projectId,
|
|
12413
|
+
changeId,
|
|
12414
|
+
options.environment
|
|
12415
|
+
);
|
|
12416
|
+
if (!change) return null;
|
|
12417
|
+
const minSample = options.minComparableSample ?? 50;
|
|
12418
|
+
const preWindowStart = new Date(
|
|
12419
|
+
Date.parse(change.occurredAt) - 30 * 6e4
|
|
12420
|
+
).toISOString();
|
|
12421
|
+
const serviceFilter = change.serviceName;
|
|
12422
|
+
const spansResult = serviceFilter ? await db.prepare(
|
|
12423
|
+
`SELECT
|
|
12424
|
+
trace_id, span_id, parent_span_id, span_kind, status_code,
|
|
12425
|
+
duration_ms, service_version, target_node_id, attributes_json,
|
|
12426
|
+
events_json, start_time, end_time, received_at
|
|
12427
|
+
FROM telemetry_spans
|
|
12428
|
+
WHERE project_id = ?
|
|
12429
|
+
AND environment = ?
|
|
12430
|
+
AND service_name = ?
|
|
12431
|
+
AND received_at >= ?
|
|
12432
|
+
ORDER BY received_at DESC
|
|
12433
|
+
LIMIT 5000`
|
|
12434
|
+
).bind(projectId, change.environment, serviceFilter, preWindowStart).all() : await db.prepare(
|
|
12435
|
+
`SELECT
|
|
12436
|
+
trace_id, span_id, parent_span_id, span_kind, status_code,
|
|
12437
|
+
duration_ms, service_version, target_node_id, attributes_json,
|
|
12438
|
+
events_json, start_time, end_time, received_at
|
|
12439
|
+
FROM telemetry_spans
|
|
12440
|
+
WHERE project_id = ?
|
|
12441
|
+
AND environment = ?
|
|
12442
|
+
AND received_at >= ?
|
|
12443
|
+
ORDER BY received_at DESC
|
|
12444
|
+
LIMIT 5000`
|
|
12445
|
+
).bind(projectId, change.environment, preWindowStart).all();
|
|
12446
|
+
const allSpans = spansResult.results;
|
|
12447
|
+
const postSpans = allSpans.filter(
|
|
12448
|
+
(s) => (s.start_time || s.received_at) >= change.occurredAt
|
|
12449
|
+
);
|
|
12450
|
+
const preSpans = allSpans.filter(
|
|
12451
|
+
(s) => (s.start_time || s.received_at) < change.occurredAt
|
|
12452
|
+
);
|
|
12453
|
+
const isRootOrServer = (s) => s.parent_span_id === null || s.span_kind === 1;
|
|
12454
|
+
const isDownstreamCall = (s) => Boolean(s.target_node_id) || s.span_kind === 2 || s.span_kind === 4;
|
|
12455
|
+
const flagKey = change.kind === "feature_flag" ? (typeof change.attributes.flagKey === "string" ? change.attributes.flagKey : change.serviceName) ?? "flag" : null;
|
|
12456
|
+
const newCohortKey = change.version ?? (typeof change.attributes.newVariant === "string" ? change.attributes.newVariant : "true");
|
|
12457
|
+
const oldCohortKey = change.previousVersion ?? (typeof change.attributes.previousVariant === "string" ? change.attributes.previousVariant : "false");
|
|
12458
|
+
const getCohortOfSpan = (s) => {
|
|
12459
|
+
if (flagKey) {
|
|
12460
|
+
const attrs = parseJsonRecord(s.attributes_json ?? "{}");
|
|
12461
|
+
const direct = attrs[`obs.feature_flags.${flagKey}`];
|
|
12462
|
+
if (direct !== void 0 && direct !== null) return String(direct);
|
|
12463
|
+
if (attrs["feature_flag.key"] === flagKey && attrs["feature_flag.variant"] !== void 0) {
|
|
12464
|
+
return String(attrs["feature_flag.variant"]);
|
|
12465
|
+
}
|
|
12466
|
+
const nested = attrs["obs.feature_flags"];
|
|
12467
|
+
if (nested && typeof nested === "object" && !Array.isArray(nested)) {
|
|
12468
|
+
const v = nested[flagKey];
|
|
12469
|
+
if (v !== void 0 && v !== null) return String(v);
|
|
12470
|
+
}
|
|
12471
|
+
return null;
|
|
12472
|
+
}
|
|
12473
|
+
return s.service_version;
|
|
12474
|
+
};
|
|
12475
|
+
const postNewReq = postSpans.filter(
|
|
12476
|
+
(s) => isRootOrServer(s) && getCohortOfSpan(s) === newCohortKey
|
|
12477
|
+
);
|
|
12478
|
+
const postNewDown = postSpans.filter(
|
|
12479
|
+
(s) => isDownstreamCall(s) && getCohortOfSpan(s) === newCohortKey
|
|
12480
|
+
);
|
|
12481
|
+
const postOldReq = postSpans.filter(
|
|
12482
|
+
(s) => isRootOrServer(s) && (oldCohortKey ? getCohortOfSpan(s) === oldCohortKey : getCohortOfSpan(s) !== null && getCohortOfSpan(s) !== newCohortKey)
|
|
12483
|
+
);
|
|
12484
|
+
const postOldDown = postSpans.filter(
|
|
12485
|
+
(s) => isDownstreamCall(s) && (oldCohortKey ? getCohortOfSpan(s) === oldCohortKey : getCohortOfSpan(s) !== null && getCohortOfSpan(s) !== newCohortKey)
|
|
12486
|
+
);
|
|
12487
|
+
const preOldReq = preSpans.filter(
|
|
12488
|
+
(s) => isRootOrServer(s) && (!oldCohortKey || getCohortOfSpan(s) === oldCohortKey)
|
|
12489
|
+
);
|
|
12490
|
+
const preOldDown = preSpans.filter(
|
|
12491
|
+
(s) => isDownstreamCall(s) && (!oldCohortKey || getCohortOfSpan(s) === oldCohortKey)
|
|
12492
|
+
);
|
|
12493
|
+
const newCohort = summarizeCohort(newCohortKey, postNewReq, postNewDown);
|
|
12494
|
+
const oldCohort = summarizeCohort(oldCohortKey, postOldReq, postOldDown);
|
|
12495
|
+
const preBaseline = summarizeCohort(
|
|
12496
|
+
`${oldCohortKey}:pre`,
|
|
12497
|
+
preOldReq,
|
|
12498
|
+
preOldDown
|
|
12499
|
+
);
|
|
12500
|
+
const overlappingSet = /* @__PURE__ */ new Set();
|
|
12501
|
+
const matchingErrorSpanIds = [];
|
|
12502
|
+
if (change.touchedPaths.length > 0) {
|
|
12503
|
+
for (const s of postNewReq.filter((sp) => sp.status_code === 2)) {
|
|
12504
|
+
const rawPaths = extractFilePathsFromSpan(s);
|
|
12505
|
+
let matchedSpan = false;
|
|
12506
|
+
for (const touched of change.touchedPaths) {
|
|
12507
|
+
if (rawPaths.some((raw) => raw.includes(touched))) {
|
|
12508
|
+
overlappingSet.add(touched);
|
|
12509
|
+
matchedSpan = true;
|
|
12510
|
+
}
|
|
12511
|
+
}
|
|
12512
|
+
if (matchedSpan) {
|
|
12513
|
+
matchingErrorSpanIds.push(s.span_id);
|
|
12514
|
+
}
|
|
12515
|
+
}
|
|
12516
|
+
}
|
|
12517
|
+
const overlappingCodePaths = Array.from(overlappingSet);
|
|
12518
|
+
let cohortVerdict = "insufficient_sample";
|
|
12519
|
+
let causalProofTier = "unverified_locus";
|
|
12520
|
+
let rateDifference = 0;
|
|
12521
|
+
let relativeRisk = 1;
|
|
12522
|
+
const sharedResourceAggressors = [];
|
|
12523
|
+
if (newCohort.requestCount < minSample) {
|
|
12524
|
+
cohortVerdict = "insufficient_sample";
|
|
12525
|
+
causalProofTier = overlappingCodePaths.length > 0 ? "corroborated_candidate" : "unverified_locus";
|
|
12526
|
+
} else if (oldCohort.requestCount >= minSample) {
|
|
12527
|
+
rateDifference = newCohort.errorRate - oldCohort.errorRate;
|
|
12528
|
+
relativeRisk = newCohort.errorRate / Math.max(oldCohort.errorRate, 1e-3);
|
|
12529
|
+
const fanoutRatio = newCohort.avgDownstreamCallsPerRequest / Math.max(oldCohort.avgDownstreamCallsPerRequest, 0.25);
|
|
12530
|
+
const durationRatio = (newCohort.avgDownstreamDurationMs ?? 0) / Math.max(oldCohort.avgDownstreamDurationMs ?? 1, 1);
|
|
12531
|
+
const hasNewVersionAggressorSignal = fanoutRatio >= 3 || durationRatio >= 3;
|
|
12532
|
+
const hasPreChangeDegradationSignal = preBaseline.requestCount >= 10 && oldCohort.errorRate > 1.5 * Math.max(preBaseline.errorRate, 0.01);
|
|
12533
|
+
const bothDegraded = newCohort.errorRate >= 0.05 && oldCohort.errorRate >= 0.05 && (hasNewVersionAggressorSignal || hasPreChangeDegradationSignal);
|
|
12534
|
+
if (bothDegraded) {
|
|
12535
|
+
if (hasNewVersionAggressorSignal) {
|
|
12536
|
+
cohortVerdict = "shared_degradation_new_version_aggressor";
|
|
12537
|
+
causalProofTier = "corroborated_candidate";
|
|
12538
|
+
const sampleTarget = postNewDown.find(
|
|
12539
|
+
(d) => d.target_node_id
|
|
12540
|
+
)?.target_node_id;
|
|
12541
|
+
if (sampleTarget) {
|
|
12542
|
+
const victimTrace = postOldReq.find((r) => r.status_code === 2)?.trace_id ?? "victim";
|
|
12543
|
+
const candidates = await findCrossTraceContention(db, {
|
|
12544
|
+
projectId,
|
|
12545
|
+
environment: change.environment,
|
|
12546
|
+
targetNodeId: sampleTarget,
|
|
12547
|
+
excludeTraceId: victimTrace,
|
|
12548
|
+
startTime: change.occurredAt,
|
|
12549
|
+
endTime: (/* @__PURE__ */ new Date()).toISOString()
|
|
12550
|
+
});
|
|
12551
|
+
sharedResourceAggressors.push(...candidates);
|
|
12552
|
+
}
|
|
12553
|
+
} else {
|
|
12554
|
+
cohortVerdict = "shared_degradation_external_cause";
|
|
12555
|
+
causalProofTier = "unverified_locus";
|
|
12556
|
+
}
|
|
12557
|
+
} else {
|
|
12558
|
+
const p95Ratio = newCohort.p95DurationMs !== null && oldCohort.p95DurationMs !== null && oldCohort.p95DurationMs > 0 ? newCohort.p95DurationMs / oldCohort.p95DurationMs : 1;
|
|
12559
|
+
if ((rateDifference >= 0.05 || p95Ratio >= 2.5) && (relativeRisk >= 5 || p95Ratio >= 2.5)) {
|
|
12560
|
+
cohortVerdict = "isolated_to_new_version";
|
|
12561
|
+
causalProofTier = "cohort_rate_isolated";
|
|
12562
|
+
} else {
|
|
12563
|
+
cohortVerdict = "no_regression_detected";
|
|
12564
|
+
causalProofTier = "unverified_locus";
|
|
12565
|
+
}
|
|
12566
|
+
}
|
|
12567
|
+
} else if (preBaseline.requestCount >= minSample) {
|
|
12568
|
+
rateDifference = newCohort.errorRate - preBaseline.errorRate;
|
|
12569
|
+
relativeRisk = newCohort.errorRate / Math.max(preBaseline.errorRate, 1e-3);
|
|
12570
|
+
const p95Ratio = newCohort.p95DurationMs !== null && preBaseline.p95DurationMs !== null && preBaseline.p95DurationMs > 0 ? newCohort.p95DurationMs / preBaseline.p95DurationMs : 1;
|
|
12571
|
+
if ((rateDifference >= 0.05 || p95Ratio >= 2.5) && (relativeRisk >= 5 || p95Ratio >= 2.5)) {
|
|
12572
|
+
cohortVerdict = "sequential_post_cutover_regression";
|
|
12573
|
+
causalProofTier = "corroborated_candidate";
|
|
12574
|
+
} else {
|
|
12575
|
+
cohortVerdict = "no_regression_detected";
|
|
12576
|
+
causalProofTier = "unverified_locus";
|
|
12577
|
+
}
|
|
12578
|
+
} else if (overlappingCodePaths.length > 0) {
|
|
12579
|
+
cohortVerdict = "insufficient_sample";
|
|
12580
|
+
causalProofTier = "corroborated_candidate";
|
|
12581
|
+
}
|
|
12582
|
+
const relationshipCertainty = overlappingCodePaths.length > 0 ? "exact" : change.relationshipCertainty;
|
|
12583
|
+
const summary = `Change ${change.id} (${change.kind} ${newCohortKey} in ${change.environment}): verdict=${cohortVerdict}, proofTier=${causalProofTier}, newErrorRate=${(newCohort.errorRate * 100).toFixed(1)}% (N=${newCohort.requestCount}), oldErrorRate=${(oldCohort.errorRate * 100).toFixed(1)}% (N=${oldCohort.requestCount})`;
|
|
12584
|
+
return {
|
|
12585
|
+
change,
|
|
12586
|
+
cohortVerdict,
|
|
12587
|
+
causalProofTier,
|
|
12588
|
+
relationshipCertainty,
|
|
12589
|
+
minComparableSample: minSample,
|
|
12590
|
+
newCohort,
|
|
12591
|
+
oldCohort,
|
|
12592
|
+
preChangeBaselineCohort: preBaseline,
|
|
12593
|
+
rateDifference,
|
|
12594
|
+
relativeRisk,
|
|
12595
|
+
overlappingCodePaths,
|
|
12596
|
+
matchingErrorSpanIds,
|
|
12597
|
+
sharedResourceAggressors,
|
|
12598
|
+
summary
|
|
9731
12599
|
};
|
|
9732
|
-
if (typeof turnIndex === "number") attrs["llm.turn"] = turnIndex;
|
|
9733
|
-
return config.tracer(
|
|
9734
|
-
"llm.anthropic.messages",
|
|
9735
|
-
async (span) => exec(span),
|
|
9736
|
-
attrs
|
|
9737
|
-
);
|
|
9738
12600
|
}
|
|
9739
12601
|
|
|
9740
|
-
// src/plugins/
|
|
9741
|
-
var
|
|
9742
|
-
|
|
9743
|
-
|
|
12602
|
+
// src/plugins/autosre-routes.ts
|
|
12603
|
+
var CHANGE_KINDS = /* @__PURE__ */ new Set([
|
|
12604
|
+
"deploy",
|
|
12605
|
+
"rollback",
|
|
12606
|
+
"feature_flag",
|
|
12607
|
+
"config_change",
|
|
12608
|
+
"prompt_rollout",
|
|
12609
|
+
"agent_version",
|
|
12610
|
+
"schema_migration",
|
|
12611
|
+
"infra_scale"
|
|
12612
|
+
]);
|
|
12613
|
+
var REMEDIATION_ACTION_KINDS = /* @__PURE__ */ new Set([
|
|
12614
|
+
"open_code_pr",
|
|
12615
|
+
"toggle_feature_flag",
|
|
12616
|
+
"rollback_deploy",
|
|
12617
|
+
"apply_config",
|
|
12618
|
+
"promote_agent_prompt"
|
|
12619
|
+
]);
|
|
12620
|
+
var parseLimit = (raw, fallback = 50) => {
|
|
12621
|
+
const parsed = Number.parseInt(raw ?? String(fallback), 10);
|
|
12622
|
+
if (!Number.isFinite(parsed)) return fallback;
|
|
12623
|
+
return Math.max(1, Math.min(200, parsed));
|
|
12624
|
+
};
|
|
12625
|
+
var isAgentAutonomyLevel = (value) => typeof value === "string" && Object.values(AgentAutonomyLevel2).includes(value);
|
|
12626
|
+
function requireHumanDashboardSession(c) {
|
|
12627
|
+
const authHeader = c.req.header("Authorization");
|
|
12628
|
+
if (authHeader) {
|
|
12629
|
+
return {
|
|
12630
|
+
ok: false,
|
|
12631
|
+
message: "Human approval requires an authenticated human dashboard session (obs_session cookie); agent/API bearer tokens cannot self-approve or reject remediation proposals."
|
|
12632
|
+
};
|
|
12633
|
+
}
|
|
12634
|
+
const cookieHeader = c.req.header("Cookie") ?? "";
|
|
12635
|
+
const match = cookieHeader.match(/(?:^|;\s*)obs_session=([^;]*)/);
|
|
12636
|
+
if (!match || !match[1]) {
|
|
12637
|
+
return {
|
|
12638
|
+
ok: false,
|
|
12639
|
+
message: "Human approval requires an authenticated human dashboard session (obs_session cookie)."
|
|
12640
|
+
};
|
|
12641
|
+
}
|
|
12642
|
+
const explicitActor = c.req.header("X-Obs-User-Actor")?.trim();
|
|
12643
|
+
const rawSession = decodeURIComponent(match[1]);
|
|
12644
|
+
const sessionActor = explicitActor || `dashboard_session:${rawSession.slice(0, 24)}`;
|
|
12645
|
+
return { ok: true, sessionActor };
|
|
12646
|
+
}
|
|
12647
|
+
var autosreRoutesPlugin = {
|
|
12648
|
+
name: "autosre-routes",
|
|
9744
12649
|
register(app, runtime) {
|
|
9745
|
-
app.post("/
|
|
12650
|
+
app.post("/v1/changes", async (c) => {
|
|
12651
|
+
const projectId = getProjectId(c);
|
|
9746
12652
|
const body = await readJsonObject(c);
|
|
9747
12653
|
if (!body) {
|
|
12654
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
12655
|
+
}
|
|
12656
|
+
const rawKind = typeof body.kind === "string" ? body.kind.trim() : "deploy";
|
|
12657
|
+
const normalizedKind = rawKind === "agent_prompt_release" ? "prompt_rollout" : rawKind;
|
|
12658
|
+
if (!CHANGE_KINDS.has(normalizedKind)) {
|
|
9748
12659
|
return c.json(
|
|
9749
|
-
{
|
|
12660
|
+
{
|
|
12661
|
+
error: `Invalid change kind: ${String(body.kind)}. Must be one of: ${Array.from(CHANGE_KINDS).join(", ")}`
|
|
12662
|
+
},
|
|
9750
12663
|
400
|
|
9751
12664
|
);
|
|
9752
12665
|
}
|
|
9753
|
-
const
|
|
9754
|
-
if (!
|
|
12666
|
+
const summary = typeof body.summary === "string" ? body.summary.trim() : "";
|
|
12667
|
+
if (!summary) {
|
|
12668
|
+
return c.json({ error: "summary is required" }, 400);
|
|
12669
|
+
}
|
|
12670
|
+
const db = runtime.getSqlDb(c.env);
|
|
12671
|
+
const touchedPaths = Array.isArray(body.touchedPaths) ? body.touchedPaths.filter((p) => typeof p === "string") : void 0;
|
|
12672
|
+
const rawAttrs = body.attributes ?? body.metadata;
|
|
12673
|
+
const attributes = rawAttrs && typeof rawAttrs === "object" && !Array.isArray(rawAttrs) ? rawAttrs : void 0;
|
|
12674
|
+
const serviceName = typeof body.serviceName === "string" ? body.serviceName : typeof body.service === "string" ? body.service : void 0;
|
|
12675
|
+
const namespace = typeof body.namespace === "string" ? body.namespace : typeof body.serviceNamespace === "string" ? body.serviceNamespace : void 0;
|
|
12676
|
+
const repositoryUrl = typeof body.repositoryUrl === "string" ? body.repositoryUrl : typeof body.repoUrl === "string" ? body.repoUrl : null;
|
|
12677
|
+
const pullRequestUrl = typeof body.pullRequestUrl === "string" ? body.pullRequestUrl : typeof body.prUrl === "string" ? body.prUrl : null;
|
|
12678
|
+
const actorId = typeof body.actorId === "string" ? body.actorId : typeof body.actor === "string" ? body.actor : null;
|
|
12679
|
+
const changeEvent = await recordChangeEvent(db, projectId, {
|
|
12680
|
+
id: typeof body.id === "string" ? body.id : void 0,
|
|
12681
|
+
environment: typeof body.environment === "string" ? body.environment : void 0,
|
|
12682
|
+
kind: normalizedKind,
|
|
12683
|
+
source: body.source === "inferred_telemetry" || body.source === "autosre_remediation" ? body.source : "explicit_api",
|
|
12684
|
+
targetNodeId: typeof body.targetNodeId === "string" ? body.targetNodeId : void 0,
|
|
12685
|
+
serviceName,
|
|
12686
|
+
namespace,
|
|
12687
|
+
version: typeof body.version === "string" ? body.version : null,
|
|
12688
|
+
previousVersion: typeof body.previousVersion === "string" ? body.previousVersion : null,
|
|
12689
|
+
commitSha: typeof body.commitSha === "string" ? body.commitSha : null,
|
|
12690
|
+
previousCommitSha: typeof body.previousCommitSha === "string" ? body.previousCommitSha : null,
|
|
12691
|
+
configDigest: typeof body.configDigest === "string" ? body.configDigest : null,
|
|
12692
|
+
summary,
|
|
12693
|
+
actorId,
|
|
12694
|
+
repositoryUrl,
|
|
12695
|
+
pullRequestUrl,
|
|
12696
|
+
causedByActionId: typeof body.causedByActionId === "string" ? body.causedByActionId : null,
|
|
12697
|
+
agentRunId: typeof body.agentRunId === "string" ? body.agentRunId : null,
|
|
12698
|
+
touchedPaths,
|
|
12699
|
+
attributes,
|
|
12700
|
+
occurredAt: typeof body.occurredAt === "string" ? body.occurredAt : void 0
|
|
12701
|
+
});
|
|
12702
|
+
return c.json({ changeEvent, change: changeEvent }, 201);
|
|
12703
|
+
});
|
|
12704
|
+
app.get("/internal/ontology", async (c) => {
|
|
12705
|
+
const projectId = getProjectId(c);
|
|
12706
|
+
const db = sqlDbFor(c.env);
|
|
12707
|
+
const environment = c.req.query("environment") || void 0;
|
|
12708
|
+
const traceId = c.req.query("traceId") || void 0;
|
|
12709
|
+
const graph = await getOntologyGraph(db, {
|
|
12710
|
+
projectId,
|
|
12711
|
+
environment
|
|
12712
|
+
});
|
|
12713
|
+
const faultLocus = traceId ? await loadTraceFaultLocusFromDb(db, projectId, traceId) : null;
|
|
12714
|
+
return c.json({
|
|
12715
|
+
...graph,
|
|
12716
|
+
faultLocus
|
|
12717
|
+
});
|
|
12718
|
+
});
|
|
12719
|
+
app.get("/internal/changes", async (c) => {
|
|
12720
|
+
const projectId = getProjectId(c);
|
|
12721
|
+
const db = sqlDbFor(c.env);
|
|
12722
|
+
const rawKind = c.req.query("kind");
|
|
12723
|
+
const normalizedKind = rawKind === "agent_prompt_release" ? "prompt_rollout" : rawKind;
|
|
12724
|
+
if (normalizedKind && !CHANGE_KINDS.has(normalizedKind)) {
|
|
12725
|
+
return c.json({ error: `Invalid kind: ${rawKind}` }, 400);
|
|
12726
|
+
}
|
|
12727
|
+
const changes2 = await listChangeEvents(db, {
|
|
12728
|
+
projectId,
|
|
12729
|
+
environment: c.req.query("environment") || void 0,
|
|
12730
|
+
serviceName: c.req.query("serviceName") || c.req.query("service") || void 0,
|
|
12731
|
+
targetNodeId: c.req.query("targetNodeId") || void 0,
|
|
12732
|
+
kind: normalizedKind || void 0,
|
|
12733
|
+
since: c.req.query("since") || void 0,
|
|
12734
|
+
limit: parseLimit(c.req.query("limit"), 50)
|
|
12735
|
+
});
|
|
12736
|
+
return c.json({ changes: changes2 });
|
|
12737
|
+
});
|
|
12738
|
+
app.get("/internal/changes/:id/blast-radius", async (c) => {
|
|
12739
|
+
const projectId = getProjectId(c);
|
|
12740
|
+
const changeId = c.req.param("id");
|
|
12741
|
+
if (!changeId) {
|
|
12742
|
+
return c.json({ error: "changeId is required" }, 400);
|
|
12743
|
+
}
|
|
12744
|
+
const db = sqlDbFor(c.env);
|
|
12745
|
+
const minSampleRaw = c.req.query("minSample") ?? c.req.query("minComparableSample");
|
|
12746
|
+
const minSample = minSampleRaw ? Number.parseInt(minSampleRaw, 10) : void 0;
|
|
12747
|
+
const blastRadius = await computeChangeBlastRadius(
|
|
12748
|
+
db,
|
|
12749
|
+
projectId,
|
|
12750
|
+
changeId,
|
|
12751
|
+
{
|
|
12752
|
+
environment: c.req.query("environment") || void 0,
|
|
12753
|
+
minComparableSample: minSample !== void 0 && Number.isFinite(minSample) ? minSample : void 0
|
|
12754
|
+
}
|
|
12755
|
+
);
|
|
12756
|
+
if (!blastRadius) {
|
|
12757
|
+
return c.json({ error: "Change event not found" }, 404);
|
|
12758
|
+
}
|
|
12759
|
+
return c.json({ blastRadius, ...blastRadius });
|
|
12760
|
+
});
|
|
12761
|
+
app.get("/internal/incidents", async (c) => {
|
|
12762
|
+
const projectId = getProjectId(c);
|
|
12763
|
+
const db = sqlDbFor(c.env);
|
|
12764
|
+
const incidents = await listIncidentInvestigations(db, {
|
|
12765
|
+
projectId,
|
|
12766
|
+
environment: c.req.query("environment") || void 0,
|
|
12767
|
+
status: c.req.query("status") || void 0,
|
|
12768
|
+
limit: parseLimit(c.req.query("limit"), 50)
|
|
12769
|
+
});
|
|
12770
|
+
return c.json({ incidents });
|
|
12771
|
+
});
|
|
12772
|
+
app.post("/internal/incidents", async (c) => {
|
|
12773
|
+
const projectId = getProjectId(c);
|
|
12774
|
+
const body = await readJsonObject(c);
|
|
12775
|
+
if (!body) {
|
|
12776
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
12777
|
+
}
|
|
12778
|
+
const targetNodeId = typeof body.targetNodeId === "string" ? body.targetNodeId.trim() : typeof body.symptomNodeId === "string" ? body.symptomNodeId.trim() : "";
|
|
12779
|
+
if (!targetNodeId) {
|
|
12780
|
+
return c.json({ error: "targetNodeId is required" }, 400);
|
|
12781
|
+
}
|
|
12782
|
+
const symptomAnchorKind = typeof body.symptomAnchorKind === "string" ? body.symptomAnchorKind : typeof body.triggerKind === "string" ? body.triggerKind : "alert";
|
|
12783
|
+
const symptomAnchorId = typeof body.symptomAnchorId === "string" ? body.symptomAnchorId : typeof body.triggerEntityId === "string" ? body.triggerEntityId : targetNodeId;
|
|
12784
|
+
const db = sqlDbFor(c.env);
|
|
12785
|
+
const result = await openOrDeduplicateIncident(db, projectId, {
|
|
12786
|
+
id: typeof body.id === "string" ? body.id : void 0,
|
|
12787
|
+
environment: typeof body.environment === "string" ? body.environment : void 0,
|
|
12788
|
+
targetNodeId,
|
|
12789
|
+
symptomCategory: typeof body.symptomCategory === "string" ? body.symptomCategory : void 0,
|
|
12790
|
+
symptomAnchorKind,
|
|
12791
|
+
symptomAnchorId,
|
|
12792
|
+
symptomFingerprint: typeof body.symptomFingerprint === "string" ? body.symptomFingerprint : typeof body.fingerprint === "string" ? body.fingerprint : null,
|
|
12793
|
+
latestBundle: body.latestBundle && typeof body.latestBundle === "object" && !Array.isArray(body.latestBundle) ? body.latestBundle : null
|
|
12794
|
+
});
|
|
12795
|
+
return c.json(result, result.deduplicated ? 200 : 201);
|
|
12796
|
+
});
|
|
12797
|
+
app.post("/internal/incidents/:id/lease", async (c) => {
|
|
12798
|
+
const projectId = getProjectId(c);
|
|
12799
|
+
const incidentId = c.req.param("id");
|
|
12800
|
+
const body = await readJsonObject(c);
|
|
12801
|
+
if (!body) {
|
|
12802
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
12803
|
+
}
|
|
12804
|
+
const runnerId = typeof body.runnerId === "string" ? body.runnerId.trim() : typeof body.leaseOwner === "string" ? body.leaseOwner.trim() : "";
|
|
12805
|
+
if (!runnerId) {
|
|
12806
|
+
return c.json({ error: "runnerId is required" }, 400);
|
|
12807
|
+
}
|
|
12808
|
+
const db = sqlDbFor(c.env);
|
|
12809
|
+
const leaseDurationSeconds = typeof body.leaseDurationSeconds === "number" && Number.isFinite(body.leaseDurationSeconds) ? body.leaseDurationSeconds : typeof body.leaseDurationMs === "number" && Number.isFinite(body.leaseDurationMs) ? Math.ceil(body.leaseDurationMs / 1e3) : void 0;
|
|
12810
|
+
const result = await acquireIncidentLease(db, projectId, incidentId, {
|
|
12811
|
+
environment: typeof body.environment === "string" ? body.environment : void 0,
|
|
12812
|
+
runnerId,
|
|
12813
|
+
leaseDurationSeconds
|
|
12814
|
+
});
|
|
12815
|
+
return c.json(result, result.acquired ? 200 : 409);
|
|
12816
|
+
});
|
|
12817
|
+
app.post("/internal/incidents/:id/dispatch", async (c) => {
|
|
12818
|
+
const projectId = getProjectId(c);
|
|
12819
|
+
const incidentId = c.req.param("id");
|
|
12820
|
+
const body = await readJsonObject(c);
|
|
12821
|
+
const webhookUrl = typeof body?.webhookUrl === "string" ? body.webhookUrl : typeof body?.url === "string" ? body.url : "";
|
|
12822
|
+
if (!webhookUrl) {
|
|
12823
|
+
return c.json({ error: "webhookUrl is required" }, 400);
|
|
12824
|
+
}
|
|
12825
|
+
const db = sqlDbFor(c.env);
|
|
12826
|
+
const result = await dispatchIncidentWebhook(db, projectId, incidentId, {
|
|
12827
|
+
url: webhookUrl,
|
|
12828
|
+
secret: typeof body?.secret === "string" ? body.secret : void 0,
|
|
12829
|
+
allowPrivateNetworks: Boolean(
|
|
12830
|
+
body?.allowPrivateNetworks ?? body?.allowPrivateWebhookTargets
|
|
12831
|
+
)
|
|
12832
|
+
});
|
|
12833
|
+
return c.json(result, result.delivered ? 200 : 502);
|
|
12834
|
+
});
|
|
12835
|
+
app.get("/internal/findings", async (c) => {
|
|
12836
|
+
const projectId = getProjectId(c);
|
|
12837
|
+
const db = sqlDbFor(c.env);
|
|
12838
|
+
const findings = await searchInvestigationFindings(db, {
|
|
12839
|
+
projectId,
|
|
12840
|
+
environment: c.req.query("environment") || void 0,
|
|
12841
|
+
fingerprint: c.req.query("fingerprint") || void 0,
|
|
12842
|
+
symptomEntityId: c.req.query("symptomEntityId") || void 0,
|
|
12843
|
+
rootCauseEntityId: c.req.query("rootCauseEntityId") || c.req.query("faultLocusEntityId") || void 0,
|
|
12844
|
+
causalVerificationStatus: c.req.query("causalVerificationStatus") || void 0,
|
|
12845
|
+
incidentId: c.req.query("incidentId") || void 0,
|
|
12846
|
+
limit: parseLimit(c.req.query("limit"), 20)
|
|
12847
|
+
});
|
|
12848
|
+
return c.json({ findings });
|
|
12849
|
+
});
|
|
12850
|
+
app.post("/internal/findings", async (c) => {
|
|
12851
|
+
const projectId = getProjectId(c);
|
|
12852
|
+
const body = await readJsonObject(c);
|
|
12853
|
+
if (!body) {
|
|
12854
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
12855
|
+
}
|
|
12856
|
+
const title = typeof body.title === "string" ? body.title.trim() : "";
|
|
12857
|
+
const summary = typeof body.summary === "string" ? body.summary.trim() : "";
|
|
12858
|
+
const symptomEntityKind = typeof body.symptomEntityKind === "string" ? body.symptomEntityKind.trim() : "";
|
|
12859
|
+
const symptomEntityId = typeof body.symptomEntityId === "string" ? body.symptomEntityId.trim() : "";
|
|
12860
|
+
const faultLocusEntityKind = typeof body.faultLocusEntityKind === "string" ? body.faultLocusEntityKind.trim() : "";
|
|
12861
|
+
const faultLocusEntityId = typeof body.faultLocusEntityId === "string" ? body.faultLocusEntityId.trim() : "";
|
|
12862
|
+
if (!title || !summary || !symptomEntityKind || !symptomEntityId || !faultLocusEntityKind || !faultLocusEntityId) {
|
|
9755
12863
|
return c.json(
|
|
9756
|
-
{
|
|
12864
|
+
{
|
|
12865
|
+
error: "title, summary, symptomEntityKind, symptomEntityId, faultLocusEntityKind, and faultLocusEntityId are required"
|
|
12866
|
+
},
|
|
9757
12867
|
400
|
|
9758
12868
|
);
|
|
9759
12869
|
}
|
|
9760
|
-
|
|
12870
|
+
const db = sqlDbFor(c.env);
|
|
12871
|
+
const input = {
|
|
12872
|
+
id: typeof body.id === "string" ? body.id : void 0,
|
|
12873
|
+
environment: typeof body.environment === "string" ? body.environment : void 0,
|
|
12874
|
+
incidentId: typeof body.incidentId === "string" ? body.incidentId : null,
|
|
12875
|
+
fingerprintInput: body.fingerprintInput && typeof body.fingerprintInput === "object" ? body.fingerprintInput : void 0,
|
|
12876
|
+
title,
|
|
12877
|
+
summary,
|
|
12878
|
+
causalProofTier: typeof body.causalProofTier === "string" ? body.causalProofTier : void 0,
|
|
12879
|
+
confidence: typeof body.confidence === "number" ? body.confidence : void 0,
|
|
12880
|
+
symptomEntityKind,
|
|
12881
|
+
symptomEntityId,
|
|
12882
|
+
faultLocusEntityKind,
|
|
12883
|
+
faultLocusEntityId,
|
|
12884
|
+
rootCauseEntityKind: typeof body.rootCauseEntityKind === "string" ? body.rootCauseEntityKind : null,
|
|
12885
|
+
rootCauseEntityId: typeof body.rootCauseEntityId === "string" ? body.rootCauseEntityId : null,
|
|
12886
|
+
contributingChangeId: typeof body.contributingChangeId === "string" ? body.contributingChangeId : null,
|
|
12887
|
+
cohortVerdict: typeof body.cohortVerdict === "string" ? body.cohortVerdict : null,
|
|
12888
|
+
contendingTraceId: typeof body.contendingTraceId === "string" ? body.contendingTraceId : null,
|
|
12889
|
+
causalChain: Array.isArray(body.causalChain) ? body.causalChain : [],
|
|
12890
|
+
citedEvidenceIds: Array.isArray(body.citedEvidenceIds) ? body.citedEvidenceIds.filter(
|
|
12891
|
+
(id) => typeof id === "string"
|
|
12892
|
+
) : [],
|
|
12893
|
+
agentRunId: typeof body.agentRunId === "string" ? body.agentRunId : null
|
|
12894
|
+
};
|
|
12895
|
+
const finding = await recordInvestigationFinding(db, projectId, input);
|
|
12896
|
+
return c.json({ finding, id: finding.id }, 201);
|
|
12897
|
+
});
|
|
12898
|
+
app.post("/internal/findings/:id/counterfactual", async (c) => {
|
|
12899
|
+
const projectId = getProjectId(c);
|
|
12900
|
+
const findingId = c.req.param("id");
|
|
12901
|
+
const body = await readJsonObject(c);
|
|
12902
|
+
if (!body) {
|
|
12903
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
12904
|
+
}
|
|
12905
|
+
const evalCaseId = typeof body.evalCaseId === "string" ? body.evalCaseId.trim() : "";
|
|
12906
|
+
const failingRunId = typeof body.failingRunId === "string" ? body.failingRunId.trim() : "";
|
|
12907
|
+
const passingRunIds = Array.isArray(body.passingRunIds) ? body.passingRunIds.filter(
|
|
12908
|
+
(id) => typeof id === "string" && id.trim() !== ""
|
|
12909
|
+
) : [];
|
|
12910
|
+
if (!evalCaseId || !failingRunId) {
|
|
9761
12911
|
return c.json(
|
|
12912
|
+
{ error: "evalCaseId and failingRunId are required" },
|
|
12913
|
+
400
|
|
12914
|
+
);
|
|
12915
|
+
}
|
|
12916
|
+
const db = sqlDbFor(c.env);
|
|
12917
|
+
try {
|
|
12918
|
+
const finding = await verifyPairedCounterfactualProof(
|
|
12919
|
+
db,
|
|
12920
|
+
projectId,
|
|
12921
|
+
findingId,
|
|
9762
12922
|
{
|
|
9763
|
-
|
|
9764
|
-
|
|
12923
|
+
environment: typeof body.environment === "string" ? body.environment : void 0,
|
|
12924
|
+
evalCaseId,
|
|
12925
|
+
failingRunId,
|
|
12926
|
+
passingRunIds,
|
|
12927
|
+
requiredTrials: typeof body.requiredTrials === "number" && Number.isFinite(body.requiredTrials) ? body.requiredTrials : void 0
|
|
12928
|
+
}
|
|
12929
|
+
);
|
|
12930
|
+
return c.json({ verified: true, finding });
|
|
12931
|
+
} catch (err) {
|
|
12932
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
12933
|
+
if (err instanceof CounterfactualVerificationError) {
|
|
12934
|
+
const isNotFound2 = message.includes("not found in project");
|
|
12935
|
+
return c.json(
|
|
12936
|
+
{ verified: false, error: message, reason: message },
|
|
12937
|
+
isNotFound2 ? 404 : 422
|
|
12938
|
+
);
|
|
12939
|
+
}
|
|
12940
|
+
return c.json({ verified: false, error: message }, 400);
|
|
12941
|
+
}
|
|
12942
|
+
});
|
|
12943
|
+
app.get("/internal/remediations", async (c) => {
|
|
12944
|
+
const projectId = getProjectId(c);
|
|
12945
|
+
const db = sqlDbFor(c.env);
|
|
12946
|
+
const proposals = await listRemediationProposals(db, {
|
|
12947
|
+
projectId,
|
|
12948
|
+
environment: c.req.query("environment") || void 0,
|
|
12949
|
+
incidentId: c.req.query("incidentId") || void 0,
|
|
12950
|
+
findingId: c.req.query("findingId") || void 0,
|
|
12951
|
+
status: c.req.query("status") || void 0,
|
|
12952
|
+
limit: parseLimit(c.req.query("limit"), 50)
|
|
12953
|
+
});
|
|
12954
|
+
return c.json({ proposals });
|
|
12955
|
+
});
|
|
12956
|
+
app.post("/internal/remediations/propose", async (c) => {
|
|
12957
|
+
const projectId = getProjectId(c);
|
|
12958
|
+
const body = await readJsonObject(c);
|
|
12959
|
+
if (!body) {
|
|
12960
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
12961
|
+
}
|
|
12962
|
+
const incidentId = typeof body.incidentId === "string" ? body.incidentId.trim() : "";
|
|
12963
|
+
const findingId = typeof body.findingId === "string" ? body.findingId.trim() : "";
|
|
12964
|
+
const idempotencyKey = typeof body.idempotencyKey === "string" ? body.idempotencyKey.trim() : "";
|
|
12965
|
+
const actionKind = typeof body.actionKind === "string" ? body.actionKind.trim() : "";
|
|
12966
|
+
const targetNodeId = typeof body.targetNodeId === "string" ? body.targetNodeId.trim() : "";
|
|
12967
|
+
if (!incidentId || !findingId || !idempotencyKey || !REMEDIATION_ACTION_KINDS.has(actionKind) || !targetNodeId) {
|
|
12968
|
+
return c.json(
|
|
12969
|
+
{
|
|
12970
|
+
error: "incidentId, findingId, idempotencyKey, valid actionKind, and targetNodeId are required"
|
|
9765
12971
|
},
|
|
9766
12972
|
400
|
|
9767
12973
|
);
|
|
9768
12974
|
}
|
|
12975
|
+
const expectedCurrentState = body.expectedCurrentState && typeof body.expectedCurrentState === "object" && !Array.isArray(body.expectedCurrentState) ? body.expectedCurrentState : {};
|
|
12976
|
+
const desiredTargetState = body.desiredTargetState && typeof body.desiredTargetState === "object" && !Array.isArray(body.desiredTargetState) ? body.desiredTargetState : {};
|
|
12977
|
+
const requestedAutonomy = isAgentAutonomyLevel(
|
|
12978
|
+
body.requestedAutonomy
|
|
12979
|
+
) ? body.requestedAutonomy : "autonomous_write";
|
|
12980
|
+
const db = sqlDbFor(c.env);
|
|
12981
|
+
const input = {
|
|
12982
|
+
environment: typeof body.environment === "string" ? body.environment : void 0,
|
|
12983
|
+
incidentId,
|
|
12984
|
+
findingId,
|
|
12985
|
+
idempotencyKey,
|
|
12986
|
+
actionKind,
|
|
12987
|
+
targetNodeId,
|
|
12988
|
+
expectedCurrentState,
|
|
12989
|
+
desiredTargetState,
|
|
12990
|
+
requestedAutonomy
|
|
12991
|
+
};
|
|
12992
|
+
try {
|
|
12993
|
+
const proposal = await proposeRemediationAction(db, projectId, input);
|
|
12994
|
+
return c.json(
|
|
12995
|
+
{
|
|
12996
|
+
proposal,
|
|
12997
|
+
...proposal,
|
|
12998
|
+
grantToken: proposal.grantToken ?? null
|
|
12999
|
+
},
|
|
13000
|
+
201
|
|
13001
|
+
);
|
|
13002
|
+
} catch (err) {
|
|
13003
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13004
|
+
const status = err instanceof RemediationGateError ? err.statusCode : 400;
|
|
13005
|
+
return c.json({ error: message }, status);
|
|
13006
|
+
}
|
|
13007
|
+
});
|
|
13008
|
+
app.post("/internal/remediations/:id/approve", async (c) => {
|
|
13009
|
+
const sessionCheck = requireHumanDashboardSession(c);
|
|
13010
|
+
if (!sessionCheck.ok) {
|
|
13011
|
+
return c.json(
|
|
13012
|
+
{ error: "Forbidden", message: sessionCheck.message },
|
|
13013
|
+
403
|
|
13014
|
+
);
|
|
13015
|
+
}
|
|
9769
13016
|
const projectId = getProjectId(c);
|
|
9770
|
-
const
|
|
9771
|
-
const
|
|
9772
|
-
const
|
|
9773
|
-
|
|
9774
|
-
|
|
9775
|
-
|
|
9776
|
-
|
|
9777
|
-
|
|
9778
|
-
|
|
9779
|
-
|
|
9780
|
-
|
|
9781
|
-
|
|
9782
|
-
|
|
9783
|
-
|
|
9784
|
-
|
|
9785
|
-
|
|
9786
|
-
|
|
9787
|
-
|
|
9788
|
-
|
|
9789
|
-
|
|
9790
|
-
|
|
9791
|
-
|
|
9792
|
-
|
|
13017
|
+
const proposalId = c.req.param("id");
|
|
13018
|
+
const body = await readJsonObject(c).catch(() => null) ?? {};
|
|
13019
|
+
const environment = typeof body.environment === "string" ? body.environment : void 0;
|
|
13020
|
+
const approvedByUserId = typeof body.approvedByUserId === "string" && body.approvedByUserId.trim() ? body.approvedByUserId.trim() : typeof body.approvedBySession === "string" && body.approvedBySession.trim() ? body.approvedBySession.trim() : sessionCheck.sessionActor;
|
|
13021
|
+
const db = sqlDbFor(c.env);
|
|
13022
|
+
try {
|
|
13023
|
+
const proposal = await approveRemediationProposal(
|
|
13024
|
+
db,
|
|
13025
|
+
projectId,
|
|
13026
|
+
proposalId,
|
|
13027
|
+
{
|
|
13028
|
+
environment,
|
|
13029
|
+
approvedByUserId
|
|
13030
|
+
}
|
|
13031
|
+
);
|
|
13032
|
+
return c.json({
|
|
13033
|
+
proposal,
|
|
13034
|
+
...proposal,
|
|
13035
|
+
grantToken: proposal.grantToken ?? null
|
|
13036
|
+
});
|
|
13037
|
+
} catch (err) {
|
|
13038
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13039
|
+
const status = err instanceof RemediationGateError ? err.statusCode : 400;
|
|
13040
|
+
return c.json({ error: message }, status);
|
|
9793
13041
|
}
|
|
9794
|
-
|
|
13042
|
+
});
|
|
13043
|
+
app.post("/internal/remediations/:id/reject", async (c) => {
|
|
13044
|
+
const sessionCheck = requireHumanDashboardSession(c);
|
|
13045
|
+
if (!sessionCheck.ok) {
|
|
13046
|
+
return c.json(
|
|
13047
|
+
{ error: "Forbidden", message: sessionCheck.message },
|
|
13048
|
+
403
|
|
13049
|
+
);
|
|
13050
|
+
}
|
|
13051
|
+
const projectId = getProjectId(c);
|
|
13052
|
+
const proposalId = c.req.param("id");
|
|
13053
|
+
const body = await readJsonObject(c).catch(() => null) ?? {};
|
|
13054
|
+
const environment = typeof body.environment === "string" ? body.environment : void 0;
|
|
13055
|
+
const reason = typeof body.reason === "string" ? body.reason : void 0;
|
|
13056
|
+
const db = sqlDbFor(c.env);
|
|
13057
|
+
try {
|
|
13058
|
+
const proposal = await rejectRemediationProposal(
|
|
13059
|
+
db,
|
|
13060
|
+
projectId,
|
|
13061
|
+
proposalId,
|
|
13062
|
+
{
|
|
13063
|
+
environment,
|
|
13064
|
+
rejectedByUserId: sessionCheck.sessionActor,
|
|
13065
|
+
reason
|
|
13066
|
+
}
|
|
13067
|
+
);
|
|
13068
|
+
return c.json({ proposal, ...proposal });
|
|
13069
|
+
} catch (err) {
|
|
13070
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13071
|
+
const status = err instanceof RemediationGateError ? err.statusCode : 400;
|
|
13072
|
+
return c.json({ error: message }, status);
|
|
13073
|
+
}
|
|
13074
|
+
});
|
|
13075
|
+
app.post("/internal/remediations/:id/claim", async (c) => {
|
|
13076
|
+
const projectId = getProjectId(c);
|
|
13077
|
+
const proposalId = c.req.param("id");
|
|
13078
|
+
const body = await readJsonObject(c);
|
|
13079
|
+
if (!body) {
|
|
13080
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
13081
|
+
}
|
|
13082
|
+
const grantToken = typeof body.grantToken === "string" ? body.grantToken.trim() : "";
|
|
13083
|
+
if (!grantToken) {
|
|
13084
|
+
return c.json({ error: "grantToken is required" }, 400);
|
|
13085
|
+
}
|
|
13086
|
+
const db = sqlDbFor(c.env);
|
|
13087
|
+
try {
|
|
13088
|
+
const proposal = await claimRemediationGrant(
|
|
13089
|
+
db,
|
|
13090
|
+
projectId,
|
|
13091
|
+
proposalId,
|
|
13092
|
+
{
|
|
13093
|
+
environment: typeof body.environment === "string" ? body.environment : void 0,
|
|
13094
|
+
grantToken,
|
|
13095
|
+
runnerId: typeof body.runnerId === "string" ? body.runnerId : void 0
|
|
13096
|
+
}
|
|
13097
|
+
);
|
|
13098
|
+
return c.json({ claimed: true, proposal, ...proposal });
|
|
13099
|
+
} catch (err) {
|
|
13100
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13101
|
+
const status = err instanceof RemediationGateError ? err.statusCode : 409;
|
|
13102
|
+
return c.json({ claimed: false, error: message, reason: message }, status);
|
|
13103
|
+
}
|
|
13104
|
+
});
|
|
13105
|
+
app.post("/internal/remediations/:id/complete", async (c) => {
|
|
13106
|
+
const projectId = getProjectId(c);
|
|
13107
|
+
const proposalId = c.req.param("id");
|
|
13108
|
+
const body = await readJsonObject(c).catch(() => null) ?? {};
|
|
13109
|
+
const db = sqlDbFor(c.env);
|
|
13110
|
+
try {
|
|
13111
|
+
const proposal = await completeRemediationProposal(
|
|
13112
|
+
db,
|
|
13113
|
+
projectId,
|
|
13114
|
+
proposalId,
|
|
13115
|
+
{
|
|
13116
|
+
environment: typeof body.environment === "string" ? body.environment : void 0,
|
|
13117
|
+
resultingChangeId: typeof body.resultingChangeId === "string" ? body.resultingChangeId : null,
|
|
13118
|
+
remediationAgentRunId: typeof body.remediationAgentRunId === "string" ? body.remediationAgentRunId : null
|
|
13119
|
+
}
|
|
13120
|
+
);
|
|
13121
|
+
return c.json({ proposal, ...proposal });
|
|
13122
|
+
} catch (err) {
|
|
13123
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13124
|
+
const status = err instanceof RemediationGateError ? err.statusCode : 400;
|
|
13125
|
+
return c.json({ error: message }, status);
|
|
13126
|
+
}
|
|
13127
|
+
});
|
|
13128
|
+
app.post("/internal/remediations/:id/fail", async (c) => {
|
|
13129
|
+
const projectId = getProjectId(c);
|
|
13130
|
+
const proposalId = c.req.param("id");
|
|
13131
|
+
const body = await readJsonObject(c).catch(() => null) ?? {};
|
|
13132
|
+
const db = sqlDbFor(c.env);
|
|
9795
13133
|
try {
|
|
9796
|
-
const
|
|
9797
|
-
|
|
9798
|
-
|
|
9799
|
-
|
|
9800
|
-
model: llm.model,
|
|
9801
|
-
question_chars: question.length
|
|
9802
|
-
});
|
|
9803
|
-
const result = await runtime.withChildSpan(
|
|
9804
|
-
"ask.runAsk",
|
|
9805
|
-
() => runAsk(question, {
|
|
9806
|
-
llm,
|
|
9807
|
-
listAnalyses: async (filters) => {
|
|
9808
|
-
const all = await getAllAnalysesForProject(projectId, {
|
|
9809
|
-
db: sqlDbFor(c.env)
|
|
9810
|
-
});
|
|
9811
|
-
return all.filter((d) => {
|
|
9812
|
-
if (filters?.group && d.group !== filters.group) return false;
|
|
9813
|
-
if (filters?.view && d.view !== filters.view) return false;
|
|
9814
|
-
return true;
|
|
9815
|
-
});
|
|
9816
|
-
},
|
|
9817
|
-
getLatestResult: async (id) => {
|
|
9818
|
-
const definitions = await store.listDefinitions(projectId);
|
|
9819
|
-
const definition = definitions.find((d) => d.id === id);
|
|
9820
|
-
if (!definition) return null;
|
|
9821
|
-
const r = await store.getLatestResult(projectId, id);
|
|
9822
|
-
return { definition, result: r };
|
|
9823
|
-
}
|
|
9824
|
-
}),
|
|
13134
|
+
const proposal = await failRemediationProposal(
|
|
13135
|
+
db,
|
|
13136
|
+
projectId,
|
|
13137
|
+
proposalId,
|
|
9825
13138
|
{
|
|
9826
|
-
"
|
|
9827
|
-
"
|
|
9828
|
-
"openinference.span.kind": "LLM"
|
|
13139
|
+
environment: typeof body.environment === "string" ? body.environment : void 0,
|
|
13140
|
+
failureReason: typeof body.failureReason === "string" ? body.failureReason : "Execution failed"
|
|
9829
13141
|
}
|
|
9830
13142
|
);
|
|
9831
|
-
|
|
9832
|
-
|
|
9833
|
-
|
|
9834
|
-
|
|
9835
|
-
|
|
9836
|
-
|
|
9837
|
-
|
|
9838
|
-
|
|
9839
|
-
|
|
9840
|
-
|
|
9841
|
-
|
|
9842
|
-
|
|
9843
|
-
|
|
9844
|
-
|
|
9845
|
-
|
|
9846
|
-
|
|
9847
|
-
|
|
9848
|
-
|
|
9849
|
-
|
|
9850
|
-
|
|
9851
|
-
|
|
9852
|
-
|
|
9853
|
-
|
|
9854
|
-
|
|
9855
|
-
|
|
9856
|
-
|
|
9857
|
-
|
|
9858
|
-
|
|
9859
|
-
|
|
9860
|
-
|
|
9861
|
-
|
|
13143
|
+
return c.json({ proposal, ...proposal });
|
|
13144
|
+
} catch (err) {
|
|
13145
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13146
|
+
const status = err instanceof RemediationGateError ? err.statusCode : 400;
|
|
13147
|
+
return c.json({ error: message }, status);
|
|
13148
|
+
}
|
|
13149
|
+
});
|
|
13150
|
+
app.post("/internal/remediations/watchdog", async (c) => {
|
|
13151
|
+
const db = sqlDbFor(c.env);
|
|
13152
|
+
const body = await readJsonObject(c).catch(() => null) ?? {};
|
|
13153
|
+
const summary = await evaluateRemediationWatchdogs(db, {
|
|
13154
|
+
nowIso: typeof body.nowIso === "string" ? body.nowIso : void 0,
|
|
13155
|
+
minComparableRequests: typeof body.minComparableRequests === "number" ? body.minComparableRequests : void 0
|
|
13156
|
+
});
|
|
13157
|
+
return c.json({ summary, ...summary });
|
|
13158
|
+
});
|
|
13159
|
+
app.get("/internal/remediations/:id", async (c) => {
|
|
13160
|
+
const projectId = getProjectId(c);
|
|
13161
|
+
const proposalId = c.req.param("id");
|
|
13162
|
+
const db = sqlDbFor(c.env);
|
|
13163
|
+
const proposal = await getRemediationProposal(
|
|
13164
|
+
db,
|
|
13165
|
+
projectId,
|
|
13166
|
+
proposalId,
|
|
13167
|
+
c.req.query("environment") || void 0
|
|
13168
|
+
);
|
|
13169
|
+
if (!proposal) {
|
|
13170
|
+
return c.json({ error: "Proposal not found" }, 404);
|
|
13171
|
+
}
|
|
13172
|
+
return c.json({ proposal });
|
|
13173
|
+
});
|
|
13174
|
+
app.get("/internal/incidents/:id", async (c) => {
|
|
13175
|
+
const projectId = getProjectId(c);
|
|
13176
|
+
const incidentId = c.req.param("id");
|
|
13177
|
+
const environment = c.req.query("environment") || void 0;
|
|
13178
|
+
const db = sqlDbFor(c.env);
|
|
13179
|
+
const incident = await getIncidentInvestigation(
|
|
13180
|
+
db,
|
|
13181
|
+
projectId,
|
|
13182
|
+
incidentId,
|
|
13183
|
+
environment
|
|
13184
|
+
);
|
|
13185
|
+
if (!incident) {
|
|
13186
|
+
return c.json({ error: "Incident not found" }, 404);
|
|
13187
|
+
}
|
|
13188
|
+
const [findings, proposals] = await Promise.all([
|
|
13189
|
+
searchInvestigationFindings(db, {
|
|
13190
|
+
projectId,
|
|
13191
|
+
environment: incident.environment,
|
|
13192
|
+
incidentId: incident.id,
|
|
13193
|
+
limit: 20
|
|
13194
|
+
}),
|
|
13195
|
+
listRemediationProposals(db, {
|
|
13196
|
+
projectId,
|
|
13197
|
+
environment: incident.environment,
|
|
13198
|
+
incidentId: incident.id,
|
|
13199
|
+
limit: 20
|
|
13200
|
+
})
|
|
13201
|
+
]);
|
|
13202
|
+
return c.json({ incident, findings, proposals });
|
|
13203
|
+
});
|
|
13204
|
+
app.get("/internal/findings/:id", async (c) => {
|
|
13205
|
+
const projectId = getProjectId(c);
|
|
13206
|
+
const findingId = c.req.param("id");
|
|
13207
|
+
const db = sqlDbFor(c.env);
|
|
13208
|
+
const finding = await getInvestigationFinding(
|
|
13209
|
+
db,
|
|
13210
|
+
projectId,
|
|
13211
|
+
findingId,
|
|
13212
|
+
c.req.query("environment") || void 0
|
|
13213
|
+
);
|
|
13214
|
+
if (!finding) {
|
|
13215
|
+
return c.json({ error: "Finding not found" }, 404);
|
|
13216
|
+
}
|
|
13217
|
+
return c.json({ finding });
|
|
13218
|
+
});
|
|
13219
|
+
app.get("/internal/changes/:id", async (c) => {
|
|
13220
|
+
const projectId = getProjectId(c);
|
|
13221
|
+
const changeId = c.req.param("id");
|
|
13222
|
+
const db = sqlDbFor(c.env);
|
|
13223
|
+
const changeEvent = await getChangeEvent(
|
|
13224
|
+
db,
|
|
13225
|
+
projectId,
|
|
13226
|
+
changeId,
|
|
13227
|
+
c.req.query("environment") || void 0
|
|
13228
|
+
);
|
|
13229
|
+
if (!changeEvent) {
|
|
13230
|
+
return c.json({ error: "Change event not found" }, 404);
|
|
9862
13231
|
}
|
|
13232
|
+
return c.json({ changeEvent, change: changeEvent });
|
|
9863
13233
|
});
|
|
9864
13234
|
}
|
|
9865
13235
|
};
|
|
@@ -9940,7 +13310,10 @@ var KNOWN_KINDS = /* @__PURE__ */ new Set([
|
|
|
9940
13310
|
"user",
|
|
9941
13311
|
"action",
|
|
9942
13312
|
"agent_run",
|
|
9943
|
-
"tool_call"
|
|
13313
|
+
"tool_call",
|
|
13314
|
+
"change",
|
|
13315
|
+
"finding",
|
|
13316
|
+
"incident"
|
|
9944
13317
|
]);
|
|
9945
13318
|
var MAX_LINKS_INLINE = 5;
|
|
9946
13319
|
var truncate = (s, n = 80) => s.length > n ? `${s.slice(0, n - 1)}\u2026` : s;
|
|
@@ -11038,6 +14411,186 @@ var connectedRoutesPlugin = {
|
|
|
11038
14411
|
manifest.rawManifest = actionManifest;
|
|
11039
14412
|
}
|
|
11040
14413
|
}
|
|
14414
|
+
} else if (kind === "change") {
|
|
14415
|
+
const db = sqlDbFor(c.env);
|
|
14416
|
+
const changeRow = await db.prepare(
|
|
14417
|
+
`SELECT * FROM change_events WHERE project_id = ? AND id = ? LIMIT 1`
|
|
14418
|
+
).bind(projectId, id).first();
|
|
14419
|
+
if (changeRow) {
|
|
14420
|
+
manifest.up = [
|
|
14421
|
+
{
|
|
14422
|
+
label: "Target Ontology Node",
|
|
14423
|
+
links: [
|
|
14424
|
+
{
|
|
14425
|
+
label: `${changeRow.target_node_id} (${changeRow.environment})`,
|
|
14426
|
+
href: `#/health?node=${encodeURIComponent(changeRow.target_node_id)}`,
|
|
14427
|
+
relationshipCertainty: "exact",
|
|
14428
|
+
reason: "Target node mutated by this change event."
|
|
14429
|
+
}
|
|
14430
|
+
]
|
|
14431
|
+
}
|
|
14432
|
+
];
|
|
14433
|
+
const findings = await db.prepare(
|
|
14434
|
+
`SELECT id, title, causal_proof_tier, confidence, incident_id
|
|
14435
|
+
FROM investigation_findings
|
|
14436
|
+
WHERE project_id = ? AND environment = ? AND contributing_change_id = ?
|
|
14437
|
+
ORDER BY created_at DESC LIMIT 10`
|
|
14438
|
+
).bind(projectId, changeRow.environment, changeRow.id).all();
|
|
14439
|
+
manifest.down = [
|
|
14440
|
+
{
|
|
14441
|
+
label: "Attributed Findings",
|
|
14442
|
+
links: findings.results.map((f) => ({
|
|
14443
|
+
label: f.title,
|
|
14444
|
+
href: `#/findings/${encodeURIComponent(f.id)}`,
|
|
14445
|
+
entityKind: "finding",
|
|
14446
|
+
entityId: f.id,
|
|
14447
|
+
relationshipCertainty: "exact",
|
|
14448
|
+
causalProofTier: f.causal_proof_tier,
|
|
14449
|
+
confidence: f.confidence,
|
|
14450
|
+
reason: "Finding attributes this change event."
|
|
14451
|
+
})),
|
|
14452
|
+
emptyReason: findings.results.length === 0 ? "No investigation findings cite this change event." : void 0
|
|
14453
|
+
}
|
|
14454
|
+
];
|
|
14455
|
+
const spans = changeRow.version ? await db.prepare(
|
|
14456
|
+
`SELECT DISTINCT trace_id, span_id, service_name, span_name
|
|
14457
|
+
FROM telemetry_spans
|
|
14458
|
+
WHERE project_id = ? AND environment = ? AND service_version = ? AND status_code = 2
|
|
14459
|
+
ORDER BY start_time DESC LIMIT 5`
|
|
14460
|
+
).bind(projectId, changeRow.environment, changeRow.version).all() : { results: [] };
|
|
14461
|
+
manifest.across = [
|
|
14462
|
+
{
|
|
14463
|
+
label: "Failing Spans on Cohort Version",
|
|
14464
|
+
links: spans.results.map((s) => ({
|
|
14465
|
+
label: `${s.service_name} \xB7 ${s.span_name}`,
|
|
14466
|
+
href: `#/traces/${s.trace_id}#span=${s.span_id}`,
|
|
14467
|
+
entityKind: "span",
|
|
14468
|
+
entityId: `${s.trace_id}:${s.span_id}`,
|
|
14469
|
+
relationshipCertainty: "exact",
|
|
14470
|
+
reason: `Span executed on version ${changeRow.version}.`
|
|
14471
|
+
})),
|
|
14472
|
+
emptyReason: spans.results.length === 0 ? "No failing spans recorded on this change version." : void 0
|
|
14473
|
+
}
|
|
14474
|
+
];
|
|
14475
|
+
}
|
|
14476
|
+
} else if (kind === "finding") {
|
|
14477
|
+
const db = sqlDbFor(c.env);
|
|
14478
|
+
const findingRow = await db.prepare(
|
|
14479
|
+
`SELECT * FROM investigation_findings WHERE project_id = ? AND id = ? LIMIT 1`
|
|
14480
|
+
).bind(projectId, id).first();
|
|
14481
|
+
if (findingRow) {
|
|
14482
|
+
const proofTier = findingRow.causal_proof_tier;
|
|
14483
|
+
manifest.up = [];
|
|
14484
|
+
if (findingRow.incident_id) {
|
|
14485
|
+
manifest.up.push({
|
|
14486
|
+
label: "Incident Investigation",
|
|
14487
|
+
links: [
|
|
14488
|
+
{
|
|
14489
|
+
label: `incident ${findingRow.incident_id}`,
|
|
14490
|
+
href: `#/incidents/${encodeURIComponent(findingRow.incident_id)}`,
|
|
14491
|
+
entityKind: "incident",
|
|
14492
|
+
entityId: findingRow.incident_id,
|
|
14493
|
+
relationshipCertainty: "exact",
|
|
14494
|
+
causalProofTier: proofTier
|
|
14495
|
+
}
|
|
14496
|
+
]
|
|
14497
|
+
});
|
|
14498
|
+
}
|
|
14499
|
+
if (findingRow.contributing_change_id) {
|
|
14500
|
+
manifest.up.push({
|
|
14501
|
+
label: "Contributing Change",
|
|
14502
|
+
links: [
|
|
14503
|
+
{
|
|
14504
|
+
label: `change ${findingRow.contributing_change_id}`,
|
|
14505
|
+
href: `#/changes/${encodeURIComponent(findingRow.contributing_change_id)}`,
|
|
14506
|
+
entityKind: "change",
|
|
14507
|
+
entityId: findingRow.contributing_change_id,
|
|
14508
|
+
relationshipCertainty: "exact",
|
|
14509
|
+
causalProofTier: proofTier,
|
|
14510
|
+
confidence: findingRow.confidence
|
|
14511
|
+
}
|
|
14512
|
+
]
|
|
14513
|
+
});
|
|
14514
|
+
}
|
|
14515
|
+
manifest.across = [
|
|
14516
|
+
{
|
|
14517
|
+
label: "Fault Locus & Symptom",
|
|
14518
|
+
links: [
|
|
14519
|
+
{
|
|
14520
|
+
label: `fault locus: ${findingRow.fault_locus_entity_id}`,
|
|
14521
|
+
href: `#/traces?q=${encodeURIComponent(findingRow.fault_locus_entity_id)}`,
|
|
14522
|
+
relationshipCertainty: "exact",
|
|
14523
|
+
causalProofTier: proofTier
|
|
14524
|
+
},
|
|
14525
|
+
...findingRow.contending_trace_id ? [
|
|
14526
|
+
{
|
|
14527
|
+
label: `contending trace ${findingRow.contending_trace_id}`,
|
|
14528
|
+
href: `#/traces/${encodeURIComponent(findingRow.contending_trace_id)}`,
|
|
14529
|
+
entityKind: "trace",
|
|
14530
|
+
entityId: findingRow.contending_trace_id,
|
|
14531
|
+
relationshipCertainty: "inferred",
|
|
14532
|
+
causalProofTier: proofTier
|
|
14533
|
+
}
|
|
14534
|
+
] : []
|
|
14535
|
+
]
|
|
14536
|
+
}
|
|
14537
|
+
];
|
|
14538
|
+
manifest.down = [
|
|
14539
|
+
{
|
|
14540
|
+
label: "Agent Lineage",
|
|
14541
|
+
links: [
|
|
14542
|
+
...findingRow.agent_run_id ? [
|
|
14543
|
+
{
|
|
14544
|
+
...linkToAgentRun(findingRow.agent_run_id),
|
|
14545
|
+
relationshipCertainty: "exact",
|
|
14546
|
+
causalProofTier: proofTier
|
|
14547
|
+
}
|
|
14548
|
+
] : []
|
|
14549
|
+
],
|
|
14550
|
+
emptyReason: !findingRow.agent_run_id ? "No agent run linked to this finding." : void 0
|
|
14551
|
+
}
|
|
14552
|
+
];
|
|
14553
|
+
}
|
|
14554
|
+
} else if (kind === "incident") {
|
|
14555
|
+
const db = sqlDbFor(c.env);
|
|
14556
|
+
const incRow = await db.prepare(
|
|
14557
|
+
`SELECT * FROM incident_investigations WHERE project_id = ? AND id = ? LIMIT 1`
|
|
14558
|
+
).bind(projectId, id).first();
|
|
14559
|
+
if (incRow) {
|
|
14560
|
+
manifest.up = [
|
|
14561
|
+
{
|
|
14562
|
+
label: "Trigger Signal",
|
|
14563
|
+
links: [
|
|
14564
|
+
{
|
|
14565
|
+
label: `${incRow.symptom_anchor_kind}: ${incRow.symptom_anchor_id}`,
|
|
14566
|
+
href: `#/alerts?id=${encodeURIComponent(incRow.symptom_anchor_id)}`,
|
|
14567
|
+
entityKind: incRow.symptom_anchor_kind === "analysis" ? "analysis" : "alert",
|
|
14568
|
+
entityId: incRow.symptom_anchor_id,
|
|
14569
|
+
relationshipCertainty: "exact"
|
|
14570
|
+
}
|
|
14571
|
+
]
|
|
14572
|
+
}
|
|
14573
|
+
];
|
|
14574
|
+
const incFindings = await db.prepare(
|
|
14575
|
+
`SELECT id, title, causal_proof_tier FROM investigation_findings
|
|
14576
|
+
WHERE project_id = ? AND incident_id = ?
|
|
14577
|
+
ORDER BY created_at DESC LIMIT 5`
|
|
14578
|
+
).bind(projectId, incRow.id).all();
|
|
14579
|
+
manifest.across = [
|
|
14580
|
+
{
|
|
14581
|
+
label: "Findings",
|
|
14582
|
+
links: incFindings.results.map((f) => ({
|
|
14583
|
+
label: f.title,
|
|
14584
|
+
href: `#/findings/${encodeURIComponent(f.id)}`,
|
|
14585
|
+
entityKind: "finding",
|
|
14586
|
+
entityId: f.id,
|
|
14587
|
+
relationshipCertainty: "exact",
|
|
14588
|
+
causalProofTier: f.causal_proof_tier
|
|
14589
|
+
})),
|
|
14590
|
+
emptyReason: incFindings.results.length > 0 ? void 0 : "No finding recorded for this incident yet."
|
|
14591
|
+
}
|
|
14592
|
+
];
|
|
14593
|
+
}
|
|
11041
14594
|
} else if (kind === "alert" || kind === "analysis") {
|
|
11042
14595
|
manifest.related = [
|
|
11043
14596
|
{
|
|
@@ -11143,7 +14696,7 @@ var defaultSpanEnrichmentPlugin = {
|
|
|
11143
14696
|
const sessionId = typeof attributes[SESSION_ID_KEY2] === "string" && attributes[SESSION_ID_KEY2].length > 0 ? attributes[SESSION_ID_KEY2] : null;
|
|
11144
14697
|
const interactionId = typeof attributes[INTERACTION_ID_KEY2] === "string" && attributes[INTERACTION_ID_KEY2].length > 0 ? attributes[INTERACTION_ID_KEY2] : null;
|
|
11145
14698
|
const telemetrySdkName = typeof resourceAttributes["telemetry.sdk.name"] === "string" && resourceAttributes["telemetry.sdk.name"].length > 0 ? resourceAttributes["telemetry.sdk.name"] : null;
|
|
11146
|
-
|
|
14699
|
+
const candidate = {
|
|
11147
14700
|
...span,
|
|
11148
14701
|
serviceName,
|
|
11149
14702
|
attributesJson: JSON.stringify(normalizedAttributes),
|
|
@@ -11151,6 +14704,15 @@ var defaultSpanEnrichmentPlugin = {
|
|
|
11151
14704
|
interactionId,
|
|
11152
14705
|
telemetrySdkName
|
|
11153
14706
|
};
|
|
14707
|
+
const lineage = extractSpanLineageAndTopology(candidate);
|
|
14708
|
+
return {
|
|
14709
|
+
...candidate,
|
|
14710
|
+
environment: lineage.environment,
|
|
14711
|
+
serviceVersion: lineage.serviceVersion,
|
|
14712
|
+
commitSha: lineage.commitSha,
|
|
14713
|
+
targetNodeId: lineage.targetNode?.id ?? null,
|
|
14714
|
+
errorSignature: lineage.errorSignature
|
|
14715
|
+
};
|
|
11154
14716
|
});
|
|
11155
14717
|
}
|
|
11156
14718
|
});
|
|
@@ -11311,8 +14873,18 @@ var rowToEvalRun = (row) => {
|
|
|
11311
14873
|
promptVersion: row.candidate_prompt_version ?? null,
|
|
11312
14874
|
modelProvider: row.candidate_model_provider ?? null,
|
|
11313
14875
|
model: row.candidate_model ?? null,
|
|
11314
|
-
modelVersion: row.candidate_model_version ?? null
|
|
14876
|
+
modelVersion: row.candidate_model_version ?? null,
|
|
14877
|
+
environment: row.environment ?? "default",
|
|
14878
|
+
serviceVersion: row.service_version ?? null,
|
|
14879
|
+
commitSha: row.commit_sha ?? null,
|
|
14880
|
+
configDigest: row.config_digest ?? null
|
|
11315
14881
|
},
|
|
14882
|
+
environment: row.environment ?? "default",
|
|
14883
|
+
serviceVersion: row.service_version ?? null,
|
|
14884
|
+
commitSha: row.commit_sha ?? null,
|
|
14885
|
+
configDigest: row.config_digest ?? null,
|
|
14886
|
+
inputPayloadHash: row.input_payload_hash ?? null,
|
|
14887
|
+
failureFingerprint: row.failure_fingerprint ?? null,
|
|
11316
14888
|
startedAt: row.started_at ?? null,
|
|
11317
14889
|
endedAt: row.ended_at ?? null,
|
|
11318
14890
|
totalCount: row.total_count ?? 0,
|
|
@@ -11443,6 +15015,12 @@ var EvalCasesStore = class {
|
|
|
11443
15015
|
if (!evalCase) throw new Error("evalCaseId does not reference a case");
|
|
11444
15016
|
}
|
|
11445
15017
|
const candidate = input.candidate ?? {};
|
|
15018
|
+
const environment = input.environment?.trim() || candidate.environment?.trim() || "default";
|
|
15019
|
+
const serviceVersion = input.serviceVersion?.trim() || candidate.serviceVersion?.trim() || null;
|
|
15020
|
+
const commitSha = input.commitSha?.trim() || candidate.commitSha?.trim() || null;
|
|
15021
|
+
const configDigest = input.configDigest?.trim() || candidate.configDigest?.trim() || null;
|
|
15022
|
+
const inputPayloadHash = input.inputPayloadHash?.trim() || null;
|
|
15023
|
+
const failureFingerprint = input.failureFingerprint?.trim() || null;
|
|
11446
15024
|
const totalCount = nonNegativeInteger(input.totalCount);
|
|
11447
15025
|
const passCount = nonNegativeInteger(input.passCount);
|
|
11448
15026
|
const failCount = nonNegativeInteger(input.failCount);
|
|
@@ -11456,10 +15034,12 @@ var EvalCasesStore = class {
|
|
|
11456
15034
|
candidate_agent_id, candidate_agent_version,
|
|
11457
15035
|
candidate_prompt_id, candidate_prompt_version,
|
|
11458
15036
|
candidate_model_provider, candidate_model, candidate_model_version,
|
|
15037
|
+
environment, service_version, commit_sha, config_digest,
|
|
15038
|
+
input_payload_hash, failure_fingerprint,
|
|
11459
15039
|
status, started_at, ended_at,
|
|
11460
15040
|
total_count, pass_count, fail_count, average_score,
|
|
11461
15041
|
metadata_json, created_at
|
|
11462
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
15042
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
11463
15043
|
).bind(
|
|
11464
15044
|
id,
|
|
11465
15045
|
projectId,
|
|
@@ -11471,6 +15051,12 @@ var EvalCasesStore = class {
|
|
|
11471
15051
|
candidate.modelProvider ?? null,
|
|
11472
15052
|
candidate.model ?? null,
|
|
11473
15053
|
candidate.modelVersion ?? null,
|
|
15054
|
+
environment,
|
|
15055
|
+
serviceVersion,
|
|
15056
|
+
commitSha,
|
|
15057
|
+
configDigest,
|
|
15058
|
+
inputPayloadHash,
|
|
15059
|
+
failureFingerprint,
|
|
11474
15060
|
status,
|
|
11475
15061
|
startedAt,
|
|
11476
15062
|
endedAt,
|
|
@@ -11537,6 +15123,14 @@ var EvalCasesStore = class {
|
|
|
11537
15123
|
average_score = (
|
|
11538
15124
|
SELECT AVG(score) FROM eval_case_results
|
|
11539
15125
|
WHERE project_id = ? AND run_id = ? AND score IS NOT NULL
|
|
15126
|
+
),
|
|
15127
|
+
input_payload_hash = COALESCE(
|
|
15128
|
+
input_payload_hash,
|
|
15129
|
+
(SELECT input_payload_hash FROM eval_case_results WHERE project_id = ? AND run_id = ? AND input_payload_hash IS NOT NULL LIMIT 1)
|
|
15130
|
+
),
|
|
15131
|
+
failure_fingerprint = COALESCE(
|
|
15132
|
+
failure_fingerprint,
|
|
15133
|
+
(SELECT failure_fingerprint FROM eval_case_results WHERE project_id = ? AND run_id = ? AND failure_fingerprint IS NOT NULL LIMIT 1)
|
|
11540
15134
|
)
|
|
11541
15135
|
WHERE project_id = ? AND id = ?`
|
|
11542
15136
|
).bind(
|
|
@@ -11549,6 +15143,10 @@ var EvalCasesStore = class {
|
|
|
11549
15143
|
projectId,
|
|
11550
15144
|
id,
|
|
11551
15145
|
projectId,
|
|
15146
|
+
id,
|
|
15147
|
+
projectId,
|
|
15148
|
+
id,
|
|
15149
|
+
projectId,
|
|
11552
15150
|
id
|
|
11553
15151
|
).run();
|
|
11554
15152
|
}
|
|
@@ -11829,7 +15427,7 @@ var badRequest = (message) => ({
|
|
|
11829
15427
|
error: "Bad Request",
|
|
11830
15428
|
message
|
|
11831
15429
|
});
|
|
11832
|
-
var
|
|
15430
|
+
var parseLimit2 = (value) => {
|
|
11833
15431
|
const parsed = Number.parseInt(value ?? "50", 10);
|
|
11834
15432
|
if (!Number.isFinite(parsed)) return 50;
|
|
11835
15433
|
return Math.min(200, Math.max(1, parsed));
|
|
@@ -11928,7 +15526,7 @@ var evalCasesRoutesPlugin = {
|
|
|
11928
15526
|
projectId: getProjectId(c),
|
|
11929
15527
|
sourceEntityType,
|
|
11930
15528
|
sourceEntityId: c.req.query("sourceEntityId"),
|
|
11931
|
-
limit:
|
|
15529
|
+
limit: parseLimit2(c.req.query("limit"))
|
|
11932
15530
|
});
|
|
11933
15531
|
return c.json({ evalCases });
|
|
11934
15532
|
});
|
|
@@ -12000,6 +15598,12 @@ var evalCasesRoutesPlugin = {
|
|
|
12000
15598
|
id: id ?? void 0,
|
|
12001
15599
|
evalCaseId,
|
|
12002
15600
|
status,
|
|
15601
|
+
environment: asOptionalString(body.environment) ?? asOptionalString(candidate.environment),
|
|
15602
|
+
serviceVersion: asOptionalString(body.serviceVersion) ?? asOptionalString(candidate.serviceVersion),
|
|
15603
|
+
commitSha: asOptionalString(body.commitSha) ?? asOptionalString(candidate.commitSha),
|
|
15604
|
+
configDigest: asOptionalString(body.configDigest) ?? asOptionalString(candidate.configDigest),
|
|
15605
|
+
inputPayloadHash: asOptionalString(body.inputPayloadHash),
|
|
15606
|
+
failureFingerprint: asOptionalString(body.failureFingerprint),
|
|
12003
15607
|
candidate: {
|
|
12004
15608
|
agentId: asOptionalString(candidate.agentId),
|
|
12005
15609
|
agentVersion: asOptionalString(candidate.agentVersion),
|
|
@@ -12007,7 +15611,11 @@ var evalCasesRoutesPlugin = {
|
|
|
12007
15611
|
promptVersion: asOptionalString(candidate.promptVersion),
|
|
12008
15612
|
modelProvider: asOptionalString(candidate.modelProvider),
|
|
12009
15613
|
model: asOptionalString(candidate.model),
|
|
12010
|
-
modelVersion: asOptionalString(candidate.modelVersion)
|
|
15614
|
+
modelVersion: asOptionalString(candidate.modelVersion),
|
|
15615
|
+
environment: asOptionalString(candidate.environment) ?? asOptionalString(body.environment),
|
|
15616
|
+
serviceVersion: asOptionalString(candidate.serviceVersion) ?? asOptionalString(body.serviceVersion),
|
|
15617
|
+
commitSha: asOptionalString(candidate.commitSha) ?? asOptionalString(body.commitSha),
|
|
15618
|
+
configDigest: asOptionalString(candidate.configDigest) ?? asOptionalString(body.configDigest)
|
|
12011
15619
|
},
|
|
12012
15620
|
startedAt: asOptionalString(body.startedAt),
|
|
12013
15621
|
endedAt: asOptionalString(body.endedAt),
|
|
@@ -12040,7 +15648,7 @@ var evalCasesRoutesPlugin = {
|
|
|
12040
15648
|
projectId: getProjectId(c),
|
|
12041
15649
|
evalCaseId: c.req.query("evalCaseId"),
|
|
12042
15650
|
status,
|
|
12043
|
-
limit:
|
|
15651
|
+
limit: parseLimit2(c.req.query("limit"))
|
|
12044
15652
|
});
|
|
12045
15653
|
return c.json({ evalRuns });
|
|
12046
15654
|
});
|
|
@@ -12089,12 +15697,15 @@ var evalCasesRoutesPlugin = {
|
|
|
12089
15697
|
const passed = body.passed ? 1 : 0;
|
|
12090
15698
|
const score = typeof body.score === "number" && Number.isFinite(body.score) ? body.score : null;
|
|
12091
15699
|
const actualOutcome = typeof body.actualOutcome === "string" ? body.actualOutcome : null;
|
|
15700
|
+
const inputPayloadHash = typeof body.inputPayloadHash === "string" && body.inputPayloadHash.trim() ? body.inputPayloadHash.trim() : null;
|
|
15701
|
+
const failureFingerprint = typeof body.failureFingerprint === "string" && body.failureFingerprint.trim() ? body.failureFingerprint.trim() : null;
|
|
12092
15702
|
const details_json = body.details === void 0 ? null : JSON.stringify(body.details);
|
|
12093
15703
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12094
15704
|
await db.prepare(
|
|
12095
15705
|
`INSERT INTO eval_case_results (
|
|
12096
|
-
id, project_id, eval_case_id, run_id, passed, score, actual_outcome,
|
|
12097
|
-
|
|
15706
|
+
id, project_id, eval_case_id, run_id, passed, score, actual_outcome,
|
|
15707
|
+
input_payload_hash, failure_fingerprint, details_json, created_at
|
|
15708
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
12098
15709
|
).bind(
|
|
12099
15710
|
id,
|
|
12100
15711
|
projectId,
|
|
@@ -12103,6 +15714,8 @@ var evalCasesRoutesPlugin = {
|
|
|
12103
15714
|
passed,
|
|
12104
15715
|
score,
|
|
12105
15716
|
actualOutcome,
|
|
15717
|
+
inputPayloadHash,
|
|
15718
|
+
failureFingerprint,
|
|
12106
15719
|
details_json,
|
|
12107
15720
|
now
|
|
12108
15721
|
).run();
|
|
@@ -12116,6 +15729,8 @@ var evalCasesRoutesPlugin = {
|
|
|
12116
15729
|
passed: body.passed,
|
|
12117
15730
|
score,
|
|
12118
15731
|
actualOutcome,
|
|
15732
|
+
inputPayloadHash,
|
|
15733
|
+
failureFingerprint,
|
|
12119
15734
|
details: body.details ?? null,
|
|
12120
15735
|
createdAt: now,
|
|
12121
15736
|
evidenceReferences: evalCaseResultEvidenceReferences(evalCase, id),
|
|
@@ -12155,6 +15770,8 @@ var evalCasesRoutesPlugin = {
|
|
|
12155
15770
|
passed: row.passed === 1,
|
|
12156
15771
|
score: row.score,
|
|
12157
15772
|
actualOutcome: row.actual_outcome,
|
|
15773
|
+
inputPayloadHash: row.input_payload_hash ?? null,
|
|
15774
|
+
failureFingerprint: row.failure_fingerprint ?? null,
|
|
12158
15775
|
details: parseJsonMaybe(row.details_json),
|
|
12159
15776
|
createdAt: row.created_at,
|
|
12160
15777
|
evidenceReferences: evalCaseResultEvidenceReferences(
|
|
@@ -12417,6 +16034,7 @@ var DEFAULT_PROFILE_FRAME_LIMIT = 50;
|
|
|
12417
16034
|
var MAX_PROFILE_FRAME_LIMIT = 200;
|
|
12418
16035
|
var EVIDENCE_INTENTS = /* @__PURE__ */ new Set([
|
|
12419
16036
|
"debug_failure",
|
|
16037
|
+
"triage_incident",
|
|
12420
16038
|
"explain_latency",
|
|
12421
16039
|
"explain_cost",
|
|
12422
16040
|
"inspect_agent_run",
|
|
@@ -13464,6 +17082,156 @@ var buildActionGraphBundle = (projectId, anchorKind, anchorId, manifest, profile
|
|
|
13464
17082
|
};
|
|
13465
17083
|
return bundle;
|
|
13466
17084
|
};
|
|
17085
|
+
var attachAutoSreTriageContext = async (db, projectId, traceId, bundle) => {
|
|
17086
|
+
try {
|
|
17087
|
+
const faultLocus = traceId ? await loadTraceFaultLocusFromDb(db, projectId, traceId) : null;
|
|
17088
|
+
const environment = (faultLocus?.faultLocusSpan?.environment ?? bundle.environment ?? "default").toLowerCase();
|
|
17089
|
+
const recentChanges = await listChangeEvents(db, {
|
|
17090
|
+
projectId,
|
|
17091
|
+
environment,
|
|
17092
|
+
limit: 5
|
|
17093
|
+
});
|
|
17094
|
+
const blastRadii = (await Promise.all(
|
|
17095
|
+
recentChanges.slice(0, 3).map(
|
|
17096
|
+
(ch) => computeChangeBlastRadius(db, projectId, ch.id, { environment })
|
|
17097
|
+
)
|
|
17098
|
+
)).filter((br) => Boolean(br));
|
|
17099
|
+
const candidateChanges = blastRadii.map((br) => ({
|
|
17100
|
+
changeId: br.change.id,
|
|
17101
|
+
kind: br.change.kind,
|
|
17102
|
+
version: br.change.version,
|
|
17103
|
+
previousVersion: br.change.previousVersion,
|
|
17104
|
+
occurredAt: br.change.occurredAt,
|
|
17105
|
+
cohortVerdict: br.cohortVerdict,
|
|
17106
|
+
causalProofTier: br.causalProofTier,
|
|
17107
|
+
rateNew: br.newCohort.errorRate,
|
|
17108
|
+
rateOld: br.oldCohort.errorRate,
|
|
17109
|
+
sampleNew: br.newCohort.requestCount,
|
|
17110
|
+
sampleOld: br.oldCohort.requestCount,
|
|
17111
|
+
overlappingCodePaths: br.overlappingCodePaths
|
|
17112
|
+
}));
|
|
17113
|
+
const contentionCandidates = blastRadii.flatMap((br) => br.sharedResourceAggressors).map((ag) => ({
|
|
17114
|
+
aggressorTraceId: ag.aggressorTraceId,
|
|
17115
|
+
aggressorSpanId: ag.aggressorSpanId,
|
|
17116
|
+
aggressorServiceName: ag.aggressorServiceName,
|
|
17117
|
+
sharedTargetNodeId: ag.sharedTargetNodeId,
|
|
17118
|
+
relationshipCertainty: ag.relationshipCertainty,
|
|
17119
|
+
totalResourceTimeMs: ag.totalResourceTimeMs
|
|
17120
|
+
}));
|
|
17121
|
+
let priorFindingsList = await searchInvestigationFindings(db, {
|
|
17122
|
+
projectId,
|
|
17123
|
+
environment,
|
|
17124
|
+
rootCauseEntityId: faultLocus?.faultLocusNodeId ?? faultLocus?.faultLocusSpan?.span_id ?? void 0,
|
|
17125
|
+
limit: 5
|
|
17126
|
+
});
|
|
17127
|
+
if (priorFindingsList.length === 0 && faultLocus?.faultLocusSpan?.span_id) {
|
|
17128
|
+
priorFindingsList = await searchInvestigationFindings(db, {
|
|
17129
|
+
projectId,
|
|
17130
|
+
environment,
|
|
17131
|
+
rootCauseEntityId: faultLocus.faultLocusSpan.span_id,
|
|
17132
|
+
limit: 5
|
|
17133
|
+
});
|
|
17134
|
+
}
|
|
17135
|
+
if (priorFindingsList.length === 0) {
|
|
17136
|
+
priorFindingsList = await searchInvestigationFindings(db, {
|
|
17137
|
+
projectId,
|
|
17138
|
+
environment,
|
|
17139
|
+
limit: 5
|
|
17140
|
+
});
|
|
17141
|
+
}
|
|
17142
|
+
const priorFindings = priorFindingsList.map((f) => ({
|
|
17143
|
+
findingId: f.id,
|
|
17144
|
+
title: f.title,
|
|
17145
|
+
causalProofTier: f.causalProofTier,
|
|
17146
|
+
causalVerificationStatus: f.causalVerificationStatus,
|
|
17147
|
+
recoveryStatus: f.recoveryStatus,
|
|
17148
|
+
contributingChangeId: f.contributingChangeId,
|
|
17149
|
+
occurredAt: f.createdAt
|
|
17150
|
+
}));
|
|
17151
|
+
for (const br of blastRadii) {
|
|
17152
|
+
const changeId = br.change.id;
|
|
17153
|
+
const changeRoute = `#/changes/${encodeURIComponent(changeId)}`;
|
|
17154
|
+
bundle.evidenceReferences.push({
|
|
17155
|
+
evidenceId: `change:${changeId}`,
|
|
17156
|
+
entityKind: "change",
|
|
17157
|
+
entityId: changeId,
|
|
17158
|
+
route: changeRoute,
|
|
17159
|
+
source: br.overlappingCodePaths.length ? "stack_frame_overlap" : "cohort_comparison",
|
|
17160
|
+
confidence: br.cohortVerdict === "isolated_to_new_version" ? 0.9 : 0.65,
|
|
17161
|
+
relationshipCertainty: br.relationshipCertainty,
|
|
17162
|
+
causalProofTier: br.causalProofTier,
|
|
17163
|
+
reason: br.summary,
|
|
17164
|
+
citations: [
|
|
17165
|
+
{
|
|
17166
|
+
label: `change ${changeId} (${br.change.version ?? br.change.kind})`,
|
|
17167
|
+
entityKind: "change",
|
|
17168
|
+
entityId: changeId,
|
|
17169
|
+
route: changeRoute
|
|
17170
|
+
}
|
|
17171
|
+
],
|
|
17172
|
+
suggestedNextPivots: [
|
|
17173
|
+
{
|
|
17174
|
+
label: "Inspect change blast radius",
|
|
17175
|
+
entityKind: "change",
|
|
17176
|
+
entityId: changeId,
|
|
17177
|
+
route: changeRoute,
|
|
17178
|
+
reason: "Inspect traffic-normalized cohort lift and contention."
|
|
17179
|
+
}
|
|
17180
|
+
]
|
|
17181
|
+
});
|
|
17182
|
+
}
|
|
17183
|
+
for (const finding of priorFindingsList) {
|
|
17184
|
+
const findingRoute = `#/findings/${encodeURIComponent(finding.id)}`;
|
|
17185
|
+
bundle.evidenceReferences.push({
|
|
17186
|
+
evidenceId: `finding:${finding.id}`,
|
|
17187
|
+
entityKind: "finding",
|
|
17188
|
+
entityId: finding.id,
|
|
17189
|
+
route: findingRoute,
|
|
17190
|
+
source: "incident_memory_fingerprint",
|
|
17191
|
+
confidence: finding.confidence,
|
|
17192
|
+
relationshipCertainty: "exact",
|
|
17193
|
+
causalProofTier: finding.causalProofTier,
|
|
17194
|
+
reason: `Prior finding (${finding.causalProofTier}): ${finding.title}`,
|
|
17195
|
+
citations: [
|
|
17196
|
+
{
|
|
17197
|
+
label: finding.title,
|
|
17198
|
+
entityKind: "finding",
|
|
17199
|
+
entityId: finding.id,
|
|
17200
|
+
route: findingRoute
|
|
17201
|
+
}
|
|
17202
|
+
],
|
|
17203
|
+
suggestedNextPivots: [
|
|
17204
|
+
{
|
|
17205
|
+
label: "Open prior finding",
|
|
17206
|
+
entityKind: "finding",
|
|
17207
|
+
entityId: finding.id,
|
|
17208
|
+
route: findingRoute,
|
|
17209
|
+
reason: "Search incident memory for similar recurring findings."
|
|
17210
|
+
}
|
|
17211
|
+
]
|
|
17212
|
+
});
|
|
17213
|
+
}
|
|
17214
|
+
bundle.autosre = {
|
|
17215
|
+
faultLocus: faultLocus?.faultLocusNodeId ? {
|
|
17216
|
+
nodeId: faultLocus.faultLocusNodeId,
|
|
17217
|
+
spanId: faultLocus.faultLocusSpan?.span_id,
|
|
17218
|
+
relationshipCertainty: faultLocus.relationshipCertainty,
|
|
17219
|
+
isBlackoutSuspected: false
|
|
17220
|
+
} : void 0,
|
|
17221
|
+
candidateChanges,
|
|
17222
|
+
contentionCandidates,
|
|
17223
|
+
priorFindings
|
|
17224
|
+
};
|
|
17225
|
+
if (bundle.budget) {
|
|
17226
|
+
bundle.budget = {
|
|
17227
|
+
...bundle.budget,
|
|
17228
|
+
estimatedTokens: estimateTokens(bundle)
|
|
17229
|
+
};
|
|
17230
|
+
}
|
|
17231
|
+
} catch {
|
|
17232
|
+
}
|
|
17233
|
+
return bundle;
|
|
17234
|
+
};
|
|
13467
17235
|
var parseBundleRequest = async (request) => {
|
|
13468
17236
|
try {
|
|
13469
17237
|
const body = await request.json();
|
|
@@ -13529,7 +17297,7 @@ var evidenceRetrievalRoutesPlugin = {
|
|
|
13529
17297
|
)).flat().filter(
|
|
13530
17298
|
(profile, index, all) => all.findIndex((candidate) => candidate.id === profile.id) === index
|
|
13531
17299
|
);
|
|
13532
|
-
|
|
17300
|
+
let bundle2 = buildActionGraphBundle(
|
|
13533
17301
|
projectId,
|
|
13534
17302
|
anchor.entityKind,
|
|
13535
17303
|
anchor.entityId,
|
|
@@ -13539,13 +17307,107 @@ var evidenceRetrievalRoutesPlugin = {
|
|
|
13539
17307
|
targetTokens,
|
|
13540
17308
|
detailLevel
|
|
13541
17309
|
);
|
|
17310
|
+
if (intent === "triage_incident") {
|
|
17311
|
+
bundle2 = await attachAutoSreTriageContext(
|
|
17312
|
+
db,
|
|
17313
|
+
projectId,
|
|
17314
|
+
traceIds[0] ?? null,
|
|
17315
|
+
bundle2
|
|
17316
|
+
);
|
|
17317
|
+
}
|
|
13542
17318
|
return c.json(await materializeBundle(db, projectId, bundle2));
|
|
13543
17319
|
}
|
|
17320
|
+
if (anchor.entityKind === "incident") {
|
|
17321
|
+
const incident = await getIncidentInvestigation(
|
|
17322
|
+
db,
|
|
17323
|
+
projectId,
|
|
17324
|
+
anchor.entityId
|
|
17325
|
+
);
|
|
17326
|
+
if (!incident) {
|
|
17327
|
+
return c.json(
|
|
17328
|
+
{ error: "Not Found", message: "Incident not found" },
|
|
17329
|
+
404
|
|
17330
|
+
);
|
|
17331
|
+
}
|
|
17332
|
+
const recentErrSpan = await db.prepare(
|
|
17333
|
+
`SELECT trace_id FROM telemetry_spans
|
|
17334
|
+
WHERE project_id = ? AND environment = ? AND status_code = 2
|
|
17335
|
+
ORDER BY start_time DESC LIMIT 1`
|
|
17336
|
+
).bind(projectId, incident.environment).first();
|
|
17337
|
+
const incRoute = `#/incidents/${encodeURIComponent(incident.id)}`;
|
|
17338
|
+
const baseBundle = {
|
|
17339
|
+
schemaVersion: BUNDLE_SCHEMA_VERSION,
|
|
17340
|
+
intent,
|
|
17341
|
+
environment: incident.environment,
|
|
17342
|
+
incidentId: incident.id,
|
|
17343
|
+
symptomFingerprint: incident.symptomFingerprint ?? void 0,
|
|
17344
|
+
anchor: { entityKind: "incident", entityId: incident.id },
|
|
17345
|
+
budget: { targetTokens, detailLevel },
|
|
17346
|
+
summary: `Incident ${incident.id} (${incident.status}) on ${incident.targetNodeId} in environment ${incident.environment}.`,
|
|
17347
|
+
derivedSummaries: [
|
|
17348
|
+
{
|
|
17349
|
+
title: `Incident ${incident.id}`,
|
|
17350
|
+
reason: `Trigger ${incident.symptomAnchorKind}:${incident.symptomAnchorId} on ${incident.targetNodeId} (status=${incident.status}).`,
|
|
17351
|
+
confidence: 1,
|
|
17352
|
+
evidenceIds: [`incident:${incident.id}`],
|
|
17353
|
+
retrievalRefIds: []
|
|
17354
|
+
}
|
|
17355
|
+
],
|
|
17356
|
+
findings: [],
|
|
17357
|
+
compactions: [],
|
|
17358
|
+
evidenceReferences: [
|
|
17359
|
+
{
|
|
17360
|
+
evidenceId: `incident:${incident.id}`,
|
|
17361
|
+
entityKind: "incident",
|
|
17362
|
+
entityId: incident.id,
|
|
17363
|
+
route: incRoute,
|
|
17364
|
+
source: "incident_investigations",
|
|
17365
|
+
confidence: 1,
|
|
17366
|
+
relationshipCertainty: "exact",
|
|
17367
|
+
reason: `Incident triggered by ${incident.symptomAnchorKind}:${incident.symptomAnchorId}.`,
|
|
17368
|
+
citations: [
|
|
17369
|
+
{
|
|
17370
|
+
label: incident.targetNodeId,
|
|
17371
|
+
entityKind: "incident",
|
|
17372
|
+
entityId: incident.id,
|
|
17373
|
+
route: incRoute
|
|
17374
|
+
}
|
|
17375
|
+
],
|
|
17376
|
+
suggestedNextPivots: [
|
|
17377
|
+
{
|
|
17378
|
+
label: "Open incident",
|
|
17379
|
+
entityKind: "incident",
|
|
17380
|
+
entityId: incident.id,
|
|
17381
|
+
route: incRoute,
|
|
17382
|
+
reason: "Inspect incident investigation and candidate changes."
|
|
17383
|
+
}
|
|
17384
|
+
]
|
|
17385
|
+
}
|
|
17386
|
+
],
|
|
17387
|
+
retrievalRefs: [],
|
|
17388
|
+
suggestedNextPivots: [
|
|
17389
|
+
{
|
|
17390
|
+
label: "Open incident",
|
|
17391
|
+
entityKind: "incident",
|
|
17392
|
+
entityId: incident.id,
|
|
17393
|
+
route: incRoute,
|
|
17394
|
+
reason: "Inspect incident investigation and candidate changes."
|
|
17395
|
+
}
|
|
17396
|
+
]
|
|
17397
|
+
};
|
|
17398
|
+
const enriched = await attachAutoSreTriageContext(
|
|
17399
|
+
db,
|
|
17400
|
+
projectId,
|
|
17401
|
+
recentErrSpan?.trace_id ?? null,
|
|
17402
|
+
baseBundle
|
|
17403
|
+
);
|
|
17404
|
+
return c.json(await materializeBundle(db, projectId, enriched));
|
|
17405
|
+
}
|
|
13544
17406
|
if (anchor.entityKind !== "trace") {
|
|
13545
17407
|
return c.json(
|
|
13546
17408
|
{
|
|
13547
17409
|
error: "Bad Request",
|
|
13548
|
-
message: "Supported evidence bundle anchors are trace, action, agent_run, and
|
|
17410
|
+
message: "Supported evidence bundle anchors are trace, action, agent_run, tool_call, and incident."
|
|
13549
17411
|
},
|
|
13550
17412
|
400
|
|
13551
17413
|
);
|
|
@@ -13563,7 +17425,7 @@ var evidenceRetrievalRoutesPlugin = {
|
|
|
13563
17425
|
limit: MAX_LOGS_FOR_BUNDLE
|
|
13564
17426
|
});
|
|
13565
17427
|
const profiles = await profilesForTrace(db, projectId, traceId);
|
|
13566
|
-
|
|
17428
|
+
let bundle = await buildTraceBundle(
|
|
13567
17429
|
projectId,
|
|
13568
17430
|
detail,
|
|
13569
17431
|
logs.logs,
|
|
@@ -13573,6 +17435,14 @@ var evidenceRetrievalRoutesPlugin = {
|
|
|
13573
17435
|
detailLevel,
|
|
13574
17436
|
hours
|
|
13575
17437
|
);
|
|
17438
|
+
if (intent === "triage_incident" || intent === "debug_failure") {
|
|
17439
|
+
bundle = await attachAutoSreTriageContext(
|
|
17440
|
+
db,
|
|
17441
|
+
projectId,
|
|
17442
|
+
traceId,
|
|
17443
|
+
bundle
|
|
17444
|
+
);
|
|
17445
|
+
}
|
|
13576
17446
|
return c.json(await materializeBundle(db, projectId, bundle));
|
|
13577
17447
|
});
|
|
13578
17448
|
app.get("/internal/evidence/stats", async (c) => {
|
|
@@ -14492,7 +18362,7 @@ var parseHours = (rawValue) => Math.min(
|
|
|
14492
18362
|
Number.parseInt(rawValue || String(DEFAULT_WINDOW_HOURS2), 10) || DEFAULT_WINDOW_HOURS2
|
|
14493
18363
|
)
|
|
14494
18364
|
);
|
|
14495
|
-
var
|
|
18365
|
+
var parseLimit3 = (rawValue) => Math.min(
|
|
14496
18366
|
MAX_ISSUE_ROWS,
|
|
14497
18367
|
Math.max(
|
|
14498
18368
|
1,
|
|
@@ -14511,7 +18381,7 @@ var issueInsightsPlugin = {
|
|
|
14511
18381
|
service: c.req.query("service") || void 0,
|
|
14512
18382
|
category: c.req.query("category") === "error" || c.req.query("category") === "latency" || c.req.query("category") === "dependency" ? c.req.query("category") : "all",
|
|
14513
18383
|
includeInternal: c.req.query("includeInternal") === "true",
|
|
14514
|
-
limit:
|
|
18384
|
+
limit: parseLimit3(c.req.query("limit"))
|
|
14515
18385
|
});
|
|
14516
18386
|
return c.json({
|
|
14517
18387
|
...overview,
|
|
@@ -18682,6 +22552,7 @@ var allPlugins = [
|
|
|
18682
22552
|
analysesRoutesPlugin,
|
|
18683
22553
|
actionRoutesPlugin,
|
|
18684
22554
|
evalCasesRoutesPlugin,
|
|
22555
|
+
autosreRoutesPlugin,
|
|
18685
22556
|
evidenceRetrievalRoutesPlugin,
|
|
18686
22557
|
askRoutesPlugin,
|
|
18687
22558
|
tailRoutesPlugin,
|
|
@@ -18704,6 +22575,7 @@ export {
|
|
|
18704
22575
|
ActionStore,
|
|
18705
22576
|
AlertsStore,
|
|
18706
22577
|
BlobStoreToR2Adapter,
|
|
22578
|
+
CounterfactualVerificationError,
|
|
18707
22579
|
D1Adapter,
|
|
18708
22580
|
EvalCaseSourceNotFoundError,
|
|
18709
22581
|
EvalCasesStore,
|
|
@@ -18711,8 +22583,10 @@ export {
|
|
|
18711
22583
|
PostgresAdapter,
|
|
18712
22584
|
ProjectsStore,
|
|
18713
22585
|
R2BlobStore,
|
|
22586
|
+
RemediationGateError,
|
|
18714
22587
|
S3BlobStore,
|
|
18715
22588
|
TailHub,
|
|
22589
|
+
acquireIncidentLease,
|
|
18716
22590
|
actionRetentionCutoff,
|
|
18717
22591
|
actionRoutesPlugin,
|
|
18718
22592
|
aiReceiverPlugin,
|
|
@@ -18720,9 +22594,15 @@ export {
|
|
|
18720
22594
|
alertsRoutesPlugin,
|
|
18721
22595
|
allPlugins,
|
|
18722
22596
|
analysesRoutesPlugin,
|
|
22597
|
+
approveRemediationProposal,
|
|
18723
22598
|
askRoutesPlugin,
|
|
22599
|
+
autosreRoutesPlugin,
|
|
18724
22600
|
botFilterPlugin,
|
|
22601
|
+
claimRemediationGrant,
|
|
18725
22602
|
compareValue,
|
|
22603
|
+
completeRemediationProposal,
|
|
22604
|
+
computeChangeBlastRadius,
|
|
22605
|
+
computeTarjanScc,
|
|
18726
22606
|
configurePostgresTypeParsers,
|
|
18727
22607
|
connectedRoutesPlugin,
|
|
18728
22608
|
consoleLogger,
|
|
@@ -18736,35 +22616,58 @@ export {
|
|
|
18736
22616
|
dashboardRoutesPlugin,
|
|
18737
22617
|
src_default as default,
|
|
18738
22618
|
defaultSpanEnrichmentPlugin,
|
|
22619
|
+
dispatchIncidentWebhook,
|
|
18739
22620
|
evalCasesRoutesPlugin,
|
|
18740
22621
|
evaluateAllRules,
|
|
22622
|
+
evaluateRemediationWatchdogs,
|
|
18741
22623
|
evidenceRetrievalRoutesPlugin,
|
|
22624
|
+
extractSpanLineageAndTopology,
|
|
22625
|
+
failRemediationProposal,
|
|
22626
|
+
findCrossTraceContention,
|
|
18742
22627
|
genAiNormalizerPlugin,
|
|
22628
|
+
getChangeEvent,
|
|
22629
|
+
getIncidentInvestigation,
|
|
22630
|
+
getInvestigationFinding,
|
|
22631
|
+
getOntologyGraph,
|
|
22632
|
+
getRemediationProposal,
|
|
18743
22633
|
identityReceiverPlugin,
|
|
18744
22634
|
isEvalCaseSourceType,
|
|
18745
22635
|
isEvalRunStatus,
|
|
18746
22636
|
issueEnrichmentPlugin,
|
|
18747
22637
|
issueInsightsPlugin,
|
|
22638
|
+
listChangeEvents,
|
|
22639
|
+
listIncidentInvestigations,
|
|
22640
|
+
listRemediationProposals,
|
|
22641
|
+
loadTraceFaultLocusFromDb,
|
|
22642
|
+
localizeTraceFaultLocus,
|
|
18748
22643
|
logsReceiverPlugin,
|
|
18749
22644
|
metricsReceiverPlugin,
|
|
18750
22645
|
onboardingRoutesPlugin,
|
|
22646
|
+
openOrDeduplicateIncident,
|
|
18751
22647
|
otlpReceiverPlugin,
|
|
18752
22648
|
platformRoutesPlugin,
|
|
18753
22649
|
profileRoutesPlugin,
|
|
18754
22650
|
projectsRoutesPlugin,
|
|
22651
|
+
proposeRemediationAction,
|
|
18755
22652
|
purgeExpiredActionGraph,
|
|
18756
22653
|
queryRoutesPlugin,
|
|
22654
|
+
recordChangeEvent,
|
|
22655
|
+
recordInvestigationFinding,
|
|
18757
22656
|
redactionProcessorPlugin,
|
|
22657
|
+
rejectRemediationProposal,
|
|
18758
22658
|
replayQueryRoutesPlugin,
|
|
18759
22659
|
replayReceiverPlugin,
|
|
22660
|
+
searchInvestigationFindings,
|
|
18760
22661
|
tailRoutesPlugin,
|
|
18761
22662
|
timelineRoutesPlugin,
|
|
18762
22663
|
uaEnrichmentPlugin,
|
|
22664
|
+
upsertOntologyFromSpans,
|
|
18763
22665
|
usageNormalizationPlugin,
|
|
18764
22666
|
usagePrivacyPlugin,
|
|
18765
22667
|
usageQueryRoutesPlugin,
|
|
18766
22668
|
usageReceiverPlugin,
|
|
18767
22669
|
usersQueryRoutesPlugin,
|
|
18768
|
-
utmEnrichmentPlugin
|
|
22670
|
+
utmEnrichmentPlugin,
|
|
22671
|
+
verifyPairedCounterfactualProof
|
|
18769
22672
|
};
|
|
18770
22673
|
//# sourceMappingURL=index.js.map
|