@obsunified/collector 2.1.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
- package/dist/chunk-ZSF5XP3B.js.map +1 -0
- package/dist/index.d.ts +309 -2
- package/dist/index.js +4159 -256
- package/dist/index.js.map +1 -1
- package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
- package/package.json +2 -2
- package/src/__tests__/autosre-closed-loop.test.ts +1275 -0
- package/src/index.ts +43 -0
- package/src/lib/analyses-runner.ts +9 -0
- package/src/lib/autosre-change-correlation.ts +780 -0
- package/src/lib/autosre-findings.ts +725 -0
- package/src/lib/autosre-ontology.ts +914 -0
- package/src/lib/autosre-remediation-gate.ts +1663 -0
- package/src/lib/eval-cases-store.ts +63 -1
- package/src/lib/evidence-references.ts +6 -0
- package/src/lib/housekeeping.test.ts +80 -0
- package/src/lib/housekeeping.ts +101 -1
- package/src/lib/store/ingest.ts +94 -6
- package/src/migrations/050_autosre_causal_ontology.sql +216 -0
- package/src/migrations/alert-state-notified.test.ts +3 -3
- package/src/migrations/parity.test.ts +4 -0
- package/src/migrations/project-scoped-keys.test.ts +1 -1
- package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
- package/src/plugins/alerts-evaluator.ts +17 -1
- package/src/plugins/autosre-routes.ts +1019 -0
- package/src/plugins/connected-routes/manifest.ts +13 -1
- package/src/plugins/connected-routes.ts +271 -0
- package/src/plugins/default-span-enrichment.ts +13 -1
- package/src/plugins/eval-cases-routes.ts +57 -2
- package/src/plugins/evidence-retrieval-routes.ts +300 -4
- package/dist/chunk-XKQTVIRE.js.map +0 -1
- /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
CausalProofTier,
|
|
3
|
+
RelationshipCertainty,
|
|
4
|
+
} from "@obsunified/types";
|
|
1
5
|
import type {
|
|
2
6
|
ActionRef,
|
|
3
7
|
AICallRef,
|
|
@@ -26,7 +30,10 @@ export type ConnectedEntityKind =
|
|
|
26
30
|
| "user"
|
|
27
31
|
| "action"
|
|
28
32
|
| "agent_run"
|
|
29
|
-
| "tool_call"
|
|
33
|
+
| "tool_call"
|
|
34
|
+
| "change"
|
|
35
|
+
| "finding"
|
|
36
|
+
| "incident";
|
|
30
37
|
|
|
31
38
|
export const KNOWN_KINDS: ReadonlySet<string> = new Set<ConnectedEntityKind>([
|
|
32
39
|
"span",
|
|
@@ -41,6 +48,9 @@ export const KNOWN_KINDS: ReadonlySet<string> = new Set<ConnectedEntityKind>([
|
|
|
41
48
|
"action",
|
|
42
49
|
"agent_run",
|
|
43
50
|
"tool_call",
|
|
51
|
+
"change",
|
|
52
|
+
"finding",
|
|
53
|
+
"incident",
|
|
44
54
|
]);
|
|
45
55
|
|
|
46
56
|
export interface ConnectedLink {
|
|
@@ -54,6 +64,8 @@ export interface ConnectedLink {
|
|
|
54
64
|
confidence?: number;
|
|
55
65
|
reason?: string;
|
|
56
66
|
causalConfidence?: CausalConfidence;
|
|
67
|
+
relationshipCertainty?: RelationshipCertainty;
|
|
68
|
+
causalProofTier?: CausalProofTier;
|
|
57
69
|
}
|
|
58
70
|
|
|
59
71
|
export interface ConnectedSection {
|
|
@@ -1049,6 +1049,277 @@ export const connectedRoutesPlugin: CollectorPlugin = {
|
|
|
1049
1049
|
manifest.rawManifest = actionManifest;
|
|
1050
1050
|
}
|
|
1051
1051
|
}
|
|
1052
|
+
} else if (kind === "change") {
|
|
1053
|
+
const db = sqlDbFor(c.env);
|
|
1054
|
+
const changeRow = await db
|
|
1055
|
+
.prepare(
|
|
1056
|
+
`SELECT * FROM change_events WHERE project_id = ? AND id = ? LIMIT 1`,
|
|
1057
|
+
)
|
|
1058
|
+
.bind(projectId, id)
|
|
1059
|
+
.first<{
|
|
1060
|
+
id: string;
|
|
1061
|
+
environment: string;
|
|
1062
|
+
kind: string;
|
|
1063
|
+
target_node_id: string;
|
|
1064
|
+
service_name: string | null;
|
|
1065
|
+
version: string | null;
|
|
1066
|
+
previous_version: string | null;
|
|
1067
|
+
commit_sha: string | null;
|
|
1068
|
+
summary: string;
|
|
1069
|
+
}>();
|
|
1070
|
+
if (changeRow) {
|
|
1071
|
+
manifest.up = [
|
|
1072
|
+
{
|
|
1073
|
+
label: "Target Ontology Node",
|
|
1074
|
+
links: [
|
|
1075
|
+
{
|
|
1076
|
+
label: `${changeRow.target_node_id} (${changeRow.environment})`,
|
|
1077
|
+
href: `#/health?node=${encodeURIComponent(changeRow.target_node_id)}`,
|
|
1078
|
+
relationshipCertainty: "exact",
|
|
1079
|
+
reason: "Target node mutated by this change event.",
|
|
1080
|
+
},
|
|
1081
|
+
],
|
|
1082
|
+
},
|
|
1083
|
+
];
|
|
1084
|
+
|
|
1085
|
+
const findings = await db
|
|
1086
|
+
.prepare(
|
|
1087
|
+
`SELECT id, title, causal_proof_tier, confidence, incident_id
|
|
1088
|
+
FROM investigation_findings
|
|
1089
|
+
WHERE project_id = ? AND environment = ? AND contributing_change_id = ?
|
|
1090
|
+
ORDER BY created_at DESC LIMIT 10`,
|
|
1091
|
+
)
|
|
1092
|
+
.bind(projectId, changeRow.environment, changeRow.id)
|
|
1093
|
+
.all<{
|
|
1094
|
+
id: string;
|
|
1095
|
+
title: string;
|
|
1096
|
+
causal_proof_tier: string;
|
|
1097
|
+
confidence: number;
|
|
1098
|
+
incident_id: string | null;
|
|
1099
|
+
}>();
|
|
1100
|
+
|
|
1101
|
+
manifest.down = [
|
|
1102
|
+
{
|
|
1103
|
+
label: "Attributed Findings",
|
|
1104
|
+
links: findings.results.map((f) => ({
|
|
1105
|
+
label: f.title,
|
|
1106
|
+
href: `#/findings/${encodeURIComponent(f.id)}`,
|
|
1107
|
+
entityKind: "finding",
|
|
1108
|
+
entityId: f.id,
|
|
1109
|
+
relationshipCertainty: "exact",
|
|
1110
|
+
causalProofTier:
|
|
1111
|
+
f.causal_proof_tier as import("@obsunified/types").CausalProofTier,
|
|
1112
|
+
confidence: f.confidence,
|
|
1113
|
+
reason: "Finding attributes this change event.",
|
|
1114
|
+
})),
|
|
1115
|
+
emptyReason:
|
|
1116
|
+
findings.results.length === 0
|
|
1117
|
+
? "No investigation findings cite this change event."
|
|
1118
|
+
: undefined,
|
|
1119
|
+
},
|
|
1120
|
+
];
|
|
1121
|
+
|
|
1122
|
+
const spans = changeRow.version
|
|
1123
|
+
? await db
|
|
1124
|
+
.prepare(
|
|
1125
|
+
`SELECT DISTINCT trace_id, span_id, service_name, span_name
|
|
1126
|
+
FROM telemetry_spans
|
|
1127
|
+
WHERE project_id = ? AND environment = ? AND service_version = ? AND status_code = 2
|
|
1128
|
+
ORDER BY start_time DESC LIMIT 5`,
|
|
1129
|
+
)
|
|
1130
|
+
.bind(projectId, changeRow.environment, changeRow.version)
|
|
1131
|
+
.all<{
|
|
1132
|
+
trace_id: string;
|
|
1133
|
+
span_id: string;
|
|
1134
|
+
service_name: string;
|
|
1135
|
+
span_name: string;
|
|
1136
|
+
}>()
|
|
1137
|
+
: { results: [] };
|
|
1138
|
+
|
|
1139
|
+
manifest.across = [
|
|
1140
|
+
{
|
|
1141
|
+
label: "Failing Spans on Cohort Version",
|
|
1142
|
+
links: spans.results.map((s) => ({
|
|
1143
|
+
label: `${s.service_name} · ${s.span_name}`,
|
|
1144
|
+
href: `#/traces/${s.trace_id}#span=${s.span_id}`,
|
|
1145
|
+
entityKind: "span",
|
|
1146
|
+
entityId: `${s.trace_id}:${s.span_id}`,
|
|
1147
|
+
relationshipCertainty: "exact",
|
|
1148
|
+
reason: `Span executed on version ${changeRow.version}.`,
|
|
1149
|
+
})),
|
|
1150
|
+
emptyReason:
|
|
1151
|
+
spans.results.length === 0
|
|
1152
|
+
? "No failing spans recorded on this change version."
|
|
1153
|
+
: undefined,
|
|
1154
|
+
},
|
|
1155
|
+
];
|
|
1156
|
+
}
|
|
1157
|
+
} else if (kind === "finding") {
|
|
1158
|
+
const db = sqlDbFor(c.env);
|
|
1159
|
+
const findingRow = await db
|
|
1160
|
+
.prepare(
|
|
1161
|
+
`SELECT * FROM investigation_findings WHERE project_id = ? AND id = ? LIMIT 1`,
|
|
1162
|
+
)
|
|
1163
|
+
.bind(projectId, id)
|
|
1164
|
+
.first<{
|
|
1165
|
+
id: string;
|
|
1166
|
+
environment: string;
|
|
1167
|
+
incident_id: string | null;
|
|
1168
|
+
title: string;
|
|
1169
|
+
causal_proof_tier: string;
|
|
1170
|
+
confidence: number;
|
|
1171
|
+
symptom_entity_kind: string;
|
|
1172
|
+
symptom_entity_id: string;
|
|
1173
|
+
fault_locus_entity_kind: string;
|
|
1174
|
+
fault_locus_entity_id: string;
|
|
1175
|
+
contributing_change_id: string | null;
|
|
1176
|
+
contending_trace_id: string | null;
|
|
1177
|
+
agent_run_id: string | null;
|
|
1178
|
+
}>();
|
|
1179
|
+
if (findingRow) {
|
|
1180
|
+
const proofTier =
|
|
1181
|
+
findingRow.causal_proof_tier as import("@obsunified/types").CausalProofTier;
|
|
1182
|
+
manifest.up = [];
|
|
1183
|
+
if (findingRow.incident_id) {
|
|
1184
|
+
manifest.up.push({
|
|
1185
|
+
label: "Incident Investigation",
|
|
1186
|
+
links: [
|
|
1187
|
+
{
|
|
1188
|
+
label: `incident ${findingRow.incident_id}`,
|
|
1189
|
+
href: `#/incidents/${encodeURIComponent(findingRow.incident_id)}`,
|
|
1190
|
+
entityKind: "incident",
|
|
1191
|
+
entityId: findingRow.incident_id,
|
|
1192
|
+
relationshipCertainty: "exact",
|
|
1193
|
+
causalProofTier: proofTier,
|
|
1194
|
+
},
|
|
1195
|
+
],
|
|
1196
|
+
});
|
|
1197
|
+
}
|
|
1198
|
+
if (findingRow.contributing_change_id) {
|
|
1199
|
+
manifest.up.push({
|
|
1200
|
+
label: "Contributing Change",
|
|
1201
|
+
links: [
|
|
1202
|
+
{
|
|
1203
|
+
label: `change ${findingRow.contributing_change_id}`,
|
|
1204
|
+
href: `#/changes/${encodeURIComponent(findingRow.contributing_change_id)}`,
|
|
1205
|
+
entityKind: "change",
|
|
1206
|
+
entityId: findingRow.contributing_change_id,
|
|
1207
|
+
relationshipCertainty: "exact",
|
|
1208
|
+
causalProofTier: proofTier,
|
|
1209
|
+
confidence: findingRow.confidence,
|
|
1210
|
+
},
|
|
1211
|
+
],
|
|
1212
|
+
});
|
|
1213
|
+
}
|
|
1214
|
+
manifest.across = [
|
|
1215
|
+
{
|
|
1216
|
+
label: "Fault Locus & Symptom",
|
|
1217
|
+
links: [
|
|
1218
|
+
{
|
|
1219
|
+
label: `fault locus: ${findingRow.fault_locus_entity_id}`,
|
|
1220
|
+
href: `#/traces?q=${encodeURIComponent(findingRow.fault_locus_entity_id)}`,
|
|
1221
|
+
relationshipCertainty: "exact",
|
|
1222
|
+
causalProofTier: proofTier,
|
|
1223
|
+
},
|
|
1224
|
+
...(findingRow.contending_trace_id
|
|
1225
|
+
? [
|
|
1226
|
+
{
|
|
1227
|
+
label: `contending trace ${findingRow.contending_trace_id}`,
|
|
1228
|
+
href: `#/traces/${encodeURIComponent(findingRow.contending_trace_id)}`,
|
|
1229
|
+
entityKind: "trace" as const,
|
|
1230
|
+
entityId: findingRow.contending_trace_id,
|
|
1231
|
+
relationshipCertainty: "inferred" as const,
|
|
1232
|
+
causalProofTier: proofTier,
|
|
1233
|
+
},
|
|
1234
|
+
]
|
|
1235
|
+
: []),
|
|
1236
|
+
],
|
|
1237
|
+
},
|
|
1238
|
+
];
|
|
1239
|
+
manifest.down = [
|
|
1240
|
+
{
|
|
1241
|
+
label: "Agent Lineage",
|
|
1242
|
+
links: [
|
|
1243
|
+
...(findingRow.agent_run_id
|
|
1244
|
+
? [
|
|
1245
|
+
{
|
|
1246
|
+
...linkToAgentRun(findingRow.agent_run_id),
|
|
1247
|
+
relationshipCertainty: "exact" as const,
|
|
1248
|
+
causalProofTier: proofTier,
|
|
1249
|
+
},
|
|
1250
|
+
]
|
|
1251
|
+
: []),
|
|
1252
|
+
],
|
|
1253
|
+
emptyReason: !findingRow.agent_run_id
|
|
1254
|
+
? "No agent run linked to this finding."
|
|
1255
|
+
: undefined,
|
|
1256
|
+
},
|
|
1257
|
+
];
|
|
1258
|
+
}
|
|
1259
|
+
} else if (kind === "incident") {
|
|
1260
|
+
const db = sqlDbFor(c.env);
|
|
1261
|
+
const incRow = await db
|
|
1262
|
+
.prepare(
|
|
1263
|
+
`SELECT * FROM incident_investigations WHERE project_id = ? AND id = ? LIMIT 1`,
|
|
1264
|
+
)
|
|
1265
|
+
.bind(projectId, id)
|
|
1266
|
+
.first<{
|
|
1267
|
+
id: string;
|
|
1268
|
+
environment: string;
|
|
1269
|
+
target_node_id: string;
|
|
1270
|
+
symptom_anchor_kind: string;
|
|
1271
|
+
symptom_anchor_id: string;
|
|
1272
|
+
status: string;
|
|
1273
|
+
}>();
|
|
1274
|
+
if (incRow) {
|
|
1275
|
+
manifest.up = [
|
|
1276
|
+
{
|
|
1277
|
+
label: "Trigger Signal",
|
|
1278
|
+
links: [
|
|
1279
|
+
{
|
|
1280
|
+
label: `${incRow.symptom_anchor_kind}: ${incRow.symptom_anchor_id}`,
|
|
1281
|
+
href: `#/alerts?id=${encodeURIComponent(incRow.symptom_anchor_id)}`,
|
|
1282
|
+
entityKind:
|
|
1283
|
+
incRow.symptom_anchor_kind === "analysis"
|
|
1284
|
+
? "analysis"
|
|
1285
|
+
: "alert",
|
|
1286
|
+
entityId: incRow.symptom_anchor_id,
|
|
1287
|
+
relationshipCertainty: "exact",
|
|
1288
|
+
},
|
|
1289
|
+
],
|
|
1290
|
+
},
|
|
1291
|
+
];
|
|
1292
|
+
const incFindings = await db
|
|
1293
|
+
.prepare(
|
|
1294
|
+
`SELECT id, title, causal_proof_tier FROM investigation_findings
|
|
1295
|
+
WHERE project_id = ? AND incident_id = ?
|
|
1296
|
+
ORDER BY created_at DESC LIMIT 5`,
|
|
1297
|
+
)
|
|
1298
|
+
.bind(projectId, incRow.id)
|
|
1299
|
+
.all<{
|
|
1300
|
+
id: string;
|
|
1301
|
+
title: string;
|
|
1302
|
+
causal_proof_tier: string;
|
|
1303
|
+
}>();
|
|
1304
|
+
manifest.across = [
|
|
1305
|
+
{
|
|
1306
|
+
label: "Findings",
|
|
1307
|
+
links: incFindings.results.map((f) => ({
|
|
1308
|
+
label: f.title,
|
|
1309
|
+
href: `#/findings/${encodeURIComponent(f.id)}`,
|
|
1310
|
+
entityKind: "finding" as const,
|
|
1311
|
+
entityId: f.id,
|
|
1312
|
+
relationshipCertainty: "exact" as const,
|
|
1313
|
+
causalProofTier:
|
|
1314
|
+
f.causal_proof_tier as import("@obsunified/types").CausalProofTier,
|
|
1315
|
+
})),
|
|
1316
|
+
emptyReason:
|
|
1317
|
+
incFindings.results.length > 0
|
|
1318
|
+
? undefined
|
|
1319
|
+
: "No finding recorded for this incident yet.",
|
|
1320
|
+
},
|
|
1321
|
+
];
|
|
1322
|
+
}
|
|
1052
1323
|
} else if (kind === "alert" || kind === "analysis") {
|
|
1053
1324
|
// Alerts and Analyses are topic-related, not identity-related.
|
|
1054
1325
|
// Their connections come from the alert_rules.analysis_id and
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
SESSION_ID_KEY,
|
|
5
5
|
} from "@obsunified/types/constants";
|
|
6
6
|
import type { CollectorPlugin } from "../framework/collector";
|
|
7
|
+
import { extractSpanLineageAndTopology } from "../lib/autosre-ontology";
|
|
7
8
|
import { parseJsonRecord } from "../lib/json";
|
|
8
9
|
|
|
9
10
|
export const defaultSpanEnrichmentPlugin: CollectorPlugin = {
|
|
@@ -77,7 +78,7 @@ export const defaultSpanEnrichmentPlugin: CollectorPlugin = {
|
|
|
77
78
|
? (resourceAttributes["telemetry.sdk.name"] as string)
|
|
78
79
|
: null;
|
|
79
80
|
|
|
80
|
-
|
|
81
|
+
const candidate: StoredSpan = {
|
|
81
82
|
...span,
|
|
82
83
|
serviceName,
|
|
83
84
|
attributesJson: JSON.stringify(normalizedAttributes),
|
|
@@ -85,6 +86,17 @@ export const defaultSpanEnrichmentPlugin: CollectorPlugin = {
|
|
|
85
86
|
interactionId,
|
|
86
87
|
telemetrySdkName,
|
|
87
88
|
};
|
|
89
|
+
|
|
90
|
+
const lineage = extractSpanLineageAndTopology(candidate);
|
|
91
|
+
|
|
92
|
+
return {
|
|
93
|
+
...candidate,
|
|
94
|
+
environment: lineage.environment,
|
|
95
|
+
serviceVersion: lineage.serviceVersion,
|
|
96
|
+
commitSha: lineage.commitSha,
|
|
97
|
+
targetNodeId: lineage.targetNode?.id ?? null,
|
|
98
|
+
errorSignature: lineage.errorSignature,
|
|
99
|
+
};
|
|
88
100
|
});
|
|
89
101
|
},
|
|
90
102
|
});
|
|
@@ -50,6 +50,8 @@ interface EvalCaseResultResponse {
|
|
|
50
50
|
passed: boolean;
|
|
51
51
|
score: number | null;
|
|
52
52
|
actualOutcome: string | null;
|
|
53
|
+
inputPayloadHash: string | null;
|
|
54
|
+
failureFingerprint: string | null;
|
|
53
55
|
details: unknown;
|
|
54
56
|
createdAt: string;
|
|
55
57
|
evidenceReferences?: EvidenceReference[];
|
|
@@ -62,6 +64,12 @@ interface EvalRunRequestBody {
|
|
|
62
64
|
evalCaseId?: unknown;
|
|
63
65
|
status?: unknown;
|
|
64
66
|
candidate?: unknown;
|
|
67
|
+
environment?: unknown;
|
|
68
|
+
serviceVersion?: unknown;
|
|
69
|
+
commitSha?: unknown;
|
|
70
|
+
configDigest?: unknown;
|
|
71
|
+
inputPayloadHash?: unknown;
|
|
72
|
+
failureFingerprint?: unknown;
|
|
65
73
|
startedAt?: unknown;
|
|
66
74
|
endedAt?: unknown;
|
|
67
75
|
totalCount?: unknown;
|
|
@@ -294,6 +302,20 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
294
302
|
id: id ?? undefined,
|
|
295
303
|
evalCaseId,
|
|
296
304
|
status,
|
|
305
|
+
environment:
|
|
306
|
+
asOptionalString(body.environment) ??
|
|
307
|
+
asOptionalString(candidate.environment),
|
|
308
|
+
serviceVersion:
|
|
309
|
+
asOptionalString(body.serviceVersion) ??
|
|
310
|
+
asOptionalString(candidate.serviceVersion),
|
|
311
|
+
commitSha:
|
|
312
|
+
asOptionalString(body.commitSha) ??
|
|
313
|
+
asOptionalString(candidate.commitSha),
|
|
314
|
+
configDigest:
|
|
315
|
+
asOptionalString(body.configDigest) ??
|
|
316
|
+
asOptionalString(candidate.configDigest),
|
|
317
|
+
inputPayloadHash: asOptionalString(body.inputPayloadHash),
|
|
318
|
+
failureFingerprint: asOptionalString(body.failureFingerprint),
|
|
297
319
|
candidate: {
|
|
298
320
|
agentId: asOptionalString(candidate.agentId),
|
|
299
321
|
agentVersion: asOptionalString(candidate.agentVersion),
|
|
@@ -302,6 +324,18 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
302
324
|
modelProvider: asOptionalString(candidate.modelProvider),
|
|
303
325
|
model: asOptionalString(candidate.model),
|
|
304
326
|
modelVersion: asOptionalString(candidate.modelVersion),
|
|
327
|
+
environment:
|
|
328
|
+
asOptionalString(candidate.environment) ??
|
|
329
|
+
asOptionalString(body.environment),
|
|
330
|
+
serviceVersion:
|
|
331
|
+
asOptionalString(candidate.serviceVersion) ??
|
|
332
|
+
asOptionalString(body.serviceVersion),
|
|
333
|
+
commitSha:
|
|
334
|
+
asOptionalString(candidate.commitSha) ??
|
|
335
|
+
asOptionalString(body.commitSha),
|
|
336
|
+
configDigest:
|
|
337
|
+
asOptionalString(candidate.configDigest) ??
|
|
338
|
+
asOptionalString(body.configDigest),
|
|
305
339
|
},
|
|
306
340
|
startedAt: asOptionalString(body.startedAt),
|
|
307
341
|
endedAt: asOptionalString(body.endedAt),
|
|
@@ -369,6 +403,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
369
403
|
passed?: unknown;
|
|
370
404
|
score?: unknown;
|
|
371
405
|
actualOutcome?: unknown;
|
|
406
|
+
inputPayloadHash?: unknown;
|
|
407
|
+
failureFingerprint?: unknown;
|
|
372
408
|
details?: unknown;
|
|
373
409
|
};
|
|
374
410
|
const parsed = await readJsonObject<typeof body>(c);
|
|
@@ -411,6 +447,16 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
411
447
|
: null;
|
|
412
448
|
const actualOutcome =
|
|
413
449
|
typeof body.actualOutcome === "string" ? body.actualOutcome : null;
|
|
450
|
+
const inputPayloadHash =
|
|
451
|
+
typeof body.inputPayloadHash === "string" &&
|
|
452
|
+
body.inputPayloadHash.trim()
|
|
453
|
+
? body.inputPayloadHash.trim()
|
|
454
|
+
: null;
|
|
455
|
+
const failureFingerprint =
|
|
456
|
+
typeof body.failureFingerprint === "string" &&
|
|
457
|
+
body.failureFingerprint.trim()
|
|
458
|
+
? body.failureFingerprint.trim()
|
|
459
|
+
: null;
|
|
414
460
|
const details_json =
|
|
415
461
|
body.details === undefined ? null : JSON.stringify(body.details);
|
|
416
462
|
const now = new Date().toISOString();
|
|
@@ -418,8 +464,9 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
418
464
|
await db
|
|
419
465
|
.prepare(
|
|
420
466
|
`INSERT INTO eval_case_results (
|
|
421
|
-
id, project_id, eval_case_id, run_id, passed, score, actual_outcome,
|
|
422
|
-
|
|
467
|
+
id, project_id, eval_case_id, run_id, passed, score, actual_outcome,
|
|
468
|
+
input_payload_hash, failure_fingerprint, details_json, created_at
|
|
469
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
423
470
|
)
|
|
424
471
|
.bind(
|
|
425
472
|
id,
|
|
@@ -429,6 +476,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
429
476
|
passed,
|
|
430
477
|
score,
|
|
431
478
|
actualOutcome,
|
|
479
|
+
inputPayloadHash,
|
|
480
|
+
failureFingerprint,
|
|
432
481
|
details_json,
|
|
433
482
|
now,
|
|
434
483
|
)
|
|
@@ -446,6 +495,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
446
495
|
passed: body.passed,
|
|
447
496
|
score,
|
|
448
497
|
actualOutcome,
|
|
498
|
+
inputPayloadHash,
|
|
499
|
+
failureFingerprint,
|
|
449
500
|
details: body.details ?? null,
|
|
450
501
|
createdAt: now,
|
|
451
502
|
evidenceReferences: evalCaseResultEvidenceReferences(evalCase, id),
|
|
@@ -482,6 +533,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
482
533
|
passed: number;
|
|
483
534
|
score: number | null;
|
|
484
535
|
actual_outcome: string | null;
|
|
536
|
+
input_payload_hash?: string | null;
|
|
537
|
+
failure_fingerprint?: string | null;
|
|
485
538
|
details_json: string | null;
|
|
486
539
|
created_at: string;
|
|
487
540
|
}>();
|
|
@@ -504,6 +557,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
504
557
|
passed: row.passed === 1,
|
|
505
558
|
score: row.score,
|
|
506
559
|
actualOutcome: row.actual_outcome,
|
|
560
|
+
inputPayloadHash: row.input_payload_hash ?? null,
|
|
561
|
+
failureFingerprint: row.failure_fingerprint ?? null,
|
|
507
562
|
details: parseJsonMaybe(row.details_json),
|
|
508
563
|
createdAt: row.created_at,
|
|
509
564
|
evidenceReferences: evalCaseResultEvidenceReferences(
|