@obsunified/collector 2.0.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +1 -1
- package/dist/{chunk-UGWDOGGH.js → chunk-KMEZKGSO.js} +22 -10
- package/dist/chunk-KMEZKGSO.js.map +1 -0
- package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
- package/dist/chunk-ZSF5XP3B.js.map +1 -0
- package/dist/index.d.ts +630 -27
- package/dist/index.js +5819 -853
- package/dist/index.js.map +1 -1
- package/dist/{ingest-auth-HFVO1zIp.d.ts → ingest-auth-C5-8Mly5.d.ts} +9 -0
- package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
- package/package.json +5 -5
- package/src/__tests__/autosre-closed-loop.test.ts +1275 -0
- package/src/analyses/derive.ts +26 -25
- package/src/analyses/investigations.ts +8 -8
- package/src/analyses/tier0.ts +22 -22
- package/src/auth/dashboard-auth.test.ts +791 -0
- package/src/auth/dashboard-auth.ts +187 -38
- package/src/auth/login-rate-limit.test.ts +129 -0
- package/src/auth/login-rate-limit.ts +143 -0
- package/src/framework/body-limit.test.ts +165 -0
- package/src/framework/collector.ts +355 -115
- package/src/framework/env.ts +9 -0
- package/src/framework/limits.ts +31 -0
- package/src/framework/scheduled-handlers.test.ts +342 -0
- package/src/index.ts +59 -1
- package/src/lib/action-retention.test.ts +472 -0
- package/src/lib/action-retention.ts +420 -0
- package/src/lib/actions-store.ts +6 -12
- package/src/lib/ai-store/calls.ts +2 -2
- package/src/lib/ai-store/sessions.ts +3 -3
- package/src/lib/ai-store/spans.ts +1 -1
- package/src/lib/ai-store/types.ts +2 -1
- package/src/lib/alerts-store.test.ts +153 -0
- package/src/lib/alerts-store.ts +122 -21
- package/src/lib/analyses-runner.test.ts +75 -0
- package/src/lib/analyses-runner.ts +69 -3
- package/src/lib/analyses-store-latest.test.ts +61 -0
- package/src/lib/analyses-store.ts +17 -10
- package/src/lib/autosre-change-correlation.ts +780 -0
- package/src/lib/autosre-findings.ts +725 -0
- package/src/lib/autosre-ontology.ts +914 -0
- package/src/lib/autosre-remediation-gate.ts +1663 -0
- package/src/lib/blob-store.ts +1 -1
- package/src/lib/eval-cases-store.ts +63 -1
- package/src/lib/evidence-references.ts +6 -0
- package/src/lib/housekeeping.test.ts +153 -0
- package/src/lib/housekeeping.ts +193 -0
- package/src/lib/identity-index/action-lookups.ts +6 -6
- package/src/lib/identity-index/key-lookups.ts +2 -2
- package/src/lib/identity-index/user-lookup.ts +8 -5
- package/src/lib/identity-index.test.ts +2 -3
- package/src/lib/metrics-store-series.test.ts +99 -0
- package/src/lib/metrics-store.test.ts +23 -0
- package/src/lib/metrics-store.ts +27 -3
- package/src/lib/projects-store.test.ts +25 -0
- package/src/lib/projects-store.ts +4 -3
- package/src/lib/sql-db-dialect.test.ts +12 -3
- package/src/lib/sql-db-postgres.integration.test.ts +1225 -0
- package/src/lib/sql-db-postgres.test.ts +314 -0
- package/src/lib/sql-db-postgres.ts +392 -54
- package/src/lib/sql-db-sqlite-time-windows.test.ts +194 -0
- package/src/lib/sql-db.ts +40 -6
- package/src/lib/store/ingest.ts +94 -6
- package/src/lib/test-utils/action-graph-fixtures.ts +307 -0
- package/src/lib/test-utils/node-sqlite-db.ts +188 -0
- package/src/lib/webhook-url.test.ts +122 -0
- package/src/lib/webhook-url.ts +155 -0
- package/src/migrations/041_ingest_key_hash_text.sql +5 -0
- package/src/migrations/042_session_replay_project_composite_pk.sql +52 -0
- package/src/migrations/043_metric_exemplars_text_timestamps.sql +5 -0
- package/src/migrations/044_actions_started_at_index.sql +10 -0
- package/src/migrations/045_user_profiles_project_composite_pk.sql +40 -0
- package/src/migrations/046_action_graph_project_composite_pk.sql +255 -0
- package/src/migrations/047_ai_span_payloads_project_composite_pk.sql +61 -0
- package/src/migrations/048_telemetry_spans_project_composite_pk.sql +92 -0
- package/src/migrations/049_alert_state_notified.sql +17 -0
- package/src/migrations/050_autosre_causal_ontology.sql +216 -0
- package/src/migrations/alert-state-notified.test.ts +111 -0
- package/src/migrations/parity.test.ts +68 -0
- package/src/migrations/project-scoped-keys.test.ts +432 -0
- package/src/migrations-postgres/009_projects_and_keys.sql +4 -4
- package/src/migrations-postgres/026_obs_dashboard_project_seed.sql +1 -1
- package/src/migrations-postgres/035_metric_exemplars.sql +5 -2
- package/src/migrations-postgres/041_ingest_key_hash_text.sql +32 -0
- package/src/migrations-postgres/042_session_replay_project_composite_pk.sql +16 -0
- package/src/migrations-postgres/043_metric_exemplars_text_timestamps.sql +47 -0
- package/src/migrations-postgres/044_actions_started_at_index.sql +8 -0
- package/src/migrations-postgres/045_user_profiles_project_composite_pk.sql +33 -0
- package/src/migrations-postgres/046_action_graph_project_composite_pk.sql +39 -0
- package/src/migrations-postgres/047_ai_span_payloads_project_composite_pk.sql +30 -0
- package/src/migrations-postgres/048_telemetry_spans_project_composite_pk.sql +37 -0
- package/src/migrations-postgres/049_alert_state_notified.sql +10 -0
- package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
- package/src/migrations-postgres/README.md +58 -5
- package/src/otlp/decode/body.test.ts +76 -0
- package/src/otlp/decode/body.ts +67 -6
- package/src/plugins/_json.ts +21 -0
- package/src/plugins/action-graph-processor.ts +6 -12
- package/src/plugins/ai-receiver.ts +5 -12
- package/src/plugins/ai-span-payloads-processor.ts +1 -2
- package/src/plugins/alerts-evaluator-retention.test.ts +170 -0
- package/src/plugins/alerts-evaluator.test.ts +464 -3
- package/src/plugins/alerts-evaluator.ts +181 -41
- package/src/plugins/alerts-routes.ts +10 -6
- package/src/plugins/ask-routes.ts +4 -5
- package/src/plugins/autosre-routes.ts +1019 -0
- package/src/plugins/connected-routes/manifest.ts +13 -1
- package/src/plugins/connected-routes.test.ts +106 -0
- package/src/plugins/connected-routes.ts +285 -5
- package/src/plugins/dashboard-routes.test.ts +120 -0
- package/src/plugins/dashboard-routes.ts +11 -2
- package/src/plugins/default-span-enrichment.ts +13 -1
- package/src/plugins/eval-cases-routes.ts +65 -19
- package/src/plugins/evidence-retrieval-routes.ts +300 -4
- package/src/plugins/identity-receiver.ts +9 -11
- package/src/plugins/json-body.test.ts +91 -0
- package/src/plugins/onboarding-routes.test.ts +105 -0
- package/src/plugins/onboarding-routes.ts +26 -11
- package/src/plugins/projects-routes.ts +2 -1
- package/src/plugins/replay-isolation.test.ts +922 -0
- package/src/plugins/replay-receiver.ts +8 -8
- package/src/plugins/usage-receiver.ts +3 -6
- package/dist/chunk-UGWDOGGH.js.map +0 -1
- package/dist/chunk-XKQTVIRE.js.map +0 -1
- /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
|
@@ -19,6 +19,7 @@ import { sourceLinkEvidenceReferences } from "../lib/evidence-references";
|
|
|
19
19
|
import { randomHex } from "../lib/hash";
|
|
20
20
|
import { sqlDbFor } from "../lib/sql-db";
|
|
21
21
|
import { getProjectId } from "./_context";
|
|
22
|
+
import { readJsonObject } from "./_json";
|
|
22
23
|
|
|
23
24
|
interface EvalCaseRequestBody {
|
|
24
25
|
sourceEntityType?: unknown;
|
|
@@ -49,6 +50,8 @@ interface EvalCaseResultResponse {
|
|
|
49
50
|
passed: boolean;
|
|
50
51
|
score: number | null;
|
|
51
52
|
actualOutcome: string | null;
|
|
53
|
+
inputPayloadHash: string | null;
|
|
54
|
+
failureFingerprint: string | null;
|
|
52
55
|
details: unknown;
|
|
53
56
|
createdAt: string;
|
|
54
57
|
evidenceReferences?: EvidenceReference[];
|
|
@@ -61,6 +64,12 @@ interface EvalRunRequestBody {
|
|
|
61
64
|
evalCaseId?: unknown;
|
|
62
65
|
status?: unknown;
|
|
63
66
|
candidate?: unknown;
|
|
67
|
+
environment?: unknown;
|
|
68
|
+
serviceVersion?: unknown;
|
|
69
|
+
commitSha?: unknown;
|
|
70
|
+
configDigest?: unknown;
|
|
71
|
+
inputPayloadHash?: unknown;
|
|
72
|
+
failureFingerprint?: unknown;
|
|
64
73
|
startedAt?: unknown;
|
|
65
74
|
endedAt?: unknown;
|
|
66
75
|
totalCount?: unknown;
|
|
@@ -123,12 +132,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
123
132
|
name: "eval-cases-routes",
|
|
124
133
|
register(app) {
|
|
125
134
|
app.post("/internal/eval-cases", async (c) => {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
body = (await c.req.json()) as EvalCaseRequestBody;
|
|
129
|
-
} catch {
|
|
130
|
-
return c.json(badRequest("body must be JSON"), 400);
|
|
131
|
-
}
|
|
135
|
+
const body = await readJsonObject<EvalCaseRequestBody>(c);
|
|
136
|
+
if (!body) return c.json(badRequest("body must be a JSON object"), 400);
|
|
132
137
|
|
|
133
138
|
const sourceType = body.sourceEntityType ?? body.source?.type;
|
|
134
139
|
if (!isEvalCaseSourceType(sourceType)) {
|
|
@@ -236,12 +241,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
236
241
|
});
|
|
237
242
|
|
|
238
243
|
app.post("/internal/eval-runs", async (c) => {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
body = (await c.req.json()) as EvalRunRequestBody;
|
|
242
|
-
} catch {
|
|
243
|
-
return c.json(badRequest("body must be JSON"), 400);
|
|
244
|
-
}
|
|
244
|
+
const body = await readJsonObject<EvalRunRequestBody>(c);
|
|
245
|
+
if (!body) return c.json(badRequest("body must be a JSON object"), 400);
|
|
245
246
|
|
|
246
247
|
const id = asOptionalString(body.id ?? body.runId)?.trim();
|
|
247
248
|
if ((body.id ?? body.runId) !== undefined && !id) {
|
|
@@ -301,6 +302,20 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
301
302
|
id: id ?? undefined,
|
|
302
303
|
evalCaseId,
|
|
303
304
|
status,
|
|
305
|
+
environment:
|
|
306
|
+
asOptionalString(body.environment) ??
|
|
307
|
+
asOptionalString(candidate.environment),
|
|
308
|
+
serviceVersion:
|
|
309
|
+
asOptionalString(body.serviceVersion) ??
|
|
310
|
+
asOptionalString(candidate.serviceVersion),
|
|
311
|
+
commitSha:
|
|
312
|
+
asOptionalString(body.commitSha) ??
|
|
313
|
+
asOptionalString(candidate.commitSha),
|
|
314
|
+
configDigest:
|
|
315
|
+
asOptionalString(body.configDigest) ??
|
|
316
|
+
asOptionalString(candidate.configDigest),
|
|
317
|
+
inputPayloadHash: asOptionalString(body.inputPayloadHash),
|
|
318
|
+
failureFingerprint: asOptionalString(body.failureFingerprint),
|
|
304
319
|
candidate: {
|
|
305
320
|
agentId: asOptionalString(candidate.agentId),
|
|
306
321
|
agentVersion: asOptionalString(candidate.agentVersion),
|
|
@@ -309,6 +324,18 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
309
324
|
modelProvider: asOptionalString(candidate.modelProvider),
|
|
310
325
|
model: asOptionalString(candidate.model),
|
|
311
326
|
modelVersion: asOptionalString(candidate.modelVersion),
|
|
327
|
+
environment:
|
|
328
|
+
asOptionalString(candidate.environment) ??
|
|
329
|
+
asOptionalString(body.environment),
|
|
330
|
+
serviceVersion:
|
|
331
|
+
asOptionalString(candidate.serviceVersion) ??
|
|
332
|
+
asOptionalString(body.serviceVersion),
|
|
333
|
+
commitSha:
|
|
334
|
+
asOptionalString(candidate.commitSha) ??
|
|
335
|
+
asOptionalString(body.commitSha),
|
|
336
|
+
configDigest:
|
|
337
|
+
asOptionalString(candidate.configDigest) ??
|
|
338
|
+
asOptionalString(body.configDigest),
|
|
312
339
|
},
|
|
313
340
|
startedAt: asOptionalString(body.startedAt),
|
|
314
341
|
endedAt: asOptionalString(body.endedAt),
|
|
@@ -376,13 +403,13 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
376
403
|
passed?: unknown;
|
|
377
404
|
score?: unknown;
|
|
378
405
|
actualOutcome?: unknown;
|
|
406
|
+
inputPayloadHash?: unknown;
|
|
407
|
+
failureFingerprint?: unknown;
|
|
379
408
|
details?: unknown;
|
|
380
409
|
};
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
return c.json(badRequest("body must be JSON"), 400);
|
|
385
|
-
}
|
|
410
|
+
const parsed = await readJsonObject<typeof body>(c);
|
|
411
|
+
if (!parsed) return c.json(badRequest("body must be a JSON object"), 400);
|
|
412
|
+
body = parsed;
|
|
386
413
|
|
|
387
414
|
if (body.passed === undefined || typeof body.passed !== "boolean") {
|
|
388
415
|
return c.json(badRequest("passed must be a boolean"), 400);
|
|
@@ -420,6 +447,16 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
420
447
|
: null;
|
|
421
448
|
const actualOutcome =
|
|
422
449
|
typeof body.actualOutcome === "string" ? body.actualOutcome : null;
|
|
450
|
+
const inputPayloadHash =
|
|
451
|
+
typeof body.inputPayloadHash === "string" &&
|
|
452
|
+
body.inputPayloadHash.trim()
|
|
453
|
+
? body.inputPayloadHash.trim()
|
|
454
|
+
: null;
|
|
455
|
+
const failureFingerprint =
|
|
456
|
+
typeof body.failureFingerprint === "string" &&
|
|
457
|
+
body.failureFingerprint.trim()
|
|
458
|
+
? body.failureFingerprint.trim()
|
|
459
|
+
: null;
|
|
423
460
|
const details_json =
|
|
424
461
|
body.details === undefined ? null : JSON.stringify(body.details);
|
|
425
462
|
const now = new Date().toISOString();
|
|
@@ -427,8 +464,9 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
427
464
|
await db
|
|
428
465
|
.prepare(
|
|
429
466
|
`INSERT INTO eval_case_results (
|
|
430
|
-
id, project_id, eval_case_id, run_id, passed, score, actual_outcome,
|
|
431
|
-
|
|
467
|
+
id, project_id, eval_case_id, run_id, passed, score, actual_outcome,
|
|
468
|
+
input_payload_hash, failure_fingerprint, details_json, created_at
|
|
469
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
432
470
|
)
|
|
433
471
|
.bind(
|
|
434
472
|
id,
|
|
@@ -438,6 +476,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
438
476
|
passed,
|
|
439
477
|
score,
|
|
440
478
|
actualOutcome,
|
|
479
|
+
inputPayloadHash,
|
|
480
|
+
failureFingerprint,
|
|
441
481
|
details_json,
|
|
442
482
|
now,
|
|
443
483
|
)
|
|
@@ -455,6 +495,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
455
495
|
passed: body.passed,
|
|
456
496
|
score,
|
|
457
497
|
actualOutcome,
|
|
498
|
+
inputPayloadHash,
|
|
499
|
+
failureFingerprint,
|
|
458
500
|
details: body.details ?? null,
|
|
459
501
|
createdAt: now,
|
|
460
502
|
evidenceReferences: evalCaseResultEvidenceReferences(evalCase, id),
|
|
@@ -491,6 +533,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
491
533
|
passed: number;
|
|
492
534
|
score: number | null;
|
|
493
535
|
actual_outcome: string | null;
|
|
536
|
+
input_payload_hash?: string | null;
|
|
537
|
+
failure_fingerprint?: string | null;
|
|
494
538
|
details_json: string | null;
|
|
495
539
|
created_at: string;
|
|
496
540
|
}>();
|
|
@@ -513,6 +557,8 @@ export const evalCasesRoutesPlugin: CollectorPlugin = {
|
|
|
513
557
|
passed: row.passed === 1,
|
|
514
558
|
score: row.score,
|
|
515
559
|
actualOutcome: row.actual_outcome,
|
|
560
|
+
inputPayloadHash: row.input_payload_hash ?? null,
|
|
561
|
+
failureFingerprint: row.failure_fingerprint ?? null,
|
|
516
562
|
details: parseJsonMaybe(row.details_json),
|
|
517
563
|
createdAt: row.created_at,
|
|
518
564
|
evidenceReferences: evalCaseResultEvidenceReferences(
|
|
@@ -12,6 +12,13 @@ import type {
|
|
|
12
12
|
TelemetryTraceDetailResponse,
|
|
13
13
|
} from "@obsunified/types";
|
|
14
14
|
import type { CollectorPlugin } from "../framework/collector";
|
|
15
|
+
import {
|
|
16
|
+
computeChangeBlastRadius,
|
|
17
|
+
listChangeEvents,
|
|
18
|
+
} from "../lib/autosre-change-correlation";
|
|
19
|
+
import { searchInvestigationFindings } from "../lib/autosre-findings";
|
|
20
|
+
import { loadTraceFaultLocusFromDb } from "../lib/autosre-ontology";
|
|
21
|
+
import { getIncidentInvestigation } from "../lib/autosre-remediation-gate";
|
|
15
22
|
import type {
|
|
16
23
|
ActionRef,
|
|
17
24
|
EntityManifestExtended,
|
|
@@ -75,7 +82,7 @@ interface ReplayEvidenceRow {
|
|
|
75
82
|
|
|
76
83
|
interface AICallEvidenceRow {
|
|
77
84
|
call_id: string;
|
|
78
|
-
trace_id: string
|
|
85
|
+
trace_id: string;
|
|
79
86
|
span_id?: string | null;
|
|
80
87
|
service_name?: string | null;
|
|
81
88
|
model_name: string;
|
|
@@ -105,6 +112,7 @@ const DEFAULT_PROFILE_FRAME_LIMIT = 50;
|
|
|
105
112
|
const MAX_PROFILE_FRAME_LIMIT = 200;
|
|
106
113
|
const EVIDENCE_INTENTS = new Set<EvidenceBundleIntent>([
|
|
107
114
|
"debug_failure",
|
|
115
|
+
"triage_incident",
|
|
108
116
|
"explain_latency",
|
|
109
117
|
"explain_cost",
|
|
110
118
|
"inspect_agent_run",
|
|
@@ -1460,6 +1468,187 @@ const buildActionGraphBundle = (
|
|
|
1460
1468
|
return bundle;
|
|
1461
1469
|
};
|
|
1462
1470
|
|
|
1471
|
+
const attachAutoSreTriageContext = async (
|
|
1472
|
+
db: SqlDb,
|
|
1473
|
+
projectId: string,
|
|
1474
|
+
traceId: string | null,
|
|
1475
|
+
bundle: EvidenceBundle,
|
|
1476
|
+
): Promise<EvidenceBundle> => {
|
|
1477
|
+
try {
|
|
1478
|
+
const faultLocus = traceId
|
|
1479
|
+
? await loadTraceFaultLocusFromDb(db, projectId, traceId)
|
|
1480
|
+
: null;
|
|
1481
|
+
const environment = (
|
|
1482
|
+
faultLocus?.faultLocusSpan?.environment ??
|
|
1483
|
+
bundle.environment ??
|
|
1484
|
+
"default"
|
|
1485
|
+
).toLowerCase();
|
|
1486
|
+
const recentChanges = await listChangeEvents(db, {
|
|
1487
|
+
projectId,
|
|
1488
|
+
environment,
|
|
1489
|
+
limit: 5,
|
|
1490
|
+
});
|
|
1491
|
+
const blastRadii = (
|
|
1492
|
+
await Promise.all(
|
|
1493
|
+
recentChanges
|
|
1494
|
+
.slice(0, 3)
|
|
1495
|
+
.map((ch) =>
|
|
1496
|
+
computeChangeBlastRadius(db, projectId, ch.id, { environment }),
|
|
1497
|
+
),
|
|
1498
|
+
)
|
|
1499
|
+
).filter((br): br is NonNullable<typeof br> => Boolean(br));
|
|
1500
|
+
|
|
1501
|
+
const candidateChanges = blastRadii.map((br) => ({
|
|
1502
|
+
changeId: br.change.id,
|
|
1503
|
+
kind: br.change.kind,
|
|
1504
|
+
version: br.change.version,
|
|
1505
|
+
previousVersion: br.change.previousVersion,
|
|
1506
|
+
occurredAt: br.change.occurredAt,
|
|
1507
|
+
cohortVerdict: br.cohortVerdict,
|
|
1508
|
+
causalProofTier: br.causalProofTier,
|
|
1509
|
+
rateNew: br.newCohort.errorRate,
|
|
1510
|
+
rateOld: br.oldCohort.errorRate,
|
|
1511
|
+
sampleNew: br.newCohort.requestCount,
|
|
1512
|
+
sampleOld: br.oldCohort.requestCount,
|
|
1513
|
+
overlappingCodePaths: br.overlappingCodePaths,
|
|
1514
|
+
}));
|
|
1515
|
+
const contentionCandidates = blastRadii
|
|
1516
|
+
.flatMap((br) => br.sharedResourceAggressors)
|
|
1517
|
+
.map((ag) => ({
|
|
1518
|
+
aggressorTraceId: ag.aggressorTraceId,
|
|
1519
|
+
aggressorSpanId: ag.aggressorSpanId,
|
|
1520
|
+
aggressorServiceName: ag.aggressorServiceName,
|
|
1521
|
+
sharedTargetNodeId: ag.sharedTargetNodeId,
|
|
1522
|
+
relationshipCertainty: ag.relationshipCertainty,
|
|
1523
|
+
totalResourceTimeMs: ag.totalResourceTimeMs,
|
|
1524
|
+
}));
|
|
1525
|
+
let priorFindingsList = await searchInvestigationFindings(db, {
|
|
1526
|
+
projectId,
|
|
1527
|
+
environment,
|
|
1528
|
+
rootCauseEntityId:
|
|
1529
|
+
faultLocus?.faultLocusNodeId ??
|
|
1530
|
+
faultLocus?.faultLocusSpan?.span_id ??
|
|
1531
|
+
undefined,
|
|
1532
|
+
limit: 5,
|
|
1533
|
+
});
|
|
1534
|
+
if (priorFindingsList.length === 0 && faultLocus?.faultLocusSpan?.span_id) {
|
|
1535
|
+
priorFindingsList = await searchInvestigationFindings(db, {
|
|
1536
|
+
projectId,
|
|
1537
|
+
environment,
|
|
1538
|
+
rootCauseEntityId: faultLocus.faultLocusSpan.span_id,
|
|
1539
|
+
limit: 5,
|
|
1540
|
+
});
|
|
1541
|
+
}
|
|
1542
|
+
if (priorFindingsList.length === 0) {
|
|
1543
|
+
priorFindingsList = await searchInvestigationFindings(db, {
|
|
1544
|
+
projectId,
|
|
1545
|
+
environment,
|
|
1546
|
+
limit: 5,
|
|
1547
|
+
});
|
|
1548
|
+
}
|
|
1549
|
+
const priorFindings = priorFindingsList.map((f) => ({
|
|
1550
|
+
findingId: f.id,
|
|
1551
|
+
title: f.title,
|
|
1552
|
+
causalProofTier: f.causalProofTier,
|
|
1553
|
+
causalVerificationStatus: f.causalVerificationStatus,
|
|
1554
|
+
recoveryStatus: f.recoveryStatus,
|
|
1555
|
+
contributingChangeId: f.contributingChangeId,
|
|
1556
|
+
occurredAt: f.createdAt,
|
|
1557
|
+
}));
|
|
1558
|
+
|
|
1559
|
+
for (const br of blastRadii) {
|
|
1560
|
+
const changeId = br.change.id;
|
|
1561
|
+
const changeRoute = `#/changes/${encodeURIComponent(changeId)}`;
|
|
1562
|
+
bundle.evidenceReferences.push({
|
|
1563
|
+
evidenceId: `change:${changeId}`,
|
|
1564
|
+
entityKind: "change",
|
|
1565
|
+
entityId: changeId,
|
|
1566
|
+
route: changeRoute,
|
|
1567
|
+
source: br.overlappingCodePaths.length
|
|
1568
|
+
? "stack_frame_overlap"
|
|
1569
|
+
: "cohort_comparison",
|
|
1570
|
+
confidence:
|
|
1571
|
+
br.cohortVerdict === "isolated_to_new_version" ? 0.9 : 0.65,
|
|
1572
|
+
relationshipCertainty: br.relationshipCertainty,
|
|
1573
|
+
causalProofTier: br.causalProofTier,
|
|
1574
|
+
reason: br.summary,
|
|
1575
|
+
citations: [
|
|
1576
|
+
{
|
|
1577
|
+
label: `change ${changeId} (${br.change.version ?? br.change.kind})`,
|
|
1578
|
+
entityKind: "change",
|
|
1579
|
+
entityId: changeId,
|
|
1580
|
+
route: changeRoute,
|
|
1581
|
+
},
|
|
1582
|
+
],
|
|
1583
|
+
suggestedNextPivots: [
|
|
1584
|
+
{
|
|
1585
|
+
label: "Inspect change blast radius",
|
|
1586
|
+
entityKind: "change",
|
|
1587
|
+
entityId: changeId,
|
|
1588
|
+
route: changeRoute,
|
|
1589
|
+
reason: "Inspect traffic-normalized cohort lift and contention.",
|
|
1590
|
+
},
|
|
1591
|
+
],
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1594
|
+
|
|
1595
|
+
for (const finding of priorFindingsList) {
|
|
1596
|
+
const findingRoute = `#/findings/${encodeURIComponent(finding.id)}`;
|
|
1597
|
+
bundle.evidenceReferences.push({
|
|
1598
|
+
evidenceId: `finding:${finding.id}`,
|
|
1599
|
+
entityKind: "finding",
|
|
1600
|
+
entityId: finding.id,
|
|
1601
|
+
route: findingRoute,
|
|
1602
|
+
source: "incident_memory_fingerprint",
|
|
1603
|
+
confidence: finding.confidence,
|
|
1604
|
+
relationshipCertainty: "exact",
|
|
1605
|
+
causalProofTier: finding.causalProofTier,
|
|
1606
|
+
reason: `Prior finding (${finding.causalProofTier}): ${finding.title}`,
|
|
1607
|
+
citations: [
|
|
1608
|
+
{
|
|
1609
|
+
label: finding.title,
|
|
1610
|
+
entityKind: "finding",
|
|
1611
|
+
entityId: finding.id,
|
|
1612
|
+
route: findingRoute,
|
|
1613
|
+
},
|
|
1614
|
+
],
|
|
1615
|
+
suggestedNextPivots: [
|
|
1616
|
+
{
|
|
1617
|
+
label: "Open prior finding",
|
|
1618
|
+
entityKind: "finding",
|
|
1619
|
+
entityId: finding.id,
|
|
1620
|
+
route: findingRoute,
|
|
1621
|
+
reason: "Search incident memory for similar recurring findings.",
|
|
1622
|
+
},
|
|
1623
|
+
],
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
bundle.autosre = {
|
|
1628
|
+
faultLocus: faultLocus?.faultLocusNodeId
|
|
1629
|
+
? {
|
|
1630
|
+
nodeId: faultLocus.faultLocusNodeId,
|
|
1631
|
+
spanId: faultLocus.faultLocusSpan?.span_id,
|
|
1632
|
+
relationshipCertainty: faultLocus.relationshipCertainty,
|
|
1633
|
+
isBlackoutSuspected: false,
|
|
1634
|
+
}
|
|
1635
|
+
: undefined,
|
|
1636
|
+
candidateChanges,
|
|
1637
|
+
contentionCandidates,
|
|
1638
|
+
priorFindings,
|
|
1639
|
+
};
|
|
1640
|
+
if (bundle.budget) {
|
|
1641
|
+
bundle.budget = {
|
|
1642
|
+
...bundle.budget,
|
|
1643
|
+
estimatedTokens: estimateTokens(bundle),
|
|
1644
|
+
};
|
|
1645
|
+
}
|
|
1646
|
+
} catch {
|
|
1647
|
+
// Gracefully degrade if AutoSRE tables are absent in minimal test setups
|
|
1648
|
+
}
|
|
1649
|
+
return bundle;
|
|
1650
|
+
};
|
|
1651
|
+
|
|
1463
1652
|
const parseBundleRequest = async (request: Request) => {
|
|
1464
1653
|
try {
|
|
1465
1654
|
const body = await request.json();
|
|
@@ -1550,7 +1739,7 @@ export const evidenceRetrievalRoutesPlugin: CollectorPlugin = {
|
|
|
1550
1739
|
all.findIndex((candidate) => candidate.id === profile.id) ===
|
|
1551
1740
|
index,
|
|
1552
1741
|
);
|
|
1553
|
-
|
|
1742
|
+
let bundle = buildActionGraphBundle(
|
|
1554
1743
|
projectId,
|
|
1555
1744
|
anchor.entityKind,
|
|
1556
1745
|
anchor.entityId,
|
|
@@ -1560,15 +1749,114 @@ export const evidenceRetrievalRoutesPlugin: CollectorPlugin = {
|
|
|
1560
1749
|
targetTokens,
|
|
1561
1750
|
detailLevel,
|
|
1562
1751
|
);
|
|
1752
|
+
if (intent === "triage_incident") {
|
|
1753
|
+
bundle = await attachAutoSreTriageContext(
|
|
1754
|
+
db,
|
|
1755
|
+
projectId,
|
|
1756
|
+
traceIds[0] ?? null,
|
|
1757
|
+
bundle,
|
|
1758
|
+
);
|
|
1759
|
+
}
|
|
1563
1760
|
return c.json(await materializeBundle(db, projectId, bundle));
|
|
1564
1761
|
}
|
|
1565
1762
|
|
|
1763
|
+
if (anchor.entityKind === "incident") {
|
|
1764
|
+
const incident = await getIncidentInvestigation(
|
|
1765
|
+
db,
|
|
1766
|
+
projectId,
|
|
1767
|
+
anchor.entityId,
|
|
1768
|
+
);
|
|
1769
|
+
if (!incident) {
|
|
1770
|
+
return c.json(
|
|
1771
|
+
{ error: "Not Found", message: "Incident not found" },
|
|
1772
|
+
404,
|
|
1773
|
+
);
|
|
1774
|
+
}
|
|
1775
|
+
const recentErrSpan = await db
|
|
1776
|
+
.prepare(
|
|
1777
|
+
`SELECT trace_id FROM telemetry_spans
|
|
1778
|
+
WHERE project_id = ? AND environment = ? AND status_code = 2
|
|
1779
|
+
ORDER BY start_time DESC LIMIT 1`,
|
|
1780
|
+
)
|
|
1781
|
+
.bind(projectId, incident.environment)
|
|
1782
|
+
.first<{ trace_id: string }>();
|
|
1783
|
+
|
|
1784
|
+
const incRoute = `#/incidents/${encodeURIComponent(incident.id)}`;
|
|
1785
|
+
const baseBundle: EvidenceBundle = {
|
|
1786
|
+
schemaVersion: BUNDLE_SCHEMA_VERSION,
|
|
1787
|
+
intent,
|
|
1788
|
+
environment: incident.environment,
|
|
1789
|
+
incidentId: incident.id,
|
|
1790
|
+
symptomFingerprint: incident.symptomFingerprint ?? undefined,
|
|
1791
|
+
anchor: { entityKind: "incident", entityId: incident.id },
|
|
1792
|
+
budget: { targetTokens, detailLevel },
|
|
1793
|
+
summary: `Incident ${incident.id} (${incident.status}) on ${incident.targetNodeId} in environment ${incident.environment}.`,
|
|
1794
|
+
derivedSummaries: [
|
|
1795
|
+
{
|
|
1796
|
+
title: `Incident ${incident.id}`,
|
|
1797
|
+
reason: `Trigger ${incident.symptomAnchorKind}:${incident.symptomAnchorId} on ${incident.targetNodeId} (status=${incident.status}).`,
|
|
1798
|
+
confidence: 1.0,
|
|
1799
|
+
evidenceIds: [`incident:${incident.id}`],
|
|
1800
|
+
retrievalRefIds: [],
|
|
1801
|
+
},
|
|
1802
|
+
],
|
|
1803
|
+
findings: [],
|
|
1804
|
+
compactions: [],
|
|
1805
|
+
evidenceReferences: [
|
|
1806
|
+
{
|
|
1807
|
+
evidenceId: `incident:${incident.id}`,
|
|
1808
|
+
entityKind: "incident",
|
|
1809
|
+
entityId: incident.id,
|
|
1810
|
+
route: incRoute,
|
|
1811
|
+
source: "incident_investigations",
|
|
1812
|
+
confidence: 1.0,
|
|
1813
|
+
relationshipCertainty: "exact",
|
|
1814
|
+
reason: `Incident triggered by ${incident.symptomAnchorKind}:${incident.symptomAnchorId}.`,
|
|
1815
|
+
citations: [
|
|
1816
|
+
{
|
|
1817
|
+
label: incident.targetNodeId,
|
|
1818
|
+
entityKind: "incident",
|
|
1819
|
+
entityId: incident.id,
|
|
1820
|
+
route: incRoute,
|
|
1821
|
+
},
|
|
1822
|
+
],
|
|
1823
|
+
suggestedNextPivots: [
|
|
1824
|
+
{
|
|
1825
|
+
label: "Open incident",
|
|
1826
|
+
entityKind: "incident",
|
|
1827
|
+
entityId: incident.id,
|
|
1828
|
+
route: incRoute,
|
|
1829
|
+
reason: "Inspect incident investigation and candidate changes.",
|
|
1830
|
+
},
|
|
1831
|
+
],
|
|
1832
|
+
},
|
|
1833
|
+
],
|
|
1834
|
+
retrievalRefs: [],
|
|
1835
|
+
suggestedNextPivots: [
|
|
1836
|
+
{
|
|
1837
|
+
label: "Open incident",
|
|
1838
|
+
entityKind: "incident",
|
|
1839
|
+
entityId: incident.id,
|
|
1840
|
+
route: incRoute,
|
|
1841
|
+
reason: "Inspect incident investigation and candidate changes.",
|
|
1842
|
+
},
|
|
1843
|
+
],
|
|
1844
|
+
};
|
|
1845
|
+
const enriched = await attachAutoSreTriageContext(
|
|
1846
|
+
db,
|
|
1847
|
+
projectId,
|
|
1848
|
+
recentErrSpan?.trace_id ?? null,
|
|
1849
|
+
baseBundle,
|
|
1850
|
+
);
|
|
1851
|
+
return c.json(await materializeBundle(db, projectId, enriched));
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1566
1854
|
if (anchor.entityKind !== "trace") {
|
|
1567
1855
|
return c.json(
|
|
1568
1856
|
{
|
|
1569
1857
|
error: "Bad Request",
|
|
1570
1858
|
message:
|
|
1571
|
-
"Supported evidence bundle anchors are trace, action, agent_run, and
|
|
1859
|
+
"Supported evidence bundle anchors are trace, action, agent_run, tool_call, and incident.",
|
|
1572
1860
|
},
|
|
1573
1861
|
400,
|
|
1574
1862
|
);
|
|
@@ -1588,7 +1876,7 @@ export const evidenceRetrievalRoutesPlugin: CollectorPlugin = {
|
|
|
1588
1876
|
limit: MAX_LOGS_FOR_BUNDLE,
|
|
1589
1877
|
});
|
|
1590
1878
|
const profiles = await profilesForTrace(db, projectId, traceId);
|
|
1591
|
-
|
|
1879
|
+
let bundle = await buildTraceBundle(
|
|
1592
1880
|
projectId,
|
|
1593
1881
|
detail,
|
|
1594
1882
|
logs.logs,
|
|
@@ -1598,6 +1886,14 @@ export const evidenceRetrievalRoutesPlugin: CollectorPlugin = {
|
|
|
1598
1886
|
detailLevel,
|
|
1599
1887
|
hours,
|
|
1600
1888
|
);
|
|
1889
|
+
if (intent === "triage_incident" || intent === "debug_failure") {
|
|
1890
|
+
bundle = await attachAutoSreTriageContext(
|
|
1891
|
+
db,
|
|
1892
|
+
projectId,
|
|
1893
|
+
traceId,
|
|
1894
|
+
bundle,
|
|
1895
|
+
);
|
|
1896
|
+
}
|
|
1601
1897
|
return c.json(await materializeBundle(db, projectId, bundle));
|
|
1602
1898
|
});
|
|
1603
1899
|
|
|
@@ -2,18 +2,15 @@ import type { IdentifyInput } from "@obsunified/types";
|
|
|
2
2
|
import type { CollectorPlugin } from "../framework/collector";
|
|
3
3
|
import { dialectFor, sqlDbFor } from "../lib/sql-db";
|
|
4
4
|
import { getProjectId } from "./_context";
|
|
5
|
+
import { readJsonObject } from "./_json";
|
|
5
6
|
|
|
6
7
|
export const identityReceiverPlugin: CollectorPlugin = {
|
|
7
8
|
name: "identity-receiver",
|
|
8
9
|
register(app) {
|
|
9
10
|
app.post("/v1/identify", async (c) => {
|
|
10
11
|
const projectId = getProjectId(c);
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
payload = await c.req.json<IdentifyInput>();
|
|
14
|
-
} catch {
|
|
15
|
-
return c.json({ error: "Invalid JSON body" }, 400);
|
|
16
|
-
}
|
|
12
|
+
const payload = await readJsonObject<IdentifyInput>(c);
|
|
13
|
+
if (!payload) return c.json({ error: "Invalid JSON body" }, 400);
|
|
17
14
|
if (!payload.userId || typeof payload.userId !== "string") {
|
|
18
15
|
return c.json({ error: "userId is required" }, 400);
|
|
19
16
|
}
|
|
@@ -51,10 +48,11 @@ export const identityReceiverPlugin: CollectorPlugin = {
|
|
|
51
48
|
? JSON.stringify(payload.properties)
|
|
52
49
|
: null;
|
|
53
50
|
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
//
|
|
57
|
-
//
|
|
51
|
+
// Profiles are keyed by (project_id, user_id) — migration 045.
|
|
52
|
+
// user_id values are caller-chosen (often an email), so two
|
|
53
|
+
// projects identifying the same id must get two independent rows;
|
|
54
|
+
// an ingest key for one project can never read or rewrite another
|
|
55
|
+
// project's profile through this endpoint.
|
|
58
56
|
//
|
|
59
57
|
// On conflict we keep the EARLIEST first_seen_at via MIN() — once
|
|
60
58
|
// observed, a user's first-seen is monotonic; an identify() call
|
|
@@ -69,7 +67,7 @@ export const identityReceiverPlugin: CollectorPlugin = {
|
|
|
69
67
|
.prepare(
|
|
70
68
|
`INSERT INTO user_profiles (project_id, user_id, visitor_id, email, name, properties_json, first_seen_at, last_seen_at)
|
|
71
69
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
72
|
-
ON CONFLICT(user_id) DO UPDATE SET
|
|
70
|
+
ON CONFLICT(project_id, user_id) DO UPDATE SET
|
|
73
71
|
visitor_id = excluded.visitor_id,
|
|
74
72
|
email = COALESCE(excluded.email, user_profiles.email),
|
|
75
73
|
name = COALESCE(excluded.name, user_profiles.name),
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `c.req.json()` resolves `null`, arrays and primitives for bodies such as
|
|
3
|
+
* `null` or `"x"`. Every JSON ingest and admin route used to dereference the
|
|
4
|
+
* result straight away and answer 500. They must all answer 400 now, and the
|
|
5
|
+
* replay receiver must reject a chunk before writing it to blob storage.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { MiddlewareHandler } from "hono";
|
|
9
|
+
import { describe, expect, it } from "vitest";
|
|
10
|
+
import { createTelemetryCollectorApp } from "../framework/collector";
|
|
11
|
+
import type { CollectorEnv } from "../framework/env";
|
|
12
|
+
import { MemSqlDb } from "../lib/test-utils/mem-sql-db";
|
|
13
|
+
import { aiReceiverPlugin } from "./ai-receiver";
|
|
14
|
+
import { askRoutesPlugin } from "./ask-routes";
|
|
15
|
+
import { evalCasesRoutesPlugin } from "./eval-cases-routes";
|
|
16
|
+
import { identityReceiverPlugin } from "./identity-receiver";
|
|
17
|
+
import { replayReceiverPlugin } from "./replay-receiver";
|
|
18
|
+
import { usageReceiverPlugin } from "./usage-receiver";
|
|
19
|
+
|
|
20
|
+
const passthrough: MiddlewareHandler<{
|
|
21
|
+
Bindings: CollectorEnv;
|
|
22
|
+
Variables: { projectId: string };
|
|
23
|
+
}> = async (c, next) => {
|
|
24
|
+
c.set("projectId", "default");
|
|
25
|
+
await next();
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const app = createTelemetryCollectorApp({
|
|
29
|
+
plugins: [
|
|
30
|
+
aiReceiverPlugin,
|
|
31
|
+
usageReceiverPlugin,
|
|
32
|
+
replayReceiverPlugin,
|
|
33
|
+
identityReceiverPlugin,
|
|
34
|
+
askRoutesPlugin,
|
|
35
|
+
evalCasesRoutesPlugin,
|
|
36
|
+
],
|
|
37
|
+
auth: { middleware: passthrough },
|
|
38
|
+
dashboardAuth: { middleware: passthrough, registerRoutes: () => {} },
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
const ROUTES = [
|
|
42
|
+
"/v1/ai",
|
|
43
|
+
"/v1/ai/evaluations",
|
|
44
|
+
"/v1/usage",
|
|
45
|
+
"/v1/replays",
|
|
46
|
+
"/v1/identify",
|
|
47
|
+
"/internal/ask",
|
|
48
|
+
"/internal/eval-cases",
|
|
49
|
+
"/internal/eval-runs",
|
|
50
|
+
"/internal/eval-cases/case-1/results",
|
|
51
|
+
];
|
|
52
|
+
|
|
53
|
+
const NON_OBJECT_BODIES = ["null", "[]", '"text"', "42", "true"];
|
|
54
|
+
|
|
55
|
+
const post = (path: string, body: string, env: Partial<CollectorEnv> = {}) =>
|
|
56
|
+
app.request(
|
|
57
|
+
path,
|
|
58
|
+
{ method: "POST", headers: { "content-type": "application/json" }, body },
|
|
59
|
+
{ DB: new MemSqlDb() as unknown as D1Database, ...env } as CollectorEnv,
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
describe("JSON body validation", () => {
|
|
63
|
+
for (const path of ROUTES) {
|
|
64
|
+
it(`${path} answers 400 for non-object JSON bodies`, async () => {
|
|
65
|
+
for (const body of NON_OBJECT_BODIES) {
|
|
66
|
+
const res = await post(path, body);
|
|
67
|
+
expect(res.status, `${path} <- ${body}`).toBe(400);
|
|
68
|
+
const json = (await res.json()) as { error?: string };
|
|
69
|
+
expect(typeof json.error, `${path} <- ${body}`).toBe("string");
|
|
70
|
+
}
|
|
71
|
+
const malformed = await post(path, "{not json");
|
|
72
|
+
expect(malformed.status, `${path} <- malformed`).toBe(400);
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
it("/v1/replays rejects a chunk without visitorId before touching blob storage", async () => {
|
|
77
|
+
const puts: string[] = [];
|
|
78
|
+
const bucket = {
|
|
79
|
+
put: async (key: string) => {
|
|
80
|
+
puts.push(key);
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
const res = await post(
|
|
84
|
+
"/v1/replays",
|
|
85
|
+
JSON.stringify({ sessionId: "sess-1", sequenceNumber: 0, events: [] }),
|
|
86
|
+
{ REPLAYS_BUCKET: bucket as unknown as R2Bucket },
|
|
87
|
+
);
|
|
88
|
+
expect(res.status).toBe(400);
|
|
89
|
+
expect(puts).toEqual([]);
|
|
90
|
+
});
|
|
91
|
+
});
|