@obsunified/collector 2.1.1 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
  2. package/dist/chunk-ZSF5XP3B.js.map +1 -0
  3. package/dist/index.d.ts +359 -2
  4. package/dist/index.js +5000 -334
  5. package/dist/index.js.map +1 -1
  6. package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
  7. package/package.json +2 -2
  8. package/src/__tests__/autosre-closed-loop.test.ts +2222 -0
  9. package/src/index.ts +43 -0
  10. package/src/lib/analyses-runner.ts +46 -0
  11. package/src/lib/autosre-change-correlation.ts +807 -0
  12. package/src/lib/autosre-findings.ts +763 -0
  13. package/src/lib/autosre-ontology-cache.test.ts +78 -0
  14. package/src/lib/autosre-ontology.ts +938 -0
  15. package/src/lib/autosre-remediation-gate.ts +2127 -0
  16. package/src/lib/eval-cases-store.ts +63 -1
  17. package/src/lib/evidence-references.ts +6 -0
  18. package/src/lib/housekeeping.test.ts +150 -0
  19. package/src/lib/housekeeping.ts +109 -1
  20. package/src/lib/store/ingest.ts +46 -6
  21. package/src/migrations/050_autosre_causal_ontology.sql +216 -0
  22. package/src/migrations/051_incident_active_dedup_unique.sql +31 -0
  23. package/src/migrations/alert-state-notified.test.ts +3 -3
  24. package/src/migrations/parity.test.ts +4 -0
  25. package/src/migrations/project-scoped-keys.test.ts +1 -1
  26. package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
  27. package/src/migrations-postgres/051_incident_active_dedup_unique.sql +31 -0
  28. package/src/plugins/alerts-evaluator.ts +29 -1
  29. package/src/plugins/autosre-routes.ts +1134 -0
  30. package/src/plugins/connected-routes/manifest.ts +10 -1
  31. package/src/plugins/connected-routes.ts +241 -0
  32. package/src/plugins/default-span-enrichment.ts +13 -1
  33. package/src/plugins/eval-cases-routes.ts +57 -2
  34. package/src/plugins/evidence-retrieval-routes.test.ts +32 -0
  35. package/src/plugins/evidence-retrieval-routes.ts +643 -5
  36. package/dist/chunk-XKQTVIRE.js.map +0 -1
  37. /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
@@ -0,0 +1,807 @@
1
+ import type {
2
+ CausalProofTier,
3
+ ChangeBlastRadiusResult,
4
+ ChangeEvent,
5
+ ChangeEventInput,
6
+ ChangeEventKind,
7
+ ChangeEventSource,
8
+ CohortAttributionVerdict,
9
+ CohortMetricsSummary,
10
+ ContentionCandidate,
11
+ OntologyNodeKind,
12
+ RelationshipCertainty,
13
+ } from "@obsunified/types";
14
+ import { buildCanonicalNodeId } from "@obsunified/types";
15
+ import { normalizeEnvironment } from "@obsunified/types/lineage";
16
+ import { randomHex } from "./hash";
17
+ import { parseJsonArray, parseJsonRecord } from "./json";
18
+ import type { SqlDb } from "./sql-db";
19
+
20
+ interface ChangeEventRow {
21
+ project_id: string;
22
+ environment: string;
23
+ id: string;
24
+ target_node_id: string;
25
+ kind: string;
26
+ source: string;
27
+ relationship_certainty: string;
28
+ summary: string;
29
+ service_name: string | null;
30
+ version: string | null;
31
+ previous_version: string | null;
32
+ commit_sha: string | null;
33
+ previous_commit_sha: string | null;
34
+ config_digest: string | null;
35
+ repository_url: string | null;
36
+ pull_request_url: string | null;
37
+ actor_id: string | null;
38
+ caused_by_action_id: string | null;
39
+ agent_run_id: string | null;
40
+ touched_paths_json: string | null;
41
+ attributes_json: string | null;
42
+ occurred_at: string;
43
+ received_at: string;
44
+ expires_at: string | null;
45
+ }
46
+
47
+ export const mapChangeEventRow = (row: ChangeEventRow): ChangeEvent => ({
48
+ id: row.id,
49
+ projectId: row.project_id,
50
+ environment: row.environment,
51
+ targetNodeId: row.target_node_id,
52
+ kind: row.kind as ChangeEventKind,
53
+ source: row.source as ChangeEventSource,
54
+ relationshipCertainty: (row.relationship_certainty ||
55
+ "exact") as RelationshipCertainty,
56
+ summary: row.summary,
57
+ serviceName: row.service_name,
58
+ version: row.version,
59
+ previousVersion: row.previous_version,
60
+ commitSha: row.commit_sha,
61
+ previousCommitSha: row.previous_commit_sha,
62
+ configDigest: row.config_digest,
63
+ repositoryUrl: row.repository_url,
64
+ pullRequestUrl: row.pull_request_url,
65
+ actorId: row.actor_id,
66
+ causedByActionId: row.caused_by_action_id,
67
+ agentRunId: row.agent_run_id,
68
+ touchedPaths: parseJsonArray(row.touched_paths_json ?? "[]").filter(
69
+ (p): p is string => typeof p === "string",
70
+ ),
71
+ attributes: parseJsonRecord(row.attributes_json ?? "{}"),
72
+ occurredAt: row.occurred_at,
73
+ receivedAt: row.received_at,
74
+ expiresAt: row.expires_at,
75
+ });
76
+
77
+ export class InvalidTimestampError extends Error {
78
+ constructor(field: string, value: string) {
79
+ super(`${field} must be a valid ISO-8601 timestamp, got "${value}"`);
80
+ this.name = "InvalidTimestampError";
81
+ }
82
+ }
83
+
84
+ /** Canonicalizes a client timestamp to `toISOString()` form so string comparison in SQL is sound. */
85
+ const normalizeTimestamp = (field: string, value: string): string => {
86
+ const ms = new Date(value).getTime();
87
+ if (!Number.isFinite(ms)) throw new InvalidTimestampError(field, value);
88
+ return new Date(ms).toISOString();
89
+ };
90
+
91
+ export async function recordChangeEvent(
92
+ db: SqlDb,
93
+ projectId: string,
94
+ input: ChangeEventInput,
95
+ ): Promise<ChangeEvent> {
96
+ const nowIso = new Date().toISOString();
97
+ const occurredAt = input.occurredAt
98
+ ? normalizeTimestamp("occurredAt", input.occurredAt)
99
+ : nowIso;
100
+ const environment = normalizeEnvironment(input.environment);
101
+ const namespace = (input.namespace || "default").trim();
102
+ const serviceName = input.serviceName?.trim() || null;
103
+
104
+ const canonicalNode = input.targetNodeId
105
+ ? {
106
+ id: input.targetNodeId,
107
+ environment,
108
+ kind: (input.kind === "feature_flag"
109
+ ? "feature_flag"
110
+ : "service") as OntologyNodeKind,
111
+ name: serviceName ?? input.targetNodeId,
112
+ namespaceOrEndpoint: namespace,
113
+ isScopedIdentity: true,
114
+ }
115
+ : input.kind === "feature_flag"
116
+ ? buildCanonicalNodeId({
117
+ kind: "feature_flag",
118
+ environment,
119
+ flagKey:
120
+ (typeof input.attributes?.flagKey === "string"
121
+ ? input.attributes.flagKey
122
+ : serviceName) ?? "flag",
123
+ })
124
+ : buildCanonicalNodeId({
125
+ kind: "service",
126
+ environment,
127
+ namespace,
128
+ serviceName: serviceName ?? "unknown-service",
129
+ });
130
+
131
+ const id = input.id?.trim() || `chg_${randomHex(10)}`;
132
+ const source: ChangeEventSource = input.source ?? "explicit_api";
133
+ const certainty: RelationshipCertainty =
134
+ input.relationshipCertainty ??
135
+ (source === "inferred_telemetry" ? "inferred" : "exact");
136
+ const touchedPaths = (input.touchedPaths ?? []).slice(0, 500);
137
+ const expiresAt = new Date(
138
+ Date.parse(occurredAt) + 30 * 24 * 3600_000,
139
+ ).toISOString();
140
+
141
+ await db.batch([
142
+ db
143
+ .prepare(
144
+ `INSERT INTO ontology_nodes (
145
+ project_id, environment, id, kind, name, namespace_or_endpoint,
146
+ is_scoped_identity, current_version, current_commit_sha,
147
+ current_config_digest, attributes_json, first_seen_at, last_seen_at
148
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
149
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
150
+ current_version = COALESCE(excluded.current_version, ontology_nodes.current_version),
151
+ current_commit_sha = COALESCE(excluded.current_commit_sha, ontology_nodes.current_commit_sha),
152
+ current_config_digest = COALESCE(excluded.current_config_digest, ontology_nodes.current_config_digest),
153
+ last_seen_at = excluded.last_seen_at`,
154
+ )
155
+ .bind(
156
+ projectId,
157
+ environment,
158
+ canonicalNode.id,
159
+ canonicalNode.kind,
160
+ canonicalNode.name,
161
+ canonicalNode.namespaceOrEndpoint,
162
+ canonicalNode.isScopedIdentity ? 1 : 0,
163
+ input.version ?? null,
164
+ input.commitSha ?? null,
165
+ input.configDigest ?? null,
166
+ occurredAt,
167
+ nowIso,
168
+ ),
169
+ db
170
+ .prepare(
171
+ `INSERT INTO change_events (
172
+ project_id, environment, id, target_node_id, kind, source,
173
+ relationship_certainty, summary, service_name, version,
174
+ previous_version, commit_sha, previous_commit_sha, config_digest,
175
+ repository_url, pull_request_url, actor_id, caused_by_action_id,
176
+ agent_run_id, touched_paths_json, attributes_json,
177
+ occurred_at, received_at, expires_at
178
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
179
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
180
+ summary = excluded.summary,
181
+ version = COALESCE(excluded.version, change_events.version),
182
+ previous_version = COALESCE(excluded.previous_version, change_events.previous_version),
183
+ commit_sha = COALESCE(excluded.commit_sha, change_events.commit_sha),
184
+ touched_paths_json = excluded.touched_paths_json,
185
+ attributes_json = excluded.attributes_json,
186
+ occurred_at = excluded.occurred_at`,
187
+ )
188
+ .bind(
189
+ projectId,
190
+ environment,
191
+ id,
192
+ canonicalNode.id,
193
+ input.kind,
194
+ source,
195
+ certainty,
196
+ input.summary,
197
+ serviceName,
198
+ input.version ?? null,
199
+ input.previousVersion ?? null,
200
+ input.commitSha ?? null,
201
+ input.previousCommitSha ?? null,
202
+ input.configDigest ?? null,
203
+ input.repositoryUrl ?? null,
204
+ input.pullRequestUrl ?? null,
205
+ input.actorId ?? null,
206
+ input.causedByActionId ?? null,
207
+ input.agentRunId ?? null,
208
+ JSON.stringify(touchedPaths),
209
+ JSON.stringify(input.attributes ?? {}),
210
+ occurredAt,
211
+ nowIso,
212
+ expiresAt,
213
+ ),
214
+ ]);
215
+
216
+ return {
217
+ id,
218
+ projectId,
219
+ environment,
220
+ targetNodeId: canonicalNode.id,
221
+ kind: input.kind,
222
+ source,
223
+ relationshipCertainty: certainty,
224
+ summary: input.summary,
225
+ serviceName,
226
+ version: input.version ?? null,
227
+ previousVersion: input.previousVersion ?? null,
228
+ commitSha: input.commitSha ?? null,
229
+ previousCommitSha: input.previousCommitSha ?? null,
230
+ configDigest: input.configDigest ?? null,
231
+ repositoryUrl: input.repositoryUrl ?? null,
232
+ pullRequestUrl: input.pullRequestUrl ?? null,
233
+ actorId: input.actorId ?? null,
234
+ causedByActionId: input.causedByActionId ?? null,
235
+ agentRunId: input.agentRunId ?? null,
236
+ touchedPaths,
237
+ attributes: input.attributes ?? {},
238
+ occurredAt,
239
+ receivedAt: nowIso,
240
+ expiresAt,
241
+ };
242
+ }
243
+
244
+ export async function listChangeEvents(
245
+ db: SqlDb,
246
+ options: {
247
+ projectId: string;
248
+ environment?: string;
249
+ serviceName?: string;
250
+ targetNodeId?: string;
251
+ kind?: string;
252
+ since?: string;
253
+ limit?: number;
254
+ },
255
+ ): Promise<ChangeEvent[]> {
256
+ const conditions: string[] = ["project_id = ?"];
257
+ const binds: unknown[] = [options.projectId];
258
+
259
+ if (options.environment) {
260
+ conditions.push("environment = ?");
261
+ binds.push(options.environment.toLowerCase());
262
+ }
263
+ if (options.serviceName) {
264
+ conditions.push("service_name = ?");
265
+ binds.push(options.serviceName);
266
+ }
267
+ if (options.targetNodeId) {
268
+ conditions.push("target_node_id = ?");
269
+ binds.push(options.targetNodeId);
270
+ }
271
+ if (options.kind) {
272
+ conditions.push("kind = ?");
273
+ binds.push(options.kind);
274
+ }
275
+ if (options.since) {
276
+ conditions.push("occurred_at >= ?");
277
+ binds.push(normalizeTimestamp("since", options.since));
278
+ }
279
+
280
+ const limit = Math.max(1, Math.min(200, options.limit ?? 50));
281
+ binds.push(limit);
282
+
283
+ const rows = await db
284
+ .prepare(
285
+ `SELECT * FROM change_events
286
+ WHERE ${conditions.join(" AND ")}
287
+ ORDER BY occurred_at DESC
288
+ LIMIT ?`,
289
+ )
290
+ .bind(...binds)
291
+ .all<ChangeEventRow>();
292
+
293
+ return rows.results.map(mapChangeEventRow);
294
+ }
295
+
296
+ export async function getChangeEvent(
297
+ db: SqlDb,
298
+ projectId: string,
299
+ changeId: string,
300
+ environment?: string,
301
+ ): Promise<ChangeEvent | null> {
302
+ const row = environment
303
+ ? await db
304
+ .prepare(
305
+ `SELECT * FROM change_events
306
+ WHERE project_id = ? AND environment = ? AND id = ?`,
307
+ )
308
+ .bind(projectId, environment.toLowerCase(), changeId)
309
+ .first<ChangeEventRow>()
310
+ : await db
311
+ .prepare(
312
+ `SELECT * FROM change_events
313
+ WHERE project_id = ? AND id = ?
314
+ ORDER BY occurred_at DESC
315
+ LIMIT 1`,
316
+ )
317
+ .bind(projectId, changeId)
318
+ .first<ChangeEventRow>();
319
+
320
+ return row ? mapChangeEventRow(row) : null;
321
+ }
322
+
323
+ interface SpanCohortSampleRow {
324
+ trace_id: string;
325
+ span_id: string;
326
+ parent_span_id: string | null;
327
+ span_kind: number;
328
+ status_code: number;
329
+ duration_ms: number;
330
+ service_version: string | null;
331
+ target_node_id: string | null;
332
+ attributes_json: string | null;
333
+ events_json: string | null;
334
+ start_time: string;
335
+ end_time: string;
336
+ received_at: string;
337
+ }
338
+
339
+ const computePercentile = (sorted: number[], p: number): number | null => {
340
+ if (sorted.length === 0) return null;
341
+ const idx = Math.min(
342
+ sorted.length - 1,
343
+ Math.max(0, Math.ceil((p / 100) * sorted.length) - 1),
344
+ );
345
+ return sorted[idx];
346
+ };
347
+
348
+ const summarizeCohort = (
349
+ cohortKey: string,
350
+ requestSpans: SpanCohortSampleRow[],
351
+ downstreamSpans: SpanCohortSampleRow[],
352
+ ): CohortMetricsSummary => {
353
+ const requestCount = requestSpans.length;
354
+ const errorCount = requestSpans.filter((s) => s.status_code === 2).length;
355
+ const errorRate = requestCount > 0 ? errorCount / requestCount : 0;
356
+ const durations = requestSpans
357
+ .map((s) => Number(s.duration_ms ?? 0))
358
+ .sort((a, b) => a - b);
359
+ const p95DurationMs = computePercentile(durations, 95);
360
+
361
+ const downstreamCount = downstreamSpans.length;
362
+ const avgDownstreamCallsPerRequest =
363
+ requestCount > 0 ? downstreamCount / requestCount : 0;
364
+ const avgDownstreamDurationMs =
365
+ downstreamCount > 0
366
+ ? downstreamSpans.reduce(
367
+ (acc, s) => acc + Number(s.duration_ms ?? 0),
368
+ 0,
369
+ ) / downstreamCount
370
+ : null;
371
+
372
+ return {
373
+ cohortKey,
374
+ requestCount,
375
+ errorCount,
376
+ errorRate,
377
+ p95DurationMs,
378
+ avgDownstreamCallsPerRequest,
379
+ avgDownstreamDurationMs,
380
+ };
381
+ };
382
+
383
+ /**
384
+ * Indexed cross-trace contention lookup (RFC 0013 § 4.B).
385
+ * Uses `idx_spans_contention_lookup` (`project_id, environment, target_node_id, start_time, end_time`).
386
+ * Enforces P1 #5: if the target node is unscoped (`is_scoped_identity = 0`), caps
387
+ * `relationshipCertainty` at `"inferred"`.
388
+ */
389
+ export async function findCrossTraceContention(
390
+ db: SqlDb,
391
+ options: {
392
+ projectId: string;
393
+ environment: string;
394
+ targetNodeId: string;
395
+ excludeTraceId: string;
396
+ startTime: string;
397
+ endTime: string;
398
+ limit?: number;
399
+ },
400
+ ): Promise<ContentionCandidate[]> {
401
+ const env = options.environment.toLowerCase();
402
+ const limit = Math.max(1, Math.min(20, options.limit ?? 5));
403
+
404
+ const [nodeRow, contentionRows] = await Promise.all([
405
+ db
406
+ .prepare(
407
+ `SELECT is_scoped_identity FROM ontology_nodes
408
+ WHERE project_id = ? AND environment = ? AND id = ?`,
409
+ )
410
+ .bind(options.projectId, env, options.targetNodeId)
411
+ .first<{ is_scoped_identity: number }>(),
412
+ db
413
+ .prepare(
414
+ `SELECT
415
+ trace_id,
416
+ span_id,
417
+ service_name,
418
+ service_version,
419
+ span_name,
420
+ COUNT(*) AS concurrent_calls,
421
+ MAX(duration_ms) AS max_duration_ms,
422
+ SUM(duration_ms) AS total_resource_time_ms
423
+ FROM telemetry_spans
424
+ WHERE project_id = ?
425
+ AND environment = ?
426
+ AND target_node_id = ?
427
+ AND trace_id <> ?
428
+ AND start_time <= ?
429
+ AND end_time >= ?
430
+ GROUP BY trace_id, span_id, service_name, service_version, span_name
431
+ ORDER BY total_resource_time_ms DESC
432
+ LIMIT ?`,
433
+ )
434
+ .bind(
435
+ options.projectId,
436
+ env,
437
+ options.targetNodeId,
438
+ options.excludeTraceId,
439
+ options.endTime,
440
+ options.startTime,
441
+ limit,
442
+ )
443
+ .all<{
444
+ trace_id: string;
445
+ span_id: string;
446
+ service_name: string | null;
447
+ service_version: string | null;
448
+ span_name: string;
449
+ concurrent_calls: number;
450
+ max_duration_ms: number;
451
+ total_resource_time_ms: number;
452
+ }>(),
453
+ ]);
454
+
455
+ const isScoped = nodeRow
456
+ ? nodeRow.is_scoped_identity === 1
457
+ : !options.targetNodeId.includes(":unscoped/");
458
+ const relationshipCertainty: RelationshipCertainty = isScoped
459
+ ? "exact"
460
+ : "inferred";
461
+
462
+ return contentionRows.results.map((r) => ({
463
+ aggressorTraceId: r.trace_id,
464
+ aggressorSpanId: r.span_id,
465
+ aggressorServiceName: r.service_name,
466
+ aggressorServiceVersion: r.service_version,
467
+ aggressorSpanName: r.span_name,
468
+ sharedTargetNodeId: options.targetNodeId,
469
+ relationshipCertainty,
470
+ concurrentCalls: Number(r.concurrent_calls ?? 1),
471
+ maxDurationMs: Number(r.max_duration_ms ?? 0),
472
+ totalResourceTimeMs: Number(r.total_resource_time_ms ?? 0),
473
+ }));
474
+ }
475
+
476
+ const extractFilePathsFromSpan = (span: SpanCohortSampleRow): string[] => {
477
+ const found: string[] = [];
478
+ const attrs = parseJsonRecord(span.attributes_json ?? "{}");
479
+ for (const key of [
480
+ "code.filepath",
481
+ "obs.code.filepath",
482
+ "exception.stacktrace",
483
+ ]) {
484
+ const val = attrs[key];
485
+ if (typeof val === "string") {
486
+ found.push(val);
487
+ }
488
+ }
489
+ if (span.events_json) {
490
+ const events = parseJsonArray(span.events_json);
491
+ for (const ev of events) {
492
+ if (ev && typeof ev === "object" && !Array.isArray(ev)) {
493
+ const evAttrs = parseJsonRecord(
494
+ JSON.stringify((ev as Record<string, unknown>).attributes ?? {}),
495
+ );
496
+ for (const key of ["code.filepath", "exception.stacktrace"]) {
497
+ if (typeof evAttrs[key] === "string") {
498
+ found.push(evAttrs[key] as string);
499
+ }
500
+ }
501
+ }
502
+ }
503
+ }
504
+ return found;
505
+ };
506
+
507
+ /**
508
+ * Computes traffic-normalized cohort rate lift, minimum sample gates, shared-degradation
509
+ * noisy-neighbor contention checks, and code-path overlap for a `change_event`
510
+ * (RFC 0013 § 4.A, P1 #1, P1 #2).
511
+ */
512
+ export async function computeChangeBlastRadius(
513
+ db: SqlDb,
514
+ projectId: string,
515
+ changeId: string,
516
+ options: {
517
+ environment?: string;
518
+ minComparableSample?: number;
519
+ } = {},
520
+ ): Promise<ChangeBlastRadiusResult | null> {
521
+ const change = await getChangeEvent(
522
+ db,
523
+ projectId,
524
+ changeId,
525
+ options.environment,
526
+ );
527
+ if (!change) return null;
528
+
529
+ const minSample = options.minComparableSample ?? 50;
530
+ const preWindowStart = new Date(
531
+ Date.parse(change.occurredAt) - 30 * 60_000,
532
+ ).toISOString();
533
+
534
+ // Load spans for the service across [occurredAt - 30m, now] using idx_spans_cohort_attribution
535
+ const serviceFilter = change.serviceName;
536
+ const spansResult = serviceFilter
537
+ ? await db
538
+ .prepare(
539
+ `SELECT
540
+ trace_id, span_id, parent_span_id, span_kind, status_code,
541
+ duration_ms, service_version, target_node_id, attributes_json,
542
+ events_json, start_time, end_time, received_at
543
+ FROM telemetry_spans
544
+ WHERE project_id = ?
545
+ AND environment = ?
546
+ AND service_name = ?
547
+ AND received_at >= ?
548
+ ORDER BY received_at DESC
549
+ LIMIT 5000`,
550
+ )
551
+ .bind(projectId, change.environment, serviceFilter, preWindowStart)
552
+ .all<SpanCohortSampleRow>()
553
+ : await db
554
+ .prepare(
555
+ `SELECT
556
+ trace_id, span_id, parent_span_id, span_kind, status_code,
557
+ duration_ms, service_version, target_node_id, attributes_json,
558
+ events_json, start_time, end_time, received_at
559
+ FROM telemetry_spans
560
+ WHERE project_id = ?
561
+ AND environment = ?
562
+ AND received_at >= ?
563
+ ORDER BY received_at DESC
564
+ LIMIT 5000`,
565
+ )
566
+ .bind(projectId, change.environment, preWindowStart)
567
+ .all<SpanCohortSampleRow>();
568
+
569
+ const allSpans = spansResult.results;
570
+ const postSpans = allSpans.filter(
571
+ (s) => (s.start_time || s.received_at) >= change.occurredAt,
572
+ );
573
+ const preSpans = allSpans.filter(
574
+ (s) => (s.start_time || s.received_at) < change.occurredAt,
575
+ );
576
+
577
+ const isRootOrServer = (s: SpanCohortSampleRow) =>
578
+ s.parent_span_id === null || s.span_kind === 1;
579
+ const isDownstreamCall = (s: SpanCohortSampleRow) =>
580
+ Boolean(s.target_node_id) || s.span_kind === 2 || s.span_kind === 4;
581
+
582
+ const flagKey =
583
+ change.kind === "feature_flag"
584
+ ? ((typeof change.attributes.flagKey === "string"
585
+ ? change.attributes.flagKey
586
+ : change.serviceName) ?? "flag")
587
+ : null;
588
+
589
+ const newCohortKey =
590
+ change.version ??
591
+ (typeof change.attributes.newVariant === "string"
592
+ ? change.attributes.newVariant
593
+ : "true");
594
+ const oldCohortKey =
595
+ change.previousVersion ??
596
+ (typeof change.attributes.previousVariant === "string"
597
+ ? change.attributes.previousVariant
598
+ : "false");
599
+
600
+ const getCohortOfSpan = (s: SpanCohortSampleRow): string | null => {
601
+ if (flagKey) {
602
+ const attrs = parseJsonRecord(s.attributes_json ?? "{}");
603
+ const direct = attrs[`obs.feature_flags.${flagKey}`];
604
+ if (direct !== undefined && direct !== null) return String(direct);
605
+ if (
606
+ attrs["feature_flag.key"] === flagKey &&
607
+ attrs["feature_flag.variant"] !== undefined
608
+ ) {
609
+ return String(attrs["feature_flag.variant"]);
610
+ }
611
+ const nested = attrs["obs.feature_flags"];
612
+ if (nested && typeof nested === "object" && !Array.isArray(nested)) {
613
+ const v = (nested as Record<string, unknown>)[flagKey];
614
+ if (v !== undefined && v !== null) return String(v);
615
+ }
616
+ return null;
617
+ }
618
+ return s.service_version;
619
+ };
620
+
621
+ const postNewReq = postSpans.filter(
622
+ (s) => isRootOrServer(s) && getCohortOfSpan(s) === newCohortKey,
623
+ );
624
+ const postNewDown = postSpans.filter(
625
+ (s) => isDownstreamCall(s) && getCohortOfSpan(s) === newCohortKey,
626
+ );
627
+
628
+ const postOldReq = postSpans.filter(
629
+ (s) =>
630
+ isRootOrServer(s) &&
631
+ (oldCohortKey
632
+ ? getCohortOfSpan(s) === oldCohortKey
633
+ : getCohortOfSpan(s) !== null && getCohortOfSpan(s) !== newCohortKey),
634
+ );
635
+ const postOldDown = postSpans.filter(
636
+ (s) =>
637
+ isDownstreamCall(s) &&
638
+ (oldCohortKey
639
+ ? getCohortOfSpan(s) === oldCohortKey
640
+ : getCohortOfSpan(s) !== null && getCohortOfSpan(s) !== newCohortKey),
641
+ );
642
+
643
+ const preOldReq = preSpans.filter(
644
+ (s) =>
645
+ isRootOrServer(s) &&
646
+ (!oldCohortKey || getCohortOfSpan(s) === oldCohortKey),
647
+ );
648
+ const preOldDown = preSpans.filter(
649
+ (s) =>
650
+ isDownstreamCall(s) &&
651
+ (!oldCohortKey || getCohortOfSpan(s) === oldCohortKey),
652
+ );
653
+
654
+ const newCohort = summarizeCohort(newCohortKey, postNewReq, postNewDown);
655
+ const oldCohort = summarizeCohort(oldCohortKey, postOldReq, postOldDown);
656
+ const preBaseline = summarizeCohort(
657
+ `${oldCohortKey}:pre`,
658
+ preOldReq,
659
+ preOldDown,
660
+ );
661
+
662
+ // Check code-path overlap between change.touchedPaths and error spans in new cohort
663
+ const overlappingSet = new Set<string>();
664
+ const matchingErrorSpanIds: string[] = [];
665
+ if (change.touchedPaths.length > 0) {
666
+ for (const s of postNewReq.filter((sp) => sp.status_code === 2)) {
667
+ const rawPaths = extractFilePathsFromSpan(s);
668
+ let matchedSpan = false;
669
+ for (const touched of change.touchedPaths) {
670
+ if (rawPaths.some((raw) => raw.includes(touched))) {
671
+ overlappingSet.add(touched);
672
+ matchedSpan = true;
673
+ }
674
+ }
675
+ if (matchedSpan) {
676
+ matchingErrorSpanIds.push(s.span_id);
677
+ }
678
+ }
679
+ }
680
+ const overlappingCodePaths = Array.from(overlappingSet);
681
+
682
+ // Evaluate Traffic-Normalized Cohort Verdict (P1 #2)
683
+ let cohortVerdict: CohortAttributionVerdict = "insufficient_sample";
684
+ let causalProofTier: CausalProofTier = "unverified_locus";
685
+ let rateDifference = 0;
686
+ let relativeRisk = 1;
687
+ const sharedResourceAggressors: ContentionCandidate[] = [];
688
+
689
+ if (newCohort.requestCount < minSample) {
690
+ cohortVerdict = "insufficient_sample";
691
+ causalProofTier =
692
+ overlappingCodePaths.length > 0
693
+ ? "corroborated_candidate"
694
+ : "unverified_locus";
695
+ } else if (oldCohort.requestCount >= minSample) {
696
+ rateDifference = newCohort.errorRate - oldCohort.errorRate;
697
+ relativeRisk = newCohort.errorRate / Math.max(oldCohort.errorRate, 0.001);
698
+
699
+ const fanoutRatio =
700
+ newCohort.avgDownstreamCallsPerRequest /
701
+ Math.max(oldCohort.avgDownstreamCallsPerRequest, 0.25);
702
+ const durationRatio =
703
+ (newCohort.avgDownstreamDurationMs ?? 0) /
704
+ Math.max(oldCohort.avgDownstreamDurationMs ?? 1, 1);
705
+ const hasNewVersionAggressorSignal =
706
+ fanoutRatio >= 3.0 || durationRatio >= 3.0;
707
+ const hasPreChangeDegradationSignal =
708
+ preBaseline.requestCount >= 10 &&
709
+ oldCohort.errorRate > 1.5 * Math.max(preBaseline.errorRate, 0.01);
710
+
711
+ const bothDegraded =
712
+ newCohort.errorRate >= 0.05 &&
713
+ oldCohort.errorRate >= 0.05 &&
714
+ (hasNewVersionAggressorSignal || hasPreChangeDegradationSignal);
715
+
716
+ if (bothDegraded) {
717
+ if (hasNewVersionAggressorSignal) {
718
+ cohortVerdict = "shared_degradation_new_version_aggressor";
719
+ causalProofTier = "corroborated_candidate";
720
+ // Populate top contention aggressors from postNewDown
721
+ const sampleTarget = postNewDown.find(
722
+ (d) => d.target_node_id,
723
+ )?.target_node_id;
724
+ if (sampleTarget) {
725
+ const victimTrace =
726
+ postOldReq.find((r) => r.status_code === 2)?.trace_id ?? "victim";
727
+ const candidates = await findCrossTraceContention(db, {
728
+ projectId,
729
+ environment: change.environment,
730
+ targetNodeId: sampleTarget,
731
+ excludeTraceId: victimTrace,
732
+ startTime: change.occurredAt,
733
+ endTime: new Date().toISOString(),
734
+ });
735
+ sharedResourceAggressors.push(...candidates);
736
+ }
737
+ } else {
738
+ cohortVerdict = "shared_degradation_external_cause";
739
+ causalProofTier = "unverified_locus";
740
+ }
741
+ } else {
742
+ const p95Ratio =
743
+ newCohort.p95DurationMs !== null &&
744
+ oldCohort.p95DurationMs !== null &&
745
+ oldCohort.p95DurationMs > 0
746
+ ? newCohort.p95DurationMs / oldCohort.p95DurationMs
747
+ : 1;
748
+
749
+ if (
750
+ (rateDifference >= 0.05 || p95Ratio >= 2.5) &&
751
+ (relativeRisk >= 5.0 || p95Ratio >= 2.5)
752
+ ) {
753
+ cohortVerdict = "isolated_to_new_version";
754
+ causalProofTier = "cohort_rate_isolated";
755
+ } else {
756
+ cohortVerdict = "no_regression_detected";
757
+ causalProofTier = "unverified_locus";
758
+ }
759
+ }
760
+ } else if (preBaseline.requestCount >= minSample) {
761
+ // Instant 100% cutover: compare post-change v_new vs pre-change baseline
762
+ rateDifference = newCohort.errorRate - preBaseline.errorRate;
763
+ relativeRisk = newCohort.errorRate / Math.max(preBaseline.errorRate, 0.001);
764
+ const p95Ratio =
765
+ newCohort.p95DurationMs !== null &&
766
+ preBaseline.p95DurationMs !== null &&
767
+ preBaseline.p95DurationMs > 0
768
+ ? newCohort.p95DurationMs / preBaseline.p95DurationMs
769
+ : 1;
770
+
771
+ if (
772
+ (rateDifference >= 0.05 || p95Ratio >= 2.5) &&
773
+ (relativeRisk >= 5.0 || p95Ratio >= 2.5)
774
+ ) {
775
+ cohortVerdict = "sequential_post_cutover_regression";
776
+ causalProofTier = "corroborated_candidate";
777
+ } else {
778
+ cohortVerdict = "no_regression_detected";
779
+ causalProofTier = "unverified_locus";
780
+ }
781
+ } else if (overlappingCodePaths.length > 0) {
782
+ cohortVerdict = "insufficient_sample";
783
+ causalProofTier = "corroborated_candidate";
784
+ }
785
+
786
+ const relationshipCertainty: RelationshipCertainty =
787
+ overlappingCodePaths.length > 0 ? "exact" : change.relationshipCertainty;
788
+
789
+ const summary = `Change ${change.id} (${change.kind} ${newCohortKey} in ${change.environment}): verdict=${cohortVerdict}, proofTier=${causalProofTier}, newErrorRate=${(newCohort.errorRate * 100).toFixed(1)}% (N=${newCohort.requestCount}), oldErrorRate=${(oldCohort.errorRate * 100).toFixed(1)}% (N=${oldCohort.requestCount})`;
790
+
791
+ return {
792
+ change,
793
+ cohortVerdict,
794
+ causalProofTier,
795
+ relationshipCertainty,
796
+ minComparableSample: minSample,
797
+ newCohort,
798
+ oldCohort,
799
+ preChangeBaselineCohort: preBaseline,
800
+ rateDifference,
801
+ relativeRisk,
802
+ overlappingCodePaths,
803
+ matchingErrorSpanIds,
804
+ sharedResourceAggressors,
805
+ summary,
806
+ };
807
+ }