@obsunified/collector 2.0.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/dist/auth.d.ts +1 -1
  2. package/dist/auth.js +1 -1
  3. package/dist/{chunk-UGWDOGGH.js → chunk-KMEZKGSO.js} +22 -10
  4. package/dist/chunk-KMEZKGSO.js.map +1 -0
  5. package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
  6. package/dist/chunk-ZSF5XP3B.js.map +1 -0
  7. package/dist/index.d.ts +630 -27
  8. package/dist/index.js +5819 -853
  9. package/dist/index.js.map +1 -1
  10. package/dist/{ingest-auth-HFVO1zIp.d.ts → ingest-auth-C5-8Mly5.d.ts} +9 -0
  11. package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
  12. package/package.json +5 -5
  13. package/src/__tests__/autosre-closed-loop.test.ts +1275 -0
  14. package/src/analyses/derive.ts +26 -25
  15. package/src/analyses/investigations.ts +8 -8
  16. package/src/analyses/tier0.ts +22 -22
  17. package/src/auth/dashboard-auth.test.ts +791 -0
  18. package/src/auth/dashboard-auth.ts +187 -38
  19. package/src/auth/login-rate-limit.test.ts +129 -0
  20. package/src/auth/login-rate-limit.ts +143 -0
  21. package/src/framework/body-limit.test.ts +165 -0
  22. package/src/framework/collector.ts +355 -115
  23. package/src/framework/env.ts +9 -0
  24. package/src/framework/limits.ts +31 -0
  25. package/src/framework/scheduled-handlers.test.ts +342 -0
  26. package/src/index.ts +59 -1
  27. package/src/lib/action-retention.test.ts +472 -0
  28. package/src/lib/action-retention.ts +420 -0
  29. package/src/lib/actions-store.ts +6 -12
  30. package/src/lib/ai-store/calls.ts +2 -2
  31. package/src/lib/ai-store/sessions.ts +3 -3
  32. package/src/lib/ai-store/spans.ts +1 -1
  33. package/src/lib/ai-store/types.ts +2 -1
  34. package/src/lib/alerts-store.test.ts +153 -0
  35. package/src/lib/alerts-store.ts +122 -21
  36. package/src/lib/analyses-runner.test.ts +75 -0
  37. package/src/lib/analyses-runner.ts +69 -3
  38. package/src/lib/analyses-store-latest.test.ts +61 -0
  39. package/src/lib/analyses-store.ts +17 -10
  40. package/src/lib/autosre-change-correlation.ts +780 -0
  41. package/src/lib/autosre-findings.ts +725 -0
  42. package/src/lib/autosre-ontology.ts +914 -0
  43. package/src/lib/autosre-remediation-gate.ts +1663 -0
  44. package/src/lib/blob-store.ts +1 -1
  45. package/src/lib/eval-cases-store.ts +63 -1
  46. package/src/lib/evidence-references.ts +6 -0
  47. package/src/lib/housekeeping.test.ts +153 -0
  48. package/src/lib/housekeeping.ts +193 -0
  49. package/src/lib/identity-index/action-lookups.ts +6 -6
  50. package/src/lib/identity-index/key-lookups.ts +2 -2
  51. package/src/lib/identity-index/user-lookup.ts +8 -5
  52. package/src/lib/identity-index.test.ts +2 -3
  53. package/src/lib/metrics-store-series.test.ts +99 -0
  54. package/src/lib/metrics-store.test.ts +23 -0
  55. package/src/lib/metrics-store.ts +27 -3
  56. package/src/lib/projects-store.test.ts +25 -0
  57. package/src/lib/projects-store.ts +4 -3
  58. package/src/lib/sql-db-dialect.test.ts +12 -3
  59. package/src/lib/sql-db-postgres.integration.test.ts +1225 -0
  60. package/src/lib/sql-db-postgres.test.ts +314 -0
  61. package/src/lib/sql-db-postgres.ts +392 -54
  62. package/src/lib/sql-db-sqlite-time-windows.test.ts +194 -0
  63. package/src/lib/sql-db.ts +40 -6
  64. package/src/lib/store/ingest.ts +94 -6
  65. package/src/lib/test-utils/action-graph-fixtures.ts +307 -0
  66. package/src/lib/test-utils/node-sqlite-db.ts +188 -0
  67. package/src/lib/webhook-url.test.ts +122 -0
  68. package/src/lib/webhook-url.ts +155 -0
  69. package/src/migrations/041_ingest_key_hash_text.sql +5 -0
  70. package/src/migrations/042_session_replay_project_composite_pk.sql +52 -0
  71. package/src/migrations/043_metric_exemplars_text_timestamps.sql +5 -0
  72. package/src/migrations/044_actions_started_at_index.sql +10 -0
  73. package/src/migrations/045_user_profiles_project_composite_pk.sql +40 -0
  74. package/src/migrations/046_action_graph_project_composite_pk.sql +255 -0
  75. package/src/migrations/047_ai_span_payloads_project_composite_pk.sql +61 -0
  76. package/src/migrations/048_telemetry_spans_project_composite_pk.sql +92 -0
  77. package/src/migrations/049_alert_state_notified.sql +17 -0
  78. package/src/migrations/050_autosre_causal_ontology.sql +216 -0
  79. package/src/migrations/alert-state-notified.test.ts +111 -0
  80. package/src/migrations/parity.test.ts +68 -0
  81. package/src/migrations/project-scoped-keys.test.ts +432 -0
  82. package/src/migrations-postgres/009_projects_and_keys.sql +4 -4
  83. package/src/migrations-postgres/026_obs_dashboard_project_seed.sql +1 -1
  84. package/src/migrations-postgres/035_metric_exemplars.sql +5 -2
  85. package/src/migrations-postgres/041_ingest_key_hash_text.sql +32 -0
  86. package/src/migrations-postgres/042_session_replay_project_composite_pk.sql +16 -0
  87. package/src/migrations-postgres/043_metric_exemplars_text_timestamps.sql +47 -0
  88. package/src/migrations-postgres/044_actions_started_at_index.sql +8 -0
  89. package/src/migrations-postgres/045_user_profiles_project_composite_pk.sql +33 -0
  90. package/src/migrations-postgres/046_action_graph_project_composite_pk.sql +39 -0
  91. package/src/migrations-postgres/047_ai_span_payloads_project_composite_pk.sql +30 -0
  92. package/src/migrations-postgres/048_telemetry_spans_project_composite_pk.sql +37 -0
  93. package/src/migrations-postgres/049_alert_state_notified.sql +10 -0
  94. package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
  95. package/src/migrations-postgres/README.md +58 -5
  96. package/src/otlp/decode/body.test.ts +76 -0
  97. package/src/otlp/decode/body.ts +67 -6
  98. package/src/plugins/_json.ts +21 -0
  99. package/src/plugins/action-graph-processor.ts +6 -12
  100. package/src/plugins/ai-receiver.ts +5 -12
  101. package/src/plugins/ai-span-payloads-processor.ts +1 -2
  102. package/src/plugins/alerts-evaluator-retention.test.ts +170 -0
  103. package/src/plugins/alerts-evaluator.test.ts +464 -3
  104. package/src/plugins/alerts-evaluator.ts +181 -41
  105. package/src/plugins/alerts-routes.ts +10 -6
  106. package/src/plugins/ask-routes.ts +4 -5
  107. package/src/plugins/autosre-routes.ts +1019 -0
  108. package/src/plugins/connected-routes/manifest.ts +13 -1
  109. package/src/plugins/connected-routes.test.ts +106 -0
  110. package/src/plugins/connected-routes.ts +285 -5
  111. package/src/plugins/dashboard-routes.test.ts +120 -0
  112. package/src/plugins/dashboard-routes.ts +11 -2
  113. package/src/plugins/default-span-enrichment.ts +13 -1
  114. package/src/plugins/eval-cases-routes.ts +65 -19
  115. package/src/plugins/evidence-retrieval-routes.ts +300 -4
  116. package/src/plugins/identity-receiver.ts +9 -11
  117. package/src/plugins/json-body.test.ts +91 -0
  118. package/src/plugins/onboarding-routes.test.ts +105 -0
  119. package/src/plugins/onboarding-routes.ts +26 -11
  120. package/src/plugins/projects-routes.ts +2 -1
  121. package/src/plugins/replay-isolation.test.ts +922 -0
  122. package/src/plugins/replay-receiver.ts +8 -8
  123. package/src/plugins/usage-receiver.ts +3 -6
  124. package/dist/chunk-UGWDOGGH.js.map +0 -1
  125. package/dist/chunk-XKQTVIRE.js.map +0 -1
  126. /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
@@ -0,0 +1,780 @@
1
+ import type {
2
+ CausalProofTier,
3
+ ChangeBlastRadiusResult,
4
+ ChangeEvent,
5
+ ChangeEventInput,
6
+ ChangeEventKind,
7
+ ChangeEventSource,
8
+ CohortAttributionVerdict,
9
+ CohortMetricsSummary,
10
+ ContentionCandidate,
11
+ OntologyNodeKind,
12
+ RelationshipCertainty,
13
+ } from "@obsunified/types";
14
+ import { buildCanonicalNodeId } from "@obsunified/types";
15
+ import { randomHex } from "./hash";
16
+ import { parseJsonArray, parseJsonRecord } from "./json";
17
+ import type { SqlDb } from "./sql-db";
18
+
19
+ interface ChangeEventRow {
20
+ project_id: string;
21
+ environment: string;
22
+ id: string;
23
+ target_node_id: string;
24
+ kind: string;
25
+ source: string;
26
+ relationship_certainty: string;
27
+ summary: string;
28
+ service_name: string | null;
29
+ version: string | null;
30
+ previous_version: string | null;
31
+ commit_sha: string | null;
32
+ previous_commit_sha: string | null;
33
+ config_digest: string | null;
34
+ repository_url: string | null;
35
+ pull_request_url: string | null;
36
+ actor_id: string | null;
37
+ caused_by_action_id: string | null;
38
+ agent_run_id: string | null;
39
+ touched_paths_json: string | null;
40
+ attributes_json: string | null;
41
+ occurred_at: string;
42
+ received_at: string;
43
+ expires_at: string | null;
44
+ }
45
+
46
+ export const mapChangeEventRow = (row: ChangeEventRow): ChangeEvent => ({
47
+ id: row.id,
48
+ projectId: row.project_id,
49
+ environment: row.environment,
50
+ targetNodeId: row.target_node_id,
51
+ kind: row.kind as ChangeEventKind,
52
+ source: row.source as ChangeEventSource,
53
+ relationshipCertainty: (row.relationship_certainty ||
54
+ "exact") as RelationshipCertainty,
55
+ summary: row.summary,
56
+ serviceName: row.service_name,
57
+ version: row.version,
58
+ previousVersion: row.previous_version,
59
+ commitSha: row.commit_sha,
60
+ previousCommitSha: row.previous_commit_sha,
61
+ configDigest: row.config_digest,
62
+ repositoryUrl: row.repository_url,
63
+ pullRequestUrl: row.pull_request_url,
64
+ actorId: row.actor_id,
65
+ causedByActionId: row.caused_by_action_id,
66
+ agentRunId: row.agent_run_id,
67
+ touchedPaths: parseJsonArray(row.touched_paths_json ?? "[]").filter(
68
+ (p): p is string => typeof p === "string",
69
+ ),
70
+ attributes: parseJsonRecord(row.attributes_json ?? "{}"),
71
+ occurredAt: row.occurred_at,
72
+ receivedAt: row.received_at,
73
+ expiresAt: row.expires_at,
74
+ });
75
+
76
+ export async function recordChangeEvent(
77
+ db: SqlDb,
78
+ projectId: string,
79
+ input: ChangeEventInput,
80
+ ): Promise<ChangeEvent> {
81
+ const nowIso = new Date().toISOString();
82
+ const occurredAt = input.occurredAt || nowIso;
83
+ const environment = (input.environment || "default").trim().toLowerCase();
84
+ const namespace = (input.namespace || "default").trim();
85
+ const serviceName = input.serviceName?.trim() || null;
86
+
87
+ const canonicalNode = input.targetNodeId
88
+ ? {
89
+ id: input.targetNodeId,
90
+ environment,
91
+ kind: (input.kind === "feature_flag"
92
+ ? "feature_flag"
93
+ : "service") as OntologyNodeKind,
94
+ name: serviceName ?? input.targetNodeId,
95
+ namespaceOrEndpoint: namespace,
96
+ isScopedIdentity: true,
97
+ }
98
+ : input.kind === "feature_flag"
99
+ ? buildCanonicalNodeId({
100
+ kind: "feature_flag",
101
+ environment,
102
+ flagKey:
103
+ (typeof input.attributes?.flagKey === "string"
104
+ ? input.attributes.flagKey
105
+ : serviceName) ?? "flag",
106
+ })
107
+ : buildCanonicalNodeId({
108
+ kind: "service",
109
+ environment,
110
+ namespace,
111
+ serviceName: serviceName ?? "unknown-service",
112
+ });
113
+
114
+ const id = input.id?.trim() || `chg_${randomHex(10)}`;
115
+ const source: ChangeEventSource = input.source ?? "explicit_api";
116
+ const certainty: RelationshipCertainty =
117
+ input.relationshipCertainty ??
118
+ (source === "inferred_telemetry" ? "inferred" : "exact");
119
+ const touchedPaths = (input.touchedPaths ?? []).slice(0, 500);
120
+ const expiresAt = new Date(
121
+ Date.parse(occurredAt) + 30 * 24 * 3600_000,
122
+ ).toISOString();
123
+
124
+ await db.batch([
125
+ db
126
+ .prepare(
127
+ `INSERT INTO ontology_nodes (
128
+ project_id, environment, id, kind, name, namespace_or_endpoint,
129
+ is_scoped_identity, current_version, current_commit_sha,
130
+ current_config_digest, attributes_json, first_seen_at, last_seen_at
131
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
132
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
133
+ current_version = COALESCE(excluded.current_version, ontology_nodes.current_version),
134
+ current_commit_sha = COALESCE(excluded.current_commit_sha, ontology_nodes.current_commit_sha),
135
+ current_config_digest = COALESCE(excluded.current_config_digest, ontology_nodes.current_config_digest),
136
+ last_seen_at = excluded.last_seen_at`,
137
+ )
138
+ .bind(
139
+ projectId,
140
+ environment,
141
+ canonicalNode.id,
142
+ canonicalNode.kind,
143
+ canonicalNode.name,
144
+ canonicalNode.namespaceOrEndpoint,
145
+ canonicalNode.isScopedIdentity ? 1 : 0,
146
+ input.version ?? null,
147
+ input.commitSha ?? null,
148
+ input.configDigest ?? null,
149
+ occurredAt,
150
+ nowIso,
151
+ ),
152
+ db
153
+ .prepare(
154
+ `INSERT INTO change_events (
155
+ project_id, environment, id, target_node_id, kind, source,
156
+ relationship_certainty, summary, service_name, version,
157
+ previous_version, commit_sha, previous_commit_sha, config_digest,
158
+ repository_url, pull_request_url, actor_id, caused_by_action_id,
159
+ agent_run_id, touched_paths_json, attributes_json,
160
+ occurred_at, received_at, expires_at
161
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
162
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
163
+ summary = excluded.summary,
164
+ version = COALESCE(excluded.version, change_events.version),
165
+ previous_version = COALESCE(excluded.previous_version, change_events.previous_version),
166
+ commit_sha = COALESCE(excluded.commit_sha, change_events.commit_sha),
167
+ touched_paths_json = excluded.touched_paths_json,
168
+ attributes_json = excluded.attributes_json,
169
+ occurred_at = excluded.occurred_at`,
170
+ )
171
+ .bind(
172
+ projectId,
173
+ environment,
174
+ id,
175
+ canonicalNode.id,
176
+ input.kind,
177
+ source,
178
+ certainty,
179
+ input.summary,
180
+ serviceName,
181
+ input.version ?? null,
182
+ input.previousVersion ?? null,
183
+ input.commitSha ?? null,
184
+ input.previousCommitSha ?? null,
185
+ input.configDigest ?? null,
186
+ input.repositoryUrl ?? null,
187
+ input.pullRequestUrl ?? null,
188
+ input.actorId ?? null,
189
+ input.causedByActionId ?? null,
190
+ input.agentRunId ?? null,
191
+ JSON.stringify(touchedPaths),
192
+ JSON.stringify(input.attributes ?? {}),
193
+ occurredAt,
194
+ nowIso,
195
+ expiresAt,
196
+ ),
197
+ ]);
198
+
199
+ return {
200
+ id,
201
+ projectId,
202
+ environment,
203
+ targetNodeId: canonicalNode.id,
204
+ kind: input.kind,
205
+ source,
206
+ relationshipCertainty: certainty,
207
+ summary: input.summary,
208
+ serviceName,
209
+ version: input.version ?? null,
210
+ previousVersion: input.previousVersion ?? null,
211
+ commitSha: input.commitSha ?? null,
212
+ previousCommitSha: input.previousCommitSha ?? null,
213
+ configDigest: input.configDigest ?? null,
214
+ repositoryUrl: input.repositoryUrl ?? null,
215
+ pullRequestUrl: input.pullRequestUrl ?? null,
216
+ actorId: input.actorId ?? null,
217
+ causedByActionId: input.causedByActionId ?? null,
218
+ agentRunId: input.agentRunId ?? null,
219
+ touchedPaths,
220
+ attributes: input.attributes ?? {},
221
+ occurredAt,
222
+ receivedAt: nowIso,
223
+ expiresAt,
224
+ };
225
+ }
226
+
227
+ export async function listChangeEvents(
228
+ db: SqlDb,
229
+ options: {
230
+ projectId: string;
231
+ environment?: string;
232
+ serviceName?: string;
233
+ targetNodeId?: string;
234
+ kind?: string;
235
+ since?: string;
236
+ limit?: number;
237
+ },
238
+ ): Promise<ChangeEvent[]> {
239
+ const conditions: string[] = ["project_id = ?"];
240
+ const binds: unknown[] = [options.projectId];
241
+
242
+ if (options.environment) {
243
+ conditions.push("environment = ?");
244
+ binds.push(options.environment.toLowerCase());
245
+ }
246
+ if (options.serviceName) {
247
+ conditions.push("service_name = ?");
248
+ binds.push(options.serviceName);
249
+ }
250
+ if (options.targetNodeId) {
251
+ conditions.push("target_node_id = ?");
252
+ binds.push(options.targetNodeId);
253
+ }
254
+ if (options.kind) {
255
+ conditions.push("kind = ?");
256
+ binds.push(options.kind);
257
+ }
258
+ if (options.since) {
259
+ conditions.push("occurred_at >= ?");
260
+ binds.push(options.since);
261
+ }
262
+
263
+ const limit = Math.max(1, Math.min(200, options.limit ?? 50));
264
+ binds.push(limit);
265
+
266
+ const rows = await db
267
+ .prepare(
268
+ `SELECT * FROM change_events
269
+ WHERE ${conditions.join(" AND ")}
270
+ ORDER BY occurred_at DESC
271
+ LIMIT ?`,
272
+ )
273
+ .bind(...binds)
274
+ .all<ChangeEventRow>();
275
+
276
+ return rows.results.map(mapChangeEventRow);
277
+ }
278
+
279
+ export async function getChangeEvent(
280
+ db: SqlDb,
281
+ projectId: string,
282
+ changeId: string,
283
+ environment?: string,
284
+ ): Promise<ChangeEvent | null> {
285
+ const row = environment
286
+ ? await db
287
+ .prepare(
288
+ `SELECT * FROM change_events
289
+ WHERE project_id = ? AND environment = ? AND id = ?`,
290
+ )
291
+ .bind(projectId, environment.toLowerCase(), changeId)
292
+ .first<ChangeEventRow>()
293
+ : await db
294
+ .prepare(
295
+ `SELECT * FROM change_events
296
+ WHERE project_id = ? AND id = ?
297
+ ORDER BY occurred_at DESC
298
+ LIMIT 1`,
299
+ )
300
+ .bind(projectId, changeId)
301
+ .first<ChangeEventRow>();
302
+
303
+ return row ? mapChangeEventRow(row) : null;
304
+ }
305
+
306
+ interface SpanCohortSampleRow {
307
+ trace_id: string;
308
+ span_id: string;
309
+ parent_span_id: string | null;
310
+ span_kind: number;
311
+ status_code: number;
312
+ duration_ms: number;
313
+ service_version: string | null;
314
+ target_node_id: string | null;
315
+ attributes_json: string | null;
316
+ events_json: string | null;
317
+ start_time: string;
318
+ end_time: string;
319
+ received_at: string;
320
+ }
321
+
322
+ const computePercentile = (sorted: number[], p: number): number | null => {
323
+ if (sorted.length === 0) return null;
324
+ const idx = Math.min(
325
+ sorted.length - 1,
326
+ Math.max(0, Math.ceil((p / 100) * sorted.length) - 1),
327
+ );
328
+ return sorted[idx];
329
+ };
330
+
331
+ const summarizeCohort = (
332
+ cohortKey: string,
333
+ requestSpans: SpanCohortSampleRow[],
334
+ downstreamSpans: SpanCohortSampleRow[],
335
+ ): CohortMetricsSummary => {
336
+ const requestCount = requestSpans.length;
337
+ const errorCount = requestSpans.filter((s) => s.status_code === 2).length;
338
+ const errorRate = requestCount > 0 ? errorCount / requestCount : 0;
339
+ const durations = requestSpans
340
+ .map((s) => Number(s.duration_ms ?? 0))
341
+ .sort((a, b) => a - b);
342
+ const p95DurationMs = computePercentile(durations, 95);
343
+
344
+ const downstreamCount = downstreamSpans.length;
345
+ const avgDownstreamCallsPerRequest =
346
+ requestCount > 0 ? downstreamCount / requestCount : 0;
347
+ const avgDownstreamDurationMs =
348
+ downstreamCount > 0
349
+ ? downstreamSpans.reduce((acc, s) => acc + Number(s.duration_ms ?? 0), 0) /
350
+ downstreamCount
351
+ : null;
352
+
353
+ return {
354
+ cohortKey,
355
+ requestCount,
356
+ errorCount,
357
+ errorRate,
358
+ p95DurationMs,
359
+ avgDownstreamCallsPerRequest,
360
+ avgDownstreamDurationMs,
361
+ };
362
+ };
363
+
364
+ /**
365
+ * Indexed cross-trace contention lookup (RFC 0013 § 4.B).
366
+ * Uses `idx_spans_contention_lookup` (`project_id, environment, target_node_id, start_time, end_time`).
367
+ * Enforces P1 #5: if the target node is unscoped (`is_scoped_identity = 0`), caps
368
+ * `relationshipCertainty` at `"inferred"`.
369
+ */
370
+ export async function findCrossTraceContention(
371
+ db: SqlDb,
372
+ options: {
373
+ projectId: string;
374
+ environment: string;
375
+ targetNodeId: string;
376
+ excludeTraceId: string;
377
+ startTime: string;
378
+ endTime: string;
379
+ limit?: number;
380
+ },
381
+ ): Promise<ContentionCandidate[]> {
382
+ const env = options.environment.toLowerCase();
383
+ const limit = Math.max(1, Math.min(20, options.limit ?? 5));
384
+
385
+ const [nodeRow, contentionRows] = await Promise.all([
386
+ db
387
+ .prepare(
388
+ `SELECT is_scoped_identity FROM ontology_nodes
389
+ WHERE project_id = ? AND environment = ? AND id = ?`,
390
+ )
391
+ .bind(options.projectId, env, options.targetNodeId)
392
+ .first<{ is_scoped_identity: number }>(),
393
+ db
394
+ .prepare(
395
+ `SELECT
396
+ trace_id,
397
+ span_id,
398
+ service_name,
399
+ service_version,
400
+ span_name,
401
+ COUNT(*) AS concurrent_calls,
402
+ MAX(duration_ms) AS max_duration_ms,
403
+ SUM(duration_ms) AS total_resource_time_ms
404
+ FROM telemetry_spans
405
+ WHERE project_id = ?
406
+ AND environment = ?
407
+ AND target_node_id = ?
408
+ AND trace_id <> ?
409
+ AND start_time <= ?
410
+ AND end_time >= ?
411
+ GROUP BY trace_id, span_id, service_name, service_version, span_name
412
+ ORDER BY total_resource_time_ms DESC
413
+ LIMIT ?`,
414
+ )
415
+ .bind(
416
+ options.projectId,
417
+ env,
418
+ options.targetNodeId,
419
+ options.excludeTraceId,
420
+ options.endTime,
421
+ options.startTime,
422
+ limit,
423
+ )
424
+ .all<{
425
+ trace_id: string;
426
+ span_id: string;
427
+ service_name: string | null;
428
+ service_version: string | null;
429
+ span_name: string;
430
+ concurrent_calls: number;
431
+ max_duration_ms: number;
432
+ total_resource_time_ms: number;
433
+ }>(),
434
+ ]);
435
+
436
+ const isScoped = nodeRow ? nodeRow.is_scoped_identity === 1 : !options.targetNodeId.includes(":unscoped/");
437
+ const relationshipCertainty: RelationshipCertainty = isScoped
438
+ ? "exact"
439
+ : "inferred";
440
+
441
+ return contentionRows.results.map((r) => ({
442
+ aggressorTraceId: r.trace_id,
443
+ aggressorSpanId: r.span_id,
444
+ aggressorServiceName: r.service_name,
445
+ aggressorServiceVersion: r.service_version,
446
+ aggressorSpanName: r.span_name,
447
+ sharedTargetNodeId: options.targetNodeId,
448
+ relationshipCertainty,
449
+ concurrentCalls: Number(r.concurrent_calls ?? 1),
450
+ maxDurationMs: Number(r.max_duration_ms ?? 0),
451
+ totalResourceTimeMs: Number(r.total_resource_time_ms ?? 0),
452
+ }));
453
+ }
454
+
455
+ const extractFilePathsFromSpan = (span: SpanCohortSampleRow): string[] => {
456
+ const found: string[] = [];
457
+ const attrs = parseJsonRecord(span.attributes_json ?? "{}");
458
+ for (const key of ["code.filepath", "obs.code.filepath", "exception.stacktrace"]) {
459
+ const val = attrs[key];
460
+ if (typeof val === "string") {
461
+ found.push(val);
462
+ }
463
+ }
464
+ if (span.events_json) {
465
+ const events = parseJsonArray(span.events_json);
466
+ for (const ev of events) {
467
+ if (ev && typeof ev === "object" && !Array.isArray(ev)) {
468
+ const evAttrs = parseJsonRecord(
469
+ JSON.stringify((ev as Record<string, unknown>).attributes ?? {}),
470
+ );
471
+ for (const key of ["code.filepath", "exception.stacktrace"]) {
472
+ if (typeof evAttrs[key] === "string") {
473
+ found.push(evAttrs[key] as string);
474
+ }
475
+ }
476
+ }
477
+ }
478
+ }
479
+ return found;
480
+ };
481
+
482
+ /**
483
+ * Computes traffic-normalized cohort rate lift, minimum sample gates, shared-degradation
484
+ * noisy-neighbor contention checks, and code-path overlap for a `change_event`
485
+ * (RFC 0013 § 4.A, P1 #1, P1 #2).
486
+ */
487
+ export async function computeChangeBlastRadius(
488
+ db: SqlDb,
489
+ projectId: string,
490
+ changeId: string,
491
+ options: {
492
+ environment?: string;
493
+ minComparableSample?: number;
494
+ } = {},
495
+ ): Promise<ChangeBlastRadiusResult | null> {
496
+ const change = await getChangeEvent(
497
+ db,
498
+ projectId,
499
+ changeId,
500
+ options.environment,
501
+ );
502
+ if (!change) return null;
503
+
504
+ const minSample = options.minComparableSample ?? 50;
505
+ const preWindowStart = new Date(
506
+ Date.parse(change.occurredAt) - 30 * 60_000,
507
+ ).toISOString();
508
+
509
+ // Load spans for the service across [occurredAt - 30m, now] using idx_spans_cohort_attribution
510
+ const serviceFilter = change.serviceName;
511
+ const spansResult = serviceFilter
512
+ ? await db
513
+ .prepare(
514
+ `SELECT
515
+ trace_id, span_id, parent_span_id, span_kind, status_code,
516
+ duration_ms, service_version, target_node_id, attributes_json,
517
+ events_json, start_time, end_time, received_at
518
+ FROM telemetry_spans
519
+ WHERE project_id = ?
520
+ AND environment = ?
521
+ AND service_name = ?
522
+ AND received_at >= ?
523
+ ORDER BY received_at DESC
524
+ LIMIT 5000`,
525
+ )
526
+ .bind(projectId, change.environment, serviceFilter, preWindowStart)
527
+ .all<SpanCohortSampleRow>()
528
+ : await db
529
+ .prepare(
530
+ `SELECT
531
+ trace_id, span_id, parent_span_id, span_kind, status_code,
532
+ duration_ms, service_version, target_node_id, attributes_json,
533
+ events_json, start_time, end_time, received_at
534
+ FROM telemetry_spans
535
+ WHERE project_id = ?
536
+ AND environment = ?
537
+ AND received_at >= ?
538
+ ORDER BY received_at DESC
539
+ LIMIT 5000`,
540
+ )
541
+ .bind(projectId, change.environment, preWindowStart)
542
+ .all<SpanCohortSampleRow>();
543
+
544
+ const allSpans = spansResult.results;
545
+ const postSpans = allSpans.filter(
546
+ (s) => (s.start_time || s.received_at) >= change.occurredAt,
547
+ );
548
+ const preSpans = allSpans.filter(
549
+ (s) => (s.start_time || s.received_at) < change.occurredAt,
550
+ );
551
+
552
+ const isRootOrServer = (s: SpanCohortSampleRow) =>
553
+ s.parent_span_id === null || s.span_kind === 1;
554
+ const isDownstreamCall = (s: SpanCohortSampleRow) =>
555
+ Boolean(s.target_node_id) || s.span_kind === 2 || s.span_kind === 4;
556
+
557
+ const flagKey =
558
+ change.kind === "feature_flag"
559
+ ? (typeof change.attributes.flagKey === "string"
560
+ ? change.attributes.flagKey
561
+ : change.serviceName) ?? "flag"
562
+ : null;
563
+
564
+ const newCohortKey =
565
+ change.version ??
566
+ (typeof change.attributes.newVariant === "string"
567
+ ? change.attributes.newVariant
568
+ : "true");
569
+ const oldCohortKey =
570
+ change.previousVersion ??
571
+ (typeof change.attributes.previousVariant === "string"
572
+ ? change.attributes.previousVariant
573
+ : "false");
574
+
575
+ const getCohortOfSpan = (s: SpanCohortSampleRow): string | null => {
576
+ if (flagKey) {
577
+ const attrs = parseJsonRecord(s.attributes_json ?? "{}");
578
+ const direct = attrs[`obs.feature_flags.${flagKey}`];
579
+ if (direct !== undefined && direct !== null) return String(direct);
580
+ if (attrs["feature_flag.key"] === flagKey && attrs["feature_flag.variant"] !== undefined) {
581
+ return String(attrs["feature_flag.variant"]);
582
+ }
583
+ const nested = attrs["obs.feature_flags"];
584
+ if (nested && typeof nested === "object" && !Array.isArray(nested)) {
585
+ const v = (nested as Record<string, unknown>)[flagKey];
586
+ if (v !== undefined && v !== null) return String(v);
587
+ }
588
+ return null;
589
+ }
590
+ return s.service_version;
591
+ };
592
+
593
+ const postNewReq = postSpans.filter(
594
+ (s) => isRootOrServer(s) && getCohortOfSpan(s) === newCohortKey,
595
+ );
596
+ const postNewDown = postSpans.filter(
597
+ (s) => isDownstreamCall(s) && getCohortOfSpan(s) === newCohortKey,
598
+ );
599
+
600
+ const postOldReq = postSpans.filter(
601
+ (s) =>
602
+ isRootOrServer(s) &&
603
+ (oldCohortKey
604
+ ? getCohortOfSpan(s) === oldCohortKey
605
+ : getCohortOfSpan(s) !== null && getCohortOfSpan(s) !== newCohortKey),
606
+ );
607
+ const postOldDown = postSpans.filter(
608
+ (s) =>
609
+ isDownstreamCall(s) &&
610
+ (oldCohortKey
611
+ ? getCohortOfSpan(s) === oldCohortKey
612
+ : getCohortOfSpan(s) !== null && getCohortOfSpan(s) !== newCohortKey),
613
+ );
614
+
615
+ const preOldReq = preSpans.filter(
616
+ (s) =>
617
+ isRootOrServer(s) &&
618
+ (!oldCohortKey || getCohortOfSpan(s) === oldCohortKey),
619
+ );
620
+ const preOldDown = preSpans.filter(
621
+ (s) =>
622
+ isDownstreamCall(s) &&
623
+ (!oldCohortKey || getCohortOfSpan(s) === oldCohortKey),
624
+ );
625
+
626
+ const newCohort = summarizeCohort(newCohortKey, postNewReq, postNewDown);
627
+ const oldCohort = summarizeCohort(oldCohortKey, postOldReq, postOldDown);
628
+ const preBaseline = summarizeCohort(
629
+ `${oldCohortKey}:pre`,
630
+ preOldReq,
631
+ preOldDown,
632
+ );
633
+
634
+ // Check code-path overlap between change.touchedPaths and error spans in new cohort
635
+ const overlappingSet = new Set<string>();
636
+ const matchingErrorSpanIds: string[] = [];
637
+ if (change.touchedPaths.length > 0) {
638
+ for (const s of postNewReq.filter((sp) => sp.status_code === 2)) {
639
+ const rawPaths = extractFilePathsFromSpan(s);
640
+ let matchedSpan = false;
641
+ for (const touched of change.touchedPaths) {
642
+ if (rawPaths.some((raw) => raw.includes(touched))) {
643
+ overlappingSet.add(touched);
644
+ matchedSpan = true;
645
+ }
646
+ }
647
+ if (matchedSpan) {
648
+ matchingErrorSpanIds.push(s.span_id);
649
+ }
650
+ }
651
+ }
652
+ const overlappingCodePaths = Array.from(overlappingSet);
653
+
654
+ // Evaluate Traffic-Normalized Cohort Verdict (P1 #2)
655
+ let cohortVerdict: CohortAttributionVerdict = "insufficient_sample";
656
+ let causalProofTier: CausalProofTier = "unverified_locus";
657
+ let rateDifference = 0;
658
+ let relativeRisk = 1;
659
+ const sharedResourceAggressors: ContentionCandidate[] = [];
660
+
661
+ if (newCohort.requestCount < minSample) {
662
+ cohortVerdict = "insufficient_sample";
663
+ causalProofTier =
664
+ overlappingCodePaths.length > 0
665
+ ? "corroborated_candidate"
666
+ : "unverified_locus";
667
+ } else if (oldCohort.requestCount >= minSample) {
668
+ rateDifference = newCohort.errorRate - oldCohort.errorRate;
669
+ relativeRisk = newCohort.errorRate / Math.max(oldCohort.errorRate, 0.001);
670
+
671
+ const fanoutRatio =
672
+ newCohort.avgDownstreamCallsPerRequest /
673
+ Math.max(oldCohort.avgDownstreamCallsPerRequest, 0.25);
674
+ const durationRatio =
675
+ (newCohort.avgDownstreamDurationMs ?? 0) /
676
+ Math.max(oldCohort.avgDownstreamDurationMs ?? 1, 1);
677
+ const hasNewVersionAggressorSignal =
678
+ fanoutRatio >= 3.0 || durationRatio >= 3.0;
679
+ const hasPreChangeDegradationSignal =
680
+ preBaseline.requestCount >= 10 &&
681
+ oldCohort.errorRate > 1.5 * Math.max(preBaseline.errorRate, 0.01);
682
+
683
+ const bothDegraded =
684
+ newCohort.errorRate >= 0.05 &&
685
+ oldCohort.errorRate >= 0.05 &&
686
+ (hasNewVersionAggressorSignal || hasPreChangeDegradationSignal);
687
+
688
+ if (bothDegraded) {
689
+ if (hasNewVersionAggressorSignal) {
690
+ cohortVerdict = "shared_degradation_new_version_aggressor";
691
+ causalProofTier = "corroborated_candidate";
692
+ // Populate top contention aggressors from postNewDown
693
+ const sampleTarget = postNewDown.find(
694
+ (d) => d.target_node_id,
695
+ )?.target_node_id;
696
+ if (sampleTarget) {
697
+ const victimTrace =
698
+ postOldReq.find((r) => r.status_code === 2)?.trace_id ?? "victim";
699
+ const candidates = await findCrossTraceContention(db, {
700
+ projectId,
701
+ environment: change.environment,
702
+ targetNodeId: sampleTarget,
703
+ excludeTraceId: victimTrace,
704
+ startTime: change.occurredAt,
705
+ endTime: new Date().toISOString(),
706
+ });
707
+ sharedResourceAggressors.push(...candidates);
708
+ }
709
+ } else {
710
+ cohortVerdict = "shared_degradation_external_cause";
711
+ causalProofTier = "unverified_locus";
712
+ }
713
+ } else {
714
+ const p95Ratio =
715
+ newCohort.p95DurationMs !== null &&
716
+ oldCohort.p95DurationMs !== null &&
717
+ oldCohort.p95DurationMs > 0
718
+ ? newCohort.p95DurationMs / oldCohort.p95DurationMs
719
+ : 1;
720
+
721
+ if (
722
+ (rateDifference >= 0.05 || p95Ratio >= 2.5) &&
723
+ (relativeRisk >= 5.0 || p95Ratio >= 2.5)
724
+ ) {
725
+ cohortVerdict = "isolated_to_new_version";
726
+ causalProofTier = "cohort_rate_isolated";
727
+ } else {
728
+ cohortVerdict = "no_regression_detected";
729
+ causalProofTier = "unverified_locus";
730
+ }
731
+ }
732
+ } else if (preBaseline.requestCount >= minSample) {
733
+ // Instant 100% cutover: compare post-change v_new vs pre-change baseline
734
+ rateDifference = newCohort.errorRate - preBaseline.errorRate;
735
+ relativeRisk =
736
+ newCohort.errorRate / Math.max(preBaseline.errorRate, 0.001);
737
+ const p95Ratio =
738
+ newCohort.p95DurationMs !== null &&
739
+ preBaseline.p95DurationMs !== null &&
740
+ preBaseline.p95DurationMs > 0
741
+ ? newCohort.p95DurationMs / preBaseline.p95DurationMs
742
+ : 1;
743
+
744
+ if (
745
+ (rateDifference >= 0.05 || p95Ratio >= 2.5) &&
746
+ (relativeRisk >= 5.0 || p95Ratio >= 2.5)
747
+ ) {
748
+ cohortVerdict = "sequential_post_cutover_regression";
749
+ causalProofTier = "corroborated_candidate";
750
+ } else {
751
+ cohortVerdict = "no_regression_detected";
752
+ causalProofTier = "unverified_locus";
753
+ }
754
+ } else if (overlappingCodePaths.length > 0) {
755
+ cohortVerdict = "insufficient_sample";
756
+ causalProofTier = "corroborated_candidate";
757
+ }
758
+
759
+ const relationshipCertainty: RelationshipCertainty =
760
+ overlappingCodePaths.length > 0 ? "exact" : change.relationshipCertainty;
761
+
762
+ const summary = `Change ${change.id} (${change.kind} ${newCohortKey} in ${change.environment}): verdict=${cohortVerdict}, proofTier=${causalProofTier}, newErrorRate=${(newCohort.errorRate * 100).toFixed(1)}% (N=${newCohort.requestCount}), oldErrorRate=${(oldCohort.errorRate * 100).toFixed(1)}% (N=${oldCohort.requestCount})`;
763
+
764
+ return {
765
+ change,
766
+ cohortVerdict,
767
+ causalProofTier,
768
+ relationshipCertainty,
769
+ minComparableSample: minSample,
770
+ newCohort,
771
+ oldCohort,
772
+ preChangeBaselineCohort: preBaseline,
773
+ rateDifference,
774
+ relativeRisk,
775
+ overlappingCodePaths,
776
+ matchingErrorSpanIds,
777
+ sharedResourceAggressors,
778
+ summary,
779
+ };
780
+ }