@obsunified/collector 2.1.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
  2. package/dist/chunk-ZSF5XP3B.js.map +1 -0
  3. package/dist/index.d.ts +309 -2
  4. package/dist/index.js +4159 -256
  5. package/dist/index.js.map +1 -1
  6. package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
  7. package/package.json +2 -2
  8. package/src/__tests__/autosre-closed-loop.test.ts +1275 -0
  9. package/src/index.ts +43 -0
  10. package/src/lib/analyses-runner.ts +9 -0
  11. package/src/lib/autosre-change-correlation.ts +780 -0
  12. package/src/lib/autosre-findings.ts +725 -0
  13. package/src/lib/autosre-ontology.ts +914 -0
  14. package/src/lib/autosre-remediation-gate.ts +1663 -0
  15. package/src/lib/eval-cases-store.ts +63 -1
  16. package/src/lib/evidence-references.ts +6 -0
  17. package/src/lib/housekeeping.test.ts +80 -0
  18. package/src/lib/housekeeping.ts +101 -1
  19. package/src/lib/store/ingest.ts +94 -6
  20. package/src/migrations/050_autosre_causal_ontology.sql +216 -0
  21. package/src/migrations/alert-state-notified.test.ts +3 -3
  22. package/src/migrations/parity.test.ts +4 -0
  23. package/src/migrations/project-scoped-keys.test.ts +1 -1
  24. package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
  25. package/src/plugins/alerts-evaluator.ts +17 -1
  26. package/src/plugins/autosre-routes.ts +1019 -0
  27. package/src/plugins/connected-routes/manifest.ts +13 -1
  28. package/src/plugins/connected-routes.ts +271 -0
  29. package/src/plugins/default-span-enrichment.ts +13 -1
  30. package/src/plugins/eval-cases-routes.ts +57 -2
  31. package/src/plugins/evidence-retrieval-routes.ts +300 -4
  32. package/dist/chunk-XKQTVIRE.js.map +0 -1
  33. /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
@@ -0,0 +1,914 @@
1
+ import type {
2
+ CanonicalNodeIdentity,
3
+ CausalTriple,
4
+ JsonValue,
5
+ OntologyEdge,
6
+ OntologyGraphResponse,
7
+ OntologyNode,
8
+ OntologyNodeKind,
9
+ RelationshipCertainty,
10
+ StoredSpan,
11
+ } from "@obsunified/types";
12
+ import { buildCanonicalNodeId, normalizeErrorSignature } from "@obsunified/types";
13
+ import { randomHex } from "./hash";
14
+ import { parseJsonArray, parseJsonRecord } from "./json";
15
+ import type { SqlDb, SqlStatement } from "./sql-db";
16
+
17
+ export interface ExtractedSpanLineage {
18
+ environment: string;
19
+ namespace: string;
20
+ serviceName: string | null;
21
+ serviceVersion: string | null;
22
+ commitSha: string | null;
23
+ configDigest: string | null;
24
+ sourceNode: CanonicalNodeIdentity | null;
25
+ targetNode: CanonicalNodeIdentity | null;
26
+ edgePredicate: string | null;
27
+ errorSignature: string | null;
28
+ }
29
+
30
+ const pickStr = (
31
+ record: Record<string, JsonValue>,
32
+ ...keys: string[]
33
+ ): string | null => {
34
+ for (const key of keys) {
35
+ const val = record[key];
36
+ if (typeof val === "string" && val.trim().length > 0) {
37
+ return val.trim();
38
+ }
39
+ if (typeof val === "number") {
40
+ return String(val);
41
+ }
42
+ }
43
+ return null;
44
+ };
45
+
46
+ /**
47
+ * Extracts runtime lineage (`environment`, `serviceVersion`, `commitSha`, `configDigest`),
48
+ * canonical source/target `OntologyNode` identities, and `errorSignature` from a span.
49
+ */
50
+ export function extractSpanLineageAndTopology(
51
+ span: StoredSpan,
52
+ ): ExtractedSpanLineage {
53
+ const attrs = parseJsonRecord(span.attributesJson);
54
+ const resource = parseJsonRecord(span.resourceAttributesJson);
55
+
56
+ const environment = (
57
+ span.environment ??
58
+ pickStr(
59
+ resource,
60
+ "deployment.environment.name",
61
+ "deployment.environment",
62
+ "obs.environment",
63
+ ) ??
64
+ pickStr(
65
+ attrs,
66
+ "deployment.environment.name",
67
+ "deployment.environment",
68
+ "obs.environment",
69
+ ) ??
70
+ "default"
71
+ ).toLowerCase();
72
+
73
+ const namespace =
74
+ pickStr(resource, "service.namespace") ??
75
+ pickStr(attrs, "service.namespace") ??
76
+ "default";
77
+
78
+ const serviceName =
79
+ span.serviceName ??
80
+ pickStr(resource, "service.name") ??
81
+ pickStr(attrs, "service.name");
82
+
83
+ const serviceVersion =
84
+ span.serviceVersion ??
85
+ pickStr(resource, "service.version", "obs.service.version") ??
86
+ pickStr(attrs, "service.version", "obs.service.version");
87
+
88
+ const commitSha =
89
+ span.commitSha ??
90
+ pickStr(
91
+ resource,
92
+ "vcs.commit.id",
93
+ "git.commit.sha",
94
+ "obs.commit.sha",
95
+ "service.commit.sha",
96
+ ) ??
97
+ pickStr(attrs, "vcs.commit.id", "git.commit.sha", "obs.commit.sha");
98
+
99
+ const configDigest =
100
+ pickStr(resource, "obs.config.digest") ??
101
+ pickStr(attrs, "obs.config.digest");
102
+
103
+ const sourceNode = serviceName
104
+ ? buildCanonicalNodeId({
105
+ kind: "service",
106
+ environment,
107
+ namespace,
108
+ serviceName,
109
+ })
110
+ : null;
111
+
112
+ // Determine downstream target node for DB, cache, queue, external API, or peer service calls
113
+ let targetNode: CanonicalNodeIdentity | null = null;
114
+ let edgePredicate: string | null = null;
115
+
116
+ const dbSystem = pickStr(attrs, "db.system");
117
+ const messagingSystem = pickStr(attrs, "messaging.system");
118
+ const peerService = pickStr(attrs, "peer.service", "obs.target.service");
119
+ const serverAddress = pickStr(
120
+ attrs,
121
+ "server.address",
122
+ "net.peer.name",
123
+ "db.connection_string",
124
+ );
125
+ const serverPort = pickStr(attrs, "server.port", "net.peer.port");
126
+ const toolName = pickStr(attrs, "obs.tool.name", "tool.name");
127
+
128
+ if (dbSystem) {
129
+ const isCache =
130
+ dbSystem.toLowerCase() === "redis" ||
131
+ dbSystem.toLowerCase() === "memcached";
132
+ targetNode = buildCanonicalNodeId({
133
+ kind: isCache ? "cache" : "database",
134
+ environment,
135
+ dbSystem,
136
+ dbName: pickStr(attrs, "db.namespace", "db.name") ?? dbSystem,
137
+ serverAddress,
138
+ serverPort,
139
+ });
140
+ edgePredicate = isCache ? "queries_cache" : "queries_database";
141
+ } else if (messagingSystem) {
142
+ targetNode = buildCanonicalNodeId({
143
+ kind: "queue_topic",
144
+ environment,
145
+ messagingSystem,
146
+ messagingDestination:
147
+ pickStr(
148
+ attrs,
149
+ "messaging.destination.name",
150
+ "messaging.destination",
151
+ ) ?? "default",
152
+ serverAddress,
153
+ });
154
+ edgePredicate =
155
+ span.spanKind === 4 ? "publishes_to_queue" : "consumes_from_queue";
156
+ } else if (peerService) {
157
+ targetNode = buildCanonicalNodeId({
158
+ kind: "service",
159
+ environment,
160
+ namespace: pickStr(attrs, "peer.namespace") ?? namespace,
161
+ serviceName: peerService,
162
+ });
163
+ edgePredicate = "calls_service";
164
+ } else if (span.spanKind === 2 && serverAddress) {
165
+ // Outbound client span to external host or internal service
166
+ targetNode = buildCanonicalNodeId({
167
+ kind: "external_api",
168
+ environment,
169
+ serverAddress,
170
+ serverPort: serverPort ?? "443",
171
+ });
172
+ edgePredicate = "calls_external_api";
173
+ } else if (toolName) {
174
+ targetNode = buildCanonicalNodeId({
175
+ kind: "tool",
176
+ environment,
177
+ serviceName:
178
+ pickStr(attrs, "obs.agent.id", "agent.id") ?? serviceName ?? "default",
179
+ toolName,
180
+ });
181
+ edgePredicate = "invokes_tool";
182
+ }
183
+
184
+ // Extract normalized error signature when status_code === 2 (ERROR)
185
+ let errorSignature: string | null = span.errorSignature ?? null;
186
+ if (!errorSignature && span.statusCode === 2) {
187
+ let rawErr =
188
+ pickStr(
189
+ attrs,
190
+ "exception.type",
191
+ "error.type",
192
+ "obs.tool.error_type",
193
+ "http.response.status_code",
194
+ "http.status_code",
195
+ "rpc.grpc.status_code",
196
+ ) ?? span.statusMessage;
197
+
198
+ if (!rawErr && span.eventsJson) {
199
+ const events = parseJsonArray(span.eventsJson);
200
+ for (const ev of events) {
201
+ if (ev && typeof ev === "object" && !Array.isArray(ev)) {
202
+ const evObj = ev as Record<string, JsonValue>;
203
+ if (evObj.name === "exception" && evObj.attributes) {
204
+ const evAttrs =
205
+ typeof evObj.attributes === "object" &&
206
+ !Array.isArray(evObj.attributes)
207
+ ? (evObj.attributes as Record<string, JsonValue>)
208
+ : {};
209
+ rawErr =
210
+ pickStr(evAttrs, "exception.type", "exception.message") ??
211
+ rawErr;
212
+ }
213
+ }
214
+ }
215
+ }
216
+
217
+ errorSignature = normalizeErrorSignature(
218
+ rawErr ?? `ERROR:${span.spanName}`,
219
+ );
220
+ }
221
+
222
+ return {
223
+ environment,
224
+ namespace,
225
+ serviceName,
226
+ serviceVersion,
227
+ commitSha,
228
+ configDigest,
229
+ sourceNode,
230
+ targetNode,
231
+ edgePredicate,
232
+ errorSignature,
233
+ };
234
+ }
235
+
236
+ // 60-second isolate cache to avoid repeating identical ontology node upserts on every span
237
+ const NODE_UPSERT_CACHE = new Map<string, number>();
238
+ const CACHE_TTL_MS = 60_000;
239
+
240
+ export function clearOntologyUpsertCache(): void {
241
+ NODE_UPSERT_CACHE.clear();
242
+ }
243
+
244
+ /**
245
+ * Upserts environment-scoped `ontology_nodes`, `ontology_edges`, and inferred version-transition
246
+ * `change_events` from ingested spans.
247
+ */
248
+ export async function upsertOntologyFromSpans(
249
+ db: SqlDb,
250
+ spans: StoredSpan[],
251
+ options: { bypassCache?: boolean } = {},
252
+ ): Promise<void> {
253
+ if (spans.length === 0) return;
254
+
255
+ const nowMs = Date.now();
256
+ const statements: SqlStatement[] = [];
257
+ const edgeAggregates = new Map<
258
+ string,
259
+ {
260
+ projectId: string;
261
+ environment: string;
262
+ id: string;
263
+ sourceNodeId: string;
264
+ predicate: string;
265
+ targetNodeId: string;
266
+ certainty: RelationshipCertainty;
267
+ callCount: number;
268
+ errorCount: number;
269
+ lastDurationMs: number;
270
+ firstSeenAt: string;
271
+ lastSeenAt: string;
272
+ }
273
+ >();
274
+
275
+ // Track version transitions per service node
276
+ const serviceNodesToCheck = new Map<
277
+ string,
278
+ {
279
+ projectId: string;
280
+ environment: string;
281
+ node: CanonicalNodeIdentity;
282
+ version: string | null;
283
+ commitSha: string | null;
284
+ configDigest: string | null;
285
+ timestamp: string;
286
+ }
287
+ >();
288
+
289
+ for (const span of spans) {
290
+ const lineage = extractSpanLineageAndTopology(span);
291
+ const ts = span.receivedAt || span.startTime || new Date().toISOString();
292
+
293
+ if (lineage.sourceNode) {
294
+ const cacheKey = `${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}::${lineage.serviceVersion ?? ""}::${lineage.commitSha ?? ""}::${lineage.configDigest ?? ""}`;
295
+ const cachedAt = NODE_UPSERT_CACHE.get(cacheKey);
296
+ if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
297
+ NODE_UPSERT_CACHE.set(cacheKey, nowMs);
298
+ serviceNodesToCheck.set(
299
+ `${span.projectId}::${lineage.environment}::${lineage.sourceNode.id}`,
300
+ {
301
+ projectId: span.projectId,
302
+ environment: lineage.environment,
303
+ node: lineage.sourceNode,
304
+ version: lineage.serviceVersion,
305
+ commitSha: lineage.commitSha,
306
+ configDigest: lineage.configDigest,
307
+ timestamp: ts,
308
+ },
309
+ );
310
+ }
311
+ }
312
+
313
+ if (lineage.targetNode) {
314
+ const targetCacheKey = `${span.projectId}::${lineage.environment}::${lineage.targetNode.id}`;
315
+ const cachedAt = NODE_UPSERT_CACHE.get(targetCacheKey);
316
+ if (options.bypassCache || !cachedAt || nowMs - cachedAt > CACHE_TTL_MS) {
317
+ NODE_UPSERT_CACHE.set(targetCacheKey, nowMs);
318
+ statements.push(
319
+ db
320
+ .prepare(
321
+ `INSERT INTO ontology_nodes (
322
+ project_id, environment, id, kind, name, namespace_or_endpoint,
323
+ is_scoped_identity, attributes_json, first_seen_at, last_seen_at
324
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
325
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
326
+ last_seen_at = excluded.last_seen_at`,
327
+ )
328
+ .bind(
329
+ span.projectId,
330
+ lineage.environment,
331
+ lineage.targetNode.id,
332
+ lineage.targetNode.kind,
333
+ lineage.targetNode.name,
334
+ lineage.targetNode.namespaceOrEndpoint,
335
+ lineage.targetNode.isScopedIdentity ? 1 : 0,
336
+ ts,
337
+ ts,
338
+ ),
339
+ );
340
+ }
341
+ }
342
+
343
+ if (lineage.sourceNode && lineage.targetNode && lineage.edgePredicate) {
344
+ const edgeId = `${lineage.sourceNode.id}--${lineage.edgePredicate}-->${lineage.targetNode.id}`;
345
+ const aggKey = `${span.projectId}::${lineage.environment}::${edgeId}`;
346
+ const certainty: RelationshipCertainty =
347
+ lineage.sourceNode.isScopedIdentity &&
348
+ lineage.targetNode.isScopedIdentity
349
+ ? "exact"
350
+ : "inferred";
351
+ const existing = edgeAggregates.get(aggKey);
352
+ const isErr = span.statusCode === 2 ? 1 : 0;
353
+ if (existing) {
354
+ existing.callCount += 1;
355
+ existing.errorCount += isErr;
356
+ existing.lastDurationMs = span.durationMs;
357
+ if (ts > existing.lastSeenAt) existing.lastSeenAt = ts;
358
+ } else {
359
+ edgeAggregates.set(aggKey, {
360
+ projectId: span.projectId,
361
+ environment: lineage.environment,
362
+ id: edgeId,
363
+ sourceNodeId: lineage.sourceNode.id,
364
+ predicate: lineage.edgePredicate,
365
+ targetNodeId: lineage.targetNode.id,
366
+ certainty,
367
+ callCount: 1,
368
+ errorCount: isErr,
369
+ lastDurationMs: span.durationMs,
370
+ firstSeenAt: ts,
371
+ lastSeenAt: ts,
372
+ });
373
+ }
374
+ }
375
+ }
376
+
377
+ // Check version transitions for service nodes before upserting them
378
+ for (const entry of serviceNodesToCheck.values()) {
379
+ if (entry.version) {
380
+ const existingNode = await db
381
+ .prepare(
382
+ `SELECT current_version, current_commit_sha
383
+ FROM ontology_nodes
384
+ WHERE project_id = ? AND environment = ? AND id = ?`,
385
+ )
386
+ .bind(entry.projectId, entry.environment, entry.node.id)
387
+ .first<{
388
+ current_version: string | null;
389
+ current_commit_sha: string | null;
390
+ }>();
391
+
392
+ if (
393
+ existingNode?.current_version &&
394
+ existingNode.current_version !== entry.version
395
+ ) {
396
+ // Check if an explicit change_event already recorded this transition
397
+ const existingChange = await db
398
+ .prepare(
399
+ `SELECT id FROM change_events
400
+ WHERE project_id = ? AND environment = ? AND target_node_id = ? AND version = ?
401
+ LIMIT 1`,
402
+ )
403
+ .bind(
404
+ entry.projectId,
405
+ entry.environment,
406
+ entry.node.id,
407
+ entry.version,
408
+ )
409
+ .first<{ id: string }>();
410
+
411
+ if (!existingChange) {
412
+ const changeId = `chg_inf_${randomHex(8)}`;
413
+ const expiresAt = new Date(
414
+ Date.now() + 30 * 24 * 3600_000,
415
+ ).toISOString();
416
+ statements.push(
417
+ db
418
+ .prepare(
419
+ `INSERT INTO change_events (
420
+ project_id, environment, id, target_node_id, kind, source,
421
+ relationship_certainty, summary, service_name, version,
422
+ previous_version, commit_sha, previous_commit_sha, config_digest,
423
+ touched_paths_json, attributes_json, occurred_at, received_at, expires_at
424
+ ) VALUES (?, ?, ?, ?, 'deploy', 'inferred_telemetry', 'inferred', ?, ?, ?, ?, ?, ?, ?, '[]', '{}', ?, ?, ?)
425
+ ON CONFLICT (project_id, environment, id) DO NOTHING`,
426
+ )
427
+ .bind(
428
+ entry.projectId,
429
+ entry.environment,
430
+ changeId,
431
+ entry.node.id,
432
+ `Inferred version transition ${existingNode.current_version} -> ${entry.version} on ${entry.node.name}`,
433
+ entry.node.name,
434
+ entry.version,
435
+ existingNode.current_version,
436
+ entry.commitSha,
437
+ existingNode.current_commit_sha,
438
+ entry.configDigest,
439
+ entry.timestamp,
440
+ entry.timestamp,
441
+ expiresAt,
442
+ ),
443
+ );
444
+ }
445
+ }
446
+ }
447
+
448
+ statements.push(
449
+ db
450
+ .prepare(
451
+ `INSERT INTO ontology_nodes (
452
+ project_id, environment, id, kind, name, namespace_or_endpoint,
453
+ is_scoped_identity, current_version, current_commit_sha,
454
+ current_config_digest, attributes_json, first_seen_at, last_seen_at
455
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)
456
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
457
+ current_version = COALESCE(excluded.current_version, ontology_nodes.current_version),
458
+ current_commit_sha = COALESCE(excluded.current_commit_sha, ontology_nodes.current_commit_sha),
459
+ current_config_digest = COALESCE(excluded.current_config_digest, ontology_nodes.current_config_digest),
460
+ last_seen_at = excluded.last_seen_at`,
461
+ )
462
+ .bind(
463
+ entry.projectId,
464
+ entry.environment,
465
+ entry.node.id,
466
+ entry.node.kind,
467
+ entry.node.name,
468
+ entry.node.namespaceOrEndpoint,
469
+ entry.node.isScopedIdentity ? 1 : 0,
470
+ entry.version,
471
+ entry.commitSha,
472
+ entry.configDigest,
473
+ entry.timestamp,
474
+ entry.timestamp,
475
+ ),
476
+ );
477
+ }
478
+
479
+ for (const edge of edgeAggregates.values()) {
480
+ statements.push(
481
+ db
482
+ .prepare(
483
+ `INSERT INTO ontology_edges (
484
+ project_id, environment, id, source_node_id, predicate, target_node_id,
485
+ relationship_certainty, call_count, error_count, last_duration_ms,
486
+ first_seen_at, last_seen_at
487
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
488
+ ON CONFLICT (project_id, environment, id) DO UPDATE SET
489
+ call_count = ontology_edges.call_count + excluded.call_count,
490
+ error_count = ontology_edges.error_count + excluded.error_count,
491
+ last_duration_ms = excluded.last_duration_ms,
492
+ last_seen_at = excluded.last_seen_at`,
493
+ )
494
+ .bind(
495
+ edge.projectId,
496
+ edge.environment,
497
+ edge.id,
498
+ edge.sourceNodeId,
499
+ edge.predicate,
500
+ edge.targetNodeId,
501
+ edge.certainty,
502
+ edge.callCount,
503
+ edge.errorCount,
504
+ edge.lastDurationMs,
505
+ edge.firstSeenAt,
506
+ edge.lastSeenAt,
507
+ ),
508
+ );
509
+ }
510
+
511
+ if (statements.length > 0) {
512
+ await db.batch(statements);
513
+ }
514
+ }
515
+
516
+ /**
517
+ * Tarjan's O(V + E) Strongly Connected Components (SCC) condensation algorithm
518
+ * (RFC 0013 § 1.B & P2 #6). Ensures cyclic service/queue topologies never cause
519
+ * infinite loops during dependency traversal.
520
+ */
521
+ export function computeTarjanScc(
522
+ nodes: OntologyNode[],
523
+ edges: OntologyEdge[],
524
+ ): {
525
+ annotatedNodes: OntologyNode[];
526
+ components: Array<{ sccId: string; nodeIds: string[]; isCyclic: boolean }>;
527
+ } {
528
+ const adjacency = new Map<string, string[]>();
529
+ const selfLoops = new Set<string>();
530
+
531
+ for (const node of nodes) {
532
+ adjacency.set(node.id, []);
533
+ }
534
+ for (const edge of edges) {
535
+ if (!adjacency.has(edge.sourceNodeId)) {
536
+ adjacency.set(edge.sourceNodeId, []);
537
+ }
538
+ if (!adjacency.has(edge.targetNodeId)) {
539
+ adjacency.set(edge.targetNodeId, []);
540
+ }
541
+ adjacency.get(edge.sourceNodeId)?.push(edge.targetNodeId);
542
+ if (edge.sourceNodeId === edge.targetNodeId) {
543
+ selfLoops.add(edge.sourceNodeId);
544
+ }
545
+ }
546
+
547
+ let index = 0;
548
+ const indices = new Map<string, number>();
549
+ const lowlink = new Map<string, number>();
550
+ const onStack = new Set<string>();
551
+ const stack: string[] = [];
552
+ const rawComponents: string[][] = [];
553
+
554
+ const strongConnect = (nodeId: string) => {
555
+ indices.set(nodeId, index);
556
+ lowlink.set(nodeId, index);
557
+ index += 1;
558
+ stack.push(nodeId);
559
+ onStack.add(nodeId);
560
+
561
+ const neighbors = adjacency.get(nodeId) ?? [];
562
+ for (const targetId of neighbors) {
563
+ if (!indices.has(targetId)) {
564
+ strongConnect(targetId);
565
+ lowlink.set(
566
+ nodeId,
567
+ Math.min(lowlink.get(nodeId) ?? 0, lowlink.get(targetId) ?? 0),
568
+ );
569
+ } else if (onStack.has(targetId)) {
570
+ lowlink.set(
571
+ nodeId,
572
+ Math.min(lowlink.get(nodeId) ?? 0, indices.get(targetId) ?? 0),
573
+ );
574
+ }
575
+ }
576
+
577
+ if (lowlink.get(nodeId) === indices.get(nodeId)) {
578
+ const component: string[] = [];
579
+ while (stack.length > 0) {
580
+ const w = stack.pop()!;
581
+ onStack.delete(w);
582
+ component.push(w);
583
+ if (w === nodeId) break;
584
+ }
585
+ component.sort();
586
+ rawComponents.push(component);
587
+ }
588
+ };
589
+
590
+ for (const nodeId of adjacency.keys()) {
591
+ if (!indices.has(nodeId)) {
592
+ strongConnect(nodeId);
593
+ }
594
+ }
595
+
596
+ const nodeSccMap = new Map<string, { sccId: string; isCyclic: boolean }>();
597
+ const components = rawComponents.map((nodeIds, idx) => {
598
+ const sccId = `scc_${idx + 1}`;
599
+ const isCyclic =
600
+ nodeIds.length > 1 || (nodeIds.length === 1 && selfLoops.has(nodeIds[0]));
601
+ for (const id of nodeIds) {
602
+ nodeSccMap.set(id, { sccId, isCyclic });
603
+ }
604
+ return { sccId, nodeIds, isCyclic };
605
+ });
606
+
607
+ const annotatedNodes = nodes.map((n) => {
608
+ const scc = nodeSccMap.get(n.id);
609
+ return {
610
+ ...n,
611
+ sccId: scc?.sccId ?? "scc_0",
612
+ inCycle: scc?.isCyclic ?? false,
613
+ };
614
+ });
615
+
616
+ return { annotatedNodes, components };
617
+ }
618
+
619
+ export interface TraceSpanLocusInput {
620
+ span_id: string;
621
+ parent_span_id: string | null;
622
+ service_name: string | null;
623
+ span_name: string;
624
+ span_kind: number;
625
+ status_code: number;
626
+ status_message: string | null;
627
+ start_time: string;
628
+ end_time: string;
629
+ duration_ms: number;
630
+ environment?: string | null;
631
+ target_node_id?: string | null;
632
+ error_signature?: string | null;
633
+ }
634
+
635
+ export interface TraceFaultLocusResult {
636
+ faultLocusSpan: TraceSpanLocusInput | null;
637
+ faultLocusNodeId: string | null;
638
+ relationshipCertainty: RelationshipCertainty;
639
+ causalProofTier: "unverified_locus";
640
+ causalTriples: CausalTriple[];
641
+ }
642
+
643
+ /**
644
+ * Per-trace cycle-guarded DFS fault locus detector (RFC 0013 § 1.B & P1 #1).
645
+ * Identifies where an error surfaced (`originated_fault_in_span`) while keeping
646
+ * `causalProofTier: "unverified_locus"` separate from `relationshipCertainty: "exact"`.
647
+ */
648
+ export function localizeTraceFaultLocus(
649
+ traceId: string,
650
+ spans: TraceSpanLocusInput[],
651
+ ): TraceFaultLocusResult {
652
+ if (spans.length === 0) {
653
+ return {
654
+ faultLocusSpan: null,
655
+ faultLocusNodeId: null,
656
+ relationshipCertainty: "exact",
657
+ causalProofTier: "unverified_locus",
658
+ causalTriples: [],
659
+ };
660
+ }
661
+
662
+ const spanById = new Map<string, TraceSpanLocusInput>();
663
+ const childrenByParent = new Map<string, TraceSpanLocusInput[]>();
664
+ for (const s of spans) {
665
+ spanById.set(s.span_id, s);
666
+ if (s.parent_span_id) {
667
+ const list = childrenByParent.get(s.parent_span_id) ?? [];
668
+ list.push(s);
669
+ childrenByParent.set(s.parent_span_id, list);
670
+ }
671
+ }
672
+
673
+ for (const list of childrenByParent.values()) {
674
+ list.sort((a, b) => a.start_time.localeCompare(b.start_time));
675
+ }
676
+
677
+ const roots = spans
678
+ .filter((s) => !s.parent_span_id || !spanById.has(s.parent_span_id))
679
+ .sort((a, b) => a.start_time.localeCompare(b.start_time));
680
+
681
+ const visited = new Set<string>();
682
+ const causalTriples: CausalTriple[] = [];
683
+ const leafErrorSpans: TraceSpanLocusInput[] = [];
684
+ const MAX_DEPTH = 256;
685
+
686
+ const walk = (span: TraceSpanLocusInput, depth: number) => {
687
+ if (visited.has(span.span_id) || depth > MAX_DEPTH) return;
688
+ visited.add(span.span_id);
689
+
690
+ const children = (childrenByParent.get(span.span_id) ?? []).filter(
691
+ (c) => !visited.has(c.span_id),
692
+ );
693
+ const errorChildren = children.filter((c) => c.status_code === 2);
694
+
695
+ if (span.status_code === 2) {
696
+ if (errorChildren.length === 0) {
697
+ leafErrorSpans.push(span);
698
+ } else {
699
+ for (const errChild of errorChildren) {
700
+ causalTriples.push({
701
+ subjectKind: "span",
702
+ subjectId: span.span_id,
703
+ predicate: "failed_due_to_child_span",
704
+ objectKind: "span",
705
+ objectId: errChild.span_id,
706
+ relationshipCertainty: "exact",
707
+ causalProofTier: "unverified_locus",
708
+ confidence: 0.85,
709
+ reason: `Parent span ${span.span_name} failed while child span ${errChild.span_name} returned ERROR`,
710
+ });
711
+ }
712
+ }
713
+ }
714
+
715
+ for (const child of children) {
716
+ walk(child, depth + 1);
717
+ }
718
+ };
719
+
720
+ for (const root of roots) {
721
+ walk(root, 0);
722
+ }
723
+ // Catch any disconnected cycle islands
724
+ for (const s of spans) {
725
+ if (!visited.has(s.span_id)) {
726
+ walk(s, 0);
727
+ }
728
+ }
729
+
730
+ leafErrorSpans.sort((a, b) => a.start_time.localeCompare(b.start_time));
731
+ const faultLocusSpan =
732
+ leafErrorSpans[0] ?? spans.find((s) => s.status_code === 2) ?? null;
733
+
734
+ let faultLocusNodeId: string | null = null;
735
+ if (faultLocusSpan) {
736
+ const env = (faultLocusSpan.environment ?? "default").toLowerCase();
737
+ faultLocusNodeId =
738
+ faultLocusSpan.target_node_id ??
739
+ (faultLocusSpan.service_name
740
+ ? buildCanonicalNodeId({
741
+ kind: "service",
742
+ environment: env,
743
+ serviceName: faultLocusSpan.service_name,
744
+ }).id
745
+ : null);
746
+
747
+ causalTriples.unshift({
748
+ subjectKind: "trace",
749
+ subjectId: traceId,
750
+ predicate: "originated_fault_in_span",
751
+ objectKind: "span",
752
+ objectId: faultLocusSpan.span_id,
753
+ relationshipCertainty: "exact",
754
+ causalProofTier: "unverified_locus",
755
+ confidence: 0.9,
756
+ reason: `Deepest/earliest failing span ${faultLocusSpan.span_name} (${faultLocusSpan.service_name ?? "unknown"}) is the trace fault locus`,
757
+ });
758
+ }
759
+
760
+ return {
761
+ faultLocusSpan,
762
+ faultLocusNodeId,
763
+ relationshipCertainty: "exact",
764
+ causalProofTier: "unverified_locus",
765
+ causalTriples,
766
+ };
767
+ }
768
+
769
+ /**
770
+ * Queries `telemetry_spans` for a trace and computes its fault locus.
771
+ */
772
+ export async function loadTraceFaultLocusFromDb(
773
+ db: SqlDb,
774
+ projectId: string,
775
+ traceId: string,
776
+ ): Promise<TraceFaultLocusResult> {
777
+ const rows = await db
778
+ .prepare(
779
+ `SELECT
780
+ span_id, parent_span_id, service_name, span_name, span_kind,
781
+ status_code, status_message, start_time, end_time, duration_ms,
782
+ environment, target_node_id, error_signature
783
+ FROM telemetry_spans
784
+ WHERE project_id = ? AND trace_id = ?
785
+ ORDER BY start_time ASC
786
+ LIMIT 500`,
787
+ )
788
+ .bind(projectId, traceId)
789
+ .all<TraceSpanLocusInput>();
790
+ return localizeTraceFaultLocus(traceId, rows.results);
791
+ }
792
+
793
+ interface OntologyNodeRow {
794
+ project_id: string;
795
+ environment: string;
796
+ id: string;
797
+ kind: string;
798
+ name: string;
799
+ namespace_or_endpoint: string;
800
+ is_scoped_identity: number;
801
+ current_version: string | null;
802
+ current_commit_sha: string | null;
803
+ current_config_digest: string | null;
804
+ attributes_json: string | null;
805
+ first_seen_at: string;
806
+ last_seen_at: string;
807
+ }
808
+
809
+ interface OntologyEdgeRow {
810
+ project_id: string;
811
+ environment: string;
812
+ id: string;
813
+ source_node_id: string;
814
+ predicate: string;
815
+ target_node_id: string;
816
+ relationship_certainty: string;
817
+ call_count: number;
818
+ error_count: number;
819
+ last_duration_ms: number | null;
820
+ first_seen_at: string;
821
+ last_seen_at: string;
822
+ }
823
+
824
+ /**
825
+ * Loads the environment-scoped operational ontology graph with Tarjan SCC
826
+ * cycle metadata and dependency blackout indicators.
827
+ */
828
+ export async function getOntologyGraph(
829
+ db: SqlDb,
830
+ options: { projectId: string; environment?: string },
831
+ ): Promise<OntologyGraphResponse> {
832
+ const env = (options.environment ?? "default").toLowerCase();
833
+
834
+ const [nodeRows, edgeRows] = await Promise.all([
835
+ db
836
+ .prepare(
837
+ `SELECT * FROM ontology_nodes
838
+ WHERE project_id = ? AND environment = ?
839
+ ORDER BY id ASC
840
+ LIMIT 500`,
841
+ )
842
+ .bind(options.projectId, env)
843
+ .all<OntologyNodeRow>(),
844
+ db
845
+ .prepare(
846
+ `SELECT * FROM ontology_edges
847
+ WHERE project_id = ? AND environment = ?
848
+ ORDER BY call_count DESC
849
+ LIMIT 1000`,
850
+ )
851
+ .bind(options.projectId, env)
852
+ .all<OntologyEdgeRow>(),
853
+ ]);
854
+
855
+ const rawNodes: OntologyNode[] = nodeRows.results.map((r) => ({
856
+ projectId: r.project_id,
857
+ environment: r.environment,
858
+ id: r.id,
859
+ kind: r.kind as OntologyNodeKind,
860
+ name: r.name,
861
+ namespaceOrEndpoint: r.namespace_or_endpoint,
862
+ isScopedIdentity: r.is_scoped_identity === 1,
863
+ currentVersion: r.current_version,
864
+ currentCommitSha: r.current_commit_sha,
865
+ currentConfigDigest: r.current_config_digest,
866
+ attributes: parseJsonRecord(r.attributes_json ?? "{}"),
867
+ firstSeenAt: r.first_seen_at,
868
+ lastSeenAt: r.last_seen_at,
869
+ }));
870
+
871
+ const edges: OntologyEdge[] = edgeRows.results.map((r) => ({
872
+ projectId: r.project_id,
873
+ environment: r.environment,
874
+ id: r.id,
875
+ sourceNodeId: r.source_node_id,
876
+ predicate: r.predicate,
877
+ targetNodeId: r.target_node_id,
878
+ relationshipCertainty: (r.relationship_certainty ||
879
+ "exact") as RelationshipCertainty,
880
+ callCount: Number(r.call_count ?? 0),
881
+ errorCount: Number(r.error_count ?? 0),
882
+ lastDurationMs:
883
+ r.last_duration_ms === null ? null : Number(r.last_duration_ms),
884
+ firstSeenAt: r.first_seen_at,
885
+ lastSeenAt: r.last_seen_at,
886
+ }));
887
+
888
+ const { annotatedNodes, components } = computeTarjanScc(rawNodes, edges);
889
+
890
+ // Detect suspected dependency blackouts: targets with high inbound error edges
891
+ const inboundErrorsByTarget = new Map<string, number>();
892
+ for (const e of edges) {
893
+ if (e.errorCount > 0) {
894
+ inboundErrorsByTarget.set(
895
+ e.targetNodeId,
896
+ (inboundErrorsByTarget.get(e.targetNodeId) ?? 0) + e.errorCount,
897
+ );
898
+ }
899
+ }
900
+
901
+ const nodes = annotatedNodes.map((n) => ({
902
+ ...n,
903
+ isBlackoutSuspected:
904
+ (inboundErrorsByTarget.get(n.id) ?? 0) >= 3 &&
905
+ edges.filter((e) => e.sourceNodeId === n.id).length === 0,
906
+ }));
907
+
908
+ return {
909
+ environment: env,
910
+ nodes,
911
+ edges,
912
+ stronglyConnectedComponents: components,
913
+ };
914
+ }