@principal-ai/subsystems-core 0.29.0

package/src/telemetry/coverage.ts

@@ -0,0 +1,480 @@
+/**
+ * Telemetry Coverage Analysis
+ *
+ * Measures observability coverage by analyzing which implementation files
+ * contain the OpenTelemetry events documented in canvas nodes.
+ *
+ * Uses codebase-composition's ImplementationFileLayerModule for accurate,
+ * framework-aware detection of implementation files.
+ */
+
+import type { FileTree } from '@principal-ai/repository-abstraction';
+import { CanvasDiscovery } from '../discovery/CanvasDiscovery';
+import type { DiscoveredCanvasWithContent } from '../discovery/types';
+import type { ExtendedCanvasNode } from '../types/canvas';
+import { isOtelNode, isStandardCanvasNode } from '../types/canvas';
+import { getNodeEventName } from '../storyboard/builder';
+import {
+  ImplementationFileLayerModule,
+  PackageLayerModule,
+  type ImplementationFileLayer,
+} from '@principal-ai/codebase-composition';
+
+/**
+ * Per-file instrumentation details
+ */
+export interface FileInstrumentation {
+  filePath: string;
+  expectedEvents: string[];      // Events that should be in this file (from canvas)
+  foundEvents: string[];         // Events actually found in the file
+  instrumentationCount: number;  // foundEvents.length (for heat map)
+  hasInstrumentation: boolean;   // instrumentationCount > 0
+  missingEvents: string[];       // expectedEvents - foundEvents
+  isImplementationFile: boolean; // Whether this file matches implementation patterns
+}
+
+/**
+ * Node status breakdown
+ */
+export interface NodeStatusBreakdown {
+  missingStatusCount: number;
+  draftCount: number;
+  approvedCount: number;
+  approvedWithFilesCount: number;
+  implementedCount: number;
+  implementedWithFilesCount: number;
+}
+
+/**
+ * Validation error for a canvas node
+ */
+export interface ValidationError {
+  nodeId: string;
+  canvasPath: string;
+  error: string;
+  status?: string;
+}
+
+/**
+ * Overall coverage metrics (file-based)
+ */
+export interface CoverageMetrics {
+  // File-based coverage
+  totalImplementationFiles: number;
+  filesWithInstrumentation: number;
+  coveragePercentage: number;
+
+  // Per-file details (for heat map and debugging)
+  fileCoverage: FileInstrumentation[];
+
+  // Canvas metadata
+  canvasFiles: string[];
+
+  // Summary stats
+  totalExpectedEvents: number;
+  totalFoundEvents: number;
+
+  // Per-package breakdown
+  packageCoverage: PackageCoverageMetrics[];
+
+  // Node status breakdown
+  nodeStatus: NodeStatusBreakdown;
+
+  // Validation errors
+  validationErrors: ValidationError[];
+}
+
+/**
+ * Per-package coverage metrics
+ */
+export interface PackageCoverageMetrics {
+  packageName: string;
+  packagePath: string;
+  totalFiles: number;
+  filesWithInstrumentation: number;
+  coveragePercentage: number;
+  filesByExtension: Record<string, number>;
+}
+
+/**
+ * Extract event name from a canvas node
+ * Uses the canonical getNodeEventName() which supports all event reference formats
+ */
+function getEventName(node: ExtendedCanvasNode): string | null {
+  return getNodeEventName(node) ?? null;
+}
+
+/**
+ * Extract instrumentation file paths from a canvas node
+ * Supports both new format (node.otel.files) and legacy format (pv.otel.files)
+ */
+function getInstrumentationFiles(node: ExtendedCanvasNode): string[] {
+  // Check OTEL nodes (top-level otel field)
+  if (isOtelNode(node) && node.otel?.files) {
+    return node.otel.files;
+  }
+  // Check standard nodes (pv.otel.files)
+  if (isStandardCanvasNode(node) && node.pv?.otel?.files) {
+    return node.pv.otel.files;
+  }
+  return [];
+}
+
+/**
+ * Check if a file contains a specific event name (simple string match)
+ */
+async function fileContainsEvent(
+  filePath: string,
+  eventName: string,
+  fileReader: (path: string) => Promise<string>
+): Promise<boolean> {
+  try {
+    const content = await fileReader(filePath);
+    return content.includes(eventName);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Get the status of a node (returns undefined if not specified)
+ * Supports both new format (node.otel.status) and legacy format (pv.status)
+ */
+function getNodeStatus(node: ExtendedCanvasNode): 'draft' | 'approved' | 'implemented' | undefined {
+  // Check OTEL nodes (otel.status)
+  if (isOtelNode(node) && node.otel?.status) {
+    return node.otel.status;
+  }
+  // Check standard nodes (pv.status)
+  if (isStandardCanvasNode(node) && node.pv?.status) {
+    return node.pv.status;
+  }
+  return undefined;
+}
+
+/**
+ * Get the scope from a node
+ * Only supports new format (node.otel.scope). Legacy format (pv.otel.scope) is rejected.
+ */
+function getNodeScope(node: ExtendedCanvasNode): string | undefined {
+  const otelNode = node as { otel?: { scope?: string } };
+  return otelNode.otel?.scope;
+}
+
+/**
+ * Build a map of file -> expected events from canvas nodes
+ * Only includes nodes with status='implemented'
+ */
+async function buildFileEventMap(
+  otelCanvases: DiscoveredCanvasWithContent[]
+): Promise<Map<string, Set<string>>> {
+  const fileEventMap = new Map<string, Set<string>>();
+
+  for (const canvas of otelCanvases) {
+    if (!canvas.content?.nodes || canvas.content.nodes.length === 0) {
+      continue;
+    }
+
+    for (const node of canvas.content.nodes) {
+      const eventName = getEventName(node);
+      if (!eventName) continue;
+
+      // Only check implemented nodes for coverage
+      const status = getNodeStatus(node);
+      if (status !== 'implemented') continue;
+
+      const files = getInstrumentationFiles(node);
+      for (const file of files) {
+        if (!fileEventMap.has(file)) {
+          fileEventMap.set(file, new Set());
+        }
+        fileEventMap.get(file)!.add(eventName);
+      }
+    }
+  }
+
+  return fileEventMap;
+}
+
+/**
+ * Validate nodes and collect validation errors
+ */
+function validateNodes(
+  otelCanvases: DiscoveredCanvasWithContent[]
+): { statusBreakdown: NodeStatusBreakdown; errors: ValidationError[] } {
+  const errors: ValidationError[] = [];
+  const statusBreakdown: NodeStatusBreakdown = {
+    missingStatusCount: 0,
+    draftCount: 0,
+    approvedCount: 0,
+    approvedWithFilesCount: 0,
+    implementedCount: 0,
+    implementedWithFilesCount: 0,
+  };
+
+  for (const canvas of otelCanvases) {
+    if (!canvas.content?.nodes || canvas.content.nodes.length === 0) {
+      continue;
+    }
+
+    for (const node of canvas.content.nodes) {
+      const status = getNodeStatus(node);
+      const eventName = getEventName(node);
+
+      // Only validate nodes with events
+      if (!eventName) continue;
+
+      const files = getInstrumentationFiles(node);
+      const hasFiles = files.length > 0;
+
+      // Validate: nodes with events MUST have a status
+      if (!status) {
+        statusBreakdown.missingStatusCount++;
+        errors.push({
+          nodeId: node.id,
+          canvasPath: canvas.path,
+          error: `Node with event "${eventName}" must have pv.status field. Set to "draft" (design phase), "approved" (ready to implement), or "implemented" (code exists)`,
+        });
+        continue; // Skip further validation for this node
+      }
+
+      // Get references for grounding validation
+      let references: string[] = [];
+      if (isOtelNode(node) && node.otel?.references) {
+        references = node.otel.references;
+      } else if (isStandardCanvasNode(node) && node.pv?.references) {
+        references = node.pv.references;
+      }
+      const hasReferences = references.length > 0;
+
+      // Count by status
+      if (status === 'draft') {
+        statusBreakdown.draftCount++;
+        // Drafts must be grounded: require either files (if implementation location is known) or references
+        if (!hasFiles && !hasReferences) {
+          errors.push({
+            nodeId: node.id,
+            canvasPath: canvas.path,
+            status,
+            error: `Node with status="draft" must have either pv.otel.files (if implementation location is known) or pv.references (to ground the event in something)`,
+          });
+        }
+      } else if (status === 'approved') {
+        statusBreakdown.approvedCount++;
+        if (hasFiles) {
+          statusBreakdown.approvedWithFilesCount++;
+        }
+      } else if (status === 'implemented') {
+        statusBreakdown.implementedCount++;
+        if (hasFiles) {
+          statusBreakdown.implementedWithFilesCount++;
+        }
+      }
+
+      // Validate approved and implemented nodes have pv.otel.files
+      if ((status === 'approved' || status === 'implemented') && !hasFiles) {
+        errors.push({
+          nodeId: node.id,
+          canvasPath: canvas.path,
+          status,
+          error: `Node with status="${status}" must have pv.otel.files specified`,
+        });
+      }
+
+      // Error if using deprecated pv.otel.scope (only for standard nodes)
+      let legacyScope: string | undefined;
+      if (isStandardCanvasNode(node)) {
+        legacyScope = node.pv?.otel?.scope;
+      }
+      if (legacyScope) {
+        errors.push({
+          nodeId: node.id,
+          canvasPath: canvas.path,
+          status,
+          error: `Node uses deprecated "pv.otel.scope" field. Use new format: convert to type: "otel-event" with top-level "otel.scope" field instead. Run: npx @principal-ai/principal-studio-cli migrate-nodes`,
+        });
+      }
+
+      // Require scope for ALL event nodes (using new format)
+      const scope = getNodeScope(node);
+      if (!scope) {
+        errors.push({
+          nodeId: node.id,
+          canvasPath: canvas.path,
+          status,
+          error: `Event node must have otel.scope specified. Scope defines which instrumentation library emits this event (e.g., "terminal-activity", "auth"). Document scopes in your .scopes.canvas file with otel-scope nodes.`,
+        });
+      }
+    }
+  }
+
+  return { statusBreakdown, errors };
+}
+
+/**
+ * Analyze instrumentation for a single file
+ */
+async function analyzeFileInstrumentation(
+  filePath: string,
+  expectedEvents: string[],
+  fileReader: (path: string) => Promise<string>,
+  isImplFile: boolean
+): Promise<FileInstrumentation> {
+  const foundEvents: string[] = [];
+
+  for (const eventName of expectedEvents) {
+    const found = await fileContainsEvent(filePath, eventName, fileReader);
+    if (found) {
+      foundEvents.push(eventName);
+    }
+  }
+
+  const missingEvents = expectedEvents.filter(e => !foundEvents.includes(e));
+
+  return {
+    filePath,
+    expectedEvents,
+    foundEvents,
+    instrumentationCount: foundEvents.length,
+    hasInstrumentation: foundEvents.length > 0,
+    missingEvents,
+    isImplementationFile: isImplFile,
+  };
+}
+
+/**
+ * Get implementation files using codebase-composition's ImplementationFileLayerModule
+ * This provides accurate detection with framework-awareness and package boundaries
+ */
+async function getImplementationFilesFromLayer(
+  fileTree: FileTree
+): Promise<{ files: Set<string>; layers: ImplementationFileLayer[] }> {
+  // Detect packages
+  const packageModule = new PackageLayerModule();
+  const packageLayers = await packageModule.discoverPackages(fileTree);
+
+  // Get implementation files per package
+  const implModule = new ImplementationFileLayerModule();
+  const implLayers = implModule.createImplementationFileLayers(packageLayers, fileTree);
+
+  // Flatten to set of file paths
+  const files = new Set<string>();
+  for (const layer of implLayers) {
+    for (const file of layer.implementationData.implementationFiles) {
+      files.add(file);
+    }
+  }
+
+  return { files, layers: implLayers };
+}
+
+/**
+ * Generate telemetry coverage report from canvas files
+ *
+ * Uses codebase-composition's ImplementationFileLayerModule for accurate,
+ * framework-aware detection of implementation files.
+ *
+ * @param fileTree - FileTree representation of the codebase
+ * @param fileReader - Function to read file contents
+ * @returns Coverage metrics based on file instrumentation
+ *
+ * @example
+ * ```typescript
+ * import { FilesystemService, NodeFileSystemAdapter } from '@principal-ai/codebase-composition/node';
+ * import { readFile } from 'fs/promises';
+ * import { resolve } from 'path';
+ *
+ * const service = new FilesystemService(new NodeFileSystemAdapter());
+ * const fileTree = await service.buildFileSystemTreeFromPath('/path/to/project');
+ * const fileReader = async (path: string) => readFile(resolve('/path/to/project', path), 'utf-8');
+ * const metrics = await analyzeCoverage(fileTree, fileReader);
+ * ```
+ */
+export async function analyzeCoverage(
+  fileTree: FileTree,
+  fileReader: (path: string) => Promise<string>
+): Promise<CoverageMetrics> {
+  // Use CanvasDiscovery to find all canvas files
+  const discovery = new CanvasDiscovery();
+  const result = await discovery.discover(fileTree, {
+    fileReader,
+    includeContent: true, // Need content to parse nodes
+  });
+
+  // Filter for .otel.canvas files only
+  const otelCanvases = result.canvases.filter(c => c.type === 'otel') as DiscoveredCanvasWithContent[];
+
+  // Validate nodes and collect status breakdown
+  const { statusBreakdown, errors } = validateNodes(otelCanvases);
+
+  // Build map of file -> expected events from canvas (only implemented nodes)
+  const fileEventMap = await buildFileEventMap(otelCanvases);
+
+  // Get all implementation files from the codebase using implementation layer
+  const { files: implementationFiles, layers: implLayers } = await getImplementationFilesFromLayer(fileTree);
+
+  // Get all files referenced in canvas (for coverage denominator)
+  const referencedFiles = new Set(fileEventMap.keys());
+
+  // Combine: we want to check all files referenced in canvas that are also implementation files
+  const filesToCheck = new Set(
+    [...referencedFiles].filter(f => implementationFiles.has(f))
+  );
+
+  // Analyze each file
+  const fileCoverage: FileInstrumentation[] = [];
+
+  for (const filePath of filesToCheck) {
+    const expectedEvents = Array.from(fileEventMap.get(filePath) || []);
+    const coverage = await analyzeFileInstrumentation(
+      filePath,
+      expectedEvents,
+      fileReader,
+      implementationFiles.has(filePath)
+    );
+    fileCoverage.push(coverage);
+  }
+
+  // Calculate metrics
+  const filesWithInstrumentation = fileCoverage.filter(f => f.hasInstrumentation).length;
+  const totalImplementationFiles = filesToCheck.size;
+  const coveragePercentage = totalImplementationFiles > 0
+    ? (filesWithInstrumentation / totalImplementationFiles) * 100
+    : 0;
+
+  const totalExpectedEvents = fileCoverage.reduce((sum, f) => sum + f.expectedEvents.length, 0);
+  const totalFoundEvents = fileCoverage.reduce((sum, f) => sum + f.foundEvents.length, 0);
+
+  // Calculate per-package metrics
+  const packageCoverage: PackageCoverageMetrics[] = implLayers.map(layer => {
+    const packageFiles = new Set(layer.implementationData.implementationFiles);
+    const packageInstrumented = fileCoverage.filter(f =>
+      packageFiles.has(f.filePath) && f.hasInstrumentation
+    );
+
+    return {
+      packageName: layer.implementationData.packageName,
+      packagePath: layer.implementationData.packagePath,
+      totalFiles: layer.implementationData.fileCount,
+      filesWithInstrumentation: packageInstrumented.length,
+      coveragePercentage:
+        layer.implementationData.fileCount > 0
+          ? (packageInstrumented.length / layer.implementationData.fileCount) * 100
+          : 0,
+      filesByExtension: layer.implementationData.filesByExtension || {},
+    };
+  });
+
+  return {
+    totalImplementationFiles,
+    filesWithInstrumentation,
+    coveragePercentage,
+    fileCoverage,
+    canvasFiles: otelCanvases.map(c => c.path),
+    totalExpectedEvents,
+    totalFoundEvents,
+    packageCoverage,
+    nodeStatus: statusBreakdown,
+    validationErrors: errors,
+  };
+}