@codeyam/codeyam-cli 0.1.21 → 0.1.23

package/analyzer-template/packages/analyze/src/lib/files/analyze/validateDependencyAnalyses.ts

@@ -6,6 +6,20 @@ import type {
   ReadonlyEntity,
 } from '~codeyam/types';
 import { ImmutableAnalysisContext } from '../../analysisContext';
+import { awsLogDebugLevel } from '~codeyam/utils';
+
+/**
+ * Tracks statistics across the recursive dependency validation.
+ * Shared by all recursive calls to provide a complete picture.
+ */
+interface ValidationStats {
+  totalVisited: number;
+  skippedDuplicate: number;
+  skippedMissing: number;
+  skippedTrivial: number;
+  maxDepth: number;
+  startTime: number;
+}
 
 export default function validateDependencyAnalyses({
   analysis,
@@ -13,12 +27,18 @@ export default function validateDependencyAnalyses({
   entity,
   context,
   dependentAnalyses,
+  _depth = 0,
+  _stats,
 }: {
   analysis?: Analysis;
   readonlyAnalysis?: ReadonlyAnalysis;
   entity: ReadonlyEntity;
   context: ImmutableAnalysisContext;
   dependentAnalyses?: ReadonlyAnalysisMap;
+  /** Recursion depth (internal, do not set) */
+  _depth?: number;
+  /** Shared stats tracker (internal, do not set) */
+  _stats?: ValidationStats;
 }): { dependentAnalyses: ReadonlyAnalysisMap } {
   if (!readonlyAnalysis) readonlyAnalysis = analysis;
   if (!readonlyAnalysis) {
@@ -32,6 +52,28 @@ export default function validateDependencyAnalyses({
     );
   }
 
+  // Initialize stats on the root call
+  const isRootCall = _depth === 0;
+  if (!_stats) {
+    _stats = {
+      totalVisited: 0,
+      skippedDuplicate: 0,
+      skippedMissing: 0,
+      skippedTrivial: 0,
+      maxDepth: 0,
+      startTime: Date.now(),
+    };
+  }
+  if (_depth > _stats.maxDepth) _stats.maxDepth = _depth;
+
+  const importCount = entity.metadata?.importedExports?.length ?? 0;
+  if (isRootCall && importCount > 0) {
+    awsLogDebugLevel(
+      2,
+      `validateDeps: ${entity.name} has ${importCount} imports to validate`,
+    );
+  }
+
   if (
     dependentAnalyses &&
     dependentAnalyses[readonlyAnalysis.filePath]?.[
@@ -45,6 +87,12 @@ export default function validateDependencyAnalyses({
     })
   ) {
     // already processed this analysis
+    if (isRootCall) {
+      awsLogDebugLevel(
+        2,
+        `validateDeps: ${entity.name} already fully processed (cache hit)`,
+      );
+    }
     return { dependentAnalyses };
   }
 
@@ -69,9 +117,20 @@ export default function validateDependencyAnalyses({
 
     // if the entity has duplicate dependencies, handle gracefully
     if (dependentAnalyses[dependencyFilePath]?.[dependencyName]) {
+      _stats.skippedDuplicate++;
       continue;
     }
 
+    _stats.totalVisited++;
+
+    // Emit periodic progress so the spinner stays alive for large trees
+    if (_stats.totalVisited % 50 === 0) {
+      awsLogDebugLevel(
+        1,
+        `Validating dependencies... (${_stats.totalVisited} resolved, depth ${_depth})`,
+      );
+    }
+
     // Check if dependency analysis exists in context
     const dependentAnalysis = context.getAnalysis(
       dependencyFilePath,
@@ -86,21 +145,20 @@ export default function validateDependencyAnalyses({
       );
 
       if (!dependentEntity) {
-        // console.warn(
-        //   `CodeYam: validateDependencyAnalyses: Missing dependent entity ${dependencyFilePath}:${dependencyName} for ${entity.filePath}:${entity.name}. This is likely a bug.`,
-        // );
+        _stats.skippedMissing++;
         continue;
       }
 
       // ignore trivial missing dependencies
-      if (!['library', 'visual'].includes(dependentEntity.entityType)) continue;
+      if (!['library', 'visual'].includes(dependentEntity.entityType)) {
+        _stats.skippedTrivial++;
+        continue;
+      }
 
       if (!dependentEntity?.metadata?.hasCircularDependency) {
         // TODO: This no longer seems to happen thanks to fixes elsewhere, but leave the warning
-        // console.warn(
-        //   `CodeYam: validateDependencyAnalyses: Missing analysis for dependency ${entity.filePath}:${entity.name} -> ${dependencyFilePath}:${dependencyName}, should be marked as having circular dependencies.`,
-        // );
       }
+      _stats.skippedMissing++;
       continue;
     }
 
@@ -136,9 +194,6 @@ export default function validateDependencyAnalyses({
     if (analysis) {
       // Track dependency relationship in analysis metadata
       if (!analysis.metadata.dependentAnalyses.includes(dependentAnalysis.id)) {
-        // console.log(
-        //   `CodeYam: validateDependencyAnalyses: Establishing dependency ${analysis.filePath}:${analysis.entityName} -> ${dependentAnalysis.filePath}:${dependentAnalysis.entityName}.`,
-        // );
         analysis.metadata.dependentAnalyses.push(dependentAnalysis.id);
       }
     }
@@ -152,9 +207,26 @@ export default function validateDependencyAnalyses({
         entity: dependentEntity,
         context: context,
         dependentAnalyses,
+        _depth: _depth + 1,
+        _stats,
       });
     }
   }
 
+  // Log summary on root call completion
+  if (isRootCall) {
+    const elapsed = Date.now() - _stats.startTime;
+    const totalDeps = Object.values(dependentAnalyses).reduce(
+      (sum, fileMap) => sum + Object.keys(fileMap).length,
+      0,
+    );
+    awsLogDebugLevel(
+      1,
+      `Dependency validation complete for ${entity.name}: ${totalDeps} dependencies resolved, ` +
+        `${_stats.skippedDuplicate} duplicates skipped, ${_stats.skippedMissing} missing, ` +
+        `${_stats.skippedTrivial} trivial, max depth ${_stats.maxDepth}, ${elapsed}ms`,
+    );
+  }
+
   return { dependentAnalyses };
 }

package/analyzer-template/packages/analyze/src/lib/files/analyzeNextRoute.ts

@@ -3,6 +3,8 @@ import { ProjectAnalyzer } from '../ProjectAnalyzer';
 import { generateSha } from '~codeyam/database';
 import { isNextRoute } from '~codeyam/utils';
 
+const warnedLayoutPaths = new Set<string>();
+
 export default function analyzeNextRoute(
   file: File,
   projectAnalyzer: ProjectAnalyzer,
@@ -56,9 +58,12 @@ export default function analyzeNextRoute(
 
           // Ensure entities array exists before trying to use it
           if (!layout.entities) {
-            console.log(
-              `CodeYam Warning: layout.entities not populated for ${layout.path}, skipping implicit dependency`,
-            );
+            if (!warnedLayoutPaths.has(layout.path)) {
+              warnedLayoutPaths.add(layout.path);
+              console.log(
+                `CodeYam Warning: layout.entities not populated for ${layout.path}, skipping implicit dependency`,
+              );
+            }
             return;
           }
 

package/analyzer-template/packages/analyze/src/lib/files/scenarios/generateDataStructure.ts

@@ -11,8 +11,10 @@ import {
   clearAttributesFromMapping,
   mergeJsonTypeDefinitions,
 } from '~codeyam/ai';
-import mergeInDependentDataStructure from './mergeInDependentDataStructure';
-import { awsLog } from '~codeyam/utils';
+import mergeInDependentDataStructure, {
+  DataStructureTimeoutError,
+} from './mergeInDependentDataStructure';
+import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import gatherDataForMocks from './gatherDataForMocks';
 import enrichArrayTypesFromChildSignatures from './enrichArrayTypesFromChildSignatures';
 import enrichUnknownTypesFromSourceEquivalencies from './enrichUnknownTypesFromSourceEquivalencies';
@@ -328,12 +330,23 @@ export default function generateDataStructure({
 
     // Recursive function to traverse the dependency tree
     const processedAnalyses = new Set<string>();
+    const traverseStart = Date.now();
+    const TRAVERSE_TIMEOUT_MS = 2_000;
+    let traverseCount = 0;
     const traverseDependencyTree = (analyses: ReadonlyAnalysisMap) => {
       for (const filePath in analyses) {
         for (const name in analyses[filePath]) {
           const analysisKey = `${filePath}:${name}`;
           if (processedAnalyses.has(analysisKey)) continue;
           processedAnalyses.add(analysisKey);
+          traverseCount++;
+
+          if (Date.now() - traverseStart > TRAVERSE_TIMEOUT_MS) {
+            throw new DataStructureTimeoutError(
+              entity.name,
+              Date.now() - traverseStart,
+            );
+          }
 
           const childAnalysis = analyses[filePath][name];
           const childEntity = childAnalysis?.entity;
@@ -443,6 +456,25 @@ export default function generateDataStructure({
     // Start traversal from direct non-mocked children
     traverseDependencyTree(nonMockedDependentAnalyses);
 
+    const traverseElapsed = Date.now() - traverseStart;
+    const childSchemaKeyCount = Object.values(childDependencySchemas).reduce(
+      (sum, fileMap) =>
+        sum +
+        Object.values(fileMap).reduce(
+          (s, schema) =>
+            s +
+            Object.keys(schema.signatureSchema || {}).length +
+            Object.keys(schema.returnValueSchema || {}).length,
+          0,
+        ),
+      0,
+    );
+    awsLogDebugLevel(
+      1,
+      `Dependency tree traversal for ${entity.name}: ${traverseCount} analyses, ` +
+        `${childSchemaKeyCount} child schema keys, ${allImportedExports.length} imports, ${traverseElapsed}ms`,
+    );
+
     // console.info(
     //   'CODEYAM DEBUG: start merge',
     //   JSON.stringify(
@@ -484,6 +516,14 @@ export default function generateDataStructure({
       dependencySchemas: isolatedDataStructure.dependencySchemas,
     });
 
+    const mergeStart = Date.now();
+    // Shared deadline for all phases from merge onward
+    const genDeadline = mergeStart + 5_000; // 5s total budget for all phases
+    awsLogDebugLevel(
+      1,
+      `Starting mergeInDependentDataStructure for ${entity.name}...`,
+    );
+
     const mergedDataStructure = mergeInDependentDataStructure({
       importedExports: allImportedExports,
       dependentAnalyses: nonMockedDependentAnalyses,
@@ -498,6 +538,14 @@ export default function generateDataStructure({
       dependencySchemas: isolatedDataStructure?.dependencySchemas || {},
     });
 
+    const mergeTimedOut = !!(mergedDataStructure as any).timedOut;
+    awsLogDebugLevel(
+      1,
+      `mergeInDependentDataStructure ${mergeTimedOut ? 'PARTIAL' : 'complete'} for ${entity.name}: ` +
+        `${Object.keys(mergedDataStructure.signatureSchema).length} sig, ` +
+        `${Object.keys(mergedDataStructure.returnValueSchema).length} ret, ${Date.now() - mergeStart}ms`,
+    );
+
     mergedDataStructure.environmentVariables =
       isolatedDataStructure.environmentVariables || [];
 
@@ -511,6 +559,7 @@ export default function generateDataStructure({
     // Merge collected child dependency schemas into mergedDataStructure.
     // This ensures that when gatherDataForMocks looks up schemas for child imports,
     // the child's schemas are available.
+    const childMergeStart = Date.now();
     for (const schemaFilePath in childDependencySchemas) {
       for (const schemaName in childDependencySchemas[schemaFilePath]) {
         mergedDataStructure.dependencySchemas[schemaFilePath] ||= {};
@@ -561,7 +610,13 @@ export default function generateDataStructure({
       }
     }
 
+    awsLogDebugLevel(
+      1,
+      `Child schema merge for ${entity.name}: ${Date.now() - childMergeStart}ms`,
+    );
+
     // Transformation tracing: capture after child schemas merged
+    const snapshotStart = Date.now();
     transformationTracer.snapshot(entity.name, 'childSchemasMerged', {
       signatureSchema: mergedDataStructure.signatureSchema,
       returnValueSchema: mergedDataStructure.returnValueSchema,
@@ -588,71 +643,134 @@ export default function generateDataStructure({
     //   ),
     // );
 
+    awsLogDebugLevel(
+      1,
+      `Transformation snapshot for ${entity.name}: ${Date.now() - snapshotStart}ms`,
+    );
+
     // Deduplicate function schemas in the merged data structure
     // The merge process might introduce duplicates even if individual schemas were clean
-    // Trace deduplication of signatureSchema
-    const sigBefore = { ...mergedDataStructure.signatureSchema };
-    mergedDataStructure.signatureSchema = deduplicateFunctionSchemas(
-      mergedDataStructure.signatureSchema,
-    );
-    transformationTracer.traceSchemaTransformResult(
-      entity.name,
-      'deduplicateFunctionSchemas',
-      sigBefore,
-      mergedDataStructure.signatureSchema,
-      { schemaType: 'signature' },
-    );
+    const dedupStart = Date.now();
 
-    // Trace deduplication of returnValueSchema
-    const rvBefore = { ...mergedDataStructure.returnValueSchema };
-    mergedDataStructure.returnValueSchema = deduplicateFunctionSchemas(
+    const sigKeyCount = Object.keys(mergedDataStructure.signatureSchema).length;
+    const retKeyCount = Object.keys(
       mergedDataStructure.returnValueSchema,
-    );
-    transformationTracer.traceSchemaTransformResult(
-      entity.name,
-      'deduplicateFunctionSchemas',
-      rvBefore,
-      mergedDataStructure.returnValueSchema,
-      { schemaType: 'returnValue' },
+    ).length;
+
+    // Skip dedup on very large schemas — the O(keys × variants × keyLength) cost
+    // is not worth it for schemas with 10K+ keys where function dedup rarely matters.
+    const DEDUP_KEY_THRESHOLD = 10_000;
+
+    if (sigKeyCount < DEDUP_KEY_THRESHOLD) {
+      const sigBefore = { ...mergedDataStructure.signatureSchema };
+      mergedDataStructure.signatureSchema = deduplicateFunctionSchemas(
+        mergedDataStructure.signatureSchema,
+      );
+      transformationTracer.traceSchemaTransformResult(
+        entity.name,
+        'deduplicateFunctionSchemas',
+        sigBefore,
+        mergedDataStructure.signatureSchema,
+        { schemaType: 'signature' },
+      );
+    } else {
+      awsLogDebugLevel(
+        2,
+        `${entity.name}: skipping sig dedup (${sigKeyCount} keys > ${DEDUP_KEY_THRESHOLD})`,
+      );
+    }
+
+    if (retKeyCount < DEDUP_KEY_THRESHOLD) {
+      const rvBefore = { ...mergedDataStructure.returnValueSchema };
+      mergedDataStructure.returnValueSchema = deduplicateFunctionSchemas(
+        mergedDataStructure.returnValueSchema,
+      );
+      transformationTracer.traceSchemaTransformResult(
+        entity.name,
+        'deduplicateFunctionSchemas',
+        rvBefore,
+        mergedDataStructure.returnValueSchema,
+        { schemaType: 'returnValue' },
+      );
+    } else {
+      awsLogDebugLevel(
+        2,
+        `${entity.name}: skipping ret dedup (${retKeyCount} keys > ${DEDUP_KEY_THRESHOLD})`,
+      );
+    }
+
+    awsLogDebugLevel(
+      1,
+      `${entity.name}: root schema dedup ${Date.now() - dedupStart}ms (sig=${sigKeyCount}, ret=${retKeyCount})`,
     );
 
     // Also deduplicate dependency schemas and clear known attributes
+    let depProcessed = 0;
+    const depStart = Date.now();
     for (const filePath in mergedDataStructure.dependencySchemas) {
       for (const depEntityName in mergedDataStructure.dependencySchemas[
         filePath
       ]) {
+        if (Date.now() > genDeadline) {
+          throw new DataStructureTimeoutError(
+            entity.name,
+            Date.now() - mergeStart,
+          );
+        }
+
         const depSchema =
           mergedDataStructure.dependencySchemas[filePath][depEntityName];
+        const depSigKeys = Object.keys(depSchema.signatureSchema || {}).length;
+        const depRetKeys = Object.keys(
+          depSchema.returnValueSchema || {},
+        ).length;
 
-        // Trace deduplication of dependency signatureSchema
-        const depSigBefore = { ...depSchema.signatureSchema };
-        depSchema.signatureSchema = deduplicateFunctionSchemas(
-          depSchema.signatureSchema,
-        );
-        transformationTracer.traceSchemaTransformResult(
-          entity.name,
-          'deduplicateFunctionSchemas',
-          depSigBefore,
-          depSchema.signatureSchema,
-          { filePath, dependencyName: depEntityName, schemaType: 'signature' },
-        );
+        const depOpStart = Date.now();
 
-        // Trace deduplication of dependency returnValueSchema
-        const depRvBefore = { ...depSchema.returnValueSchema };
-        depSchema.returnValueSchema = deduplicateFunctionSchemas(
-          depSchema.returnValueSchema,
-        );
-        transformationTracer.traceSchemaTransformResult(
-          entity.name,
-          'deduplicateFunctionSchemas',
-          depRvBefore,
-          depSchema.returnValueSchema,
-          {
-            filePath,
-            dependencyName: depEntityName,
-            schemaType: 'returnValue',
-          },
-        );
+        if (depSigKeys < DEDUP_KEY_THRESHOLD) {
+          const depSigBefore = { ...depSchema.signatureSchema };
+          depSchema.signatureSchema = deduplicateFunctionSchemas(
+            depSchema.signatureSchema,
+          );
+          transformationTracer.traceSchemaTransformResult(
+            entity.name,
+            'deduplicateFunctionSchemas',
+            depSigBefore,
+            depSchema.signatureSchema,
+            {
+              filePath,
+              dependencyName: depEntityName,
+              schemaType: 'signature',
+            },
+          );
+        }
+
+        if (depRetKeys < DEDUP_KEY_THRESHOLD) {
+          const depRvBefore = { ...depSchema.returnValueSchema };
+          depSchema.returnValueSchema = deduplicateFunctionSchemas(
+            depSchema.returnValueSchema,
+          );
+          transformationTracer.traceSchemaTransformResult(
+            entity.name,
+            'deduplicateFunctionSchemas',
+            depRvBefore,
+            depSchema.returnValueSchema,
+            {
+              filePath,
+              dependencyName: depEntityName,
+              schemaType: 'returnValue',
+            },
+          );
+        }
+
+        const depOpElapsed = Date.now() - depOpStart;
+        if (depOpElapsed > 1000) {
+          awsLogDebugLevel(
+            2,
+            `${entity.name}: slow dep dedup ${depEntityName} sig=${depSigKeys} ret=${depRetKeys} ${depOpElapsed}ms`,
+          );
+        }
+        depProcessed++;
 
         // Infer function types from child component usage BEFORE fillInSchemaGapsAndUnknowns.
         // When a hook returns a property with type 'unknown' that is passed to a child
@@ -729,13 +847,33 @@ export default function generateDataStructure({
         // Fill in type gaps BEFORE clearing attributes, so evidence like .includes()
         // is used for type inference before being deleted
 
-        depSchema.signatureSchema = fillInDirectSchemaGapsAndUnknowns({
-          schema: depSchema.signatureSchema,
-        });
+        const FILL_IN_KEY_THRESHOLD = 5_000;
+        const fillStart = Date.now();
+
+        if (depSigKeys < FILL_IN_KEY_THRESHOLD) {
+          depSchema.signatureSchema = fillInDirectSchemaGapsAndUnknowns({
+            schema: depSchema.signatureSchema,
+          });
+        }
+
+        if (depRetKeys < FILL_IN_KEY_THRESHOLD) {
+          depSchema.returnValueSchema = fillInDirectSchemaGapsAndUnknowns({
+            schema: depSchema.returnValueSchema,
+          });
+        } else {
+          awsLogDebugLevel(
+            2,
+            `${entity.name}: skipping fillInGaps for ${depEntityName} ret (${depRetKeys} keys > ${FILL_IN_KEY_THRESHOLD})`,
+          );
+        }
 
-        depSchema.returnValueSchema = fillInDirectSchemaGapsAndUnknowns({
-          schema: depSchema.returnValueSchema,
-        });
+        const fillElapsed = Date.now() - fillStart;
+        if (fillElapsed > 1000) {
+          awsLogDebugLevel(
+            2,
+            `${entity.name}: slow fillInGaps ${depEntityName} sig=${depSigKeys} ret=${depRetKeys} ${fillElapsed}ms`,
+          );
+        }
 
         // Trace clearAttributesFromMapping on dependency schemas
         transformationTracer.traceSchemaTransform(
@@ -760,15 +898,46 @@ export default function generateDataStructure({
       }
     }
 
+    awsLogDebugLevel(
+      1,
+      `${entity.name}: dep schema processing: ${depProcessed} deps in ${Date.now() - depStart}ms`,
+    );
+    awsLogDebugLevel(
+      1,
+      `Dedup + dep schema processing for ${entity.name}: ${Date.now() - dedupStart}ms`,
+    );
+
     // Transformation tracing: capture after deduplication
+    const dedupSnapshotStart = Date.now();
     transformationTracer.snapshot(entity.name, 'deduplicated', {
       signatureSchema: mergedDataStructure.signatureSchema,
       returnValueSchema: mergedDataStructure.returnValueSchema,
       dependencySchemas: mergedDataStructure.dependencySchemas,
     });
 
+    awsLogDebugLevel(
+      1,
+      `Dedup snapshot for ${entity.name}: ${Date.now() - dedupSnapshotStart}ms`,
+    );
+
     analysis.metadata ||= {};
     analysis.metadata.mergedDataStructure = mergedDataStructure;
+    if (mergeTimedOut) {
+      (analysis.metadata as any).mergeTimedOut = true;
+    }
+
+    const checkGenDeadline = (phase: string) => {
+      if (Date.now() > genDeadline) {
+        throw new DataStructureTimeoutError(
+          entity.name,
+          Date.now() - mergeStart,
+        );
+      }
+      awsLogDebugLevel(
+        1,
+        `${entity.name}: ${phase} complete (${Date.now() - mergeStart}ms total)`,
+      );
+    };
 
     // Enrich mocked dependency array types with fields from child component signatures
     // This ensures that when getSurveysAction returns { data: array }, and SurveyCard
@@ -787,6 +956,8 @@ export default function generateDataStructure({
         ),
     );
 
+    checkGenDeadline('enrichArrayTypes');
+
     // Transformation tracing: capture after array types enriched
     transformationTracer.snapshot(entity.name, 'arrayTypesEnriched', {
       signatureSchema: mergedDataStructure.signatureSchema,
@@ -811,6 +982,8 @@ export default function generateDataStructure({
         ),
     );
 
+    checkGenDeadline('enrichUnknownTypes');
+
     // Transformation tracing: capture after unknown types enriched
     transformationTracer.snapshot(entity.name, 'unknownTypesEnriched', {
       signatureSchema: mergedDataStructure.signatureSchema,
@@ -831,6 +1004,8 @@ export default function generateDataStructure({
       { entityName: entity.name },
     );
 
+    checkGenDeadline('gatherDataForMocks');
+
     // Transformation tracing: capture after dataForMocks gathered
     transformationTracer.snapshot(entity.name, 'dataForMocksGathered', {
       signatureSchema: mergedDataStructure.signatureSchema,
@@ -860,6 +1035,8 @@ export default function generateDataStructure({
       }
     }
 
+    checkGenDeadline('mergeChildDataForMocks');
+
     const finalizedArgumentsSchema = { ...mergedDataStructure.signatureSchema };
 
     // Trace fillInDirectSchemaGapsAndUnknowns
@@ -880,9 +1057,13 @@ export default function generateDataStructure({
       { stage: 'finalization' },
     );
 
+    checkGenDeadline('fillInGaps+clearAttributes');
+
     const argumentsSchema = convertDotNotation(finalizedArgumentsSchema)
       .signature as JsonTypeDefinition[];
 
+    checkGenDeadline('convertDotNotation');
+
     analysis.metadata.scenariosDataStructure = {
       arguments: argumentsSchema,
       dataForMocks,