@codeyam/codeyam-cli 0.1.0-staging.483fdc2 → 0.1.0-staging.50df560

package/analyzer-template/packages/ai/src/lib/isolateScopes.ts

@@ -743,6 +743,48 @@ function processScope({
   };
 }
 
+/**
+ * Check if a code string contains only type declarations (type aliases, interfaces, etc.)
+ * that don't contribute to runtime behavior. This is used to determine if the root scope
+ * should be considered "empty" for scope extraction purposes.
+ */
+function containsOnlyTypeDeclarations(code: string): boolean {
+  try {
+    const sourceFile = ts.createSourceFile(
+      'temp.ts',
+      code,
+      ts.ScriptTarget.Latest,
+      true,
+      ts.ScriptKind.TS,
+    );
+
+    // Check each statement - if any is NOT a type-only declaration, return false
+    for (const statement of sourceFile.statements) {
+      if (
+        ts.isTypeAliasDeclaration(statement) ||
+        ts.isInterfaceDeclaration(statement) ||
+        ts.isEnumDeclaration(statement) ||
+        // Import declarations that are type-only
+        (ts.isImportDeclaration(statement) &&
+          statement.importClause?.isTypeOnly) ||
+        // Export declarations that are type-only
+        (ts.isExportDeclaration(statement) && statement.isTypeOnly)
+      ) {
+        // This is a type-only declaration, continue checking
+        continue;
+      }
+      // Found a non-type statement
+      return false;
+    }
+
+    // All statements are type-only (or there are no statements)
+    return sourceFile.statements.length > 0;
+  } catch {
+    // If parsing fails, assume it's not type-only
+    return false;
+  }
+}
+
 /**
  * Main entry point.
  *
@@ -792,9 +834,15 @@ export default function isolateScopes(
   const jsxLimit = options?.jsxLimit ?? 10;
   processJSXForScope(isolatedScopes, generateSyntheticName, jsxLimit);
 
-  // If the root scope text is empty and there's only one child scope,
-  // return that child scope as the root scope
-  if (isolatedScopes.text.trim().length === 0) {
+  // If the root scope text is empty (or only contains type declarations) and there's
+  // only one child scope, return that child scope as the root scope.
+  // Type declarations (type aliases, interfaces) don't contribute to runtime behavior,
+  // so we treat them as "empty" for scope extraction purposes.
+  const rootTextIsEffectivelyEmpty =
+    isolatedScopes.text.trim().length === 0 ||
+    containsOnlyTypeDeclarations(isolatedScopes.text);
+
+  if (rootTextIsEffectivelyEmpty) {
     const childScopeArray = Object.values(isolatedScopes.childScopes);
     if (childScopeArray.length === 1) {
       return childScopeArray[0];

package/analyzer-template/packages/ai/src/lib/mergeJsonTypeDefinitions.ts

@@ -11,6 +11,11 @@ export default function mergeJsonTypeDefinitions(
     if (!typeDef) continue;
     if (typeof typeDef === 'string') continue;
     for (const [key, value] of Object.entries(typeDef)) {
+      // Preserve _nullable boolean markers set by convertDotNotation
+      if (key === '_nullable' && typeof value === 'boolean') {
+        mergedDef[key] = value;
+        continue;
+      }
       if (typeof value === 'string') {
         if (!mergedDef[key]) {
           mergedDef[key] = value;

package/analyzer-template/packages/ai/src/lib/promptGenerators/collapseNullableObjects.ts

@@ -0,0 +1,118 @@
+import { JsonTypeDefinition } from '~codeyam/types';
+
+/**
+ * Check whether all non-`_nullable` children of an object are simple strings.
+ * "Leaf-like" nullable objects (all-string children) get collapsed to
+ * `"object | null"` so the LLM knows the field can be null.
+ * Objects with nested structure (object/array children) keep their shape
+ * so the LLM can still generate proper data when the field is truthy.
+ */
+function isLeafLikeObject(obj: Record<string, any>): boolean {
+  for (const key of Object.keys(obj)) {
+    if (key === '_nullable') continue;
+    const value = obj[key];
+    if (typeof value !== 'string') return false;
+  }
+  return true;
+}
+
+/**
+ * Collapse *leaf-like* nullable objects/arrays in the data structure to type strings.
+ *
+ * When convertDotNotation marks an object or array as `_nullable: true`
+ * (meaning the schema had `object | undefined`, `object | null`, etc.),
+ * we decide whether to collapse it:
+ *
+ * - **Leaf-like** (all children are simple strings): collapse to `"object | null"`.
+ *   Example: `{ _nullable: true, type: "string", path: "string" }` → `"object | null"`
+ *
+ * - **Rich** (has object/array children): strip `_nullable` and keep the structure
+ *   so the LLM can generate proper nested data when the field is truthy.
+ *   Example: `{ _nullable: true, items: [...], name: "string" }` → `{ items: [...], name: "string" }`
+ *
+ * Non-nullable objects are recursively processed but left as nested structures.
+ */
+export default function collapseNullableObjects(
+  data: JsonTypeDefinition,
+): JsonTypeDefinition {
+  if (typeof data !== 'object' || data === null) {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    return data.map((item: any) =>
+      typeof item === 'object' && item !== null
+        ? collapseNullableObjects(item as JsonTypeDefinition)
+        : item,
+    ) as any as JsonTypeDefinition;
+  }
+
+  const result: JsonTypeDefinition = {};
+
+  for (const key of Object.keys(data)) {
+    // Skip the _nullable marker itself (shouldn't appear at top level, but be safe)
+    if (key === '_nullable') continue;
+
+    const value = data[key];
+
+    if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      const hasNullable = (value as any)._nullable === true;
+      if (hasNullable && isLeafLikeObject(value as Record<string, any>)) {
+        // Leaf-like nullable object → collapse to type string
+        const childKeys = Object.keys(value as any).filter(
+          (k) => k !== '_nullable',
+        );
+        console.log(
+          `[collapseNullableObjects] collapsed "${key}" → "object | null" (leaf-like, children: [${childKeys.join(', ')}])`,
+        );
+        result[key] = 'object | null';
+      } else if (hasNullable) {
+        // Rich nullable object → strip _nullable, keep structure
+        const childKeys = Object.keys(value as any).filter(
+          (k) => k !== '_nullable',
+        );
+        console.log(
+          `[collapseNullableObjects] kept structure for nullable "${key}" (rich, children: [${childKeys.join(', ')}])`,
+        );
+        result[key] = collapseNullableObjects(value as JsonTypeDefinition);
+      } else {
+        // Non-nullable → recurse
+        result[key] = collapseNullableObjects(value as JsonTypeDefinition);
+      }
+    } else if (Array.isArray(value)) {
+      const hasNullable = (value as any)._nullable === true;
+      if (
+        hasNullable &&
+        value.length <= 1 &&
+        value.every((v) => typeof v === 'string')
+      ) {
+        // Leaf-like nullable array → collapse to type string
+        console.log(
+          `[collapseNullableObjects] collapsed "${key}" → "array | null" (leaf-like)`,
+        );
+        result[key] = 'array | null';
+      } else if (hasNullable) {
+        // Rich nullable array → strip _nullable, keep structure
+        console.log(
+          `[collapseNullableObjects] kept structure for nullable array "${key}" (${value.length} elements)`,
+        );
+        result[key] = value.map((item) =>
+          typeof item === 'object' && item !== null
+            ? collapseNullableObjects(item as JsonTypeDefinition)
+            : item,
+        ) as JsonTypeDefinition[];
+      } else {
+        // Non-nullable → recurse elements
+        result[key] = value.map((item) =>
+          typeof item === 'object' && item !== null
+            ? collapseNullableObjects(item as JsonTypeDefinition)
+            : item,
+        ) as JsonTypeDefinition[];
+      }
+    } else {
+      result[key] = value;
+    }
+  }
+
+  return result;
+}

package/analyzer-template/packages/ai/src/lib/promptGenerators/generateEntityScenarioDataGenerator.ts

@@ -7,6 +7,7 @@ import {
 } from '~codeyam/types';
 import noErrorAttributes from './noErrorAttributes';
 import simplifyKeysForLLM from './simplifyKeysForLLM';
+import collapseNullableObjects from './collapseNullableObjects';
 
 /**
  * Sort object keys by the size of their JSON-stringified values (smallest first).
@@ -99,11 +100,17 @@ ${JSON.stringify(defaultScenarioData.data, null, 2)}
   // while the reconcileMockDataKeys function will still match the simplified response back to original keys.
   // Then sort keys by size (smallest first) so small keys appear before large keys,
   // increasing the chance the LLM generates all keys for large schemas.
-  const cleanedDataForMocks = structure.dataForMocks
-    ? simplifyKeysForLLM(
-        noErrorAttributes(structure.dataForMocks) as JsonTypeDefinition,
-      )
+
+  const afterNoError = structure.dataForMocks
+    ? (noErrorAttributes(structure.dataForMocks) as JsonTypeDefinition)
+    : null;
+
+  const afterSimplify = afterNoError ? simplifyKeysForLLM(afterNoError) : null;
+
+  const cleanedDataForMocks = afterSimplify
+    ? collapseNullableObjects(afterSimplify)
     : null;
+
   const sortedDataForMocks =
     cleanedDataForMocks &&
     typeof cleanedDataForMocks === 'object' &&
@@ -111,6 +118,19 @@ ${JSON.stringify(defaultScenarioData.data, null, 2)}
       ? sortKeysBySize(cleanedDataForMocks as Record<string, any>)
       : cleanedDataForMocks;
 
+  // Log nullable-related content in final prompt JSON
+  if (sortedDataForMocks) {
+    const finalJson = JSON.stringify(sortedDataForMocks, null, 2);
+    const hasNullableMarker = finalJson.includes('_nullable');
+    const nullCollapsed = (finalJson.match(/"object \| null"/g) || []).length;
+    const arrayCollapsed = (finalJson.match(/"array \| null"/g) || []).length;
+    if (hasNullableMarker || nullCollapsed > 0 || arrayCollapsed > 0) {
+      console.log(
+        `[generateEntityScenarioDataGenerator] final prompt: ${nullCollapsed} "object | null", ${arrayCollapsed} "array | null"${hasNullableMarker ? ', WARNING: _nullable still present!' : ''}`,
+      );
+    }
+  }
+
   // When mockData has already been generated via chunks, skip it in the main call
   const mockDataSection = options?.mockDataAlreadyGenerated
     ? '## mockData Structure\nmockData has been pre-generated—return `{}` for mockData.'

package/analyzer-template/packages/analyze/index.ts

@@ -41,6 +41,8 @@ export { default as mergeValidatedDataStructures } from './src/lib/files/scenari
 
 export { discoverDirectDependencies } from './src/lib/files/analyze/dependencyResolver';
 
+export { transformationTracer } from './src/lib/files/scenarios/TransformationTracer';
+
 export { default as getAnalysisError } from './src/lib/utils/getAnalysisError';
 export { default as measureAndReportExecutionTime } from './src/lib/utils/measureAndReportExecutionTime';
 

package/analyzer-template/packages/analyze/src/lib/ProjectAnalyzer.ts

@@ -46,6 +46,12 @@ export class ProjectAnalyzer {
       lib.asts.createProgramFromTsConfigPath(configPath, excludePatterns),
     );
 
+    if (this.programs.length === 0) {
+      throw new Error(
+        'No TypeScript programs could be created — no tsconfig.json found',
+      );
+    }
+
     // Primary program is the first one (usually root) for backward compatibility
     this.program = this.programs[0];
     this.typeChecker = this.program.getTypeChecker();
@@ -334,17 +340,23 @@ export class ProjectAnalyzer {
    * Refresh the TypeScript program to pick up file changes
    */
   refreshProgram(): void {
-    // Memory logging: Capture state before refresh
-    const memBefore = process.memoryUsage();
-
     // Apply the same unapprovedPaths filtering when refreshing
     const excludePatterns =
       this.project.metadata?.unapprovedPaths?.filter(Boolean) || [];
 
-    // Refresh all programs
-    this.programs = this.tsConfigPaths.map((configPath) =>
-      lib.asts.createProgramFromTsConfigPath(configPath, excludePatterns),
-    );
+    // Release old programs and caches BEFORE creating new ones to reduce peak memory.
+    // Without this, both old (~1.7GB) and new (~1.7GB) programs coexist during creation,
+    // causing OOM on monorepos with multiple tsconfigs.
+    this.programs = [];
+    this.sourceFileByPath.clear();
+    this.typeCheckerByPath.clear();
+
+    // Create programs one at a time so each old program can be GC'd before the next
+    for (const configPath of this.tsConfigPaths) {
+      this.programs.push(
+        lib.asts.createProgramFromTsConfigPath(configPath, excludePatterns),
+      );
+    }
 
     // Update primary program for backward compatibility
     this.program = this.programs[0];

package/analyzer-template/packages/analyze/src/lib/asts/index.ts

@@ -95,8 +95,13 @@ export function createProgramFromTsConfigPath(
     throw new Error('Could not parse tsconfig.json');
   }
 
-  // Filter file names based on exclude patterns to reduce memory usage
-  let fileNames = parsedCommandLine.fileNames;
+  // Always filter out node_modules from root files to prevent OOM on large projects.
+  // This only removes them as root compilation targets — TypeScript's module resolution
+  // still follows imports into node_modules for type info when processing source files.
+  let fileNames = parsedCommandLine.fileNames.filter(
+    (f) => !f.includes('/node_modules/'),
+  );
+
   if (excludePatterns && excludePatterns.length > 0) {
     const regexPatterns = excludePatterns
       .map((pattern) => {

package/analyzer-template/packages/analyze/src/lib/asts/nodes/getNodeType.ts

@@ -39,6 +39,7 @@ export function getNodeType(node: ts.Node, sourceFile: ts.SourceFile) {
     case 'StringLiteral':
     case 'NumericLiteral':
     case 'ObjectLiteralExpression':
+    case 'ArrayLiteralExpression':
       nodeType = 'data';
       break;
     case 'TypeAliasDeclaration':

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities/prepareDataStructures.ts

@@ -18,6 +18,7 @@ import {
   getCompoundConditionals,
   getChildBoundaryGatingConditions,
   getJsxRenderingUsages,
+  skipWorkerPool,
 } from '~codeyam/ai';
 import type { AnalysisOptions } from '../analyzeEntities';
 
@@ -271,6 +272,23 @@ export default async function prepareEntityDataStructures(
   };
 
   const entities = Array.from(allEntities);
+
+  // The worker thread creates its own ProjectAnalyzer (ts.Program) on first use,
+  // which takes 50-70s. For small entity counts the overhead far exceeds the actual
+  // processing time (<2s). Skip the worker and run on the main thread instead,
+  // reusing the existing ProjectAnalyzer.
+  const entitiesNeedingGeneration = entities.filter(
+    (e) =>
+      !e.metadata?.isolatedDataStructure || options.force || options.forceAll,
+  ).length;
+  const WORKER_ENTITY_THRESHOLD = 500;
+  if (entitiesNeedingGeneration < WORKER_ENTITY_THRESHOLD) {
+    console.log(
+      `CodeYam: Skipping worker thread for DS prep (${entitiesNeedingGeneration} entities < ${WORKER_ENTITY_THRESHOLD} threshold)`,
+    );
+    skipWorkerPool();
+  }
+
   const sequential =
     DEBUG_SEQUENTIAL_EXECUTION || options.sequentialDataStructurePrep;
 
@@ -555,6 +573,42 @@ async function getEntityDataStructureAndFunctionCalls({
     },
   );
 
+  // Add same-file child components to dependencySchemas.
+  // Same-file components (e.g., AgentCard defined in the same file as AgentTranscriptsPage)
+  // appear in externalFunctionCalls and functionResults but NOT in importedExports or
+  // nodeModuleImports. Without this, their signatures are missing from dependencySchemas
+  // and enrichArrayTypesFromChildSignatures can't find child component field accesses.
+  const allDependencyNames = new Set(allDependencies.map((d) => d.name));
+
+  for (const efc of dataStructure.externalFunctionCalls) {
+    // Skip if already covered by importedExports or nodeModuleImports
+    if (allDependencyNames.has(efc.name)) continue;
+
+    const signatureSchema = getFunctionSignature(dataStructure, {
+      functionName: efc.name,
+    });
+
+    const returnValueSchema = getReturnValue(dataStructure, {
+      functionName: efc.name,
+    });
+
+    // Only add if we have signature or return value data
+    if (!signatureSchema && !returnValueSchema) continue;
+
+    const clonedSignatureSchema = { ...(signatureSchema ?? {}) };
+    const clonedReturnValueSchema = { ...(returnValueSchema ?? {}) };
+
+    const entityFilePath = entity.filePath;
+    dependencySchemas[entityFilePath] ||= {};
+    dependencySchemas[entityFilePath][efc.name] = {
+      signatureSchema: deduplicateFunctionSchemas(clonedSignatureSchema),
+      returnValueSchema: deduplicateFunctionSchemas(clonedReturnValueSchema),
+      usageEquivalencies: getUsageEquivalencies(dataStructure, efc.name) ?? {},
+      sourceEquivalencies:
+        getSourceEquivalencies(dataStructure, efc.name) ?? {},
+    };
+  }
+
   const rawReturnValue =
     getReturnValue(dataStructure, { fillInUnknowns: false }) ?? {};
 
@@ -870,9 +924,21 @@ export function determineMockedDependencies(entity: Entity) {
       // A schema with only "EntityName(): function" just indicates the entity is callable,
       // it doesn't mean there's actual return data to mock
       const hasReturnValue = hasActualReturnValueData(entityReturnValueSchema);
-      const complexReturnValue = Object.values(
-        entityReturnValueSchema ?? {},
-      ).some((value) => value === 'object' || value === 'array');
+      // Check for complex return types (object/array), accounting for nullable types
+      // e.g., 'object', 'object | null', 'array | undefined' should all count as complex.
+      // Also detect when functionCallReturnValue has nested child properties — this
+      // proves it's a structured type even if its declared type is still 'unknown'.
+      const returnValueKeys = Object.keys(entityReturnValueSchema ?? {});
+      const complexReturnValue =
+        Object.values(entityReturnValueSchema ?? {}).some((value) => {
+          const baseType = value?.split(' | ')[0];
+          return baseType === 'object' || baseType === 'array';
+        }) ||
+        returnValueKeys.some(
+          (key) =>
+            key.includes('functionCallReturnValue.') ||
+            key.includes('functionCallReturnValue['),
+        );
 
       // Check if this is a validation function that should always be mocked
       const isValidationFn = isValidationFunction(entityName);

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities.ts

@@ -5,6 +5,7 @@ import {
   measureAndReportExecutionTime,
   ProjectAnalyzer,
 } from '~codeyam/analyze';
+import { transformationTracer } from '../scenarios/TransformationTracer';
 import { getFileByPathSafe } from '../../utils/getFileByPath';
 import trackEntityCircularDependencies from './trackEntityCircularDependencies';
 import validateDependencyAnalyses from './validateDependencyAnalyses';
@@ -307,6 +308,18 @@ export default async function analyzeEntities({
   awsLog(
     `CodeYam: analyzeEntities: Analysis completed with ${analyses.length} analyses`,
   );
+
+  // Flush transformation trace if tracing is enabled
+  // Wrapped in try/catch because trace data can exceed V8's max string length
+  // for large projects, and we must not crash the analysis process over tracing.
+  try {
+    transformationTracer.flush();
+  } catch (e) {
+    console.warn(
+      `CodeYam Warning: Failed to flush transformation trace: ${e.message}`,
+    );
+  }
+
   return analyses;
 
   // PHASE 3: Pure graph traversal

package/analyzer-template/packages/analyze/src/lib/files/analyze/dependencyResolver.ts

@@ -100,12 +100,6 @@ export function getCandidatesReadyForCapture(
       continue;
     }
 
-    // Skip notExported entities - they can't be imported into the simulation
-    // environment and will always fail capture. Don't mark them as ready.
-    if (candidate.entity?.metadata?.notExported) {
-      continue;
-    }
-
     // Get all transitive dependencies
     const candidateKey = `${candidate.filePath}:${candidate.entityName}`;
     const { missingDependencies } = getTransitiveDependencies(

package/analyzer-template/packages/analyze/src/lib/files/analyze/findOrCreateEntity.ts

@@ -94,12 +94,15 @@ export default async function findOrCreateEntity({
   try {
     code = fileAnalyzer.getEntityCode(localEntityName);
   } catch (e) {
-    console.log('CodeYam Error: could not get entity code', {
-      filePath: file.path,
-      localEntityName,
-      entityName,
-      error: e,
-    });
+    console.log(
+      `CodeYam Error: Could not extract code for "${localEntityName}" from ${file.path} — export not found in file, may have been moved or renamed during refactoring`,
+      {
+        filePath: file.path,
+        localEntityName,
+        entityName,
+        error: String(e),
+      },
+    );
     return null;
   }
   const sha = generateSha(file.path, entityName, code);
@@ -138,6 +141,18 @@ export default async function findOrCreateEntity({
           break;
         }
       }
+
+      // Mark previous versions as superseded
+      const entitiesToSupersede = previousEntities.filter(
+        (e) => !e.metadata?.isSuperseded,
+      );
+      if (entitiesToSupersede.length > 0) {
+        for (const prev of entitiesToSupersede) {
+          prev.metadata = prev.metadata || {};
+          prev.metadata.isSuperseded = true;
+        }
+        await upsertEntities(entitiesToSupersede);
+      }
     }
 
     entity = {

package/analyzer-template/packages/analyze/src/lib/files/analyze/gatherEntityMap.ts

@@ -111,18 +111,15 @@ export default async function gatherEntityMap({
       });
 
       if (!importedEntity) {
-        console.log(
-          `CodeYam Error: Entity not found while gathering entity map: ${importedFile.path}: ${importedExportName}`,
-          {
-            entityName: entity.name,
-            entityFilePath: entity.filePath,
-            importedFilePath: importedFile.path,
-            importedExportName: importedExportName,
-          },
-        );
-        throw new Error(
-          `Entity not found while gathering entity map: ${importedFile.path}: ${importedExport.name}`,
-        );
+        const msg = [
+          `Entity not found: "${importedExportName}" in ${importedFile.path}`,
+          `  While analyzing dependencies of: ${entity.name} (${entity.filePath})`,
+          `  This usually means "${importedExportName}" was moved or renamed but`,
+          `  ${entity.filePath} still has stale import metadata referencing the old location.`,
+          `  Fix: Update or re-analyze ${entity.filePath} so its importedExports reflect current code.`,
+        ].join('\n');
+        console.log(`CodeYam Error: ${msg}`);
+        throw new Error(msg);
       }
 
       // Add newly created entity to context