@codeyam/codeyam-cli 0.1.0-staging.036391e → 0.1.0-staging.07d8f7b

package/analyzer-template/packages/ai/src/lib/mergeJsonTypeDefinitions.ts

@@ -11,6 +11,11 @@ export default function mergeJsonTypeDefinitions(
     if (!typeDef) continue;
     if (typeof typeDef === 'string') continue;
     for (const [key, value] of Object.entries(typeDef)) {
+      // Preserve _nullable boolean markers set by convertDotNotation
+      if (key === '_nullable' && typeof value === 'boolean') {
+        mergedDef[key] = value;
+        continue;
+      }
       if (typeof value === 'string') {
         if (!mergedDef[key]) {
           mergedDef[key] = value;

package/analyzer-template/packages/ai/src/lib/promptGenerators/collapseNullableObjects.ts

@@ -0,0 +1,118 @@
+import { JsonTypeDefinition } from '~codeyam/types';
+
+/**
+ * Check whether all non-`_nullable` children of an object are simple strings.
+ * "Leaf-like" nullable objects (all-string children) get collapsed to
+ * `"object | null"` so the LLM knows the field can be null.
+ * Objects with nested structure (object/array children) keep their shape
+ * so the LLM can still generate proper data when the field is truthy.
+ */
+function isLeafLikeObject(obj: Record<string, any>): boolean {
+  for (const key of Object.keys(obj)) {
+    if (key === '_nullable') continue;
+    const value = obj[key];
+    if (typeof value !== 'string') return false;
+  }
+  return true;
+}
+
+/**
+ * Collapse *leaf-like* nullable objects/arrays in the data structure to type strings.
+ *
+ * When convertDotNotation marks an object or array as `_nullable: true`
+ * (meaning the schema had `object | undefined`, `object | null`, etc.),
+ * we decide whether to collapse it:
+ *
+ * - **Leaf-like** (all children are simple strings): collapse to `"object | null"`.
+ *   Example: `{ _nullable: true, type: "string", path: "string" }` → `"object | null"`
+ *
+ * - **Rich** (has object/array children): strip `_nullable` and keep the structure
+ *   so the LLM can generate proper nested data when the field is truthy.
+ *   Example: `{ _nullable: true, items: [...], name: "string" }` → `{ items: [...], name: "string" }`
+ *
+ * Non-nullable objects are recursively processed but left as nested structures.
+ */
+export default function collapseNullableObjects(
+  data: JsonTypeDefinition,
+): JsonTypeDefinition {
+  if (typeof data !== 'object' || data === null) {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    return data.map((item: any) =>
+      typeof item === 'object' && item !== null
+        ? collapseNullableObjects(item as JsonTypeDefinition)
+        : item,
+    ) as any as JsonTypeDefinition;
+  }
+
+  const result: JsonTypeDefinition = {};
+
+  for (const key of Object.keys(data)) {
+    // Skip the _nullable marker itself (shouldn't appear at top level, but be safe)
+    if (key === '_nullable') continue;
+
+    const value = data[key];
+
+    if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      const hasNullable = (value as any)._nullable === true;
+      if (hasNullable && isLeafLikeObject(value as Record<string, any>)) {
+        // Leaf-like nullable object → collapse to type string
+        const childKeys = Object.keys(value as any).filter(
+          (k) => k !== '_nullable',
+        );
+        console.log(
+          `[collapseNullableObjects] collapsed "${key}" → "object | null" (leaf-like, children: [${childKeys.join(', ')}])`,
+        );
+        result[key] = 'object | null';
+      } else if (hasNullable) {
+        // Rich nullable object → strip _nullable, keep structure
+        const childKeys = Object.keys(value as any).filter(
+          (k) => k !== '_nullable',
+        );
+        console.log(
+          `[collapseNullableObjects] kept structure for nullable "${key}" (rich, children: [${childKeys.join(', ')}])`,
+        );
+        result[key] = collapseNullableObjects(value as JsonTypeDefinition);
+      } else {
+        // Non-nullable → recurse
+        result[key] = collapseNullableObjects(value as JsonTypeDefinition);
+      }
+    } else if (Array.isArray(value)) {
+      const hasNullable = (value as any)._nullable === true;
+      if (
+        hasNullable &&
+        value.length <= 1 &&
+        value.every((v) => typeof v === 'string')
+      ) {
+        // Leaf-like nullable array → collapse to type string
+        console.log(
+          `[collapseNullableObjects] collapsed "${key}" → "array | null" (leaf-like)`,
+        );
+        result[key] = 'array | null';
+      } else if (hasNullable) {
+        // Rich nullable array → strip _nullable, keep structure
+        console.log(
+          `[collapseNullableObjects] kept structure for nullable array "${key}" (${value.length} elements)`,
+        );
+        result[key] = value.map((item) =>
+          typeof item === 'object' && item !== null
+            ? collapseNullableObjects(item as JsonTypeDefinition)
+            : item,
+        ) as JsonTypeDefinition[];
+      } else {
+        // Non-nullable → recurse elements
+        result[key] = value.map((item) =>
+          typeof item === 'object' && item !== null
+            ? collapseNullableObjects(item as JsonTypeDefinition)
+            : item,
+        ) as JsonTypeDefinition[];
+      }
+    } else {
+      result[key] = value;
+    }
+  }
+
+  return result;
+}

package/analyzer-template/packages/ai/src/lib/promptGenerators/generateEntityScenarioDataGenerator.ts

@@ -7,6 +7,7 @@ import {
 } from '~codeyam/types';
 import noErrorAttributes from './noErrorAttributes';
 import simplifyKeysForLLM from './simplifyKeysForLLM';
+import collapseNullableObjects from './collapseNullableObjects';
 
 /**
  * Sort object keys by the size of their JSON-stringified values (smallest first).
@@ -99,11 +100,17 @@ ${JSON.stringify(defaultScenarioData.data, null, 2)}
   // while the reconcileMockDataKeys function will still match the simplified response back to original keys.
   // Then sort keys by size (smallest first) so small keys appear before large keys,
   // increasing the chance the LLM generates all keys for large schemas.
-  const cleanedDataForMocks = structure.dataForMocks
-    ? simplifyKeysForLLM(
-        noErrorAttributes(structure.dataForMocks) as JsonTypeDefinition,
-      )
+
+  const afterNoError = structure.dataForMocks
+    ? (noErrorAttributes(structure.dataForMocks) as JsonTypeDefinition)
+    : null;
+
+  const afterSimplify = afterNoError ? simplifyKeysForLLM(afterNoError) : null;
+
+  const cleanedDataForMocks = afterSimplify
+    ? collapseNullableObjects(afterSimplify)
     : null;
+
   const sortedDataForMocks =
     cleanedDataForMocks &&
     typeof cleanedDataForMocks === 'object' &&
@@ -111,6 +118,19 @@ ${JSON.stringify(defaultScenarioData.data, null, 2)}
       ? sortKeysBySize(cleanedDataForMocks as Record<string, any>)
       : cleanedDataForMocks;
 
+  // Log nullable-related content in final prompt JSON
+  if (sortedDataForMocks) {
+    const finalJson = JSON.stringify(sortedDataForMocks, null, 2);
+    const hasNullableMarker = finalJson.includes('_nullable');
+    const nullCollapsed = (finalJson.match(/"object \| null"/g) || []).length;
+    const arrayCollapsed = (finalJson.match(/"array \| null"/g) || []).length;
+    if (hasNullableMarker || nullCollapsed > 0 || arrayCollapsed > 0) {
+      console.log(
+        `[generateEntityScenarioDataGenerator] final prompt: ${nullCollapsed} "object | null", ${arrayCollapsed} "array | null"${hasNullableMarker ? ', WARNING: _nullable still present!' : ''}`,
+      );
+    }
+  }
+
   // When mockData has already been generated via chunks, skip it in the main call
   const mockDataSection = options?.mockDataAlreadyGenerated
     ? '## mockData Structure\nmockData has been pre-generated—return `{}` for mockData.'

package/analyzer-template/packages/analyze/src/lib/ProjectAnalyzer.ts

@@ -46,6 +46,12 @@ export class ProjectAnalyzer {
       lib.asts.createProgramFromTsConfigPath(configPath, excludePatterns),
     );
 
+    if (this.programs.length === 0) {
+      throw new Error(
+        'No TypeScript programs could be created — no tsconfig.json found',
+      );
+    }
+
     // Primary program is the first one (usually root) for backward compatibility
     this.program = this.programs[0];
     this.typeChecker = this.program.getTypeChecker();

package/analyzer-template/packages/analyze/src/lib/asts/nodes/getNodeType.ts

@@ -39,6 +39,7 @@ export function getNodeType(node: ts.Node, sourceFile: ts.SourceFile) {
     case 'StringLiteral':
     case 'NumericLiteral':
     case 'ObjectLiteralExpression':
+    case 'ArrayLiteralExpression':
       nodeType = 'data';
       break;
     case 'TypeAliasDeclaration':

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities/prepareDataStructures.ts

@@ -18,6 +18,7 @@ import {
   getCompoundConditionals,
   getChildBoundaryGatingConditions,
   getJsxRenderingUsages,
+  skipWorkerPool,
 } from '~codeyam/ai';
 import type { AnalysisOptions } from '../analyzeEntities';
 
@@ -271,6 +272,23 @@ export default async function prepareEntityDataStructures(
   };
 
   const entities = Array.from(allEntities);
+
+  // The worker thread creates its own ProjectAnalyzer (ts.Program) on first use,
+  // which takes 50-70s. For small entity counts the overhead far exceeds the actual
+  // processing time (<2s). Skip the worker and run on the main thread instead,
+  // reusing the existing ProjectAnalyzer.
+  const entitiesNeedingGeneration = entities.filter(
+    (e) =>
+      !e.metadata?.isolatedDataStructure || options.force || options.forceAll,
+  ).length;
+  const WORKER_ENTITY_THRESHOLD = 500;
+  if (entitiesNeedingGeneration < WORKER_ENTITY_THRESHOLD) {
+    console.log(
+      `CodeYam: Skipping worker thread for DS prep (${entitiesNeedingGeneration} entities < ${WORKER_ENTITY_THRESHOLD} threshold)`,
+    );
+    skipWorkerPool();
+  }
+
   const sequential =
     DEBUG_SEQUENTIAL_EXECUTION || options.sequentialDataStructurePrep;
 

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities.ts

@@ -310,7 +310,15 @@ export default async function analyzeEntities({
   );
 
   // Flush transformation trace if tracing is enabled
-  transformationTracer.flush();
+  // Wrapped in try/catch because trace data can exceed V8's max string length
+  // for large projects, and we must not crash the analysis process over tracing.
+  try {
+    transformationTracer.flush();
+  } catch (e) {
+    console.warn(
+      `CodeYam Warning: Failed to flush transformation trace: ${e.message}`,
+    );
+  }
 
   return analyses;
 

package/analyzer-template/packages/analyze/src/lib/files/analyze/dependencyResolver.ts

@@ -100,12 +100,6 @@ export function getCandidatesReadyForCapture(
       continue;
     }
 
-    // Skip notExported entities - they can't be imported into the simulation
-    // environment and will always fail capture. Don't mark them as ready.
-    if (candidate.entity?.metadata?.notExported) {
-      continue;
-    }
-
     // Get all transitive dependencies
     const candidateKey = `${candidate.filePath}:${candidate.entityName}`;
     const { missingDependencies } = getTransitiveDependencies(

package/analyzer-template/packages/analyze/src/lib/files/analyze/findOrCreateEntity.ts

@@ -138,6 +138,18 @@ export default async function findOrCreateEntity({
           break;
         }
       }
+
+      // Mark previous versions as superseded
+      const entitiesToSupersede = previousEntities.filter(
+        (e) => !e.metadata?.isSuperseded,
+      );
+      if (entitiesToSupersede.length > 0) {
+        for (const prev of entitiesToSupersede) {
+          prev.metadata = prev.metadata || {};
+          prev.metadata.isSuperseded = true;
+        }
+        await upsertEntities(entitiesToSupersede);
+      }
     }
 
     entity = {

package/analyzer-template/packages/analyze/src/lib/files/scenarios/TransformationTracer.ts

@@ -404,22 +404,15 @@ export class TransformationTracer {
   }
 
   /**
-   * Write the trace to the output file.
+   * Compute the summary data for the current traces.
    */
-  flush(): void {
-    if (!this.isEnabled()) return;
-    if (this.traces.size === 0) {
-      this.log('flush: no traces to write');
-      return;
-    }
-
-    const entities = Array.from(this.traces.keys());
-    const stageCounts = entities
-      .map((e) => `${e}(${this.traces.get(e)!.stages.length})`)
-      .join(', ');
-    this.log(`flush: writing ${entities.length} entities: ${stageCounts}`);
-
-    // Compute summary
+  private computeFlushSummary(): {
+    stageChangeCounts: Record<
+      string,
+      { added: number; removed: number; changed: number }
+    >;
+    entitiesWithMostChanges: string[];
+  } {
     const stageChangeCounts: Record<
       string,
       { added: number; removed: number; changed: number }
@@ -484,17 +477,39 @@ export class TransformationTracer {
       .slice(0, 10)
       .map(([name]) => name);
 
-    const traceFile: TraceFile = {
-      meta: {
-        timestamp: new Date().toISOString(),
-        projectSlug: this.projectSlug,
-        entityCount: this.traces.size,
-      },
-      summary: {
-        stageChangeCounts,
-        entitiesWithMostChanges,
-      },
-      entities: Object.fromEntries(this.traces),
+    return { stageChangeCounts, entitiesWithMostChanges };
+  }
+
+  /**
+   * Write the trace to the output file.
+   * Uses streaming per-entity writes to avoid exceeding V8's max string length
+   * on large projects.
+   */
+  flush(): void {
+    if (!this.isEnabled()) return;
+    if (this.traces.size === 0) {
+      this.log('flush: no traces to write');
+      return;
+    }
+
+    const entities = Array.from(this.traces.keys());
+    const stageCounts = entities
+      .map((e) => `${e}(${this.traces.get(e)!.stages.length})`)
+      .join(', ');
+    this.log(`flush: writing ${entities.length} entities: ${stageCounts}`);
+
+    const { stageChangeCounts, entitiesWithMostChanges } =
+      this.computeFlushSummary();
+
+    const meta = {
+      timestamp: new Date().toISOString(),
+      projectSlug: this.projectSlug,
+      entityCount: this.traces.size,
+    };
+
+    const summary = {
+      stageChangeCounts,
+      entitiesWithMostChanges,
     };
 
     // Ensure directory exists
@@ -503,8 +518,30 @@ export class TransformationTracer {
       fs.mkdirSync(dir, { recursive: true });
     }
 
-    fs.writeFileSync(this.outputPath, JSON.stringify(traceFile, null, 2));
-    this.log(`flush: wrote trace to ${this.outputPath}`);
+    // Stream JSON per-entity to avoid exceeding V8's max string length.
+    // Each entity is serialized individually so no single JSON.stringify
+    // call needs to produce the entire trace.
+    const fd = fs.openSync(this.outputPath, 'w');
+    try {
+      fs.writeSync(fd, '{\n  "meta": ');
+      fs.writeSync(fd, JSON.stringify(meta, null, 2));
+      fs.writeSync(fd, ',\n  "summary": ');
+      fs.writeSync(fd, JSON.stringify(summary, null, 2));
+      fs.writeSync(fd, ',\n  "entities": {');
+
+      let first = true;
+      for (const [entityName, entityTrace] of this.traces) {
+        if (!first) fs.writeSync(fd, ',');
+        fs.writeSync(fd, `\n    ${JSON.stringify(entityName)}: `);
+        fs.writeSync(fd, JSON.stringify(entityTrace, null, 2));
+        first = false;
+      }
+
+      fs.writeSync(fd, '\n  }\n}\n');
+      this.log(`flush: wrote trace to ${this.outputPath}`);
+    } finally {
+      fs.closeSync(fd);
+    }
   }
 
   /**

package/analyzer-template/packages/analyze/src/lib/files/scenarios/gatherDataForMocks.ts

@@ -353,8 +353,6 @@ function processCall(
   // Preprocess to filter signatures and extract functionCallReturnValue
   let preprocessedSchema = preprocessSchemaForMocks(mergedSchema);
 
-  const functionNames: string[] = [];
-
   const isFunctionNameWithoutFunction = (
     path: string,
     functionName: string,
@@ -439,31 +437,58 @@ function processCall(
     }
   }
 
-  const relevantMergedDependencySchema = Object.keys(preprocessedSchema).reduce(
-    (acc, path) => {
-      if (path.endsWith(')')) {
-        const pathParts = splitOutsideParenthesesAndArrays(path);
-        const functionName = joinParenthesesAndArrays([
-          ...pathParts.slice(0, -1),
-          pathParts[pathParts.length - 1].split('(')[0],
-        ]);
-        functionNames.push(functionName);
-        for (const existingPath in acc) {
-          if (isFunctionNameWithoutFunction(existingPath, functionName)) {
-            delete acc[existingPath];
-          }
-        }
-      }
+  // Two-pass function name filtering to avoid O(k²) retroactive deletion.
+  // Pass 1: Discover all unique function names from paths ending with ')'
+  const functionNameSet = new Set<string>();
+  const schemaKeys = Object.keys(preprocessedSchema);
+  for (const path of schemaKeys) {
+    if (path.endsWith(')')) {
+      const pathParts = splitOutsideParenthesesAndArrays(path);
+      const functionName = joinParenthesesAndArrays([
+        ...pathParts.slice(0, -1),
+        pathParts[pathParts.length - 1].split('(')[0],
+      ]);
+      functionNameSet.add(functionName);
+    }
+  }
 
-      if (functionNames.some((fn) => isFunctionNameWithoutFunction(path, fn))) {
-        return acc;
-      }
+  // Index function names by first segment for O(1) bucket lookup instead of
+  // checking all function names for every path. Most paths only need to check
+  // 1-2 function names instead of all F.
+  const fnByFirstSegment = new Map<string, string[]>();
+  for (const fn of functionNameSet) {
+    const dotIdx = fn.indexOf('.');
+    const firstSeg = dotIdx >= 0 ? fn.slice(0, dotIdx) : fn;
+    let bucket = fnByFirstSegment.get(firstSeg);
+    if (!bucket) {
+      bucket = [];
+      fnByFirstSegment.set(firstSeg, bucket);
+    }
+    bucket.push(fn);
+  }
 
-      acc[path] = preprocessedSchema[path];
-      return acc;
-    },
-    {} as Record<string, string>,
-  );
+  // Pass 2: Filter paths that match any discovered function name prefix.
+  // Use indexed lookup: extract path's first segment, check only matching function names.
+  const relevantMergedDependencySchema: Record<string, string> = {};
+  for (const path of schemaKeys) {
+    // Extract first segment of path (before first '.', '(', or '<')
+    const dotIdx = path.indexOf('.');
+    const parenIdx = path.indexOf('(');
+    const angleIdx = path.indexOf('<');
+    let minIdx = path.length;
+    if (dotIdx >= 0 && dotIdx < minIdx) minIdx = dotIdx;
+    if (parenIdx >= 0 && parenIdx < minIdx) minIdx = parenIdx;
+    if (angleIdx >= 0 && angleIdx < minIdx) minIdx = angleIdx;
+    const firstSeg = path.slice(0, minIdx);
+
+    const candidates = fnByFirstSegment.get(firstSeg);
+    if (
+      !candidates ||
+      !candidates.some((fn) => isFunctionNameWithoutFunction(path, fn))
+    ) {
+      relevantMergedDependencySchema[path] = preprocessedSchema[path];
+    }
+  }
 
   const filledSchema = fillInDirectSchemaGapsAndUnknowns({
     scopeName: importedExport.name,

package/analyzer-template/packages/analyze/src/lib/files/scenarios/generateDataStructure.ts

@@ -461,6 +461,17 @@ export default function generateDataStructure({
 
     const { isolatedDataStructure } = entity.metadata;
 
+    if (!isolatedDataStructure) {
+      awsLog(
+        'CodeYam: Skipping data structure generation — isolatedDataStructure is undefined (scope analysis may have failed)',
+        {
+          entityName: entity.name,
+          filePath: entity.filePath,
+        },
+      );
+      return analysis;
+    }
+
     // Transformation tracing: start entity and capture isolated data structure
     transformationTracer.startEntity({
       name: entity.name,
@@ -643,6 +654,78 @@ export default function generateDataStructure({
           },
         );
 
+        // Infer function types from child component usage BEFORE fillInSchemaGapsAndUnknowns.
+        // When a hook returns a property with type 'unknown' that is passed to a child
+        // component which calls it as a function (e.g., isEntityPending(entity)),
+        // promote it to 'function'. Without this, name heuristics like isLikelyBoolean
+        // would mistype "isEntityPending" as boolean because of the "is" prefix.
+        //
+        // sourceEquivalencies are in isolatedDataStructure.dependencySchemas (not in
+        // mergedDataStructure.dependencySchemas — they're dropped during the merge).
+        // The child's signatureSchema (with function call evidence) IS in the merged schemas.
+        {
+          const isolatedDepSchemas =
+            entity.metadata.isolatedDataStructure?.dependencySchemas;
+          if (isolatedDepSchemas) {
+            for (const childFilePath in isolatedDepSchemas) {
+              const childIsolated = isolatedDepSchemas[childFilePath];
+              for (const childName in childIsolated) {
+                const sourceEq = childIsolated[childName]?.sourceEquivalencies;
+                if (!sourceEq) continue;
+
+                // Get the child's signatureSchema from the MERGED schemas (richer)
+                const mergedChildSchema =
+                  mergedDataStructure.dependencySchemas[childFilePath]?.[
+                    childName
+                  ];
+                const sigSchema = mergedChildSchema?.signatureSchema;
+                if (!sigSchema) continue;
+
+                for (const childPath in sourceEq) {
+                  for (const source of sourceEq[childPath]) {
+                    const fcrvSuffix = '.functionCallReturnValue.';
+                    const fcrvIndex = source.schemaPath.indexOf(fcrvSuffix);
+                    if (fcrvIndex === -1) continue;
+
+                    const propName = source.schemaPath.slice(
+                      fcrvIndex + fcrvSuffix.length,
+                    );
+
+                    // Find the corresponding child signature path
+                    const entityCallPrefix = childName + '().';
+                    const sigPathBase = childPath.startsWith(entityCallPrefix)
+                      ? childPath.slice(entityCallPrefix.length)
+                      : childPath;
+
+                    // Check if the child's signatureSchema has a function call version
+                    let isFunction = false;
+                    for (const sigKey in sigSchema) {
+                      if (
+                        sigKey.startsWith(sigPathBase + '(') &&
+                        sigSchema[sigKey] === 'function'
+                      ) {
+                        isFunction = true;
+                        break;
+                      }
+                    }
+
+                    if (isFunction) {
+                      for (const rvKey in depSchema.returnValueSchema) {
+                        if (
+                          rvKey.endsWith('.' + propName) &&
+                          depSchema.returnValueSchema[rvKey] === 'unknown'
+                        ) {
+                          depSchema.returnValueSchema[rvKey] = 'function';
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+
         // Fill in type gaps BEFORE clearing attributes, so evidence like .includes()
         // is used for type inference before being deleted