@codeyam/codeyam-cli 0.1.0-staging.bbe4da9 → 0.1.0-staging.c9dc00c

package/analyzer-template/packages/ai/src/lib/analyzeScope.ts

@@ -24,7 +24,7 @@ import mergeStatements, { Statement } from './mergeStatements';
 import { analyzeScopeWithAst } from './astScopes';
 import { FileAnalyzer } from '~codeyam/analyze';
 import { writeFile } from 'fs/promises';
-import { saveEntityStatements } from '~codeyam/supabase';
+import { saveEntityStatements } from '~codeyam/database';
 import {
   ScopeDataStructure,
   ScopeInfo,
@@ -32,6 +32,7 @@ import {
 import FunctionCallManager from './dataStructure/equivalencyManagers/FunctionCallManager';
 import { ReactFrameworkManager } from './dataStructure/equivalencyManagers/frameworks/ReactFrameworkManager';
 import { JavascriptFrameworkManager } from './dataStructure/equivalencyManagers/frameworks/JavascriptFrameworkManager';
+import { MuiManager } from './dataStructure/equivalencyManagers/frameworks/MuiManager';
 import ParentScopeManager from './dataStructure/equivalencyManagers/ParentScopeManager';
 import { AI } from '~codeyam/ai';
 import Piscina from 'piscina';
@@ -48,6 +49,7 @@ let workerPool: Piscina<
   AnalyzeScopeWorkerInput,
   AnalyzeScopeWorkerOutput
 > | null = null;
+let workerPoolDestroyed = false;
 
 /**
  * Check if we're in Node.js main thread
@@ -64,11 +66,48 @@ function getIsMainThread(): boolean {
   return isMainThread;
 }
 
+/**
+ * Destroy the worker pool, freeing its ~2GB heap.
+ * Called after data structure preparation completes (the producing phase transition).
+ * The worker is not needed after this point — subsequent calls to ensureWorkerPool() will throw.
+ */
+export async function destroyWorkerPool() {
+  if (workerPool) {
+    const memBefore = process.memoryUsage();
+    console.log('CodeYam: Destroying worker pool...', {
+      rssBefore: `${Math.round(memBefore.rss / 1024 / 1024)}MB`,
+      heapBefore: `${Math.round(memBefore.heapUsed / 1024 / 1024)}MB`,
+    });
+
+    await workerPool.destroy();
+    workerPool = null;
+
+    // Force GC if available to reclaim worker memory faster
+    if (global.gc) {
+      global.gc();
+    }
+
+    const memAfter = process.memoryUsage();
+    console.log('CodeYam: Worker pool destroyed', {
+      rssAfter: `${Math.round(memAfter.rss / 1024 / 1024)}MB`,
+      heapAfter: `${Math.round(memAfter.heapUsed / 1024 / 1024)}MB`,
+      rssDelta: `${Math.round((memAfter.rss - memBefore.rss) / 1024 / 1024)}MB`,
+    });
+  }
+  workerPoolDestroyed = true;
+}
+
 /**
  * Lazy initialization of worker pool.
  * Called on first use to ensure environment variables are set.
  */
 function ensureWorkerPool() {
+  if (workerPoolDestroyed) {
+    throw new Error(
+      'CodeYam: Worker pool was destroyed (post-producing phase). ' +
+        'analyzeScope should not be called after data structure preparation completes.',
+    );
+  }
   if (workerPool !== null) return;
 
   // Only initialize worker pool in Node.js environment
@@ -84,10 +123,17 @@ function ensureWorkerPool() {
   // Note: WORKER_PATH is null when __mocks__/analyzeScope.ts is active
   const { WORKER_PATH } = analyzeScopeWorkerPaths();
 
+  // Disable workers when tracing is enabled - we need access to ScopeDataStructure
+  // instance in the main thread to capture pre-serialization state
+  const tracingEnabled =
+    process.env.CODEYAM_TRACE_TRANSFORMS === '1' ||
+    process.env.CODEYAM_TRACE_TRANSFORMS === 'true';
+
   if (
     process.env.USE_WORKER_THREADS &&
     WORKER_PATH &&
-    fs.existsSync(WORKER_PATH)
+    fs.existsSync(WORKER_PATH) &&
+    !tracingEnabled
   ) {
     console.log('CodeYam: Using worker threads for analyzeScope', {
       WORKER_PATH,
@@ -120,6 +166,7 @@ function ensureWorkerPool() {
       WORKER_PATH,
       workerExists: WORKER_PATH ? fs.existsSync(WORKER_PATH) : false,
       USE_WORKER_THREADS: process.env.USE_WORKER_THREADS,
+      tracingEnabled,
     });
     workerPool = null; // Explicitly set to indicate "no workers"
   }
@@ -163,13 +210,27 @@ export async function analyzeScopeLocal({
     // Collect conditional usages from all statement analyses
     const allConditionalUsages: Record<
       string,
-      Array<{
-        path: string;
-        conditionType: 'truthiness' | 'comparison' | 'switch';
-        comparedValues?: string[];
-        location: 'if' | 'ternary' | 'logical-and' | 'switch';
-      }>
+      import('./astScopes/types').ConditionalUsage[]
     > = {};
+    // Collect conditional effects (setter calls inside conditionals)
+    const allConditionalEffects: import('./astScopes/types').ConditionalEffect[] =
+      [];
+    // Collect compound conditionals (grouped conditions that must all be true)
+    const allCompoundConditionals: import('./astScopes/types').CompoundConditional[] =
+      [];
+    // Collect derived variables (variables computed from boolean expressions)
+    const allDerivedVariables: Record<
+      string,
+      import('./astScopes/types').DerivedVariableInfo
+    > = {};
+    // Collect child boundary gating conditions across all statements
+    const allChildBoundaryGatingConditions: Record<
+      string,
+      import('./astScopes/types').ConditionalUsage[]
+    > = {};
+    // Collect JSX rendering usages (arrays via .map(), strings via interpolation)
+    const allJsxRenderingUsages: import('./astScopes/types').JsxRenderingUsage[] =
+      [];
 
     const astAnalysisResultWithStatements = statementScopes.map(
       (statementScope) => {
@@ -190,6 +251,39 @@ export async function analyzeScopeLocal({
           }
           allConditionalUsages[path].push(...usages);
         }
+        // Collect conditional effects from AST analysis
+        if (astAnalysisResult.conditionalEffects?.length > 0) {
+          allConditionalEffects.push(...astAnalysisResult.conditionalEffects);
+        }
+        // Collect compound conditionals from AST analysis
+        if (astAnalysisResult.compoundConditionals?.length > 0) {
+          allCompoundConditionals.push(
+            ...astAnalysisResult.compoundConditionals,
+          );
+        }
+        // Collect derived variables from AST analysis
+        if (astAnalysisResult.derivedVariables) {
+          for (const [varName, derivationInfo] of Object.entries(
+            astAnalysisResult.derivedVariables,
+          )) {
+            allDerivedVariables[varName] = derivationInfo;
+          }
+        }
+        // Collect child boundary gating conditions from AST analysis
+        if (astAnalysisResult.childBoundaryGatingConditions) {
+          for (const [childName, usages] of Object.entries(
+            astAnalysisResult.childBoundaryGatingConditions,
+          )) {
+            if (!allChildBoundaryGatingConditions[childName]) {
+              allChildBoundaryGatingConditions[childName] = [];
+            }
+            allChildBoundaryGatingConditions[childName].push(...usages);
+          }
+        }
+        // Collect JSX rendering usages from AST analysis
+        if (astAnalysisResult.jsxRenderingUsages?.length) {
+          allJsxRenderingUsages.push(...astAnalysisResult.jsxRenderingUsages);
+        }
         return {
           statementScope,
           astAnalysisResult,
@@ -259,8 +353,34 @@ export async function analyzeScopeLocal({
               },
             );
 
+            // DEBUG: Log child scope replacement context for fetcher statements
+            if (
+              statementScope.text.includes('Fetcher') ||
+              statementScope.text.includes('fetcher')
+            ) {
+              console.log(
+                'CodeYam DEBUG childScope replacement:',
+                JSON.stringify(
+                  {
+                    statementText: statementScope.text.slice(0, 80),
+                    statementStart: statementScope.start,
+                    statementEnd: statementScope.end,
+                    childBoundaries: statementScope.childBoundaries,
+                    relevantChildScopes: Object.keys(relevantChildScopes),
+                    astStructure: astAnalysisResult.structure,
+                    astEquivalentVariables:
+                      astAnalysisResult.equivalentVariables,
+                    astIsComplete: astAnalysisResult.isComplete,
+                    equivalentVariables,
+                  },
+                  null,
+                  2,
+                ),
+              );
+            }
+
             for (const mapping of [structure, equivalentVariables]) {
-              for (let [key, value] of Object.entries(mapping)) {
+              for (let [key, rawValue] of Object.entries(mapping)) {
                 for (const [childScopeId, childScope] of sortedEntries) {
                   const scopeText = fileAnalyzer.sourceFile.text.slice(
                     childScope.start,
@@ -280,15 +400,49 @@ export async function analyzeScopeLocal({
                     return text;
                   };
 
-                  if (value.includes(scopeText)) {
-                    value = replaceChildScopeId(value);
-                    mapping[key] = value;
+                  // Handle both string and string[] values (for OR expressions)
+                  const processValue = (value: string): string => {
+                    if (value.includes(scopeText)) {
+                      // DEBUG: Log when replacement happens
+                      if (key.includes('Fetcher') || key.includes('fetcher')) {
+                        console.log(
+                          'CodeYam DEBUG value replacement:',
+                          JSON.stringify({
+                            key,
+                            oldValue: value,
+                            childScopeId,
+                            scopeTextPreview: scopeText.slice(0, 50),
+                          }),
+                        );
+                      }
+                      return replaceChildScopeId(value);
+                    }
+                    return value;
+                  };
+
+                  if (Array.isArray(rawValue)) {
+                    const newValues = rawValue.map(processValue);
+                    (mapping as Record<string, string | string[]>)[key] =
+                      newValues;
+                    rawValue = newValues;
+                  } else if (typeof rawValue === 'string') {
+                    const newValue = processValue(rawValue);
+                    if (newValue !== rawValue) {
+                      (mapping as Record<string, string | string[]>)[key] =
+                        newValue;
+                      rawValue = newValue;
+                    }
                   }
+
                   if (key.includes(scopeText)) {
                     const oldKey = key;
                     key = replaceChildScopeId(key);
-                    mapping[key] = mapping[oldKey];
-                    delete mapping[oldKey];
+                    (mapping as Record<string, string | string[]>)[key] = (
+                      mapping as Record<string, string | string[]>
+                    )[oldKey];
+                    delete (mapping as Record<string, string | string[]>)[
+                      oldKey
+                    ];
                   }
                 }
               }
@@ -435,6 +589,7 @@ export async function analyzeScopeLocal({
           new FunctionCallManager(),
           new ReactFrameworkManager(),
           new JavascriptFrameworkManager(),
+          new MuiManager(),
           new ParentScopeManager(),
         ],
         scope.name,
@@ -451,11 +606,83 @@ export async function analyzeScopeLocal({
       );
     }
 
+    // Enrich conditional usages with derived variable info
+    // This must happen after all statements are processed so derived variables from
+    // earlier statements can be used to enrich usages from later statements
+    if (Object.keys(allDerivedVariables).length > 0) {
+      for (const [path, usages] of Object.entries(allConditionalUsages)) {
+        for (const usage of usages) {
+          if (!usage.derivedFrom && allDerivedVariables[path]) {
+            usage.derivedFrom = allDerivedVariables[path];
+          }
+        }
+      }
+    }
+
     // Add conditional usages from this scope's analysis
     if (Object.keys(allConditionalUsages).length > 0) {
       scopeDataStructure.addConditionalUsages(allConditionalUsages);
     }
 
+    // Add conditional effects from this scope's analysis
+    if (allConditionalEffects.length > 0) {
+      scopeDataStructure.addConditionalEffects(allConditionalEffects);
+    }
+
+    // Add compound conditionals from this scope's analysis
+    if (allCompoundConditionals.length > 0) {
+      scopeDataStructure.addCompoundConditionals(allCompoundConditionals);
+    }
+
+    // Merge extracted gating conditions from JSX extraction (processJSXForScope)
+    // These were detected BEFORE JSX was simplified, so they have the full context
+    if (scope.extractedGatingConditions) {
+      for (const [childName, conditions] of Object.entries(
+        scope.extractedGatingConditions,
+      )) {
+        if (!allChildBoundaryGatingConditions[childName]) {
+          allChildBoundaryGatingConditions[childName] = [];
+        }
+        for (const condition of conditions) {
+          // Convert to full ConditionalUsage format
+          allChildBoundaryGatingConditions[childName].push({
+            path: condition.path,
+            conditionType: condition.conditionType,
+            location: condition.location,
+            isNegated: condition.isNegated,
+            controlsJsxRendering: true,
+          });
+        }
+      }
+    }
+
+    // Enrich child boundary gating conditions with derived variable info
+    // This must happen after all statements are processed so derived variables from
+    // earlier statements can be used to enrich gating conditions from later statements
+    if (Object.keys(allDerivedVariables).length > 0) {
+      for (const [_childName, conditions] of Object.entries(
+        allChildBoundaryGatingConditions,
+      )) {
+        for (const condition of conditions) {
+          if (!condition.derivedFrom && allDerivedVariables[condition.path]) {
+            condition.derivedFrom = allDerivedVariables[condition.path];
+          }
+        }
+      }
+    }
+
+    // Add child boundary gating conditions from this scope's analysis
+    if (Object.keys(allChildBoundaryGatingConditions).length > 0) {
+      scopeDataStructure.addChildBoundaryGatingConditions(
+        allChildBoundaryGatingConditions,
+      );
+    }
+
+    // Add JSX rendering usages from this scope's analysis
+    if (allJsxRenderingUsages.length > 0) {
+      scopeDataStructure.addJsxRenderingUsages(allJsxRenderingUsages);
+    }
+
     await Promise.all(
       Object.values(scope.childScopes).map(async (childScope) => {
         // This is awkward but need to identify class scopes
@@ -534,6 +761,7 @@ export default async function analyzeScope(
         end: scope.end,
         variables: scope.variables,
         childScopes: scope.childScopes,
+        extractedGatingConditions: scope.extractedGatingConditions,
       },
       model: args.model,
     };

package/analyzer-template/packages/ai/src/lib/astScopes/arrayDerivationDetector.ts

@@ -0,0 +1,199 @@
+/**
+ * Shared utilities for detecting array-derived expressions in conditionals.
+ *
+ * These utilities detect patterns like:
+ * - arr.some(fn) → { sourcePath: 'arr', operation: 'arraySome' }
+ * - arr.every(fn) → { sourcePath: 'arr', operation: 'arrayEvery' }
+ * - arr.includes(x) → { sourcePath: 'arr', operation: 'arrayIncludes' }
+ * - arr.length → { sourcePath: 'arr', operation: 'arrayLength' }
+ *
+ * Used by both:
+ * - variableDeclarationHandler.ts (for variable assignments)
+ * - processExpression.ts (for inline conditionals)
+ */
+
+import ts from 'typescript';
+
+/** Array predicate method operations */
+export type ArrayPredicateOperation =
+  | 'arrayIncludes'
+  | 'arraySome'
+  | 'arrayEvery';
+
+/** Array length operation */
+export type ArrayLengthOperation = 'arrayLength';
+
+/** All array-derived operations */
+export type ArrayDerivedOperation =
+  | ArrayPredicateOperation
+  | ArrayLengthOperation;
+
+/** Result of detecting an array derivation pattern */
+export interface ArrayDerivationResult {
+  sourcePath: string;
+  operation: ArrayDerivedOperation;
+}
+
+/**
+ * Extracts a simple path string from an expression.
+ * Returns the path for identifiers and property access chains.
+ */
+export function getSimplePath(node: ts.Expression): string | null {
+  if (ts.isIdentifier(node)) {
+    return node.text;
+  }
+  if (ts.isPropertyAccessExpression(node)) {
+    const objectPath = getSimplePath(node.expression);
+    if (objectPath) {
+      return `${objectPath}.${node.name.text}`;
+    }
+  }
+  // Handle optional chaining: a?.b
+  if (ts.isNonNullExpression(node)) {
+    return getSimplePath(node.expression);
+  }
+  return null;
+}
+
+/**
+ * Maps array method names to derivation operation names
+ */
+function getArrayPredicateOperation(
+  methodName: string,
+): ArrayPredicateOperation | null {
+  switch (methodName) {
+    case 'includes':
+      return 'arrayIncludes';
+    case 'some':
+      return 'arraySome';
+    case 'every':
+      return 'arrayEvery';
+    default:
+      return null;
+  }
+}
+
+/**
+ * Detects if an expression is an array predicate call and extracts the array path.
+ * Patterns detected:
+ * - arr.includes(x) → { sourcePath: 'arr', operation: 'arrayIncludes' }
+ * - arr?.includes(x) → { sourcePath: 'arr', operation: 'arrayIncludes' }
+ * - arr.some(fn) → { sourcePath: 'arr', operation: 'arraySome' }
+ * - arr.every(fn) → { sourcePath: 'arr', operation: 'arrayEvery' }
+ * - Boolean(x && arr?.includes(y)) → { sourcePath: 'arr', operation: 'arrayIncludes' }
+ */
+export function detectArrayPredicatePattern(
+  expr: ts.Expression,
+): ArrayDerivationResult | null {
+  // Unwrap Boolean() calls: Boolean(x) → x
+  if (
+    ts.isCallExpression(expr) &&
+    ts.isIdentifier(expr.expression) &&
+    expr.expression.text === 'Boolean' &&
+    expr.arguments.length === 1
+  ) {
+    return detectArrayPredicatePattern(expr.arguments[0]);
+  }
+
+  // Handle logical AND: a && b - check both sides for array predicates
+  if (
+    ts.isBinaryExpression(expr) &&
+    expr.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken
+  ) {
+    // Try the right side first (more likely to have the predicate)
+    const rightResult = detectArrayPredicatePattern(expr.right);
+    if (rightResult) return rightResult;
+    // Try the left side
+    const leftResult = detectArrayPredicatePattern(expr.left);
+    if (leftResult) return leftResult;
+    return null;
+  }
+
+  // Handle parenthesized expressions
+  if (ts.isParenthesizedExpression(expr)) {
+    return detectArrayPredicatePattern(expr.expression);
+  }
+
+  if (!ts.isCallExpression(expr)) {
+    return null;
+  }
+
+  const callExpr = expr.expression;
+
+  // Handle both regular property access (arr.includes) and optional chaining (arr?.includes)
+  let objectExpr: ts.Expression | null = null;
+  let methodName: string | null = null;
+
+  if (ts.isPropertyAccessExpression(callExpr)) {
+    objectExpr = callExpr.expression;
+    methodName = callExpr.name.text;
+  } else if (
+    ts.isElementAccessExpression(callExpr) &&
+    ts.isStringLiteral(callExpr.argumentExpression)
+  ) {
+    // Handle arr["includes"]() syntax (rare but possible)
+    objectExpr = callExpr.expression;
+    methodName = callExpr.argumentExpression.text;
+  }
+
+  if (!objectExpr || !methodName) {
+    return null;
+  }
+
+  const operation = getArrayPredicateOperation(methodName);
+  if (!operation) {
+    return null;
+  }
+
+  // Get the source path for the array
+  const sourcePath = getSimplePath(objectExpr);
+  if (!sourcePath) {
+    return null;
+  }
+
+  return { sourcePath, operation };
+}
+
+/**
+ * Detects if an expression is an array.length access.
+ * Patterns detected:
+ * - arr.length → { sourcePath: 'arr', operation: 'arrayLength' }
+ * - arr?.length → { sourcePath: 'arr', operation: 'arrayLength' }
+ */
+export function detectArrayLengthPattern(
+  expr: ts.Expression,
+): ArrayDerivationResult | null {
+  // Handle property access: arr.length
+  if (ts.isPropertyAccessExpression(expr)) {
+    if (expr.name.text === 'length') {
+      const sourcePath = getSimplePath(expr.expression);
+      if (sourcePath) {
+        return { sourcePath, operation: 'arrayLength' };
+      }
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Detects any array-derived pattern (predicate methods or .length).
+ * This is the main entry point for inline conditional detection.
+ */
+export function detectArrayDerivedPattern(
+  expr: ts.Expression,
+): ArrayDerivationResult | null {
+  // Check for array predicate methods first (.some, .every, .includes)
+  const predicateResult = detectArrayPredicatePattern(expr);
+  if (predicateResult) {
+    return predicateResult;
+  }
+
+  // Check for .length access
+  const lengthResult = detectArrayLengthPattern(expr);
+  if (lengthResult) {
+    return lengthResult;
+  }
+
+  return null;
+}