@codeyam/codeyam-cli 0.1.0-staging.1 → 0.1.0-staging.596f0eb

package/analyzer-template/packages/ai/src/lib/analyzeScope.ts

@@ -24,7 +24,7 @@ import mergeStatements, { Statement } from './mergeStatements';
 import { analyzeScopeWithAst } from './astScopes';
 import { FileAnalyzer } from '~codeyam/analyze';
 import { writeFile } from 'fs/promises';
-import { saveEntityStatements } from '~codeyam/supabase';
+import { saveEntityStatements } from '~codeyam/database';
 import {
   ScopeDataStructure,
   ScopeInfo,
@@ -160,6 +160,16 @@ export async function analyzeScopeLocal({
 
     const llmCalls: any[] = [];
     const allEnvironmentVariables: Set<string> = new Set();
+    // Collect conditional usages from all statement analyses
+    const allConditionalUsages: Record<
+      string,
+      Array<{
+        path: string;
+        conditionType: 'truthiness' | 'comparison' | 'switch';
+        comparedValues?: string[];
+        location: 'if' | 'ternary' | 'logical-and' | 'switch';
+      }>
+    > = {};
 
     const astAnalysisResultWithStatements = statementScopes.map(
       (statementScope) => {
@@ -171,6 +181,15 @@ export async function analyzeScopeLocal({
         astAnalysisResult.environmentVariables.forEach((envVar) =>
           allEnvironmentVariables.add(envVar),
         );
+        // Collect conditional usages from AST analysis
+        for (const [path, usages] of Object.entries(
+          astAnalysisResult.conditionalUsages,
+        )) {
+          if (!allConditionalUsages[path]) {
+            allConditionalUsages[path] = [];
+          }
+          allConditionalUsages[path].push(...usages);
+        }
         return {
           statementScope,
           astAnalysisResult,
@@ -240,6 +259,32 @@ export async function analyzeScopeLocal({
               },
             );
 
+            // DEBUG: Log child scope replacement context for fetcher statements
+            if (
+              statementScope.text.includes('Fetcher') ||
+              statementScope.text.includes('fetcher')
+            ) {
+              console.log(
+                'CodeYam DEBUG childScope replacement:',
+                JSON.stringify(
+                  {
+                    statementText: statementScope.text.slice(0, 80),
+                    statementStart: statementScope.start,
+                    statementEnd: statementScope.end,
+                    childBoundaries: statementScope.childBoundaries,
+                    relevantChildScopes: Object.keys(relevantChildScopes),
+                    astStructure: astAnalysisResult.structure,
+                    astEquivalentVariables:
+                      astAnalysisResult.equivalentVariables,
+                    astIsComplete: astAnalysisResult.isComplete,
+                    equivalentVariables,
+                  },
+                  null,
+                  2,
+                ),
+              );
+            }
+
             for (const mapping of [structure, equivalentVariables]) {
               for (let [key, value] of Object.entries(mapping)) {
                 for (const [childScopeId, childScope] of sortedEntries) {
@@ -262,6 +307,18 @@ export async function analyzeScopeLocal({
                   };
 
                   if (value.includes(scopeText)) {
+                    // DEBUG: Log when replacement happens
+                    if (key.includes('Fetcher') || key.includes('fetcher')) {
+                      console.log(
+                        'CodeYam DEBUG value replacement:',
+                        JSON.stringify({
+                          key,
+                          oldValue: value,
+                          childScopeId,
+                          scopeTextPreview: scopeText.slice(0, 50),
+                        }),
+                      );
+                    }
                     value = replaceChildScopeId(value);
                     mapping[key] = value;
                   }
@@ -432,6 +489,11 @@ export async function analyzeScopeLocal({
       );
     }
 
+    // Add conditional usages from this scope's analysis
+    if (Object.keys(allConditionalUsages).length > 0) {
+      scopeDataStructure.addConditionalUsages(allConditionalUsages);
+    }
+
     await Promise.all(
       Object.values(scope.childScopes).map(async (childScope) => {
         // This is awkward but need to identify class scopes

package/analyzer-template/packages/ai/src/lib/astScopes/astScopeAnalyzer.ts

@@ -155,9 +155,6 @@ export class ASTScopeAnalyzer {
    * Performs the analysis by identifying and processing patterns in the AST
    */
   analyze(): AstScopeAnalysisResult {
-    // Debug log the source code
-    // console.log('Analyzing source code:', this.scope.text);
-
     // Create analysis context
     const context: AnalysisContext = this.createAnalysisContext();
 
@@ -167,38 +164,9 @@ export class ASTScopeAnalyzer {
       this.statementInfo.start,
       this.statementInfo.end,
     );
-    // console.info('SOURCEFILE', this.sourceFile.getText());
-    // console.info('RANGE', this.statementInfo.start, this.statementInfo.end, this.sourceFile.getText().slice(this.statementInfo.start, this.statementInfo.end));
-
-    // console.info('RELEVANT NODES:\n\n', this.statementInfo.text);
-    // relevantNodes.forEach((node, i) =>
-    //   console.info(`RELEVANT NODE ${i}`, node.getText(this.sourceFile)),
-    // );
 
-    // console.info(
-    //   'CHILD BOUNDARIES',
-    //   this.statementInfo.childBoundaries.map((b) =>
-    //     this.sourceFile.getText().slice(b.start, b.end),
-    //   ),
-    // );
     this.processNodes(relevantNodes, context);
 
-    // for (const childBoundary of this.statementInfo.childBoundaries) {
-    //   const boundaryText = this.sourceFile.getText().slice(childBoundary.start, childBoundary.end);
-    //   console.info('CHILD', boundaryText);
-    //   for (const equivalentVariableKey in this.equivalentVariables) {
-    //     const equivalentVariableValue = this.equivalentVariables[equivalentVariableKey];
-    //     for (const path of [equivalentVariableKey, equivalentVariableValue]) {
-    //       console.info('CHECKING', path);
-    //       console.info("REPLACE", path.replace(boundaryText, ''))
-    //       if (path.replace(boundaryText, '').replace(/[\s\(\)]*/g, '') === '') {
-    //         this.equivalentVariables[equivalentVariableKey] = 'cyScope3'
-    //         delete this.equivalentVariables[equivalentVariableKey];
-    //       }
-    //     }
-    //   }
-    // }
-
     // Calculate completeness
     const completeness = this.calculateCompleteness();
 
@@ -277,6 +245,7 @@ export class ASTScopeAnalyzer {
         processExpression({ node, context, targetPath }),
       addConditionalUsage: (usage) => this.addConditionalUsage(usage),
       getConditionalUsages: () => this.conditionalUsages,
+      updateSchemaType: (path, newType) => this.updateSchemaType(path, newType),
     };
     return context;
   }
@@ -285,12 +254,13 @@ export class ASTScopeAnalyzer {
     node: ts.Node,
     sourceFile: ts.SourceFile,
   ): { nodeStart: number; nodeEnd: number } {
-    const sourceText = sourceFile.text;
-    const sourceGetText = sourceFile.getText();
-    const leadingContentOffset = sourceText.length - sourceGetText.length;
+    // Use getStart(sourceFile) to get the position after leading trivia.
+    // Note: We intentionally don't adjust for leadingContentOffset here because
+    // the positions from isolateStatements are calculated using getStart() too,
+    // which uses the same sourceFile.text coordinate system.
     return {
-      nodeStart: node.getStart(sourceFile) - leadingContentOffset,
-      nodeEnd: node.getEnd() - leadingContentOffset,
+      nodeStart: node.getStart(sourceFile),
+      nodeEnd: node.getEnd(),
     };
   }
 
@@ -658,6 +628,46 @@ export class ASTScopeAnalyzer {
     }
   }
 
+  /**
+   * Updates the schema type for a path, typically used when switch statements
+   * reveal the valid values for a parameter (creating a union type like "'a' | 'b' | 'c'")
+   *
+   * This method:
+   * 1. Updates the type for the given path
+   * 2. Also updates the type for any equivalent path (e.g., if "state" -> "signature[0]",
+   *    we update both "state" and "signature[0]")
+   * 3. For property paths like "data.status", if "data" -> "signature[0]",
+   *    we also update "signature[0].status"
+   */
+  private updateSchemaType(
+    path: string | StructuredPath,
+    newType: string,
+  ): void {
+    const pathStr = path.toString();
+
+    // Update the type for the given path
+    this.structure[pathStr] = newType;
+
+    // Also update the type for any equivalent path
+    // e.g., if "state" -> "signature[0]", update "signature[0]" as well
+    const equivalentPath = this.equivalentVariables[pathStr];
+    if (equivalentPath) {
+      this.structure[equivalentPath] = newType;
+    }
+
+    // For property paths like "data.status", check if the base has an equivalence
+    // e.g., if "data" -> "signature[0]", transform "data.status" to "signature[0].status"
+    for (const [varPath, equivalence] of Object.entries(
+      this.equivalentVariables,
+    )) {
+      if (pathStr.startsWith(varPath + '.')) {
+        const suffix = pathStr.slice(varPath.length);
+        const transformedPath = equivalence + suffix;
+        this.structure[transformedPath] = newType;
+      }
+    }
+  }
+
   /**
    * Get a simplified type string from a TypeScript Type object
    * This converts TypeScript's internal type representation to our type system

package/analyzer-template/packages/ai/src/lib/astScopes/methodSemantics.ts

@@ -343,6 +343,56 @@ export class ArraySpliceSemantics implements MethodSemantics {
   }
 }
 
+/**
+ * Array push method semantics
+ *
+ * push() adds one or more elements to the end of an array and returns the new length.
+ * Critically for data flow analysis, pushed elements become array elements.
+ */
+export class ArrayPushSemantics implements MethodSemantics {
+  getReturnType(): string {
+    return 'number'; // push returns the new length of the array
+  }
+
+  addEquivalences(
+    methodCallPath: StructuredPath,
+    sourcePath: StructuredPath,
+    context: AnalysisContext,
+  ): void {
+    // Mark source as array
+    context.addType(sourcePath, 'array');
+
+    // Get the last function call segment to access the arguments
+    const functionCallSegment = methodCallPath.getLastFunctionCallSegment();
+
+    if (functionCallSegment && functionCallSegment.args.length > 0) {
+      // Each argument is an element being added to the array
+      // Establish equivalence: array[] === pushed element
+      // This enables tracing through patterns like:
+      //   allEntities.push({...entity})
+      //   allEntities.map(e => e.prop) // e.prop traces back to entity.prop
+      //
+      // We use methodCallPath.withParameter(i) because that's the path used when
+      // processExpression processes each argument. The spread equivalencies are
+      // set up against signature[i] paths, so we need to connect to those.
+      //
+      // Note: The path transformation in paths.ts now correctly handles the case
+      // where .signature[n] follows a function call - it won't add
+      // .functionCallReturnValue before the parameter path.
+      for (let i = 0; i < functionCallSegment.args.length; i++) {
+        context.addEquivalence(
+          sourcePath.withElement('*'),
+          methodCallPath.withParameter(i),
+        );
+      }
+    }
+  }
+
+  isComplete(): boolean {
+    return true;
+  }
+}
+
 /**
  * Array join method semantics
  *
@@ -423,6 +473,194 @@ export class ArrayMapSemantics implements MethodSemantics {
   }
 }
 
+/**
+ * Array flatMap method semantics
+ *
+ * flatMap() maps each element using a mapping function, then flattens the result into a new array
+ */
+export class ArrayFlatMapSemantics implements MethodSemantics {
+  getReturnType(): string {
+    return 'array';
+  }
+
+  addEquivalences(
+    methodCallPath: StructuredPath,
+    sourcePath: StructuredPath,
+    context: AnalysisContext,
+  ): void {
+    // Mark source as array
+    context.addType(sourcePath, 'array');
+
+    // Get the last function call segment to access the arguments
+    const lastFunctionCallSegment = methodCallPath.getLastFunctionCallSegment();
+
+    if (lastFunctionCallSegment && lastFunctionCallSegment.args.length > 0) {
+      // The first argument is the callback
+      const callbackPath = lastFunctionCallSegment.args[0];
+
+      // Add the callback to the structure
+      context.addType(callbackPath, 'function');
+
+      // Mark that the callback's first parameter receives array elements
+      context.addEquivalence(
+        callbackPath.withParameter(0),
+        sourcePath.withElement('*'),
+      );
+    }
+  }
+
+  isComplete(): boolean {
+    return true;
+  }
+}
+
+/**
+ * Array at method semantics
+ *
+ * at() returns the element at the given index, supporting negative indices
+ */
+export class ArrayAtSemantics implements MethodSemantics {
+  getReturnType(): string {
+    return 'unknown'; // Returns element type or undefined
+  }
+
+  addEquivalences(
+    methodCallPath: StructuredPath,
+    sourcePath: StructuredPath,
+    context: AnalysisContext,
+  ): void {
+    // Mark source as array
+    context.addType(sourcePath, 'array');
+
+    // The return value is equivalent to an array element
+    context.addEquivalence(
+      methodCallPath.withReturnValues(),
+      sourcePath.withElement('*'),
+    );
+  }
+
+  isComplete(): boolean {
+    return true;
+  }
+}
+
+/**
+ * Array findLast method semantics
+ *
+ * findLast() returns the last element that passes the test, or undefined
+ */
+export class ArrayFindLastSemantics implements MethodSemantics {
+  getReturnType(): string {
+    return 'unknown'; // Could be any type from the array or undefined
+  }
+
+  addEquivalences(
+    methodCallPath: StructuredPath,
+    sourcePath: StructuredPath,
+    context: AnalysisContext,
+  ): void {
+    // Mark source as array
+    context.addType(sourcePath, 'array');
+
+    // Get the last function call segment to access the arguments
+    const functionCallSegment = methodCallPath.getLastFunctionCallSegment();
+
+    if (functionCallSegment && functionCallSegment.args.length > 0) {
+      // The first argument is the callback
+      const callbackPath = functionCallSegment.args[0];
+
+      // Add the callback to the structure
+      context.addType(callbackPath, 'function');
+
+      // Mark that the callback's first parameter receives array elements
+      context.addEquivalence(
+        callbackPath.withParameter(0),
+        sourcePath.withElement('*'),
+      );
+    }
+  }
+
+  isComplete(): boolean {
+    return true;
+  }
+}
+
+/**
+ * Object.fromEntries method semantics
+ *
+ * fromEntries() transforms a list of key-value pairs into an object
+ */
+export class ObjectFromEntriesSemantics implements MethodSemantics {
+  getReturnType(): string {
+    return 'object';
+  }
+
+  addEquivalences(
+    methodCallPath: StructuredPath,
+    sourcePath: StructuredPath,
+    context: AnalysisContext,
+  ): void {
+    // The source path would be Object, not particularly useful
+    // The argument is an iterable of key-value pairs
+    const functionCallSegment = methodCallPath.getLastFunctionCallSegment();
+
+    if (functionCallSegment && functionCallSegment.args.length > 0) {
+      // Mark the argument as an array (iterable of entries)
+      const entriesPath = functionCallSegment.args[0];
+      context.addType(entriesPath, 'array');
+    }
+  }
+
+  isComplete(): boolean {
+    return true;
+  }
+}
+
+/**
+ * Promise.then method semantics
+ *
+ * then() registers a callback to be called when the promise resolves,
+ * and returns a new promise that resolves to the callback's return value
+ */
+export class PromiseThenSemantics implements MethodSemantics {
+  getReturnType(): string {
+    return 'unknown'; // Returns a Promise resolving to the callback's return
+  }
+
+  addEquivalences(
+    methodCallPath: StructuredPath,
+    sourcePath: StructuredPath,
+    context: AnalysisContext,
+  ): void {
+    // Get the callback argument
+    const functionCallSegment = methodCallPath.getLastFunctionCallSegment();
+
+    if (functionCallSegment && functionCallSegment.args.length > 0) {
+      const callbackPath = functionCallSegment.args[0];
+
+      // Mark the callback as a function
+      context.addType(callbackPath, 'function');
+
+      // The callback's first parameter receives the resolved value of the source promise
+      // For promise.then(response => ...), response = promise's resolved value
+      // Note: Don't add .functionCallReturnValue - the system adds it automatically
+      // via toRightHandSideString() when storing equivalences
+      context.addEquivalence(callbackPath.withParameter(0), sourcePath);
+
+      // The .then() return value (when chained) resolves to the callback's return
+      // This enables chained .then() calls to track through
+      context.addEquivalence(
+        methodCallPath.withProperty('functionCallReturnValue'),
+        callbackPath.withProperty('returnValue'),
+      );
+    }
+  }
+
+  isComplete(): boolean {
+    return true;
+  }
+}
+
 /**
  * UseMemo React hook semantics
  *
@@ -513,12 +751,22 @@ export function createMethodRegistry(): MethodSemanticsRegistry {
   // Register array methods
   registry.register('filter', new ArrayFilterSemantics(), 'Array');
   registry.register('map', new ArrayMapSemantics(), 'Array');
+  registry.register('flatMap', new ArrayFlatMapSemantics(), 'Array');
   registry.register('join', new ArrayJoinSemantics(), 'Array');
   registry.register('find', new ArrayFindSemantics(), 'Array');
+  registry.register('findLast', new ArrayFindLastSemantics(), 'Array');
+  registry.register('at', new ArrayAtSemantics(), 'Array');
   registry.register('reduce', new ArrayReduceSemantics(), 'Array');
   registry.register('concat', new ArrayConcatSemantics(), 'Array');
   registry.register('slice', new ArraySliceSemantics(), 'Array');
   registry.register('splice', new ArraySpliceSemantics(), 'Array');
+  registry.register('push', new ArrayPushSemantics(), 'Array');
+
+  // Register Object methods
+  registry.register('fromEntries', new ObjectFromEntriesSemantics(), 'Object');
+
+  // Register Promise methods
+  registry.register('then', new PromiseThenSemantics(), 'Promise');
 
   // Register React hooks
   registry.register('useState', new UseStateSemantics(), 'React');

package/analyzer-template/packages/ai/src/lib/astScopes/paths.ts

@@ -17,6 +17,25 @@ interface ToStringWithSegmentsOptions {
   withNestedReturnValues?: boolean;
 }
 
+/**
+ * Check if a string is a valid JavaScript identifier (can use dot notation)
+ * Valid identifiers start with a letter, underscore, or dollar sign
+ * and contain only letters, digits, underscores, and dollar signs
+ *
+ * Also allows computed property markers like *f.path* (wrapped in asterisks)
+ * which are used internally to represent dynamic property accesses
+ */
+function isValidIdentifier(name: string): boolean {
+  if (!name || name.length === 0) return false;
+  // First char: letter, underscore, or dollar sign
+  // Rest: letter, digit, underscore, or dollar sign
+  // Also allow computed property markers like *expression*
+  return (
+    /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(name) ||
+    (name.startsWith('*') && name.endsWith('*'))
+  );
+}
+
 /**
  * A structured representation of a variable or property path
  *
@@ -335,8 +354,15 @@ export class StructuredPath {
       switch (segment.type) {
         case PathRelationship.PROPERTY:
           const propSegment = segment as PropertySegment;
-          if (result) result += '.';
-          result += propSegment.name;
+          // Use bracket notation for property names that aren't valid identifiers
+          // (e.g. names with spaces, hyphens, starting with numbers, etc.)
+          if (isValidIdentifier(propSegment.name)) {
+            if (result) result += '.';
+            result += propSegment.name;
+          } else {
+            // Use bracket notation with quoted string for non-identifier names
+            result += `['${propSegment.name}']`;
+          }
           break;
 
         case PathRelationship.ELEMENT:
@@ -389,7 +415,20 @@ export class StructuredPath {
             result += `(${argsStr})`;
           }
 
-          if (i === lastFunctionCallIndex && doRenderFunctionCallReturnValue) {
+          // Add .functionCallReturnValue if:
+          // 1. This is the last function call in the path
+          // 2. We're rendering with return values (right-hand side of equivalency)
+          // 3. The next segment is NOT a parameter (signature[n])
+          // Parameter segments represent function inputs, not outputs, so they
+          // shouldn't go through .functionCallReturnValue
+          const nextSegmentIsParameter =
+            i + 1 < segments.length &&
+            segments[i + 1].type === PathRelationship.PARAMETER;
+          if (
+            i === lastFunctionCallIndex &&
+            doRenderFunctionCallReturnValue &&
+            !nextSegmentIsParameter
+          ) {
             // Add the return value marker
             result += '.functionCallReturnValue';
           }

package/analyzer-template/packages/ai/src/lib/astScopes/patterns/switchStatementHandler.ts

@@ -40,13 +40,22 @@ export class SwitchStatementHandler implements PatternHandler {
         }
       }
 
+      const discriminantPathStr = discriminantPath.toLeftHandSideString();
+
       // Record the conditional usage
       context.addConditionalUsage({
-        path: discriminantPath.toLeftHandSideString(),
+        path: discriminantPathStr,
         conditionType: 'switch',
         comparedValues: caseValues.length > 0 ? caseValues : undefined,
         location: 'switch',
       });
+
+      // Update the schema type to be a union of the case values
+      // This ensures the LLM knows the valid values for this field
+      if (caseValues.length > 0) {
+        const unionType = caseValues.map((v) => `'${v}'`).join(' | ');
+        context.updateSchemaType(discriminantPathStr, unionType);
+      }
     }
 
     // Process each clause in the switch statement