@codeyam/codeyam-cli 0.1.0-staging.b8a55ba → 0.1.0-staging.bd4126b

package/analyzer-template/packages/ai/src/lib/mergeStatements.ts

@@ -7,58 +7,12 @@ import {
 
 export interface Statement {
   structure: { [key: string]: string };
-  equivalentVariables: { [key: string]: string };
+  // Supports multiple equivalencies per key for OR expressions (e.g., x = a || b)
+  equivalentVariables: { [key: string]: string | string[] };
   llmCall?: LlmCall;
 }
 
 export default function mergeStatements(statements: Statement[]) {
-  // Debug: Log input statements to understand what we're merging
-  const hasStateStructure = statements.some((s) =>
-    Object.keys(s.structure || {}).some(
-      (k) => k === 'state' || k.includes('state'),
-    ),
-  );
-  const hasStateEquiv = statements.some((s) =>
-    Object.keys(s.equivalentVariables || {}).some(
-      (k) => k === 'state' || k.includes('state'),
-    ),
-  );
-  if (hasStateStructure || hasStateEquiv) {
-    console.log(
-      `[UNION-TYPE] mergeStatements INPUT: ${statements.length} statements`,
-    );
-    statements.forEach((s, i) => {
-      const stateInStructure = Object.entries(s.structure || {}).filter(
-        ([k]) => k === 'state' || k.includes('state'),
-      );
-      const stateInEquiv = Object.entries(s.equivalentVariables || {}).filter(
-        ([k]) => k === 'state' || k.includes('state'),
-      );
-      const sigInEquiv = Object.entries(s.equivalentVariables || {}).filter(
-        ([k, v]) => k === 'state' || v === 'signature[0]',
-      );
-      if (
-        stateInStructure.length > 0 ||
-        stateInEquiv.length > 0 ||
-        sigInEquiv.length > 0
-      ) {
-        console.log(`[UNION-TYPE]   Statement ${i}:`);
-        if (stateInStructure.length > 0)
-          console.log(
-            `[UNION-TYPE]     structure: ${JSON.stringify(Object.fromEntries(stateInStructure))}`,
-          );
-        if (stateInEquiv.length > 0)
-          console.log(
-            `[UNION-TYPE]     equivalentVariables (state): ${JSON.stringify(Object.fromEntries(stateInEquiv))}`,
-          );
-        if (sigInEquiv.length > 0)
-          console.log(
-            `[UNION-TYPE]     equivalentVariables (sig): ${JSON.stringify(Object.fromEntries(sigInEquiv))}`,
-          );
-      }
-    });
-  }
-
   // This should be handled by the AST Analyzer but it currently has a bug
   const ensureFunctionCallReturnValue = (path: string, value: string) => {
     if (path.endsWith(')') && value !== 'function') {
@@ -166,12 +120,60 @@ export default function mergeStatements(statements: Statement[]) {
     return acc;
   }, {});
 
+  // Track variable reassignments: when a variable is reassigned (creating ::cyDuplicateKey::),
+  // subsequent references to that variable in VALUES should use the duplicate key.
+  // This ensures configData → fetcher.data.data uses the FIRST fetcher, while
+  // settingsData → fetcher.data.data uses fetcher::cyDuplicateKey1::.data.data
+  const activeRemappings: Record<string, string> = {};
+
+  // Helper to normalize value to array and process
+  const normalizeValue = (value: string | string[]): string[] =>
+    Array.isArray(value) ? value : [value];
+
   const equivalentVariables = statements.reduce((acc: any, result) => {
-    for (const [key, value] of Object.entries(
+    for (const [key, rawValue] of Object.entries(
       result.equivalentVariables ?? {},
     )) {
+      // Handle arrays from AST analyzer (OR expressions like x = a || b)
+      // These should be preserved as arrays to track all sources
+      if (Array.isArray(rawValue)) {
+        const keyOptions = deBinaryPath(key);
+        const validValues = rawValue.filter(
+          (v): v is string => typeof v === 'string' && v.length > 0,
+        );
+        if (validValues.length === 0) continue;
+
+        // Process each array value through deBinaryPath
+        const allValueOptions: string[] = [];
+        for (const value of validValues) {
+          const valueOptions = deBinaryPath(value);
+          allValueOptions.push(...valueOptions);
+        }
+
+        // Remove duplicates while preserving order
+        const uniqueValueOptions = [...new Set(allValueOptions)];
+        const finalValue =
+          uniqueValueOptions.length === 1
+            ? uniqueValueOptions[0]
+            : uniqueValueOptions;
+
+        delete result.equivalentVariables[key];
+        if (keyOptions.length >= 2) {
+          for (const keyOption of keyOptions) {
+            result.equivalentVariables[keyOption] = finalValue;
+          }
+        } else {
+          result.equivalentVariables[keyOptions[0]] = finalValue;
+        }
+        continue;
+      }
+
+      // Original logic for string values with potential binary expressions in the string
+      const firstValue = rawValue;
+      if (!firstValue) continue;
+
       const keyOptions = deBinaryPath(key);
-      const valueOptions = deBinaryPath(value);
+      const valueOptions = deBinaryPath(firstValue);
       delete result.equivalentVariables[key];
       if (keyOptions.length === 2) {
         if (valueOptions.length === 2) {
@@ -193,19 +195,54 @@ export default function mergeStatements(statements: Statement[]) {
       }
     }
 
-    for (const [key, value] of Object.entries(
+    for (const [key, rawValue] of Object.entries(
       result.equivalentVariables ?? {},
     )) {
+      // Normalize to array for consistent handling
+      const values = normalizeValue(rawValue);
+
+      // Apply any active remappings to the VALUE, but ONLY for property paths
+      // e.g., if fetcher was remapped to fetcher::cyDuplicateKey1::,
+      // then "fetcher.data.data" should become "fetcher::cyDuplicateKey1::.data.data"
+      // However, we should NOT remap standalone variable references like "description"
+      // because that would lose type information from the original variable.
+      const remappedValues = values.map((value) => {
+        let remappedValue = value;
+        for (const [originalKey, remappedKey] of Object.entries(
+          activeRemappings,
+        )) {
+          // Only remap if the value is a property path (contains a dot after the key)
+          // This preserves type information for simple variable references
+          const keyRegex = new RegExp(
+            `^${originalKey.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?=\\.)`,
+          );
+          if (keyRegex.test(remappedValue)) {
+            remappedValue = remappedValue.replace(keyRegex, remappedKey);
+          }
+        }
+        return remappedValue;
+      });
+
+      // Use array if multiple values, string if single
+      const remappedValue =
+        remappedValues.length === 1 ? remappedValues[0] : remappedValues;
+      const firstRemappedValue = remappedValues[0];
+
       if (acc[key]) {
-        if (acc[key] === value) continue;
+        if (acc[key] === remappedValue) continue;
 
         let duplicateId = 1;
         while (acc[`${key}::cyDuplicateKey${duplicateId}::`]) {
           duplicateId++;
         }
-        acc[`${key}::cyDuplicateKey${duplicateId}::`] = value;
+        const duplicateKey = `${key}::cyDuplicateKey${duplicateId}::`;
+        acc[duplicateKey] = remappedValue;
+
+        // Track this remapping for subsequent statements
+        // Any future references to 'key' in VALUES should use duplicateKey instead
+        activeRemappings[key] = duplicateKey;
       } else {
-        acc[key] = value;
+        acc[key] = remappedValue;
       }
     }
     return acc;
@@ -217,45 +254,32 @@ export default function mergeStatements(statements: Statement[]) {
   // If we have equivalentVariables[A] = B, and structure[A] is more specific than structure[B],
   // then update structure[B] to match structure[A]
   // This handles cases like: "state" -> "signature[0]" where "state" has a union type
-
-  // Debug: Log union types in merged structure
-  const unionTypesInStructure = Object.entries(structure).filter(
-    ([_, v]) => typeof v === 'string' && v.includes("'") && v.includes(' | '),
-  );
-  if (unionTypesInStructure.length > 0) {
-    console.log(
-      `[UNION-TYPE] mergeStatements: Found ${unionTypesInStructure.length} union types in merged structure`,
-    );
-    unionTypesInStructure.forEach(([k, v]) =>
-      console.log(`[UNION-TYPE]   structure["${k}"] = ${v}`),
-    );
-    console.log(
-      `[UNION-TYPE] mergeStatements: equivalentVariables = ${JSON.stringify(equivalentVariables)}`,
-    );
-  }
-
-  for (const [source, target] of Object.entries(equivalentVariables) as [
-    string,
+  for (const [source, rawTarget] of Object.entries(equivalentVariables) as [
     string,
+    string | string[],
   ][]) {
+    // Normalize to array for consistent handling
+    const targets = Array.isArray(rawTarget) ? rawTarget : [rawTarget];
     const sourceType = structure[source];
-    const targetType = structure[target];
 
-    // Check if source has a more specific type (union type vs generic type)
-    if (sourceType) {
-      const sourceIsUnion =
-        sourceType.includes("'") && sourceType.includes(' | ');
+    for (const target of targets) {
+      if (typeof target !== 'string') continue;
 
-      if (sourceIsUnion) {
-        // Target doesn't exist or is generic - propagate the union type
-        const targetIsGeneric =
-          !targetType || targetType === 'string' || targetType === 'unknown';
+      const targetType = structure[target];
 
-        if (targetIsGeneric) {
-          console.log(
-            `[UNION-TYPE] mergeStatements: Propagating "${source}" (${sourceType}) -> "${target}"`,
-          );
-          structure[target] = sourceType;
+      // Check if source has a more specific type (union type vs generic type)
+      if (sourceType) {
+        const sourceIsUnion =
+          sourceType.includes("'") && sourceType.includes(' | ');
+
+        if (sourceIsUnion) {
+          // Target doesn't exist or is generic - propagate the union type
+          const targetIsGeneric =
+            !targetType || targetType === 'string' || targetType === 'unknown';
+
+          if (targetIsGeneric) {
+            structure[target] = sourceType;
+          }
         }
       }
     }

package/analyzer-template/packages/ai/src/lib/promptGenerators/collapseNullableObjects.ts

@@ -0,0 +1,118 @@
+import { JsonTypeDefinition } from '~codeyam/types';
+
+/**
+ * Check whether all non-`_nullable` children of an object are simple strings.
+ * "Leaf-like" nullable objects (all-string children) get collapsed to
+ * `"object | null"` so the LLM knows the field can be null.
+ * Objects with nested structure (object/array children) keep their shape
+ * so the LLM can still generate proper data when the field is truthy.
+ */
+function isLeafLikeObject(obj: Record<string, any>): boolean {
+  for (const key of Object.keys(obj)) {
+    if (key === '_nullable') continue;
+    const value = obj[key];
+    if (typeof value !== 'string') return false;
+  }
+  return true;
+}
+
+/**
+ * Collapse *leaf-like* nullable objects/arrays in the data structure to type strings.
+ *
+ * When convertDotNotation marks an object or array as `_nullable: true`
+ * (meaning the schema had `object | undefined`, `object | null`, etc.),
+ * we decide whether to collapse it:
+ *
+ * - **Leaf-like** (all children are simple strings): collapse to `"object | null"`.
+ *   Example: `{ _nullable: true, type: "string", path: "string" }` → `"object | null"`
+ *
+ * - **Rich** (has object/array children): strip `_nullable` and keep the structure
+ *   so the LLM can generate proper nested data when the field is truthy.
+ *   Example: `{ _nullable: true, items: [...], name: "string" }` → `{ items: [...], name: "string" }`
+ *
+ * Non-nullable objects are recursively processed but left as nested structures.
+ */
+export default function collapseNullableObjects(
+  data: JsonTypeDefinition,
+): JsonTypeDefinition {
+  if (typeof data !== 'object' || data === null) {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    return data.map((item: any) =>
+      typeof item === 'object' && item !== null
+        ? collapseNullableObjects(item as JsonTypeDefinition)
+        : item,
+    ) as any as JsonTypeDefinition;
+  }
+
+  const result: JsonTypeDefinition = {};
+
+  for (const key of Object.keys(data)) {
+    // Skip the _nullable marker itself (shouldn't appear at top level, but be safe)
+    if (key === '_nullable') continue;
+
+    const value = data[key];
+
+    if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      const hasNullable = (value as any)._nullable === true;
+      if (hasNullable && isLeafLikeObject(value as Record<string, any>)) {
+        // Leaf-like nullable object → collapse to type string
+        const childKeys = Object.keys(value as any).filter(
+          (k) => k !== '_nullable',
+        );
+        console.log(
+          `[collapseNullableObjects] collapsed "${key}" → "object | null" (leaf-like, children: [${childKeys.join(', ')}])`,
+        );
+        result[key] = 'object | null';
+      } else if (hasNullable) {
+        // Rich nullable object → strip _nullable, keep structure
+        const childKeys = Object.keys(value as any).filter(
+          (k) => k !== '_nullable',
+        );
+        console.log(
+          `[collapseNullableObjects] kept structure for nullable "${key}" (rich, children: [${childKeys.join(', ')}])`,
+        );
+        result[key] = collapseNullableObjects(value as JsonTypeDefinition);
+      } else {
+        // Non-nullable → recurse
+        result[key] = collapseNullableObjects(value as JsonTypeDefinition);
+      }
+    } else if (Array.isArray(value)) {
+      const hasNullable = (value as any)._nullable === true;
+      if (
+        hasNullable &&
+        value.length <= 1 &&
+        value.every((v) => typeof v === 'string')
+      ) {
+        // Leaf-like nullable array → collapse to type string
+        console.log(
+          `[collapseNullableObjects] collapsed "${key}" → "array | null" (leaf-like)`,
+        );
+        result[key] = 'array | null';
+      } else if (hasNullable) {
+        // Rich nullable array → strip _nullable, keep structure
+        console.log(
+          `[collapseNullableObjects] kept structure for nullable array "${key}" (${value.length} elements)`,
+        );
+        result[key] = value.map((item) =>
+          typeof item === 'object' && item !== null
+            ? collapseNullableObjects(item as JsonTypeDefinition)
+            : item,
+        ) as JsonTypeDefinition[];
+      } else {
+        // Non-nullable → recurse elements
+        result[key] = value.map((item) =>
+          typeof item === 'object' && item !== null
+            ? collapseNullableObjects(item as JsonTypeDefinition)
+            : item,
+        ) as JsonTypeDefinition[];
+      }
+    } else {
+      result[key] = value;
+    }
+  }
+
+  return result;
+}

package/analyzer-template/packages/ai/src/lib/promptGenerators/gatherAttributesMap.ts

@@ -139,16 +139,19 @@ export function gatherAttributesMap(
       // Use merged type if available, otherwise fall back to isolated type
       const typeValue =
         mergedSignatureSchema[key] ?? isolatedSignatureSchema[key];
-      const equivalentSignatureVariable =
-        equivalentSignatureVariables[keyParts[0]];
-      const equivalentSignatureVariableParts = splitOutsideParenthesesAndArrays(
-        equivalentSignatureVariable,
-      );
+      // Handle array case (OR expressions) - use first element if array
+      const rawEquivalent = equivalentSignatureVariables[keyParts[0]];
+      const equivalentSignatureVariable = Array.isArray(rawEquivalent)
+        ? rawEquivalent[0]
+        : rawEquivalent;
+      const equivalentSignatureVariableParts = equivalentSignatureVariable
+        ? splitOutsideParenthesesAndArrays(equivalentSignatureVariable)
+        : [];
       if (
         equivalentSignatureVariable &&
-        !equivalentSignatureVariableParts[0].includes('(')
+        !equivalentSignatureVariableParts[0]?.includes('(')
       ) {
-        const equivalentKey = equivalentSignatureVariables[keyParts[0]];
+        const equivalentKey = equivalentSignatureVariable;
         const equivalentPath = joinParenthesesAndArrays([
           equivalentKey,
           ...keyParts.slice(1),

package/analyzer-template/packages/ai/src/lib/promptGenerators/generateChangesEntityScenarioDataGenerator.ts

@@ -66,7 +66,7 @@ ${incompleteResponse}
     {
       name: scenario.name,
       description: scenario.description,
-      keyAttributeInstructions: scenario.metadata.keyAttributeInstructions,
+      coveredFlows: scenario.metadata?.coveredFlows || [],
     },
     null,
     2,

package/analyzer-template/packages/ai/src/lib/promptGenerators/generateChangesEntityScenariosGenerator.ts

@@ -1,19 +1,11 @@
-import {
-  DataStructure,
-  Entity,
-  Scenario,
-  ReadonlyAnalysisMap,
-  Analysis,
-} from '~codeyam/types';
-import gatherRelevantDependentKeyAttributes from '../gatherRelevantDependentKeyAttributes';
+import { DataStructure, Entity, ExecutionFlow, Scenario } from '~codeyam/types';
 
 interface GenerateChangesEntityScenariosGeneratorArgs {
   entity: Pick<Entity, 'name' | 'code' | 'metadata'>;
   mergedDataStructure: Omit<DataStructure, 'equivalentSignatureVariables'>;
-  keyAttributes: Analysis['metadata']['keyAttributes'];
-  addedKeyAttributes: Analysis['metadata']['keyAttributes'];
-  removedKeyAttributes: Analysis['metadata']['keyAttributes'];
-  dependentAnalyses: ReadonlyAnalysisMap;
+  executionFlows?: ExecutionFlow[];
+  addedFlows?: ExecutionFlow[];
+  removedFlows?: ExecutionFlow[];
   existingScenarios: Scenario[];
   commitDiff: string;
   error: boolean;
@@ -22,80 +14,26 @@ interface GenerateChangesEntityScenariosGeneratorArgs {
 
 export default function generateChangesEntityScenariosGenerator({
   entity,
-  mergedDataStructure,
-  keyAttributes,
-  addedKeyAttributes,
-  removedKeyAttributes,
-  dependentAnalyses,
+  executionFlows,
+  addedFlows,
+  removedFlows,
   existingScenarios,
   commitDiff,
   error,
   changedDataStructureFields = [],
 }: GenerateChangesEntityScenariosGeneratorArgs) {
-  const relevantDependentKeyAttributesFlat =
-    gatherRelevantDependentKeyAttributes(
-      entity,
-      mergedDataStructure,
-      dependentAnalyses,
-    );
-
-  // Restructure flat array into nested format: { [filePath]: { [entityName]: KeyAttribute[] } }
-  const relevantDependentKeyAttributes: Record<
-    string,
-    Record<string, any[]>
-  > = {};
-  for (const attr of relevantDependentKeyAttributesFlat) {
-    const filePath = (attr as any).filePath;
-    const entityName = (attr as any).originalEntityName;
-
-    if (!filePath || !entityName) continue;
-
-    if (!relevantDependentKeyAttributes[filePath]) {
-      relevantDependentKeyAttributes[filePath] = {};
-    }
-    if (!relevantDependentKeyAttributes[filePath][entityName]) {
-      relevantDependentKeyAttributes[filePath][entityName] = [];
-    }
-    relevantDependentKeyAttributes[filePath][entityName].push(attr);
-  }
-
-  const formatKeyAttrs = (attrs: Analysis['metadata']['keyAttributes']) =>
-    attrs.map((ka) => ({
-      dataStructurePath: ka.dataStructurePath,
-      localVariable: ka.internalPath,
-      description: ka.description,
-      // Include indexed validValueOptions for structured references
-      validValueOptions: ka.validValueOptions.map((value, index) => ({
-        index,
-        value,
-      })),
-      ...(ka.errorValueOptions?.length && {
-        errorValueOptions: ka.errorValueOptions.map((value, index) => ({
-          index,
-          value,
-        })),
-      }),
-      // Include new fields when present
-      ...(ka.valueType && { valueType: ka.valueType }),
-      ...(ka.dependencies && { dependencies: ka.dependencies }),
-      // Include source locations showing where this attribute is used in conditionals
-      ...(ka.sourceLocations?.length && {
-        codeUsages: ka.sourceLocations.map((loc) => ({
-          line: loc.lineNumber,
-          code: loc.codeSnippet,
-          context: loc.description,
-        })),
-      }),
-    }));
-
   const addedSection =
-    addedKeyAttributes?.length > 0
-      ? JSON.stringify(formatKeyAttrs(addedKeyAttributes), null, 2)
+    addedFlows && addedFlows.length > 0
+      ? JSON.stringify(addedFlows, null, 2)
       : 'None';
 
   const removedSection =
-    removedKeyAttributes?.length > 0
-      ? JSON.stringify(removedKeyAttributes, null, 2)
+    removedFlows && removedFlows.length > 0
+      ? JSON.stringify(
+          removedFlows.map((f) => ({ id: f.id, name: f.name })),
+          null,
+          2,
+        )
       : 'None';
 
   const changedFieldsSection =
@@ -107,18 +45,18 @@ Fields that changed type, were added, or removed:
 ${JSON.stringify(changedDataStructureFields, null, 2)}
 \`\`\`
 
-**Update keyAttributeInstructions for affected scenarios** (e.g., if "stats.visualEntities" changed from array to number).
+**Update scenarios if their coveredFlows depend on these changed fields.**
 `
       : '';
 
   const existingScenariosJson = JSON.stringify(
     existingScenarios
-      .filter((s) => (error ? !!s.metadata.error : !s.metadata.error))
+      .filter((s) => (error ? !!s.metadata?.error : !s.metadata?.error))
       .map((s) => ({
         name: s.name,
         description: s.description,
-        keyAttributeInstructions: s.metadata.keyAttributeInstructions,
-        playwrightInstructions: s.metadata.playwrightInstructions,
+        coveredFlows: s.metadata?.coveredFlows || [],
+        playwrightInstructions: s.metadata?.playwrightInstructions,
       })),
     null,
     2,
@@ -132,77 +70,29 @@ ${commitDiff}
 \`\`\``
     : '';
 
-  const dependencySection =
-    !error &&
-    relevantDependentKeyAttributes &&
-    Object.keys(relevantDependentKeyAttributes).length > 0
-      ? `
-## Dependency Key Attributes
+  const executionFlowsSection =
+    executionFlows && executionFlows.length > 0
+      ? `## Current Execution Flows
+All distinct outcomes/behaviors this component can currently produce:
 \`\`\`json
-${JSON.stringify(
-  Object.keys(relevantDependentKeyAttributes).reduce((acc: any, path) => {
-    acc[path] = Object.keys(relevantDependentKeyAttributes[path]).reduce(
-      (entityAcc: any, entityName) => {
-        entityAcc[entityName] = relevantDependentKeyAttributes[path][
-          entityName
-        ].map((ka: any) => ({
-          dataStructurePath: ka.dataStructurePath,
-          localVariable: ka.path || ka.internalPath,
-          description: ka.description,
-          // Include indexed validValueOptions for structured references
-          validValueOptions: (ka.validValueOptions || []).map(
-            (value: string, index: number) => ({
-              index,
-              value,
-            }),
-          ),
-          ...(ka.errorValueOptions?.length && {
-            errorValueOptions: ka.errorValueOptions.map(
-              (value: string, index: number) => ({
-                index,
-                value,
-              }),
-            ),
-          }),
-          // Include new fields when present
-          ...(ka.valueType && { valueType: ka.valueType }),
-          ...(ka.dependencies && { dependencies: ka.dependencies }),
-          // Include source locations showing where this attribute is used in conditionals
-          ...(ka.sourceLocations?.length && {
-            codeUsages: ka.sourceLocations.map((loc: any) => ({
-              line: loc.lineNumber,
-              code: loc.codeSnippet,
-              context: loc.description,
-            })),
-          }),
-        }));
-        return entityAcc;
-      },
-      {},
-    );
-    return acc;
-  }, {}),
-  null,
-  2,
-)}
-\`\`\``
+${JSON.stringify(executionFlows, null, 2)}
+\`\`\`
+`
       : '';
 
-  return `## Added Key Attributes
+  return `## Added Execution Flows
+New flows that need scenario coverage:
 \`\`\`json
 ${addedSection}
 \`\`\`
 
-## Removed Key Attributes
+## Removed Execution Flows
+Flows that no longer exist (remove from scenarios):
 \`\`\`json
 ${removedSection}
 \`\`\`
 ${changedFieldsSection}
-## Current Key Attributes
-\`\`\`json
-${JSON.stringify(formatKeyAttrs(keyAttributes), null, 2)}
-\`\`\`
-
+${executionFlowsSection}
 ## Existing Scenarios
 \`\`\`json
 ${existingScenariosJson}
@@ -212,5 +102,5 @@ ${diffSection}
 ## Current Code
 \`\`\`typescript
 ${entity.code}
-\`\`\`${dependencySection}`;
+\`\`\``;
 }