@codeyam/codeyam-cli 0.1.0-staging.8aea589 → 0.1.0-staging.8df382d

package/analyzer-template/packages/ai/src/lib/promptGenerators/collapseNullableObjects.ts

@@ -0,0 +1,118 @@
+import { JsonTypeDefinition } from '~codeyam/types';
+
+/**
+ * Check whether all non-`_nullable` children of an object are simple strings.
+ * "Leaf-like" nullable objects (all-string children) get collapsed to
+ * `"object | null"` so the LLM knows the field can be null.
+ * Objects with nested structure (object/array children) keep their shape
+ * so the LLM can still generate proper data when the field is truthy.
+ */
+function isLeafLikeObject(obj: Record<string, any>): boolean {
+  for (const key of Object.keys(obj)) {
+    if (key === '_nullable') continue;
+    const value = obj[key];
+    if (typeof value !== 'string') return false;
+  }
+  return true;
+}
+
+/**
+ * Collapse *leaf-like* nullable objects/arrays in the data structure to type strings.
+ *
+ * When convertDotNotation marks an object or array as `_nullable: true`
+ * (meaning the schema had `object | undefined`, `object | null`, etc.),
+ * we decide whether to collapse it:
+ *
+ * - **Leaf-like** (all children are simple strings): collapse to `"object | null"`.
+ *   Example: `{ _nullable: true, type: "string", path: "string" }` → `"object | null"`
+ *
+ * - **Rich** (has object/array children): strip `_nullable` and keep the structure
+ *   so the LLM can generate proper nested data when the field is truthy.
+ *   Example: `{ _nullable: true, items: [...], name: "string" }` → `{ items: [...], name: "string" }`
+ *
+ * Non-nullable objects are recursively processed but left as nested structures.
+ */
+export default function collapseNullableObjects(
+  data: JsonTypeDefinition,
+): JsonTypeDefinition {
+  if (typeof data !== 'object' || data === null) {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    return data.map((item: any) =>
+      typeof item === 'object' && item !== null
+        ? collapseNullableObjects(item as JsonTypeDefinition)
+        : item,
+    ) as any as JsonTypeDefinition;
+  }
+
+  const result: JsonTypeDefinition = {};
+
+  for (const key of Object.keys(data)) {
+    // Skip the _nullable marker itself (shouldn't appear at top level, but be safe)
+    if (key === '_nullable') continue;
+
+    const value = data[key];
+
+    if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      const hasNullable = (value as any)._nullable === true;
+      if (hasNullable && isLeafLikeObject(value as Record<string, any>)) {
+        // Leaf-like nullable object → collapse to type string
+        const childKeys = Object.keys(value as any).filter(
+          (k) => k !== '_nullable',
+        );
+        console.log(
+          `[collapseNullableObjects] collapsed "${key}" → "object | null" (leaf-like, children: [${childKeys.join(', ')}])`,
+        );
+        result[key] = 'object | null';
+      } else if (hasNullable) {
+        // Rich nullable object → strip _nullable, keep structure
+        const childKeys = Object.keys(value as any).filter(
+          (k) => k !== '_nullable',
+        );
+        console.log(
+          `[collapseNullableObjects] kept structure for nullable "${key}" (rich, children: [${childKeys.join(', ')}])`,
+        );
+        result[key] = collapseNullableObjects(value as JsonTypeDefinition);
+      } else {
+        // Non-nullable → recurse
+        result[key] = collapseNullableObjects(value as JsonTypeDefinition);
+      }
+    } else if (Array.isArray(value)) {
+      const hasNullable = (value as any)._nullable === true;
+      if (
+        hasNullable &&
+        value.length <= 1 &&
+        value.every((v) => typeof v === 'string')
+      ) {
+        // Leaf-like nullable array → collapse to type string
+        console.log(
+          `[collapseNullableObjects] collapsed "${key}" → "array | null" (leaf-like)`,
+        );
+        result[key] = 'array | null';
+      } else if (hasNullable) {
+        // Rich nullable array → strip _nullable, keep structure
+        console.log(
+          `[collapseNullableObjects] kept structure for nullable array "${key}" (${value.length} elements)`,
+        );
+        result[key] = value.map((item) =>
+          typeof item === 'object' && item !== null
+            ? collapseNullableObjects(item as JsonTypeDefinition)
+            : item,
+        ) as JsonTypeDefinition[];
+      } else {
+        // Non-nullable → recurse elements
+        result[key] = value.map((item) =>
+          typeof item === 'object' && item !== null
+            ? collapseNullableObjects(item as JsonTypeDefinition)
+            : item,
+        ) as JsonTypeDefinition[];
+      }
+    } else {
+      result[key] = value;
+    }
+  }
+
+  return result;
+}

package/analyzer-template/packages/ai/src/lib/promptGenerators/gatherAttributesMap.ts

@@ -1,13 +1,48 @@
-import { Entity } from '~codeyam/types';
+import { DataStructure } from '~codeyam/types';
 import cleanOutBoundary from '../cleanOutBoundary';
 import {
   joinParenthesesAndArrays,
   splitOutsideParenthesesAndArrays,
 } from '../splitOutsideParentheses';
 
+/**
+ * Checks if a bracket-only path is a valid string key access (not just an index).
+ * Valid: ['*'], ['someKey'] - string key access
+ * Invalid: [0], [1] - numeric index only (meaningless without parent path)
+ */
+function isValidBracketStringKey(attribute: string): boolean {
+  // Match ['string'] or ["string"] patterns
+  return /^\[['"][^'"]+['"]\]$/.test(attribute);
+}
+
+/**
+ * Extracts the string key from a bracket notation like ['*'] or ["key"]
+ * Returns the key without quotes and brackets, e.g., '*' or 'key'
+ */
+function extractBracketStringKey(attribute: string): string | null {
+  const match = attribute.match(/^\[['"]([^'"]+)['"]\]$/);
+  return match ? match[1] : null;
+}
+
+/**
+ * Normalizes a path by extracting string keys from bracket notation.
+ * If the path is a single bracket string key like ['*'], returns the key ('*').
+ * Otherwise returns the path unchanged.
+ */
+function normalizePathForInternalUse(path: string): string {
+  const extracted = extractBracketStringKey(path);
+  return extracted !== null ? extracted : path;
+}
+
 function validAttribute(attribute: string) {
-  if (!attribute || attribute.length === 0 || attribute.startsWith('['))
-    return false;
+  if (!attribute || attribute.length === 0) return false;
+
+  // Allow string-key bracket access like ['*'] or ['someKey']
+  // These are valid data paths from objects with dynamic/special keys
+  if (attribute.startsWith('[')) {
+    return isValidBracketStringKey(attribute);
+  }
+
   const attributeParts = splitOutsideParenthesesAndArrays(attribute);
   if (
     attributeParts.some((part) =>
@@ -30,34 +65,98 @@ function validAttribute(attribute: string) {
   return true;
 }
 
+/**
+ * Gathers attributes from data structures for use in prompts.
+ *
+ * @param isolatedDataStructure - The entity's own data structure (determines which paths to include)
+ * @param mergedDataStructureOrEquivalentVars - Either the merged data structure (with richer types from
+ *   dependent analyses) OR equivalentSignatureVariables for backwards compatibility
+ * @param equivalentSignatureVariablesOrReverse - Either equivalentSignatureVariables OR reverse flag
+ * @param reverseArg - The reverse flag (only used when all 4 args provided)
+ */
 export function gatherAttributesMap(
-  entity: Pick<Entity, 'metadata'>,
-  reverse = false,
+  isolatedDataStructure: Omit<DataStructure, 'equivalentSignatureVariables'>,
+  mergedDataStructureOrEquivalentVars?:
+    | Omit<DataStructure, 'equivalentSignatureVariables'>
+    | DataStructure['equivalentSignatureVariables'],
+  equivalentSignatureVariablesOrReverse?:
+    | DataStructure['equivalentSignatureVariables']
+    | boolean,
+  reverseArg?: boolean,
 ) {
+  // Handle backwards compatibility: detect if called with old signature
+  // Old: (mergedDataStructure, equivalentSignatureVariables?, reverse?)
+  // New: (isolatedDataStructure, mergedDataStructure, equivalentSignatureVariables?, reverse?)
+  let mergedDataStructure: Omit<
+    DataStructure,
+    'equivalentSignatureVariables'
+  > | null;
+  let equivalentSignatureVariables: DataStructure['equivalentSignatureVariables'];
+  let reverse: boolean;
+
+  if (
+    mergedDataStructureOrEquivalentVars &&
+    typeof mergedDataStructureOrEquivalentVars === 'object' &&
+    'dependencySchemas' in mergedDataStructureOrEquivalentVars
+  ) {
+    // New signature: (isolated, merged, equivalentVars?, reverse?)
+    mergedDataStructure = mergedDataStructureOrEquivalentVars as Omit<
+      DataStructure,
+      'equivalentSignatureVariables'
+    >;
+    equivalentSignatureVariables =
+      (equivalentSignatureVariablesOrReverse as DataStructure['equivalentSignatureVariables']) ??
+      {};
+    reverse = reverseArg ?? false;
+  } else {
+    // Old signature: (merged, equivalentVars?, reverse?)
+    // Use isolatedDataStructure as both isolated and merged for backwards compatibility
+    mergedDataStructure = null; // Will use isolatedDataStructure for types
+    equivalentSignatureVariables =
+      (mergedDataStructureOrEquivalentVars as DataStructure['equivalentSignatureVariables']) ??
+      {};
+    reverse =
+      (equivalentSignatureVariablesOrReverse as boolean | undefined) ?? false;
+  }
+
   const attributesMap: Record<string, string> = {};
   const associationMap: Record<string, string> = {};
   // Maps internalPath -> full data structure path (e.g., "useDiffModal().diffView")
   const dataStructurePathMap: Record<string, string> = {};
-  const { signatureSchema, equivalentSignatureVariables, dependencySchemas } =
-    entity.metadata.isolatedDataStructure;
-  for (const key in signatureSchema) {
+
+  // Use isolatedDataStructure for paths, but look up types in mergedDataStructure when available
+  const isolatedSignatureSchema = isolatedDataStructure.signatureSchema;
+  const mergedSignatureSchema =
+    mergedDataStructure?.signatureSchema ?? isolatedSignatureSchema;
+  const isolatedDependencySchemas = isolatedDataStructure.dependencySchemas;
+  const mergedDependencySchemas =
+    mergedDataStructure?.dependencySchemas ?? isolatedDependencySchemas;
+
+  // Process signature schema - use isolatedDataStructure keys, but merged types
+  for (const key in isolatedSignatureSchema) {
     const keyParts = splitOutsideParenthesesAndArrays(key);
     if (keyParts[0].startsWith('signature')) {
-      const equivalentSignatureVariable =
-        equivalentSignatureVariables[keyParts[0]];
-      const equivalentSignatureVariableParts = splitOutsideParenthesesAndArrays(
-        equivalentSignatureVariable,
-      );
+      // Use merged type if available, otherwise fall back to isolated type
+      const typeValue =
+        mergedSignatureSchema[key] ?? isolatedSignatureSchema[key];
+      // Handle array case (OR expressions) - use first element if array
+      const rawEquivalent = equivalentSignatureVariables[keyParts[0]];
+      const equivalentSignatureVariable = Array.isArray(rawEquivalent)
+        ? rawEquivalent[0]
+        : rawEquivalent;
+      const equivalentSignatureVariableParts = equivalentSignatureVariable
+        ? splitOutsideParenthesesAndArrays(equivalentSignatureVariable)
+        : [];
       if (
         equivalentSignatureVariable &&
-        !equivalentSignatureVariableParts[0].includes('(')
+        !equivalentSignatureVariableParts[0]?.includes('(')
       ) {
-        const equivalentKey = equivalentSignatureVariables[keyParts[0]];
+        const equivalentKey = equivalentSignatureVariable;
         const equivalentPath = joinParenthesesAndArrays([
           equivalentKey,
           ...keyParts.slice(1),
         ]);
-        attributesMap[equivalentPath] = signatureSchema[key];
+        attributesMap[equivalentPath] = typeValue;
         associationMap[reverse ? equivalentPath : key] = reverse
           ? key
           : equivalentPath;
@@ -65,7 +164,7 @@ export function gatherAttributesMap(
         dataStructurePathMap[equivalentPath] = equivalentPath;
       } else {
         const relativePath = joinParenthesesAndArrays(keyParts.slice(1));
-        attributesMap[relativePath] = signatureSchema[key];
+        attributesMap[relativePath] = typeValue;
         associationMap[reverse ? relativePath : key] = reverse
           ? key
           : relativePath;
@@ -75,25 +174,41 @@ export function gatherAttributesMap(
     }
   }
 
-  for (const filePath in dependencySchemas) {
-    for (const entityName in dependencySchemas[filePath]) {
-      const returnValueDataStructure =
-        dependencySchemas[filePath][entityName].returnValueSchema ?? {};
+  // Process dependency schemas - iterate through isolatedDependencySchemas (for which paths to include)
+  // but use mergedDependencySchemas for types (richer type information)
+  for (const filePath in isolatedDependencySchemas) {
+    for (const entityName in isolatedDependencySchemas[filePath]) {
+      // Get the isolated return value schema (determines which paths exist)
+      const isolatedReturnValueSchema =
+        isolatedDependencySchemas[filePath][entityName].returnValueSchema ?? {};
 
-      // First pass: collect all function paths
+      // Get the merged return value schema (for richer types)
+      const mergedReturnValueSchema =
+        mergedDependencySchemas?.[filePath]?.[entityName]?.returnValueSchema ??
+        isolatedReturnValueSchema;
+
+      // First pass: collect all function paths from merged schema (for broader coverage)
       const functionPaths: string[] = [];
-      for (const path in returnValueDataStructure) {
-        if (returnValueDataStructure[path] === 'function') {
+      for (const path in mergedReturnValueSchema) {
+        if (mergedReturnValueSchema[path] === 'function') {
           functionPaths.push(path);
         }
       }
 
-      // Second pass: process all paths
-      for (const path in returnValueDataStructure) {
+      // Second pass: process paths from merged schema, but only include paths that
+      // are rooted in an isolated path (i.e., isolatedDataStructure knows about this dependency)
+      for (const path in mergedReturnValueSchema) {
         const pathParts = splitOutsideParenthesesAndArrays(path);
         if (pathParts.length < 2) continue;
 
-        if (returnValueDataStructure[path] === 'function') {
+        // Filter out paths that contain signature[N] anywhere
+        // These represent function input parameters, not data from data sources
+        // Key attributes must reference data sources, not function inputs
+        if (pathParts.some((part) => part.startsWith('signature['))) {
+          continue;
+        }
+
+        if (mergedReturnValueSchema[path] === 'function') {
           continue;
         }
 
@@ -141,7 +256,7 @@ export function gatherAttributesMap(
               const relativePath = path;
               if (!validAttribute(relativePath)) continue;
 
-              attributesMap[relativePath] = returnValueDataStructure[path];
+              attributesMap[relativePath] = mergedReturnValueSchema[path];
               associationMap[reverse ? relativePath : path] = reverse
                 ? path
                 : relativePath;
@@ -153,10 +268,14 @@ export function gatherAttributesMap(
           // Fall through to normal processing for other return value properties
         }
 
-        const relativePath = joinParenthesesAndArrays(pathParts.slice(2));
-        if (!validAttribute(relativePath)) continue;
+        const rawRelativePath = joinParenthesesAndArrays(pathParts.slice(2));
+        if (!validAttribute(rawRelativePath)) continue;
+
+        // Normalize the path for internal use (e.g., ['*'] -> '*')
+        const relativePath = normalizePathForInternalUse(rawRelativePath);
 
-        attributesMap[relativePath] = returnValueDataStructure[path];
+        // Use the type from mergedReturnValueSchema
+        attributesMap[relativePath] = mergedReturnValueSchema[path];
         associationMap[reverse ? relativePath : path] = reverse
           ? path
           : relativePath;

package/analyzer-template/packages/ai/src/lib/promptGenerators/generateChangesEntityDocumentationGenerator.ts

@@ -4,44 +4,19 @@ export default function generateChangesEntityDocumentationGenerator(
   entity: Entity,
   previousDocumentation: string,
 ) {
-  return `A function recently changed. Here is the current documentation for that function:
+  return `## Task
+Review if the existing documentation is still accurate after code changes. If accurate, respond "No changes needed". Otherwise, provide updated documentation.
 
-"""
+## Current Documentation
 ${previousDocumentation}
-"""  
 
-and here is the new code for ${entity.name}:
-  
-"""
+## Updated Code
+\`\`\`typescript
 ${entity.code}
-"""
-
-If the existing documentation is still accurate please respond with "No changes needed". 
-
-Otherwise, please update the documentation to reflect the changes in the code. Using these instructions:
-
-- Start with an overview that describes the overall intent of the code. Be concise in your language. Immediately start with the intent of the code. Here is an example Overview:
-
-\`\`\` Overview
-Displays details about a specific Git commit. It shows the commit message, author, date, and provides a link to the commit on GitHub. Users can copy the commit SHA to their clipboard, with a visual indication of the copy action.
 \`\`\`
 
-Note that it does not mention the name of the function but dives directly into what the code does in concise language followed by a small amount of detail.
-
-- Next, specify the inputs (with details about types, how the input is used, and examples) and expected outputs (with details about types and examples).
-
-- Finally, provide examples of how the code can be used with clear inputs and outputs for each example. Provide diverse examples (there's no need to include repetitive examples). 
-
-Write the documentation as markdown. Be concise. Use bullets when possible. No one wants to read long paragraphs of text. 
-
-Provide no other text than the documentation in proper markdown using the headers below.
-
-## Overview
-
-## Inputs
-
-## Output
+## Overview Example
+Good: "Displays details about a specific Git commit. Shows the commit message, author, date, and provides a link to GitHub."
 
-## Examples
-`;
+Bad: "The CommitDetails function takes a commit object..." (mentions the function name)`;
 }

package/analyzer-template/packages/ai/src/lib/promptGenerators/generateChangesEntityScenarioDataGenerator.ts

@@ -52,77 +52,69 @@ export default function generateChangesEntityScenarioDataGenerator(
   defaultScenarioData?: ScenarioData,
   incompleteResponse?: string,
 ) {
-  return `${
-    incompleteResponse
-      ? `## Incomplete Response
-
-Here is the incomplete response. Please use this and the following data to complete the response, only returning the missing data.
-
-Just pick up where this response left off. It will be improper json but we will combine the responses into proper json.
-
-\`\`\` START INCOMPLETE RESPONSE \`\`\`
+  const incompleteSection = incompleteResponse
+    ? `## Incomplete Response
+Continue from where this left off (combine for valid JSON):
+\`\`\`
 ${incompleteResponse}
-\`\`\` END INCOMPLETE RESPONSE \`\`\`
+\`\`\`
 
 `
-      : ''
-  }## Scenario
-  
-This scenario needs mock data to support it. Here are the keyAttributeInstructions for generating this scenario:
-\`\`\`
-${JSON.stringify(
-  {
-    name: scenario.name,
-    description: scenario.description,
-    keyAttributeInstructions: scenario.metadata.keyAttributeInstructions,
-  },
-  null,
-  2,
-)}
-\`\`\`
+    : '';
 
-## Existing Scenario
-  
-Here are the "keyAttributeInstructions" and data for this scenario before this more recent code change. Please maintain the existing data as effectively as possible while fulfilling the new data structure and "keyAttributeInstructions".
-\`\`\`
-${JSON.stringify(
-  {
-    name: existingScenario.name,
-    data: filterData(structure, existingScenario.metadata.data),
-  },
-  null,
-  2,
-)}
-\`\`\`
-${
-  defaultScenarioData
-    ? `
-## Default Scenario
+  const scenarioJson = JSON.stringify(
+    {
+      name: scenario.name,
+      description: scenario.description,
+      coveredFlows: scenario.metadata?.coveredFlows || [],
+    },
+    null,
+    2,
+  );
 
-Data for the default scenario. Do not modify it. The data you are generating will be merged into it to create the full data.
-\`\`\`
+  const existingJson = JSON.stringify(
+    {
+      name: existingScenario.name,
+      data: filterData(structure, existingScenario.metadata.data),
+    },
+    null,
+    2,
+  );
+
+  const defaultSection = defaultScenarioData
+    ? `
+## Default Scenario Data (merge base, do not modify)
+\`\`\`json
 ${JSON.stringify(defaultScenarioData.data, null, 2)}
 \`\`\`
 `
-    : ''
-}
-${
-  structure.dataForMocks
-    ? `The mockData structure (for the default scenario provide robust values for the entire structure):
-\`\`\`
+    : '';
+
+  const mockDataSection = structure.dataForMocks
+    ? `## mockData Structure
+\`\`\`json
 ${JSON.stringify(structure.dataForMocks, null, 2)}
-\`\`\`
-`
-    : 'There is no mockData for this file. Return {} for the mockData.'
-}
-    
-${
-  structure.arguments
-    ? `The argumentsData structure (for the default scenario provide robust values for the entire structure):
-\`\`\`
+\`\`\``
+    : '## mockData Structure\nNone—return `{}` for mockData.';
+
+  const argumentsSection = structure.arguments
+    ? `## argumentsData Structure
+\`\`\`json
 ${JSON.stringify(structure.arguments, null, 2)}
+\`\`\``
+    : '## argumentsData Structure\nNone—return `[]` for argumentsData.';
+
+  return `${incompleteSection}## Scenario (new instructions)
+\`\`\`json
+${scenarioJson}
 \`\`\`
-`
-    : 'There is no argumentsData for this file. Return [] for the argumentsData.'
-}`;
+
+## Existing Scenario (preserve where possible)
+\`\`\`json
+${existingJson}
+\`\`\`
+${defaultSection}
+${mockDataSection}
+
+${argumentsSection}`;
 }