@codeyam/codeyam-cli 0.1.0-staging.1 → 0.1.0-staging.596f0eb

package/analyzer-template/packages/ai/src/lib/generateChangesEntityScenarios.ts

@@ -46,6 +46,7 @@ export default async function generateChangesEntityScenarios({
 }: GenerateChangesEntityScenariosArgs) {
   const relevantDependentKeyAttributes = gatherRelevantDependentKeyAttributes(
     entity,
+    analysis.metadata.mergedDataStructure,
     dependentAnalyses,
   );
   if (
@@ -71,6 +72,7 @@ export default async function generateChangesEntityScenarios({
 
   const prompt = generateChangesEntityScenariosGenerator({
     entity,
+    mergedDataStructure: analysis.metadata.mergedDataStructure,
     keyAttributes,
     addedKeyAttributes,
     removedKeyAttributes,
@@ -269,325 +271,139 @@ export default async function generateChangesEntityScenarios({
 function generateChangesSystemMessage(
   context?: string,
   scenarioCount?: number,
-  isError?: boolean,
+  _isError?: boolean,
 ): string {
-  return `We have a function that was recently changed. We want to know if the existing data scenarios are still valid, if any should be changed or removed at all, or if any should be added to effectively capture the change that just took place.
-
-${
-  context
-    ? `## Business Logic Context
-
+  const contextSection = context
+    ? `## Business Context
 ${context}
 
-Focus your scenario updates on demonstrating this specific change. ${scenarioCount ? `Generate ${scenarioCount} total scenario${scenarioCount > 1 ? 's' : ''}.` : ''}
+Focus scenario updates on demonstrating this change.${scenarioCount ? ` Generate ${scenarioCount} total scenario${scenarioCount > 1 ? 's' : ''}.` : ''}
 
 `
-    : ''
-}
-
-${
-  !context && scenarioCount
-    ? `Generate ${scenarioCount} total scenario${scenarioCount > 1 ? 's' : ''}.
+    : scenarioCount
+      ? `Generate ${scenarioCount} total scenario${scenarioCount > 1 ? 's' : ''}.
 
 `
-    : ''
-}
-
-Also if any of the existing scenarios have playwright instructions, we want to ensure that they are still valid and if not, update them to reflect the changes.
+      : '';
 
-It is important to focus on the added or removed key attributes, the change to the code, and the playwright instructions. If a dependency was involved in the change then look at its key attributes, if it has any, as well. 
+  return `You are updating data scenarios after a code change. Determine which existing scenarios need updating, which should be removed, and if new ones are needed.
 
-If the change involves a dependency that has key attributes listed then be sure to include key attribute instructions for the key attributes of that dependency in the scenarios. We want to ensure dependencies demonstrate their state in this function as well.
+${contextSection}## Update Rules
 
-We want to ensure that we have scenarios that capture the changes and how the function behaves differently now. The added or removed key attributes and the playwright instructions are the most important part of this.
+### Removed Key Attributes
+- Update scenarios using removed attributes—replace with new attributes or remove if no longer relevant
+- CRITICAL: "${DEFAULT_SCENARIO_NAME}" must NEVER contain removed attributes—update or replace them
+- Try to maintain 3+ scenarios; create replacements if removing too many
 
-If a key attribute was removed:
-  - Look at scenarios that use that key attribute.
-  - If the scenario still makes sense as a scenario for this code then adjust the \`keyAttributeInstructions\` to add new key attributes that make up for the removed key attributes.
-  - If the scenario no longer makes sense then remove it.
-  - Try to ensure scenarios that scenarios are not removed without replacing them with another scenario unless the existing scenario is truly no longer needed. If there are fewer than three scenarios remaining please create new scenarios to ensure there are at least three scenarios that capture the behavior of the function.
+### Added Key Attributes
+- Create or update scenarios to demonstrate new behavior
 
-IMPORTANT: If the ${DEFAULT_SCENARIO_NAME} scenario contains a removed key attribute then it must be updated to remove that key attribute, usually replacing it with a new key attribute. This is the default scenario. It should not contain any removed key attributes, but it must always exist. If it does then it is not a valid scenario.
+### Playwright Instructions
+- Validate existing instructions still work after code changes
+- Update any that reference changed UI elements
 
-If a key attribute was added:
-  - Create new or edit existing scenarios to capture the new behavior that the added key attribute introduces.
+### Unchanged Scenarios
+- Return name only (no other fields needed)
 
-If any of the playwright instructions are no longer valid:
-  - Update the playwright instructions to reflect the changes in the code.
+### Default Scenario
+- First scenario must be "${DEFAULT_SCENARIO_NAME}" (exact name)
+- NEVER include playwrightInstructions—captures initial state only
+- Must include all current key attributes with valid values
 
-If new scenarios are warranted, note in the description for the scenario how the output of the function is affected and what the user should look for to understand what has changed. This is very important for the user to understand the impact of the change. For new scenarios modify the ${DEFAULT_SCENARIO_NAME} keyAttributeInstructions as little as possible to highlight the change or range of changes.
+## Response Format
 
-If no changes are required to the existing scenarios (all keyAttributes, their values, and any playwright instructions are still valid) and there are no new scenarios then provide a brief overview of the change and why no scenarios are changed or new scenarios are needed. For example:
-
-\`\`\`
+If no changes needed:
+\`\`\`json
 {
-  "overview": "The styles of the returned component are updated. The existing scenarios will show these changes. No new scenarios are needed.",
+  "overview": "Style updates only. Existing scenarios remain valid.",
   "dataScenarios": "No changes required"
 }
 \`\`\`
 
-Otherwise return all relevant scenarios, only removing ones that no longer make sense. Always try to replace removed scenarios with new scenarios if the behavior the scenario was testing is still relevant, even if the key attributes have changed.
-
-If an existing scenario is still valid and requires no changes (all keyAttributes, their values, and any playwright instructions are still valid) then simply return it, but with nothing but the name provided in the response.
-
-When creating a new scenario follow the pattern of the existing scenarios, providing "keyAttributeInstructions" that another AI can read to generate the data for each scenario. The "keyAttributeInstructions" should detail exactly how to populate the data structure with data for the function to behave as described in the scenario. New scenarios should have distinctly different "keyAttributeInstructions" from existing scenarios to higlight diverse states for the function.
-
-You can assume the other AI will generate reasonable data to satisfy the data structure, so focus the "keyAttributeInstructions" on the interesting parts of the data that should be generated.
-
-## Instructions
-
-- For each key attribute select from one of the valid values provided in the key attribtutes list.
-- If the function is a front-end (visual) function, then pay special attention to key attributes that will change the visual appearance of the function. In the "${DEFAULT_SCENARIO_NAME}" scenario if there are attributes such as "children", ensure they are populated so that they front end function renders visually interesting results.
-- Each scenario should have: 
-  - a unique name (should be short and concise with absolutely no special characters other than spaces).
-  - a "testName" written in the style of Jest the focuses on specific expectations for the test. Try to avoid generic names like "it(\"should work\")" or "it(\"should render\")" and describe more specific expectations.
-  - a description that describes why the key attributes were changed from the default scenario.
-  - "keyAttributeInstructions" specifying how the key attributes should be set.
-  - Optional "playwrightInstructions" that provide the necessary code to run in Playwright to put the function into the correct state to demonstrate the scenario.
-- Focus on positive scenarios that showcase the intended behavior of this function. We have another prompt for generating error scenarios.
-
-## Dependencies
-
-This function may dependend on other functions that have key attributes as well. You can leverage those to generate data scenarios also by creating key attribute instructions for those key attributes in the same manner. Generally speaking key attributes for dependencies are necessary for robust scenarios.
-
-## Relative Dates
-
-If a relative date is required you can describe it in the "keyAttributeInstructions", e.g. 'today', 'tomorrow', 'next week', 'last month'.
-
-## Rules
-
-- If the key attributes require react elements, keep them simple (e.g. <div> or <span>). Do not reference custom jsx elements as they will not exist.
-
-- Do not reference any external libraries, such as React, in the "keyAttributeInstructions". Assume no external libraries are available. Do not reference custom jsx elements. If needed describe how to mock the enternal library data using plain javascript that can be run in a node environment.
-
-## Response Structure
-
-Respond with the data scenarios in a JSON object format, without additional information or delimiters. Ensure clarity and specificity in each data scenario to facilitate accurate data generation for testing or demonstration purposes.
-
-{
-  "dataScenarios": [
-    {
-      "name": string;
-      "testName": string;
-      "description": string;
-      "keyAttributeInstructions": {
-        ["keyAttributePath"]: string;
-      },
-      "playwrightInstructions?": {
-        "selectorFn": string;
-        "selectorArgs": string[];
-        "actionFn": string;
-        "actionArgs": string[];
-      }[]
-    }
-  ]
-}
-
-Optionally you can include playwright instructions for front-end functions if any are needed to put the function into the correct state to demonstrate the scenario.
-
-This is often the case if the key attributes are only used through a button click or form submission or if the data required for this scenario is not listed as a key attribute and must be provided directly by the user.
-
-If the data won't exist or be used without user actions then use the "playwrightInstructions" to provide the necessary code to run in Playwright.
-
-Please check all existing scenarios for any outdated or invalid playwright instructions. If any exist, update them to reflect the changes in the code. and return the full scenario with the updated playwright instructions.
-
-For \`playwrightInstructions\`, provide an array of strings representing the necessary code to run.
+Otherwise:
 \`\`\`json
 {
-  "playwrightInstructions": [
-    {
-      "selectorFn": "getByLabel",
-      "selectorArgs": ["email"],
-      "actionFn": "fill",
-      "actionArgs": ["test@example.com"]
-    },
-    {
-      "selectorFn": "getByText",
-      "selectorArgs": ["Submit"],
-      "actionFn": "click",
-      "actionArgs": []
-    }
-  ]
-}
-\`\`\`
-
-For an existing scenario that does not need to change, only provide the name of the scenario in the response:
-
-{
-  "overview": string;
+  "overview": "Added empty state scenario for new display behavior.",
   "dataScenarios": [
-    {
-      "name": string;
-    }
-  ]
-}
-
-## Example
-
-Assuming a change that impacted how the component displayed if there were no notifications, the full response might be:
-
-\`\`\`
-{
-  "overview": "This change impacted how the component displays when there are no notifications. Creating one new scenario with no notifications to test this. All other scenarios are still valid.",
-  "dataScenarios": [
-    {
-      "name": ${DEFAULT_SCENARIO_NAME}
-    },
-    {
-      "name": "Many notifications, a few unread"
-    },
+    { "name": "${DEFAULT_SCENARIO_NAME}" },
+    { "name": "Many notifications" },
     {
       "name": "No notifications",
-      "testName": "it(\"renders a default message when there are no notifications\")",
-      "description": "A user has no notifications at all.",
+      "testName": "it(\\"shows empty message when no notifications\\")",
+      "description": "User has no notifications.",
       "keyAttributeInstructions": {
-        "notifications[]": "set to an empty array",
+        "notifications[]": { "localVariable": "notifications[]", "instruction": "Empty array" }
       }
     }
   ]
 }
 \`\`\`
 
-Note the terseness of the description for the new scenario. It should not start with "This scenario..." or "In this scenario...". It should be a simple declarative statement that describes the scenario.
-
-The first data scenario is your response should always be the ${DEFAULT_SCENARIO_NAME} with that exact name.
-
-Remember to only use the valid values provided in the key attributes list in the "keyAttributeInstructions".
-
-Remember to include keyAttributeInstructions for dependencies as well, especially if the change involves one.
-
-Do not use any special characters in the scenario names. Keep them short and concise and only use letters, numbers, and spaces.
-
-Remember to update playwright instructions, if any exist, as needed. Scenarios can not have invalid playwright instructions or the simulation will fail.
+## Rules
+- Scenario names: letters, numbers, spaces only
+- testName: Jest style with specific expectations
+- Use values from \`validValueOptions\` in key attributes list
+- Include dependency key attributes if dependencies involved
+- React elements: simple only (\`<div>\`, \`<span>\`)
+- No external library references—plain JS/TS values
+- Relative dates OK: "today", "tomorrow", "last month"
 `;
 }
 
-const ERROR_SYSTEM_MESSAGE = `We have a function that was recently changed. We want to know if the change introduced any potential bugs or fixed any existing error scenarios.
-
-Please look at the diff and the resulting code to determine if there are any new ways to cause the function to fail that are not properly handled by existing error scenarios. 
-
-If the changes to the code do not introduce any more potential errors then do not create any new error scenarios. We've provided all existing error scenarios. Do not duplicate any of them. If any of them are no longer valid then remove them. If they are still valid then return their name with no additional information.
-
-If new error scenarios are warranted, note in the description for the scenario how the change introduced an error, provide a code snippet (or multiple) showing where it will occur, a description of how it can be fixed along with example code, and provide detailed instructions on how to generate the necessary data to cause the error.
-
-For each new error scenario also provide two evaluation dynamics:
-1) How likely is this error to occur? (1-10)
-- 1 = Unlikely. It would require passing in data that is clearly wrong and not suited for the function.
-- 10 = Guaranteed. There is a bug in the function that will happen regardless of the data passed in.
-
-2) How severe is the error? (1 or 2)
-- 1 = Minor. The error will not break the application. It may cause confusion or unexpected behavior, but will not cause a direct failure.
-- 2 = Major. The error will cause the function to fail and will need to be fixed.
-
-If there are no existing error scenarios and no new potential errors have been introduced then return the following:
-
-\`\`\`
-{
-  "errorDataScenarios": "No errors found"
-}
-\`\`\`
-
-## New Error Scenarios Instructions
+const ERROR_SYSTEM_MESSAGE = `You are reviewing error scenarios after a code change. Determine if the change introduced new potential bugs or fixed existing ones.
 
-You can assume the other AI will generate reasonable data to satisfy the data structure, so focus the "keyAttributeInstructions" on the parts of the data that must be generated to cause the error.
+## Task
+- Existing error scenarios: return name only if still valid, remove if fixed
+- New errors from the change: create new error scenarios
+- No new errors and no existing ones: return \`{ "errorDataScenarios": "No errors found" }\`
 
-We can only generate data for the exact data structure provided, regardless of the code for the function. Do not provide "keyAttributeInstructions" for data that is not in the data structure.
+## What Qualifies as an Error
+- MUST cause actual thrown error (not awkward behavior)
+- MUST be fixable in THIS function's code
+- MUST NOT be catchable by static type analysis
+- MUST NOT assume dependent functions fail
 
-All data must be plain that can be run in a node environment (not in the browser). Do not reference any external libraries in the "keyAttributeInstructions". If an external library is required by the function, then provide instructructions on how to fake or mock that data.
+## Likelihood Scale (1-10)
+- 1-3: Requires unusual/wrong data
+- 4-6: Plausible edge case
+- 7-9: Common real-world scenario
+- 10: ONLY if guaranteed bug regardless of input
 
-For each error scenario, start with a data mapping between the phrase you want to use in the "keyAttributeInstructions" and the exact variable in the data strucure. Then provide a name and a description that explains the scenario and why it is interesting. The name should be unique and short (no more than 30 characters), and the description should provide context for the error scenario.
+Be conservative with high scores.
 
-Focus on "Major" errors. Only include "Minor" errors if they are very likely to occur.
+## Severity Scale (1-2)
+- 1 = Minor: Unexpected behavior, no crash
+- 2 = Major: Function fails (focus here)
 
 ## Rules
+- Only reference data in the provided data structure
+- React elements: simple only (\`<div>\`, \`<span>\`)
+- No external library references—mock with plain JS
+- Each scenario causes a DISTINCT error type
 
-- Do not provide "keyAttributeInstructions" for data that is not in the data structure. 
-
-- If the data structure calls for react elements, keep them simple (e.g. <div> or <span>). Do not reference custom jsx elements as they will not exist.
-
-- Do not reference any external libraries, such as React, in the "keyAttributeInstructions". Assume no external libraries are available. Do not reference custom jsx elements.
-
-- Only create new error scenarios if the scenario will literally cause an error. We are not looking for awkward or unintended results, only literal errors. Do not create an error scenario if you are not confident it will cause an error. 
-
-- Do not assume that another function might cause an error. We are looking for errors that are a direct result of the code in this function.
-
-- Do not create errors that can be caught through normal type detection. We only want errors that are reasonably likely to happen and won't be caught by basic static code analysis.
+Note: In JS/TS, any non-zero value satisfies a boolean check—this won't error.
 
-- Only create new error scenarios that can be fixed by changing the code in this function. Do not create error scenarios that require changes to other functions or external libraries.
-
-## Response
-Respond with the error scenarios in a JSON object format, without additional information or delimiters. Ensure clarity and specificity in each error scenario to facilitate accurate data generation for testing or demonstration purposes.
-
-{
-  "errorDataScenarios": [
-    {
-      "dataMapping": { [key: string]: string };
-      "name": string;
-      "codeSnippet": string;
-      "description": string;
-      "fix": string;
-      "fixSnippet": string;
-      "likelihood": number;
-      "likelihoodDescription": string;
-      "severity": number;
-      "severityDescription": string;      
-      "keyAttributeInstructions": { [key: string]: string };
-    }
-  ]
-}
-
-## Example
-Given the following data structure:
-
-\`\`\`
-{
-  "props": {
-    "notifications": {
-      "message": "string",
-      "read": "boolean"
-    }[]
-  }
-}
-\`\`\`
-
-And assuming a change that impacted how the component displayed if there were no notifications, the full response might be:
-
-\`\`\`
+## Response Format
+\`\`\`json
 {
   "errorDataScenarios": [
+    { "name": "Existing valid error" },
     {
-      name: "Error: Existing error scenario 1"
-    },
-    {
-      name: "Error: Existing error scenario 2"
-    },
-    {
-      "name": "Error: Non-UTF8 characters",
-      "codeSnippet": "...\nconst encodedMessage = encodeURIComponent(message);\n...",
-      "description": "If the message contains non-utf8 characters the function will error out. The \`encodeURIComponent\` call will throw an error which is not caught.",
-      "fix": "Wrap the \`encodeURIComponent\` call in a try/catch block.",
-      "fixSnippet: "...\nlet encodedMessage: string | undefined;\ntry {\n  encodedMessage = encodeURIComponent(message)\n} catch (e) {\n  console.error('Error encoding message:', e)\n..."
+      "dataMapping": { "notifications array": "props.notifications" },
+      "name": "Non-utf8 encoding error",
+      "codeSnippet": "const encoded = encodeURIComponent(message);",
+      "description": "encodeURIComponent throws on non-utf8 characters",
+      "fix": "Wrap in try/catch",
+      "fixSnippet": "try { encoded = encodeURIComponent(message); } catch (e) { ... }",
       "likelihood": 7,
-      "likelihoodDescription": "The international nature of the function makes it likely that a user will input a message with non-utf8 characters.",
+      "likelihoodDescription": "International users may input non-utf8 characters",
       "severity": 2,
-      "severityDescription": "The function will error out if a message contains non-utf8 characters.",
+      "severityDescription": "Function throws uncaught error",
       "keyAttributeInstructions": {
-        "notifications[]": "set to an array of objects containing at least one notification with a message that contains non-utf8 characters.",
+        "notifications[]": "Array with message containing non-utf8 characters"
       }
     }
   ]
 }
 \`\`\`
-
-Please write both the \`codeSnippet\` and \`fixSnippet\` with appropriate line breaks to increase readability.
-
-Assume that all functions this function depends on exist even if they are not provided in the prompt. We have isolated this function for analysis but those dependencies do exist.
-
-Only create error scenarios based on the code that is in the function provided. 
-
-Common Errors In Your Analysis:
-- Typescript/javascript: 
-    - Any object or primitive other than 0 can be used to satisfy a boolean. This will not cause an error.
-
-Do not mark anything as a "10" for likelihood unless the error will literally happen every time the function is called. In general be cautious about high likelihood scores so that errors that are truly most likely to happen show up as higher priority.
 `;

package/analyzer-template/packages/ai/src/lib/generateEntityDataStructure.ts

@@ -23,9 +23,13 @@ export default async function generateEntityDataStructure({
   //   // code: entity.code.length,
   // });
 
-  const fileScope = isolateScopes(fileAnalyzer.sourceFile.getText(), {
+  // Use getFullText() to include leading trivia (comments/whitespace) so that
+  // positions from isolateScopes match AST node positions which include trivia
+  const fileText = fileAnalyzer.sourceFile.getFullText();
+  const fileScope = isolateScopes(fileText, {
     entityNames: [entity.name],
   });
+
   let entityScope =
     fileScope.name === entity.name
       ? fileScope
@@ -39,12 +43,10 @@ export default async function generateEntityDataStructure({
   }
 
   if (!entityScope) {
-    if (entity.code === fileAnalyzer.sourceFile.getText()) {
+    if (entity.code === fileText) {
       entityScope = fileScope;
     } else {
-      const entityStartIndex = fileAnalyzer.sourceFile
-        .getText()
-        .indexOf(entity.code);
+      const entityStartIndex = fileText.indexOf(entity.code);
       const entityIndex = entityStartIndex + entity.code.length / 2;
       entityScope = Object.values(fileScope.childScopes).find(
         (s) => entityIndex > s.start && entityIndex < s.end,

package/analyzer-template/packages/ai/src/lib/generateEntityDocumentation.ts

@@ -35,8 +35,7 @@ export default async function generateEntityDocumentation({
 
   const { completion: documentation, stats } = await completionCall({
     type: 'analyzeDocumentation',
-    systemMessage:
-      'You are an expert coder and educator. You think a lot about how people can understand new things, recognizing that it is a mixture of direct facts and numerous examples from different perspectives.',
+    systemMessage: SYSTEM_MESSAGE,
     prompt,
     model,
     jsonResponse: false,
@@ -68,3 +67,18 @@ export default async function generateEntityDocumentation({
 
   return { documentation, llmCall };
 }
+
+const SYSTEM_MESSAGE = `You are a technical documentation writer. Generate clear, concise markdown documentation.
+
+## Style
+- Be concise—use bullets, not paragraphs
+- Provide diverse examples (no repetitive variations)
+- In Overview, describe intent directly without mentioning the function name
+
+## Required Sections
+Return documentation with exactly these markdown headers:
+- ## Overview
+- ## Inputs
+- ## Output
+- ## Examples
+`;