@codeyam/codeyam-cli 0.1.0-staging.1 → 0.1.0-staging.1669d45

package/analyzer-template/project/controller/startController.ts

@@ -13,8 +13,8 @@ export enum ControllerPhase {
   Startup = 'STARTUP',
   RepoCheckout = 'REPO_CHECKOUT',
   Analysis = 'ANALYSIS',
+  AnalysisProducing = 'ANALYSIS_PRODUCING',
   Serving = 'SERVING',
-  Orchestrating = 'ORCHESTRATING',
   Complete = 'COMPLETE',
 }
 
@@ -30,6 +30,7 @@ type ControllerContext =
   | { phase: ControllerPhase.Startup; info: null }
   | { phase: ControllerPhase.RepoCheckout; info: null }
   | { phase: ControllerPhase.Analysis; info: null }
+  | { phase: ControllerPhase.AnalysisProducing; info: null }
   | { phase: ControllerPhase.Serving; info: ServingCodeYamInfo }
   | { phase: ControllerPhase.Complete; info: null };
 
@@ -100,6 +101,11 @@ export class Controller extends EventEmitter {
     this.setPhase({ phase: ControllerPhase.Analysis, info: null });
   }
 
+  setAnalysisProducingPhase(): void {
+    this.setPhase({ phase: ControllerPhase.AnalysisProducing, info: null });
+    this.emit('analysisProducing');
+  }
+
   setServingPhase(info: ServingCodeYamInfo): void {
     this.setPhase({ phase: ControllerPhase.Serving, info });
   }
@@ -116,6 +122,15 @@ export class Controller extends EventEmitter {
     );
   }
 
+  hasStartedProducing(): boolean {
+    // Analysis is producing once we've moved past the Analysis phase
+    return (
+      this.context.phase === ControllerPhase.AnalysisProducing ||
+      this.context.phase === ControllerPhase.Serving ||
+      this.context.phase === ControllerPhase.Complete
+    );
+  }
+
   async close(): Promise<void> {
     if (!this.server) return;
 

package/analyzer-template/project/createEntitiesAndSortFiles.ts

@@ -11,9 +11,12 @@ import {
   updateCommitMetadata,
   upsertEntities,
   upsertEntityBranches,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import { LazyFileStore } from './LazyFileStore';
 
+// Track files we've already warned about to avoid duplicate warnings
+const warnedMissingFiles = new Set<string>();
+
 export interface EntityWithImportCount extends Entity {
   importCount?: number;
   started?: string;
@@ -89,12 +92,17 @@ async function findAndGatherEntity({
         ? fileStore.getByPath(importedEntityFilePath)
         : project.files.find((f) => f.path === importedEntityFilePath);
       if (!entityFile) {
-        console.log(
-          'CodeYam Error: Entity file not found for create and sort entities',
-          importedEntityName,
-          importedEntityFilePath,
-        );
-        throw new Error('Entity file not found for create and sort entities');
+        // File not in project - likely a mocked dependency or external library
+        // Skip this import rather than throwing, similar to gatherEntityMap.ts
+        if (!warnedMissingFiles.has(importedEntityFilePath)) {
+          warnedMissingFiles.add(importedEntityFilePath);
+          console.log(
+            'CodeYam Warning: Skipping dependency in create and sort - file not in project.files:',
+            importedEntityName,
+            importedEntityFilePath,
+          );
+        }
+        continue;
       }
 
       const info = await findAndGatherEntity({

package/analyzer-template/project/executeLibraryFunctionDirect.ts

@@ -33,11 +33,15 @@ export default async function executeLibraryFunctionDirect(
   // Generate the execution script
   const script = directExecutionScript(file, entity, analysis, scenario);
 
+  // Use .ts extension for TypeScript source files so tsx can resolve TS imports
+  const isTypeScript = file.path.endsWith('.ts') || file.path.endsWith('.tsx');
+  const scriptExtension = isTypeScript ? '.ts' : '.mjs';
+
   // Write the script to a temporary file
   const scriptPath = path.join(
     PROJECT_RELATIVE_PATH,
     '.codeyam-execution',
-    `${entity.sha}_${scenario.id}.mjs`,
+    `${entity.sha}_${scenario.id}${scriptExtension}`,
   );
 
   await writeFile(scriptPath, script);
@@ -54,10 +58,10 @@ export default async function executeLibraryFunctionDirect(
       command: 'npx',
       args: ['tsx', scriptPath],
       workingDir: PROJECT_RELATIVE_PATH,
-      onData: async (data) => {
+      onData: (data) => {
         stdout += data;
       },
-      onError: async (data) => {
+      onError: (data) => {
         stderr += data;
       },
       processName: 'library-function-execution',

package/analyzer-template/project/getFilesWithEntitiesFromRepo.ts

@@ -3,7 +3,7 @@ import {
   saveFiles,
   generateSha,
   loadEntities,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import { File, Project } from '~codeyam/types';
 import { ProjectAnalyzer } from '~codeyam/analyze';
 

package/analyzer-template/project/getScenarioUrl.ts

@@ -1,6 +1,17 @@
-import { Analysis, Project, ProjectFramework, Scenario, File } from '~codeyam/types';
+import {
+  Analysis,
+  Project,
+  ProjectFramework,
+  Scenario,
+  File,
+} from '~codeyam/types';
 import getScenarioPagePaths from './getScenarioPagePath';
-import { frameworkRouteFileNameToRoute, isFrameworkRoute, PROJECT_RELATIVE_PATH, getAppDirectoryForFile } from '~codeyam/utils';
+import {
+  frameworkRouteFileNameToRoute,
+  isFrameworkRoute,
+  PROJECT_RELATIVE_PATH,
+  getAppDirectoryForFile,
+} from '~codeyam/utils';
 
 export interface GetScenarioUrlArgs {
   project: Project;
@@ -10,6 +21,48 @@ export interface GetScenarioUrlArgs {
   framework: ProjectFramework;
 }
 
+/**
+ * Extract route params from scenario data for URL generation.
+ * Handles array values (for catch-all routes) by joining them.
+ *
+ * Scenario data can be stored in two locations:
+ * - scenario.data.argumentsData (primary)
+ * - scenario.metadata.data.argumentsData (fallback)
+ *
+ * @param scenario - The scenario containing mock data
+ * @returns A record of param names to string values, or undefined if no params
+ */
+function extractRouteParamsFromScenario(
+  scenario: Scenario,
+): Record<string, string | string[]> | undefined {
+  // Check both possible locations for argumentsData
+  const argumentsData =
+    scenario.data?.argumentsData ?? scenario.metadata?.data?.argumentsData;
+
+  if (!Array.isArray(argumentsData) || argumentsData.length === 0) {
+    return undefined;
+  }
+
+  const params = (argumentsData[0] as Record<string, unknown>)?.params;
+  if (!params || typeof params !== 'object') {
+    return undefined;
+  }
+
+  // Convert all values to strings or string arrays
+  const result: Record<string, string | string[]> = {};
+  for (const [key, value] of Object.entries(
+    params as Record<string, unknown>,
+  )) {
+    if (Array.isArray(value)) {
+      result[key] = value.map(String);
+    } else if (value !== null && value !== undefined) {
+      result[key] = String(value);
+    }
+  }
+
+  return Object.keys(result).length > 0 ? result : undefined;
+}
+
 /**
  * Get the URL path for a scenario (the path that goes in the iframe src)
  * This is the same logic used in generateCodeYamPage.ts to determine the iframe src
@@ -40,7 +93,10 @@ export default function getScenarioUrl({
     case ProjectFramework.Next:
       // the routePath looks like: relative/path/app/123/static/c054b7ee-3961-4697-8354-886062a95a9d/default/page.tsx
       // the path should look like: static/123/c054b7ee-3961-4697-8354-886062a95a9d/default/0
-      const appDirectory = getAppDirectoryForFile(file.path, project.metadata?.webapps);
+      const appDirectory = getAppDirectoryForFile(
+        file.path,
+        project.metadata?.webapps,
+      );
       path = routePath
         .replace(PROJECT_RELATIVE_PATH + '/' + appDirectory + '/', '')
         .split('/')
@@ -50,7 +106,10 @@ export default function getScenarioUrl({
     case ProjectFramework.NextPages:
       // the routePath looks like: relative/path/app/123/static/c054b7ee-3961-4697-8354-886062a95a9d/default/page.tsx
       // the path should look like: static/123/c054b7ee-3961-4697-8354-886062a95a9d/default/0
-      const pagesDirectory = getAppDirectoryForFile(file.path, project.metadata?.webapps);
+      const pagesDirectory = getAppDirectoryForFile(
+        file.path,
+        project.metadata?.webapps,
+      );
       path = routePath
         .replace(PROJECT_RELATIVE_PATH + '/' + pagesDirectory + '/', '')
         .split('/')
@@ -62,7 +121,16 @@ export default function getScenarioUrl({
   }
 
   if (isFrameworkRoute(file, analysis.entity, framework)) {
-    path = frameworkRouteFileNameToRoute(project.slug, file, analysis, framework, scenario.name);
+    // Extract route params from scenario mock data for URL generation
+    const routeParams = extractRouteParamsFromScenario(scenario);
+    path = frameworkRouteFileNameToRoute(
+      project.slug,
+      file,
+      analysis,
+      framework,
+      scenario.name,
+      routeParams,
+    );
   }
 
   return `/${path}`;

package/analyzer-template/project/loadReadyToBeCaptured.ts

@@ -1,5 +1,9 @@
 import { AnalysisContext, discoverDirectDependencies } from '~codeyam/analyze';
-import { loadAnalysis, loadReadyToBeCapturedAnalyses } from '~codeyam/supabase';
+import {
+  loadAnalysis,
+  loadEntities,
+  loadReadyToBeCapturedAnalyses,
+} from '~codeyam/database';
 import { Analysis, Project } from '~codeyam/types';
 import { LazyFileStore } from './LazyFileStore';
 
@@ -11,6 +15,132 @@ interface LoadReadyToBeCapturedArgs {
   fileStore?: LazyFileStore;
 }
 
+/**
+ * Recursively gather all entities referenced in importedExports of entities already in context.
+ * This fills the gap where discoverDirectDependencies skips mocked/type/implicit imports,
+ * but writeScenarioComponents still needs those entities in the context.
+ */
+async function gatherAllReferencedEntities(
+  context: AnalysisContext,
+  project: Project,
+): Promise<void> {
+  const processedKeys = new Set<string>();
+  let iterationCount = 0;
+  const MAX_ITERATIONS = 20; // Safety limit to prevent infinite loops
+
+  const getAllEntities = () =>
+    Object.values(context.getMutableEntityMap()).flatMap(Object.values);
+
+  while (iterationCount < MAX_ITERATIONS) {
+    iterationCount++;
+
+    // Collect all (filePath, entityName) pairs from importedExports of entities in context
+    const missingEntities = new Map<
+      string,
+      { filePath: string; name: string }
+    >();
+
+    for (const entity of getAllEntities()) {
+      for (const ie of entity.metadata?.importedExports ?? []) {
+        const filePath = ie.resolvedFilePath ?? ie.filePath;
+        const entityName = ie.resolvedName ?? ie.name;
+        const key = `${filePath}:${entityName}`;
+
+        // Skip if already in context or already processed
+        if (context.getEntity(filePath, entityName) || processedKeys.has(key)) {
+          continue;
+        }
+
+        missingEntities.set(key, { filePath, name: entityName });
+      }
+    }
+
+    if (missingEntities.size === 0) {
+      break; // No more missing entities
+    }
+
+    console.log(
+      `CodeYam: Entity gathering iteration ${iterationCount}: Loading ${missingEntities.size} missing entities`,
+    );
+
+    // Mark all as processed before loading to prevent re-processing
+    for (const key of missingEntities.keys()) {
+      processedKeys.add(key);
+    }
+
+    // Group by file path for batch loading
+    const byFilePath = new Map<string, string[]>();
+    for (const { filePath, name } of missingEntities.values()) {
+      if (!byFilePath.has(filePath)) {
+        byFilePath.set(filePath, []);
+      }
+      byFilePath.get(filePath)!.push(name);
+    }
+
+    // Batch load entities by file path
+    const loadPromises = Array.from(byFilePath.entries()).map(
+      async ([filePath, names]) => {
+        const entities = await loadEntities({
+          projectId: project.id,
+          filePaths: [filePath],
+        });
+
+        if (!entities) return [];
+
+        // Filter to only the entities we need (loadEntities returns all for the file)
+        // Special handling: if we requested 'default' and got a default export entity,
+        // include it even though the name doesn't match (entities are stored by actual name, not 'default')
+        return entities.filter((e) => {
+          if (names.includes(e.name)) {
+            return true;
+          }
+          // If we requested 'default' and this is a default export, include it
+          if (
+            names.includes('default') &&
+            e.metadata &&
+            !e.metadata.notExported &&
+            !e.metadata.namedExport
+          ) {
+            return true;
+          }
+          return false;
+        });
+      },
+    );
+
+    const results = await Promise.all(loadPromises);
+    const loadedEntities = results.flat();
+
+    // Add loaded entities to context
+    for (const entity of loadedEntities) {
+      context.addEntity(entity);
+
+      // Also register under 'default' alias if it's a default export
+      if (
+        entity.metadata &&
+        !entity.metadata.notExported &&
+        !entity.metadata.namedExport
+      ) {
+        context.addEntity(entity, 'default');
+      }
+    }
+
+    console.log(
+      `CodeYam: Loaded ${loadedEntities.length} entities in iteration ${iterationCount}`,
+    );
+  }
+
+  if (iterationCount >= MAX_ITERATIONS) {
+    console.log(
+      `CodeYam Warning: Entity gathering reached max iterations (${MAX_ITERATIONS}), some entities may be missing`,
+    );
+  }
+
+  console.log(
+    `CodeYam: Entity gathering complete after ${iterationCount} iteration(s), context has ${getAllEntities().length} entities`,
+  );
+}
+
 /**
  * Load the minimal set of analyses required to successfully run scenario capture.
  * This includes the root analyses (with scenarios) plus all recursive mock dependencies.
@@ -145,7 +275,7 @@ export default async function loadReadyToBeCaptured({
     `CodeYam: Loaded ${allRequiredAnalyses.size} total analyses for scenario capture`,
   );
 
-  // Populate the provided analysis context
+  // Populate the provided analysis context with analyses and their direct entities
   for (const analysis of allRequiredAnalyses.values()) {
     // Add the entity if it exists
     if (analysis.entity) {
@@ -156,6 +286,12 @@ export default async function loadReadyToBeCaptured({
     context.addAnalysis(analysis, analysis.filePath, analysis.entityName);
   }
 
+  // Recursively gather ALL entities referenced in importedExports.
+  // This is necessary because discoverDirectDependencies only finds entities that need
+  // ANALYSES loaded (skipping mocked imports, type imports, etc.), but writeScenarioComponents
+  // needs ALL entities in the context for import path rewriting and recursion.
+  await gatherAllReferencedEntities(context, project);
+
   // Extract the root analyses with scenarios from allRequiredAnalyses (they were reloaded with mocks)
   const finalReadyToBeCaptured = readyToBeCapturedAnalyses.map(
     (originalAnalysis) => {

package/analyzer-template/project/mocks/analyzeFileMock.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/require-await */
 import {
   Analysis,
   AnalysisMap,
@@ -47,7 +48,7 @@ const standaloneComponentAnalysis = {
         'The counter is initialized at 0, with the ability to increment and decrement the count.',
       metadata: {
         dataMapping: {},
-        keyAttributeInstructions: {},
+        coveredFlows: [],
         data: {
           argumentsData: [],
           mockData: {},
@@ -221,7 +222,7 @@ async function analyzeReturnsSimplePropsFile(): Promise<{
                 'A user who is under 18 years old and is currently active.',
               metadata: {
                 dataMapping: {},
-                keyAttributeInstructions: {},
+                coveredFlows: [],
                 data: {
                   argumentsData: [
                     {
@@ -242,7 +243,7 @@ async function analyzeReturnsSimplePropsFile(): Promise<{
                 'A user who is between 18 and 39 years old and is currently active.',
               metadata: {
                 dataMapping: {},
-                keyAttributeInstructions: {},
+                coveredFlows: [],
                 data: {
                   argumentsData: [
                     {
@@ -263,7 +264,7 @@ async function analyzeReturnsSimplePropsFile(): Promise<{
                 'A user who is 40 years or older and is currently active.',
               metadata: {
                 dataMapping: {},
-                keyAttributeInstructions: {},
+                coveredFlows: [],
                 data: {
                   argumentsData: [
                     {
@@ -284,7 +285,7 @@ async function analyzeReturnsSimplePropsFile(): Promise<{
                 'A user who is under 18 years old and is currently inactive.',
               metadata: {
                 dataMapping: {},
-                keyAttributeInstructions: {},
+                coveredFlows: [],
                 data: {
                   argumentsData: [
                     {
@@ -305,7 +306,7 @@ async function analyzeReturnsSimplePropsFile(): Promise<{
                 'A user who is between 18 and 39 years old and is currently inactive.',
               metadata: {
                 dataMapping: {},
-                keyAttributeInstructions: {},
+                coveredFlows: [],
                 data: {
                   argumentsData: [
                     {
@@ -378,7 +379,7 @@ async function analyzeReturnsImportedComponentsFile(): Promise<{
               description: 'Default scenario description',
               metadata: {
                 dataMapping: {},
-                keyAttributeInstructions: {},
+                coveredFlows: [],
                 data: {
                   argumentsData: [],
                   mockData: {},

package/analyzer-template/project/orchestrateCapture/KyselyAnalysisLoader.ts

@@ -1,5 +1,5 @@
 import type { Commit, Project } from '~codeyam/types';
-import { getDatabase } from '~codeyam/supabase';
+import { getDatabase } from '~codeyam/database';
 import { sql } from 'kysely';
 import type { AnalysisLoader, ReadyAnalysisData } from './taskRunner';
 
@@ -46,21 +46,17 @@ export default class KyselyAnalysisLoader implements AnalysisLoader {
             'status',
           ])
           // Base condition: readyToBeCaptured = true
-          .where(
-            sql<string>` ${sql.ref('status')} -> 'readyToBeCaptured'`,
-            '=',
-            'true',
+          // Use -> operator which works in both PostgreSQL and SQLite
+          // Both return a value that compares equal to string 'true'
+          .where((eb) =>
+            eb(sql`${sql.ref('status')} -> 'readyToBeCaptured'`, '=', 'true'),
           )
           // Project filter (always required)
           .where('project_id', '=', this.config.project.id);
 
         // Optional: branch commit SHA filter
         if (this.config.commit?.metadata?.baseline) {
-          query = query.where(
-            'branch_commit_sha',
-            '=',
-            this.config.commit.sha,
-          );
+          query = query.where('branch_commit_sha', '=', this.config.commit.sha);
         }
 
         // Optional: commit ID filter
@@ -98,11 +94,11 @@ export default class KyselyAnalysisLoader implements AnalysisLoader {
             return false;
           }
 
-          // Check: has incomplete scenarios
+          // Check: has scenarios needing capture
           // Include if:
           // - No scenarios field
           // - Empty scenarios array
-          // - At least one scenario without finishedAt or error
+          // - At least one scenario that needs capture (no finishedAt OR has error)
           const scenarios = status?.scenarios;
           if (!scenarios || !Array.isArray(scenarios)) {
             return true; // No scenarios field
@@ -111,12 +107,16 @@ export default class KyselyAnalysisLoader implements AnalysisLoader {
             return true; // Empty array
           }
 
-          // Check if at least one scenario is incomplete
-          const hasIncompleteScenario = scenarios.some(
+          // Check if at least one scenario needs capture
+          // A scenario needs capture if it has no finishedAt AND no error
+          // (i.e., it hasn't been attempted yet)
+          // Scenarios with finishedAt are complete (successful)
+          // Scenarios with error are complete (failed, not retried automatically)
+          const needsCapture = scenarios.some(
             (scenario: any) => !scenario.finishedAt && !scenario.error,
           );
 
-          return hasIncompleteScenario;
+          return needsCapture;
         });
 
         // Map to ReadyAnalysisData, excluding the status field