@codeyam/codeyam-cli 0.1.0-staging.bbe4da9 → 0.1.0-staging.d0ad4ae

package/analyzer-template/playwright/takeElementScreenshot.ts

@@ -36,28 +36,43 @@ async function safeAwaitWithReturn<T>(
   }
 }
 
-/** Wait until DOM mutations stop for `quietMs` (default 500 ms). */
-async function waitForDomIdle(page: Page, quietMs = 500) {
+/** Wait until DOM mutations stop for `quietMs` (default 500 ms), or until `maxWaitMs` elapsed. */
+async function waitForDomIdle(page: Page, quietMs = 500, maxWaitMs = 5000) {
   await page.evaluate(
-    ({ quietMs }) =>
+    ({ quietMs, maxWaitMs }) =>
       new Promise<void>((resolve) => {
         let timer: NodeJS.Timeout;
+        const startTime = Date.now();
+
+        const finish = () => {
+          clearTimeout(timer);
+          observer.disconnect();
+          resolve();
+        };
+
         const reset = () => {
           clearTimeout(timer);
-          timer = setTimeout(() => {
-            observer.disconnect();
-            resolve();
-          }, quietMs);
+          // Check if we've exceeded max wait time
+          if (Date.now() - startTime >= maxWaitMs) {
+            finish();
+            return;
+          }
+          timer = setTimeout(finish, quietMs);
         };
+
         const observer = new MutationObserver(reset);
         observer.observe(document, {
           childList: true,
           subtree: true,
           attributes: true,
         });
+
+        // Set up the max wait timeout as a fallback
+        setTimeout(finish, maxWaitMs);
+
         reset();
       }),
-    { quietMs },
+    { quietMs, maxWaitMs },
   );
 }
 
@@ -226,10 +241,10 @@ export default async function takeElementScreenshot({
       `[${screenshotId}] Step 1 complete: Page loaded (success=${gotoResult}), elapsed: ${Date.now() - startTime}ms`,
     );
     awsLog(
-      `[${screenshotId}] Step 2: Waiting for DOM idle (timeout=${domIdle}ms)...`,
+      `[${screenshotId}] Step 2: Waiting for DOM idle (quietMs=500, maxWait=${domIdle}ms)...`,
     );
     const domIdleResult = await safeAwait('dom-idle', () =>
-      waitForDomIdle(page, domIdle),
+      waitForDomIdle(page, 500, domIdle),
     );
     awsLog(
       `[${screenshotId}] Step 2 complete: DOM idle (success=${domIdleResult}), elapsed: ${Date.now() - startTime}ms`,
@@ -262,7 +277,7 @@ export default async function takeElementScreenshot({
         page.waitForLoadState('networkidle', { timeout: pageIdle }),
       );
       await safeAwait('post-script-dom-idle', () =>
-        waitForDomIdle(page, domIdle),
+        waitForDomIdle(page, 500, domIdle),
       );
       awsLog(
         `[${screenshotId}] Step 4 complete: Instructions executed, elapsed: ${Date.now() - startTime}ms`,
@@ -390,10 +405,19 @@ export default async function takeElementScreenshot({
       `[${screenshotId}] Step 7 complete: Screenshot taken=${shotTaken}, elapsed: ${Date.now() - startTime}ms`,
     );
 
+    // Track first screenshot in case fullPage fallback is also uniform
+    let firstScreenshotWasUniform = false;
+    let firstScreenshotBackup: Buffer | null = null;
+
     if (shotTaken) {
       const isUniform = await isImageUniform(screenshotPath).catch(() => false);
       if (isUniform) {
         awsLog(`[${screenshotId}] Screenshot is uniform, trying full screen`);
+        // Backup the first screenshot - it's smaller than fullPage fallback
+        firstScreenshotWasUniform = true;
+        firstScreenshotBackup = await fs
+          .readFile(screenshotPath)
+          .catch((): null => null);
         shotTaken = false; // reset shotTaken to false, we will set it to true later if successful
       }
     }
@@ -415,6 +439,19 @@ export default async function takeElementScreenshot({
       awsLog(
         `[${screenshotId}] Step 7b complete: Full page screenshot taken=${shotTaken}, elapsed: ${Date.now() - startTime}ms`,
       );
+
+      // If fullPage is also uniform, restore the smaller first screenshot
+      if (shotTaken && firstScreenshotWasUniform && firstScreenshotBackup) {
+        const fullPageUniform = await isImageUniform(screenshotPath).catch(
+          () => false,
+        );
+        if (fullPageUniform) {
+          awsLog(
+            `[${screenshotId}] Full page also uniform, restoring smaller first screenshot`,
+          );
+          await fs.writeFile(screenshotPath, firstScreenshotBackup);
+        }
+      }
     }
 
     if (shotTaken)

package/analyzer-template/playwright/takeScreenshot.ts

@@ -9,7 +9,7 @@ import { uploadAssetFile } from './uploadAssets';
 import {
   updateFreshAnalysisStatus,
   updateFreshAnalysisStatusWithScenarios,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import invalidateCloudFrontPath from './invalidateCloudFrontPath';
 import { awsLog, awsLogDebugLevel, pushAnalysisError } from '~codeyam/utils';
 import { PlaywrightStep } from './executeInstuctions';
@@ -108,6 +108,7 @@ export default async function takeScreenshot({
 
     let isUniform = false;
     let screenshotTaken = false;
+    let firstUniformScreenshot: Buffer | null = null;
 
     for (const urlSuffix of dimensionRetries) {
       const result = await takeElementScreenshot({
@@ -118,6 +119,15 @@ export default async function takeScreenshot({
       isUniform = result.isUniform;
       screenshotTaken = result.screenshotTaken;
 
+      // Backup the first screenshot in case all retries are uniform
+      // (the first one without dimension overrides is typically smallest)
+      if (urlSuffix === '' && screenshotTaken && isUniform) {
+        const fs = await import('fs/promises');
+        firstUniformScreenshot = await fs
+          .readFile(screenshotPath)
+          .catch((): null => null);
+      }
+
       // Stop retrying if screenshot is good or if it wasn't taken (different problem)
       if (!isUniform || !screenshotTaken) {
         break;
@@ -133,6 +143,18 @@ export default async function takeScreenshot({
       });
     }
 
+    // If all retries produced uniform screenshots, restore the first one
+    // (it's the smallest since it has no dimension overrides)
+    if (isUniform && screenshotTaken && firstUniformScreenshot) {
+      awsLog(`CodeYam: All retries uniform, restoring first screenshot`, {
+        filePath: analysis.filePath,
+        entityName: analysis.entityName,
+        scenarioName: scenario.name,
+      });
+      const fs = await import('fs/promises');
+      await fs.writeFile(screenshotPath, firstUniformScreenshot);
+    }
+
     if (isUniform) {
       awsLog(`CodeYam: Screenshot is uniform`, {
         filePath: analysis.filePath,
@@ -172,12 +194,6 @@ export default async function takeScreenshot({
         const freshScenario = freshScenarios.find(
           (s) => s.name === scenario.name,
         );
-        freshScenario.metadata.screenshotPaths ||= [];
-        if (
-          !freshScenario.metadata.screenshotPaths.includes(screenshotFileName)
-        ) {
-          freshScenario.metadata.screenshotPaths.unshift(screenshotFileName);
-        }
 
         const statusScenario = freshStatus.scenarios.find(
           (s) => s.name === scenario.name,
@@ -186,14 +202,26 @@ export default async function takeScreenshot({
           statusScenario.screenshotFinishedAt = new Date().toISOString();
         }
 
-        if (!screenshotTaken) {
+        // Only add to screenshotPaths if the screenshot was actually taken
+        if (screenshotTaken) {
+          freshScenario.metadata.screenshotPaths ||= [];
+          if (
+            !freshScenario.metadata.screenshotPaths.includes(screenshotFileName)
+          ) {
+            freshScenario.metadata.screenshotPaths.unshift(screenshotFileName);
+          }
+        } else {
           freshScenario.metadata.noScreenshotSaved = true;
-          statusScenario.error = 'Screenshot not taken properly';
+          if (!statusScenario.error) {
+            statusScenario.error = 'Screenshot not taken properly';
+          }
         }
 
         if (isUniform) {
           freshScenario.metadata.isUniformScreenshot = true;
-          statusScenario.error = 'Screenshot is uniform';
+          if (!statusScenario.error) {
+            statusScenario.error = 'Screenshot is uniform';
+          }
         }
       },
     );

package/analyzer-template/playwright/updateBackgroundJob.ts

@@ -1,4 +1,4 @@
-import { updateBackgroundJobProgress } from '~codeyam/supabase';
+import { updateBackgroundJobProgress } from '~codeyam/database';
 import { BackgroundJob } from '~codeyam/types';
 
 export default async function updateBackgroundJob(

package/analyzer-template/playwright/waitForServer.ts

@@ -17,10 +17,18 @@ export interface WaitForServerResult {
   shutdown: boolean;
 }
 
+export interface WaitForServerOptions {
+  // In local CLI mode, treat connection failures as shutdown signal (server should be running)
+  // In cloud mode, connection failures are normal between work items
+  isLocalMode?: boolean;
+}
+
 export default async function waitForServer(
   url: string,
   shouldStop: () => boolean,
+  options?: WaitForServerOptions,
 ): Promise<WaitForServerResult> {
+  const isLocalMode = options?.isLocalMode ?? false;
   let retryDelay = INITIAL_RETRY_DELAY;
   const startTime = Date.now();
   let consecutiveConnectionFailures = 0;
@@ -49,14 +57,21 @@ export default async function waitForServer(
         return { shutdown: true };
       }
 
-      // Check if connection failed (server exited)
+      // Check if connection failed (server exited or not started yet)
       if (result.connectionFailed) {
-        consecutiveConnectionFailures++;
-        // If we've had many consecutive connection failures, the controller has likely exited
-        if (consecutiveConnectionFailures >= MAX_CONNECTION_FAILURES_FOR_SHUTDOWN) {
-          awsLog('CodeYam: Controller connection lost, treating as shutdown');
-          return { shutdown: true };
+        // Only treat connection failures as shutdown in local mode
+        // In cloud mode, connection failures are expected between work items
+        if (isLocalMode) {
+          consecutiveConnectionFailures++;
+          if (
+            consecutiveConnectionFailures >=
+            MAX_CONNECTION_FAILURES_FOR_SHUTDOWN
+          ) {
+            awsLog('CodeYam: Controller connection lost, treating as shutdown');
+            return { shutdown: true };
+          }
         }
+        // In cloud mode, just continue waiting (don't count failures)
       } else {
         consecutiveConnectionFailures = 0;
       }

package/analyzer-template/project/LazyFileStore.ts

@@ -5,7 +5,7 @@ import {
   loadEntities,
   saveFiles,
   generateSha,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 
 export interface LazyFileStoreOptions {
   project: Project;

package/analyzer-template/project/TESTING.md

@@ -0,0 +1,83 @@
+# Testing writeScenarioComponents
+
+This document describes how to test changes to `writeScenarioComponents.ts`.
+
+## 1. Run the specific test
+
+```bash
+npx jest background/src/lib/virtualized/project/__tests__/writeScenarioComponents.test.ts -t "test name here" --no-coverage
+```
+
+## 2. Run all tests in the file
+
+```bash
+npx jest background/src/lib/virtualized/project/__tests__/writeScenarioComponents.test.ts --no-coverage -w 2
+```
+
+## 3. Rebuild and reinstall the CLI
+
+The globally installed CLI must be reinstalled to pick up changes:
+
+```bash
+# Remove the globally installed CLI and rebuild
+rm -rf /Users/jaredcosulich/.nvm/versions/node/v20.16.0/lib/node_modules/@codeyam/codeyam-cli && pnpm cli
+```
+
+## 4. Restart the server in the client project
+
+After rebuilding the CLI, restart the codeyam server in the client project:
+
+```bash
+cd /Users/jaredcosulich/workspace/codeyam/formbricks && codeyam stop & codeyam
+```
+
+## 5. Force analyzer update and run debug
+
+The analyzer is cached in `/tmp/codeyam/local-dev/{project-slug}/codeyam/analyzer`.
+To force it to use the new code:
+
+```bash
+# Remove cached analyzer
+rm -rf /tmp/codeyam/local-dev/formbricks-formbricks/codeyam/analyzer
+
+# Run debug (from the project directory)
+cd /Users/jaredcosulich/workspace/codeyam/formbricks && codeyam debug <analysis-id>
+```
+
+## 6. Verify the fix
+
+```bash
+# Check version in logs (should match version in writeScenarioComponents.ts)
+grep "CodeYam \[v" /tmp/codeyam/local-dev/formbricks-formbricks/codeyam/log.txt | tail -3
+
+# Check generated files
+head -30 /tmp/codeyam/local-dev/formbricks-formbricks/project/path/to/generated/file.tsx
+
+# List data scenario files
+ls -la /tmp/codeyam/local-dev/formbricks-formbricks/project/apps/web/lib/*_data_Default_Scenario*
+```
+
+## 7. Start dev server and test in browser
+
+```bash
+cd /tmp/codeyam/local-dev/formbricks-formbricks/project/apps/web && pnpm next dev --turbopack -p 3112
+```
+
+Test the actual simulation page (not just codeyam-sample):
+
+```bash
+# Format: /static/{project-slug}/{analysis-id}/{entity-name}/{scenario-name}/default
+curl http://localhost:3112/static/formbricks-formbricks/7925cfe4-7789-470d-bddc-2a85e48ef356/SurveysPage/Default_Scenario/default
+```
+
+Visit in browser: http://localhost:3112/static/formbricks-formbricks/7925cfe4-7789-470d-bddc-2a85e48ef356/SurveysPage/Default_Scenario/default
+
+## Version Tracking
+
+The version is defined in `writeScenarioComponents.ts`:
+
+```typescript
+const WRITE_SCENARIO_COMPONENTS_VERSION = '2.5.13-strip-use-server-directive';
+```
+
+Increment this when making changes to help track which version is running.

package/analyzer-template/project/analyzeBaselineCommit.ts

@@ -6,7 +6,7 @@ import {
   upsertAnalysisBranches,
   updateCommitMetadata,
   updateProjectMetadata,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import { awsLog } from '~codeyam/utils';
 import { LazyFileStore } from './LazyFileStore';
 

package/analyzer-template/project/analyzeBranchCommit.ts

@@ -12,7 +12,7 @@ import {
   loadFile,
   updateCommitMetadata,
   updateProjectMetadata,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import checkApprovedPath from './checkApprovedPath';
 import { LazyFileStore } from './LazyFileStore';
 

package/analyzer-template/project/analyzeFileEntities.ts

@@ -12,7 +12,7 @@ import {
   loadEntity,
   updateCommitMetadata,
   upsertCommits,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import { analyzeFileMocks } from './mocks/analyzeFileMock';
 import { LazyFileStore } from './LazyFileStore';
 

package/analyzer-template/project/analyzeRegularCommit.ts

@@ -16,7 +16,7 @@ import {
   updateCommitMetadata,
   upsertAnalysesWithScenarios,
   upsertAnalysisBranches,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import { LazyFileStore } from './LazyFileStore';
 

package/analyzer-template/project/captureLibraryFunctionDirect.ts

@@ -4,7 +4,7 @@ import {
   updateFreshAnalysisStatus,
   updateFreshAnalysisStatusWithScenarios,
   loadAnalysis,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import executeLibraryFunctionDirect from './executeLibraryFunctionDirect';
 
 /**