@codeyam/codeyam-cli 0.1.0-bleeding-edge.8afd3ee

package/analyzer-template/project/writeUniversalMocks.ts

@@ -0,0 +1,171 @@
+import { Analysis, ProjectFramework } from '~codeyam/types';
+import writeFile from '../common/writeFile';
+import path from 'path';
+import { ProjectAnalyzer } from '~codeyam/analyze';
+import getAppPath from './getAppPath';
+import { getRelativePath } from '~codeyam/generate';
+import {
+  safeFileName,
+  CODEYAM_MOCKS_DIRNAME,
+  PROJECT_RELATIVE_PATH,
+} from '~codeyam/utils';
+import { LazyFileStore } from './LazyFileStore';
+
+interface WriteUniversalMocksArgs {
+  analysis: Analysis;
+  projectAnalyzer: ProjectAnalyzer;
+  framework: ProjectFramework;
+  rootPath?: string;
+  fileStore?: LazyFileStore;
+}
+
+export default async function writeUniversalMocks({
+  analysis,
+  projectAnalyzer,
+  framework,
+  rootPath,
+  fileStore,
+}: WriteUniversalMocksArgs) {
+  const { project } = projectAnalyzer;
+
+  for (const universalMock of project.metadata?.universalMocks ?? []) {
+    console.log(
+      'CodeYam: Initializing universal mock',
+      universalMock.filePath,
+      universalMock.entityName,
+    );
+
+    try {
+      let mockPath = '';
+      let mockContent = '';
+
+      // Use fileStore for O(1) lookup when available
+      let file = fileStore
+        ? fileStore.getByPath(universalMock.filePath)
+        : project.files.find((f) => f.path === universalMock.filePath);
+
+      // Ensure content is loaded for the file
+      if (
+        file &&
+        fileStore &&
+        !fileStore.isContentLoaded(universalMock.filePath)
+      ) {
+        file = await fileStore.ensureContent(universalMock.filePath);
+      }
+
+      if (file) {
+        mockContent = file.content;
+        mockPath = path.join(
+          rootPath ?? PROJECT_RELATIVE_PATH,
+          universalMock.filePath,
+        );
+        const fileAnalyzer = projectAnalyzer.getFileAnalyzer(file);
+
+        // Use getAllDeclaredEntityNodes to avoid incorrectly matching identifier references
+        if (
+          fileAnalyzer.getAllDeclaredEntityNodes()[universalMock.entityName]
+        ) {
+          const entityCode = fileAnalyzer.getDeclaredEntityCode(
+            universalMock.entityName,
+          );
+          mockContent = file.content.replace(entityCode, universalMock.content);
+        }
+      } else {
+        const appPath = getAppPath(
+          project,
+          framework,
+          false,
+          universalMock.filePath,
+        );
+        // For node_modules, strip extension before creating mock path
+        // This handles both old format (with extension) and new format (without)
+        const filePathForMockFile = universalMock.nodeModule
+          ? universalMock.filePath.replace(/\.(ts|tsx|js|jsx)$/, '')
+          : universalMock.filePath;
+        mockPath = path.join(
+          appPath,
+          CODEYAM_MOCKS_DIRNAME,
+          safeFileName(filePathForMockFile) + '.tsx',
+        );
+        const exportName =
+          universalMock.entityName === 'default'
+            ? 'default'
+            : `{ ${universalMock.entityName} }`;
+        if (exportName === 'default') {
+          mockContent = `export default ${universalMock.content}`;
+        } else {
+          mockContent = `${universalMock.content}\n\nexport { ${universalMock.entityName} };`;
+        }
+      }
+
+      const mockDataImport = new RegExp(
+        `import { scenarios } from ['"]\.+\/[Mm]ockData(.js)*['"];`,
+        'g',
+      );
+      if (mockContent.match(mockDataImport)) {
+        const appPath = getAppPath(
+          projectAnalyzer.project,
+          framework,
+          false,
+          universalMock.filePath,
+        );
+        const mocksDir = `${appPath}/${CODEYAM_MOCKS_DIRNAME}/`;
+        const relativeMocksDir = getRelativePath(mockPath, mocksDir);
+        const mockDataPath = `${relativeMocksDir}/${analysis.id}/MockData`;
+        const importStatement = `import { scenarios } from "${mockDataPath}";`;
+        mockContent = mockContent.replace(mockDataImport, '');
+        mockContent = `${importStatement}\n${mockContent}`;
+      }
+
+      if (file) {
+        // File exists in project - overwrite it with the mock content
+        file.content = mockContent;
+      } else {
+        // File doesn't exist in project (e.g., node_module or external file)
+        // We need to rewrite all imports to point to our mock file
+
+        // For node_modules, strip any extension to match import statements
+        // This handles both old format (with extension) and new format (without)
+        const filePathForMatching = universalMock.nodeModule
+          ? universalMock.filePath.replace(/\.(ts|tsx|js|jsx)$/, '')
+          : universalMock.filePath;
+
+        const importRegEx = new RegExp(
+          `['"]${filePathForMatching.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}['"]`,
+          'g',
+        );
+        const allSourceFiles = projectAnalyzer.getAllSourceFiles();
+        for (const sourceFile of allSourceFiles) {
+          if (importRegEx.test(sourceFile.content)) {
+            const relativeMockPath = getRelativePath(sourceFile.path, mockPath);
+            const newContent = sourceFile.content.replace(
+              importRegEx,
+              `'${relativeMockPath}'`,
+            );
+            console.log(
+              'CodeYam: Rewriting import in',
+              sourceFile.path,
+              'from',
+              universalMock.filePath,
+              'to',
+              relativeMockPath,
+            );
+            await writeFile(sourceFile.absolutePath, newContent);
+          }
+        }
+      }
+
+      // console.log('CodeYam: Writing universal mock', mockPath, mockContent);
+      await writeFile(mockPath, mockContent);
+    } catch (e) {
+      console.error(
+        'CodeYam Error: Writing universal mock',
+        analysis.id,
+        analysis.entityName,
+        universalMock,
+        e,
+      );
+      throw e;
+    }
+  }
+}

package/analyzer-template/scripts/comboWorkerLoop.cjs

@@ -0,0 +1,474 @@
+#!/usr/bin/env node
+const { spawn } = require('child_process');
+const path = require('path');
+const fs = require('fs').promises;
+const {
+  SQSClient,
+  ReceiveMessageCommand,
+  DeleteMessageCommand,
+  ChangeMessageVisibilityCommand,
+  SendMessageCommand,
+} = require('@aws-sdk/client-sqs');
+
+const FIXED_PROJECT_PATH = '/usr/src/app/project';
+const FIXED_TEMP_PATH = '/usr/src/app/tmp';
+
+/**
+ * This script manages the continuous Playwright capture process and handles
+ * work orders from an SQS queue to run project simulations. It requires:
+ *   $PROJECT_SLUG
+ *   $FRAMEWORK
+ *   $PACKAGE_MANAGER
+ *   $SQS_QUEUE_URL (work queue)
+ *   $SQS_CONTROL_QUEUE_URL (control queue)
+ */
+
+// Track child processes for cleanup
+const childProcesses = new Set();
+
+const isProcessAlive = (pid) => {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return false;
+  }
+};
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+let isCleaningUp = false;
+const cleanup = async () => {
+  if (isCleaningUp) return;
+  isCleaningUp = true;
+
+  console.log('Cleaning up child processes...');
+  const pids = Array.from(childProcesses).map((proc) => proc.pid);
+
+  // First attempt: SIGTERM
+  for (const proc of childProcesses) {
+    try {
+      process.kill(proc.pid, 'SIGTERM');
+    } catch (err) {
+      // ignore errors, we'll check status in final sweep
+    }
+  }
+
+  await sleep(20000); // give TERM a significant amount of time to take effect
+
+  // Second attempt: SIGKILL for survivors
+  for (const proc of childProcesses) {
+    if (isProcessAlive(proc.pid)) {
+      try {
+        process.kill(proc.pid, 'SIGKILL');
+        console.log(`Killed [SIGKILL] process ${proc.pid}`);
+      } catch (err) {
+        // ignore errors, we'll check status in final sweep
+      }
+    } else {
+      console.log(`Terminated [SIGTERM] process ${proc.pid}`);
+    }
+  }
+
+  await sleep(3000); // KILL should be pretty much immediaste
+
+  for (const pid of pids) {
+    if (isProcessAlive(pid)) {
+      console.error(`Process ${pid} is still running after cleanup`);
+    }
+  }
+};
+
+// Setup cleanup handlers
+const handleSignal = async (signal) => {
+  console.log(`Received ${signal}`);
+  await cleanup();
+  process.exit(0);
+};
+
+process.on('SIGTERM', () => handleSignal('SIGTERM'));
+process.on('SIGINT', () => handleSignal('SIGINT'));
+
+const validateEnvironment = () => {
+  const requiredVariables = [
+    'PROJECT_SLUG',
+    'FRAMEWORK',
+    'PACKAGE_MANAGER',
+    'SQS_QUEUE_URL',
+    'SQS_CONTROL_QUEUE_URL',
+  ];
+
+  const missingVariables = requiredVariables.filter(
+    (variable) => !process.env[variable],
+  );
+
+  if (missingVariables.length > 0) {
+    throw new Error(
+      `Missing environment variables: ${missingVariables.join(', ')}`,
+    );
+  }
+};
+
+const validateFiles = async () => {
+  const requiredFiles = ['dist/playwright/capture.js', 'dist/project/start.js'];
+
+  for (const file of requiredFiles) {
+    try {
+      await fs.access(file);
+    } catch (error) {
+      throw new Error(`Required file '${file}' is missing.`);
+    }
+  }
+};
+
+const createLinePrefixer = (stream, processName, outputStream) => {
+  let buffer = '';
+
+  stream.on('data', (chunk) => {
+    buffer += chunk.toString();
+    const lines = buffer.split('\n');
+
+    // All but the last element are complete lines
+    const completeLines = lines.slice(0, -1);
+    buffer = lines[lines.length - 1]; // Keep incomplete line
+
+    // Output complete lines with prefix
+    completeLines.forEach((line) => {
+      outputStream.write(`[${processName}] ${line}\n`);
+    });
+  });
+
+  stream.on('end', () => {
+    // Flush remaining buffer if any
+    if (buffer) {
+      outputStream.write(`[${processName}] ${buffer}\n`);
+    }
+  });
+};
+
+const spawnProcess = (command, args, processName, env = {}) => {
+  return new Promise((resolve, reject) => {
+    const childProcess = spawn(command, args, {
+      shell: true,
+      stdio: ['inherit', 'pipe', 'pipe'], // Inherit stdin, pipe stdout/stderr
+      env: { ...process.env, ...env },
+    });
+
+    console.log(`Started ${processName} process with PID ${childProcess.pid}`);
+    childProcesses.add(childProcess);
+
+    // Set up prefixed output for stdout and stderr
+    createLinePrefixer(childProcess.stdout, processName, process.stdout);
+    createLinePrefixer(childProcess.stderr, processName, process.stderr);
+
+    childProcess.on('close', (code, signal) => {
+      console.log(
+        `${processName} process exited with code ${code}, signal ${signal}`,
+      );
+      childProcesses.delete(childProcess);
+      if (code === 0) {
+        resolve();
+      } else if (code === null && signal && isCleaningUp) {
+        // Process was killed by signal during cleanup - this is expected, not an error
+        console.log(
+          `${processName} process was terminated by ${signal} during cleanup`,
+        );
+        resolve();
+      } else {
+        reject(
+          new Error(
+            `${processName} process exited with code ${code}, signal ${signal}`,
+          ),
+        );
+      }
+    });
+
+    childProcess.on('error', (error) => {
+      console.error(`Error starting ${processName} process:`, error);
+      childProcesses.delete(childProcess);
+      reject(error);
+    });
+  });
+};
+
+const startPlaywrightCapture = () => {
+  console.log('Starting Playwright capture service...');
+  return spawnProcess(
+    'node',
+    [
+      '--enable-source-maps',
+      './dist/playwright/capture.js',
+      '--slug',
+      process.env.PROJECT_SLUG,
+    ],
+    'playwright',
+  );
+};
+
+const runProjectSimulation = async (analysisIds) => {
+  try {
+    // server now uses rsync, so project directory is guaranteed clean, just ensure it exists
+    await fs.mkdir(FIXED_PROJECT_PATH, { recursive: true });
+
+    const analysisIdString = analysisIds.join(',');
+    console.log(
+      `Starting project simulation for analysis IDs: ${analysisIdString}`,
+    );
+
+    return spawnProcess(
+      'node',
+      [
+        '--enable-source-maps',
+        './dist/project/start.js',
+        '--packageManager',
+        process.env.PACKAGE_MANAGER,
+        '--absoluteProjectRootPath',
+        FIXED_PROJECT_PATH,
+        '--absoluteTempPath',
+        FIXED_TEMP_PATH,
+        '--port',
+        '3000',
+        '--framework',
+        process.env.FRAMEWORK,
+      ],
+      'project',
+      { ANALYSIS_IDS: analysisIdString, READY_TO_BE_CAPTURED: 'true' },
+    );
+  } catch (error) {
+    throw new Error(`Failed to run project simulation: ${error.message}`);
+  }
+};
+
+const isQueueNotExistsError = (error) => {
+  return (
+    error.name === 'QueueDoesNotExist' ||
+    error.code === 'AWS.SimpleQueueService.NonExistentQueue'
+  );
+};
+
+const extendMessageVisibility = async (sqs, message, queueUrl) => {
+  try {
+    await sqs.send(
+      new ChangeMessageVisibilityCommand({
+        QueueUrl: queueUrl,
+        ReceiptHandle: message.ReceiptHandle,
+        VisibilityTimeout: 1200, // 20 minutes
+      }),
+    );
+    console.log('Extended message visibility timeout');
+  } catch (error) {
+    if (isQueueNotExistsError(error)) {
+      console.log(
+        'Queue was deleted during visibility extension, completing work before shutting down.',
+      );
+    } else {
+      // Log the error but don't exit - message may have been processed already
+      // or receipt handle became invalid for other reasons
+      console.warn(
+        'Failed to extend message visibility (continuing work):',
+        error.message,
+      );
+    }
+  }
+};
+
+const processWorkMessage = async (sqs, message, queueUrl, controlQueueUrl) => {
+  try {
+    const body = JSON.parse(message.Body);
+
+    if (
+      body.type !== 'work_items' ||
+      !body.analysisIds ||
+      !Array.isArray(body.analysisIds)
+    ) {
+      throw new Error('Invalid message format');
+    }
+
+    const analysisIdString = body.analysisIds.join(',');
+    console.log(`Processing analysis IDs: ${analysisIdString}`);
+
+    // Immediately extend message visibility upon receipt
+    await extendMessageVisibility(sqs, message, queueUrl);
+
+    // Set up visibility extension interval with proper async handling
+    const visibilityInterval = setInterval(
+      async () => {
+        try {
+          await extendMessageVisibility(sqs, message, queueUrl);
+        } catch (error) {
+          console.warn(
+            'Failed to extend visibility in interval:',
+            error.message,
+          );
+        }
+      },
+      300000, // every 5 minutes
+    );
+
+    // Set up safety timeout (5 + 5 * batch size minutes)
+    const timeoutMinutes = 5 + 5 * body.analysisIds.length;
+    const timeoutMs = timeoutMinutes * 60 * 1000;
+    let timeoutHandle;
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutHandle = setTimeout(() => {
+        reject(new Error(`Processing timeout after ${timeoutMinutes} minutes`));
+        timeoutHandle = null;
+      }, timeoutMs);
+    });
+
+    try {
+      // Race between the actual work and the timeout
+      await Promise.race([
+        runProjectSimulation(body.analysisIds),
+        timeoutPromise,
+      ]);
+
+      console.log(`Successfully processed analysis IDs: ${analysisIdString}`);
+    } finally {
+      if (visibilityInterval) clearInterval(visibilityInterval);
+      if (timeoutHandle) clearTimeout(timeoutHandle);
+
+      // Send batch completion notification to control queue
+      try {
+        await sqs.send(
+          new SendMessageCommand({
+            QueueUrl: controlQueueUrl,
+            MessageBody: JSON.stringify({
+              type: 'batch_completed',
+              analysisIds: body.analysisIds,
+            }),
+          }),
+        );
+        console.log(
+          `Sent batch completion notification for ${body.analysisIds.length} analyses`,
+        );
+      } catch (error) {
+        // Log error but don't fail - orchestrator will eventually timeout/abandon
+        if (!isQueueNotExistsError(error)) {
+          console.warn(
+            'Failed to send batch completion notification:',
+            error.message,
+          );
+        }
+      }
+
+      // Whether the work succeeded or failed, mark the work item complete
+      try {
+        await sqs.send(
+          new DeleteMessageCommand({
+            QueueUrl: queueUrl,
+            ReceiptHandle: message.ReceiptHandle,
+          }),
+        );
+      } catch (error) {
+        // Silently ignore queue-not-exists errors when deleting messages
+        if (!isQueueNotExistsError(error)) {
+          console.error('Error deleting message:', error);
+        }
+      }
+    }
+  } catch (error) {
+    console.error('Error processing message:', error);
+  }
+};
+
+const checkForWorkMessages = async (sqs, workQueueUrl, controlQueueUrl) => {
+  try {
+    const response = await sqs.send(
+      new ReceiveMessageCommand({
+        QueueUrl: workQueueUrl,
+        MaxNumberOfMessages: 1,
+        WaitTimeSeconds: 20,
+        VisibilityTimeout: 1200, // 20 minutes initial timeout
+      }),
+    );
+
+    if (response.Messages && response.Messages.length > 0) {
+      const message = response.Messages[0];
+      await processWorkMessage(sqs, message, workQueueUrl, controlQueueUrl);
+    } else {
+      console.log('No work messages received, waiting for more...');
+    }
+    return response;
+  } catch (error) {
+    if (isQueueNotExistsError(error)) {
+      console.log('Work queue has been deleted, initiating graceful shutdown');
+      await cleanup();
+      process.exit(0);
+    }
+    console.error('Error checking work messages:', error);
+    // Wait a bit before retrying for other types of errors
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+  }
+};
+
+const IDLE_TIMEOUT = 30 * 60 * 1000; // self-terminate after long inactivity
+
+const pollSQSMessages = async (sqs, workQueueUrl, controlQueueUrl) => {
+  let shutdownReceived = false;
+  let lastMessageTime = Date.now();
+
+  while (!shutdownReceived) {
+    const idleTime = Date.now() - lastMessageTime;
+    if (idleTime > IDLE_TIMEOUT) {
+      console.log(
+        `CodeYam Error: No messages received for ${IDLE_TIMEOUT / 1000} seconds, self-terminating`,
+      );
+      break;
+    }
+
+    // Only check work queue if no shutdown signal received
+    const response = await checkForWorkMessages(
+      sqs,
+      workQueueUrl,
+      controlQueueUrl,
+    );
+    // Update lastMessageTime if we received any messages
+    if (response?.Messages?.length > 0) {
+      lastMessageTime = Date.now();
+    }
+  }
+
+  console.log('Gracefully shutting down after completing work');
+  await cleanup();
+};
+
+const handleSQSPolling = async (sqs, workQueueUrl, controlQueueUrl) => {
+  console.log('Starting SQS message processing...');
+
+  try {
+    // Run Playwright capture and SQS polling concurrently
+    await Promise.race([
+      startPlaywrightCapture(),
+      pollSQSMessages(sqs, workQueueUrl, controlQueueUrl),
+    ]);
+    console.log('Exiting main loop');
+  } catch (error) {
+    console.error('Error in handleSQSPolling:', error);
+    // Only cleanup if we're not already cleaning up to avoid double cleanup
+    if (!isCleaningUp) {
+      await cleanup();
+    }
+    process.exit(1);
+  }
+};
+
+const main = async () => {
+  try {
+    validateEnvironment();
+    await validateFiles();
+
+    const sqs = new SQSClient({});
+    const workQueueUrl = process.env.SQS_QUEUE_URL;
+    const controlQueueUrl = process.env.SQS_CONTROL_QUEUE_URL;
+
+    await handleSQSPolling(sqs, workQueueUrl, controlQueueUrl);
+    process.exit(0);
+  } catch (error) {
+    console.error('Fatal error:', error);
+    await cleanup();
+    process.exit(1);
+  }
+};
+
+main();