@codeyam/codeyam-cli 0.1.0-staging.b8a55ba → 0.1.0-staging.c90f8c9

package/analyzer-template/packages/aws/src/lib/s3/uploadFileToS3.ts

@@ -24,11 +24,18 @@ export default async function uploadFileToS3({
   });
 
   try {
+    let lastLoggedMB = 0;
     upload.on('httpUploadProgress', (progress) => {
-      console.log(progress);
+      const loadedMB = Math.floor((progress.loaded ?? 0) / (1024 * 1024));
+      // Log every 10MB to avoid spam
+      if (loadedMB >= lastLoggedMB + 10) {
+        lastLoggedMB = loadedMB;
+        console.log(`Uploading ${key}... ${loadedMB}MB`);
+      }
     });
 
     await upload.done();
+    console.log(`Uploaded ${key}`);
   } catch (error) {
     console.log('Error. File not uploaded.', error);
     throw error;

package/analyzer-template/packages/database/src/lib/kysely/tables/debugReportsTable.ts

@@ -6,7 +6,7 @@ import { schemaField, defaultNow } from '../schemaHelpers';
 /**
  * Metadata for a debug bundle upload.
  * Note: "bundle" refers to the tarball upload (base + delta).
- * The markdown "debug report" from /debug-codeyam is stored in feedback.debugReport.
+ * The markdown "debug report" from /codeyam:diagnose is stored in feedback.debugReport.
  */
 interface DebugBundleMetadata {
   timestamp: string;
@@ -25,6 +25,22 @@ interface DebugBundleMetadata {
     platform: string;
     arch: string;
   };
+  feedback?: {
+    source: string;
+    issueType?: string;
+    description?: string;
+    email?: string;
+    entitySha?: string;
+    scenarioId?: string;
+    analysisId?: string;
+    currentUrl?: string;
+    recentActivity?: string[];
+    debugReport?: string;
+    entityName?: string;
+    entityType?: string;
+    scenarioName?: string;
+    errorMessage?: string;
+  };
 }
 
 // Table name remains 'debug_reports' in the database for backwards compatibility

package/analyzer-template/packages/database/src/lib/loadAnalyses.ts

@@ -27,6 +27,7 @@ function buildAnalysesQuery({
   commitIds,
   branchCommitSha,
   limit,
+  excludeMetadata,
 }: LoadAnalysesArgs & {
   ids?: string[];
   fileIds?: string[];
@@ -37,7 +38,32 @@ function buildAnalysesQuery({
   const { jsonObjectFrom, jsonArrayFrom } = getJsonHelper();
 
   // Create CTE for filtering analyses
-  let filteredAnalysisCTE = db.selectFrom('analyses').selectAll('analyses');
+  // When excludeMetadata is true, select specific columns to avoid loading 64MB+ of metadata
+  let filteredAnalysisCTE = excludeMetadata
+    ? db
+        .selectFrom('analyses')
+        .select([
+          'analyses.id',
+          'analyses.project_id',
+          'analyses.file_id',
+          'analyses.commit_id',
+          'analyses.entity_sha',
+          'analyses.entity_name',
+          'analyses.entity_type',
+          'analyses.file_path',
+          'analyses.status',
+          'analyses.created_at',
+          'analyses.updated_at',
+          'analyses.tree_sha',
+          'analyses.analyzed_tree_sha',
+          'analyses.dependency_analyzed_tree_sha',
+          'analyses.previous_analysis_id',
+          'analyses.branch_commit_sha',
+          'analyses.indirect',
+          'analyses.committed_at',
+          'analyses.completed_at',
+        ])
+    : db.selectFrom('analyses').selectAll('analyses');
 
   // Apply filters
   if (projectId) {
@@ -96,6 +122,35 @@ function buildAnalysesQuery({
   }
 
   // Main query with JSON aggregations referencing the CTE
+  // When excludeMetadata is true, skip the entity subquery (saves loading entity metadata again)
+  if (excludeMetadata) {
+    return db
+      .with('filtered_analyses', () => filteredAnalysisCTE)
+      .selectFrom('filtered_analyses')
+      .selectAll('filtered_analyses')
+      .select((eb: ExpressionBuilder<any, any>) => [
+        // Skip entity subquery - not needed for list views and loads 41MB+ of entity metadata
+
+        jsonArrayFrom(
+          eb
+            .selectFrom('scenarios')
+            .select(aliasedTableColumns('scenarios', ScenariosTableColumns))
+            .whereRef('scenarios.analysis_id', '=', 'filtered_analyses.id'),
+        ).as('scenarios'),
+
+        jsonArrayFrom(
+          eb
+            .selectFrom('analysis_branches')
+            .select(['id', 'branch_id'])
+            .whereRef(
+              'analysis_branches.analysis_id',
+              '=',
+              'filtered_analyses.id',
+            ),
+        ).as('analysis_branches'),
+      ]);
+  }
+
   return db
     .with('filtered_analyses', () => filteredAnalysisCTE)
     .selectFrom('filtered_analyses')
@@ -138,6 +193,8 @@ export interface LoadAnalysesArgs {
   entityShas?: string[];
   branchCommitSha?: string;
   limit?: number;
+  /** Exclude large metadata columns for faster list queries */
+  excludeMetadata?: boolean;
 }
 
 export default async function loadAnalyses(

package/analyzer-template/packages/database/src/lib/loadCommits.ts

@@ -19,6 +19,8 @@ interface LoadCommitsArgs {
   shas?: string[];
   fileNames?: string[];
   limit?: number;
+  /** Skip loading related data (analyses, entities, branches) for faster queries */
+  skipRelations?: boolean;
 }
 
 async function loadBranchesForCommits(
@@ -137,6 +139,7 @@ export default async function loadCommits({
   shas,
   fileNames,
   limit = 10,
+  skipRelations = false,
 }: LoadCommitsArgs): Promise<Commit[]> {
   if (!projectId && !ids) {
     throw new Error('Must provide projectId or ids');
@@ -208,6 +211,19 @@ export default async function loadCommits({
       return [];
     }
 
+    // When skipRelations is true, just return commits without loading related data
+    // This is much faster for queries that only need commit metadata
+    if (skipRelations) {
+      const dbCommits: DbCommit[] = commits.map((commit: any) => ({
+        ...commit,
+        branch: undefined,
+        mergedBranch: undefined,
+        analyses: [],
+        entities: [],
+      }));
+      return dbCommits.map(dbToCommit);
+    }
+
     const commitIds = commits.map((c: any) => c.id);
 
     // Load all relations in parallel

package/analyzer-template/packages/database/src/lib/loadEntities.ts

@@ -9,6 +9,8 @@ export interface LoadEntitiesArgs {
   filePaths?: string[];
   names?: string[];
   shas?: string[];
+  /** Exclude large metadata column for faster list queries */
+  excludeMetadata?: boolean;
 }
 
 export default async function loadEntities({
@@ -18,6 +20,7 @@ export default async function loadEntities({
   filePaths,
   names,
   shas,
+  excludeMetadata,
 }: LoadEntitiesArgs): Promise<Entity[] | null> {
   if (
     (fileIds && fileIds.length == 0) ||
@@ -39,6 +42,7 @@ export default async function loadEntities({
         filePaths,
         names,
         shas: chunk,
+        excludeMetadata,
       });
 
       if (chunkEntities) {
@@ -51,9 +55,28 @@ export default async function loadEntities({
   const db = getDatabase();
 
   try {
-    const query = db
-      .selectFrom('entities')
-      .selectAll('entities')
+    // When excludeMetadata is true, select all columns except the large metadata column
+    // This dramatically speeds up list queries (41MB+ of JSON avoided)
+    const baseQuery = excludeMetadata
+      ? db
+          .selectFrom('entities')
+          .select([
+            'entities.project_id',
+            'entities.file_id',
+            'entities.commit_id',
+            'entities.name',
+            'entities.sha',
+            'entities.entity_type',
+            'entities.file_path',
+            'entities.description',
+            'entities.documentation',
+            'entities.quality',
+            'entities.created_at',
+            'entities.updated_at',
+          ])
+      : db.selectFrom('entities').selectAll('entities');
+
+    const query = baseQuery
       .$if(!!branchId, (qb) =>
         qb
           .innerJoin(

package/analyzer-template/packages/database/src/lib/loadReadyToBeCapturedAnalyses.ts

@@ -40,15 +40,19 @@ export default async function loadReadyToBeCapturedAnalyses(
           .whereRef('scenarios.analysis_id', '=', 'analyses.id'),
       ).as('scenarios'),
     ])
+    // Cross-database compatible: use -> (not ->>) for boolean JSON fields
+    // PostgreSQL: jsonb `true` auto-casts to string 'true' for comparison
+    // SQLite: -> returns text 'true' which equals string 'true'
+    // (Note: ->> returns integer 1 in SQLite for booleans, breaking string comparisons)
     .where(
-      // Use ->> to extract as text for proper comparison (-> returns JSONB which doesn't compare to string)
-      sql<string>` ${sql.ref('analyses.status')} ->> 'readyToBeCaptured'`,
+      sql<string>`${sql.ref('analyses.status')} -> 'readyToBeCaptured'`,
       '=',
       'true',
     )
     // Exclude entities that are not exported - they cannot be captured because they can't be imported
+    // Use -> operator for cross-database boolean comparison
     .where(
-      sql<boolean>`(${sql.ref('entities.metadata')} ->> 'notExported') IS NULL OR (${sql.ref('entities.metadata')} ->> 'notExported') != 'true'`,
+      sql<boolean>`(${sql.ref('entities.metadata')} -> 'notExported') IS NULL OR (${sql.ref('entities.metadata')} -> 'notExported') != 'true'`,
     );
 
   if (projectId) {

package/analyzer-template/packages/generate/index.ts

@@ -1,6 +1,8 @@
 export { default as getComponentImportStatements } from './src/lib/getComponentImportStatements';
 export { default as componentScenarioPageRemix } from './src/lib/componentScenarioPage/componentScenarioPageRemix';
 export { default as componentScenarioPageNext } from './src/lib/componentScenarioPage/componentScenarioPageNext';
+export { default as generateScenarioServerComponent } from './src/lib/componentScenarioPage/generateScenarioServerComponent';
+export { default as generateScenarioClientWrapper } from './src/lib/componentScenarioPage/generateScenarioClientWrapper';
 export { default as getComponentScenarioPath } from './src/lib/getComponentScenarioPath';
 export { default as libDemoComponent } from './src/lib/libDemoComponent';
 export { default as directExecutionScript } from './src/lib/directExecutionScript';
@@ -9,4 +11,5 @@ export { default as safeFolder } from './src/lib/safeFolder';
 export { default as deepMerge } from './src/lib/deepMerge';
 export { default as simpleRootRemix } from './src/lib/simpleRootRemix';
 export { default as mergeRootRemix } from './src/lib/mergeRootRemix';
+export { default as scenarioComponentForServer } from './src/lib/scenarioComponentForServer';
 export { PLACEHOLDER_IMAGE_SRC } from './src/lib/constants';

package/analyzer-template/packages/generate/src/lib/componentScenarioPage/componentScenarioPageNext.ts

@@ -6,6 +6,7 @@ import escapeQuotes from '../escapeQuotes';
 import getIFrameMessageListenerCode from './getIFrameMessageListenerCode';
 import getImageReplacementCode from './getImageReplacementCode';
 import getNextJsErrorClosingCode from './getNextJsErrorClosingCode';
+import generateScenarioServerComponent from './generateScenarioServerComponent';
 
 const VERSION = '0.5.1';
 
@@ -28,6 +29,21 @@ export default function componentScenarioPageNext({
 }) {
   const fileName = file.name.split('.')[0];
 
+  // For async components (Server Components), generate a Server Component page
+  // that uses a separate client wrapper for interactive features
+  if (entity.metadata?.isAsync) {
+    return generateScenarioServerComponent({
+      file,
+      entity,
+      analysis,
+      currentPath,
+      appPath,
+      scenario,
+      relativeMocksDir,
+    });
+  }
+
+  // For non-async components, use the existing client component approach
   return `/* eslint-disable */
   "use client";
   import {
@@ -58,7 +74,7 @@ ${getComponentImportStatement({
 ${getImageReplacementCode()}
 
 function Scenario({ name, width, height, children }: { name: string, width: string, height: string, children: ReactNode }) {
-  
+
   useEffect(() => {
     if (typeof window === 'undefined') return;
 

package/analyzer-template/packages/generate/src/lib/componentScenarioPage/generateScenarioClientWrapper.ts

@@ -0,0 +1,193 @@
+import type { Scenario } from '~codeyam/types';
+import { safeFileName } from '~codeyam/utils';
+import getImageReplacementCode from './getImageReplacementCode';
+import getNextJsErrorClosingCode from './getNextJsErrorClosingCode';
+
+/**
+ * Generates the ScenarioClientWrapper component code.
+ * This is a Client Component that handles all the interactive features
+ * (iframe messaging, console interception, error closing, etc.)
+ * while allowing the actual page to be a Server Component.
+ */
+export default function generateScenarioClientWrapper(
+  scenario: Scenario,
+): string {
+  const safeScenarioName = safeFileName(scenario.name);
+
+  return `"use client";
+/* eslint-disable */
+// ScenarioClientWrapper - Client Component for interactive features
+// This wraps Server Components to add client-side interactivity
+
+import {
+  useCallback,
+  useEffect,
+  useState,
+  type ReactNode,
+} from 'react';
+import { scenarios } from "./MockData_${safeScenarioName}";
+
+// Inject image interception code at module level
+${getImageReplacementCode()}
+
+interface ScenarioClientWrapperProps {
+  scenarioName: string;
+  fileName: string;
+  version: string;
+  children: ReactNode;
+}
+
+export function ScenarioClientWrapper({
+  scenarioName,
+  fileName,
+  version,
+  children,
+}: ScenarioClientWrapperProps) {
+  const [internalVersion, setInternalVersion] = useState(1);
+  const [searchParams] = useState(() =>
+    typeof window !== 'undefined'
+      ? new URLSearchParams(window.location.search)
+      : new URLSearchParams()
+  );
+  const widthOverride = searchParams.get('width');
+  const heightOverride = searchParams.get('height') || 'auto';
+
+  // Set page metadata
+  useEffect(() => {
+    document.title = \`\${fileName}: \${scenarioName} CodeYam Demo v\${version}\`;
+    let metaDesc = document.querySelector('meta[name="description"]');
+    if (!metaDesc) {
+      metaDesc = document.createElement('meta');
+      metaDesc.setAttribute('name', 'description');
+      document.head.appendChild(metaDesc);
+    }
+    metaDesc.setAttribute('content', \`This is a demo of the \${fileName} component\`);
+  }, [fileName, scenarioName, version]);
+
+  // Message posting helper
+  const respond = useCallback((message: any) => {
+    try {
+      window.parent.postMessage({
+        type: 'codeyam-update',
+        name: scenarioName,
+        message,
+      }, '*');
+    } catch (e) {
+      console.warn('Error posting message to parent', message, e);
+    }
+  }, [scenarioName]);
+
+  // Handle data override messages from parent
+  useEffect(() => {
+    const eventListener = (event: MessageEvent) => {
+      if (event.data.type === 'codeyam-override-data') {
+        const parsedData = JSON.parse(event.data.data);
+        console.log("Override data received");
+        respond("Set override data to " + JSON.stringify(parsedData, null, 2));
+        scenarios().updateData(scenarioName, parsedData);
+        setInternalVersion(prev => prev + 1);
+      }
+    };
+    window.addEventListener('message', eventListener);
+    return () => window.removeEventListener('message', eventListener);
+  }, [respond, scenarioName]);
+
+  // Console interception
+  useEffect(() => {
+    const oldConsoleLog = console.log;
+    console.log = function(...args: any[]) {
+      oldConsoleLog(...args);
+      try {
+        window.parent.postMessage({
+          type: 'codeyam-log',
+          name: scenarioName,
+          data: {
+            log: {
+              name: "console.log",
+              args
+            }
+          }
+        }, '*');
+      } catch (e) {
+        oldConsoleLog('Error posting message to parent', e);
+      }
+    };
+
+    const oldConsoleError = console.error;
+    console.error = function(...args: any[]) {
+      oldConsoleError(...args);
+      window.parent.postMessage({
+        type: 'codeyam-log',
+        name: scenarioName,
+        data: {
+          log: {
+            name: "console.error",
+            args
+          }
+        }
+      }, '*');
+    };
+
+    return () => {
+      console.log = oldConsoleLog;
+      console.error = oldConsoleError;
+    };
+  }, [scenarioName]);
+
+  // Next.js error closing
+  ${getNextJsErrorClosingCode()}
+
+  // Height calculation and resize messaging
+  useEffect(() => {
+    if (typeof window === 'undefined') return;
+
+    const calculateHeight = () => {
+      const bodyHeight = document.body.offsetHeight;
+      const htmlHeight = document.documentElement.offsetHeight;
+      const scrollHeight = Math.max(
+        document.body.scrollHeight,
+        document.documentElement.scrollHeight,
+        document.body.offsetHeight,
+        document.documentElement.offsetHeight,
+        document.body.clientHeight,
+        document.documentElement.clientHeight
+      );
+      return Math.max(100, bodyHeight, htmlHeight, scrollHeight);
+    };
+
+    const calculatedHeight = calculateHeight();
+    parent.postMessage({ type: 'codeyam-resize', name: scenarioName, height: calculatedHeight }, '*');
+
+    const eventListener = (event: MessageEvent) => {
+      if (event.data.name === scenarioName) {
+        if (event.data.type === 'codeyam-respond') {
+          const calculatedHeight = calculateHeight();
+          parent.postMessage({ type: 'codeyam-resize', name: scenarioName, height: calculatedHeight }, '*');
+        }
+      }
+    };
+
+    window.addEventListener('message', eventListener);
+    return () => window.removeEventListener('message', eventListener);
+  }, [scenarioName]);
+
+  const width = widthOverride || 'auto';
+
+  return (
+    <div className='flex flex-col justify-center items-center h-screen' data-version={internalVersion}>
+      <div className='w-full flex items-center justify-center'>
+        <div id='component' style={{
+          minWidth: width !== 'auto' ? \`min(\${width}, 100vw)\` : undefined,
+          height: heightOverride !== 'auto' ? heightOverride : undefined,
+          minHeight: heightOverride !== 'auto' ? heightOverride : undefined,
+          display: 'block',
+          padding: '6px'
+        }}>
+          {children}
+        </div>
+      </div>
+    </div>
+  );
+}
+`;
+}

package/analyzer-template/packages/generate/src/lib/componentScenarioPage/generateScenarioServerComponent.ts

@@ -0,0 +1,73 @@
+import { Entity, Analysis, type File, type Scenario } from '~codeyam/types';
+import { safeFileName } from '~codeyam/utils';
+import getComponentImportStatement from '../getComponentImportStatement';
+import scenarioComponentForServer from '../scenarioComponentForServer';
+
+const VERSION = '0.5.1';
+
+/**
+ * Generates a Server Component page for async components (Server Components in Next.js).
+ *
+ * This is needed because:
+ * 1. Async components use await, which only works in Server Components
+ * 2. Client Components ("use client") don't support async/await in the component body
+ * 3. We use Next.js composition pattern: Server Component page renders the async component,
+ *    then wraps it in a Client Component (ScenarioClientWrapper) for interactive features
+ */
+export default function generateScenarioServerComponent({
+  file,
+  entity,
+  analysis,
+  currentPath,
+  appPath,
+  scenario,
+  relativeMocksDir,
+}: {
+  file: File;
+  entity: Entity;
+  analysis: Analysis;
+  currentPath: string;
+  appPath: string;
+  scenario: Scenario;
+  relativeMocksDir: string;
+}) {
+  const fileName = file.name.split('.')[0];
+  const safeScenarioName = safeFileName(scenario.name);
+
+  return `/* eslint-disable */
+// Server Component page for async component: ${entity.name}
+// This page does NOT have "use client" - it runs on the server
+import { ScenarioClientWrapper } from "${relativeMocksDir}/ScenarioClientWrapper";
+${analysis.scenarios?.length > 0 ? `import { scenarios } from "${relativeMocksDir}/MockData_${safeScenarioName}";` : ''}
+${getComponentImportStatement({
+  file,
+  entity,
+  analysis,
+  currentPath,
+  appPath,
+  scenario,
+  primary: true,
+  namedExport:
+    entity.name !== 'default' &&
+    (entity.metadata?.notExported || entity.metadata?.namedExport) &&
+    entity.entityType !== 'library',
+})}
+
+function getScenarioProps({ argumentIndex, propName }: { argumentIndex: number, propName?: string }) {
+  const data = scenarios().data();
+  const argument = data['arguments'][argumentIndex];
+  return propName ? argument[propName] : argument;
+}
+
+export default async function Index() {
+  return (
+    <ScenarioClientWrapper
+      scenarioName="${scenario.name}"
+      fileName="${fileName}"
+      version="${VERSION}"
+    >
+      ${scenarioComponentForServer(entity, analysis, scenario)}
+    </ScenarioClientWrapper>
+  );
+}`;
+}