@codeyam/codeyam-cli 0.1.0-staging.c90f8c9 → 0.1.0-staging.c9dc00c

package/analyzer-template/project/createEntitiesAndSortFiles.ts

@@ -28,6 +28,14 @@ export interface SortedEntitiesAndMap {
   entityMap: { [key: string]: { [key: string]: EntityWithImportCount } };
 }
 
+// Timing stats for debugging
+let findOrCreateEntityTime = 0;
+let gatherEntityMapTime = 0;
+let recursiveCallTime = 0;
+let findOrCreateEntityCalls = 0;
+let gatherEntityMapCalls = 0;
+let recursiveCalls = 0;
+
 async function findAndGatherEntity({
   project,
   branch,
@@ -37,6 +45,7 @@ async function findAndGatherEntity({
   projectAnalyzer,
   context,
   fileStore,
+  depth = 0,
 }: {
   project: Project;
   branch: Branch;
@@ -46,6 +55,7 @@ async function findAndGatherEntity({
   projectAnalyzer: ProjectAnalyzer;
   context: AnalysisContext;
   fileStore?: LazyFileStore;
+  depth?: number;
 }) {
   let entityWithImportCount: EntityWithImportCount = context.getEntity(
     file.path,
@@ -53,6 +63,7 @@ async function findAndGatherEntity({
   ) as EntityWithImportCount;
   if (entityWithImportCount?.importCount === undefined) {
     try {
+      const t0 = performance.now();
       entityWithImportCount = await findOrCreateEntity({
         file,
         branchId: branch?.id,
@@ -61,6 +72,8 @@ async function findAndGatherEntity({
         projectAnalyzer,
         context,
       });
+      findOrCreateEntityTime += performance.now() - t0;
+      findOrCreateEntityCalls++;
     } catch (e) {
       console.log(
         'CodeYam Warning: Entity not found in create and sort entities',
@@ -72,12 +85,15 @@ async function findAndGatherEntity({
     // Add entity to context
     context.addEntity(entityWithImportCount);
 
+    const t1 = performance.now();
     await gatherEntityMap({
       entity: entityWithImportCount,
       branchId: branch?.id,
       context,
       projectAnalyzer,
     });
+    gatherEntityMapTime += performance.now() - t1;
+    gatherEntityMapCalls++;
 
     entityWithImportCount.importCount = 0;
     for (const importedEntity of entityWithImportCount.metadata
@@ -105,6 +121,8 @@ async function findAndGatherEntity({
         continue;
       }
 
+      const t2 = performance.now();
+      recursiveCalls++;
       const info = await findAndGatherEntity({
         project,
         branch,
@@ -114,7 +132,11 @@ async function findAndGatherEntity({
         projectAnalyzer,
         context,
         fileStore,
+        depth: depth + 1,
       });
+      if (depth === 0) {
+        recursiveCallTime += performance.now() - t2;
+      }
       entityWithImportCount.importCount +=
         info.entityWithImportCount.importCount;
     }
@@ -147,6 +169,14 @@ export default async function createEntitiesAndSortFiles({
   branchCommit, // just for progress updates
   fileStore,
 }: EntitiesAndSortFiles): Promise<SortedEntitiesAndMap> {
+  // Reset timing stats
+  findOrCreateEntityTime = 0;
+  gatherEntityMapTime = 0;
+  recursiveCallTime = 0;
+  findOrCreateEntityCalls = 0;
+  gatherEntityMapCalls = 0;
+  recursiveCalls = 0;
+
   const sortedEntities: Entity[] = [];
 
   // We slightly abuse the AnalysisContext here, initializing and later throwing it away
@@ -161,6 +191,16 @@ export default async function createEntitiesAndSortFiles({
   // Enable baseline mode to skip DB reads during entity creation
   context.setBaselineMode(true);
   let ii = 0;
+  let updateCommitMetadataTime = 0;
+  let getFileAnalyzerTime = 0;
+  let getAllEntityNodesTime = 0;
+  const loopStart = performance.now();
+  let lastLogTime = loopStart;
+
+  console.log(
+    `CodeYam: createEntitiesAndSortFiles starting, ${filePaths.length} files to process`,
+  );
+
   for (const currentFilePath of filePaths) {
     if (filePath && currentFilePath !== filePath) {
       ii++;
@@ -179,16 +219,23 @@ export default async function createEntitiesAndSortFiles({
     }
 
     if (branchCommit) {
+      const t0 = performance.now();
       branchCommit.metadata = await updateCommitMetadata({
         commitId: branchCommit.id,
         runStatusUpdate: {
           filesCompleted: ii + 1,
         },
       });
+      updateCommitMetadataTime += performance.now() - t0;
     }
 
+    const t1 = performance.now();
     const fileAnalyzer = projectAnalyzer.getFileAnalyzer(file);
+    getFileAnalyzerTime += performance.now() - t1;
+
+    const t2 = performance.now();
     const allEntityNodes = fileAnalyzer.getAllEntityNodesOnlyAliases();
+    getAllEntityNodesTime += performance.now() - t2;
 
     for (const entityName of Object.keys(allEntityNodes)) {
       const { entityWithImportCount: entity } = await findAndGatherEntity({
@@ -206,8 +253,44 @@ export default async function createEntitiesAndSortFiles({
     }
 
     ii++;
+
+    // Log timing stats every 5 seconds
+    const now = performance.now();
+    if (now - lastLogTime > 5000) {
+      lastLogTime = now;
+      const elapsed = ((now - loopStart) / 1000).toFixed(1);
+      console.log(`CodeYam: createEntitiesAndSortFiles progress`, {
+        filesProcessed: ii,
+        totalFiles: filePaths.length,
+        elapsedSec: elapsed,
+        timings: {
+          updateCommitMetadata: `${(updateCommitMetadataTime / 1000).toFixed(1)}s`,
+          getFileAnalyzer: `${(getFileAnalyzerTime / 1000).toFixed(1)}s`,
+          getAllEntityNodes: `${(getAllEntityNodesTime / 1000).toFixed(1)}s`,
+          findOrCreateEntity: `${(findOrCreateEntityTime / 1000).toFixed(1)}s (${findOrCreateEntityCalls} calls)`,
+          gatherEntityMap: `${(gatherEntityMapTime / 1000).toFixed(1)}s (${gatherEntityMapCalls} calls)`,
+          recursiveCalls: `${(recursiveCallTime / 1000).toFixed(1)}s (${recursiveCalls} calls)`,
+        },
+      });
+    }
   }
 
+  // Final timing log
+  const totalElapsed = ((performance.now() - loopStart) / 1000).toFixed(1);
+  console.log(`CodeYam: createEntitiesAndSortFiles loop complete`, {
+    filesProcessed: ii,
+    totalFiles: filePaths.length,
+    elapsedSec: totalElapsed,
+    timings: {
+      updateCommitMetadata: `${(updateCommitMetadataTime / 1000).toFixed(1)}s`,
+      getFileAnalyzer: `${(getFileAnalyzerTime / 1000).toFixed(1)}s`,
+      getAllEntityNodes: `${(getAllEntityNodesTime / 1000).toFixed(1)}s`,
+      findOrCreateEntity: `${(findOrCreateEntityTime / 1000).toFixed(1)}s (${findOrCreateEntityCalls} calls)`,
+      gatherEntityMap: `${(gatherEntityMapTime / 1000).toFixed(1)}s (${gatherEntityMapCalls} calls)`,
+      recursiveCalls: `${(recursiveCallTime / 1000).toFixed(1)}s (${recursiveCalls} calls)`,
+    },
+  });
+
   await setImportedExports({
     entities: Object.values(sortedEntities).flat(),
     context,

package/analyzer-template/project/loadReadyToBeCaptured.ts

@@ -7,6 +7,28 @@ import {
 import { Analysis, Project } from '~codeyam/types';
 import { LazyFileStore } from './LazyFileStore';
 
+const DB_CONCURRENCY = 1;
+
+async function mapWithConcurrency<T, R>(
+  items: T[],
+  fn: (item: T) => Promise<R>,
+  concurrency: number,
+): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let index = 0;
+  const workers = Array.from(
+    { length: Math.min(concurrency, items.length) },
+    async () => {
+      while (index < items.length) {
+        const i = index++;
+        results[i] = await fn(items[i]);
+      }
+    },
+  );
+  await Promise.all(workers);
+  return results;
+}
+
 interface LoadReadyToBeCapturedArgs {
   project: Project;
   branchCommitSha?: string;
@@ -77,8 +99,9 @@ async function gatherAllReferencedEntities(
       byFilePath.get(filePath)!.push(name);
     }
 
-    // Batch load entities by file path
-    const loadPromises = Array.from(byFilePath.entries()).map(
+    // Batch load entities by file path (concurrency-limited to avoid exhausting DB connections)
+    const results = await mapWithConcurrency(
+      Array.from(byFilePath.entries()),
       async ([filePath, names]) => {
         const entities = await loadEntities({
           projectId: project.id,
@@ -106,9 +129,8 @@ async function gatherAllReferencedEntities(
           return false;
         });
       },
+      DB_CONCURRENCY,
     );
-
-    const results = await Promise.all(loadPromises);
     const loadedEntities = results.flat();
 
     // Add loaded entities to context
@@ -198,48 +220,50 @@ export default async function loadReadyToBeCaptured({
     foundNewDependencies = false;
     const currentBatch = Array.from(stillNeeded);
 
-    // Load analyses for current batch
-    const loadPromises = currentBatch.map(async (keyString) => {
-      const [filePath, entityName] = keyString.split(':');
-
-      // Find the file ID for this file path - use fileStore for O(1) lookup when available
-      const file = fileStore
-        ? fileStore.getByPath(filePath)
-        : project.files.find((f) => f.path === filePath);
-      if (!file) {
-        console.log(
-          `CodeYam Warning: File not found in project: ${filePath} for key ${keyString}`,
-        );
-        return null;
-      }
-
-      try {
-        const analysis = await loadAnalysis({
-          projectId: project.id,
-          fileId: file.id,
-          entityName,
-          includeScenarios: isFirstBatch, // Only include scenarios for the root analyses
-        });
-
-        if (analysis) {
-          console.log(`CodeYam: Loaded analysis ${filePath}:${entityName}`);
-          return { key: keyString, analysis };
-        } else {
+    // Load analyses for current batch (concurrency-limited to avoid exhausting DB connections)
+    const results = await mapWithConcurrency(
+      currentBatch,
+      async (keyString) => {
+        const [filePath, entityName] = keyString.split(':');
+
+        // Find the file ID for this file path - use fileStore for O(1) lookup when available
+        const file = fileStore
+          ? fileStore.getByPath(filePath)
+          : project.files.find((f) => f.path === filePath);
+        if (!file) {
           console.log(
-            `CodeYam Warning: Analysis not found: ${filePath}:${entityName}`,
+            `CodeYam Warning: File not found in project: ${filePath} for key ${keyString}`,
           );
           return null;
         }
-      } catch (error) {
-        console.log(
-          `CodeYam Error loading analysis ${filePath}:${entityName}:`,
-          error,
-        );
-        return null;
-      }
-    });
 
-    const results = await Promise.all(loadPromises);
+        try {
+          const analysis = await loadAnalysis({
+            projectId: project.id,
+            fileId: file.id,
+            entityName,
+            includeScenarios: isFirstBatch, // Only include scenarios for the root analyses
+          });
+
+          if (analysis) {
+            console.log(`CodeYam: Loaded analysis ${filePath}:${entityName}`);
+            return { key: keyString, analysis };
+          } else {
+            console.log(
+              `CodeYam Warning: Analysis not found: ${filePath}:${entityName}`,
+            );
+            return null;
+          }
+        } catch (error) {
+          console.log(
+            `CodeYam Error loading analysis ${filePath}:${entityName}:`,
+            error,
+          );
+          return null;
+        }
+      },
+      DB_CONCURRENCY,
+    );
 
     // Process loaded analyses and find new dependencies
     for (const result of results) {

package/analyzer-template/project/orchestrateCapture/AwsCaptureTaskRunner.ts

@@ -35,7 +35,10 @@ export default class AwsCaptureTaskRunner implements CaptureTaskRunner {
   private sqs: SQSClient;
   private queueInfo: QueueInfo;
   private taskArns: string[];
-  private batchCompletedCallback?: (analysisIds: string[]) => void;
+  private batchCompletedCallback?: (
+    analysisIds: string[],
+    success: boolean,
+  ) => void;
   private controlPollingInterval?: NodeJS.Timeout;
   private lastTaskHealthCheck = Date.now();
   private isCleanedUp = false;
@@ -213,7 +216,9 @@ export default class AwsCaptureTaskRunner implements CaptureTaskRunner {
     }
   }
 
-  onBatchCompleted(callback: (analysisIds: string[]) => void): void {
+  onBatchCompleted(
+    callback: (analysisIds: string[], success: boolean) => void,
+  ): void {
     this.batchCompletedCallback = callback;
   }
 
@@ -255,9 +260,12 @@ export default class AwsCaptureTaskRunner implements CaptureTaskRunner {
               body.analysisIds &&
               Array.isArray(body.analysisIds)
             ) {
-              // Notify callback
+              // Notify callback with success status (default false for backwards compatibility)
               if (this.batchCompletedCallback) {
-                this.batchCompletedCallback(body.analysisIds);
+                this.batchCompletedCallback(
+                  body.analysisIds,
+                  body.success ?? false,
+                );
               }
             }
 

package/analyzer-template/project/orchestrateCapture/SequentialCaptureTaskRunner.ts

@@ -49,7 +49,10 @@ export interface SequentialCaptureTaskRunnerContext {
 
 export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
   private queue: string[][] = [];
-  private batchCompletedCallback?: (analysisIds: string[]) => void;
+  private batchCompletedCallback?: (
+    analysisIds: string[],
+    success: boolean,
+  ) => void;
   private isProcessing = false;
   private isCleanedUp = false;
   private processingLoop: Promise<void> | null = null;
@@ -197,15 +200,15 @@ export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
       try {
         await this.processBatch(batch);
 
-        // Notify callback of completion
+        // Notify callback of successful completion
         if (this.batchCompletedCallback) {
-          this.batchCompletedCallback(batch);
+          this.batchCompletedCallback(batch, true);
         }
       } catch (error) {
         console.error('CodeYam: Error processing batch:', error);
-        // Still notify callback so orchestration can track failures
+        // Notify callback of failure so orchestration can track retries
         if (this.batchCompletedCallback) {
-          this.batchCompletedCallback(batch);
+          this.batchCompletedCallback(batch, false);
         }
       } finally {
         this.isProcessing = false;
@@ -399,7 +402,9 @@ export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
     console.log('CodeYam: Sequential task runner cleanup complete');
   }
 
-  onBatchCompleted(callback: (analysisIds: string[]) => void): void {
+  onBatchCompleted(
+    callback: (analysisIds: string[], success: boolean) => void,
+  ): void {
     this.batchCompletedCallback = callback;
   }
 }

package/analyzer-template/project/orchestrateCapture/taskRunner.ts

@@ -115,7 +115,9 @@ export interface CaptureTaskRunner {
    * Register callback for when workers complete batch processing.
    *
    * Critical for retry logic and progress tracking in orchestration.
-   * Callback receives the analysis IDs that were completed.
+   * Callback receives the analysis IDs and whether the batch succeeded.
    */
-  onBatchCompleted(callback: (analysisIds: string[]) => void): void;
+  onBatchCompleted(
+    callback: (analysisIds: string[], success: boolean) => void,
+  ): void;
 }

package/analyzer-template/project/orchestrateCapture.ts

@@ -99,15 +99,20 @@ export default async function orchestrateCapture({
   const attemptCount = new Map<string, number>();
   const maxAttempts = 2;
 
+  // Track which analyses had a worker report success — only these can be "captured"
+  const confirmedSuccessAnalyses = new Set<string>();
+
   // Register batch completion callback with task runner
-  taskRunner.onBatchCompleted((analysisIds: string[]) => {
+  taskRunner.onBatchCompleted((analysisIds: string[], success: boolean) => {
     console.log(
-      `CodeYam Capture: Received batch completion for ${analysisIds.length} analyses: ${analysisIds.join(', ')}`,
+      `CodeYam Capture: Received batch completion (success=${success}) for ${analysisIds.length} analyses: ${analysisIds.join(', ')}`,
     );
-    // Remove from inFlight and increment attempt count for each analysis
     for (const analysisId of analysisIds) {
       inFlight.delete(analysisId);
       attemptCount.set(analysisId, (attemptCount.get(analysisId) || 0) + 1);
+      if (success) {
+        confirmedSuccessAnalyses.add(analysisId);
+      }
     }
   });
 
@@ -195,9 +200,13 @@ export default async function orchestrateCapture({
         knownAnalyses.add(id);
       }
 
-      // an analysis we've seen, but is no longer "ready to be captured", is captured!
+      // An analysis is "captured" only if:
+      // 1. A worker reported success for its batch (in confirmedSuccessAnalyses)
+      // 2. It no longer appears in the ready-to-be-captured list (DB confirms)
+      // This prevents transient empty DB results from inflating the captured count.
       const capturedAnalysisIds = Array.from(knownAnalyses).filter(
-        (id) => !allReadyAnalysisIds.includes(id),
+        (id) =>
+          confirmedSuccessAnalyses.has(id) && !allReadyAnalysisIds.includes(id),
       );
 
       // Remove captured analyses from inFlight tracking
@@ -397,7 +406,9 @@ export default async function orchestrateCapture({
           // check if all enqueued analyses have been either captured or abandoned
           const allAnalysesSettled = Array.from(enqueuedAnalyses).every(
             (id) => {
-              const captured = !allReadyAnalysisIds.includes(id);
+              const captured =
+                confirmedSuccessAnalyses.has(id) &&
+                !allReadyAnalysisIds.includes(id);
               const abandoned = (attemptCount.get(id) || 0) >= maxAttempts;
               return captured || abandoned;
             },
@@ -437,5 +448,33 @@ export default async function orchestrateCapture({
   } finally {
     // Always clean up task runner resources in finally block
     await taskRunner.cleanup();
+
+    const trackingCommit = metadataCommit ?? commit;
+    if (trackingCommit?.id) {
+      const captureCompletedAt = new Date().toISOString();
+
+      await updateCommitMetadata({
+        commitId: trackingCommit.id,
+        runStatusUpdate: {
+          captureCompletedAt,
+          capturePid: undefined,
+        },
+        // completedAt = max(analysisCompletedAt, captureCompletedAt)
+        // Normally capture finishes after analysis, but using max means illogical
+        // orderings leave a detectable signal (completedAt !== captureCompletedAt).
+        updateCallback(metadata) {
+          const run = metadata.currentRun!;
+          run.completedAt =
+            run.analysisCompletedAt &&
+            run.analysisCompletedAt > captureCompletedAt
+              ? run.analysisCompletedAt
+              : captureCompletedAt;
+        },
+      }).catch((error) => {
+        console.log(
+          `Warning: Failed to update capture completion metadata: ${error}`,
+        );
+      });
+    }
   }
 }

package/analyzer-template/project/start.ts

@@ -17,7 +17,8 @@ import { startScenarioCapture } from './startScenarioCapture';
 import { createController } from './controller/startController';
 import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import * as v8 from 'v8';
-import { initializeAIServiceMode } from '~codeyam/ai';
+import * as os from 'os';
+import { initializeAIServiceMode, destroyWorkerPool } from '~codeyam/ai';
 import runAnalysis, { EnvConfig, SharedContext } from './runAnalysis';
 import {
   loadOrCreateCommit,
@@ -119,28 +120,31 @@ async function main({
     // and attempted to access project/commit/branch data before it was loaded.
     const sharedContext = await loadSharedContext(envConfig);
 
-    // Heap monitoring: pulse every 1s, log every 30s when normal, warn every 1s when high
+    // Memory monitoring: tracks both V8 heap and RSS (which includes worker threads)
+    const containerMemoryMB = process.env.ECS_CONTAINER_MEMORY_MB
+      ? parseInt(process.env.ECS_CONTAINER_MEMORY_MB, 10)
+      : Math.round(os.totalmem() / 1024 / 1024);
     let lastLogTime = 0;
     setInterval(() => {
       const mem = process.memoryUsage();
       const heap = v8.getHeapStatistics();
-      const usagePercent = (mem.heapUsed / heap.heap_size_limit) * 100;
+      const heapMB = Math.round(mem.heapUsed / 1024 / 1024);
+      const rssMB = Math.round(mem.rss / 1024 / 1024);
+      const heapLimitMB = Math.round(heap.heap_size_limit / 1024 / 1024);
+      const rssPercent = (rssMB / containerMemoryMB) * 100;
       const now = Date.now();
 
-      if (usagePercent >= 80) {
-        // High usage: warn every second
+      if (rssPercent >= 80) {
         console.warn(
-          `CodeYam Heap Warning: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ HIGH USAGE`,
+          `CodeYam Memory Warning: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ HIGH`,
         );
-      } else if (usagePercent >= 50 && now - lastLogTime >= 5000) {
-        // Elevated usage: log every 5 seconds
+      } else if (rssPercent >= 50 && now - lastLogTime >= 5000) {
         console.log(
-          `CodeYam Heap Notice: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ Elevated`,
+          `CodeYam Memory Notice: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ Elevated`,
         );
       } else if (now - lastLogTime >= 30000) {
-        // Normal usage: log every 30 seconds
         console.log(
-          `CodeYam Heap Stats: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) 📊 OK`,
+          `CodeYam Memory Stats: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB 📊 OK`,
         );
       }
 
@@ -187,6 +191,11 @@ async function main({
               analyzerPid: undefined,
               capturePid: process.pid,
             },
+            updateCallback(metadata) {
+              if (metadata.currentRun?.captureCompletedAt) {
+                metadata.currentRun.completedAt = new Date().toISOString();
+              }
+            },
           }).catch((error) => {
             awsLog(`Warning: Failed to update commit metadata: ${error}`);
           });
@@ -219,6 +228,10 @@ async function main({
         }
         awsLog('CodeYam: Analysis producing, starting orchestration...');
 
+        // Worker pool is no longer needed after data structure prep phase.
+        // Freeing it reclaims ~2GB before capture workers ramp up.
+        await destroyWorkerPool();
+
         // Create task runner based on orchestration mode
         let taskRunner;
 
@@ -229,6 +242,17 @@ async function main({
             framework: framework,
             packageManager: packageManager,
           });
+
+          const captureCommit =
+            sharedContext.commit ?? sharedContext.branchCommit;
+          if (captureCommit) {
+            await updateCommitMetadata({
+              commitId: captureCommit.id,
+              runStatusUpdate: {
+                captureTaskArns: taskRunner.getTaskArns(),
+              },
+            });
+          }
         } else if (orchestrateCapture === 'local-sequential') {
           taskRunner = await SequentialCaptureTaskRunner.create(
             {