@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.27d5a59

package/analyzer-template/project/createEntitiesAndSortFiles.ts

@@ -28,6 +28,14 @@ export interface SortedEntitiesAndMap {
   entityMap: { [key: string]: { [key: string]: EntityWithImportCount } };
 }
 
+// Timing stats for debugging
+let findOrCreateEntityTime = 0;
+let gatherEntityMapTime = 0;
+let recursiveCallTime = 0;
+let findOrCreateEntityCalls = 0;
+let gatherEntityMapCalls = 0;
+let recursiveCalls = 0;
+
 async function findAndGatherEntity({
   project,
   branch,
@@ -37,6 +45,7 @@ async function findAndGatherEntity({
   projectAnalyzer,
   context,
   fileStore,
+  depth = 0,
 }: {
   project: Project;
   branch: Branch;
@@ -46,6 +55,7 @@ async function findAndGatherEntity({
   projectAnalyzer: ProjectAnalyzer;
   context: AnalysisContext;
   fileStore?: LazyFileStore;
+  depth?: number;
 }) {
   let entityWithImportCount: EntityWithImportCount = context.getEntity(
     file.path,
@@ -53,6 +63,7 @@ async function findAndGatherEntity({
   ) as EntityWithImportCount;
   if (entityWithImportCount?.importCount === undefined) {
     try {
+      const t0 = performance.now();
       entityWithImportCount = await findOrCreateEntity({
         file,
         branchId: branch?.id,
@@ -61,6 +72,8 @@ async function findAndGatherEntity({
         projectAnalyzer,
         context,
       });
+      findOrCreateEntityTime += performance.now() - t0;
+      findOrCreateEntityCalls++;
     } catch (e) {
       console.log(
         'CodeYam Warning: Entity not found in create and sort entities',
@@ -72,12 +85,15 @@ async function findAndGatherEntity({
     // Add entity to context
     context.addEntity(entityWithImportCount);
 
+    const t1 = performance.now();
     await gatherEntityMap({
       entity: entityWithImportCount,
       branchId: branch?.id,
       context,
       projectAnalyzer,
     });
+    gatherEntityMapTime += performance.now() - t1;
+    gatherEntityMapCalls++;
 
     entityWithImportCount.importCount = 0;
     for (const importedEntity of entityWithImportCount.metadata
@@ -105,6 +121,8 @@ async function findAndGatherEntity({
         continue;
       }
 
+      const t2 = performance.now();
+      recursiveCalls++;
       const info = await findAndGatherEntity({
         project,
         branch,
@@ -114,7 +132,11 @@ async function findAndGatherEntity({
         projectAnalyzer,
         context,
         fileStore,
+        depth: depth + 1,
       });
+      if (depth === 0) {
+        recursiveCallTime += performance.now() - t2;
+      }
       entityWithImportCount.importCount +=
         info.entityWithImportCount.importCount;
     }
@@ -147,6 +169,14 @@ export default async function createEntitiesAndSortFiles({
   branchCommit, // just for progress updates
   fileStore,
 }: EntitiesAndSortFiles): Promise<SortedEntitiesAndMap> {
+  // Reset timing stats
+  findOrCreateEntityTime = 0;
+  gatherEntityMapTime = 0;
+  recursiveCallTime = 0;
+  findOrCreateEntityCalls = 0;
+  gatherEntityMapCalls = 0;
+  recursiveCalls = 0;
+
   const sortedEntities: Entity[] = [];
 
   // We slightly abuse the AnalysisContext here, initializing and later throwing it away
@@ -161,6 +191,16 @@ export default async function createEntitiesAndSortFiles({
   // Enable baseline mode to skip DB reads during entity creation
   context.setBaselineMode(true);
   let ii = 0;
+  let updateCommitMetadataTime = 0;
+  let getFileAnalyzerTime = 0;
+  let getAllEntityNodesTime = 0;
+  const loopStart = performance.now();
+  let lastLogTime = loopStart;
+
+  console.log(
+    `CodeYam: createEntitiesAndSortFiles starting, ${filePaths.length} files to process`,
+  );
+
   for (const currentFilePath of filePaths) {
     if (filePath && currentFilePath !== filePath) {
       ii++;
@@ -179,16 +219,23 @@ export default async function createEntitiesAndSortFiles({
     }
 
     if (branchCommit) {
+      const t0 = performance.now();
       branchCommit.metadata = await updateCommitMetadata({
         commitId: branchCommit.id,
         runStatusUpdate: {
           filesCompleted: ii + 1,
         },
       });
+      updateCommitMetadataTime += performance.now() - t0;
     }
 
+    const t1 = performance.now();
     const fileAnalyzer = projectAnalyzer.getFileAnalyzer(file);
+    getFileAnalyzerTime += performance.now() - t1;
+
+    const t2 = performance.now();
     const allEntityNodes = fileAnalyzer.getAllEntityNodesOnlyAliases();
+    getAllEntityNodesTime += performance.now() - t2;
 
     for (const entityName of Object.keys(allEntityNodes)) {
       const { entityWithImportCount: entity } = await findAndGatherEntity({
@@ -206,8 +253,44 @@ export default async function createEntitiesAndSortFiles({
     }
 
     ii++;
+
+    // Log timing stats every 5 seconds
+    const now = performance.now();
+    if (now - lastLogTime > 5000) {
+      lastLogTime = now;
+      const elapsed = ((now - loopStart) / 1000).toFixed(1);
+      console.log(`CodeYam: createEntitiesAndSortFiles progress`, {
+        filesProcessed: ii,
+        totalFiles: filePaths.length,
+        elapsedSec: elapsed,
+        timings: {
+          updateCommitMetadata: `${(updateCommitMetadataTime / 1000).toFixed(1)}s`,
+          getFileAnalyzer: `${(getFileAnalyzerTime / 1000).toFixed(1)}s`,
+          getAllEntityNodes: `${(getAllEntityNodesTime / 1000).toFixed(1)}s`,
+          findOrCreateEntity: `${(findOrCreateEntityTime / 1000).toFixed(1)}s (${findOrCreateEntityCalls} calls)`,
+          gatherEntityMap: `${(gatherEntityMapTime / 1000).toFixed(1)}s (${gatherEntityMapCalls} calls)`,
+          recursiveCalls: `${(recursiveCallTime / 1000).toFixed(1)}s (${recursiveCalls} calls)`,
+        },
+      });
+    }
   }
 
+  // Final timing log
+  const totalElapsed = ((performance.now() - loopStart) / 1000).toFixed(1);
+  console.log(`CodeYam: createEntitiesAndSortFiles loop complete`, {
+    filesProcessed: ii,
+    totalFiles: filePaths.length,
+    elapsedSec: totalElapsed,
+    timings: {
+      updateCommitMetadata: `${(updateCommitMetadataTime / 1000).toFixed(1)}s`,
+      getFileAnalyzer: `${(getFileAnalyzerTime / 1000).toFixed(1)}s`,
+      getAllEntityNodes: `${(getAllEntityNodesTime / 1000).toFixed(1)}s`,
+      findOrCreateEntity: `${(findOrCreateEntityTime / 1000).toFixed(1)}s (${findOrCreateEntityCalls} calls)`,
+      gatherEntityMap: `${(gatherEntityMapTime / 1000).toFixed(1)}s (${gatherEntityMapCalls} calls)`,
+      recursiveCalls: `${(recursiveCallTime / 1000).toFixed(1)}s (${recursiveCalls} calls)`,
+    },
+  });
+
   await setImportedExports({
     entities: Object.values(sortedEntities).flat(),
     context,

package/analyzer-template/project/loadReadyToBeCaptured.ts

@@ -7,6 +7,28 @@ import {
 import { Analysis, Project } from '~codeyam/types';
 import { LazyFileStore } from './LazyFileStore';
 
+const DB_CONCURRENCY = 1;
+
+async function mapWithConcurrency<T, R>(
+  items: T[],
+  fn: (item: T) => Promise<R>,
+  concurrency: number,
+): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let index = 0;
+  const workers = Array.from(
+    { length: Math.min(concurrency, items.length) },
+    async () => {
+      while (index < items.length) {
+        const i = index++;
+        results[i] = await fn(items[i]);
+      }
+    },
+  );
+  await Promise.all(workers);
+  return results;
+}
+
 interface LoadReadyToBeCapturedArgs {
   project: Project;
   branchCommitSha?: string;
@@ -77,8 +99,9 @@ async function gatherAllReferencedEntities(
       byFilePath.get(filePath)!.push(name);
     }
 
-    // Batch load entities by file path
-    const loadPromises = Array.from(byFilePath.entries()).map(
+    // Batch load entities by file path (concurrency-limited to avoid exhausting DB connections)
+    const results = await mapWithConcurrency(
+      Array.from(byFilePath.entries()),
       async ([filePath, names]) => {
         const entities = await loadEntities({
           projectId: project.id,
@@ -106,9 +129,8 @@ async function gatherAllReferencedEntities(
           return false;
         });
       },
+      DB_CONCURRENCY,
     );
-
-    const results = await Promise.all(loadPromises);
     const loadedEntities = results.flat();
 
     // Add loaded entities to context
@@ -198,48 +220,50 @@ export default async function loadReadyToBeCaptured({
     foundNewDependencies = false;
     const currentBatch = Array.from(stillNeeded);
 
-    // Load analyses for current batch
-    const loadPromises = currentBatch.map(async (keyString) => {
-      const [filePath, entityName] = keyString.split(':');
-
-      // Find the file ID for this file path - use fileStore for O(1) lookup when available
-      const file = fileStore
-        ? fileStore.getByPath(filePath)
-        : project.files.find((f) => f.path === filePath);
-      if (!file) {
-        console.log(
-          `CodeYam Warning: File not found in project: ${filePath} for key ${keyString}`,
-        );
-        return null;
-      }
-
-      try {
-        const analysis = await loadAnalysis({
-          projectId: project.id,
-          fileId: file.id,
-          entityName,
-          includeScenarios: isFirstBatch, // Only include scenarios for the root analyses
-        });
-
-        if (analysis) {
-          console.log(`CodeYam: Loaded analysis ${filePath}:${entityName}`);
-          return { key: keyString, analysis };
-        } else {
+    // Load analyses for current batch (concurrency-limited to avoid exhausting DB connections)
+    const results = await mapWithConcurrency(
+      currentBatch,
+      async (keyString) => {
+        const [filePath, entityName] = keyString.split(':');
+
+        // Find the file ID for this file path - use fileStore for O(1) lookup when available
+        const file = fileStore
+          ? fileStore.getByPath(filePath)
+          : project.files.find((f) => f.path === filePath);
+        if (!file) {
           console.log(
-            `CodeYam Warning: Analysis not found: ${filePath}:${entityName}`,
+            `CodeYam Warning: File not found in project: ${filePath} for key ${keyString}`,
           );
           return null;
         }
-      } catch (error) {
-        console.log(
-          `CodeYam Error loading analysis ${filePath}:${entityName}:`,
-          error,
-        );
-        return null;
-      }
-    });
 
-    const results = await Promise.all(loadPromises);
+        try {
+          const analysis = await loadAnalysis({
+            projectId: project.id,
+            fileId: file.id,
+            entityName,
+            includeScenarios: isFirstBatch, // Only include scenarios for the root analyses
+          });
+
+          if (analysis) {
+            console.log(`CodeYam: Loaded analysis ${filePath}:${entityName}`);
+            return { key: keyString, analysis };
+          } else {
+            console.log(
+              `CodeYam Warning: Analysis not found: ${filePath}:${entityName}`,
+            );
+            return null;
+          }
+        } catch (error) {
+          console.log(
+            `CodeYam Error loading analysis ${filePath}:${entityName}:`,
+            error,
+          );
+          return null;
+        }
+      },
+      DB_CONCURRENCY,
+    );
 
     // Process loaded analyses and find new dependencies
     for (const result of results) {

package/analyzer-template/project/orchestrateCapture/AwsCaptureTaskRunner.ts

@@ -35,7 +35,10 @@ export default class AwsCaptureTaskRunner implements CaptureTaskRunner {
   private sqs: SQSClient;
   private queueInfo: QueueInfo;
   private taskArns: string[];
-  private batchCompletedCallback?: (analysisIds: string[]) => void;
+  private batchCompletedCallback?: (
+    analysisIds: string[],
+    success: boolean,
+  ) => void;
   private controlPollingInterval?: NodeJS.Timeout;
   private lastTaskHealthCheck = Date.now();
   private isCleanedUp = false;
@@ -213,7 +216,9 @@ export default class AwsCaptureTaskRunner implements CaptureTaskRunner {
     }
   }
 
-  onBatchCompleted(callback: (analysisIds: string[]) => void): void {
+  onBatchCompleted(
+    callback: (analysisIds: string[], success: boolean) => void,
+  ): void {
     this.batchCompletedCallback = callback;
   }
 
@@ -255,9 +260,12 @@ export default class AwsCaptureTaskRunner implements CaptureTaskRunner {
               body.analysisIds &&
               Array.isArray(body.analysisIds)
             ) {
-              // Notify callback
+              // Notify callback with success status (default false for backwards compatibility)
               if (this.batchCompletedCallback) {
-                this.batchCompletedCallback(body.analysisIds);
+                this.batchCompletedCallback(
+                  body.analysisIds,
+                  body.success ?? false,
+                );
               }
             }
 

package/analyzer-template/project/orchestrateCapture/SequentialCaptureTaskRunner.ts

@@ -49,7 +49,10 @@ export interface SequentialCaptureTaskRunnerContext {
 
 export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
   private queue: string[][] = [];
-  private batchCompletedCallback?: (analysisIds: string[]) => void;
+  private batchCompletedCallback?: (
+    analysisIds: string[],
+    success: boolean,
+  ) => void;
   private isProcessing = false;
   private isCleanedUp = false;
   private processingLoop: Promise<void> | null = null;
@@ -94,7 +97,13 @@ export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
 
     // In CAPTURE_ONLY mode, run directly from the main project folder (skip rsync)
     // This preserves any manual edits made to generated files
-    const captureOnly = process.env.CAPTURE_ONLY === 'true';
+    const captureOnlyEnvValue = process.env.CAPTURE_ONLY;
+    const captureOnly = captureOnlyEnvValue === 'true';
+
+    console.log(
+      `CodeYam: CAPTURE_ONLY env = ${captureOnlyEnvValue === undefined ? 'undefined' : `"${captureOnlyEnvValue}"`}`,
+    );
+
     const capturerPath = captureOnly
       ? parentDir
       : path.normalize(path.join(parentDir, 'capturer-1'));
@@ -191,15 +200,15 @@ export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
       try {
         await this.processBatch(batch);
 
-        // Notify callback of completion
+        // Notify callback of successful completion
         if (this.batchCompletedCallback) {
-          this.batchCompletedCallback(batch);
+          this.batchCompletedCallback(batch, true);
         }
       } catch (error) {
         console.error('CodeYam: Error processing batch:', error);
-        // Still notify callback so orchestration can track failures
+        // Notify callback of failure so orchestration can track retries
         if (this.batchCompletedCallback) {
-          this.batchCompletedCallback(batch);
+          this.batchCompletedCallback(batch, false);
         }
       } finally {
         this.isProcessing = false;
@@ -393,7 +402,9 @@ export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
     console.log('CodeYam: Sequential task runner cleanup complete');
   }
 
-  onBatchCompleted(callback: (analysisIds: string[]) => void): void {
+  onBatchCompleted(
+    callback: (analysisIds: string[], success: boolean) => void,
+  ): void {
     this.batchCompletedCallback = callback;
   }
 }

package/analyzer-template/project/orchestrateCapture/taskRunner.ts

@@ -115,7 +115,9 @@ export interface CaptureTaskRunner {
    * Register callback for when workers complete batch processing.
    *
    * Critical for retry logic and progress tracking in orchestration.
-   * Callback receives the analysis IDs that were completed.
+   * Callback receives the analysis IDs and whether the batch succeeded.
    */
-  onBatchCompleted(callback: (analysisIds: string[]) => void): void;
+  onBatchCompleted(
+    callback: (analysisIds: string[], success: boolean) => void,
+  ): void;
 }

package/analyzer-template/project/orchestrateCapture.ts

@@ -32,7 +32,10 @@ import type {
 } from './orchestrateCapture/taskRunner';
 import KyselyAnalysisLoader from './orchestrateCapture/KyselyAnalysisLoader';
 
-const MAX_BATCH_SIZE = 6;
+// Temporarily set to 1 (effectively disabling batching) to work around reliability
+// issues where multi-analysis batches cause cascading failures. Revisit once we
+// reassess the SQS protocol and worker error handling.
+const MAX_BATCH_SIZE = 1;
 const POLLING_INTERVAL = 30 * 1000; // 30 seconds
 const TASK_HEALTH_CHECK_INTERVAL = 60 * 1000; // 1 minute
 const ANALYSIS_TIMEOUT = 20 * 60 * 1000;
@@ -99,15 +102,20 @@ export default async function orchestrateCapture({
   const attemptCount = new Map<string, number>();
   const maxAttempts = 2;
 
+  // Track which analyses had a worker report success — only these can be "captured"
+  const confirmedSuccessAnalyses = new Set<string>();
+
   // Register batch completion callback with task runner
-  taskRunner.onBatchCompleted((analysisIds: string[]) => {
+  taskRunner.onBatchCompleted((analysisIds: string[], success: boolean) => {
     console.log(
-      `CodeYam Capture: Received batch completion for ${analysisIds.length} analyses: ${analysisIds.join(', ')}`,
+      `CodeYam Capture: Received batch completion (success=${success}) for ${analysisIds.length} analyses: ${analysisIds.join(', ')}`,
     );
-    // Remove from inFlight and increment attempt count for each analysis
     for (const analysisId of analysisIds) {
       inFlight.delete(analysisId);
       attemptCount.set(analysisId, (attemptCount.get(analysisId) || 0) + 1);
+      if (success) {
+        confirmedSuccessAnalyses.add(analysisId);
+      }
     }
   });
 
@@ -195,9 +203,13 @@ export default async function orchestrateCapture({
         knownAnalyses.add(id);
       }
 
-      // an analysis we've seen, but is no longer "ready to be captured", is captured!
+      // An analysis is "captured" only if:
+      // 1. A worker reported success for its batch (in confirmedSuccessAnalyses)
+      // 2. It no longer appears in the ready-to-be-captured list (DB confirms)
+      // This prevents transient empty DB results from inflating the captured count.
       const capturedAnalysisIds = Array.from(knownAnalyses).filter(
-        (id) => !allReadyAnalysisIds.includes(id),
+        (id) =>
+          confirmedSuccessAnalyses.has(id) && !allReadyAnalysisIds.includes(id),
       );
 
       // Remove captured analyses from inFlight tracking
@@ -355,6 +367,32 @@ export default async function orchestrateCapture({
         }
       }
 
+      // Early exit when all enqueued work is settled (captured or abandoned)
+      // This avoids waiting for the 20-minute ANALYSIS_TIMEOUT when capture is complete
+      // IMPORTANT: Only check this when isDraining (analysis phase complete) to avoid
+      // exiting while slow-running analyses are still in progress
+      if (
+        isDraining &&
+        !firstLoop &&
+        enqueuedAnalyses.size > 0 &&
+        inFlight.size === 0 &&
+        remaining.length === 0 &&
+        toRetry.length === 0
+      ) {
+        const allAnalysesSettled = Array.from(enqueuedAnalyses).every((id) => {
+          const captured = !allReadyAnalysisIds.includes(id);
+          const abandoned = (attemptCount.get(id) || 0) >= maxAttempts;
+          return captured || abandoned;
+        });
+
+        if (allAnalysesSettled) {
+          console.log(
+            'CodeYam Capture: All enqueued analyses settled (captured or abandoned), exiting early',
+          );
+          break;
+        }
+      }
+
       if (isDraining) {
         // Exit if analysis complete but no entities ready for capture
         // Only check this after at least one poll (not firstLoop) to ensure we've
@@ -371,7 +409,9 @@ export default async function orchestrateCapture({
           // check if all enqueued analyses have been either captured or abandoned
           const allAnalysesSettled = Array.from(enqueuedAnalyses).every(
             (id) => {
-              const captured = !allReadyAnalysisIds.includes(id);
+              const captured =
+                confirmedSuccessAnalyses.has(id) &&
+                !allReadyAnalysisIds.includes(id);
               const abandoned = (attemptCount.get(id) || 0) >= maxAttempts;
               return captured || abandoned;
             },
@@ -411,5 +451,33 @@ export default async function orchestrateCapture({
   } finally {
     // Always clean up task runner resources in finally block
     await taskRunner.cleanup();
+
+    const trackingCommit = metadataCommit ?? commit;
+    if (trackingCommit?.id) {
+      const captureCompletedAt = new Date().toISOString();
+
+      await updateCommitMetadata({
+        commitId: trackingCommit.id,
+        runStatusUpdate: {
+          captureCompletedAt,
+          capturePid: undefined,
+        },
+        // completedAt = max(analysisCompletedAt, captureCompletedAt)
+        // Normally capture finishes after analysis, but using max means illogical
+        // orderings leave a detectable signal (completedAt !== captureCompletedAt).
+        updateCallback(metadata) {
+          const run = metadata.currentRun!;
+          run.completedAt =
+            run.analysisCompletedAt &&
+            run.analysisCompletedAt > captureCompletedAt
+              ? run.analysisCompletedAt
+              : captureCompletedAt;
+        },
+      }).catch((error) => {
+        console.log(
+          `Warning: Failed to update capture completion metadata: ${error}`,
+        );
+      });
+    }
   }
 }