@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.2a88920

package/analyzer-template/project/loadReadyToBeCaptured.ts

@@ -7,6 +7,28 @@ import {
 import { Analysis, Project } from '~codeyam/types';
 import { LazyFileStore } from './LazyFileStore';
 
+const DB_CONCURRENCY = 1;
+
+async function mapWithConcurrency<T, R>(
+  items: T[],
+  fn: (item: T) => Promise<R>,
+  concurrency: number,
+): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let index = 0;
+  const workers = Array.from(
+    { length: Math.min(concurrency, items.length) },
+    async () => {
+      while (index < items.length) {
+        const i = index++;
+        results[i] = await fn(items[i]);
+      }
+    },
+  );
+  await Promise.all(workers);
+  return results;
+}
+
 interface LoadReadyToBeCapturedArgs {
   project: Project;
   branchCommitSha?: string;
@@ -77,8 +99,9 @@ async function gatherAllReferencedEntities(
       byFilePath.get(filePath)!.push(name);
     }
 
-    // Batch load entities by file path
-    const loadPromises = Array.from(byFilePath.entries()).map(
+    // Batch load entities by file path (concurrency-limited to avoid exhausting DB connections)
+    const results = await mapWithConcurrency(
+      Array.from(byFilePath.entries()),
       async ([filePath, names]) => {
         const entities = await loadEntities({
           projectId: project.id,
@@ -106,9 +129,8 @@ async function gatherAllReferencedEntities(
           return false;
         });
       },
+      DB_CONCURRENCY,
     );
-
-    const results = await Promise.all(loadPromises);
     const loadedEntities = results.flat();
 
     // Add loaded entities to context
@@ -198,48 +220,50 @@ export default async function loadReadyToBeCaptured({
     foundNewDependencies = false;
     const currentBatch = Array.from(stillNeeded);
 
-    // Load analyses for current batch
-    const loadPromises = currentBatch.map(async (keyString) => {
-      const [filePath, entityName] = keyString.split(':');
-
-      // Find the file ID for this file path - use fileStore for O(1) lookup when available
-      const file = fileStore
-        ? fileStore.getByPath(filePath)
-        : project.files.find((f) => f.path === filePath);
-      if (!file) {
-        console.log(
-          `CodeYam Warning: File not found in project: ${filePath} for key ${keyString}`,
-        );
-        return null;
-      }
-
-      try {
-        const analysis = await loadAnalysis({
-          projectId: project.id,
-          fileId: file.id,
-          entityName,
-          includeScenarios: isFirstBatch, // Only include scenarios for the root analyses
-        });
-
-        if (analysis) {
-          console.log(`CodeYam: Loaded analysis ${filePath}:${entityName}`);
-          return { key: keyString, analysis };
-        } else {
+    // Load analyses for current batch (concurrency-limited to avoid exhausting DB connections)
+    const results = await mapWithConcurrency(
+      currentBatch,
+      async (keyString) => {
+        const [filePath, entityName] = keyString.split(':');
+
+        // Find the file ID for this file path - use fileStore for O(1) lookup when available
+        const file = fileStore
+          ? fileStore.getByPath(filePath)
+          : project.files.find((f) => f.path === filePath);
+        if (!file) {
           console.log(
-            `CodeYam Warning: Analysis not found: ${filePath}:${entityName}`,
+            `CodeYam Warning: File not found in project: ${filePath} for key ${keyString}`,
           );
           return null;
         }
-      } catch (error) {
-        console.log(
-          `CodeYam Error loading analysis ${filePath}:${entityName}:`,
-          error,
-        );
-        return null;
-      }
-    });
 
-    const results = await Promise.all(loadPromises);
+        try {
+          const analysis = await loadAnalysis({
+            projectId: project.id,
+            fileId: file.id,
+            entityName,
+            includeScenarios: isFirstBatch, // Only include scenarios for the root analyses
+          });
+
+          if (analysis) {
+            console.log(`CodeYam: Loaded analysis ${filePath}:${entityName}`);
+            return { key: keyString, analysis };
+          } else {
+            console.log(
+              `CodeYam Warning: Analysis not found: ${filePath}:${entityName}`,
+            );
+            return null;
+          }
+        } catch (error) {
+          console.log(
+            `CodeYam Error loading analysis ${filePath}:${entityName}:`,
+            error,
+          );
+          return null;
+        }
+      },
+      DB_CONCURRENCY,
+    );
 
     // Process loaded analyses and find new dependencies
     for (const result of results) {

package/analyzer-template/project/orchestrateCapture/AwsCaptureTaskRunner.ts

@@ -35,7 +35,10 @@ export default class AwsCaptureTaskRunner implements CaptureTaskRunner {
   private sqs: SQSClient;
   private queueInfo: QueueInfo;
   private taskArns: string[];
-  private batchCompletedCallback?: (analysisIds: string[]) => void;
+  private batchCompletedCallback?: (
+    analysisIds: string[],
+    success: boolean,
+  ) => void;
   private controlPollingInterval?: NodeJS.Timeout;
   private lastTaskHealthCheck = Date.now();
   private isCleanedUp = false;
@@ -213,7 +216,9 @@ export default class AwsCaptureTaskRunner implements CaptureTaskRunner {
     }
   }
 
-  onBatchCompleted(callback: (analysisIds: string[]) => void): void {
+  onBatchCompleted(
+    callback: (analysisIds: string[], success: boolean) => void,
+  ): void {
     this.batchCompletedCallback = callback;
   }
 
@@ -255,9 +260,12 @@ export default class AwsCaptureTaskRunner implements CaptureTaskRunner {
               body.analysisIds &&
               Array.isArray(body.analysisIds)
             ) {
-              // Notify callback
+              // Notify callback with success status (default false for backwards compatibility)
               if (this.batchCompletedCallback) {
-                this.batchCompletedCallback(body.analysisIds);
+                this.batchCompletedCallback(
+                  body.analysisIds,
+                  body.success ?? false,
+                );
               }
             }
 

package/analyzer-template/project/orchestrateCapture/SequentialCaptureTaskRunner.ts

@@ -49,7 +49,10 @@ export interface SequentialCaptureTaskRunnerContext {
 
 export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
   private queue: string[][] = [];
-  private batchCompletedCallback?: (analysisIds: string[]) => void;
+  private batchCompletedCallback?: (
+    analysisIds: string[],
+    success: boolean,
+  ) => void;
   private isProcessing = false;
   private isCleanedUp = false;
   private processingLoop: Promise<void> | null = null;
@@ -94,7 +97,13 @@ export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
 
     // In CAPTURE_ONLY mode, run directly from the main project folder (skip rsync)
     // This preserves any manual edits made to generated files
-    const captureOnly = process.env.CAPTURE_ONLY === 'true';
+    const captureOnlyEnvValue = process.env.CAPTURE_ONLY;
+    const captureOnly = captureOnlyEnvValue === 'true';
+
+    console.log(
+      `CodeYam: CAPTURE_ONLY env = ${captureOnlyEnvValue === undefined ? 'undefined' : `"${captureOnlyEnvValue}"`}`,
+    );
+
     const capturerPath = captureOnly
       ? parentDir
       : path.normalize(path.join(parentDir, 'capturer-1'));
@@ -191,15 +200,15 @@ export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
       try {
         await this.processBatch(batch);
 
-        // Notify callback of completion
+        // Notify callback of successful completion
         if (this.batchCompletedCallback) {
-          this.batchCompletedCallback(batch);
+          this.batchCompletedCallback(batch, true);
         }
       } catch (error) {
         console.error('CodeYam: Error processing batch:', error);
-        // Still notify callback so orchestration can track failures
+        // Notify callback of failure so orchestration can track retries
         if (this.batchCompletedCallback) {
-          this.batchCompletedCallback(batch);
+          this.batchCompletedCallback(batch, false);
         }
       } finally {
         this.isProcessing = false;
@@ -393,7 +402,9 @@ export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
     console.log('CodeYam: Sequential task runner cleanup complete');
   }
 
-  onBatchCompleted(callback: (analysisIds: string[]) => void): void {
+  onBatchCompleted(
+    callback: (analysisIds: string[], success: boolean) => void,
+  ): void {
     this.batchCompletedCallback = callback;
   }
 }

package/analyzer-template/project/orchestrateCapture/taskRunner.ts

@@ -115,7 +115,9 @@ export interface CaptureTaskRunner {
    * Register callback for when workers complete batch processing.
    *
    * Critical for retry logic and progress tracking in orchestration.
-   * Callback receives the analysis IDs that were completed.
+   * Callback receives the analysis IDs and whether the batch succeeded.
    */
-  onBatchCompleted(callback: (analysisIds: string[]) => void): void;
+  onBatchCompleted(
+    callback: (analysisIds: string[], success: boolean) => void,
+  ): void;
 }

package/analyzer-template/project/orchestrateCapture.ts

@@ -99,15 +99,20 @@ export default async function orchestrateCapture({
   const attemptCount = new Map<string, number>();
   const maxAttempts = 2;
 
+  // Track which analyses had a worker report success — only these can be "captured"
+  const confirmedSuccessAnalyses = new Set<string>();
+
   // Register batch completion callback with task runner
-  taskRunner.onBatchCompleted((analysisIds: string[]) => {
+  taskRunner.onBatchCompleted((analysisIds: string[], success: boolean) => {
     console.log(
-      `CodeYam Capture: Received batch completion for ${analysisIds.length} analyses: ${analysisIds.join(', ')}`,
+      `CodeYam Capture: Received batch completion (success=${success}) for ${analysisIds.length} analyses: ${analysisIds.join(', ')}`,
     );
-    // Remove from inFlight and increment attempt count for each analysis
     for (const analysisId of analysisIds) {
       inFlight.delete(analysisId);
       attemptCount.set(analysisId, (attemptCount.get(analysisId) || 0) + 1);
+      if (success) {
+        confirmedSuccessAnalyses.add(analysisId);
+      }
     }
   });
 
@@ -195,9 +200,13 @@ export default async function orchestrateCapture({
         knownAnalyses.add(id);
       }
 
-      // an analysis we've seen, but is no longer "ready to be captured", is captured!
+      // An analysis is "captured" only if:
+      // 1. A worker reported success for its batch (in confirmedSuccessAnalyses)
+      // 2. It no longer appears in the ready-to-be-captured list (DB confirms)
+      // This prevents transient empty DB results from inflating the captured count.
       const capturedAnalysisIds = Array.from(knownAnalyses).filter(
-        (id) => !allReadyAnalysisIds.includes(id),
+        (id) =>
+          confirmedSuccessAnalyses.has(id) && !allReadyAnalysisIds.includes(id),
       );
 
       // Remove captured analyses from inFlight tracking
@@ -355,6 +364,32 @@ export default async function orchestrateCapture({
         }
       }
 
+      // Early exit when all enqueued work is settled (captured or abandoned)
+      // This avoids waiting for the 20-minute ANALYSIS_TIMEOUT when capture is complete
+      // IMPORTANT: Only check this when isDraining (analysis phase complete) to avoid
+      // exiting while slow-running analyses are still in progress
+      if (
+        isDraining &&
+        !firstLoop &&
+        enqueuedAnalyses.size > 0 &&
+        inFlight.size === 0 &&
+        remaining.length === 0 &&
+        toRetry.length === 0
+      ) {
+        const allAnalysesSettled = Array.from(enqueuedAnalyses).every((id) => {
+          const captured = !allReadyAnalysisIds.includes(id);
+          const abandoned = (attemptCount.get(id) || 0) >= maxAttempts;
+          return captured || abandoned;
+        });
+
+        if (allAnalysesSettled) {
+          console.log(
+            'CodeYam Capture: All enqueued analyses settled (captured or abandoned), exiting early',
+          );
+          break;
+        }
+      }
+
       if (isDraining) {
         // Exit if analysis complete but no entities ready for capture
         // Only check this after at least one poll (not firstLoop) to ensure we've
@@ -371,7 +406,9 @@ export default async function orchestrateCapture({
           // check if all enqueued analyses have been either captured or abandoned
           const allAnalysesSettled = Array.from(enqueuedAnalyses).every(
             (id) => {
-              const captured = !allReadyAnalysisIds.includes(id);
+              const captured =
+                confirmedSuccessAnalyses.has(id) &&
+                !allReadyAnalysisIds.includes(id);
               const abandoned = (attemptCount.get(id) || 0) >= maxAttempts;
               return captured || abandoned;
             },
@@ -411,5 +448,33 @@ export default async function orchestrateCapture({
   } finally {
     // Always clean up task runner resources in finally block
     await taskRunner.cleanup();
+
+    const trackingCommit = metadataCommit ?? commit;
+    if (trackingCommit?.id) {
+      const captureCompletedAt = new Date().toISOString();
+
+      await updateCommitMetadata({
+        commitId: trackingCommit.id,
+        runStatusUpdate: {
+          captureCompletedAt,
+          capturePid: undefined,
+        },
+        // completedAt = max(analysisCompletedAt, captureCompletedAt)
+        // Normally capture finishes after analysis, but using max means illogical
+        // orderings leave a detectable signal (completedAt !== captureCompletedAt).
+        updateCallback(metadata) {
+          const run = metadata.currentRun!;
+          run.completedAt =
+            run.analysisCompletedAt &&
+            run.analysisCompletedAt > captureCompletedAt
+              ? run.analysisCompletedAt
+              : captureCompletedAt;
+        },
+      }).catch((error) => {
+        console.log(
+          `Warning: Failed to update capture completion metadata: ${error}`,
+        );
+      });
+    }
   }
 }

package/analyzer-template/project/reconcileMockDataKeys.ts

@@ -7,6 +7,9 @@
  *
  * Keys are now original call signatures (e.g., "useFetcher<User>()", "db.select(query)").
  *
+ * Also handles flat dotted keys (e.g., "trpc.fastener.getAttributeValues.useQuery(...)")
+ * by converting them to nested structure.
+ *
  * @param mockData The mock data object to reconcile
  * @param expectedKeys The expected keys from the structure
  */
@@ -29,6 +32,26 @@ export function reconcileMockDataKeys(
       continue;
     }
 
+    // Check if this is a flat dotted key that should be converted to nested structure
+    // e.g., "trpc.fastener.getAttributeValues.useQuery({ ... })"
+    const flatKeyParts = parseFlatDottedKey(key);
+    if (flatKeyParts && flatKeyParts.length > 1) {
+      // Check if the parts match expected keys (indicating this should be nested)
+      const partsMatchExpected = flatKeyParts.some((part) =>
+        expectedKeys.includes(part),
+      );
+      if (partsMatchExpected) {
+        // Convert flat key to nested structure and merge
+        deepMergeIntoResult(
+          result,
+          flatKeyParts,
+          reconciledValue,
+          expectedKeys,
+        );
+        continue;
+      }
+    }
+
     // Try to find closest match using existing logic
     const closestKey = findClosestKey(key, expectedKeys);
     if (closestKey) {
@@ -42,6 +65,94 @@ export function reconcileMockDataKeys(
   return result;
 }
 
+/**
+ * Parse a flat dotted key into path segments.
+ * Handles keys like "trpc.fastener.getAttributeValues.useQuery({ ... })"
+ * where the last part might contain parentheses with function arguments.
+ *
+ * Returns null if the key doesn't look like a dotted path.
+ */
+function parseFlatDottedKey(key: string): string[] | null {
+  // Don't treat keys that start with a function call as dotted paths
+  // e.g., "useQuery({ id: 123 })" should not be parsed
+  if (key.match(/^[a-zA-Z_][a-zA-Z0-9_]*\s*\(/)) {
+    // Check if there's a dot BEFORE the first parenthesis
+    const parenIndex = key.indexOf('(');
+    const dotBeforeParen = key.slice(0, parenIndex).includes('.');
+    if (!dotBeforeParen) {
+      return null;
+    }
+  }
+
+  // Must contain a dot to be a dotted path
+  if (!key.includes('.')) {
+    return null;
+  }
+
+  const parts: string[] = [];
+  let current = '';
+  let depth = 0;
+
+  for (let i = 0; i < key.length; i++) {
+    const char = key[i];
+
+    if (char === '(' || char === '{' || char === '[') {
+      depth++;
+      current += char;
+    } else if (char === ')' || char === '}' || char === ']') {
+      depth--;
+      current += char;
+    } else if (char === '.' && depth === 0) {
+      if (current) {
+        parts.push(current);
+      }
+      current = '';
+    } else {
+      current += char;
+    }
+  }
+
+  if (current) {
+    parts.push(current);
+  }
+
+  // Need at least 2 parts for it to be a meaningful dotted path
+  return parts.length >= 2 ? parts : null;
+}
+
+/**
+ * Deep merge a value into the result object following the path segments.
+ * Reconciles the leaf key against expected keys.
+ */
+function deepMergeIntoResult(
+  result: Record<string, unknown>,
+  parts: string[],
+  value: unknown,
+  expectedKeys: string[],
+): void {
+  let current = result;
+
+  // Navigate/create path for all but the last part
+  for (let i = 0; i < parts.length - 1; i++) {
+    const part = parts[i];
+    if (!(part in current)) {
+      current[part] = {};
+    }
+    const next = current[part];
+    if (next && typeof next === 'object' && !Array.isArray(next)) {
+      current = next as Record<string, unknown>;
+    } else {
+      // Can't traverse further, structure mismatch
+      return;
+    }
+  }
+
+  // Handle the leaf key
+  const leafKey = parts[parts.length - 1];
+  const closestLeafKey = findClosestKey(leafKey, expectedKeys) || leafKey;
+  current[closestLeafKey] = value;
+}
+
 /**
  * Find the closest matching key from expectedKeys for a given key.
  *
@@ -100,38 +211,70 @@ function findClosestKey(key: string, expectedKeys: string[]): string | null {
   }
 
   // If key has object literal arguments (like "useQuery({ id: 123, ... })"),
-  // try matching expected keys with the same function name and object arguments.
+  // try matching expected keys with the same function path AND first object property.
   // This handles cases where LLM generates literal values but expected keys have variable references.
-  // e.g., "useQuery({ id: 123 })" matches "useQuery({ id: Number(id) })"
+  // e.g., "useQuery({ attributes: ['a', 'b'] })" matches "useQuery({ attributes: items || [] })"
   // e.g., "trpc.users.getById.useQuery({ userId: 'user-123' })" matches "trpc.users.getById.useQuery({ userId })"
+  // Uses first property matching when multiple candidates exist for disambiguation.
   const objectArgMatch = key.match(/^([a-zA-Z_][a-zA-Z0-9_.]*)\s*\(\s*\{/);
   if (objectArgMatch) {
     const funcPath = objectArgMatch[1]; // Can be "funcName" or "path.to.funcName"
+    // Extract the first property name from the object argument
+    const firstPropMatch = key.match(
+      /\(\s*\{\s*\n?\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*:/,
+    );
+    const firstProp = firstPropMatch ? firstPropMatch[1] : null;
+
+    // Collect all matching candidates
+    const candidates: string[] = [];
     for (const expected of expectedKeys) {
       // Match if expected key has the same function path and starts with object argument
       const expectedObjArgMatch = expected.match(
         new RegExp(`^${escapeRegExp(funcPath)}\\s*\\(\\s*\\{`),
       );
       if (expectedObjArgMatch) {
-        return expected;
+        candidates.push(expected);
       }
     }
+
+    // If we have a first property, prefer matching by property name
+    if (firstProp && candidates.length > 1) {
+      // Look for a candidate with the same first property
+      const propRegex = new RegExp(
+        `\\(\\s*\\{[\\s\\n]*${escapeRegExp(firstProp)}\\s*:`,
+      );
+      const propMatch = candidates.find((c) => propRegex.test(c));
+      if (propMatch) {
+        return propMatch;
+      }
+    }
+
+    // Fall back to first candidate if no property match found
+    if (candidates.length > 0) {
+      return candidates[0];
+    }
   }
 
-  // If key has a single argument with a literal value (string, number, etc.),
+  // If key has a single argument (literal value OR variable reference),
   // try matching expected keys with the same function name but different argument.
-  // This handles cases where LLM generates "useCustomSizes('example-slug')" but expected
-  // key is "useCustomSizes(projectSlug)" with a variable reference.
-  // e.g., "useCustomSizes('example-slug')" matches "useCustomSizes(projectSlug)"
+  // This handles cases where:
+  // - LLM generates "useCustomSizes('example-slug')" but expected key is "useCustomSizes(projectSlug)"
+  // - LLM generates "useCustomSizes(someProjectSlug)" but expected key is "useCustomSizes(projectSlug)"
+  //
+  // The second case is important because when LLM generates multiple keys with different
+  // variable names (e.g., "useCustomSizes(someProjectSlug)" and "useCustomSizes(projectSlug)"),
+  // without reconciliation, normalizeMockData merges them into "useCustomSizes()" which
+  // doesn't match the expected key "useCustomSizes(projectSlug)", causing undefined data access.
   const singleArgMatch = key.match(
     /^([a-zA-Z_][a-zA-Z0-9_.]*)\s*\(([^,)]+)\)$/,
   );
   if (singleArgMatch) {
     const funcPath = singleArgMatch[1];
     const arg = singleArgMatch[2].trim();
-    // Only apply this matching if the argument looks like a literal (quoted string or number)
+    // Apply this matching if the argument is a literal OR a simple variable reference
     const isLiteralArg = /^['"`].*['"`]$/.test(arg) || /^\d+$/.test(arg);
-    if (isLiteralArg) {
+    const isVariableRef = /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(arg);
+    if (isLiteralArg || isVariableRef) {
       for (const expected of expectedKeys) {
         // Match if expected key has the same function path with a single argument
         const expectedSingleArgMatch = expected.match(

package/analyzer-template/project/runAnalysis.ts

@@ -313,6 +313,10 @@ export default async function runAnalysis({
   // Route to appropriate analyzer
   try {
     if (runReadyToBeCapturedPath) {
+      // Signal that analysis can produce captures immediately since analyses are already ready.
+      // This is needed for recapture to work - orchestration waits for the 'analysisProducing' event.
+      controller?.setAnalysisProducingPhase();
+
       return await analyzeReadyToBeCaptured({
         project,
         projectAnalyzer,