@codeyam/codeyam-cli 0.1.0-staging.76566f9 → 0.1.0-staging.790201e

package/analyzer-template/packages/analyze/src/lib/files/scenarios/gatherDataForMocks.ts

@@ -1,7 +1,8 @@
-// import { writeFile } from 'fs/promises';
 import {
   convertDotNotation,
   fillInDirectSchemaGapsAndUnknowns,
+  buildSchemaIndexes,
+  type SchemaIndexes,
   joinParenthesesAndArrays,
   mergeJsonTypeDefinitions,
   splitOutsideParenthesesAndArrays,
@@ -166,6 +167,7 @@ function processCall(
   variableNameOccurrence: number | undefined,
   options: GatherDataForMocksOptions | undefined,
   hookCallIndex: number,
+  prebuiltIndexes?: SchemaIndexes,
 ): { key: string; value: JsonTypeDefinition } | null {
   const callName = importedExport.calls?.[callIndex] ?? importedExport.name;
 
@@ -270,21 +272,87 @@ function processCall(
     baseHasMultipleTypeParamVariants = typeParamVariantsInBase.size > 1;
   }
 
-  // For type-parameterized calls where the base schema has MULTIPLE different type
-  // parameter variants, use ONLY the call-specific schema to avoid contamination.
-  // Otherwise, merge with base schema for complete coverage (e.g., array element details).
-  const mergedSchema =
-    hasCallSpecificSchema && hasTypeParams && baseHasMultipleTypeParamVariants
-      ? rawSchema // Use ONLY the call-specific schema when base has mixed type params
-      : Object.keys(rawSchema).length === 0
-        ? baseSchema
-        : { ...baseSchema, ...rawSchema };
+  // For type-parameterized calls where we have a call-specific schema, we need to
+  // be careful about merging with the base schema. The base schema may contain
+  // merged data from ALL type parameter variants, which would pollute the result.
+  let mergedSchema: Record<string, string>;
+  if (hasCallSpecificSchema && hasTypeParams) {
+    if (baseHasMultipleTypeParamVariants) {
+      // Base schema has paths with different type parameters - filter to only
+      // include paths matching the current type parameter.
+      const typeParamMatch = callName.match(/<([^>]+)>/);
+      const currentTypeParam = typeParamMatch ? typeParamMatch[1] : null;
+      const baseFunctionName = cleanFunctionName(callName.split('(')[0]);
+
+      if (currentTypeParam) {
+        // Filter base schema to only include paths that:
+        // 1. Match current type param (e.g., useLoaderData<LoaderData>())
+        // 2. OR don't have any type param (generic paths like useLoaderData())
+        const filteredBaseSchema: Record<string, string> = {};
+        for (const [path, type] of Object.entries(baseSchema)) {
+          // Check if path has a type parameter for this function
+          const pathTypeParamMatch = path.match(
+            new RegExp(`^${escapeRegExp(baseFunctionName)}<([^>]+)>\\(`),
+          );
+          if (pathTypeParamMatch) {
+            // Path has a type param - only include if it matches current
+            if (pathTypeParamMatch[1] === currentTypeParam) {
+              filteredBaseSchema[path] = type;
+            }
+            // Otherwise skip (it's for a different type param variant)
+          } else {
+            // Path doesn't have a type param for this function - include it
+            // (could be generic paths or paths for other functions)
+            filteredBaseSchema[path] = type;
+          }
+        }
+        mergedSchema = { ...filteredBaseSchema, ...rawSchema };
+      } else {
+        // Couldn't extract type param - fall back to rawSchema only
+        mergedSchema = rawSchema;
+      }
+    } else {
+      // Base schema has zero or one type-parameterized variant.
+      // When there's only one variant, the base schema paths are safe to merge
+      // (no contamination from other variants). This is important for cases like:
+      // - Array element details (gitStatus[].path) that are in base but not call-specific
+      // - Enriched paths that should be preserved
+      // Filter to only include paths matching current type param or generic paths.
+      const typeParamMatch = callName.match(/<([^>]+)>/);
+      const currentTypeParam = typeParamMatch ? typeParamMatch[1] : null;
+      const baseFunctionName = cleanFunctionName(callName.split('(')[0]);
+
+      if (currentTypeParam) {
+        const filteredBaseSchema: Record<string, string> = {};
+        for (const [path, type] of Object.entries(baseSchema)) {
+          const pathTypeParamMatch = path.match(
+            new RegExp(`^${escapeRegExp(baseFunctionName)}<([^>]+)>\\(`),
+          );
+          if (pathTypeParamMatch) {
+            // Path has a type param - only include if it matches current
+            if (pathTypeParamMatch[1] === currentTypeParam) {
+              filteredBaseSchema[path] = type;
+            }
+          } else {
+            // Path doesn't have a type param - include generic paths
+            filteredBaseSchema[path] = type;
+          }
+        }
+        mergedSchema = { ...filteredBaseSchema, ...rawSchema };
+      } else {
+        // No type param to extract - merge normally
+        mergedSchema = { ...baseSchema, ...rawSchema };
+      }
+    }
+  } else if (Object.keys(rawSchema).length === 0) {
+    mergedSchema = baseSchema;
+  } else {
+    mergedSchema = { ...baseSchema, ...rawSchema };
+  }
 
   // Preprocess to filter signatures and extract functionCallReturnValue
   let preprocessedSchema = preprocessSchemaForMocks(mergedSchema);
 
-  const functionNames: string[] = [];
-
   const isFunctionNameWithoutFunction = (
     path: string,
     functionName: string,
@@ -369,38 +437,66 @@ function processCall(
     }
   }
 
-  const relevantMergedDependencySchema = Object.keys(preprocessedSchema).reduce(
-    (acc, path) => {
-      if (path.endsWith(')')) {
-        const pathParts = splitOutsideParenthesesAndArrays(path);
-        const functionName = joinParenthesesAndArrays([
-          ...pathParts.slice(0, -1),
-          pathParts[pathParts.length - 1].split('(')[0],
-        ]);
-        functionNames.push(functionName);
-        for (const existingPath in acc) {
-          if (isFunctionNameWithoutFunction(existingPath, functionName)) {
-            delete acc[existingPath];
-          }
-        }
-      }
+  // Two-pass function name filtering to avoid O(k²) retroactive deletion.
+  // Pass 1: Discover all unique function names from paths ending with ')'
+  const functionNameSet = new Set<string>();
+  const schemaKeys = Object.keys(preprocessedSchema);
+  for (const path of schemaKeys) {
+    if (path.endsWith(')')) {
+      const pathParts = splitOutsideParenthesesAndArrays(path);
+      const functionName = joinParenthesesAndArrays([
+        ...pathParts.slice(0, -1),
+        pathParts[pathParts.length - 1].split('(')[0],
+      ]);
+      functionNameSet.add(functionName);
+    }
+  }
 
-      if (functionNames.some((fn) => isFunctionNameWithoutFunction(path, fn))) {
-        return acc;
-      }
+  // Index function names by first segment for O(1) bucket lookup instead of
+  // checking all function names for every path. Most paths only need to check
+  // 1-2 function names instead of all F.
+  const fnByFirstSegment = new Map<string, string[]>();
+  for (const fn of functionNameSet) {
+    const dotIdx = fn.indexOf('.');
+    const firstSeg = dotIdx >= 0 ? fn.slice(0, dotIdx) : fn;
+    let bucket = fnByFirstSegment.get(firstSeg);
+    if (!bucket) {
+      bucket = [];
+      fnByFirstSegment.set(firstSeg, bucket);
+    }
+    bucket.push(fn);
+  }
 
-      acc[path] = preprocessedSchema[path];
-      return acc;
-    },
-    {} as Record<string, string>,
-  );
+  // Pass 2: Filter paths that match any discovered function name prefix.
+  // Use indexed lookup: extract path's first segment, check only matching function names.
+  const relevantMergedDependencySchema: Record<string, string> = {};
+  for (const path of schemaKeys) {
+    // Extract first segment of path (before first '.', '(', or '<')
+    const dotIdx = path.indexOf('.');
+    const parenIdx = path.indexOf('(');
+    const angleIdx = path.indexOf('<');
+    let minIdx = path.length;
+    if (dotIdx >= 0 && dotIdx < minIdx) minIdx = dotIdx;
+    if (parenIdx >= 0 && parenIdx < minIdx) minIdx = parenIdx;
+    if (angleIdx >= 0 && angleIdx < minIdx) minIdx = angleIdx;
+    const firstSeg = path.slice(0, minIdx);
+
+    const candidates = fnByFirstSegment.get(firstSeg);
+    if (
+      !candidates ||
+      !candidates.some((fn) => isFunctionNameWithoutFunction(path, fn))
+    ) {
+      relevantMergedDependencySchema[path] = preprocessedSchema[path];
+    }
+  }
 
-  const schema = convertDotNotation(
-    fillInDirectSchemaGapsAndUnknowns({
-      scopeName: importedExport.name,
-      schema: relevantMergedDependencySchema,
-    }),
-  );
+  const filledSchema = fillInDirectSchemaGapsAndUnknowns({
+    scopeName: importedExport.name,
+    schema: relevantMergedDependencySchema,
+    prebuiltIndexes,
+  });
+
+  const schema = convertDotNotation(filledSchema);
 
   // Determine if this is an object-style or hook-style dependency:
   // - Hook-style: calls = ['useAuth()'] - call starts with name + '('
@@ -459,17 +555,6 @@ function processCall(
     // Use method-chain result if found, otherwise keep previous returnValue
     if (current !== undefined) {
       returnValue = current as typeof returnValue;
-      // Log when method-chain navigation succeeds for supabase-style calls
-      if (callName.includes('createClient') || callName.includes('from(')) {
-        console.log(
-          'CODEYAM: Method-chain navigation succeeded for:',
-          callName,
-          '-> keys:',
-          typeof returnValue === 'object' && returnValue
-            ? Object.keys(returnValue as object).slice(0, 5)
-            : typeof returnValue,
-        );
-      }
     }
   }
 
@@ -512,6 +597,33 @@ export default function gatherDataForMocks(
   dependencySchemas: DataStructure['dependencySchemas'],
   options?: GatherDataForMocksOptions,
 ) {
+  // Build schema indexes ONCE from all dependency schemas combined.
+  // This avoids rebuilding indexes (~1-2 seconds for 18k keys) for each processCall.
+  //
+  // IMPORTANT: We must preprocess schemas BEFORE building indexes!
+  // The schemas contain .functionCallReturnValue paths that get stripped during processing.
+  // If we build indexes from original paths, lookups with preprocessed paths will fail.
+  // Example: 'hook().functionCallReturnValue.x.length' → index maps 'hook().functionCallReturnValue.x' → ['length']
+  //          But preprocessed path 'hook().x.length' looks for 'hook().x' which doesn't exist in the index.
+  const allSchemaKeys: Record<string, string> = {};
+  if (dependencySchemas) {
+    for (const filePath in dependencySchemas) {
+      for (const entityName in dependencySchemas[filePath]) {
+        const schema =
+          dependencySchemas[filePath][entityName]?.returnValueSchema;
+        if (schema) {
+          // Preprocess each schema to remove .functionCallReturnValue before collecting
+          const preprocessed = preprocessSchemaForMocks(schema);
+          Object.assign(allSchemaKeys, preprocessed);
+        }
+      }
+    }
+  }
+  const prebuiltIndexes =
+    Object.keys(allSchemaKeys).length > 0
+      ? buildSchemaIndexes(allSchemaKeys)
+      : undefined;
+
   // Track the global index for each hook name across all imports
   // This enables canonical keys like EntityName::useLoaderData::0, EntityName::useFetcher::0, etc.
   const hookCallIndices: Record<string, number> = {};
@@ -537,7 +649,108 @@ export default function gatherDataForMocks(
         importedExport.callVariableNames.length === directCallCount &&
         !allDirectCallsIdentical(importedExport.calls ?? []);
 
-      if (hasMultipleDistinctCalls) {
+      // Check for multiple distinct method-chain calls WITHOUT callVariableNames.
+      // This pattern is common with tRPC, Supabase, etc. where a single import
+      // (e.g., `trpc`) is used with multiple different method chains (e.g.,
+      // `trpc.useUtils()`, `trpc.quote.getQuotesByOrg.useQuery()`).
+      // These should each get their own entry in dataForMocks.
+      //
+      // IMPORTANT: This is DIFFERENT from the rest spread pattern where a function
+      // like useLoaderData is called directly, and other calls are method chains
+      // on its return value (e.g., `useLoaderData<typeof loader>()` with
+      // `useLoaderData<typeof loader>().uncommittedEntitiesList.filter(...)`).
+      // In that case, all the additional calls start with the base function call.
+      const directCalls = (importedExport.calls ?? []).filter(
+        (sig) => !sig.includes('.functionCallReturnValue.'),
+      );
+      const uniqueDirectCalls = [...new Set(directCalls)];
+
+      // Check if any call is a direct invocation of the import (e.g., `useLoaderData<typeof loader>()`)
+      // as opposed to all being property accesses (e.g., `trpc.useUtils()`, `trpc.quote.xxx()`).
+      // If there's a direct function call, the other calls are likely method chains on its result.
+      const baseName = importedExport.name;
+      const hasDirectFunctionCall = directCalls.some((sig) => {
+        // Pattern: baseName() or baseName<...>()
+        // This matches "useLoaderData()", "useLoaderData<typeof loader>()", etc.
+        // but NOT "trpc.useUtils()" which has a dot before the parens
+        const baseCallPattern = new RegExp(`^${baseName}(<[^>]*>)?\\(`);
+        return baseCallPattern.test(sig);
+      });
+
+      const hasMultipleDistinctMethodChains =
+        !hasMultipleDistinctCalls &&
+        uniqueDirectCalls.length > 1 &&
+        !allDirectCallsIdentical(directCalls) &&
+        !hasDirectFunctionCall; // Exclude rest spread patterns
+
+      if (hasMultipleDistinctMethodChains) {
+        // For multiple method-chain calls on the same import, create a NESTED structure
+        // under the base import name. This allows constructMockCode to generate a
+        // combined mock object with all method chains.
+        //
+        // Example: For calls ['trpc.useUtils()', 'trpc.quote.getQuotesByOrg.useQuery()'],
+        // create: { "trpc": { "useUtils()": {...}, "quote": { "getQuotesByOrg": { "useQuery()": {...} } } } }
+        const nestedResult: Record<string, unknown> = {};
+
+        for (let i = 0; i < uniqueDirectCalls.length; i++) {
+          // Find the index of this call in the original calls array
+          const callIndex = (importedExport.calls ?? []).indexOf(
+            uniqueDirectCalls[i],
+          );
+          if (callIndex === -1) continue;
+
+          const hookCallIndex = hookCallIndices[hookName] ?? 0;
+          hookCallIndices[hookName] = hookCallIndex + 1;
+
+          const processResult = processCall(
+            importedExport,
+            dependencySchemas,
+            callIndex,
+            undefined,
+            options,
+            hookCallIndex,
+            prebuiltIndexes,
+          );
+
+          if (processResult) {
+            // The key is like "trpc.useUtils()" or "trpc.quote.getQuotesByOrg.useQuery()"
+            // We need to nest the value at the path AFTER the base name
+            const callSignature = processResult.key;
+
+            // Split the call signature into path parts, handling parens
+            // e.g., "trpc.useUtils()" -> ["trpc", "useUtils()"]
+            // e.g., "trpc.quote.getQuotesByOrg.useQuery()" -> ["trpc", "quote", "getQuotesByOrg", "useQuery()"]
+            const pathParts = splitOutsideParenthesesAndArrays(callSignature);
+
+            // Skip the base name (first part) and nest the value at the remaining path
+            if (pathParts.length > 1 && pathParts[0] === baseName) {
+              const nestedPath = pathParts.slice(1);
+
+              // Set the value at the nested path, merging with existing structure
+              let current = nestedResult;
+              for (let j = 0; j < nestedPath.length - 1; j++) {
+                const part = nestedPath[j];
+                if (!(part in current)) {
+                  current[part] = {};
+                }
+                current = current[part] as Record<string, unknown>;
+              }
+              // Set the leaf value
+              const leafKey = nestedPath[nestedPath.length - 1];
+              current[leafKey] = processResult.value;
+            }
+          }
+        }
+
+        // Add the nested result under the base import name
+        if (Object.keys(nestedResult).length > 0) {
+          addResultToAccumulator(
+            acc,
+            baseName,
+            nestedResult as JsonTypeDefinition,
+          );
+        }
+      } else if (hasMultipleDistinctCalls) {
         // Track how many times each variable name has been seen
         // to disambiguate keys when the same variable is reused
         const variableNameCounts: Record<string, number> = {};
@@ -561,7 +774,42 @@ export default function gatherDataForMocks(
             occurrence,
             options,
             hookCallIndex,
+            prebuiltIndexes,
           );
+          if (processResult) {
+            addResultToAccumulator(acc, processResult.key, processResult.value);
+          }
+        }
+      } else if (
+        // Handle multiple UNIQUE type-parameterized direct calls when variable count doesn't match.
+        // This happens when a hook is called with different type parameters (useFetcher<TypeA>(), useFetcher<TypeB>()),
+        // but the callVariableNames count doesn't match directCallCount (common with rest spread patterns).
+        // Each unique call signature should get its own entry in dataForMocks.
+        uniqueDirectCalls.length > 1 &&
+        !allDirectCallsIdentical(directCalls) &&
+        hasDirectFunctionCall
+      ) {
+        // Process each UNIQUE direct call signature
+        for (let i = 0; i < uniqueDirectCalls.length; i++) {
+          const callSignature = uniqueDirectCalls[i];
+
+          // Find the index of this call in the original calls array
+          const callIndex = (importedExport.calls ?? []).indexOf(callSignature);
+          if (callIndex === -1) continue;
+
+          const hookCallIndex = hookCallIndices[hookName] ?? 0;
+          hookCallIndices[hookName] = hookCallIndex + 1;
+
+          const processResult = processCall(
+            importedExport,
+            dependencySchemas,
+            callIndex,
+            undefined,
+            options,
+            hookCallIndex,
+            prebuiltIndexes,
+          );
+
           if (processResult) {
             addResultToAccumulator(acc, processResult.key, processResult.value);
           }
@@ -580,6 +828,7 @@ export default function gatherDataForMocks(
           undefined,
           options,
           hookCallIndex,
+          prebuiltIndexes,
         );
 
         // Check if we got a meaningful result (not empty object or null)
@@ -624,6 +873,7 @@ export default function gatherDataForMocks(
                 0, // Use 0 to trigger variable-qualified lookup
                 options,
                 hookCallIndex,
+                prebuiltIndexes,
               );
               if (varQualifiedResult && varQualifiedResult.value) {
                 mergedValue = mergeJsonTypeDefinitions(
@@ -660,6 +910,7 @@ export default function gatherDataForMocks(
               0, // Use 0 to trigger variable-qualified lookup
               options,
               hookCallIndex,
+              prebuiltIndexes,
             );
             if (varQualifiedResult && varQualifiedResult.value) {
               // Deep merge the value into the accumulated result
@@ -681,18 +932,5 @@ export default function gatherDataForMocks(
     {} as { [importAlias: string]: JsonTypeDefinition },
   );
 
-  // Log dataForMocks keys when supabase/createClient is involved
-  const supabaseKeys = Object.keys(result).filter(
-    (k) => k.includes('createClient') || k.includes('from('),
-  );
-  if (supabaseKeys.length > 0) {
-    console.log(
-      'CODEYAM: dataForMocks includes supabase keys:',
-      supabaseKeys,
-      '| All keys:',
-      Object.keys(result),
-    );
-  }
-
   return result;
 }

package/analyzer-template/packages/analyze/src/lib/files/scenarios/generateChangesScenarios.ts

@@ -5,15 +5,11 @@ import {
   Entity,
   DEFAULT_SCENARIO_NAME,
 } from '~codeyam/types';
-import deepEqual from '../../utils/deepEqual';
 import measureAndReportExecutionTime from '../../utils/measureAndReportExecutionTime';
-import {
-  AI,
-  generateChangesEntityScenarios,
-  generateChangesEntityKeyAttributes,
-} from '~codeyam/ai';
+import { AI, generateChangesEntityScenarios } from '~codeyam/ai';
 import { awsLog } from '~codeyam/utils';
 import detectChangedDataStructureFields from './detectChangedDataStructureFields';
+import generateExecutionFlows from './generateExecutionFlows';
 
 export interface GenerateChangesScenariosArgs {
   previousAnalysis: Analysis;
@@ -40,8 +36,8 @@ export default async function generateChangesScenarios({
 
   const existingScenarios = (previousAnalysis.scenarios ?? []).filter(
     (s) =>
-      (error ? s.metadata.error : !s.metadata.error) &&
-      !s.metadata.sameAsDefault,
+      (error ? s.metadata?.error : !s.metadata?.error) &&
+      !s.metadata?.sameAsDefault,
   );
   const commitDiff = commit.files.find(
     (f) => f.fileName === entity.filePath,
@@ -87,8 +83,7 @@ export default async function generateChangesScenarios({
           name: DEFAULT_SCENARIO_NAME,
           description: 'The only external state the component can be in.',
           metadata: {
-            dataMapping: {},
-            keyAttributeInstructions: {},
+            coveredFlows: [],
           },
         },
       ];
@@ -96,53 +91,20 @@ export default async function generateChangesScenarios({
     return;
   }
 
-  // if (!analysis.metadata.keyAttributes) {
-  const existingKeyAttributes = (
-    previousAnalysis.metadata?.keyAttributes ?? []
-  ).filter(
-    (keyAttribute, index, self) =>
-      self.findIndex((ka) => ka.internalPath === keyAttribute.internalPath) ===
-      index,
-  );
-
-  const {
-    result: { keyAttributes, llmCall: keyAttributesLLMCall },
-  } = await measureAndReportExecutionTime(
-    () =>
-      generateChangesEntityKeyAttributes({
-        entity,
-        existingKeyAttributes,
-        analysis,
-        commitDiff,
-        model,
-      }),
-    `Generating changes entity key attributes`,
-    updateProgress,
-  );
-
-  analysis.metadata.keyAttributes = keyAttributes;
-
-  if (keyAttributesLLMCall) {
-    analysis.metadata.llmCalls ||= [];
-    analysis.metadata.llmCalls.push({
-      name: `generateChangesEntityKeyAttributes`,
-      id: keyAttributesLLMCall.id,
+  // Generate execution flows if not already present
+  if (!analysis.metadata.executionFlows) {
+    await generateExecutionFlows({
+      entity,
+      analysis,
+      model,
+      updateProgress,
+      dependentAnalyses,
     });
   }
-  // }
 
-  const addedKeyAttributes = analysis.metadata.keyAttributes?.filter(
-    (ka) =>
-      !previousAnalysis.metadata.keyAttributes?.find(
-        (ka2) => ka.internalPath === ka2.internalPath,
-      ),
-  );
-  const removedKeyAttributes = previousAnalysis.metadata.keyAttributes?.filter(
-    (ka) =>
-      !analysis.metadata.keyAttributes.find(
-        (ka2) => ka.internalPath === ka2.internalPath,
-      ),
-  );
+  // Get previous execution flows for comparison
+  const previousExecutionFlows =
+    previousAnalysis.metadata?.executionFlows ?? [];
 
   // Detect which specific data structure fields changed
   const changedDataStructureFields = detectChangedDataStructureFields(
@@ -156,10 +118,8 @@ export default async function generateChangesScenarios({
     () =>
       generateChangesEntityScenarios({
         entity,
-        keyAttributes: analysis.metadata.keyAttributes,
-        addedKeyAttributes,
-        removedKeyAttributes,
-        dependentAnalyses,
+        executionFlows: analysis.metadata.executionFlows,
+        previousExecutionFlows,
         existingScenarios,
         commitDiff,
         analysis,
@@ -175,10 +135,13 @@ export default async function generateChangesScenarios({
   for (const scenario of scenarios) {
     if (
       scenario.previousVersionId &&
-      !scenario.metadata.revisedFromPreviousVersion
+      !scenario.metadata?.revisedFromPreviousVersion
     ) {
       if (dataStructureChanged) {
-        scenario.metadata.revisedFromPreviousVersion = true;
+        scenario.metadata = {
+          ...scenario.metadata,
+          revisedFromPreviousVersion: true,
+        };
         continue;
       }
 
@@ -186,7 +149,10 @@ export default async function generateChangesScenarios({
         (s) => s.name === scenario.name,
       );
       if (!existingScenarioStatus || existingScenarioStatus.error) {
-        scenario.metadata.revisedFromPreviousVersion = true;
+        scenario.metadata = {
+          ...scenario.metadata,
+          revisedFromPreviousVersion: true,
+        };
       }
     }
   }
@@ -210,7 +176,7 @@ export default async function generateChangesScenarios({
 
     if (analysis.scenarios) {
       for (const existingScenario of analysis.scenarios.filter((s) =>
-        error ? !!s.metadata.error : !s.metadata.error,
+        error ? !!s.metadata?.error : !s.metadata?.error,
       )) {
         const scenarioIndex = scenarios.findIndex(
           (s) => s.name === existingScenario.name,