@codeyam/codeyam-cli 0.1.0-staging.323686 → 0.1.0-staging.4813bf3

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/fillInSchemaGapsAndUnknowns.ts

@@ -152,8 +152,11 @@ export default function fillInSchemaGapsAndUnknowns(
           schema[previousSubPath],
         )
       ) {
-        const newValue =
-          checkIfKnownType(previousSubPath, functionKeysMapping) ?? 'object';
+        // When fillInUnknowns=false, add missing gaps as 'unknown' to preserve raw schema.
+        // When fillInUnknowns=true, infer the type using checkIfKnownType.
+        const newValue = fillInUnknowns
+          ? (checkIfKnownType(previousSubPath, functionKeysMapping) ?? 'object')
+          : 'unknown';
 
         if (
           !schema[previousSubPath] ||
@@ -166,10 +169,14 @@ export default function fillInSchemaGapsAndUnknowns(
 
       // Handle the wrong function chaining function().function()
       if (isFunction && !existingSchema(previousSubPath)) {
+        // When fillInUnknowns=false, use 'unknown' for non-function paths.
+        // When fillInUnknowns=true, infer the type.
         schema[previousSubPath] = previousSubPath.endsWith(')')
           ? 'function'
-          : (checkIfKnownType(previousSubPath, functionKeysMapping) ??
-            'object');
+          : fillInUnknowns
+            ? (checkIfKnownType(previousSubPath, functionKeysMapping) ??
+              'object')
+            : 'unknown';
         changeMade = true;
       }
 
@@ -205,16 +212,88 @@ export default function fillInSchemaGapsAndUnknowns(
   }
 }
 
+/**
+ * Pre-built indexes that can be reused across multiple calls to fillInDirectSchemaGapsAndUnknowns.
+ * Build once with buildSchemaIndexes(), then pass to all calls processing the same schema.
+ */
+export interface SchemaIndexes {
+  functionKeysMapping: Record<string, string[]>;
+  prefixMaxDepth: Map<string, number>;
+}
+
+/**
+ * Build indexes for a schema that can be reused across multiple calls.
+ * This avoids rebuilding indexes (~1-2 seconds for 18k keys) on every call.
+ *
+ * @param schema - The schema to build indexes from (typically the full dependency schema)
+ * @returns Indexes that can be passed to fillInDirectSchemaGapsAndUnknowns
+ */
+export function buildSchemaIndexes(
+  schema: Record<string, string>,
+): SchemaIndexes {
+  // Build functionKeysMapping + prefixMaxDepth in a single pass
+  // Avoid repeated split/join work by caching parts and building prefixes incrementally.
+  const functionKeysMapping: Record<string, string[]> = {};
+  const prefixMaxDepth = new Map<string, number>();
+  const partsCache = new Map<string, string[]>();
+
+  for (const key in schema) {
+    const parts = getCachedParts(partsCache, key);
+    const lastPart = parts[parts.length - 1];
+    const prefixes = buildPrefixParts(parts);
+
+    if (
+      key.endsWith(')') ||
+      schema[key] === 'function' ||
+      lastPart === 'length'
+    ) {
+      const allButLastPart = parts.length > 1 ? prefixes[parts.length - 2] : '';
+      functionKeysMapping[allButLastPart] ||= [];
+      functionKeysMapping[allButLastPart].push(lastPart);
+    }
+
+    for (let i = 0; i < prefixes.length; i++) {
+      const prefix = prefixes[i];
+      const existingMax = prefixMaxDepth.get(prefix) ?? 0;
+      if (parts.length > existingMax) {
+        prefixMaxDepth.set(prefix, parts.length);
+      }
+    }
+  }
+
+  return { functionKeysMapping, prefixMaxDepth };
+}
+
 export function fillInDirectSchemaGapsAndUnknowns({
   scopeName,
   schema,
   mergedSchema,
   attempts = 0,
+  // Pre-built indexes from buildSchemaIndexes() - use for cross-call caching
+  prebuiltIndexes,
+  // Internal: pre-computed indexes to avoid rebuilding on recursion
+  _functionKeysMapping,
+  _prefixMaxDepth,
+  _partsCache,
+  _prefixPartsCache,
+  _knownTypeCache,
+  _guessTypeCache,
+  _nameHintCache,
 }: {
   scopeName?: string;
   schema: Record<string, string>;
   mergedSchema?: Record<string, string>;
   attempts?: number;
+  /** Pre-built indexes from buildSchemaIndexes() - pass to reuse across multiple calls */
+  prebuiltIndexes?: SchemaIndexes;
+  // Internal: pre-computed indexes to avoid rebuilding on recursion (used by recursive calls)
+  _functionKeysMapping?: Record<string, string[]>;
+  _prefixMaxDepth?: Map<string, number>;
+  _partsCache?: Map<string, string[]>;
+  _prefixPartsCache?: Map<string, string[]>;
+  _knownTypeCache?: Map<string, string | undefined>;
+  _guessTypeCache?: Map<string, string | undefined>;
+  _nameHintCache?: Map<string, string | undefined>;
 }) {
   try {
     const existingSchema = (path: any) => {
@@ -225,105 +304,239 @@ export function fillInDirectSchemaGapsAndUnknowns({
       }
     };
 
-    const functionKeysMapping = [...Object.keys(schema)]
-      .filter((key) => {
-        if (key.endsWith(')') || schema[key] === 'function') return true;
-        const parts = splitOutsideParenthesesAndArrays(key);
-        if (parts[parts.length - 1] === 'length') return true;
-      })
-      .reduce(
-        (acc, key) => {
-          const parts = splitOutsideParenthesesAndArrays(key);
-          const allButLastPart = joinParenthesesAndArrays(parts.slice(0, -1));
+    const partsCache = _partsCache ?? new Map<string, string[]>();
+    const prefixPartsCache = _prefixPartsCache ?? new Map<string, string[]>();
+    const knownTypeCache =
+      _knownTypeCache ?? new Map<string, string | undefined>();
+    const guessTypeCache =
+      _guessTypeCache ?? new Map<string, string | undefined>();
+    const nameHintCache =
+      _nameHintCache ?? new Map<string, string | undefined>();
+
+    const getParts = (path: string) => getCachedParts(partsCache, path);
+    const getPrefixParts = (path: string) => {
+      const cached = prefixPartsCache.get(path);
+      if (cached) return cached;
+      const prefixes = buildPrefixParts(getParts(path));
+      prefixPartsCache.set(path, prefixes);
+      return prefixes;
+    };
+
+    // Index resolution priority:
+    // 1. _functionKeysMapping/_prefixMaxDepth (from recursive calls within this invocation)
+    // 2. prebuiltIndexes (from cross-call caching - built once, reused across multiple calls)
+    // 3. Build new indexes (fallback - expensive for large schemas)
+    const functionKeysMapping =
+      _functionKeysMapping ??
+      prebuiltIndexes?.functionKeysMapping ??
+      (() => {
+        const mapping: Record<string, string[]> = {};
+        for (const key in schema) {
+          const parts = getParts(key);
           const lastPart = parts[parts.length - 1];
-          acc[allButLastPart] ||= [];
-          acc[allButLastPart].push(lastPart);
-          return acc;
-        },
-        {} as Record<string, string[]>,
-      );
+          if (
+            key.endsWith(')') ||
+            schema[key] === 'function' ||
+            lastPart === 'length'
+          ) {
+            const prefixes = getPrefixParts(key);
+            const allButLastPart =
+              parts.length > 1 ? prefixes[parts.length - 2] : '';
+            mapping[allButLastPart] ||= [];
+            mapping[allButLastPart].push(lastPart);
+          }
+        }
+        return mapping;
+      })();
+
+    // Build prefix index for O(1) lookups in checkIfObjectOrFunction
+    // Maps each prefix to the max depth of keys that have this prefix
+    // This replaces O(n) scans with O(1) lookups, fixing the O(n²) bottleneck
+    let prefixMaxDepth =
+      _prefixMaxDepth ?? prebuiltIndexes?.prefixMaxDepth ?? null;
+    if (!prefixMaxDepth) {
+      prefixMaxDepth = new Map<string, number>();
+      const targetSchema = mergedSchema ?? schema;
+      for (const key in targetSchema) {
+        const keyParts = getParts(key);
+        const prefixes = buildPrefixParts(keyParts);
+        // Record each prefix of this key with its max depth
+        for (let i = 0; i < prefixes.length; i++) {
+          const prefix = prefixes[i];
+          const existingMax = prefixMaxDepth.get(prefix) ?? 0;
+          if (keyParts.length > existingMax) {
+            prefixMaxDepth.set(prefix, keyParts.length);
+          }
+        }
+      }
+    }
 
-    let changeMade = false;
-    const guessTypeForPath = (path: string, defaultType = 'string') => {
-      let knownType = checkIfKnownType(path, functionKeysMapping);
-      if (!knownType) {
-        // Use mergedSchema for type inference if available (has more complete nested paths)
-        // Fall back to schema if mergedSchema is not provided
-        knownType = checkIfObjectOrFunction(path, mergedSchema ?? schema);
+    // O(1) replacement for checkIfObjectOrFunction
+    const checkIfObjectOrFunctionFast = (path: string): string | undefined => {
+      const maxDepth = prefixMaxDepth!.get(path);
+
+      if (maxDepth === undefined) return undefined; // No key starts with this path
+
+      const pathParts = getParts(path);
+      // Check if path's last part ends with ')' → function
+      const lastPart = pathParts[pathParts.length - 1];
+      if (lastPart?.endsWith(')')) {
+        return 'function';
       }
 
-      if (knownType && schema[path] !== knownType) {
-        return knownType;
+      // Check if there are deeper keys
+      if (maxDepth > pathParts.length) {
+        return 'object';
       }
 
-      const keyParts = splitOutsideParenthesesAndArrays(path);
-      const lastPart = keyParts[keyParts.length - 1];
+      return undefined;
+    };
+
+    let changeMade = false;
+    const getKnownType = (path: string) => {
+      if (knownTypeCache.has(path)) return knownTypeCache.get(path);
+      const known = checkIfKnownType(path, functionKeysMapping);
+      knownTypeCache.set(path, known);
+      return known;
+    };
+    const guessTypeFromName = (lastPart: string, defaultType: string) => {
+      if (nameHintCache.has(lastPart)) {
+        return nameHintCache.get(lastPart) ?? defaultType;
+      }
 
+      let hint: string | undefined;
       if (isLikelyBoolean(lastPart)) {
-        return 'boolean';
+        hint = 'boolean';
       } else if (isLikelyDate(lastPart)) {
-        return 'date';
+        hint = 'date';
       } else if (isLikelyAction(lastPart)) {
-        return 'function';
+        hint = 'function';
       } else if (pluralize.isPlural(lastPart)) {
-        return 'number';
-      } else {
-        return defaultType;
+        // Default to number instead of array - safer for JSX rendering
+        // Arrays cause "Objects are not valid as a React child" errors when rendered directly
+        hint = 'number';
       }
+
+      nameHintCache.set(lastPart, hint);
+      return hint ?? defaultType;
+    };
+    const guessTypeForPath = (path: string, defaultType = 'string') => {
+      const cacheKey = `${path}|${defaultType}`;
+      if (guessTypeCache.has(cacheKey)) {
+        return guessTypeCache.get(cacheKey);
+      }
+
+      let knownType = getKnownType(path);
+      if (!knownType) {
+        // Use optimized O(1) prefix lookup instead of O(n) scan
+        knownType = checkIfObjectOrFunctionFast(path);
+      }
+
+      if (knownType) {
+        guessTypeCache.set(cacheKey, knownType);
+        return knownType;
+      }
+
+      const keyParts = getParts(path);
+      const lastPart = keyParts[keyParts.length - 1];
+      const guessed = guessTypeFromName(lastPart, defaultType);
+      guessTypeCache.set(cacheKey, guessed);
+      return guessed;
     };
 
     for (const key in schema) {
       if (!schema[key]) schema[key] = 'unknown';
       if (schema[key].includes(' | unknown')) {
-        schema[key] = schema[key].replace(' | unknown', '');
-        changeMade = true;
+        const cleaned = schema[key].replace(' | unknown', '');
+        if (schema[key] !== cleaned) {
+          schema[key] = cleaned;
+          changeMade = true;
+        }
       } else if (schema[key].startsWith('unknown | ')) {
         // Handle "unknown | undefined" -> "string | undefined"
         const remainingType = schema[key].substring('unknown | '.length);
         const newType = guessTypeForPath(key, 'string');
         if (newType) {
-          schema[key] = `${newType} | ${remainingType}`;
-          changeMade = true;
+          const replacement = `${newType} | ${remainingType}`;
+          if (schema[key] !== replacement) {
+            schema[key] = replacement;
+            changeMade = true;
+          }
         }
       } else if (schema[key] === 'unknown') {
         const newType = guessTypeForPath(key, 'string');
-        if (newType) {
+        // Don't convert 'unknown' to 'object' — 'object' is just inferred from
+        // having child properties, which isn't more useful than 'unknown' and
+        // prevents enrichment from filling in the actual type later.
+        if (newType && newType !== 'object' && schema[key] !== newType) {
           schema[key] = newType;
           changeMade = true;
         }
       }
 
-      const keyParts = splitOutsideParenthesesAndArrays(key);
+      const keyParts = getParts(key);
+      const prefixParts = getPrefixParts(key);
       for (let i = 0; i < keyParts.length; i++) {
-        const subPathParts = keyParts.slice(0, i + 1);
-        const lastSubPathPart = subPathParts[subPathParts.length - 1];
-
-        const previousSubPathParts = subPathParts.slice(0, i);
-        if (previousSubPathParts.length === 0) continue;
-
-        const previousSubPath = joinParenthesesAndArrays(previousSubPathParts);
-
-        if (
-          cleanOutBoundary(lastSubPathPart).match(/\[\d*\]/) &&
-          !lastSubPathPart.match(/signature\[\d+\]/)
-        ) {
-          // Fix 39: Don't overwrite explicit 'object' types with 'array'
-          // This handles spurious [] paths from components like JsonNode that handle
-          // both arrays and objects. When metadata is explicitly typed as 'object',
-          // paths like metadata[] (from dynamic iteration) should not change it.
-          if (
-            schema[previousSubPath] !== 'array' &&
-            schema[previousSubPath] !== 'object'
-          ) {
-            schema[previousSubPath] = 'array';
-            changeMade = true;
+        if (i === 0) continue;
+        const lastSubPathPart = keyParts[i];
+        const previousSubPath = prefixParts[i - 1];
+        const isSignatureIndex =
+          lastSubPathPart.includes('signature[') &&
+          SIGNATURE_INDEX_REGEX.test(lastSubPathPart);
+
+        if (lastSubPathPart.includes('[')) {
+          const cleaned = cleanOutBoundary(lastSubPathPart);
+          if (cleaned.match(/\[\d*\]/) && !isSignatureIndex) {
+            // Fix 39: Don't overwrite explicit 'object' types with 'array'
+            // This handles spurious [] paths from components like JsonNode that handle
+            // both arrays and objects. When metadata is explicitly typed as 'object',
+            // paths like metadata[] (from dynamic iteration) should not change it.
+            const existingType = schema[previousSubPath];
+            const baseType = existingType?.split(' | ')[0];
+
+            // Also check if this path has meaningful object property children (non-[] children).
+            // If it does, it's an object with mixed access patterns (e.g., agent.entries
+            // AND agent[].type), not a pure array. Don't infer 'array' in that case.
+            // Exclude .length since it exists on arrays too and is NOT evidence of object-ness.
+            let hasObjectPropertyChildren = false;
+            if (baseType !== 'array' && baseType !== 'object') {
+              const dotPrefix = previousSubPath + '.';
+              const lengthPath = previousSubPath + '.length';
+              for (const schemaKey in schema) {
+                if (
+                  schemaKey.startsWith(dotPrefix) &&
+                  schemaKey !== lengthPath
+                ) {
+                  hasObjectPropertyChildren = true;
+                  break;
+                }
+              }
+            }
+
+            if (
+              baseType !== 'array' &&
+              baseType !== 'object' &&
+              !hasObjectPropertyChildren
+            ) {
+              // Preserve nullability from existing type (e.g., 'unknown | undefined' -> 'array | undefined')
+              let newType = 'array';
+              if (existingType?.includes(' | undefined')) {
+                newType = 'array | undefined';
+              } else if (existingType?.includes(' | null')) {
+                newType = 'array | null';
+              }
+              if (schema[previousSubPath] !== newType) {
+                schema[previousSubPath] = newType;
+                changeMade = true;
+              }
+            }
           }
         }
 
         const isFunction =
           lastSubPathPart.endsWith(')') ||
           lastSubPathPart === 'functionCallReturnValue' ||
-          lastSubPathPart.match(/signature\[\d+\]/);
+          isSignatureIndex;
 
         if (
           !isFunction &&
@@ -331,15 +544,36 @@ export function fillInDirectSchemaGapsAndUnknowns({
             schema[previousSubPath],
           )
         ) {
-          const newValue =
-            checkIfKnownType(previousSubPath, functionKeysMapping) ?? 'object';
-
+          const existingType = schema[previousSubPath];
+          const baseExistingType = existingType?.split(' | ')[0];
+          // Skip if the base type is already a structured type
           if (
-            !schema[previousSubPath] ||
-            schema[previousSubPath] !== newValue
+            !['object', 'array', 'function', 'async-function'].includes(
+              baseExistingType,
+            )
           ) {
-            schema[previousSubPath] = newValue;
-            changeMade = true;
+            let newValue = getKnownType(previousSubPath) ?? 'object';
+            // Don't overwrite 'unknown' with 'object' — 'object' is just
+            // inferred from child properties and isn't more useful than 'unknown'
+            if (
+              newValue === 'object' &&
+              existingType &&
+              existingType.split(' | ')[0] === 'unknown'
+            ) {
+              // preserve existing 'unknown' (possibly with nullability)
+            } else {
+              // Preserve nullability from existing type
+              if (existingType?.includes(' | undefined')) {
+                newValue = `${newValue} | undefined`;
+              } else if (existingType?.includes(' | null')) {
+                newValue = `${newValue} | null`;
+              }
+
+              if (!existingType || existingType !== newValue) {
+                schema[previousSubPath] = newValue;
+                changeMade = true;
+              }
+            }
           }
         }
 
@@ -347,8 +581,7 @@ export function fillInDirectSchemaGapsAndUnknowns({
         if (isFunction && !existingSchema(previousSubPath)) {
           schema[previousSubPath] = previousSubPath.endsWith(')')
             ? 'function'
-            : (checkIfKnownType(previousSubPath, functionKeysMapping) ??
-              'object');
+            : (getKnownType(previousSubPath) ?? 'object');
           changeMade = true;
         }
 
@@ -358,10 +591,12 @@ export function fillInDirectSchemaGapsAndUnknowns({
           const functionReturnValuePath = `${previousSubPath}.functionCallReturnValue`;
 
           // Extract the method name from lastSubPathPart (e.g., "then()" -> "then")
-          const methodName = lastSubPathPart.split('(')[0];
-          const isPromiseMethod = ['then', 'catch', 'finally'].includes(
-            methodName,
-          );
+          const parenIndex = lastSubPathPart.indexOf('(');
+          const methodName =
+            parenIndex === -1
+              ? lastSubPathPart
+              : lastSubPathPart.slice(0, parenIndex);
+          const isPromiseMethod = PROMISE_METHODS.has(methodName);
           const isPreviousAsync = schema[previousSubPath] === 'async-function';
 
           // Don't set to 'function' if this is a promise method on an async function
@@ -386,6 +621,15 @@ export function fillInDirectSchemaGapsAndUnknowns({
         schema,
         mergedSchema,
         attempts: ++attempts,
+        prebuiltIndexes,
+        // Pass pre-computed indexes to avoid rebuilding on each recursive call
+        _functionKeysMapping: functionKeysMapping,
+        _prefixMaxDepth: prefixMaxDepth,
+        _partsCache: partsCache,
+        _prefixPartsCache: prefixPartsCache,
+        _knownTypeCache: knownTypeCache,
+        _guessTypeCache: guessTypeCache,
+        _nameHintCache: nameHintCache,
       });
     }
 
@@ -395,9 +639,10 @@ export function fillInDirectSchemaGapsAndUnknowns({
     // However, if the parent could be an 'object' or other type, keep .length as it may be
     // a custom property.
     for (const key of Object.keys(schema)) {
-      const parts = splitOutsideParenthesesAndArrays(key);
+      const parts = getParts(key);
       if (parts[parts.length - 1] === 'length') {
-        const parentPath = joinParenthesesAndArrays(parts.slice(0, -1));
+        const prefixes = getPrefixParts(key);
+        const parentPath = parts.length > 1 ? prefixes[parts.length - 2] : '';
         const parentType = schema[parentPath];
         if (parentType) {
           // Split union types and filter out nullable parts
@@ -434,6 +679,35 @@ export function fillInDirectSchemaGapsAndUnknowns({
   }
 }
 
+const SIGNATURE_INDEX_REGEX = /signature\[\d+\]/;
+const PROMISE_METHODS = new Set(['then', 'catch', 'finally']);
+
+function getCachedParts(cache: Map<string, string[]>, path: string): string[] {
+  const cached = cache.get(path);
+  if (cached) return cached;
+  const parts = splitOutsideParenthesesAndArrays(path);
+  cache.set(path, parts);
+  return parts;
+}
+
+function buildPrefixParts(parts: string[]): string[] {
+  if (parts.length === 0) return [];
+  const prefixes = new Array(parts.length);
+  let prefix = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      prefix = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      prefix += part;
+    } else {
+      prefix += `.${part}`;
+    }
+    prefixes[i] = prefix;
+  }
+  return prefixes;
+}
+
 function checkIfObjectOrFunction(path: string, schema: ScopeNode['schema']) {
   const pathParts = splitOutsideParenthesesAndArrays(path);
   for (const structureKey in schema) {
@@ -506,20 +780,20 @@ function checkIfKnownType(
   let hasStrongArrayEvidence = false;
   let hasOnlyAmbiguousArrayMethods = true;
   let hasLengthAccess = false;
-  let hasArrayStringAmbiguousMethod = false;
   let hasIncludesOrIndexOfWithVariable = false;
 
-  for (const functionOrAttributeName of functionAndAttributesKeysMapping[key] ??
-    []) {
+  const entries = functionAndAttributesKeysMapping[key];
+  if (!entries || entries.length === 0) return;
+
+  for (const functionOrAttributeName of entries) {
     // Extract the method name without arguments for checking against ambiguous list
     const methodName = functionOrAttributeName.split('(')[0].trim();
 
+    const knownType = knownMethodCalls(functionOrAttributeName);
     const couldBeArray =
-      knownMethodCalls(functionOrAttributeName).includes('array') ||
-      functionOrAttributeName === 'length';
+      knownType.includes('array') || functionOrAttributeName === 'length';
     const couldBeString =
-      knownMethodCalls(functionOrAttributeName).includes('string') ||
-      functionOrAttributeName === 'length';
+      knownType.includes('string') || functionOrAttributeName === 'length';
 
     // Track .length access
     if (functionOrAttributeName === 'length') {
@@ -527,10 +801,6 @@ function checkIfKnownType(
     }
 
     // Track methods that exist on both arrays and strings (like includes, indexOf)
-    if (couldBeArray && couldBeString && functionOrAttributeName !== 'length') {
-      hasArrayStringAmbiguousMethod = true;
-    }
-
     // Check if .includes() or .indexOf() is called with a variable argument (not a string literal).
     // Pattern: arr.includes(item) -> likely array (checking if item exists)
     // Pattern: str.includes('substring') -> likely string (checking for substring)
@@ -572,10 +842,11 @@ function checkIfKnownType(
     return 'array';
   }
 
-  // When .includes() or .indexOf() is called with a variable argument AND we have .length,
-  // prefer array. This pattern (arr.length > 0 && arr.includes(item)) is common for arrays.
-  // But str.includes('literal') is common for strings - that case is handled by hasStringLiteralArgument.
-  if (isArray && hasLengthAccess && hasIncludesOrIndexOfWithVariable) {
+  // When .includes() or .indexOf() is called with a variable argument, prefer array.
+  // Pattern: arr.includes(item) -> likely array (checking if item exists in collection)
+  // String literals are already filtered by hasStringLiteralArgument, so if we get here
+  // with a variable argument, it's strong evidence of array membership testing.
+  if (isArray && hasIncludesOrIndexOfWithVariable) {
     return 'array';
   }