@codeyam/codeyam-cli 0.1.0-staging.1 → 0.1.0-staging.28f73cf

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/PathManager.ts

@@ -0,0 +1,203 @@
+/**
+ * PathManager
+ *
+ * Centralized utility for schema path manipulation with built-in caching.
+ * Handles splitting, joining, and validating schema paths used throughout
+ * the ScopeDataStructure analysis.
+ *
+ * ## Context in the Analysis Pipeline
+ *
+ * Schema paths are dot-separated strings representing data access patterns:
+ * - `signature[0].user.id` - First argument's user.id property
+ * - `items[].name` - name property of array elements
+ * - `functionCallReturnValue(fetch).data` - Return value from fetch() call
+ *
+ * These paths are created during AST analysis and used extensively in:
+ * 1. **Equivalency tracking** - Linking data flow between scopes
+ * 2. **Schema building** - Determining the shape of function inputs/outputs
+ * 3. **Database queries** - Finding source/usage relationships
+ *
+ * ## Performance Notes
+ *
+ * Path operations are called thousands of times per analysis. Caching
+ * provides ~10x speedup for repeated operations on the same paths.
+ *
+ * ## Related Modules
+ *
+ * - `splitOutsideParentheses.ts` - Low-level split/join with parentheses awareness
+ * - `cleanPath.ts` - Normalizes paths by removing noise
+ * - `uniqueIdUtils.ts` - Creates unique keys from scope+path combinations
+ */
+
+import {
+  joinParenthesesAndArrays,
+  splitOutsideParenthesesAndArrays,
+} from '../../splitOutsideParentheses';
+import cleanOutBoundary from '../../cleanOutBoundary';
+
+export class PathManager {
+  private pathPartsCache: Map<string, string[]> = new Map();
+  private pathJoinCache: Map<string, string> = new Map();
+  private isValidPathCache: Map<string, boolean> = new Map();
+  private stripGenericsCache: Map<string, string> = new Map();
+
+  /**
+   * Splits a schema path into its component parts.
+   * Results are cached for performance.
+   *
+   * IMPORTANT: The returned array must NOT be mutated by callers.
+   * This allows us to return the cached array directly without cloning,
+   * providing significant performance gains (structuredClone is expensive).
+   *
+   * @example
+   * splitPath('user.profile.name') // ['user', 'profile', 'name']
+   * splitPath('items[].id') // ['items[]', 'id']
+   */
+  splitPath(path: string): readonly string[] {
+    const cached = this.pathPartsCache.get(path);
+    if (cached) {
+      return cached;
+    }
+    const pathParts = splitOutsideParenthesesAndArrays(path);
+    this.pathPartsCache.set(path, pathParts);
+    return pathParts;
+  }
+
+  /**
+   * Joins path parts back into a schema path string.
+   * Results are cached for performance.
+   *
+   * @example
+   * joinPathParts(['user', 'profile', 'name']) // 'user.profile.name'
+   */
+  joinPathParts(pathParts: readonly string[]): string {
+    const cacheKey = pathParts.join('|');
+    if (this.pathJoinCache.has(cacheKey)) {
+      return this.pathJoinCache.get(cacheKey)!;
+    }
+    const result = joinParenthesesAndArrays(pathParts as string[]);
+    this.pathJoinCache.set(cacheKey, result);
+    return result;
+  }
+
+  /**
+   * Validates whether a string is a valid schema path.
+   * Results are cached for performance since this is called very frequently.
+   * Rejects:
+   * - Non-strings
+   * - Empty strings
+   * - Boolean literals ('true', 'false')
+   * - Numeric strings
+   * - Quoted strings (starting with ' or ")
+   * - Paths containing whitespace outside of brackets
+   */
+  isValidPath(path: string): boolean {
+    // Quick type checks (not cacheable)
+    if (typeof path !== 'string' || !path) {
+      return false;
+    }
+
+    // Check cache first
+    if (this.isValidPathCache.has(path)) {
+      return this.isValidPathCache.get(path)!;
+    }
+
+    // Compute validity
+    let result = true;
+
+    if (path === 'true' || path === 'false') {
+      result = false;
+    } else if (!isNaN(Number(path))) {
+      result = false;
+    } else if (path.charCodeAt(0) === 39 || path.charCodeAt(0) === 34) {
+      // Check for ' (39) or " (34) at start - faster than regex
+      result = false;
+    } else {
+      // Check for whitespace in path parts
+      result = this.splitPath(path).every((part) => {
+        const cleaned = cleanOutBoundary(
+          cleanOutBoundary(cleanOutBoundary(part, '<', '>'), '[', ']'),
+        );
+        // Use indexOf instead of regex for speed
+        return (
+          cleaned.indexOf(' ') === -1 &&
+          cleaned.indexOf('\t') === -1 &&
+          cleaned.indexOf('\n') === -1
+        );
+      });
+    }
+
+    this.isValidPathCache.set(path, result);
+    return result;
+  }
+
+  /**
+   * Strips generic type parameters from a name.
+   * Cached for performance since this is called frequently.
+   *
+   * @example
+   * stripGenerics('MyComponent<Props>') // 'MyComponent'
+   * stripGenerics('Array<T>') // 'Array'
+   * stripGenerics('Simple') // 'Simple'
+   */
+  stripGenerics(name: string): string {
+    if (this.stripGenericsCache.has(name)) {
+      return this.stripGenericsCache.get(name)!;
+    }
+
+    // Find position of '<' - if not found, return original
+    const ltIndex = name.indexOf('<');
+    const result = ltIndex === -1 ? name : name.slice(0, ltIndex);
+
+    this.stripGenericsCache.set(name, result);
+    return result;
+  }
+
+  /**
+   * Clears all caches. Useful for testing or memory management.
+   */
+  clearCache(): void {
+    this.pathPartsCache.clear();
+    this.pathJoinCache.clear();
+    this.isValidPathCache.clear();
+    this.stripGenericsCache.clear();
+  }
+
+  /**
+   * Returns cache statistics for debugging/monitoring.
+   */
+  getCacheStats(): {
+    pathParts: number;
+    pathJoin: number;
+    isValidPath: number;
+    stripGenerics: number;
+  } {
+    return {
+      pathParts: this.pathPartsCache.size,
+      pathJoin: this.pathJoinCache.size,
+      isValidPath: this.isValidPathCache.size,
+      stripGenerics: this.stripGenericsCache.size,
+    };
+  }
+}
+
+// Default instance for simple use cases
+export const defaultPathManager = new PathManager();
+
+// Convenience functions that use the default instance
+// These can be used when a shared cache is acceptable
+export function splitPath(path: string): readonly string[] {
+  return defaultPathManager.splitPath(path);
+}
+
+export function joinPathParts(pathParts: readonly string[]): string {
+  return defaultPathManager.joinPathParts(pathParts);
+}
+
+export function isValidPath(path: string): boolean {
+  return defaultPathManager.isValidPath(path);
+}
+
+export function stripGenerics(name: string): string {
+  return defaultPathManager.stripGenerics(name);
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/README.md

@@ -0,0 +1,294 @@
+# ScopeDataStructure Helpers
+
+This directory contains extracted helper modules from `ScopeDataStructure.ts`, the core engine for CodeYam's static analysis.
+
+## Architecture Overview
+
+```
+┌─────────────────────────────────────────────────────────────────────────┐
+│                        ScopeDataStructure                               │
+│                                                                         │
+│  Two-Phase Analysis:                                                    │
+│  ┌─────────────────────┐      ┌──────────────────────────────────────┐ │
+│  │ Phase 1: Discovery  │ ──▶  │ Phase 2: Schema Building            │ │
+│  │ (onlyEquivalencies) │      │ (captureCompleteSchema)              │ │
+│  └─────────────────────┘      └──────────────────────────────────────┘ │
+│                                                                         │
+│  Core Data Structures:                                                  │
+│  ┌──────────────┐  ┌───────────────────┐  ┌────────────────────────┐  │
+│  │ scopeNodes   │  │ equivalencyDB     │  │ externalFunctionCalls  │  │
+│  │ (per-scope)  │  │ (cross-scope)     │  │ (API tracking)         │  │
+│  └──────────────┘  └───────────────────┘  └────────────────────────┘  │
+└─────────────────────────────────────────────────────────────────────────┘
+
+Helper Modules:
+┌─────────────┐  ┌─────────────┐  ┌─────────────────┐  ┌──────────────┐
+│ PathManager │  │ UniqueId    │  │ VisitedTracker  │  │ ScopeTree    │
+│             │  │ Utils       │  │                 │  │ Manager      │
+└─────────────┘  └─────────────┘  └─────────────────┘  └──────────────┘
+```
+
+## Module Index
+
+### PathManager.ts
+
+**Purpose**: Path string manipulation with caching.
+
+**Key Functions**:
+
+- `splitPath(path)` - Split `user.profile.name` → `['user', 'profile', 'name']`
+- `joinPathParts(parts)` - Join `['user', 'id']` → `user.id`
+- `isValidPath(path)` - Validate path format
+
+**When to Use**: Whenever you need to parse or construct schema paths.
+
+---
+
+### uniqueIdUtils.ts
+
+**Purpose**: Create unique identifiers for scope/path combinations.
+
+**Key Functions**:
+
+- `uniqueId({scopeNodeName, schemaPath, value})` - Create lookup key
+- `uniqueScopeAndPaths(array)` - Deduplicate by scope+path
+
+**Key Format**: `scopeNodeName::schemaPath[::value]`
+
+**When to Use**: Building cache keys, deduplicating equivalencies.
+
+---
+
+### VisitedTracker.ts
+
+**Purpose**: Prevent infinite loops during recursive schema traversal.
+
+**Key Methods**:
+
+- `checkAndMarkGlobalVisited(scope, path, value)` - Fine-grained (includes type)
+- `checkAndMarkComplexSourceVisited(scope, path)` - Coarse (ignores type)
+- `resetGlobalVisited()` - Called between analysis phases
+
+**When to Use**: Inside recursive methods like `addToSchema()` or `followEquivalencies()`.
+
+---
+
+### ScopeTreeManager.ts
+
+**Purpose**: Track parent-child relationships between function scopes.
+
+**Key Methods**:
+
+- `addPath(['outer', 'inner'])` - Build tree as scopes are discovered
+- `findNode(name)` - Look up scope in hierarchy
+- `getTree()` - Get full tree structure
+
+**When to Use**: Understanding scope nesting, debugging scope resolution.
+
+---
+
+### selectBestValue.ts
+
+**Purpose**: Choose the most specific type when merging schemas.
+
+**Logic**: Prefers `'string'` over `'unknown'`, handles union types.
+
+**When to Use**: Merging type information from multiple sources.
+
+---
+
+### DebugTracer.ts
+
+**Purpose**: Structured debug/trace system for selective tracing without code changes.
+
+**Key Features**:
+
+- `trace(label, context)` - Simple trace point
+- `traceEnter/traceExit(label)` - Track call depth
+- Path/scope pattern filtering - Only trace what you care about
+- Max depth limiting - Prevent runaway output
+
+**Example Usage**:
+
+```typescript
+const tracer = new DebugTracer({
+  enabled: true,
+  pathPatterns: [/user\.id/], // Only trace paths matching this pattern
+  maxDepth: 10,
+});
+
+tracer.traceEnter('addToSchema', { path, scope: scopeNode.name });
+// ... do work ...
+tracer.traceExit('addToSchema', { wrote: true });
+```
+
+**When to Use**: Debugging specific paths through the analysis without modifying code.
+
+---
+
+## Key Concepts for Debugging
+
+### Equivalencies
+
+An **equivalency** connects two data paths that refer to the same data:
+
+```
+// In code:
+const { user } = props;
+return <Name name={user.name} />;
+
+// Creates equivalencies:
+props.user.name ←→ user.name ←→ <Name>.props.name
+```
+
+Equivalencies are stored in `scopeNode.equivalencies` as:
+
+```typescript
+{
+  'user.name': [
+    { scopeNodeName: 'MyComponent', schemaPath: 'signature[0].user.name', ... }
+  ]
+}
+```
+
+### Schema Paths
+
+Schema paths describe the shape of data:
+
+| Path                             | Meaning                              |
+| -------------------------------- | ------------------------------------ |
+| `signature[0]`                   | First function argument              |
+| `signature[0].user.id`           | `id` property of `user` in first arg |
+| `items[].name`                   | `name` property of array elements    |
+| `returnValue`                    | Function return value                |
+| `functionCallReturnValue(fetch)` | Return value from `fetch()` call     |
+
+### The Two Phases
+
+1. **Phase 1 (Discovery)**: `onlyEquivalencies = true`
+   - Processes all scopes
+   - Builds equivalency relationships
+   - Does NOT populate schemas
+
+2. **Phase 2 (Schema Building)**: `captureCompleteSchema()`
+   - Resets visited tracker
+   - Follows equivalencies to build complete schemas
+   - Populates `equivalencyDatabase` for source/usage tracking
+
+## ScopeDataStructure Debugging Methods
+
+ScopeDataStructure includes built-in debugging helpers. These methods are accessible on any ScopeDataStructure instance:
+
+### `explainPath(scopeName, path)`
+
+Traces a path through equivalencies and shows where data comes from:
+
+```typescript
+const explanation = sds.explainPath('MyComponent', 'user.id');
+// {
+//   path: 'user.id',
+//   scope: 'MyComponent',
+//   schemaValue: 'string',
+//   equivalencies: [{ scope: 'MyComponent', path: 'signature[0].user.id' }],
+//   source: { scope: 'fetchUser', path: 'functionCallReturnValue(fetch).data.user.id' }
+// }
+```
+
+### `traceEquivalencyChain(scopeName, path, maxDepth?)`
+
+Shows the full chain of equivalencies for a path:
+
+```typescript
+const chain = sds.traceEquivalencyChain('MyComponent', 'user.id');
+// Returns array of steps showing how the data is traced back to its source
+```
+
+### `findPaths(scopeName, pattern)`
+
+Finds all paths matching a regex pattern:
+
+```typescript
+const paths = sds.findPaths('MyComponent', /user/);
+// Returns paths containing "user" with their values and equivalency status
+```
+
+### `diagnoseMissingPath(scopeName, path)`
+
+Explains why a path might be missing from the schema:
+
+```typescript
+const diagnosis = sds.diagnoseMissingPath('MyComponent', 'user.id');
+// { exists: false, possibleReasons: [...], suggestions: [...] }
+```
+
+### `dumpState(options?)`
+
+Dumps the current analysis state for inspection:
+
+```typescript
+const state = sds.dumpState({ includeSchemas: true, scopeFilter: 'MyComp' });
+// { scopeCount, metrics, scopes: [...] }
+```
+
+---
+
+## Common Debugging Patterns
+
+### "Why is this path missing from the schema?"
+
+**Quick approach**: Use `diagnoseMissingPath()`:
+
+```typescript
+const diagnosis = sds.diagnoseMissingPath('MyComponent', 'user.id');
+console.log(diagnosis.possibleReasons);
+console.log(diagnosis.suggestions);
+```
+
+**Manual investigation**:
+
+1. Check if the path exists in equivalencies: `scopeNode.equivalencies[path]`
+2. Check if it was visited: Enable debug logging in `addToSchema()`
+3. Check if it was filtered: Look at `isValidPath()` rejections
+
+### "Why is this type wrong?"
+
+**Quick approach**: Use `traceEquivalencyChain()`:
+
+```typescript
+const chain = sds.traceEquivalencyChain('MyComponent', 'user.id');
+// Each step shows the schemaValue - find where the wrong type appears
+```
+
+**Manual investigation**:
+
+1. Find where the type is set: Search for the path in `addToSchema()` calls
+2. Check `selectBestValue()` logic: Multiple sources may be merged
+3. Check equivalency chain: Type may come from a linked path
+
+### "Infinite loop or hang"
+
+1. Check `VisitedTracker` usage: Missing cycle detection?
+2. Look for recursive equivalencies: A → B → C → A
+3. Enable metrics: `maxEquivalencyChainDepth` tracks depth
+
+## Testing
+
+Each module has tests in `__tests__/`:
+
+- `PathManager.test.ts`
+- `uniqueIdUtils.test.ts`
+- `VisitedTracker.test.ts`
+- `ScopeTreeManager.test.ts`
+- `selectBestValue.test.ts`
+
+Run with: `npx jest packages/ai/src/lib/dataStructure/helpers`
+
+## Adding New Helpers
+
+When extracting new functionality:
+
+1. **Single Responsibility**: One concept per module
+2. **Document Context**: Explain WHY, not just WHAT
+3. **Link Related Modules**: Help readers navigate
+4. **Write Tests First**: Tests are documentation
+5. **Keep Interfaces Stable**: ScopeDataStructure's public API shouldn't change

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/ScopeTreeManager.ts

@@ -0,0 +1,161 @@
+/**
+ * ScopeTreeManager
+ *
+ * Manages the hierarchical tree structure of scopes in the analysis.
+ * Tracks parent-child relationships between function scopes.
+ *
+ * ## Purpose
+ *
+ * When analyzing nested functions, we need to track which functions
+ * are defined inside other functions. This affects:
+ * - Variable scope inheritance (inner functions can access outer variables)
+ * - Equivalency propagation (data flows between parent and child scopes)
+ * - Schema merging (combining schemas from nested call hierarchies)
+ *
+ * ## Tree Structure Example
+ *
+ * For code like:
+ * ```typescript
+ * function outer(data) {
+ *   function inner(item) {
+ *     return item.name;
+ *   }
+ *   return data.map(inner);
+ * }
+ * ```
+ *
+ * The tree would be:
+ * ```
+ * outer
+ *   └── inner
+ * ```
+ *
+ * ## Key Operations
+ *
+ * - `addPath(['outer', 'inner'])` - Builds tree as scopes are discovered
+ * - `findNode('inner')` - Locates scope node for equivalency lookups
+ * - `getTree()` - Returns full tree for debugging/visualization
+ *
+ * ## Related Modules
+ *
+ * - `ScopeDataStructure.addScopeAnalysis()` - Adds paths when processing scopes
+ * - `ScopeDataStructure.getAnalysisTree()` - Exposes tree to callers
+ */
+
+export interface ScopeTreeNode {
+  name: string;
+  children: ScopeTreeNode[];
+}
+
+export const ROOT_SCOPE_NAME = 'root';
+
+export class ScopeTreeManager {
+  private tree: ScopeTreeNode;
+
+  constructor(rootName: string = ROOT_SCOPE_NAME) {
+    this.tree = { name: rootName, children: [] };
+  }
+
+  /**
+   * Gets the name of the root scope.
+   * This is frequently used as the default scope name throughout the analysis.
+   */
+  getRootName(): string {
+    return this.tree.name;
+  }
+
+  /**
+   * Sets the name of the root scope.
+   * Called during initialization when the actual root scope name is known.
+   */
+  setRootName(name: string): void {
+    this.tree.name = name;
+  }
+
+  /**
+   * Gets the entire tree structure.
+   * Used by getAnalysisTree() to return the scope hierarchy.
+   */
+  getTree(): ScopeTreeNode {
+    return this.tree;
+  }
+
+  /**
+   * Adds a scope path to the tree.
+   * Creates intermediate nodes as needed.
+   *
+   * @param pathParts - Array of scope names representing the path from root
+   *
+   * @example
+   * // For a nested function: root -> parentFunc -> childFunc
+   * addPath(['root', 'parentFunc', 'childFunc'])
+   */
+  addPath(pathParts: string[]): void {
+    let currentNode: { children: ScopeTreeNode[] } = { children: [this.tree] };
+
+    for (const pathPart of pathParts) {
+      const existingChild = currentNode.children.find(
+        (child) => child.name === pathPart,
+      );
+
+      if (existingChild) {
+        currentNode = existingChild;
+      } else {
+        const newChild: ScopeTreeNode = {
+          name: pathPart,
+          children: [],
+        };
+        currentNode.children.push(newChild);
+        currentNode = newChild;
+      }
+    }
+  }
+
+  /**
+   * Finds a node by name in the tree (breadth-first search).
+   * Returns undefined if not found.
+   */
+  findNode(name: string): ScopeTreeNode | undefined {
+    const queue: ScopeTreeNode[] = [this.tree];
+
+    while (queue.length > 0) {
+      const node = queue.shift()!;
+      if (node.name === name) {
+        return node;
+      }
+      queue.push(...node.children);
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Gets all scope names in the tree (flattened).
+   */
+  getAllNames(): string[] {
+    const names: string[] = [];
+    const queue: ScopeTreeNode[] = [this.tree];
+
+    while (queue.length > 0) {
+      const node = queue.shift()!;
+      names.push(node.name);
+      queue.push(...node.children);
+    }
+
+    return names;
+  }
+
+  /**
+   * Returns the depth of the tree (for debugging/metrics).
+   */
+  getDepth(): number {
+    const getNodeDepth = (node: ScopeTreeNode): number => {
+      if (node.children.length === 0) {
+        return 1;
+      }
+      return 1 + Math.max(...node.children.map(getNodeDepth));
+    };
+
+    return getNodeDepth(this.tree);
+  }
+}